mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Merge branch 'master' into complete_zk_api
This commit is contained in:
alesapin
commit
278bbf6d51
11
.gitignore
vendored
11
.gitignore
vendored
@ -125,4 +125,15 @@ website/package-lock.json
|
||||
# Toolchains
|
||||
/cmake/toolchain/*
|
||||
|
||||
# ANTLR extension cache
|
||||
.antlr
|
||||
|
||||
# ANTLR generated files
|
||||
/src/Parsers/New/*.interp
|
||||
/src/Parsers/New/*.tokens
|
||||
/src/Parsers/New/ClickHouseParserBaseVisitor.*
|
||||
|
||||
# pytest-profiling
|
||||
/prof
|
||||
|
||||
*.iml
|
||||
|
||||
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -172,6 +172,9 @@
|
||||
[submodule "contrib/fmtlib"]
|
||||
path = contrib/fmtlib
|
||||
url = https://github.com/fmtlib/fmt.git
|
||||
[submodule "contrib/antlr4-runtime"]
|
||||
path = contrib/antlr4-runtime
|
||||
url = https://github.com/ClickHouse-Extras/antlr4-runtime.git
|
||||
[submodule "contrib/sentry-native"]
|
||||
path = contrib/sentry-native
|
||||
url = https://github.com/ClickHouse-Extras/sentry-native.git
|
||||
|
||||
@ -257,6 +257,8 @@ if (WITH_COVERAGE AND COMPILER_GCC)
|
||||
set(WITHOUT_COVERAGE "-fno-profile-arcs -fno-test-coverage")
|
||||
endif()
|
||||
|
||||
set(COMPILER_FLAGS "${COMPILER_FLAGS}")
|
||||
|
||||
set (CMAKE_BUILD_COLOR_MAKEFILE ON)
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}")
|
||||
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}")
|
||||
|
||||
@ -16,5 +16,4 @@ ClickHouse® is an open-source column-oriented database management system that a
|
||||
* You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person.
|
||||
|
||||
## Upcoming Events
|
||||
* [SF Bay Area ClickHouse Meetup (online)](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/274498897/) on 2 December 2020.
|
||||
* [SF Bay Area ClickHouse Virtual Office Hours (online)](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/274273549/) on 20 January 2020.
|
||||
|
||||
@ -76,12 +76,6 @@
|
||||
# define NO_SANITIZE_THREAD
|
||||
#endif
|
||||
|
||||
#if defined __GNUC__ && !defined __clang__
|
||||
# define OPTIMIZE(x) __attribute__((__optimize__(x)))
|
||||
#else
|
||||
# define OPTIMIZE(x)
|
||||
#endif
|
||||
|
||||
/// A macro for suppressing warnings about unused variables or function results.
|
||||
/// Useful for structured bindings which have no standard way to declare this.
|
||||
#define UNUSED(...) (void)(__VA_ARGS__)
|
||||
|
||||
@ -5,7 +5,6 @@ LIBRARY()
|
||||
|
||||
ADDINCL(
|
||||
GLOBAL clickhouse/base
|
||||
GLOBAL contrib/libs/cctz/include
|
||||
)
|
||||
|
||||
CFLAGS (GLOBAL -DARCADIA_BUILD)
|
||||
@ -24,7 +23,7 @@ ELSEIF (OS_LINUX)
|
||||
ENDIF ()
|
||||
|
||||
PEERDIR(
|
||||
contrib/libs/cctz/src
|
||||
contrib/libs/cctz
|
||||
contrib/libs/cxxsupp/libcxx-filesystem
|
||||
contrib/libs/poco/Net
|
||||
contrib/libs/poco/Util
|
||||
|
||||
@ -4,7 +4,6 @@ LIBRARY()
|
||||
|
||||
ADDINCL(
|
||||
GLOBAL clickhouse/base
|
||||
GLOBAL contrib/libs/cctz/include
|
||||
)
|
||||
|
||||
CFLAGS (GLOBAL -DARCADIA_BUILD)
|
||||
@ -23,7 +22,7 @@ ELSEIF (OS_LINUX)
|
||||
ENDIF ()
|
||||
|
||||
PEERDIR(
|
||||
contrib/libs/cctz/src
|
||||
contrib/libs/cctz
|
||||
contrib/libs/cxxsupp/libcxx-filesystem
|
||||
contrib/libs/poco/Net
|
||||
contrib/libs/poco/Util
|
||||
|
||||
@ -104,6 +104,11 @@ void Connection::connect(const char* db,
|
||||
if (mysql_options(driver.get(), MYSQL_OPT_LOCAL_INFILE, &enable_local_infile_arg))
|
||||
throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get()));
|
||||
|
||||
/// Enables auto-reconnect.
|
||||
bool reconnect = true;
|
||||
if (mysql_options(driver.get(), MYSQL_OPT_RECONNECT, reinterpret_cast<const char *>(&reconnect)))
|
||||
throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get()));
|
||||
|
||||
/// Specifies particular ssl key and certificate if it needs
|
||||
if (mysql_ssl_set(driver.get(), ifNotEmpty(ssl_key), ifNotEmpty(ssl_cert), ifNotEmpty(ssl_ca), nullptr, nullptr))
|
||||
throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get()));
|
||||
@ -115,11 +120,6 @@ void Connection::connect(const char* db,
|
||||
if (mysql_set_character_set(driver.get(), "UTF8"))
|
||||
throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get()));
|
||||
|
||||
/// Enables auto-reconnect.
|
||||
bool reconnect = true;
|
||||
if (mysql_options(driver.get(), MYSQL_OPT_RECONNECT, reinterpret_cast<const char *>(&reconnect)))
|
||||
throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get()));
|
||||
|
||||
is_connected = true;
|
||||
}
|
||||
|
||||
|
||||
@ -26,6 +26,7 @@ void Pool::Entry::incrementRefCount()
|
||||
mysql_thread_init();
|
||||
}
|
||||
|
||||
|
||||
void Pool::Entry::decrementRefCount()
|
||||
{
|
||||
if (!data)
|
||||
@ -150,28 +151,39 @@ Pool::Entry Pool::tryGet()
|
||||
|
||||
initialize();
|
||||
|
||||
/// Searching for connection which was established but wasn't used.
|
||||
for (auto & connection : connections)
|
||||
/// Try to pick an idle connection from already allocated
|
||||
for (auto connection_it = connections.cbegin(); connection_it != connections.cend();)
|
||||
{
|
||||
if (connection->ref_count == 0)
|
||||
Connection * connection_ptr = *connection_it;
|
||||
/// Fixme: There is a race condition here b/c we do not synchronize with Pool::Entry's copy-assignment operator
|
||||
if (connection_ptr->ref_count == 0)
|
||||
{
|
||||
Entry res(connection, this);
|
||||
return res.tryForceConnected() ? res : Entry();
|
||||
Entry res(connection_ptr, this);
|
||||
if (res.tryForceConnected()) /// Tries to reestablish connection as well
|
||||
return res;
|
||||
|
||||
auto & logger = Poco::Util::Application::instance().logger();
|
||||
logger.information("Idle connection to mysql server cannot be recovered, dropping it.");
|
||||
|
||||
/// This one is disconnected, cannot be reestablished and so needs to be disposed of.
|
||||
connection_it = connections.erase(connection_it);
|
||||
::delete connection_ptr; /// TODO: Manual memory management is awkward (matches allocConnection() method)
|
||||
}
|
||||
else
|
||||
++connection_it;
|
||||
}
|
||||
|
||||
/// Throws if pool is overflowed.
|
||||
if (connections.size() >= max_connections)
|
||||
throw Poco::Exception("mysqlxx::Pool is full");
|
||||
|
||||
/// Allocates new connection.
|
||||
Connection * conn = allocConnection(true);
|
||||
if (conn)
|
||||
return Entry(conn, this);
|
||||
Connection * connection_ptr = allocConnection(true);
|
||||
if (connection_ptr)
|
||||
return {connection_ptr, this};
|
||||
|
||||
return Entry();
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
void Pool::removeConnection(Connection* connection)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
@ -199,11 +211,9 @@ void Pool::Entry::forceConnected() const
|
||||
throw Poco::RuntimeException("Tried to access NULL database connection.");
|
||||
|
||||
Poco::Util::Application & app = Poco::Util::Application::instance();
|
||||
if (data->conn.ping())
|
||||
return;
|
||||
|
||||
bool first = true;
|
||||
do
|
||||
while (!tryForceConnected())
|
||||
{
|
||||
if (first)
|
||||
first = false;
|
||||
@ -225,7 +235,26 @@ void Pool::Entry::forceConnected() const
|
||||
pool->rw_timeout,
|
||||
pool->enable_local_infile);
|
||||
}
|
||||
while (!data->conn.ping());
|
||||
}
|
||||
|
||||
|
||||
bool Pool::Entry::tryForceConnected() const
|
||||
{
|
||||
auto * const mysql_driver = data->conn.getDriver();
|
||||
const auto prev_connection_id = mysql_thread_id(mysql_driver);
|
||||
if (data->conn.ping()) /// Attempts to reestablish lost connection
|
||||
{
|
||||
const auto current_connection_id = mysql_thread_id(mysql_driver);
|
||||
if (prev_connection_id != current_connection_id)
|
||||
{
|
||||
auto & logger = Poco::Util::Application::instance().logger();
|
||||
logger.information("Connection to mysql server has been reestablished. Connection id changed: %d -> %d",
|
||||
prev_connection_id, current_connection_id);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -127,10 +127,7 @@ public:
|
||||
void forceConnected() const;
|
||||
|
||||
/// Connects to database. If connection is failed then returns false.
|
||||
bool tryForceConnected() const
|
||||
{
|
||||
return data->conn.ping();
|
||||
}
|
||||
bool tryForceConnected() const;
|
||||
|
||||
void incrementRefCount();
|
||||
void decrementRefCount();
|
||||
|
||||
@ -1,2 +1,5 @@
|
||||
add_executable (mysqlxx_test mysqlxx_test.cpp)
|
||||
target_link_libraries (mysqlxx_test PRIVATE mysqlxx)
|
||||
|
||||
add_executable (mysqlxx_pool_test mysqlxx_pool_test.cpp)
|
||||
target_link_libraries (mysqlxx_pool_test PRIVATE mysqlxx)
|
||||
|
||||
98
base/mysqlxx/tests/mysqlxx_pool_test.cpp
Normal file
98
base/mysqlxx/tests/mysqlxx_pool_test.cpp
Normal file
@ -0,0 +1,98 @@
|
||||
#include <mysqlxx/mysqlxx.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
|
||||
|
||||
namespace
|
||||
{
|
||||
mysqlxx::Pool::Entry getWithFailover(mysqlxx::Pool & connections_pool)
|
||||
{
|
||||
using namespace std::chrono;
|
||||
|
||||
constexpr size_t max_tries = 3;
|
||||
|
||||
mysqlxx::Pool::Entry worker_connection;
|
||||
|
||||
for (size_t try_no = 1; try_no <= max_tries; ++try_no)
|
||||
{
|
||||
try
|
||||
{
|
||||
worker_connection = connections_pool.tryGet();
|
||||
|
||||
if (!worker_connection.isNull())
|
||||
{
|
||||
return worker_connection;
|
||||
}
|
||||
}
|
||||
catch (const Poco::Exception & e)
|
||||
{
|
||||
if (e.displayText().find("mysqlxx::Pool is full") != std::string::npos)
|
||||
{
|
||||
std::cerr << e.displayText() << std::endl;
|
||||
}
|
||||
|
||||
std::cerr << "Connection to " << connections_pool.getDescription() << " failed: " << e.displayText() << std::endl;
|
||||
}
|
||||
|
||||
std::clog << "Connection to all replicas failed " << try_no << " times" << std::endl;
|
||||
std::this_thread::sleep_for(1s);
|
||||
}
|
||||
|
||||
std::stringstream message;
|
||||
message << "Connections to all replicas failed: " << connections_pool.getDescription();
|
||||
|
||||
throw Poco::Exception(message.str());
|
||||
}
|
||||
}
|
||||
|
||||
int main(int, char **)
|
||||
{
|
||||
using namespace std::chrono;
|
||||
|
||||
const char * remote_mysql = "localhost";
|
||||
const std::string test_query = "SHOW DATABASES";
|
||||
|
||||
mysqlxx::Pool mysql_conn_pool("", remote_mysql, "default", "10203040", 3306);
|
||||
|
||||
size_t iteration = 0;
|
||||
while (++iteration)
|
||||
{
|
||||
std::clog << "Iteration: " << iteration << std::endl;
|
||||
try
|
||||
{
|
||||
std::clog << "Acquiring DB connection ...";
|
||||
mysqlxx::Pool::Entry worker = getWithFailover(mysql_conn_pool);
|
||||
std::clog << "ok" << std::endl;
|
||||
|
||||
std::clog << "Preparing query (5s sleep) ...";
|
||||
std::this_thread::sleep_for(5s);
|
||||
mysqlxx::Query query = worker->query();
|
||||
query << test_query;
|
||||
std::clog << "ok" << std::endl;
|
||||
|
||||
std::clog << "Querying result (5s sleep) ...";
|
||||
std::this_thread::sleep_for(5s);
|
||||
mysqlxx::UseQueryResult result = query.use();
|
||||
std::clog << "ok" << std::endl;
|
||||
|
||||
std::clog << "Fetching result data (5s sleep) ...";
|
||||
std::this_thread::sleep_for(5s);
|
||||
size_t rows_count = 0;
|
||||
while (result.fetch())
|
||||
++rows_count;
|
||||
std::clog << "ok" << std::endl;
|
||||
|
||||
std::clog << "Read " << rows_count << " rows." << std::endl;
|
||||
}
|
||||
catch (const Poco::Exception & e)
|
||||
{
|
||||
std::cerr << "Iteration FAILED:\n" << e.displayText() << std::endl;
|
||||
}
|
||||
|
||||
std::clog << "====================" << std::endl;
|
||||
std::this_thread::sleep_for(3s);
|
||||
}
|
||||
}
|
||||
@ -24,7 +24,7 @@ option (WEVERYTHING "Enable -Weverything option with some exceptions." ON)
|
||||
# Control maximum size of stack frames. It can be important if the code is run in fibers with small stack size.
|
||||
# Only in release build because debug has too large stack frames.
|
||||
if ((NOT CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") AND (NOT SANITIZE) AND (NOT CMAKE_CXX_COMPILER_ID MATCHES "AppleClang"))
|
||||
add_warning(frame-larger-than=32768)
|
||||
add_warning(frame-larger-than=65536)
|
||||
endif ()
|
||||
|
||||
if (COMPILER_CLANG)
|
||||
|
||||
1
contrib/CMakeLists.txt
vendored
1
contrib/CMakeLists.txt
vendored
@ -21,6 +21,7 @@ endif()
|
||||
|
||||
set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1)
|
||||
|
||||
add_subdirectory (antlr4-runtime-cmake)
|
||||
add_subdirectory (boost-cmake)
|
||||
add_subdirectory (cctz-cmake)
|
||||
add_subdirectory (consistent-hashing-sumbur)
|
||||
|
||||
1
contrib/antlr4-runtime
vendored
Submodule
1
contrib/antlr4-runtime
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit a2fa7b76e2ee16d2ad955e9214a90bbf79da66fc
|
||||
156
contrib/antlr4-runtime-cmake/CMakeLists.txt
Normal file
156
contrib/antlr4-runtime-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,156 @@
|
||||
set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/antlr4-runtime)
|
||||
|
||||
set (SRCS
|
||||
${LIBRARY_DIR}/ANTLRErrorListener.cpp
|
||||
${LIBRARY_DIR}/ANTLRErrorStrategy.cpp
|
||||
${LIBRARY_DIR}/ANTLRFileStream.cpp
|
||||
${LIBRARY_DIR}/ANTLRInputStream.cpp
|
||||
${LIBRARY_DIR}/atn/AbstractPredicateTransition.cpp
|
||||
${LIBRARY_DIR}/atn/ActionTransition.cpp
|
||||
${LIBRARY_DIR}/atn/AmbiguityInfo.cpp
|
||||
${LIBRARY_DIR}/atn/ArrayPredictionContext.cpp
|
||||
${LIBRARY_DIR}/atn/ATN.cpp
|
||||
${LIBRARY_DIR}/atn/ATNConfig.cpp
|
||||
${LIBRARY_DIR}/atn/ATNConfigSet.cpp
|
||||
${LIBRARY_DIR}/atn/ATNDeserializationOptions.cpp
|
||||
${LIBRARY_DIR}/atn/ATNDeserializer.cpp
|
||||
${LIBRARY_DIR}/atn/ATNSerializer.cpp
|
||||
${LIBRARY_DIR}/atn/ATNSimulator.cpp
|
||||
${LIBRARY_DIR}/atn/ATNState.cpp
|
||||
${LIBRARY_DIR}/atn/AtomTransition.cpp
|
||||
${LIBRARY_DIR}/atn/BasicBlockStartState.cpp
|
||||
${LIBRARY_DIR}/atn/BasicState.cpp
|
||||
${LIBRARY_DIR}/atn/BlockEndState.cpp
|
||||
${LIBRARY_DIR}/atn/BlockStartState.cpp
|
||||
${LIBRARY_DIR}/atn/ContextSensitivityInfo.cpp
|
||||
${LIBRARY_DIR}/atn/DecisionEventInfo.cpp
|
||||
${LIBRARY_DIR}/atn/DecisionInfo.cpp
|
||||
${LIBRARY_DIR}/atn/DecisionState.cpp
|
||||
${LIBRARY_DIR}/atn/EmptyPredictionContext.cpp
|
||||
${LIBRARY_DIR}/atn/EpsilonTransition.cpp
|
||||
${LIBRARY_DIR}/atn/ErrorInfo.cpp
|
||||
${LIBRARY_DIR}/atn/LexerAction.cpp
|
||||
${LIBRARY_DIR}/atn/LexerActionExecutor.cpp
|
||||
${LIBRARY_DIR}/atn/LexerATNConfig.cpp
|
||||
${LIBRARY_DIR}/atn/LexerATNSimulator.cpp
|
||||
${LIBRARY_DIR}/atn/LexerChannelAction.cpp
|
||||
${LIBRARY_DIR}/atn/LexerCustomAction.cpp
|
||||
${LIBRARY_DIR}/atn/LexerIndexedCustomAction.cpp
|
||||
${LIBRARY_DIR}/atn/LexerModeAction.cpp
|
||||
${LIBRARY_DIR}/atn/LexerMoreAction.cpp
|
||||
${LIBRARY_DIR}/atn/LexerPopModeAction.cpp
|
||||
${LIBRARY_DIR}/atn/LexerPushModeAction.cpp
|
||||
${LIBRARY_DIR}/atn/LexerSkipAction.cpp
|
||||
${LIBRARY_DIR}/atn/LexerTypeAction.cpp
|
||||
${LIBRARY_DIR}/atn/LL1Analyzer.cpp
|
||||
${LIBRARY_DIR}/atn/LookaheadEventInfo.cpp
|
||||
${LIBRARY_DIR}/atn/LoopEndState.cpp
|
||||
${LIBRARY_DIR}/atn/NotSetTransition.cpp
|
||||
${LIBRARY_DIR}/atn/OrderedATNConfigSet.cpp
|
||||
${LIBRARY_DIR}/atn/ParseInfo.cpp
|
||||
${LIBRARY_DIR}/atn/ParserATNSimulator.cpp
|
||||
${LIBRARY_DIR}/atn/PlusBlockStartState.cpp
|
||||
${LIBRARY_DIR}/atn/PlusLoopbackState.cpp
|
||||
${LIBRARY_DIR}/atn/PrecedencePredicateTransition.cpp
|
||||
${LIBRARY_DIR}/atn/PredicateEvalInfo.cpp
|
||||
${LIBRARY_DIR}/atn/PredicateTransition.cpp
|
||||
${LIBRARY_DIR}/atn/PredictionContext.cpp
|
||||
${LIBRARY_DIR}/atn/PredictionMode.cpp
|
||||
${LIBRARY_DIR}/atn/ProfilingATNSimulator.cpp
|
||||
${LIBRARY_DIR}/atn/RangeTransition.cpp
|
||||
${LIBRARY_DIR}/atn/RuleStartState.cpp
|
||||
${LIBRARY_DIR}/atn/RuleStopState.cpp
|
||||
${LIBRARY_DIR}/atn/RuleTransition.cpp
|
||||
${LIBRARY_DIR}/atn/SemanticContext.cpp
|
||||
${LIBRARY_DIR}/atn/SetTransition.cpp
|
||||
${LIBRARY_DIR}/atn/SingletonPredictionContext.cpp
|
||||
${LIBRARY_DIR}/atn/StarBlockStartState.cpp
|
||||
${LIBRARY_DIR}/atn/StarLoopbackState.cpp
|
||||
${LIBRARY_DIR}/atn/StarLoopEntryState.cpp
|
||||
${LIBRARY_DIR}/atn/TokensStartState.cpp
|
||||
${LIBRARY_DIR}/atn/Transition.cpp
|
||||
${LIBRARY_DIR}/atn/WildcardTransition.cpp
|
||||
${LIBRARY_DIR}/BailErrorStrategy.cpp
|
||||
${LIBRARY_DIR}/BaseErrorListener.cpp
|
||||
${LIBRARY_DIR}/BufferedTokenStream.cpp
|
||||
${LIBRARY_DIR}/CharStream.cpp
|
||||
${LIBRARY_DIR}/CommonToken.cpp
|
||||
${LIBRARY_DIR}/CommonTokenFactory.cpp
|
||||
${LIBRARY_DIR}/CommonTokenStream.cpp
|
||||
${LIBRARY_DIR}/ConsoleErrorListener.cpp
|
||||
${LIBRARY_DIR}/DefaultErrorStrategy.cpp
|
||||
${LIBRARY_DIR}/dfa/DFA.cpp
|
||||
${LIBRARY_DIR}/dfa/DFASerializer.cpp
|
||||
${LIBRARY_DIR}/dfa/DFAState.cpp
|
||||
${LIBRARY_DIR}/dfa/LexerDFASerializer.cpp
|
||||
${LIBRARY_DIR}/DiagnosticErrorListener.cpp
|
||||
${LIBRARY_DIR}/Exceptions.cpp
|
||||
${LIBRARY_DIR}/FailedPredicateException.cpp
|
||||
${LIBRARY_DIR}/InputMismatchException.cpp
|
||||
${LIBRARY_DIR}/InterpreterRuleContext.cpp
|
||||
${LIBRARY_DIR}/IntStream.cpp
|
||||
${LIBRARY_DIR}/Lexer.cpp
|
||||
${LIBRARY_DIR}/LexerInterpreter.cpp
|
||||
${LIBRARY_DIR}/LexerNoViableAltException.cpp
|
||||
${LIBRARY_DIR}/ListTokenSource.cpp
|
||||
${LIBRARY_DIR}/misc/InterpreterDataReader.cpp
|
||||
${LIBRARY_DIR}/misc/Interval.cpp
|
||||
${LIBRARY_DIR}/misc/IntervalSet.cpp
|
||||
${LIBRARY_DIR}/misc/MurmurHash.cpp
|
||||
${LIBRARY_DIR}/misc/Predicate.cpp
|
||||
${LIBRARY_DIR}/NoViableAltException.cpp
|
||||
${LIBRARY_DIR}/Parser.cpp
|
||||
${LIBRARY_DIR}/ParserInterpreter.cpp
|
||||
${LIBRARY_DIR}/ParserRuleContext.cpp
|
||||
${LIBRARY_DIR}/ProxyErrorListener.cpp
|
||||
${LIBRARY_DIR}/RecognitionException.cpp
|
||||
${LIBRARY_DIR}/Recognizer.cpp
|
||||
${LIBRARY_DIR}/RuleContext.cpp
|
||||
${LIBRARY_DIR}/RuleContextWithAltNum.cpp
|
||||
${LIBRARY_DIR}/RuntimeMetaData.cpp
|
||||
${LIBRARY_DIR}/support/Any.cpp
|
||||
${LIBRARY_DIR}/support/Arrays.cpp
|
||||
${LIBRARY_DIR}/support/CPPUtils.cpp
|
||||
${LIBRARY_DIR}/support/guid.cpp
|
||||
${LIBRARY_DIR}/support/StringUtils.cpp
|
||||
${LIBRARY_DIR}/Token.cpp
|
||||
${LIBRARY_DIR}/TokenSource.cpp
|
||||
${LIBRARY_DIR}/TokenStream.cpp
|
||||
${LIBRARY_DIR}/TokenStreamRewriter.cpp
|
||||
${LIBRARY_DIR}/tree/ErrorNode.cpp
|
||||
${LIBRARY_DIR}/tree/ErrorNodeImpl.cpp
|
||||
${LIBRARY_DIR}/tree/IterativeParseTreeWalker.cpp
|
||||
${LIBRARY_DIR}/tree/ParseTree.cpp
|
||||
${LIBRARY_DIR}/tree/ParseTreeListener.cpp
|
||||
${LIBRARY_DIR}/tree/ParseTreeVisitor.cpp
|
||||
${LIBRARY_DIR}/tree/ParseTreeWalker.cpp
|
||||
${LIBRARY_DIR}/tree/pattern/Chunk.cpp
|
||||
${LIBRARY_DIR}/tree/pattern/ParseTreeMatch.cpp
|
||||
${LIBRARY_DIR}/tree/pattern/ParseTreePattern.cpp
|
||||
${LIBRARY_DIR}/tree/pattern/ParseTreePatternMatcher.cpp
|
||||
${LIBRARY_DIR}/tree/pattern/RuleTagToken.cpp
|
||||
${LIBRARY_DIR}/tree/pattern/TagChunk.cpp
|
||||
${LIBRARY_DIR}/tree/pattern/TextChunk.cpp
|
||||
${LIBRARY_DIR}/tree/pattern/TokenTagToken.cpp
|
||||
${LIBRARY_DIR}/tree/TerminalNode.cpp
|
||||
${LIBRARY_DIR}/tree/TerminalNodeImpl.cpp
|
||||
${LIBRARY_DIR}/tree/Trees.cpp
|
||||
${LIBRARY_DIR}/tree/xpath/XPath.cpp
|
||||
${LIBRARY_DIR}/tree/xpath/XPathElement.cpp
|
||||
${LIBRARY_DIR}/tree/xpath/XPathLexer.cpp
|
||||
${LIBRARY_DIR}/tree/xpath/XPathLexerErrorListener.cpp
|
||||
${LIBRARY_DIR}/tree/xpath/XPathRuleAnywhereElement.cpp
|
||||
${LIBRARY_DIR}/tree/xpath/XPathRuleElement.cpp
|
||||
${LIBRARY_DIR}/tree/xpath/XPathTokenAnywhereElement.cpp
|
||||
${LIBRARY_DIR}/tree/xpath/XPathTokenElement.cpp
|
||||
${LIBRARY_DIR}/tree/xpath/XPathWildcardAnywhereElement.cpp
|
||||
${LIBRARY_DIR}/tree/xpath/XPathWildcardElement.cpp
|
||||
${LIBRARY_DIR}/UnbufferedCharStream.cpp
|
||||
${LIBRARY_DIR}/UnbufferedTokenStream.cpp
|
||||
${LIBRARY_DIR}/Vocabulary.cpp
|
||||
${LIBRARY_DIR}/WritableToken.cpp
|
||||
)
|
||||
|
||||
add_library (antlr4-runtime ${SRCS})
|
||||
|
||||
target_include_directories (antlr4-runtime SYSTEM PUBLIC ${LIBRARY_DIR})
|
||||
@ -131,6 +131,7 @@ function clone_submodules
|
||||
cd "$FASTTEST_SOURCE"
|
||||
|
||||
SUBMODULES_TO_UPDATE=(
|
||||
contrib/antlr4-runtime
|
||||
contrib/boost
|
||||
contrib/zlib-ng
|
||||
contrib/libxml2
|
||||
|
||||
@ -28,6 +28,7 @@ RUN apt-get update \
|
||||
libssl-dev \
|
||||
libcurl4-openssl-dev \
|
||||
gdb \
|
||||
software-properties-common \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/cache/debconf \
|
||||
@ -37,6 +38,22 @@ RUN apt-get update \
|
||||
ENV TZ=Europe/Moscow
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
ENV DOCKER_CHANNEL stable
|
||||
ENV DOCKER_VERSION 5:19.03.13~3-0~ubuntu-bionic
|
||||
RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
|
||||
RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}"
|
||||
|
||||
RUN apt-get update \
|
||||
&& env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
|
||||
docker-ce \
|
||||
&& rm -rf \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/cache/debconf \
|
||||
/tmp/* \
|
||||
&& apt-get clean
|
||||
|
||||
RUN dockerd --version; docker --version
|
||||
|
||||
RUN python3 -m pip install \
|
||||
PyMySQL \
|
||||
aerospike \
|
||||
@ -60,28 +77,6 @@ RUN python3 -m pip install \
|
||||
tzlocal \
|
||||
urllib3
|
||||
|
||||
ENV DOCKER_CHANNEL stable
|
||||
ENV DOCKER_VERSION 17.09.1-ce
|
||||
|
||||
RUN set -eux; \
|
||||
\
|
||||
# this "case" statement is generated via "update.sh"
|
||||
\
|
||||
if ! wget -nv -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \
|
||||
echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${x86_64}'"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
\
|
||||
tar --extract \
|
||||
--file docker.tgz \
|
||||
--strip-components 1 \
|
||||
--directory /usr/local/bin/ \
|
||||
; \
|
||||
rm docker.tgz; \
|
||||
\
|
||||
dockerd --version; \
|
||||
docker --version
|
||||
|
||||
COPY modprobe.sh /usr/local/bin/modprobe
|
||||
COPY dockerd-entrypoint.sh /usr/local/bin/
|
||||
COPY compose/ /compose/
|
||||
|
||||
@ -8,6 +8,7 @@ RUN apt-get update -y \
|
||||
apt-get install --yes --no-install-recommends \
|
||||
brotli \
|
||||
expect \
|
||||
zstd \
|
||||
lsof \
|
||||
ncdu \
|
||||
netcat-openbsd \
|
||||
|
||||
@ -8,6 +8,7 @@ RUN apt-get --allow-unauthenticated update -y \
|
||||
apt-get --allow-unauthenticated install --yes --no-install-recommends \
|
||||
alien \
|
||||
brotli \
|
||||
zstd \
|
||||
cmake \
|
||||
devscripts \
|
||||
expect \
|
||||
|
||||
@ -24,6 +24,7 @@ RUN apt-get update -y \
|
||||
tree \
|
||||
moreutils \
|
||||
brotli \
|
||||
zstd \
|
||||
gdb \
|
||||
lsof \
|
||||
unixodbc \
|
||||
|
||||
@ -579,6 +579,7 @@ Tags:
|
||||
- `disk` — a disk within a volume.
|
||||
- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume’s disks.
|
||||
- `move_factor` — when the amount of available space gets lower than this factor, data automatically start to move on the next volume if any (by default, 0.1).
|
||||
- `prefer_not_to_merge` — Disables merging of data parts on this volume. When this setting is enabled, merging data on this volume is not allowed. This allows controlling how ClickHouse works with slow disks.
|
||||
|
||||
Cofiguration examples:
|
||||
|
||||
@ -607,6 +608,18 @@ Cofiguration examples:
|
||||
</volumes>
|
||||
<move_factor>0.2</move_factor>
|
||||
</moving_from_ssd_to_hdd>
|
||||
|
||||
<small_jbod_with_external_no_merges>
|
||||
<volumes>
|
||||
<main>
|
||||
<disk>jbod1</disk>
|
||||
</main>
|
||||
<external>
|
||||
<disk>external</disk>
|
||||
<prefer_not_to_merge>true</prefer_not_to_merge>
|
||||
</external>
|
||||
</volumes>
|
||||
</small_jbod_with_external_no_merges>
|
||||
</policies>
|
||||
...
|
||||
</storage_configuration>
|
||||
|
||||
@ -23,4 +23,44 @@ Please note that `errors_count` is updated once per query to the cluster, but `e
|
||||
- [distributed_replica_error_cap setting](../../operations/settings/settings.md#settings-distributed_replica_error_cap)
|
||||
- [distributed_replica_error_half_life setting](../../operations/settings/settings.md#settings-distributed_replica_error_half_life)
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
:) SELECT * FROM system.clusters LIMIT 2 FORMAT Vertical;
|
||||
```
|
||||
|
||||
```text
|
||||
Row 1:
|
||||
──────
|
||||
cluster: test_cluster
|
||||
shard_num: 1
|
||||
shard_weight: 1
|
||||
replica_num: 1
|
||||
host_name: clickhouse01
|
||||
host_address: 172.23.0.11
|
||||
port: 9000
|
||||
is_local: 1
|
||||
user: default
|
||||
default_database:
|
||||
errors_count: 0
|
||||
estimated_recovery_time: 0
|
||||
|
||||
Row 2:
|
||||
──────
|
||||
cluster: test_cluster
|
||||
shard_num: 1
|
||||
shard_weight: 1
|
||||
replica_num: 2
|
||||
host_name: clickhouse02
|
||||
host_address: 172.23.0.12
|
||||
port: 9000
|
||||
is_local: 0
|
||||
user: default
|
||||
default_database:
|
||||
errors_count: 0
|
||||
estimated_recovery_time: 0
|
||||
|
||||
2 rows in set. Elapsed: 0.002 sec.
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/clusters) <!--hide-->
|
||||
|
||||
@ -23,4 +23,50 @@ The `system.columns` table contains the following columns (the column type is sh
|
||||
- `is_in_sampling_key` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Flag that indicates whether the column is in the sampling key expression.
|
||||
- `compression_codec` ([String](../../sql-reference/data-types/string.md)) — Compression codec name.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
:) select * from system.columns LIMIT 2 FORMAT Vertical;
|
||||
```
|
||||
|
||||
```text
|
||||
Row 1:
|
||||
──────
|
||||
database: system
|
||||
table: aggregate_function_combinators
|
||||
name: name
|
||||
type: String
|
||||
default_kind:
|
||||
default_expression:
|
||||
data_compressed_bytes: 0
|
||||
data_uncompressed_bytes: 0
|
||||
marks_bytes: 0
|
||||
comment:
|
||||
is_in_partition_key: 0
|
||||
is_in_sorting_key: 0
|
||||
is_in_primary_key: 0
|
||||
is_in_sampling_key: 0
|
||||
compression_codec:
|
||||
|
||||
Row 2:
|
||||
──────
|
||||
database: system
|
||||
table: aggregate_function_combinators
|
||||
name: is_internal
|
||||
type: UInt8
|
||||
default_kind:
|
||||
default_expression:
|
||||
data_compressed_bytes: 0
|
||||
data_uncompressed_bytes: 0
|
||||
marks_bytes: 0
|
||||
comment:
|
||||
is_in_partition_key: 0
|
||||
is_in_sorting_key: 0
|
||||
is_in_primary_key: 0
|
||||
is_in_sampling_key: 0
|
||||
compression_codec:
|
||||
|
||||
2 rows in set. Elapsed: 0.002 sec.
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/columns) <!--hide-->
|
||||
|
||||
@ -11,3 +11,21 @@ Columns:
|
||||
- `keep_free_space` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Amount of disk space that should stay free on disk in bytes. Defined in the `keep_free_space_bytes` parameter of disk configuration.
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/disks) <!--hide-->
|
||||
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
:) SELECT * FROM system.disks;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─name────┬─path─────────────────┬───free_space─┬──total_space─┬─keep_free_space─┐
|
||||
│ default │ /var/lib/clickhouse/ │ 276392587264 │ 490652508160 │ 0 │
|
||||
└─────────┴──────────────────────┴──────────────┴──────────────┴─────────────────┘
|
||||
|
||||
1 rows in set. Elapsed: 0.001 sec.
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
@ -8,3 +8,26 @@ Columns:
|
||||
- `is_aggregate`(`UInt8`) — Whether the function is aggregate.
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/functions) <!--hide-->
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT * FROM system.functions LIMIT 10;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─name─────────────────────┬─is_aggregate─┬─case_insensitive─┬─alias_to─┐
|
||||
│ sumburConsistentHash │ 0 │ 0 │ │
|
||||
│ yandexConsistentHash │ 0 │ 0 │ │
|
||||
│ demangle │ 0 │ 0 │ │
|
||||
│ addressToLine │ 0 │ 0 │ │
|
||||
│ JSONExtractRaw │ 0 │ 0 │ │
|
||||
│ JSONExtractKeysAndValues │ 0 │ 0 │ │
|
||||
│ JSONExtract │ 0 │ 0 │ │
|
||||
│ JSONExtractString │ 0 │ 0 │ │
|
||||
│ JSONExtractFloat │ 0 │ 0 │ │
|
||||
│ JSONExtractInt │ 0 │ 0 │ │
|
||||
└──────────────────────────┴──────────────┴──────────────────┴──────────┘
|
||||
|
||||
10 rows in set. Elapsed: 0.002 sec.
|
||||
```
|
||||
@ -10,4 +10,45 @@ Columns:
|
||||
- `type` (String) — Setting type (implementation specific string value).
|
||||
- `changed` (UInt8) — Whether the setting was explicitly defined in the config or explicitly changed.
|
||||
|
||||
**Example**
|
||||
```sql
|
||||
:) SELECT * FROM system.merge_tree_settings LIMIT 4 FORMAT Vertical;
|
||||
```
|
||||
|
||||
```text
|
||||
Row 1:
|
||||
──────
|
||||
name: index_granularity
|
||||
value: 8192
|
||||
changed: 0
|
||||
description: How many rows correspond to one primary key value.
|
||||
type: SettingUInt64
|
||||
|
||||
Row 2:
|
||||
──────
|
||||
name: min_bytes_for_wide_part
|
||||
value: 0
|
||||
changed: 0
|
||||
description: Minimal uncompressed size in bytes to create part in wide format instead of compact
|
||||
type: SettingUInt64
|
||||
|
||||
Row 3:
|
||||
──────
|
||||
name: min_rows_for_wide_part
|
||||
value: 0
|
||||
changed: 0
|
||||
description: Minimal number of rows to create part in wide format instead of compact
|
||||
type: SettingUInt64
|
||||
|
||||
Row 4:
|
||||
──────
|
||||
name: merge_max_block_size
|
||||
value: 8192
|
||||
changed: 0
|
||||
description: How many rows in blocks should be formed for merge operations.
|
||||
type: SettingUInt64
|
||||
|
||||
4 rows in set. Elapsed: 0.001 sec.
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/merge_tree_settings) <!--hide-->
|
||||
|
||||
@ -6,4 +6,27 @@ You can use this table for tests, or if you need to do a brute force search.
|
||||
|
||||
Reads from this table are not parallelized.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
:) SELECT * FROM system.numbers LIMIT 10;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─number─┐
|
||||
│ 0 │
|
||||
│ 1 │
|
||||
│ 2 │
|
||||
│ 3 │
|
||||
│ 4 │
|
||||
│ 5 │
|
||||
│ 6 │
|
||||
│ 7 │
|
||||
│ 8 │
|
||||
│ 9 │
|
||||
└────────┘
|
||||
|
||||
10 rows in set. Elapsed: 0.001 sec.
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/numbers) <!--hide-->
|
||||
|
||||
@ -4,4 +4,27 @@ The same as [system.numbers](../../operations/system-tables/numbers.md) but read
|
||||
|
||||
Used for tests.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
:) SELECT * FROM system.numbers_mt LIMIT 10;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─number─┐
|
||||
│ 0 │
|
||||
│ 1 │
|
||||
│ 2 │
|
||||
│ 3 │
|
||||
│ 4 │
|
||||
│ 5 │
|
||||
│ 6 │
|
||||
│ 7 │
|
||||
│ 8 │
|
||||
│ 9 │
|
||||
└────────┘
|
||||
|
||||
10 rows in set. Elapsed: 0.001 sec.
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/numbers_mt) <!--hide-->
|
||||
|
||||
@ -6,4 +6,18 @@ This table is used if a `SELECT` query doesn’t specify the `FROM` clause.
|
||||
|
||||
This is similar to the `DUAL` table found in other DBMSs.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
:) SELECT * FROM system.one LIMIT 10;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─dummy─┐
|
||||
│ 0 │
|
||||
└───────┘
|
||||
|
||||
1 rows in set. Elapsed: 0.001 sec.
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/one) <!--hide-->
|
||||
|
||||
@ -14,4 +14,51 @@ Columns:
|
||||
- `query` (String) – The query text. For `INSERT`, it doesn’t include the data to insert.
|
||||
- `query_id` (String) – Query ID, if defined.
|
||||
|
||||
|
||||
```sql
|
||||
:) SELECT * FROM system.processes LIMIT 10 FORMAT Vertical;
|
||||
```
|
||||
|
||||
```text
|
||||
Row 1:
|
||||
──────
|
||||
is_initial_query: 1
|
||||
user: default
|
||||
query_id: 35a360fa-3743-441d-8e1f-228c938268da
|
||||
address: ::ffff:172.23.0.1
|
||||
port: 47588
|
||||
initial_user: default
|
||||
initial_query_id: 35a360fa-3743-441d-8e1f-228c938268da
|
||||
initial_address: ::ffff:172.23.0.1
|
||||
initial_port: 47588
|
||||
interface: 1
|
||||
os_user: bharatnc
|
||||
client_hostname: tower
|
||||
client_name: ClickHouse
|
||||
client_revision: 54437
|
||||
client_version_major: 20
|
||||
client_version_minor: 7
|
||||
client_version_patch: 2
|
||||
http_method: 0
|
||||
http_user_agent:
|
||||
quota_key:
|
||||
elapsed: 0.000582537
|
||||
is_cancelled: 0
|
||||
read_rows: 0
|
||||
read_bytes: 0
|
||||
total_rows_approx: 0
|
||||
written_rows: 0
|
||||
written_bytes: 0
|
||||
memory_usage: 0
|
||||
peak_memory_usage: 0
|
||||
query: SELECT * from system.processes LIMIT 10 FORMAT Vertical;
|
||||
thread_ids: [67]
|
||||
ProfileEvents.Names: ['Query','SelectQuery','ReadCompressedBytes','CompressedReadBufferBlocks','CompressedReadBufferBytes','IOBufferAllocs','IOBufferAllocBytes','ContextLock','RWLockAcquiredReadLocks']
|
||||
ProfileEvents.Values: [1,1,36,1,10,1,89,16,1]
|
||||
Settings.Names: ['use_uncompressed_cache','load_balancing','log_queries','max_memory_usage']
|
||||
Settings.Values: ['0','in_order','1','10000000000']
|
||||
|
||||
1 rows in set. Elapsed: 0.002 sec.
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/processes) <!--hide-->
|
||||
|
||||
@ -10,6 +10,7 @@ Columns:
|
||||
- `disks` ([Array(String)](../../sql-reference/data-types/array.md)) — Disk names, defined in the storage policy.
|
||||
- `max_data_part_size` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Maximum size of a data part that can be stored on volume disks (0 — no limit).
|
||||
- `move_factor` ([Float64](../../sql-reference/data-types/float.md)) — Ratio of free disk space. When the ratio exceeds the value of configuration parameter, ClickHouse start to move data to the next volume in order.
|
||||
- `prefer_not_to_merge` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Value of the `prefer_not_to_merge` setting. When this setting is enabled, merging data on this volume is not allowed. This allows controlling how ClickHouse works with slow disks.
|
||||
|
||||
If the storage policy contains more then one volume, then information for each volume is stored in the individual row of the table.
|
||||
|
||||
|
||||
@ -52,4 +52,56 @@ This table contains the following columns (the column type is shown in brackets)
|
||||
|
||||
The `system.tables` table is used in `SHOW TABLES` query implementation.
|
||||
|
||||
```sql
|
||||
:) SELECT * FROM system.tables LIMIT 2 FORMAT Vertical;
|
||||
```
|
||||
|
||||
```text
|
||||
Row 1:
|
||||
──────
|
||||
database: system
|
||||
name: aggregate_function_combinators
|
||||
uuid: 00000000-0000-0000-0000-000000000000
|
||||
engine: SystemAggregateFunctionCombinators
|
||||
is_temporary: 0
|
||||
data_paths: []
|
||||
metadata_path: /var/lib/clickhouse/metadata/system/aggregate_function_combinators.sql
|
||||
metadata_modification_time: 1970-01-01 03:00:00
|
||||
dependencies_database: []
|
||||
dependencies_table: []
|
||||
create_table_query:
|
||||
engine_full:
|
||||
partition_key:
|
||||
sorting_key:
|
||||
primary_key:
|
||||
sampling_key:
|
||||
storage_policy:
|
||||
total_rows: ᴺᵁᴸᴸ
|
||||
total_bytes: ᴺᵁᴸᴸ
|
||||
|
||||
Row 2:
|
||||
──────
|
||||
database: system
|
||||
name: asynchronous_metrics
|
||||
uuid: 00000000-0000-0000-0000-000000000000
|
||||
engine: SystemAsynchronousMetrics
|
||||
is_temporary: 0
|
||||
data_paths: []
|
||||
metadata_path: /var/lib/clickhouse/metadata/system/asynchronous_metrics.sql
|
||||
metadata_modification_time: 1970-01-01 03:00:00
|
||||
dependencies_database: []
|
||||
dependencies_table: []
|
||||
create_table_query:
|
||||
engine_full:
|
||||
partition_key:
|
||||
sorting_key:
|
||||
primary_key:
|
||||
sampling_key:
|
||||
storage_policy:
|
||||
total_rows: ᴺᵁᴸᴸ
|
||||
total_bytes: ᴺᵁᴸᴸ
|
||||
|
||||
2 rows in set. Elapsed: 0.004 sec.
|
||||
```
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/tables) <!--hide-->
|
||||
|
||||
@ -9,7 +9,7 @@ toc_title: DELETE
|
||||
ALTER TABLE [db.]table [ON CLUSTER cluster] DELETE WHERE filter_expr
|
||||
```
|
||||
|
||||
Allows to delete data matching the specified filtering expression. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
Deletes data matching the specified filtering expression. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
!!! note "Note"
|
||||
The `ALTER TABLE` prefix makes this syntax different from most other systems supporting SQL. It is intended to signify that unlike similar queries in OLTP databases this is a heavy operation not designed for frequent use.
|
||||
|
||||
@ -19,5 +19,4 @@ The first two commands are lightweight in a sense that they only change metadata
|
||||
Also, they are replicated, syncing indices metadata via ZooKeeper.
|
||||
|
||||
!!! note "Note"
|
||||
Index manipulation is supported only for tables with [`*MergeTree`](../../../../engines/table-engines/mergetree-family/mergetree.md) engine (including
|
||||
[replicated](../../../../engines/table-engines/mergetree-family/replication.md) variants).
|
||||
Index manipulation is supported only for tables with [`*MergeTree`](../../../../engines/table-engines/mergetree-family/mergetree.md) engine (including [replicated](../../../../engines/table-engines/mergetree-family/replication.md) variants).
|
||||
|
||||
@ -241,6 +241,46 @@ ALTER TABLE hits MOVE PART '20190301_14343_16206_438' TO VOLUME 'slow'
|
||||
ALTER TABLE hits MOVE PARTITION '2019-09-01' TO DISK 'fast_ssd'
|
||||
```
|
||||
|
||||
## UPDATE IN PARTITION {#update-in-partition}
|
||||
|
||||
Manipulates data in the specifies partition matching the specified filtering expression. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] [IN PARTITION partition_id] WHERE filter_expr
|
||||
```
|
||||
|
||||
### Example
|
||||
|
||||
``` sql
|
||||
ALTER TABLE mt UPDATE x = x + 1 IN PARTITION 2 WHERE p = 2;
|
||||
```
|
||||
|
||||
### See Also
|
||||
|
||||
- [UPDATE](../../../sql-reference/statements/alter/update.md#alter-table-update-statements)
|
||||
|
||||
## DELETE IN PARTITION {#delete-in-partition}
|
||||
|
||||
Deletes data in the specifies partition matching the specified filtering expression. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db.]table DELETE [IN PARTITION partition_id] WHERE filter_expr
|
||||
```
|
||||
|
||||
### Example
|
||||
|
||||
``` sql
|
||||
ALTER TABLE mt DELETE IN PARTITION 2 WHERE p = 2;
|
||||
```
|
||||
|
||||
### See Also
|
||||
|
||||
- [DELETE](../../../sql-reference/statements/alter/delete.md#alter-mutations)
|
||||
|
||||
## How to Set Partition Expression {#alter-how-to-specify-part-expr}
|
||||
|
||||
You can specify the partition expression in `ALTER ... PARTITION` queries in different ways:
|
||||
@ -258,4 +298,6 @@ All the rules above are also true for the [OPTIMIZE](../../../sql-reference/stat
|
||||
OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL;
|
||||
```
|
||||
|
||||
`IN PARTITION` specifies the partition to which the [UPDATE](../../../sql-reference/statements/alter/update.md#alter-table-update-statements) or [DELETE](../../../sql-reference/statements/alter/delete.md#alter-mutations) expressions are applied as a result of the `ALTER TABLE` query. New parts are created only from the specified partition. In this way, `IN PARTITION` helps to reduce the load when the table is divided into many partitions, and you only need to update the data point-by-point.
|
||||
|
||||
The examples of `ALTER ... PARTITION` queries are demonstrated in the tests [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) and [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql).
|
||||
|
||||
@ -9,7 +9,7 @@ toc_title: UPDATE
|
||||
ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr
|
||||
```
|
||||
|
||||
Allows to manipulate data matching the specified filtering expression. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
Manipulates data matching the specified filtering expression. Implemented as a [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
!!! note "Note"
|
||||
The `ALTER TABLE` prefix makes this syntax different from most other systems supporting SQL. It is intended to signify that unlike similar queries in OLTP databases this is a heavy operation not designed for frequent use.
|
||||
|
||||
@ -152,7 +152,7 @@ ClickHouse can manage background processes in [MergeTree](../../engines/table-en
|
||||
Provides possibility to stop background merges for tables in the MergeTree family:
|
||||
|
||||
``` sql
|
||||
SYSTEM STOP MERGES [[db.]merge_tree_family_table_name]
|
||||
SYSTEM STOP MERGES [ON VOLUME <volume_name> | [db.]merge_tree_family_table_name]
|
||||
```
|
||||
|
||||
!!! note "Note"
|
||||
@ -163,7 +163,7 @@ SYSTEM STOP MERGES [[db.]merge_tree_family_table_name]
|
||||
Provides possibility to start background merges for tables in the MergeTree family:
|
||||
|
||||
``` sql
|
||||
SYSTEM START MERGES [[db.]merge_tree_family_table_name]
|
||||
SYSTEM START MERGES [ON VOLUME <volume_name> | [db.]merge_tree_family_table_name]
|
||||
```
|
||||
|
||||
### STOP TTL MERGES {#query_language-stop-ttl-merges}
|
||||
|
||||
@ -57,7 +57,7 @@ Identifiers are:
|
||||
|
||||
Identifiers can be quoted or non-quoted. The latter is preferred.
|
||||
|
||||
Non-quoted identifiers must match the regex `^[0-9a-zA-Z_]*[a-zA-Z_]$` and can not be equal to [keywords](#syntax-keywords). Examples: `x, _1, X_y__Z123_.`
|
||||
Non-quoted identifiers must match the regex `^[a-zA-Z_][0-9a-zA-Z_]*$` and can not be equal to [keywords](#syntax-keywords). Examples: `x`, `_1`, `X_y__Z123_`.
|
||||
|
||||
If you want to use identifiers the same as keywords or you want to use other symbols in identifiers, quote it using double quotes or backticks, for example, `"id"`, `` `id` ``.
|
||||
|
||||
|
||||
@ -565,6 +565,7 @@ ALTER TABLE example_table
|
||||
- `disk` — диск, находящийся внутри тома.
|
||||
- `max_data_part_size_bytes` — максимальный размер куска данных, который может находится на любом из дисков этого тома.
|
||||
- `move_factor` — доля доступного свободного места на томе, если места становится меньше, то данные начнут перемещение на следующий том, если он есть (по умолчанию 0.1).
|
||||
- `prefer_not_to_merge` — Отключает слияние кусков данных, хранящихся на данном томе. Если данная настройка включена, то слияние данных, хранящихся на данном томе, не допускается. Это позволяет контролировать работу ClickHouse с медленными дисками.
|
||||
|
||||
Примеры конфигураций:
|
||||
|
||||
@ -593,6 +594,19 @@ ALTER TABLE example_table
|
||||
</volumes>
|
||||
<move_factor>0.2</move_factor>
|
||||
</moving_from_ssd_to_hdd>
|
||||
|
||||
<small_jbod_with_external_no_merges>
|
||||
<volumes>
|
||||
<main>
|
||||
<disk>jbod1</disk>
|
||||
</main>
|
||||
<external>
|
||||
<disk>external</disk>
|
||||
<prefer_not_to_merge>true</prefer_not_to_merge>
|
||||
</external>
|
||||
</volumes>
|
||||
</small_jbod_with_external_no_merges>
|
||||
|
||||
</policies>
|
||||
...
|
||||
</storage_configuration>
|
||||
|
||||
@ -10,6 +10,7 @@
|
||||
- `disks` ([Array(String)](../../sql-reference/data-types/array.md)) — имена дисков, содержащихся в политике хранения.
|
||||
- `max_data_part_size` ([UInt64](../../sql-reference/data-types/int-uint.md)) — максимальный размер куска данных, который может храниться на дисках тома (0 — без ограничений).
|
||||
- `move_factor` — доля доступного свободного места на томе, если места становится меньше, то данные начнут перемещение на следующий том, если он есть (по умолчанию 0.1).
|
||||
- `prefer_not_to_merge` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Значение настройки `prefer_not_to_merge`. Если данная настройка включена, то слияние данных, хранящихся на данном томе, не допускается. Это позволяет контролировать работу ClickHouse с медленными дисками.
|
||||
|
||||
Если политика хранения содержит несколько томов, то каждому тому соответствует отдельная запись в таблице.
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@ toc_title: DELETE
|
||||
ALTER TABLE [db.]table [ON CLUSTER cluster] DELETE WHERE filter_expr
|
||||
```
|
||||
|
||||
Позволяет удалить данные, соответствующие указанному выражению фильтрации. Реализовано как [мутация](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
Удаляет данные, соответствующие указанному выражению фильтрации. Реализовано как [мутация](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
!!! note "Note"
|
||||
Префикс `ALTER TABLE` делает этот синтаксис отличным от большинства других систем, поддерживающих SQL. Он предназначен для обозначения того, что в отличие от аналогичных запросов в базах данных OLTP это тяжелая операция, не предназначенная для частого использования.
|
||||
|
||||
@ -243,6 +243,46 @@ ALTER TABLE hits MOVE PART '20190301_14343_16206_438' TO VOLUME 'slow'
|
||||
ALTER TABLE hits MOVE PARTITION '2019-09-01' TO DISK 'fast_ssd'
|
||||
```
|
||||
|
||||
## UPDATE IN PARTITION {#update-in-partition}
|
||||
|
||||
Манипулирует данными в указанной партиции, соответствующими заданному выражению фильтрации. Реализовано как мутация [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
Синтаксис:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] [IN PARTITION partition_id] WHERE filter_expr
|
||||
```
|
||||
|
||||
### Пример
|
||||
|
||||
``` sql
|
||||
ALTER TABLE mt UPDATE x = x + 1 IN PARTITION 2 WHERE p = 2;
|
||||
```
|
||||
|
||||
### Смотрите также
|
||||
|
||||
- [UPDATE](../../../sql-reference/statements/alter/update.md#alter-table-update-statements)
|
||||
|
||||
## DELETE IN PARTITION {#delete-in-partition}
|
||||
|
||||
Удаляет данные в указанной партиции, соответствующие указанному выражению фильтрации. Реализовано как мутация [mutation](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
Синтаксис:
|
||||
|
||||
``` sql
|
||||
ALTER TABLE [db.]table DELETE [IN PARTITION partition_id] WHERE filter_expr
|
||||
```
|
||||
|
||||
### Пример
|
||||
|
||||
``` sql
|
||||
ALTER TABLE mt DELETE IN PARTITION 2 WHERE p = 2;
|
||||
```
|
||||
|
||||
### Смотрите также
|
||||
|
||||
- [DELETE](../../../sql-reference/statements/alter/delete.md#alter-mutations)
|
||||
|
||||
## Как задавать имя партиции в запросах ALTER {#alter-how-to-specify-part-expr}
|
||||
|
||||
Чтобы задать нужную партицию в запросах `ALTER ... PARTITION`, можно использовать:
|
||||
@ -262,6 +302,8 @@ ALTER TABLE hits MOVE PARTITION '2019-09-01' TO DISK 'fast_ssd'
|
||||
OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL;
|
||||
```
|
||||
|
||||
`IN PARTITION` указывает на партицию, для которой применяются выражения [UPDATE](../../../sql-reference/statements/alter/update.md#alter-table-update-statements) или [DELETE](../../../sql-reference/statements/alter/delete.md#alter-mutations) в результате запроса `ALTER TABLE`. Новые куски создаются только в указанной партиции. Таким образом, `IN PARTITION` помогает снизить нагрузку, когда таблица разбита на множество партиций, а вам нужно обновить данные лишь точечно.
|
||||
|
||||
Примеры запросов `ALTER ... PARTITION` можно посмотреть в тестах: [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) и [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql).
|
||||
|
||||
[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/partition/) <!--hide-->
|
||||
@ -9,7 +9,7 @@ toc_title: UPDATE
|
||||
ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr
|
||||
```
|
||||
|
||||
Позволяет манипулировать данными, соответствующими заданному выражению фильтрации. Реализовано как [мутация](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
Манипулирует данными, соответствующими заданному выражению фильтрации. Реализовано как [мутация](../../../sql-reference/statements/alter/index.md#mutations).
|
||||
|
||||
!!! note "Note"
|
||||
Префикс `ALTER TABLE` делает этот синтаксис отличным от большинства других систем, поддерживающих SQL. Он предназначен для обозначения того, что в отличие от аналогичных запросов в базах данных OLTP это тяжелая операция, не предназначенная для частого использования.
|
||||
|
||||
@ -130,7 +130,7 @@ ClickHouse может управлять фоновыми процессами
|
||||
Позволяет остановить фоновые мержи для таблиц семейства MergeTree:
|
||||
|
||||
``` sql
|
||||
SYSTEM STOP MERGES [[db.]merge_tree_family_table_name]
|
||||
SYSTEM STOP MERGES [ON VOLUME <volume_name> | [db.]merge_tree_family_table_name]
|
||||
```
|
||||
|
||||
!!! note "Note"
|
||||
@ -141,7 +141,7 @@ SYSTEM STOP MERGES [[db.]merge_tree_family_table_name]
|
||||
Включает фоновые мержи для таблиц семейства MergeTree:
|
||||
|
||||
``` sql
|
||||
SYSTEM START MERGES [[db.]merge_tree_family_table_name]
|
||||
SYSTEM START MERGES [ON VOLUME <volume_name> | [db.]merge_tree_family_table_name]
|
||||
```
|
||||
|
||||
### STOP TTL MERGES {#query_language-stop-ttl-merges}
|
||||
|
||||
@ -1,13 +1,108 @@
|
||||
# 条件函数 {#tiao-jian-han-shu}
|
||||
|
||||
## 如果（cond，那么，否则），cond？ 运算符然后：else {#ifcond-then-else-cond-operator-then-else}
|
||||
## if {#if}
|
||||
|
||||
控制条件分支。 与大多数系统不同，ClickHouse始终评估两个表达式 `then` 和 `else`。
|
||||
|
||||
**语法**
|
||||
|
||||
``` sql
|
||||
SELECT if(cond, then, else)
|
||||
```
|
||||
|
||||
如果条件 `cond` 的计算结果为非零值，则返回表达式 `then` 的结果，并且跳过表达式 `else` 的结果（如果存在）。 如果 `cond` 为零或 `NULL`，则将跳过 `then` 表达式的结果，并返回 `else` 表达式的结果（如果存在）。
|
||||
|
||||
**参数**
|
||||
|
||||
- `cond` – 条件结果可以为零或不为零。 类型是 UInt8，Nullable(UInt8) 或 NULL。
|
||||
- `then` - 如果满足条件则返回的表达式。
|
||||
- `else` - 如果不满足条件则返回的表达式。
|
||||
|
||||
**返回值**
|
||||
|
||||
该函数执行 `then` 和 `else` 表达式并返回其结果，这取决于条件 `cond` 最终是否为零。
|
||||
|
||||
**示例**
|
||||
|
||||
查询:
|
||||
|
||||
``` sql
|
||||
SELECT if(1, plus(2, 2), plus(2, 6))
|
||||
```
|
||||
|
||||
结果:
|
||||
|
||||
``` text
|
||||
┌─plus(2, 2)─┐
|
||||
│ 4 │
|
||||
└────────────┘
|
||||
```
|
||||
|
||||
查询:
|
||||
|
||||
``` sql
|
||||
SELECT if(0, plus(2, 2), plus(2, 6))
|
||||
```
|
||||
|
||||
结果:
|
||||
|
||||
``` text
|
||||
┌─plus(2, 6)─┐
|
||||
│ 8 │
|
||||
└────────────┘
|
||||
```
|
||||
|
||||
- `then` 和 `else` 必须具有最低的通用类型。
|
||||
|
||||
**示例:**
|
||||
|
||||
给定表`LEFT_RIGHT`:
|
||||
|
||||
``` sql
|
||||
SELECT *
|
||||
FROM LEFT_RIGHT
|
||||
|
||||
┌─left─┬─right─┐
|
||||
│ ᴺᵁᴸᴸ │ 4 │
|
||||
│ 1 │ 3 │
|
||||
│ 2 │ 2 │
|
||||
│ 3 │ 1 │
|
||||
│ 4 │ ᴺᵁᴸᴸ │
|
||||
└──────┴───────┘
|
||||
```
|
||||
|
||||
下面的查询比较了 `left` 和 `right` 的值:
|
||||
|
||||
``` sql
|
||||
SELECT
|
||||
left,
|
||||
right,
|
||||
if(left < right, 'left is smaller than right', 'right is greater or equal than left') AS is_smaller
|
||||
FROM LEFT_RIGHT
|
||||
WHERE isNotNull(left) AND isNotNull(right)
|
||||
|
||||
┌─left─┬─right─┬─is_smaller──────────────────────────┐
|
||||
│ 1 │ 3 │ left is smaller than right │
|
||||
│ 2 │ 2 │ right is greater or equal than left │
|
||||
│ 3 │ 1 │ right is greater or equal than left │
|
||||
└──────┴───────┴─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
注意：在此示例中未使用'NULL'值，请检查[条件中的NULL值](#null-values-in-conditionals) 部分。
|
||||
|
||||
## 三元运算符 {#ternary-operator}
|
||||
|
||||
与 `if` 函数相同。
|
||||
|
||||
语法: `cond ? then : else`
|
||||
|
||||
如果`cond ！= 0`则返回`then`，如果`cond = 0`则返回`else`。
|
||||
`cond`必须是`UInt8`类型，`then`和`else`必须存在最低的共同类型。
|
||||
|
||||
`then`和`else`可以是`NULL`
|
||||
- `cond`必须是`UInt8`类型，`then`和`else`必须存在最低的共同类型。
|
||||
|
||||
## 多 {#multiif}
|
||||
- `then`和`else`可以是`NULL`
|
||||
|
||||
## multiIf {#multiif}
|
||||
|
||||
允许您在查询中更紧凑地编写[CASE](../operators/index.md#operator_case)运算符。
|
||||
|
||||
@ -27,18 +122,74 @@
|
||||
|
||||
**示例**
|
||||
|
||||
存在如下一张表
|
||||
再次使用表 `LEFT_RIGHT` 。
|
||||
|
||||
┌─x─┬────y─┐
|
||||
│ 1 │ ᴺᵁᴸᴸ │
|
||||
│ 2 │ 3 │
|
||||
└───┴──────┘
|
||||
``` sql
|
||||
SELECT
|
||||
left,
|
||||
right,
|
||||
multiIf(left < right, 'left is smaller', left > right, 'left is greater', left = right, 'Both equal', 'Null value') AS result
|
||||
FROM LEFT_RIGHT
|
||||
|
||||
执行查询 `SELECT multiIf(isNull(y), x, y < 3, y, NULL) FROM t_null`。结果：
|
||||
┌─left─┬─right─┬─result──────────┐
|
||||
│ ᴺᵁᴸᴸ │ 4 │ Null value │
|
||||
│ 1 │ 3 │ left is smaller │
|
||||
│ 2 │ 2 │ Both equal │
|
||||
│ 3 │ 1 │ left is greater │
|
||||
│ 4 │ ᴺᵁᴸᴸ │ Null value │
|
||||
└──────┴───────┴─────────────────┘
|
||||
```
|
||||
## 直接使用条件结果 {#using-conditional-results-directly}
|
||||
|
||||
┌─multiIf(isNull(y), x, less(y, 3), y, NULL)─┐
|
||||
│ 1 │
|
||||
│ ᴺᵁᴸᴸ │
|
||||
└────────────────────────────────────────────┘
|
||||
条件结果始终为 `0`、 `1` 或 `NULL`。 因此，你可以像这样直接使用条件结果：
|
||||
|
||||
``` sql
|
||||
SELECT left < right AS is_small
|
||||
FROM LEFT_RIGHT
|
||||
|
||||
┌─is_small─┐
|
||||
│ ᴺᵁᴸᴸ │
|
||||
│ 1 │
|
||||
│ 0 │
|
||||
│ 0 │
|
||||
│ ᴺᵁᴸᴸ │
|
||||
└──────────┘
|
||||
```
|
||||
|
||||
## 条件中的NULL值 {#null-values-in-conditionals}
|
||||
|
||||
当条件中包含 `NULL` 值时，结果也将为 `NULL`。
|
||||
|
||||
``` sql
|
||||
SELECT
|
||||
NULL < 1,
|
||||
2 < NULL,
|
||||
NULL < NULL,
|
||||
NULL = NULL
|
||||
|
||||
┌─less(NULL, 1)─┬─less(2, NULL)─┬─less(NULL, NULL)─┬─equals(NULL, NULL)─┐
|
||||
│ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │
|
||||
└───────────────┴───────────────┴──────────────────┴────────────────────┘
|
||||
```
|
||||
|
||||
因此，如果类型是 `Nullable`，你应该仔细构造查询。
|
||||
|
||||
以下示例说明这一点。
|
||||
|
||||
``` sql
|
||||
SELECT
|
||||
left,
|
||||
right,
|
||||
multiIf(left < right, 'left is smaller', left > right, 'right is smaller', 'Both equal') AS faulty_result
|
||||
FROM LEFT_RIGHT
|
||||
|
||||
┌─left─┬─right─┬─faulty_result────┐
|
||||
│ ᴺᵁᴸᴸ │ 4 │ Both equal │
|
||||
│ 1 │ 3 │ left is smaller │
|
||||
│ 2 │ 2 │ Both equal │
|
||||
│ 3 │ 1 │ right is smaller │
|
||||
│ 4 │ ᴺᵁᴸᴸ │ Both equal │
|
||||
└──────┴───────┴──────────────────┘
|
||||
```
|
||||
|
||||
[来源文章](https://clickhouse.tech/docs/en/query_language/functions/conditional_functions/) <!--hide-->
|
||||
|
||||
@ -1,5 +1,71 @@
|
||||
# 编码函数 {#bian-ma-han-shu}
|
||||
|
||||
## char {#char}
|
||||
|
||||
返回长度为传递参数数量的字符串，并且每个字节都有对应参数的值。接受数字Numeric类型的多个参数。如果参数的值超出了UInt8数据类型的范围，则将其转换为UInt8，并可能进行舍入和溢出。
|
||||
|
||||
**语法**
|
||||
|
||||
``` sql
|
||||
char(number_1, [number_2, ..., number_n]);
|
||||
```
|
||||
|
||||
**参数**
|
||||
|
||||
- `number_1, number_2, ..., number_n` — 数值参数解释为整数。类型: [Int](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md).
|
||||
|
||||
**返回值**
|
||||
|
||||
- 给定字节数的字符串。
|
||||
|
||||
类型: `String`。
|
||||
|
||||
**示例**
|
||||
|
||||
查询:
|
||||
|
||||
``` sql
|
||||
SELECT char(104.1, 101, 108.9, 108.9, 111) AS hello
|
||||
```
|
||||
|
||||
结果:
|
||||
|
||||
``` text
|
||||
┌─hello─┐
|
||||
│ hello │
|
||||
└───────┘
|
||||
```
|
||||
|
||||
你可以通过传递相应的字节来构造任意编码的字符串。 这是UTF-8的示例:
|
||||
|
||||
查询:
|
||||
|
||||
``` sql
|
||||
SELECT char(0xD0, 0xBF, 0xD1, 0x80, 0xD0, 0xB8, 0xD0, 0xB2, 0xD0, 0xB5, 0xD1, 0x82) AS hello;
|
||||
```
|
||||
|
||||
结果:
|
||||
|
||||
``` text
|
||||
┌─hello──┐
|
||||
│ привет │
|
||||
└────────┘
|
||||
```
|
||||
|
||||
查询:
|
||||
|
||||
``` sql
|
||||
SELECT char(0xE4, 0xBD, 0xA0, 0xE5, 0xA5, 0xBD) AS hello;
|
||||
```
|
||||
|
||||
结果:
|
||||
|
||||
``` text
|
||||
┌─hello─┐
|
||||
│ 你好 │
|
||||
└───────┘
|
||||
```
|
||||
|
||||
## hex {#hex}
|
||||
|
||||
接受`String`，`unsigned integer`，`Date`或`DateTime`类型的参数。返回包含参数的十六进制表示的字符串。使用大写字母`A-F`。不使用`0x`前缀或`h`后缀。对于字符串，所有字节都简单地编码为两个十六进制数字。数字转换为大端（«易阅读»）格式。对于数字，去除其中较旧的零，但仅限整个字节。例如，`hex（1）='01'`。 `Date`被编码为自Unix时间开始以来的天数。 `DateTime`编码为自Unix时间开始以来的秒数。
|
||||
@ -17,11 +83,11 @@
|
||||
|
||||
接受FixedString（16）值。返回包含36个字符的文本格式的字符串。
|
||||
|
||||
## 位掩码列表(num) {#bitmasktolistnum}
|
||||
## bitmaskToList(num) {#bitmasktolistnum}
|
||||
|
||||
接受一个整数。返回一个字符串，其中包含一组2的幂列表，其列表中的所有值相加等于这个整数。列表使用逗号分割，按升序排列。
|
||||
|
||||
## 位掩码阵列(num) {#bitmasktoarraynum}
|
||||
## bitmaskToArray(num) {#bitmasktoarraynum}
|
||||
|
||||
接受一个整数。返回一个UInt64类型数组，其中包含一组2的幂列表，其列表中的所有值相加等于这个整数。数组中的数字按升序排列。
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
您可以向它传递任何类型的参数，但传递的参数将不会使用在任何随机数生成过程中。
|
||||
此参数的唯一目的是防止公共子表达式消除，以便在相同的查询中使用相同的随机函数生成不同的随机数。
|
||||
|
||||
## 兰德 {#rand}
|
||||
## rand, rand32 {#rand}
|
||||
|
||||
返回一个UInt32类型的随机数字，所有UInt32类型的数字被生成的概率均相等。此函数线性同于的方式生成随机数。
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# 字符串函数 {#zi-fu-chuan-han-shu}
|
||||
|
||||
## 空 {#string-functions-empty}
|
||||
## empty {#string-functions-empty}
|
||||
|
||||
对于空字符串返回1，对于非空字符串返回0。
|
||||
结果类型是UInt8。
|
||||
@ -13,13 +13,13 @@
|
||||
结果类型是UInt8。
|
||||
该函数也适用于数组。
|
||||
|
||||
## 长度 {#length}
|
||||
## length {#length}
|
||||
|
||||
返回字符串的字节长度。
|
||||
结果类型是UInt64。
|
||||
该函数也适用于数组。
|
||||
|
||||
## 长度8 {#lengthutf8}
|
||||
## lengthUTF8 {#lengthutf8}
|
||||
|
||||
假定字符串以UTF-8编码组成的文本，返回此字符串的Unicode字符长度。如果传入的字符串不是UTF-8编码，则函数可能返回一个预期外的值（不会抛出异常）。
|
||||
结果类型是UInt64。
|
||||
@ -29,16 +29,16 @@
|
||||
假定字符串以UTF-8编码组成的文本，返回此字符串的Unicode字符长度。如果传入的字符串不是UTF-8编码，则函数可能返回一个预期外的值（不会抛出异常）。
|
||||
结果类型是UInt64。
|
||||
|
||||
## 字符长度,字符长度 {#character-length-character-length}
|
||||
## character_length,CHARACTER_LENGTH {#character-length-character-length}
|
||||
|
||||
假定字符串以UTF-8编码组成的文本，返回此字符串的Unicode字符长度。如果传入的字符串不是UTF-8编码，则函数可能返回一个预期外的值（不会抛出异常）。
|
||||
结果类型是UInt64。
|
||||
|
||||
## 低一点 {#lower-lcase}
|
||||
## lower, lcase {#lower-lcase}
|
||||
|
||||
将字符串中的ASCII转换为小写。
|
||||
|
||||
## 上,ucase {#upper-ucase}
|
||||
## upper, ucase {#upper-ucase}
|
||||
|
||||
将字符串中的ASCII转换为大写。
|
||||
|
||||
@ -84,7 +84,7 @@ SELECT toValidUTF8('\x61\xF0\x80\x80\x80b')
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
## 反向 {#reverse}
|
||||
## reverse {#reverse}
|
||||
|
||||
反转字符串。
|
||||
|
||||
@ -118,11 +118,11 @@ SELECT format('{} {}', 'Hello', 'World')
|
||||
|
||||
与[concat](#concat-s1-s2)相同，区别在于，你需要保证concat(s1, s2, s3) -\> s4是单射的，它将用于GROUP BY的优化。
|
||||
|
||||
## 子串(s,offset,length),mid(s,offset,length),substr(s,offset,length) {#substrings-offset-length-mids-offset-length-substrs-offset-length}
|
||||
## substring(s,offset,length),mid(s,offset,length),substr(s,offset,length) {#substrings-offset-length-mids-offset-length-substrs-offset-length}
|
||||
|
||||
以字节为单位截取指定位置字符串，返回以’offset’位置为开头，长度为’length’的子串。’offset’从1开始（与标准SQL相同）。’offset’和’length’参数必须是常量。
|
||||
|
||||
## substringf8(s,offset,length) {#substringutf8s-offset-length}
|
||||
## substringUTF8(s,offset,length) {#substringutf8s-offset-length}
|
||||
|
||||
与’substring’相同，但其操作单位为Unicode字符，函数假设字符串是以UTF-8进行编码的文本。如果不是则可能返回一个预期外的结果（不会抛出异常）。
|
||||
|
||||
@ -150,7 +150,7 @@ SELECT format('{} {}', 'Hello', 'World')
|
||||
|
||||
返回是否以指定的后缀结尾。如果字符串以指定的后缀结束，则返回1，否则返回0。
|
||||
|
||||
## 开始使用（s，前缀) {#startswiths-prefix}
|
||||
## startsWith（s，前缀) {#startswiths-prefix}
|
||||
|
||||
返回是否以指定的前缀开头。如果字符串以指定的前缀开头，则返回1，否则返回0。
|
||||
|
||||
|
||||
@ -151,7 +151,7 @@ DROP [ROW] POLICY [IF EXISTS] name [,...] ON [database.]table [,...] [ON CLUSTER
|
||||
|
||||
删除配额。
|
||||
|
||||
已删除的配额将从分配配额的所有实体撤销。
|
||||
已删除的配额将从分配该配额的所有实体撤销。
|
||||
|
||||
### 语法 {#drop-quota-syntax}
|
||||
|
||||
@ -161,9 +161,9 @@ DROP QUOTA [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
|
||||
|
||||
## DROP SETTINGS PROFILE {#drop-settings-profile-statement}
|
||||
|
||||
删除配额。
|
||||
删除settings配置。
|
||||
|
||||
已删除的配额将从分配配额的所有实体撤销。
|
||||
已删除的settings配置将从分配该settings配置的所有实体撤销。
|
||||
|
||||
### 语法 {#drop-settings-profile-syntax}
|
||||
|
||||
@ -177,7 +177,7 @@ DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name]
|
||||
EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT format]
|
||||
```
|
||||
|
||||
返回单 `UInt8`-type column，其中包含单个值 `0` 如果表或数据库不存在，或 `1` 如果该表存在于指定的数据库中。
|
||||
返回单个 `UInt8` 类型的列，其中包含单个值 `0` 如果表或数据库不存在，或 `1` 如果该表存在于指定的数据库中。
|
||||
|
||||
## KILL QUERY {#kill-query-statement}
|
||||
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
#include <AggregateFunctions/parseAggregateFunctionParameters.h>
|
||||
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ExpressionListParsers.h>
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Core/Defines.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -25,6 +25,13 @@ Array getAggregateFunctionParametersArray(const ASTPtr & expression_list, const
|
||||
for (size_t i = 0; i < parameters.size(); ++i)
|
||||
{
|
||||
const auto * literal = parameters[i]->as<ASTLiteral>();
|
||||
|
||||
ASTPtr func_literal;
|
||||
if (!literal)
|
||||
if (const auto * func = parameters[i]->as<ASTFunction>())
|
||||
if ((func_literal = func->toLiteral()))
|
||||
literal = func_literal->as<ASTLiteral>();
|
||||
|
||||
if (!literal)
|
||||
{
|
||||
throw Exception(
|
||||
|
||||
@ -6,18 +6,6 @@ if (USE_CLANG_TIDY)
|
||||
set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")
|
||||
endif ()
|
||||
|
||||
if(COMPILER_PIPE)
|
||||
set(MAX_COMPILER_MEMORY 2500)
|
||||
else()
|
||||
set(MAX_COMPILER_MEMORY 1500)
|
||||
endif()
|
||||
if(MAKE_STATIC_LIBRARIES)
|
||||
set(MAX_LINKER_MEMORY 3500)
|
||||
else()
|
||||
set(MAX_LINKER_MEMORY 2500)
|
||||
endif()
|
||||
include(../cmake/limit_jobs.cmake)
|
||||
|
||||
set (CONFIG_VERSION ${CMAKE_CURRENT_BINARY_DIR}/Common/config_version.h)
|
||||
set (CONFIG_COMMON ${CMAKE_CURRENT_BINARY_DIR}/Common/config.h)
|
||||
|
||||
@ -49,6 +37,7 @@ add_subdirectory (Dictionaries)
|
||||
add_subdirectory (Disks)
|
||||
add_subdirectory (Storages)
|
||||
add_subdirectory (Parsers)
|
||||
add_subdirectory (Parsers/New)
|
||||
add_subdirectory (IO)
|
||||
add_subdirectory (Functions)
|
||||
add_subdirectory (Interpreters)
|
||||
@ -186,12 +175,12 @@ endif()
|
||||
|
||||
if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES)
|
||||
add_library (dbms STATIC ${dbms_headers} ${dbms_sources})
|
||||
target_link_libraries (dbms PRIVATE jemalloc libdivide ${DBMS_COMMON_LIBRARIES})
|
||||
target_link_libraries (dbms PRIVATE clickhouse_parsers_new jemalloc libdivide ${DBMS_COMMON_LIBRARIES})
|
||||
set (all_modules dbms)
|
||||
else()
|
||||
add_library (dbms SHARED ${dbms_headers} ${dbms_sources})
|
||||
target_link_libraries (dbms PUBLIC ${all_modules} ${DBMS_COMMON_LIBRARIES})
|
||||
target_link_libraries (clickhouse_interpreters PRIVATE jemalloc libdivide)
|
||||
target_link_libraries (clickhouse_interpreters PRIVATE clickhouse_parsers_new jemalloc libdivide)
|
||||
list (APPEND all_modules dbms)
|
||||
# force all split libs to be linked
|
||||
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-as-needed")
|
||||
@ -329,6 +318,8 @@ dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${MINISELECT_INCLUDE_DIR})
|
||||
|
||||
if (ZSTD_LIBRARY)
|
||||
dbms_target_link_libraries(PRIVATE ${ZSTD_LIBRARY})
|
||||
target_link_libraries (clickhouse_common_io PUBLIC ${ZSTD_LIBRARY})
|
||||
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${ZSTD_INCLUDE_DIR})
|
||||
if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR)
|
||||
dbms_target_include_directories(SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
|
||||
endif ()
|
||||
|
||||
@ -526,6 +526,8 @@
|
||||
M(557, UNKNOWN_UNION) \
|
||||
M(558, EXPECTED_ALL_OR_DISTINCT) \
|
||||
M(559, INVALID_GRPC_QUERY_INFO) \
|
||||
M(560, ZSTD_ENCODER_FAILED) \
|
||||
M(561, ZSTD_DECODER_FAILED) \
|
||||
\
|
||||
M(999, KEEPER_EXCEPTION) \
|
||||
M(1000, POCO_EXCEPTION) \
|
||||
|
||||
@ -67,6 +67,10 @@ static bool renameat2(const std::string & old_path, const std::string & new_path
|
||||
/// Other cases when EINVAL can be returned should never happen.
|
||||
if (errno == EINVAL)
|
||||
return false;
|
||||
/// We should never get ENOSYS on Linux, because we check kernel version in supportsRenameat2Impl().
|
||||
/// However, we can get in on WSL.
|
||||
if (errno == ENOSYS)
|
||||
return false;
|
||||
|
||||
if (errno == EEXIST)
|
||||
throwFromErrno("Cannot rename " + old_path + " to " + new_path + " because the second path already exists", ErrorCodes::ATOMIC_RENAME_FAIL);
|
||||
|
||||
@ -799,9 +799,8 @@ namespace MySQLReplication
|
||||
break;
|
||||
}
|
||||
case WRITE_ROWS_EVENT_V1:
|
||||
case WRITE_ROWS_EVENT_V2:
|
||||
{
|
||||
if (do_replicate())
|
||||
case WRITE_ROWS_EVENT_V2: {
|
||||
if (doReplicate())
|
||||
event = std::make_shared<WriteRowsEvent>(table_map, std::move(event_header));
|
||||
else
|
||||
event = std::make_shared<DryRunEvent>(std::move(event_header));
|
||||
@ -810,9 +809,8 @@ namespace MySQLReplication
|
||||
break;
|
||||
}
|
||||
case DELETE_ROWS_EVENT_V1:
|
||||
case DELETE_ROWS_EVENT_V2:
|
||||
{
|
||||
if (do_replicate())
|
||||
case DELETE_ROWS_EVENT_V2: {
|
||||
if (doReplicate())
|
||||
event = std::make_shared<DeleteRowsEvent>(table_map, std::move(event_header));
|
||||
else
|
||||
event = std::make_shared<DryRunEvent>(std::move(event_header));
|
||||
@ -821,9 +819,8 @@ namespace MySQLReplication
|
||||
break;
|
||||
}
|
||||
case UPDATE_ROWS_EVENT_V1:
|
||||
case UPDATE_ROWS_EVENT_V2:
|
||||
{
|
||||
if (do_replicate())
|
||||
case UPDATE_ROWS_EVENT_V2: {
|
||||
if (doReplicate())
|
||||
event = std::make_shared<UpdateRowsEvent>(table_map, std::move(event_header));
|
||||
else
|
||||
event = std::make_shared<DryRunEvent>(std::move(event_header));
|
||||
|
||||
@ -549,7 +549,7 @@ namespace MySQLReplication
|
||||
std::shared_ptr<TableMapEvent> table_map;
|
||||
size_t checksum_signature_length = 4;
|
||||
|
||||
inline bool do_replicate() { return (replicate_do_db.empty() || table_map->schema == replicate_do_db); }
|
||||
inline bool doReplicate() { return (replicate_do_db.empty() || table_map->schema == replicate_do_db); }
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -399,6 +399,8 @@ class IColumn;
|
||||
M(Bool, enable_global_with_statement, false, "Propagate WITH statements to UNION queries and all subqueries", 0) \
|
||||
M(Bool, aggregate_functions_null_for_empty, false, "Rewrite all aggregate functions in a query, adding -OrNull suffix to them", 0) \
|
||||
\
|
||||
M(Bool, use_antlr_parser, false, "Parse incoming queries using ANTLR-generated parser", 0) \
|
||||
\
|
||||
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
|
||||
\
|
||||
M(UInt64, max_memory_usage_for_all_queries, 0, "Obsolete. Will be removed after 2020-10-20", 0) \
|
||||
|
||||
@ -357,20 +357,23 @@ static DataTypePtr create(const ASTPtr & arguments)
|
||||
throw Exception("Unexpected level of parameters to aggregate function", ErrorCodes::SYNTAX_ERROR);
|
||||
function_name = parametric->name;
|
||||
|
||||
const ASTs & parameters = parametric->arguments->children;
|
||||
params_row.resize(parameters.size());
|
||||
|
||||
for (size_t i = 0; i < parameters.size(); ++i)
|
||||
if (parametric->arguments)
|
||||
{
|
||||
const auto * literal = parameters[i]->as<ASTLiteral>();
|
||||
if (!literal)
|
||||
throw Exception(
|
||||
ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS,
|
||||
"Parameters to aggregate functions must be literals. "
|
||||
"Got parameter '{}' for function '{}'",
|
||||
parameters[i]->formatForErrorMessage(), function_name);
|
||||
const ASTs & parameters = parametric->arguments->children;
|
||||
params_row.resize(parameters.size());
|
||||
|
||||
params_row[i] = literal->value;
|
||||
for (size_t i = 0; i < parameters.size(); ++i)
|
||||
{
|
||||
const auto * literal = parameters[i]->as<ASTLiteral>();
|
||||
if (!literal)
|
||||
throw Exception(
|
||||
ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS,
|
||||
"Parameters to aggregate functions must be literals. "
|
||||
"Got parameter '{}' for function '{}'",
|
||||
parameters[i]->formatForErrorMessage(), function_name);
|
||||
|
||||
params_row[i] = literal->value;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (auto opt_name = tryGetIdentifierName(arguments->children[0]))
|
||||
|
||||
@ -72,20 +72,24 @@ static std::pair<DataTypePtr, DataTypeCustomDescPtr> create(const ASTPtr & argum
|
||||
throw Exception("Unexpected level of parameters to aggregate function", ErrorCodes::SYNTAX_ERROR);
|
||||
function_name = parametric->name;
|
||||
|
||||
const ASTs & parameters = parametric->arguments->as<ASTExpressionList &>().children;
|
||||
params_row.resize(parameters.size());
|
||||
|
||||
for (size_t i = 0; i < parameters.size(); ++i)
|
||||
if (parametric->arguments)
|
||||
{
|
||||
const ASTLiteral * lit = parameters[i]->as<ASTLiteral>();
|
||||
if (!lit)
|
||||
throw Exception(
|
||||
ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS,
|
||||
"Parameters to aggregate functions must be literals. "
|
||||
"Got parameter '{}' for function '{}'",
|
||||
parameters[i]->formatForErrorMessage(), function_name);
|
||||
const ASTs & parameters = parametric->arguments->as<ASTExpressionList &>().children;
|
||||
params_row.resize(parameters.size());
|
||||
|
||||
params_row[i] = lit->value;
|
||||
for (size_t i = 0; i < parameters.size(); ++i)
|
||||
{
|
||||
const ASTLiteral * lit = parameters[i]->as<ASTLiteral>();
|
||||
if (!lit)
|
||||
throw Exception(
|
||||
ErrorCodes::PARAMETERS_TO_AGGREGATE_FUNCTIONS_MUST_BE_LITERALS,
|
||||
"Parameters to aggregate functions must be literals. "
|
||||
"Got parameter '{}' for function '{}'",
|
||||
parameters[i]->formatForErrorMessage(),
|
||||
function_name);
|
||||
|
||||
params_row[i] = lit->value;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (auto opt_name = tryGetIdentifierName(arguments->children[0]))
|
||||
|
||||
@ -99,38 +99,40 @@ void buildLayoutConfiguration(
|
||||
root->appendChild(layout_element);
|
||||
AutoPtr<Element> layout_type_element(doc->createElement(layout->layout_type));
|
||||
layout_element->appendChild(layout_type_element);
|
||||
for (const auto & param : layout->parameters->children)
|
||||
{
|
||||
const ASTPair * pair = param->as<ASTPair>();
|
||||
if (!pair)
|
||||
|
||||
if (layout->parameters)
|
||||
for (const auto & param : layout->parameters->children)
|
||||
{
|
||||
throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Dictionary layout parameters must be key/value pairs, got '{}' instead",
|
||||
param->formatForErrorMessage());
|
||||
const ASTPair * pair = param->as<ASTPair>();
|
||||
if (!pair)
|
||||
{
|
||||
throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Dictionary layout parameters must be key/value pairs, got '{}' instead",
|
||||
param->formatForErrorMessage());
|
||||
}
|
||||
|
||||
const ASTLiteral * value_literal = pair->second->as<ASTLiteral>();
|
||||
if (!value_literal)
|
||||
{
|
||||
throw DB::Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Dictionary layout parameter value must be a literal, got '{}' instead",
|
||||
pair->second->formatForErrorMessage());
|
||||
}
|
||||
|
||||
const auto value_field = value_literal->value;
|
||||
|
||||
if (value_field.getType() != Field::Types::UInt64
|
||||
&& value_field.getType() != Field::Types::String)
|
||||
{
|
||||
throw DB::Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Dictionary layout parameter value must be an UInt64 or String, got '{}' instead",
|
||||
value_field.getTypeName());
|
||||
}
|
||||
|
||||
AutoPtr<Element> layout_type_parameter_element(doc->createElement(pair->first));
|
||||
AutoPtr<Text> value_to_append(doc->createTextNode(toString(value_field)));
|
||||
layout_type_parameter_element->appendChild(value_to_append);
|
||||
layout_type_element->appendChild(layout_type_parameter_element);
|
||||
}
|
||||
|
||||
const ASTLiteral * value_literal = pair->second->as<ASTLiteral>();
|
||||
if (!value_literal)
|
||||
{
|
||||
throw DB::Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Dictionary layout parameter value must be a literal, got '{}' instead",
|
||||
pair->second->formatForErrorMessage());
|
||||
}
|
||||
|
||||
const auto value_field = value_literal->value;
|
||||
|
||||
if (value_field.getType() != Field::Types::UInt64
|
||||
&& value_field.getType() != Field::Types::String)
|
||||
{
|
||||
throw DB::Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Dictionary layout parameter value must be an UInt64 or String, got '{}' instead",
|
||||
value_field.getTypeName());
|
||||
}
|
||||
|
||||
AutoPtr<Element> layout_type_parameter_element(doc->createElement(pair->first));
|
||||
AutoPtr<Text> value_to_append(doc->createTextNode(toString(value_field)));
|
||||
layout_type_parameter_element->appendChild(value_to_append);
|
||||
layout_type_element->appendChild(layout_type_parameter_element);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@ -141,16 +141,16 @@ struct NumericArraySource : public ArraySourceImpl<NumericArraySource<T>>
|
||||
|
||||
/// The methods can be virtual or not depending on the template parameter. See IStringSource.
|
||||
#if !__clang__
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wsuggest-override"
|
||||
# pragma GCC diagnostic push
|
||||
# pragma GCC diagnostic ignored "-Wsuggest-override"
|
||||
#elif __clang_major__ >= 11
|
||||
#pragma GCC diagnostic push
|
||||
#ifdef HAS_SUGGEST_OVERRIDE
|
||||
#pragma GCC diagnostic ignored "-Wsuggest-override"
|
||||
#endif
|
||||
#ifdef HAS_SUGGEST_DESTRUCTOR_OVERRIDE
|
||||
#pragma GCC diagnostic ignored "-Wsuggest-destructor-override"
|
||||
#endif
|
||||
# pragma GCC diagnostic push
|
||||
# ifdef HAS_SUGGEST_OVERRIDE
|
||||
# pragma GCC diagnostic ignored "-Wsuggest-override"
|
||||
# endif
|
||||
# ifdef HAS_SUGGEST_DESTRUCTOR_OVERRIDE
|
||||
# pragma GCC diagnostic ignored "-Wsuggest-destructor-override"
|
||||
# endif
|
||||
#endif
|
||||
|
||||
template <typename Base>
|
||||
@ -234,7 +234,7 @@ struct ConstSource : public Base
|
||||
};
|
||||
|
||||
#if !__clang__ || __clang_major__ >= 11
|
||||
#pragma GCC diagnostic pop
|
||||
# pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
struct StringSource
|
||||
@ -355,9 +355,9 @@ struct UTF8StringSource : public StringSource
|
||||
|
||||
Slice getSliceFromLeft(size_t offset) const
|
||||
{
|
||||
auto begin = &elements[prev_offset];
|
||||
auto end = elements.data() + offsets[row_num] - 1;
|
||||
auto res_begin = skipCodePointsForward(begin, offset, end);
|
||||
const auto * begin = &elements[prev_offset];
|
||||
const auto * end = elements.data() + offsets[row_num] - 1;
|
||||
const auto * res_begin = skipCodePointsForward(begin, offset, end);
|
||||
|
||||
if (res_begin >= end)
|
||||
return {begin, 0};
|
||||
@ -367,14 +367,14 @@ struct UTF8StringSource : public StringSource
|
||||
|
||||
Slice getSliceFromLeft(size_t offset, size_t length) const
|
||||
{
|
||||
auto begin = &elements[prev_offset];
|
||||
auto end = elements.data() + offsets[row_num] - 1;
|
||||
auto res_begin = skipCodePointsForward(begin, offset, end);
|
||||
const auto * begin = &elements[prev_offset];
|
||||
const auto * end = elements.data() + offsets[row_num] - 1;
|
||||
const auto * res_begin = skipCodePointsForward(begin, offset, end);
|
||||
|
||||
if (res_begin >= end)
|
||||
return {begin, 0};
|
||||
|
||||
auto res_end = skipCodePointsForward(res_begin, length, end);
|
||||
const auto * res_end = skipCodePointsForward(res_begin, length, end);
|
||||
|
||||
if (res_end >= end)
|
||||
return {res_begin, size_t(end - res_begin)};
|
||||
@ -384,19 +384,19 @@ struct UTF8StringSource : public StringSource
|
||||
|
||||
Slice getSliceFromRight(size_t offset) const
|
||||
{
|
||||
auto begin = &elements[prev_offset];
|
||||
auto end = elements.data() + offsets[row_num] - 1;
|
||||
auto res_begin = skipCodePointsBackward(end, offset, begin);
|
||||
const auto * begin = &elements[prev_offset];
|
||||
const auto * end = elements.data() + offsets[row_num] - 1;
|
||||
const auto * res_begin = skipCodePointsBackward(end, offset, begin);
|
||||
|
||||
return {res_begin, size_t(end - res_begin)};
|
||||
}
|
||||
|
||||
Slice getSliceFromRight(size_t offset, size_t length) const
|
||||
{
|
||||
auto begin = &elements[prev_offset];
|
||||
auto end = elements.data() + offsets[row_num] - 1;
|
||||
auto res_begin = skipCodePointsBackward(end, offset, begin);
|
||||
auto res_end = skipCodePointsForward(res_begin, length, end);
|
||||
const auto * begin = &elements[prev_offset];
|
||||
const auto * end = elements.data() + offsets[row_num] - 1;
|
||||
const auto * res_begin = skipCodePointsBackward(end, offset, begin);
|
||||
const auto * res_end = skipCodePointsForward(res_begin, length, end);
|
||||
|
||||
if (res_end >= end)
|
||||
return {res_begin, size_t(end - res_begin)};
|
||||
@ -495,7 +495,7 @@ struct IStringSource
|
||||
virtual bool isEnd() const = 0;
|
||||
virtual size_t getSizeForReserve() const = 0;
|
||||
virtual Slice getWhole() const = 0;
|
||||
virtual ~IStringSource() {}
|
||||
virtual ~IStringSource() = default;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <IO/ZlibDeflatingWriteBuffer.h>
|
||||
#include <IO/ZlibInflatingReadBuffer.h>
|
||||
#include <IO/ZstdDeflatingWriteBuffer.h>
|
||||
#include <IO/ZstdInflatingReadBuffer.h>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include <Common/config.h>
|
||||
@ -34,6 +36,8 @@ std::string toContentEncodingName(CompressionMethod method)
|
||||
return "br";
|
||||
case CompressionMethod::Xz:
|
||||
return "xz";
|
||||
case CompressionMethod::Zstd:
|
||||
return "zstd";
|
||||
case CompressionMethod::None:
|
||||
return "";
|
||||
}
|
||||
@ -61,11 +65,13 @@ CompressionMethod chooseCompressionMethod(const std::string & path, const std::s
|
||||
return CompressionMethod::Brotli;
|
||||
if (*method_str == "LZMA" || *method_str == "xz")
|
||||
return CompressionMethod::Xz;
|
||||
if (*method_str == "zstd" || *method_str == "zst")
|
||||
return CompressionMethod::Zstd;
|
||||
if (hint.empty() || hint == "auto" || hint == "none")
|
||||
return CompressionMethod::None;
|
||||
|
||||
throw Exception(
|
||||
"Unknown compression method " + hint + ". Only 'auto', 'none', 'gzip', 'br', 'xz' are supported as compression methods",
|
||||
"Unknown compression method " + hint + ". Only 'auto', 'none', 'gzip', 'br', 'xz', 'zstd' are supported as compression methods",
|
||||
ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
@ -81,6 +87,8 @@ std::unique_ptr<ReadBuffer> wrapReadBufferWithCompressionMethod(
|
||||
#endif
|
||||
if (method == CompressionMethod::Xz)
|
||||
return std::make_unique<LZMAInflatingReadBuffer>(std::move(nested), buf_size, existing_memory, alignment);
|
||||
if (method == CompressionMethod::Zstd)
|
||||
return std::make_unique<ZstdInflatingReadBuffer>(std::move(nested), buf_size, existing_memory, alignment);
|
||||
|
||||
if (method == CompressionMethod::None)
|
||||
return nested;
|
||||
@ -102,6 +110,9 @@ std::unique_ptr<WriteBuffer> wrapWriteBufferWithCompressionMethod(
|
||||
if (method == CompressionMethod::Xz)
|
||||
return std::make_unique<LZMADeflatingWriteBuffer>(std::move(nested), level, buf_size, existing_memory, alignment);
|
||||
|
||||
if (method == CompressionMethod::Zstd)
|
||||
return std::make_unique<ZstdDeflatingWriteBuffer>(std::move(nested), level, buf_size, existing_memory, alignment);
|
||||
|
||||
if (method == CompressionMethod::None)
|
||||
return nested;
|
||||
|
||||
|
||||
@ -28,6 +28,9 @@ enum class CompressionMethod
|
||||
/// LZMA2-based content compression
|
||||
/// This option corresponds to HTTP Content-Encoding: xz
|
||||
Xz,
|
||||
/// Zstd compressor
|
||||
/// This option corresponds to HTTP Content-Encoding: zstd
|
||||
Zstd,
|
||||
Brotli
|
||||
};
|
||||
|
||||
|
||||
95
src/IO/ZstdDeflatingWriteBuffer.cpp
Normal file
95
src/IO/ZstdDeflatingWriteBuffer.cpp
Normal file
@ -0,0 +1,95 @@
|
||||
#include <IO/ZstdDeflatingWriteBuffer.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ZSTD_ENCODER_FAILED;
|
||||
}
|
||||
|
||||
ZstdDeflatingWriteBuffer::ZstdDeflatingWriteBuffer(
|
||||
std::unique_ptr<WriteBuffer> out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment)
|
||||
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment), out(std::move(out_))
|
||||
{
|
||||
cctx = ZSTD_createCCtx();
|
||||
if (cctx == nullptr)
|
||||
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "zstd stream encoder init failed: zstd version: {}", ZSTD_VERSION_STRING);
|
||||
size_t ret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level);
|
||||
if (ZSTD_isError(ret))
|
||||
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "zstd stream encoder option setting failed: error code: {}; zstd version: {}", ret, ZSTD_VERSION_STRING);
|
||||
ret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1);
|
||||
if (ZSTD_isError(ret))
|
||||
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "zstd stream encoder option setting failed: error code: {}; zstd version: {}", ret, ZSTD_VERSION_STRING);
|
||||
|
||||
input = {nullptr, 0, 0};
|
||||
output = {nullptr, 0, 0};
|
||||
}
|
||||
|
||||
|
||||
ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer()
|
||||
{
|
||||
try
|
||||
{
|
||||
finish();
|
||||
|
||||
ZSTD_freeCCtx(cctx);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
void ZstdDeflatingWriteBuffer::nextImpl()
|
||||
{
|
||||
if (!offset())
|
||||
return;
|
||||
|
||||
ZSTD_EndDirective mode = ZSTD_e_flush;
|
||||
|
||||
input.src = reinterpret_cast<unsigned char *>(working_buffer.begin());
|
||||
input.size = offset();
|
||||
input.pos = 0;
|
||||
|
||||
bool finished = false;
|
||||
do
|
||||
{
|
||||
out->nextIfAtEnd();
|
||||
|
||||
output.dst = reinterpret_cast<unsigned char *>(out->buffer().begin());
|
||||
output.size = out->buffer().size();
|
||||
output.pos = out->offset();
|
||||
|
||||
|
||||
ZSTD_compressStream2(cctx, &output, &input, mode);
|
||||
out->position() = out->buffer().begin() + output.pos;
|
||||
finished = (input.pos == input.size);
|
||||
} while (!finished);
|
||||
|
||||
}
|
||||
|
||||
void ZstdDeflatingWriteBuffer::finish()
|
||||
{
|
||||
if (flushed)
|
||||
return;
|
||||
|
||||
next();
|
||||
|
||||
out->nextIfAtEnd();
|
||||
|
||||
input.src = reinterpret_cast<unsigned char *>(working_buffer.begin());
|
||||
input.size = offset();
|
||||
input.pos = 0;
|
||||
|
||||
output.dst = reinterpret_cast<unsigned char *>(out->buffer().begin());
|
||||
output.size = out->buffer().size();
|
||||
output.pos = out->offset();
|
||||
|
||||
size_t remaining = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
|
||||
if (ZSTD_isError(remaining))
|
||||
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "zstd stream encoder end failed: zstd version: {}", ZSTD_VERSION_STRING);
|
||||
out->position() = out->buffer().begin() + output.pos;
|
||||
flushed = true;
|
||||
}
|
||||
|
||||
}
|
||||
40
src/IO/ZstdDeflatingWriteBuffer.h
Normal file
40
src/IO/ZstdDeflatingWriteBuffer.h
Normal file
@ -0,0 +1,40 @@
|
||||
#pragma once
|
||||
|
||||
#include <IO/BufferWithOwnMemory.h>
|
||||
#include <IO/CompressionMethod.h>
|
||||
#include <IO/WriteBuffer.h>
|
||||
|
||||
#include <zstd.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Performs compression using zstd library and writes compressed data to out_ WriteBuffer.
|
||||
class ZstdDeflatingWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
|
||||
{
|
||||
public:
|
||||
ZstdDeflatingWriteBuffer(
|
||||
std::unique_ptr<WriteBuffer> out_,
|
||||
int compression_level,
|
||||
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
|
||||
char * existing_memory = nullptr,
|
||||
size_t alignment = 0);
|
||||
|
||||
/// Flush all pending data and write zstd footer to the underlying buffer.
|
||||
/// After the first call to this function, subsequent calls will have no effect and
|
||||
/// an attempt to write to this buffer will result in exception.
|
||||
void finish();
|
||||
|
||||
~ZstdDeflatingWriteBuffer() override;
|
||||
|
||||
private:
|
||||
void nextImpl() override;
|
||||
|
||||
std::unique_ptr<WriteBuffer> out;
|
||||
ZSTD_CCtx * cctx;
|
||||
ZSTD_inBuffer input;
|
||||
ZSTD_outBuffer output;
|
||||
bool flushed = false;
|
||||
};
|
||||
|
||||
}
|
||||
63
src/IO/ZstdInflatingReadBuffer.cpp
Normal file
63
src/IO/ZstdInflatingReadBuffer.cpp
Normal file
@ -0,0 +1,63 @@
|
||||
#include <IO/ZstdInflatingReadBuffer.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ZSTD_DECODER_FAILED;
|
||||
}
|
||||
|
||||
ZstdInflatingReadBuffer::ZstdInflatingReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t buf_size, char * existing_memory, size_t alignment)
|
||||
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment), in(std::move(in_))
|
||||
{
|
||||
dctx = ZSTD_createDCtx();
|
||||
input = {nullptr, 0, 0};
|
||||
output = {nullptr, 0, 0};
|
||||
|
||||
if (dctx == nullptr)
|
||||
{
|
||||
throw Exception(ErrorCodes::ZSTD_DECODER_FAILED, "zstd_stream_decoder init failed: zstd version: {}", ZSTD_VERSION_STRING);
|
||||
}
|
||||
}
|
||||
|
||||
ZstdInflatingReadBuffer::~ZstdInflatingReadBuffer()
|
||||
{
|
||||
ZSTD_freeDCtx(dctx);
|
||||
}
|
||||
|
||||
bool ZstdInflatingReadBuffer::nextImpl()
|
||||
{
|
||||
if (eof)
|
||||
return false;
|
||||
|
||||
if (input.pos >= input.size)
|
||||
{
|
||||
in->nextIfAtEnd();
|
||||
input.src = reinterpret_cast<unsigned char *>(in->position());
|
||||
input.pos = 0;
|
||||
input.size = in->buffer().end() - in->position();
|
||||
}
|
||||
|
||||
output.dst = reinterpret_cast<unsigned char *>(internal_buffer.begin());
|
||||
output.size = internal_buffer.size();
|
||||
output.pos = 0;
|
||||
|
||||
size_t ret = ZSTD_decompressStream(dctx, &output, &input);
|
||||
if (ZSTD_isError(ret))
|
||||
throw Exception(
|
||||
ErrorCodes::ZSTD_DECODER_FAILED, "Zstd stream decoding failed: error code: {}; zstd version: {}", ret, ZSTD_VERSION_STRING);
|
||||
|
||||
in->position() = in->buffer().begin() + input.pos;
|
||||
working_buffer.resize(output.pos);
|
||||
|
||||
if (in->eof())
|
||||
{
|
||||
eof = true;
|
||||
return working_buffer.size() != 0;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
37
src/IO/ZstdInflatingReadBuffer.h
Normal file
37
src/IO/ZstdInflatingReadBuffer.h
Normal file
@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
#include <IO/BufferWithOwnMemory.h>
|
||||
#include <IO/CompressionMethod.h>
|
||||
#include <IO/ReadBuffer.h>
|
||||
|
||||
#include <zstd.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
}
|
||||
|
||||
class ZstdInflatingReadBuffer : public BufferWithOwnMemory<ReadBuffer>
|
||||
{
|
||||
public:
|
||||
ZstdInflatingReadBuffer(
|
||||
std::unique_ptr<ReadBuffer> in_,
|
||||
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
|
||||
char * existing_memory = nullptr,
|
||||
size_t alignment = 0);
|
||||
|
||||
~ZstdInflatingReadBuffer() override;
|
||||
|
||||
private:
|
||||
bool nextImpl() override;
|
||||
|
||||
std::unique_ptr<ReadBuffer> in;
|
||||
ZSTD_DCtx * dctx;
|
||||
ZSTD_inBuffer input;
|
||||
ZSTD_outBuffer output;
|
||||
bool eof = false;
|
||||
};
|
||||
|
||||
}
|
||||
@ -82,3 +82,6 @@ target_link_libraries (zlib_ng_bug PRIVATE ${ZLIB_LIBRARIES})
|
||||
|
||||
add_executable (ryu_test ryu_test.cpp)
|
||||
target_link_libraries (ryu_test PRIVATE ryu)
|
||||
|
||||
add_executable (zstd_buffers zstd_buffers.cpp)
|
||||
target_link_libraries (zstd_buffers PRIVATE clickhouse_common_io)
|
||||
|
||||
66
src/IO/tests/zstd_buffers.cpp
Normal file
66
src/IO/tests/zstd_buffers.cpp
Normal file
@ -0,0 +1,66 @@
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
|
||||
#include <IO/ReadBufferFromFile.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <IO/ZstdDeflatingWriteBuffer.h>
|
||||
#include <IO/ZstdInflatingReadBuffer.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
|
||||
|
||||
int main(int, char **)
|
||||
try
|
||||
{
|
||||
std::cout << std::fixed << std::setprecision(2);
|
||||
|
||||
size_t n = 10000000;
|
||||
Stopwatch stopwatch;
|
||||
|
||||
|
||||
{
|
||||
auto buf
|
||||
= std::make_unique<DB::WriteBufferFromFile>("test_zstd_buffers.zst", DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_CREAT | O_TRUNC);
|
||||
DB::ZstdDeflatingWriteBuffer zstd_buf(std::move(buf), /*compression level*/ 3);
|
||||
|
||||
stopwatch.restart();
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
{
|
||||
DB::writeIntText(i, zstd_buf);
|
||||
DB::writeChar('\t', zstd_buf);
|
||||
}
|
||||
zstd_buf.finish();
|
||||
|
||||
stopwatch.stop();
|
||||
|
||||
std::cout << "Writing done. Elapsed: " << stopwatch.elapsedSeconds() << " s."
|
||||
<< ", " << (zstd_buf.count() / stopwatch.elapsedSeconds() / 1000000) << " MB/s" << std::endl;
|
||||
}
|
||||
|
||||
{
|
||||
auto buf = std::make_unique<DB::ReadBufferFromFile>("test_zstd_buffers.zst");
|
||||
DB::ZstdInflatingReadBuffer zstd_buf(std::move(buf));
|
||||
|
||||
stopwatch.restart();
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
{
|
||||
size_t x;
|
||||
DB::readIntText(x, zstd_buf);
|
||||
zstd_buf.ignore();
|
||||
|
||||
if (x != i)
|
||||
throw DB::Exception("Failed!, read: " + std::to_string(x) + ", expected: " + std::to_string(i), 0);
|
||||
}
|
||||
stopwatch.stop();
|
||||
std::cout << "Reading done. Elapsed: " << stopwatch.elapsedSeconds() << " s."
|
||||
<< ", " << (zstd_buf.count() / stopwatch.elapsedSeconds() / 1000000) << " MB/s" << std::endl;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
catch (const DB::Exception & e)
|
||||
{
|
||||
std::cerr << e.what() << ", " << e.displayText() << std::endl;
|
||||
return 1;
|
||||
}
|
||||
@ -3,11 +3,16 @@ OWNER(g:clickhouse)
|
||||
|
||||
LIBRARY()
|
||||
|
||||
ADDINCL(
|
||||
contrib/libs/zstd
|
||||
)
|
||||
|
||||
PEERDIR(
|
||||
clickhouse/src/Common
|
||||
contrib/libs/brotli/dec
|
||||
contrib/libs/brotli/enc
|
||||
contrib/libs/poco/NetSSL_OpenSSL
|
||||
contrib/libs/zstd
|
||||
)
|
||||
|
||||
|
||||
@ -58,6 +63,8 @@ SRCS(
|
||||
WriteHelpers.cpp
|
||||
ZlibDeflatingWriteBuffer.cpp
|
||||
ZlibInflatingReadBuffer.cpp
|
||||
ZstdDeflatingWriteBuffer.cpp
|
||||
ZstdInflatingReadBuffer.cpp
|
||||
copyData.cpp
|
||||
createReadBufferFromFileBase.cpp
|
||||
createWriteBufferFromFileBase.cpp
|
||||
|
||||
@ -2,11 +2,16 @@ OWNER(g:clickhouse)
|
||||
|
||||
LIBRARY()
|
||||
|
||||
ADDINCL(
|
||||
contrib/libs/zstd
|
||||
)
|
||||
|
||||
PEERDIR(
|
||||
clickhouse/src/Common
|
||||
contrib/libs/brotli/dec
|
||||
contrib/libs/brotli/enc
|
||||
contrib/libs/poco/NetSSL_OpenSSL
|
||||
contrib/libs/zstd
|
||||
)
|
||||
|
||||
|
||||
|
||||
@ -757,39 +757,102 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
|
||||
|
||||
/// If the function has an argument-lambda expression, you need to determine its type before the recursive call.
|
||||
bool has_lambda_arguments = false;
|
||||
size_t num_arguments = node.arguments->children.size();
|
||||
for (size_t arg = 0; arg < num_arguments; ++arg)
|
||||
|
||||
if (node.arguments)
|
||||
{
|
||||
auto & child = node.arguments->children[arg];
|
||||
|
||||
const auto * function = child->as<ASTFunction>();
|
||||
const auto * identifier = child->as<ASTIdentifier>();
|
||||
if (function && function->name == "lambda")
|
||||
size_t num_arguments = node.arguments->children.size();
|
||||
for (size_t arg = 0; arg < num_arguments; ++arg)
|
||||
{
|
||||
/// If the argument is a lambda expression, just remember its approximate type.
|
||||
if (function->arguments->children.size() != 2)
|
||||
throw Exception("lambda requires two arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
auto & child = node.arguments->children[arg];
|
||||
|
||||
const auto * lambda_args_tuple = function->arguments->children.at(0)->as<ASTFunction>();
|
||||
|
||||
if (!lambda_args_tuple || lambda_args_tuple->name != "tuple")
|
||||
throw Exception("First argument of lambda must be a tuple", ErrorCodes::TYPE_MISMATCH);
|
||||
|
||||
has_lambda_arguments = true;
|
||||
argument_types.emplace_back(std::make_shared<DataTypeFunction>(DataTypes(lambda_args_tuple->arguments->children.size())));
|
||||
/// Select the name in the next cycle.
|
||||
argument_names.emplace_back();
|
||||
}
|
||||
else if (function && function->name == "untuple")
|
||||
{
|
||||
auto columns = doUntuple(function, data);
|
||||
|
||||
if (columns.empty())
|
||||
continue;
|
||||
|
||||
for (const auto & column : columns)
|
||||
const auto * function = child->as<ASTFunction>();
|
||||
const auto * identifier = child->as<ASTIdentifier>();
|
||||
if (function && function->name == "lambda")
|
||||
{
|
||||
if (auto name_type = getNameAndTypeFromAST(column, data))
|
||||
/// If the argument is a lambda expression, just remember its approximate type.
|
||||
if (function->arguments->children.size() != 2)
|
||||
throw Exception("lambda requires two arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
const auto * lambda_args_tuple = function->arguments->children.at(0)->as<ASTFunction>();
|
||||
|
||||
if (!lambda_args_tuple || lambda_args_tuple->name != "tuple")
|
||||
throw Exception("First argument of lambda must be a tuple", ErrorCodes::TYPE_MISMATCH);
|
||||
|
||||
has_lambda_arguments = true;
|
||||
argument_types.emplace_back(std::make_shared<DataTypeFunction>(DataTypes(lambda_args_tuple->arguments->children.size())));
|
||||
/// Select the name in the next cycle.
|
||||
argument_names.emplace_back();
|
||||
}
|
||||
else if (function && function->name == "untuple")
|
||||
{
|
||||
auto columns = doUntuple(function, data);
|
||||
|
||||
if (columns.empty())
|
||||
continue;
|
||||
|
||||
for (const auto & column : columns)
|
||||
{
|
||||
if (auto name_type = getNameAndTypeFromAST(column, data))
|
||||
{
|
||||
argument_types.push_back(name_type->type);
|
||||
argument_names.push_back(name_type->name);
|
||||
}
|
||||
else
|
||||
arguments_present = false;
|
||||
}
|
||||
|
||||
node.arguments->children.erase(node.arguments->children.begin() + arg);
|
||||
node.arguments->children.insert(node.arguments->children.begin() + arg, columns.begin(), columns.end());
|
||||
num_arguments += columns.size() - 1;
|
||||
arg += columns.size() - 1;
|
||||
}
|
||||
else if (checkFunctionIsInOrGlobalInOperator(node) && arg == 1 && prepared_set)
|
||||
{
|
||||
ColumnWithTypeAndName column;
|
||||
column.type = std::make_shared<DataTypeSet>();
|
||||
|
||||
/// If the argument is a set given by an enumeration of values (so, the set was already built), give it a unique name,
|
||||
/// so that sets with the same literal representation do not fuse together (they can have different types).
|
||||
if (!prepared_set->empty())
|
||||
column.name = data.getUniqueName("__set");
|
||||
else
|
||||
column.name = child->getColumnName();
|
||||
|
||||
if (!data.hasColumn(column.name))
|
||||
{
|
||||
auto column_set = ColumnSet::create(1, prepared_set);
|
||||
/// If prepared_set is not empty, we have a set made with literals.
|
||||
/// Create a const ColumnSet to make constant folding work
|
||||
if (!prepared_set->empty())
|
||||
column.column = ColumnConst::create(std::move(column_set), 1);
|
||||
else
|
||||
column.column = std::move(column_set);
|
||||
data.addColumn(column);
|
||||
}
|
||||
|
||||
argument_types.push_back(column.type);
|
||||
argument_names.push_back(column.name);
|
||||
}
|
||||
else if (identifier && (functionIsJoinGet(node.name) || functionIsDictGet(node.name)) && arg == 0)
|
||||
{
|
||||
auto table_id = IdentifierSemantic::extractDatabaseAndTable(*identifier);
|
||||
table_id = data.context.resolveStorageID(table_id, Context::ResolveOrdinary);
|
||||
auto column_string = ColumnString::create();
|
||||
column_string->insert(table_id.getDatabaseName() + "." + table_id.getTableName());
|
||||
ColumnWithTypeAndName column(
|
||||
ColumnConst::create(std::move(column_string), 1),
|
||||
std::make_shared<DataTypeString>(),
|
||||
data.getUniqueName("__" + node.name));
|
||||
data.addColumn(column);
|
||||
argument_types.push_back(column.type);
|
||||
argument_names.push_back(column.name);
|
||||
}
|
||||
else
|
||||
{
|
||||
/// If the argument is not a lambda expression, call it recursively and find out its type.
|
||||
visit(child, data);
|
||||
|
||||
if (auto name_type = getNameAndTypeFromAST(child, data))
|
||||
{
|
||||
argument_types.push_back(name_type->type);
|
||||
argument_names.push_back(name_type->name);
|
||||
@ -797,125 +860,66 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
|
||||
else
|
||||
arguments_present = false;
|
||||
}
|
||||
|
||||
node.arguments->children.erase(node.arguments->children.begin() + arg);
|
||||
node.arguments->children.insert(node.arguments->children.begin() + arg, columns.begin(), columns.end());
|
||||
num_arguments += columns.size() - 1;
|
||||
arg += columns.size() - 1;
|
||||
}
|
||||
else if (checkFunctionIsInOrGlobalInOperator(node) && arg == 1 && prepared_set)
|
||||
|
||||
if (data.only_consts && !arguments_present)
|
||||
return;
|
||||
|
||||
if (has_lambda_arguments && !data.only_consts)
|
||||
{
|
||||
ColumnWithTypeAndName column;
|
||||
column.type = std::make_shared<DataTypeSet>();
|
||||
function_builder->getLambdaArgumentTypes(argument_types);
|
||||
|
||||
/// If the argument is a set given by an enumeration of values (so, the set was already built), give it a unique name,
|
||||
/// so that sets with the same literal representation do not fuse together (they can have different types).
|
||||
if (!prepared_set->empty())
|
||||
column.name = data.getUniqueName("__set");
|
||||
else
|
||||
column.name = child->getColumnName();
|
||||
|
||||
if (!data.hasColumn(column.name))
|
||||
/// Call recursively for lambda expressions.
|
||||
for (size_t i = 0; i < node.arguments->children.size(); ++i)
|
||||
{
|
||||
auto column_set = ColumnSet::create(1, prepared_set);
|
||||
/// If prepared_set is not empty, we have a set made with literals.
|
||||
/// Create a const ColumnSet to make constant folding work
|
||||
if (!prepared_set->empty())
|
||||
column.column = ColumnConst::create(std::move(column_set), 1);
|
||||
else
|
||||
column.column = std::move(column_set);
|
||||
data.addColumn(column);
|
||||
}
|
||||
ASTPtr child = node.arguments->children[i];
|
||||
|
||||
argument_types.push_back(column.type);
|
||||
argument_names.push_back(column.name);
|
||||
}
|
||||
else if (identifier && (functionIsJoinGet(node.name) || functionIsDictGet(node.name)) && arg == 0)
|
||||
{
|
||||
auto table_id = IdentifierSemantic::extractDatabaseAndTable(*identifier);
|
||||
table_id = data.context.resolveStorageID(table_id, Context::ResolveOrdinary);
|
||||
auto column_string = ColumnString::create();
|
||||
column_string->insert(table_id.getDatabaseName() + "." + table_id.getTableName());
|
||||
ColumnWithTypeAndName column(
|
||||
ColumnConst::create(std::move(column_string), 1),
|
||||
std::make_shared<DataTypeString>(),
|
||||
data.getUniqueName("__" + node.name));
|
||||
data.addColumn(column);
|
||||
argument_types.push_back(column.type);
|
||||
argument_names.push_back(column.name);
|
||||
}
|
||||
else
|
||||
{
|
||||
/// If the argument is not a lambda expression, call it recursively and find out its type.
|
||||
visit(child, data);
|
||||
|
||||
if (auto name_type = getNameAndTypeFromAST(child, data))
|
||||
{
|
||||
argument_types.push_back(name_type->type);
|
||||
argument_names.push_back(name_type->name);
|
||||
}
|
||||
else
|
||||
arguments_present = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (data.only_consts && !arguments_present)
|
||||
return;
|
||||
|
||||
if (has_lambda_arguments && !data.only_consts)
|
||||
{
|
||||
function_builder->getLambdaArgumentTypes(argument_types);
|
||||
|
||||
/// Call recursively for lambda expressions.
|
||||
for (size_t i = 0; i < node.arguments->children.size(); ++i)
|
||||
{
|
||||
ASTPtr child = node.arguments->children[i];
|
||||
|
||||
const auto * lambda = child->as<ASTFunction>();
|
||||
if (lambda && lambda->name == "lambda")
|
||||
{
|
||||
const DataTypeFunction * lambda_type = typeid_cast<const DataTypeFunction *>(argument_types[i].get());
|
||||
const auto * lambda_args_tuple = lambda->arguments->children.at(0)->as<ASTFunction>();
|
||||
const ASTs & lambda_arg_asts = lambda_args_tuple->arguments->children;
|
||||
NamesAndTypesList lambda_arguments;
|
||||
|
||||
for (size_t j = 0; j < lambda_arg_asts.size(); ++j)
|
||||
const auto * lambda = child->as<ASTFunction>();
|
||||
if (lambda && lambda->name == "lambda")
|
||||
{
|
||||
auto opt_arg_name = tryGetIdentifierName(lambda_arg_asts[j]);
|
||||
if (!opt_arg_name)
|
||||
throw Exception("lambda argument declarations must be identifiers", ErrorCodes::TYPE_MISMATCH);
|
||||
const DataTypeFunction * lambda_type = typeid_cast<const DataTypeFunction *>(argument_types[i].get());
|
||||
const auto * lambda_args_tuple = lambda->arguments->children.at(0)->as<ASTFunction>();
|
||||
const ASTs & lambda_arg_asts = lambda_args_tuple->arguments->children;
|
||||
NamesAndTypesList lambda_arguments;
|
||||
|
||||
lambda_arguments.emplace_back(*opt_arg_name, lambda_type->getArgumentTypes()[j]);
|
||||
for (size_t j = 0; j < lambda_arg_asts.size(); ++j)
|
||||
{
|
||||
auto opt_arg_name = tryGetIdentifierName(lambda_arg_asts[j]);
|
||||
if (!opt_arg_name)
|
||||
throw Exception("lambda argument declarations must be identifiers", ErrorCodes::TYPE_MISMATCH);
|
||||
|
||||
lambda_arguments.emplace_back(*opt_arg_name, lambda_type->getArgumentTypes()[j]);
|
||||
}
|
||||
|
||||
data.actions_stack.pushLevel(lambda_arguments);
|
||||
visit(lambda->arguments->children.at(1), data);
|
||||
auto lambda_dag = data.actions_stack.popLevel();
|
||||
|
||||
String result_name = lambda->arguments->children.at(1)->getColumnName();
|
||||
lambda_dag->removeUnusedActions(Names(1, result_name));
|
||||
|
||||
auto lambda_actions = std::make_shared<ExpressionActions>(lambda_dag);
|
||||
|
||||
DataTypePtr result_type = lambda_actions->getSampleBlock().getByName(result_name).type;
|
||||
|
||||
Names captured;
|
||||
Names required = lambda_actions->getRequiredColumns();
|
||||
for (const auto & required_arg : required)
|
||||
if (findColumn(required_arg, lambda_arguments) == lambda_arguments.end())
|
||||
captured.push_back(required_arg);
|
||||
|
||||
/// We can not name `getColumnName()`,
|
||||
/// because it does not uniquely define the expression (the types of arguments can be different).
|
||||
String lambda_name = data.getUniqueName("__lambda");
|
||||
|
||||
auto function_capture = std::make_unique<FunctionCaptureOverloadResolver>(
|
||||
lambda_actions, captured, lambda_arguments, result_type, result_name);
|
||||
auto function_capture_adapter = std::make_shared<FunctionOverloadResolverAdaptor>(std::move(function_capture));
|
||||
data.addFunction(function_capture_adapter, captured, lambda_name);
|
||||
|
||||
argument_types[i] = std::make_shared<DataTypeFunction>(lambda_type->getArgumentTypes(), result_type);
|
||||
argument_names[i] = lambda_name;
|
||||
}
|
||||
|
||||
data.actions_stack.pushLevel(lambda_arguments);
|
||||
visit(lambda->arguments->children.at(1), data);
|
||||
auto lambda_dag = data.actions_stack.popLevel();
|
||||
|
||||
String result_name = lambda->arguments->children.at(1)->getColumnName();
|
||||
lambda_dag->removeUnusedActions(Names(1, result_name));
|
||||
|
||||
auto lambda_actions = std::make_shared<ExpressionActions>(lambda_dag);
|
||||
|
||||
DataTypePtr result_type = lambda_actions->getSampleBlock().getByName(result_name).type;
|
||||
|
||||
Names captured;
|
||||
Names required = lambda_actions->getRequiredColumns();
|
||||
for (const auto & required_arg : required)
|
||||
if (findColumn(required_arg, lambda_arguments) == lambda_arguments.end())
|
||||
captured.push_back(required_arg);
|
||||
|
||||
/// We can not name `getColumnName()`,
|
||||
/// because it does not uniquely define the expression (the types of arguments can be different).
|
||||
String lambda_name = data.getUniqueName("__lambda");
|
||||
|
||||
auto function_capture = std::make_unique<FunctionCaptureOverloadResolver>(
|
||||
lambda_actions, captured, lambda_arguments, result_type, result_name);
|
||||
auto function_capture_adapter = std::make_shared<FunctionOverloadResolverAdaptor>(std::move(function_capture));
|
||||
data.addFunction(function_capture_adapter, captured, lambda_name);
|
||||
|
||||
argument_types[i] = std::make_shared<DataTypeFunction>(lambda_type->getArgumentTypes(), result_type);
|
||||
argument_names[i] = lambda_name;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -97,10 +97,10 @@ public:
|
||||
function_node->name == "any" || function_node->name == "anyLast"))
|
||||
{
|
||||
KeepAggregateFunctionVisitor::Data keep_data{data.group_by_keys, false};
|
||||
KeepAggregateFunctionVisitor(keep_data).visit(function_node->arguments);
|
||||
if (function_node->arguments) KeepAggregateFunctionVisitor(keep_data).visit(function_node->arguments);
|
||||
|
||||
/// Place argument of an aggregate function instead of function
|
||||
if (!keep_data.keep_aggregator && !function_node->arguments->children.empty())
|
||||
if (!keep_data.keep_aggregator && function_node->arguments && !function_node->arguments->children.empty())
|
||||
{
|
||||
String alias = function_node->alias;
|
||||
ast = (function_node->arguments->children[0])->clone();
|
||||
|
||||
@ -20,7 +20,7 @@ namespace
|
||||
|
||||
const ASTFunction * getInternalFunction(const ASTFunction & func)
|
||||
{
|
||||
if (func.arguments->children.size() == 1)
|
||||
if (func.arguments && func.arguments->children.size() == 1)
|
||||
return func.arguments->children[0]->as<ASTFunction>();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@ -2029,7 +2029,7 @@ void Context::checkCanBeDropped(const String & database, const String & table, c
|
||||
"1. Size ({}) is greater than max_[table/partition]_size_to_drop ({})\n"
|
||||
"2. File '{}' intended to force DROP {}\n"
|
||||
"How to fix this:\n"
|
||||
"1. Either increase (or set to zero) max_[table/partition]_size_to_drop in server config\n",
|
||||
"1. Either increase (or set to zero) max_[table/partition]_size_to_drop in server config\n"
|
||||
"2. Either create forcing file {} and make sure that ClickHouse has write permission for it.\n"
|
||||
"Example:\nsudo touch '{}' && sudo chmod 666 '{}'",
|
||||
backQuoteIfNeed(database), backQuoteIfNeed(table),
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
#include <filesystem>
|
||||
|
||||
#include <Interpreters/DDLWorker.h>
|
||||
#include <Parsers/ASTAlterQuery.h>
|
||||
#include <Parsers/ASTDropQuery.h>
|
||||
@ -11,7 +13,6 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <Storages/IStorage.h>
|
||||
#include <Storages/StorageDistributed.h>
|
||||
#include <DataStreams/IBlockInputStream.h>
|
||||
#include <Interpreters/executeQuery.h>
|
||||
@ -31,18 +32,14 @@
|
||||
#include <Common/quoteString.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Storages/StorageReplicatedMergeTree.h>
|
||||
#include <Poco/Timestamp.h>
|
||||
#include <Poco/Net/NetException.h>
|
||||
#include <common/sleep.h>
|
||||
#include <common/getFQDNOrHostName.h>
|
||||
#include <random>
|
||||
#include <pcg_random.hpp>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -226,7 +223,7 @@ public:
|
||||
const std::string & lock_message_ = "")
|
||||
:
|
||||
zookeeper(zookeeper_),
|
||||
lock_path(lock_prefix_ + "/" + lock_name_),
|
||||
lock_path(fs::path(lock_prefix_) / lock_name_),
|
||||
lock_message(lock_message_),
|
||||
log(&Poco::Logger::get("zkutil::Lock"))
|
||||
{
|
||||
@ -394,7 +391,7 @@ void DDLWorker::recoverZooKeeper()
|
||||
DDLTaskPtr DDLWorker::initAndCheckTask(const String & entry_name, String & out_reason, const ZooKeeperPtr & zookeeper)
|
||||
{
|
||||
String node_data;
|
||||
String entry_path = queue_dir + "/" + entry_name;
|
||||
String entry_path = fs::path(queue_dir) / entry_name;
|
||||
|
||||
if (!zookeeper->tryGet(entry_path, node_data))
|
||||
{
|
||||
@ -423,7 +420,7 @@ DDLTaskPtr DDLWorker::initAndCheckTask(const String & entry_name, String & out_r
|
||||
try
|
||||
{
|
||||
createStatusDirs(entry_path, zookeeper);
|
||||
zookeeper->tryCreate(entry_path + "/finished/" + host_fqdn_id, status, zkutil::CreateMode::Persistent);
|
||||
zookeeper->tryCreate(fs::path(entry_path) / "finished" / host_fqdn_id, status, zkutil::CreateMode::Persistent);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
@ -504,7 +501,7 @@ void DDLWorker::scheduleTasks()
|
||||
continue;
|
||||
}
|
||||
|
||||
bool already_processed = zookeeper->exists(task->entry_path + "/finished/" + task->host_id_str);
|
||||
bool already_processed = zookeeper->exists(fs::path(task->entry_path) / "finished" / task->host_id_str);
|
||||
if (!server_startup && !task->was_executed && already_processed)
|
||||
{
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
@ -862,10 +859,10 @@ bool DDLWorker::tryExecuteQueryOnLeaderReplica(
|
||||
};
|
||||
|
||||
String shard_node_name = get_shard_name(task.cluster->getShardsAddresses().at(task.host_shard_num));
|
||||
String shard_path = node_path + "/shards/" + shard_node_name;
|
||||
String is_executed_path = shard_path + "/executed";
|
||||
String tries_to_execute_path = shard_path + "/tries_to_execute";
|
||||
zookeeper->createAncestors(shard_path + "/");
|
||||
String shard_path = fs::path(node_path) / "shards" / shard_node_name;
|
||||
String is_executed_path = fs::path(shard_path) / "executed";
|
||||
String tries_to_execute_path = fs::path(shard_path) / "tries_to_execute";
|
||||
zookeeper->createAncestors(fs::path(shard_path) / ""); /* appends "/" at the end of shard_path */
|
||||
|
||||
/// Node exists, or we will create or we will get an exception
|
||||
zookeeper->tryCreate(tries_to_execute_path, "0", zkutil::CreateMode::Persistent);
|
||||
@ -994,8 +991,8 @@ void DDLWorker::cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zo
|
||||
return;
|
||||
|
||||
String node_name = *it;
|
||||
String node_path = queue_dir + "/" + node_name;
|
||||
String lock_path = node_path + "/lock";
|
||||
String node_path = fs::path(queue_dir) / node_name;
|
||||
String lock_path = fs::path(node_path) / "lock";
|
||||
|
||||
Coordination::Stat stat;
|
||||
String dummy;
|
||||
@ -1018,7 +1015,7 @@ void DDLWorker::cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zo
|
||||
continue;
|
||||
|
||||
/// Skip if there are active nodes (it is weak guard)
|
||||
if (zookeeper->exists(node_path + "/active", &stat) && stat.numChildren > 0)
|
||||
if (zookeeper->exists(fs::path(node_path) / "active", &stat) && stat.numChildren > 0)
|
||||
{
|
||||
LOG_INFO(log, "Task {} should be deleted, but there are active workers. Skipping it.", node_name);
|
||||
continue;
|
||||
@ -1044,7 +1041,7 @@ void DDLWorker::cleanupQueue(Int64 current_time_seconds, const ZooKeeperPtr & zo
|
||||
for (const String & child : children)
|
||||
{
|
||||
if (child != "lock")
|
||||
zookeeper->tryRemoveRecursive(node_path + "/" + child);
|
||||
zookeeper->tryRemoveRecursive(fs::path(node_path) / child);
|
||||
}
|
||||
|
||||
/// Remove the lock node and its parent atomically
|
||||
@ -1068,12 +1065,12 @@ void DDLWorker::createStatusDirs(const std::string & node_path, const ZooKeeperP
|
||||
Coordination::Requests ops;
|
||||
{
|
||||
Coordination::CreateRequest request;
|
||||
request.path = node_path + "/active";
|
||||
request.path = fs::path(node_path) / "active";
|
||||
ops.emplace_back(std::make_shared<Coordination::CreateRequest>(std::move(request)));
|
||||
}
|
||||
{
|
||||
Coordination::CreateRequest request;
|
||||
request.path = node_path + "/finished";
|
||||
request.path = fs::path(node_path) / "finished";
|
||||
ops.emplace_back(std::make_shared<Coordination::CreateRequest>(std::move(request)));
|
||||
}
|
||||
Coordination::Responses responses;
|
||||
@ -1091,7 +1088,7 @@ String DDLWorker::enqueueQuery(DDLLogEntry & entry)
|
||||
|
||||
auto zookeeper = getAndSetZooKeeper();
|
||||
|
||||
String query_path_prefix = queue_dir + "/query-";
|
||||
String query_path_prefix = fs::path(queue_dir) / "query-";
|
||||
zookeeper->createAncestors(query_path_prefix);
|
||||
|
||||
String node_path = zookeeper->create(query_path_prefix, entry.toString(), zkutil::CreateMode::PersistentSequential);
|
||||
@ -1121,7 +1118,7 @@ void DDLWorker::runMainThread()
|
||||
try
|
||||
{
|
||||
auto zookeeper = getAndSetZooKeeper();
|
||||
zookeeper->createAncestors(queue_dir + "/");
|
||||
zookeeper->createAncestors(fs::path(queue_dir) / "");
|
||||
initialized = true;
|
||||
}
|
||||
catch (const Coordination::Exception & e)
|
||||
@ -1294,12 +1291,12 @@ public:
|
||||
node_path);
|
||||
}
|
||||
|
||||
Strings new_hosts = getNewAndUpdate(getChildrenAllowNoNode(zookeeper, node_path + "/finished"));
|
||||
Strings new_hosts = getNewAndUpdate(getChildrenAllowNoNode(zookeeper, fs::path(node_path) / "finished"));
|
||||
++try_number;
|
||||
if (new_hosts.empty())
|
||||
continue;
|
||||
|
||||
current_active_hosts = getChildrenAllowNoNode(zookeeper, node_path + "/active");
|
||||
current_active_hosts = getChildrenAllowNoNode(zookeeper, fs::path(node_path) / "active");
|
||||
|
||||
MutableColumns columns = sample.cloneEmptyColumns();
|
||||
for (const String & host_id : new_hosts)
|
||||
@ -1307,7 +1304,7 @@ public:
|
||||
ExecutionStatus status(-1, "Cannot obtain error message");
|
||||
{
|
||||
String status_data;
|
||||
if (zookeeper->tryGet(node_path + "/finished/" + host_id, status_data))
|
||||
if (zookeeper->tryGet(fs::path(node_path) / "finished" / host_id, status_data))
|
||||
status.tryDeserializeText(status_data);
|
||||
}
|
||||
|
||||
@ -1412,9 +1409,9 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & cont
|
||||
|
||||
if (const auto * query_alter = query_ptr->as<ASTAlterQuery>())
|
||||
{
|
||||
for (const auto & command : query_alter->command_list->commands)
|
||||
for (const auto & command : query_alter->command_list->children)
|
||||
{
|
||||
if (!isSupportedAlterType(command->type))
|
||||
if (!isSupportedAlterType(command->as<ASTAlterCommand&>().type))
|
||||
throw Exception("Unsupported type of ALTER query", ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
}
|
||||
|
||||
@ -423,11 +423,11 @@ bool ExpressionAnalyzer::makeAggregateDescriptions(ActionsDAGPtr & actions)
|
||||
for (const ASTFunction * node : aggregates())
|
||||
{
|
||||
AggregateDescription aggregate;
|
||||
getRootActionsNoMakeSet(node->arguments, true, actions);
|
||||
if (node->arguments) getRootActionsNoMakeSet(node->arguments, true, actions);
|
||||
|
||||
aggregate.column_name = node->getColumnName();
|
||||
|
||||
const ASTs & arguments = node->arguments->children;
|
||||
const ASTs & arguments = node->arguments ? node->arguments->children : ASTs();
|
||||
aggregate.argument_names.resize(arguments.size());
|
||||
DataTypes types(arguments.size());
|
||||
|
||||
@ -820,8 +820,9 @@ void SelectQueryExpressionAnalyzer::appendAggregateFunctionsArguments(Expression
|
||||
|
||||
/// TODO: data.aggregates -> aggregates()
|
||||
for (const ASTFunction * node : data.aggregates)
|
||||
for (auto & argument : node->arguments->children)
|
||||
getRootActions(argument, only_types, step.actions());
|
||||
if (node->arguments)
|
||||
for (auto & argument : node->arguments->children)
|
||||
getRootActions(argument, only_types, step.actions());
|
||||
}
|
||||
|
||||
bool SelectQueryExpressionAnalyzer::appendHaving(ExpressionActionsChain & chain, bool only_types)
|
||||
|
||||
@ -49,7 +49,7 @@ struct ExpressionAnalyzerData
|
||||
SubqueriesForSets subqueries_for_sets;
|
||||
PreparedSets prepared_sets;
|
||||
|
||||
/// Columns after ARRAY JOIN. It there is no ARRAY JOIN, it's source_columns.
|
||||
/// Columns after ARRAY JOIN. If there is no ARRAY JOIN, it's source_columns.
|
||||
NamesAndTypesList columns_after_array_join;
|
||||
/// Columns after Columns after ARRAY JOIN and JOIN. If there is no JOIN, it's columns_after_array_join.
|
||||
NamesAndTypesList columns_after_join;
|
||||
|
||||
@ -105,7 +105,7 @@ public:
|
||||
{
|
||||
if (auto * function_node = ast->as<ASTFunction>())
|
||||
{
|
||||
if (!(function_node->arguments->children.empty()))
|
||||
if (function_node->arguments && !function_node->arguments->children.empty())
|
||||
visit(function_node, data);
|
||||
}
|
||||
}
|
||||
|
||||
@ -56,8 +56,9 @@ BlockIO InterpreterAlterQuery::execute()
|
||||
PartitionCommands partition_commands;
|
||||
MutationCommands mutation_commands;
|
||||
LiveViewCommands live_view_commands;
|
||||
for (ASTAlterCommand * command_ast : alter.command_list->commands)
|
||||
for (const auto & child : alter.command_list->children)
|
||||
{
|
||||
auto * command_ast = child->as<ASTAlterCommand>();
|
||||
if (auto alter_command = AlterCommand::parse(command_ast))
|
||||
alter_commands.emplace_back(std::move(*alter_command));
|
||||
else if (auto partition_command = PartitionCommand::parse(command_ast))
|
||||
@ -124,8 +125,8 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccess() const
|
||||
{
|
||||
AccessRightsElements required_access;
|
||||
const auto & alter = query_ptr->as<ASTAlterQuery &>();
|
||||
for (ASTAlterCommand * command : alter.command_list->commands)
|
||||
boost::range::push_back(required_access, getRequiredAccessForCommand(*command, alter.database, alter.table));
|
||||
for (const auto & child : alter.command_list->children)
|
||||
boost::range::push_back(required_access, getRequiredAccessForCommand(child->as<ASTAlterCommand&>(), alter.database, alter.table));
|
||||
return required_access;
|
||||
}
|
||||
|
||||
|
||||
@ -130,6 +130,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
|
||||
auto engine = std::make_shared<ASTFunction>();
|
||||
auto storage = std::make_shared<ASTStorage>();
|
||||
engine->name = old_style_database ? "Ordinary" : "Atomic";
|
||||
engine->no_empty_args = true;
|
||||
storage->set(storage->engine, engine);
|
||||
create.set(create.storage, storage);
|
||||
}
|
||||
@ -600,6 +601,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
|
||||
{
|
||||
auto engine_ast = std::make_shared<ASTFunction>();
|
||||
engine_ast->name = "Memory";
|
||||
engine_ast->no_empty_args = true;
|
||||
auto storage_ast = std::make_shared<ASTStorage>();
|
||||
storage_ast->set(storage_ast->engine, engine_ast);
|
||||
create.set(create.storage, storage_ast);
|
||||
|
||||
@ -87,8 +87,7 @@ public:
|
||||
return;
|
||||
|
||||
/// TODO: monotonicity for functions of several arguments
|
||||
auto arguments = ast_function.arguments;
|
||||
if (arguments->children.size() != 1)
|
||||
if (!ast_function.arguments || ast_function.arguments->children.size() != 1)
|
||||
{
|
||||
data.reject();
|
||||
return;
|
||||
|
||||
@ -442,10 +442,10 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
|
||||
auto type_literal = std::make_shared<ASTLiteral>(columns_desc.getPhysical(column).type->getName());
|
||||
|
||||
const auto & update_expr = kv.second;
|
||||
auto updated_column = makeASTFunction("CAST",
|
||||
auto updated_column = makeASTFunction("cast",
|
||||
makeASTFunction("if",
|
||||
getPartitionAndPredicateExpressionForMutationCommand(command),
|
||||
makeASTFunction("CAST",
|
||||
makeASTFunction("cast",
|
||||
update_expr->clone(),
|
||||
type_literal),
|
||||
std::make_shared<ASTIdentifier>(column)),
|
||||
|
||||
@ -478,7 +478,7 @@ ASTs InterpreterAlterImpl::getRewrittenQueries(
|
||||
auto rewritten_rename_query = std::make_shared<ASTRenameQuery>();
|
||||
rewritten_alter_query->database = mapped_to_database;
|
||||
rewritten_alter_query->table = alter_query.table;
|
||||
rewritten_alter_query->set(rewritten_alter_query->command_list, std::make_shared<ASTAlterCommandList>());
|
||||
rewritten_alter_query->set(rewritten_alter_query->command_list, std::make_shared<ASTExpressionList>());
|
||||
|
||||
String default_after_column;
|
||||
for (const auto & command_query : alter_query.command_list->children)
|
||||
@ -542,7 +542,7 @@ ASTs InterpreterAlterImpl::getRewrittenQueries(
|
||||
}
|
||||
|
||||
rewritten_command->children.push_back(rewritten_command->col_decl);
|
||||
rewritten_alter_query->command_list->add(rewritten_command);
|
||||
rewritten_alter_query->command_list->children.push_back(rewritten_command);
|
||||
}
|
||||
}
|
||||
else if (alter_command->type == MySQLParser::ASTAlterCommand::DROP_COLUMN)
|
||||
@ -550,7 +550,7 @@ ASTs InterpreterAlterImpl::getRewrittenQueries(
|
||||
auto rewritten_command = std::make_shared<ASTAlterCommand>();
|
||||
rewritten_command->type = ASTAlterCommand::DROP_COLUMN;
|
||||
rewritten_command->column = std::make_shared<ASTIdentifier>(alter_command->column_name);
|
||||
rewritten_alter_query->command_list->add(rewritten_command);
|
||||
rewritten_alter_query->command_list->children.push_back(rewritten_command);
|
||||
}
|
||||
else if (alter_command->type == MySQLParser::ASTAlterCommand::RENAME_COLUMN)
|
||||
{
|
||||
@ -561,7 +561,7 @@ ASTs InterpreterAlterImpl::getRewrittenQueries(
|
||||
rewritten_command->type = ASTAlterCommand::RENAME_COLUMN;
|
||||
rewritten_command->column = std::make_shared<ASTIdentifier>(alter_command->old_name);
|
||||
rewritten_command->rename_to = std::make_shared<ASTIdentifier>(alter_command->column_name);
|
||||
rewritten_alter_query->command_list->add(rewritten_command);
|
||||
rewritten_alter_query->command_list->children.push_back(rewritten_command);
|
||||
}
|
||||
}
|
||||
else if (alter_command->type == MySQLParser::ASTAlterCommand::MODIFY_COLUMN)
|
||||
@ -590,7 +590,7 @@ ASTs InterpreterAlterImpl::getRewrittenQueries(
|
||||
rewritten_command->children.push_back(rewritten_command->column);
|
||||
}
|
||||
|
||||
rewritten_alter_query->command_list->add(rewritten_command);
|
||||
rewritten_alter_query->command_list->children.push_back(rewritten_command);
|
||||
}
|
||||
|
||||
if (!alter_command->old_name.empty() && alter_command->old_name != new_column_name)
|
||||
@ -599,7 +599,7 @@ ASTs InterpreterAlterImpl::getRewrittenQueries(
|
||||
rewritten_command->type = ASTAlterCommand::RENAME_COLUMN;
|
||||
rewritten_command->column = std::make_shared<ASTIdentifier>(alter_command->old_name);
|
||||
rewritten_command->rename_to = std::make_shared<ASTIdentifier>(new_column_name);
|
||||
rewritten_alter_query->command_list->add(rewritten_command);
|
||||
rewritten_alter_query->command_list->children.push_back(rewritten_command);
|
||||
}
|
||||
}
|
||||
else if (alter_command->type == MySQLParser::ASTAlterCommand::RENAME_TABLE)
|
||||
@ -624,7 +624,7 @@ ASTs InterpreterAlterImpl::getRewrittenQueries(
|
||||
ASTs rewritten_queries;
|
||||
|
||||
/// Order is very important. We always execute alter first and then execute rename
|
||||
if (!rewritten_alter_query->command_list->commands.empty())
|
||||
if (!rewritten_alter_query->command_list->children.empty())
|
||||
rewritten_queries.push_back(rewritten_alter_query);
|
||||
|
||||
if (!rewritten_rename_query->elements.empty())
|
||||
|
||||
@ -29,7 +29,7 @@ static bool tryExtractConstValueFromCondition(const ASTPtr & condition, bool & v
|
||||
/// cast of numeric constant in condition to UInt8
|
||||
if (const auto * function = condition->as<ASTFunction>())
|
||||
{
|
||||
if (function->name == "CAST")
|
||||
if (function->name == "cast")
|
||||
{
|
||||
if (const auto * expr_list = function->arguments->as<ASTExpressionList>())
|
||||
{
|
||||
@ -64,13 +64,17 @@ void OptimizeIfWithConstantConditionVisitor::visit(ASTPtr & current_ast)
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!function_node->arguments)
|
||||
throw Exception("Wrong number of arguments for function 'if' (0 instead of 3)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
if (function_node->arguments->children.size() != 3)
|
||||
throw Exception(
|
||||
"Wrong number of arguments for function 'if' (" + toString(function_node->arguments->children.size()) + " instead of 3)",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
visit(function_node->arguments);
|
||||
const auto * args = function_node->arguments->as<ASTExpressionList>();
|
||||
|
||||
if (args->children.size() != 3)
|
||||
throw Exception("Wrong number of arguments for function 'if' (" + toString(args->children.size()) + " instead of 3)",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
ASTPtr condition_expr = args->children[0];
|
||||
ASTPtr then_expr = args->children[1];
|
||||
ASTPtr else_expr = args->children[2];
|
||||
|
||||
@ -164,14 +164,17 @@ void QueryNormalizer::visitChildren(const ASTPtr & node, Data & data)
|
||||
if (func_node->name == "lambda")
|
||||
first_pos = 1;
|
||||
|
||||
auto & func_children = func_node->arguments->children;
|
||||
|
||||
for (size_t i = first_pos; i < func_children.size(); ++i)
|
||||
if (func_node->arguments)
|
||||
{
|
||||
auto & child = func_children[i];
|
||||
auto & func_children = func_node->arguments->children;
|
||||
|
||||
if (needVisitChild(child))
|
||||
visit(child, data);
|
||||
for (size_t i = first_pos; i < func_children.size(); ++i)
|
||||
{
|
||||
auto & child = func_children[i];
|
||||
|
||||
if (needVisitChild(child))
|
||||
visit(child, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (!node->as<ASTSelectQuery>())
|
||||
|
||||
@ -22,7 +22,7 @@ static bool removeInjectiveFunction(ASTPtr & ast, const Context & context, const
|
||||
if (!func)
|
||||
return false;
|
||||
|
||||
if (func->arguments->children.size() != 1)
|
||||
if (!func->arguments || func->arguments->children.size() != 1)
|
||||
return false;
|
||||
|
||||
if (!function_factory.get(func->name, context)->isInjective({}))
|
||||
|
||||
@ -62,7 +62,7 @@ void RewriteAnyFunctionMatcher::visit(ASTPtr & ast, Data & data)
|
||||
|
||||
void RewriteAnyFunctionMatcher::visit(const ASTFunction & func, ASTPtr & ast, Data & data)
|
||||
{
|
||||
if (func.arguments->children.empty() || !func.arguments->children[0])
|
||||
if (!func.arguments || func.arguments->children.empty() || !func.arguments->children[0])
|
||||
return;
|
||||
|
||||
if (func.name != "any" && func.name != "anyLast")
|
||||
|
||||
@ -28,6 +28,7 @@ class ASTQueryWithTableAndOutput;
|
||||
class ASTIdentifier;
|
||||
class Context;
|
||||
|
||||
// TODO(ilezhankin): refactor and merge |ASTTableIdentifier|
|
||||
struct StorageID
|
||||
{
|
||||
String database_name;
|
||||
|
||||
@ -74,6 +74,10 @@ std::shared_ptr<TSystemLog> createSystemLog(
|
||||
engine += " TTL " + ttl;
|
||||
engine += " ORDER BY (event_date, event_time)";
|
||||
}
|
||||
// Validate engine definition grammatically to prevent some configuration errors
|
||||
ParserStorage storage_parser;
|
||||
parseQuery(storage_parser, engine.data(), engine.data() + engine.size(),
|
||||
"Storage to create table for " + config_prefix, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
|
||||
|
||||
size_t flush_interval_milliseconds = config.getUInt64(config_prefix + ".flush_interval_milliseconds",
|
||||
DEFAULT_SYSTEM_LOG_FLUSH_INTERVAL_MILLISECONDS);
|
||||
|
||||
@ -126,6 +126,8 @@ void TranslateQualifiedNamesMatcher::visit(ASTFunction & node, const ASTPtr &, D
|
||||
{
|
||||
ASTPtr & func_arguments = node.arguments;
|
||||
|
||||
if (!func_arguments) return;
|
||||
|
||||
String func_name_lowercase = Poco::toLower(node.name);
|
||||
if (func_name_lowercase == "count" &&
|
||||
func_arguments->children.size() == 1 &&
|
||||
|
||||
@ -380,8 +380,9 @@ std::vector<const ASTFunction *> getAggregates(ASTPtr & query, const ASTSelectQu
|
||||
|
||||
/// There can not be other aggregate functions within the aggregate functions.
|
||||
for (const ASTFunction * node : data.aggregates)
|
||||
for (auto & arg : node->arguments->children)
|
||||
assertNoAggregates(arg, "inside another aggregate function");
|
||||
if (node->arguments)
|
||||
for (auto & arg : node->arguments->children)
|
||||
assertNoAggregates(arg, "inside another aggregate function");
|
||||
return data.aggregates;
|
||||
}
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ namespace DB
|
||||
|
||||
ASTPtr addTypeConversionToAST(ASTPtr && ast, const String & type_name)
|
||||
{
|
||||
auto func = makeASTFunction("CAST", ast, std::make_shared<ASTLiteral>(type_name));
|
||||
auto func = makeASTFunction("cast", ast, std::make_shared<ASTLiteral>(type_name));
|
||||
|
||||
if (ASTWithAlias * ast_with_alias = dynamic_cast<ASTWithAlias *>(ast.get()))
|
||||
{
|
||||
|
||||
@ -15,18 +15,23 @@
|
||||
#include <DataStreams/InputStreamFromASTInsertQuery.h>
|
||||
#include <DataStreams/CountingBlockOutputStream.h>
|
||||
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTInsertQuery.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/ASTSelectQuery.h>
|
||||
#include <Parsers/ASTSelectWithUnionQuery.h>
|
||||
#include <Parsers/ASTShowProcesslistQuery.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/ParserQuery.h>
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
#include <Parsers/ASTWatchQuery.h>
|
||||
#include <Parsers/Lexer.h>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include <Parsers/New/parseQuery.h> // Y_IGNORE
|
||||
#endif
|
||||
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include <Parsers/ParserQuery.h>
|
||||
#include <Parsers/queryToString.h>
|
||||
|
||||
#include <Storages/StorageInput.h>
|
||||
|
||||
#include <Access/EnabledQuota.h>
|
||||
@ -147,10 +152,11 @@ static void logQuery(const String & query, const Context & context, bool interna
|
||||
const auto & initial_query_id = client_info.initial_query_id;
|
||||
const auto & current_user = client_info.current_user;
|
||||
|
||||
LOG_DEBUG(&Poco::Logger::get("executeQuery"), "(from {}{}{}) {}",
|
||||
LOG_DEBUG(&Poco::Logger::get("executeQuery"), "(from {}{}{}, using {} parser) {}",
|
||||
client_info.current_address.toString(),
|
||||
(current_user != "default" ? ", user: " + current_user : ""),
|
||||
(!initial_query_id.empty() && current_query_id != initial_query_id ? ", initial_query_id: " + initial_query_id : std::string()),
|
||||
(context.getSettingsRef().use_antlr_parser ? "new" : "old"),
|
||||
joinLines(query));
|
||||
|
||||
if (client_info.client_trace_context.trace_id)
|
||||
@ -321,19 +327,33 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
|
||||
ParserQuery parser(end);
|
||||
ASTPtr ast;
|
||||
const char * query_end;
|
||||
|
||||
/// Don't limit the size of internal queries.
|
||||
size_t max_query_size = 0;
|
||||
if (!internal)
|
||||
max_query_size = settings.max_query_size;
|
||||
if (!internal) max_query_size = settings.max_query_size;
|
||||
|
||||
try
|
||||
{
|
||||
/// TODO Parser should fail early when max_query_size limit is reached.
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
if (settings.use_antlr_parser)
|
||||
{
|
||||
ast = parseQuery(begin, end, max_query_size, settings.max_parser_depth);
|
||||
}
|
||||
else
|
||||
{
|
||||
ParserQuery parser(end);
|
||||
|
||||
/// TODO: parser should fail early when max_query_size limit is reached.
|
||||
ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth);
|
||||
}
|
||||
#else
|
||||
ParserQuery parser(end);
|
||||
|
||||
/// TODO: parser should fail early when max_query_size limit is reached.
|
||||
ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth);
|
||||
#endif
|
||||
|
||||
/// Interpret SETTINGS clauses as early as possible (before invoking the corresponding interpreter),
|
||||
/// to allow settings to take effect.
|
||||
@ -874,7 +894,7 @@ void executeQuery(
|
||||
end = istr.buffer().end();
|
||||
istr.position() += end - begin;
|
||||
/// Actually we don't know will query has additional data or not.
|
||||
/// But we can't check istr.eof(), because begin and end pointers will became invalid
|
||||
/// But we can't check istr.eof(), because begin and end pointers will become invalid
|
||||
may_have_tail = true;
|
||||
}
|
||||
else
|
||||
|
||||
@ -43,7 +43,7 @@ void addDefaultRequiredExpressionsRecursively(Block & block, const String & requ
|
||||
RequiredSourceColumnsVisitor(columns_context).visit(column_default_expr);
|
||||
NameSet required_columns_names = columns_context.requiredColumns();
|
||||
|
||||
auto cast_func = makeASTFunction("CAST", column_default_expr, std::make_shared<ASTLiteral>(columns.get(required_column).type->getName()));
|
||||
auto cast_func = makeASTFunction("cast", column_default_expr, std::make_shared<ASTLiteral>(columns.get(required_column).type->getName()));
|
||||
default_expr_list_accum->children.emplace_back(setAlias(cast_func, required_column));
|
||||
added_columns.emplace(required_column);
|
||||
|
||||
@ -79,7 +79,7 @@ ASTPtr convertRequiredExpressions(Block & block, const NamesAndTypesList & requi
|
||||
continue;
|
||||
|
||||
auto cast_func = makeASTFunction(
|
||||
"CAST", std::make_shared<ASTIdentifier>(required_column.name), std::make_shared<ASTLiteral>(required_column.type->getName()));
|
||||
"cast", std::make_shared<ASTIdentifier>(required_column.name), std::make_shared<ASTLiteral>(required_column.type->getName()));
|
||||
|
||||
conversion_expr_list->children.emplace_back(setAlias(cast_func, required_column.name));
|
||||
|
||||
|
||||
@ -345,30 +345,6 @@ void ASTAlterCommand::formatImpl(
|
||||
}
|
||||
|
||||
|
||||
ASTPtr ASTAlterCommandList::clone() const
|
||||
{
|
||||
auto res = std::make_shared<ASTAlterCommandList>();
|
||||
for (ASTAlterCommand * command : commands)
|
||||
res->add(command->clone());
|
||||
return res;
|
||||
}
|
||||
|
||||
void ASTAlterCommandList::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
|
||||
{
|
||||
std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' ');
|
||||
|
||||
for (size_t i = 0; i < commands.size(); ++i)
|
||||
{
|
||||
static_cast<IAST *>(commands[i])->formatImpl(settings, state, frame);
|
||||
|
||||
std::string comma = (i < (commands.size() - 1)) ? "," : "";
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << comma << (settings.hilite ? hilite_none : "");
|
||||
|
||||
settings.ostr << settings.nl_or_ws;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** Get the text that identifies this element. */
|
||||
String ASTAlterQuery::getID(char delim) const
|
||||
{
|
||||
|
||||
@ -1,9 +1,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <Parsers/IAST.h>
|
||||
#include <Parsers/ASTQueryWithTableAndOutput.h>
|
||||
#include <Parsers/ASTExpressionList.h>
|
||||
#include <Parsers/ASTQueryWithOnCluster.h>
|
||||
#include <Parsers/ASTQueryWithTableAndOutput.h>
|
||||
#include <Parsers/ASTTTLElement.h>
|
||||
#include <Parsers/IAST.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -179,31 +180,12 @@ protected:
|
||||
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
|
||||
};
|
||||
|
||||
class ASTAlterCommandList : public IAST
|
||||
{
|
||||
public:
|
||||
std::vector<ASTAlterCommand *> commands;
|
||||
|
||||
void add(const ASTPtr & command)
|
||||
{
|
||||
commands.push_back(command->as<ASTAlterCommand>());
|
||||
children.push_back(command);
|
||||
}
|
||||
|
||||
String getID(char) const override { return "AlterCommandList"; }
|
||||
|
||||
ASTPtr clone() const override;
|
||||
|
||||
protected:
|
||||
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
|
||||
};
|
||||
|
||||
class ASTAlterQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster
|
||||
{
|
||||
public:
|
||||
bool is_live_view{false}; /// true for ALTER LIVE VIEW
|
||||
|
||||
ASTAlterCommandList * command_list = nullptr;
|
||||
ASTExpressionList * command_list = nullptr;
|
||||
|
||||
String getID(char) const override;
|
||||
|
||||
|
||||
@ -4,10 +4,10 @@
|
||||
#include <Parsers/ASTQueryWithOnCluster.h>
|
||||
#include <Parsers/ASTDictionary.h>
|
||||
#include <Parsers/ASTDictionaryAttributeDeclaration.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTSelectWithUnionQuery.h>
|
||||
#include <Interpreters/StorageID.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
@ -65,7 +65,7 @@ ASTPtr ASTDictionaryLayout::clone() const
|
||||
{
|
||||
auto res = std::make_shared<ASTDictionaryLayout>();
|
||||
res->layout_type = layout_type;
|
||||
res->set(res->parameters, parameters->clone());
|
||||
if (parameters) res->set(res->parameters, parameters->clone());
|
||||
res->has_brackets = has_brackets;
|
||||
return res;
|
||||
}
|
||||
@ -86,7 +86,7 @@ void ASTDictionaryLayout::formatImpl(const FormatSettings & settings,
|
||||
if (has_brackets)
|
||||
settings.ostr << "(";
|
||||
|
||||
parameters->formatImpl(settings, state, frame);
|
||||
if (parameters) parameters->formatImpl(settings, state, frame);
|
||||
|
||||
if (has_brackets)
|
||||
settings.ostr << ")";
|
||||
|
||||
@ -29,12 +29,13 @@ void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const
|
||||
}
|
||||
|
||||
writeChar('(', ostr);
|
||||
for (auto it = arguments->children.begin(); it != arguments->children.end(); ++it)
|
||||
{
|
||||
if (it != arguments->children.begin())
|
||||
writeCString(", ", ostr);
|
||||
(*it)->appendColumnName(ostr);
|
||||
}
|
||||
if (arguments)
|
||||
for (auto it = arguments->children.begin(); it != arguments->children.end(); ++it)
|
||||
{
|
||||
if (it != arguments->children.begin())
|
||||
writeCString(", ", ostr);
|
||||
(*it)->appendColumnName(ostr);
|
||||
}
|
||||
writeChar(')', ostr);
|
||||
}
|
||||
|
||||
@ -64,6 +65,35 @@ void ASTFunction::updateTreeHashImpl(SipHash & hash_state) const
|
||||
}
|
||||
|
||||
|
||||
ASTPtr ASTFunction::toLiteral() const
|
||||
{
|
||||
if (!arguments) return {};
|
||||
|
||||
if (name == "array")
|
||||
{
|
||||
Array array;
|
||||
|
||||
for (const auto & arg : arguments->children)
|
||||
{
|
||||
if (auto * literal = arg->as<ASTLiteral>())
|
||||
array.push_back(literal->value);
|
||||
else if (auto * func = arg->as<ASTFunction>())
|
||||
{
|
||||
if (auto func_literal = func->toLiteral())
|
||||
array.push_back(func_literal->as<ASTLiteral>()->value);
|
||||
}
|
||||
else
|
||||
/// Some of the Array arguments is not literal
|
||||
return {};
|
||||
}
|
||||
|
||||
return std::make_shared<ASTLiteral>(array);
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
/** A special hack. If it's [I]LIKE or NOT [I]LIKE expression and the right hand side is a string literal,
|
||||
* we will highlight unescaped metacharacters % and _ in string literal for convenience.
|
||||
* Motivation: most people are unaware that _ is a metacharacter and forgot to properly escape it with two backslashes.
|
||||
@ -378,12 +408,14 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
|
||||
settings.ostr << (settings.hilite ? hilite_function : "") << ')';
|
||||
}
|
||||
|
||||
if (arguments)
|
||||
{
|
||||
if ((arguments && !arguments->children.empty()) || !no_empty_args)
|
||||
settings.ostr << '(' << (settings.hilite ? hilite_none : "");
|
||||
|
||||
if (arguments)
|
||||
{
|
||||
bool special_hilite_regexp = settings.hilite
|
||||
&& (name == "match" || name == "extract" || name == "extractAll" || name == "replaceRegexpOne" || name == "replaceRegexpAll");
|
||||
&& (name == "match" || name == "extract" || name == "extractAll" || name == "replaceRegexpOne"
|
||||
|| name == "replaceRegexpAll");
|
||||
|
||||
for (size_t i = 0, size = arguments->children.size(); i < size; ++i)
|
||||
{
|
||||
@ -397,10 +429,11 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
|
||||
if (!special_hilite)
|
||||
arguments->children[i]->formatImpl(settings, state, nested_dont_need_parens);
|
||||
}
|
||||
|
||||
settings.ostr << (settings.hilite ? hilite_function : "") << ')';
|
||||
}
|
||||
|
||||
if ((arguments && !arguments->children.empty()) || !no_empty_args)
|
||||
settings.ostr << (settings.hilite ? hilite_function : "") << ')';
|
||||
|
||||
settings.ostr << (settings.hilite ? hilite_none : "");
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user