mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 23:52:03 +00:00
Merge branch 'master' into sanych73-prepared_statements
This commit is contained in:
Alexey Milovidov
commit
eb4f0920b1
@ -1,8 +1,5 @@
|
||||
<yandex>
|
||||
<!-- <zookeeper>
|
||||
<node>
|
||||
<host>localhost</host>
|
||||
<port>2181</port>
|
||||
</node>
|
||||
</zookeeper>-->
|
||||
<zookeeper>
|
||||
<implementation>testkeeper</implementation>
|
||||
</zookeeper>
|
||||
</yandex>
|
||||
|
||||
@ -19,6 +19,7 @@ namespace ErrorCodes
|
||||
extern const int STD_EXCEPTION;
|
||||
extern const int UNKNOWN_EXCEPTION;
|
||||
extern const int CANNOT_TRUNCATE_FILE;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
std::string errnoToString(int code, int e)
|
||||
|
||||
727
dbms/src/Common/ZooKeeper/TestKeeper.cpp
Normal file
727
dbms/src/Common/ZooKeeper/TestKeeper.cpp
Normal file
@ -0,0 +1,727 @@
|
||||
#include <Common/ZooKeeper/TestKeeper.h>
|
||||
#include <Common/setThreadName.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Core/Types.h>
|
||||
|
||||
#include <sstream>
|
||||
#include <iomanip>
|
||||
|
||||
|
||||
namespace Coordination
|
||||
{
|
||||
|
||||
static String parentPath(const String & path)
|
||||
{
|
||||
auto rslash_pos = path.rfind('/');
|
||||
if (rslash_pos > 0)
|
||||
return path.substr(0, rslash_pos);
|
||||
return "/";
|
||||
}
|
||||
|
||||
static String baseName(const String & path)
|
||||
{
|
||||
auto rslash_pos = path.rfind('/');
|
||||
return path.substr(rslash_pos + 1);
|
||||
}
|
||||
|
||||
|
||||
struct TestKeeperRequest : virtual Request
|
||||
{
|
||||
virtual bool isMutable() const { return false; }
|
||||
virtual ResponsePtr createResponse() const = 0;
|
||||
virtual ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const = 0;
|
||||
virtual void processWatches(TestKeeper::Watches & /*watches*/, TestKeeper::Watches & /*list_watches*/) const {}
|
||||
};
|
||||
|
||||
|
||||
static void processWatchesImpl(const String & path, TestKeeper::Watches & watches, TestKeeper::Watches & list_watches)
|
||||
{
|
||||
WatchResponse watch_response;
|
||||
watch_response.path = path;
|
||||
|
||||
auto it = watches.find(watch_response.path);
|
||||
if (it != watches.end())
|
||||
{
|
||||
for (auto & callback : it->second)
|
||||
if (callback)
|
||||
callback(watch_response);
|
||||
|
||||
watches.erase(it);
|
||||
}
|
||||
|
||||
WatchResponse watch_list_response;
|
||||
watch_list_response.path = parentPath(path);
|
||||
|
||||
it = list_watches.find(watch_list_response.path);
|
||||
if (it != list_watches.end())
|
||||
{
|
||||
for (auto & callback : it->second)
|
||||
if (callback)
|
||||
callback(watch_list_response);
|
||||
|
||||
list_watches.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
struct TestKeeperCreateRequest final : CreateRequest, TestKeeperRequest
|
||||
{
|
||||
TestKeeperCreateRequest() {}
|
||||
TestKeeperCreateRequest(const CreateRequest & base) : CreateRequest(base) {}
|
||||
ResponsePtr createResponse() const override;
|
||||
ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const override;
|
||||
|
||||
void processWatches(TestKeeper::Watches & node_watches, TestKeeper::Watches & list_watches) const override
|
||||
{
|
||||
processWatchesImpl(getPath(), node_watches, list_watches);
|
||||
}
|
||||
};
|
||||
|
||||
struct TestKeeperRemoveRequest final : RemoveRequest, TestKeeperRequest
|
||||
{
|
||||
TestKeeperRemoveRequest() {}
|
||||
TestKeeperRemoveRequest(const RemoveRequest & base) : RemoveRequest(base) {}
|
||||
bool isMutable() const override { return true; }
|
||||
ResponsePtr createResponse() const override;
|
||||
ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const override;
|
||||
|
||||
void processWatches(TestKeeper::Watches & node_watches, TestKeeper::Watches & list_watches) const override
|
||||
{
|
||||
processWatchesImpl(getPath(), node_watches, list_watches);
|
||||
}
|
||||
};
|
||||
|
||||
struct TestKeeperExistsRequest final : ExistsRequest, TestKeeperRequest
|
||||
{
|
||||
ResponsePtr createResponse() const override;
|
||||
ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const override;
|
||||
};
|
||||
|
||||
struct TestKeeperGetRequest final : GetRequest, TestKeeperRequest
|
||||
{
|
||||
TestKeeperGetRequest() {}
|
||||
ResponsePtr createResponse() const override;
|
||||
ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const override;
|
||||
};
|
||||
|
||||
struct TestKeeperSetRequest final : SetRequest, TestKeeperRequest
|
||||
{
|
||||
TestKeeperSetRequest() {}
|
||||
TestKeeperSetRequest(const SetRequest & base) : SetRequest(base) {}
|
||||
bool isMutable() const override { return true; }
|
||||
ResponsePtr createResponse() const override;
|
||||
ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const override;
|
||||
|
||||
void processWatches(TestKeeper::Watches & node_watches, TestKeeper::Watches & list_watches) const override
|
||||
{
|
||||
processWatchesImpl(getPath(), node_watches, list_watches);
|
||||
}
|
||||
};
|
||||
|
||||
struct TestKeeperListRequest final : ListRequest, TestKeeperRequest
|
||||
{
|
||||
ResponsePtr createResponse() const override;
|
||||
ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const override;
|
||||
};
|
||||
|
||||
struct TestKeeperCheckRequest final : CheckRequest, TestKeeperRequest
|
||||
{
|
||||
TestKeeperCheckRequest() {}
|
||||
TestKeeperCheckRequest(const CheckRequest & base) : CheckRequest(base) {}
|
||||
ResponsePtr createResponse() const override;
|
||||
ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const override;
|
||||
};
|
||||
|
||||
struct TestKeeperMultiRequest final : MultiRequest, TestKeeperRequest
|
||||
{
|
||||
TestKeeperMultiRequest(const Requests & generic_requests)
|
||||
{
|
||||
requests.reserve(generic_requests.size());
|
||||
|
||||
for (const auto & generic_request : generic_requests)
|
||||
{
|
||||
if (auto * concrete_request_create = dynamic_cast<const CreateRequest *>(generic_request.get()))
|
||||
{
|
||||
auto create = std::make_shared<TestKeeperCreateRequest>(*concrete_request_create);
|
||||
requests.push_back(create);
|
||||
}
|
||||
else if (auto * concrete_request_remove = dynamic_cast<const RemoveRequest *>(generic_request.get()))
|
||||
{
|
||||
requests.push_back(std::make_shared<TestKeeperRemoveRequest>(*concrete_request_remove));
|
||||
}
|
||||
else if (auto * concrete_request_set = dynamic_cast<const SetRequest *>(generic_request.get()))
|
||||
{
|
||||
requests.push_back(std::make_shared<TestKeeperSetRequest>(*concrete_request_set));
|
||||
}
|
||||
else if (auto * concrete_request_check = dynamic_cast<const CheckRequest *>(generic_request.get()))
|
||||
{
|
||||
requests.push_back(std::make_shared<TestKeeperCheckRequest>(*concrete_request_check));
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal command as part of multi ZooKeeper request", ZBADARGUMENTS);
|
||||
}
|
||||
}
|
||||
|
||||
void processWatches(TestKeeper::Watches & node_watches, TestKeeper::Watches & list_watches) const override
|
||||
{
|
||||
for (const auto & generic_request : requests)
|
||||
dynamic_cast<const TestKeeperRequest &>(*generic_request).processWatches(node_watches, list_watches);
|
||||
}
|
||||
|
||||
ResponsePtr createResponse() const override;
|
||||
ResponsePtr process(TestKeeper::Container & container, int64_t zxid) const override;
|
||||
};
|
||||
|
||||
|
||||
ResponsePtr TestKeeperCreateRequest::process(TestKeeper::Container & container, int64_t zxid) const
|
||||
{
|
||||
CreateResponse response;
|
||||
if (container.count(path))
|
||||
{
|
||||
response.error = Error::ZNODEEXISTS;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto it = container.find(parentPath(path));
|
||||
|
||||
if (it == container.end())
|
||||
{
|
||||
response.error = Error::ZNONODE;
|
||||
}
|
||||
else if (it->second.is_ephemeral)
|
||||
{
|
||||
response.error = Error::ZNOCHILDRENFOREPHEMERALS;
|
||||
}
|
||||
else
|
||||
{
|
||||
TestKeeper::Node created_node;
|
||||
created_node.seq_num = 0;
|
||||
created_node.stat.czxid = zxid;
|
||||
created_node.stat.mzxid = zxid;
|
||||
created_node.stat.ctime = std::chrono::system_clock::now().time_since_epoch() / std::chrono::milliseconds(1);
|
||||
created_node.stat.mtime = created_node.stat.ctime;
|
||||
created_node.stat.numChildren = 0;
|
||||
created_node.stat.dataLength = data.length();
|
||||
created_node.data = data;
|
||||
created_node.is_ephemeral = is_ephemeral;
|
||||
created_node.is_sequental = is_sequential;
|
||||
std::string path_created = path;
|
||||
|
||||
if (is_sequential)
|
||||
{
|
||||
auto seq_num = it->second.seq_num;
|
||||
++it->second.seq_num;
|
||||
|
||||
std::stringstream seq_num_str;
|
||||
seq_num_str << std::setw(10) << std::setfill('0') << seq_num;
|
||||
|
||||
path_created += seq_num_str.str();
|
||||
}
|
||||
|
||||
response.path_created = path_created;
|
||||
container.emplace(std::move(path_created), std::move(created_node));
|
||||
|
||||
++it->second.stat.cversion;
|
||||
++it->second.stat.numChildren;
|
||||
|
||||
response.error = Error::ZOK;
|
||||
}
|
||||
}
|
||||
|
||||
return std::make_shared<CreateResponse>(response);
|
||||
}
|
||||
|
||||
ResponsePtr TestKeeperRemoveRequest::process(TestKeeper::Container & container, int64_t) const
|
||||
{
|
||||
RemoveResponse response;
|
||||
|
||||
auto it = container.find(path);
|
||||
if (it == container.end())
|
||||
{
|
||||
response.error = Error::ZNONODE;
|
||||
}
|
||||
else if (version != -1 && version != it->second.stat.version)
|
||||
{
|
||||
response.error = Error::ZBADVERSION;
|
||||
}
|
||||
else if (it->second.stat.numChildren)
|
||||
{
|
||||
response.error = Error::ZNOTEMPTY;
|
||||
}
|
||||
else
|
||||
{
|
||||
container.erase(it);
|
||||
auto & parent = container.at(parentPath(path));
|
||||
--parent.stat.numChildren;
|
||||
++parent.stat.cversion;
|
||||
response.error = Error::ZOK;
|
||||
}
|
||||
|
||||
return std::make_shared<RemoveResponse>(response);
|
||||
}
|
||||
|
||||
ResponsePtr TestKeeperExistsRequest::process(TestKeeper::Container & container, int64_t) const
|
||||
{
|
||||
ExistsResponse response;
|
||||
|
||||
auto it = container.find(path);
|
||||
if (it != container.end())
|
||||
{
|
||||
response.stat = it->second.stat;
|
||||
response.error = Error::ZOK;
|
||||
}
|
||||
else
|
||||
{
|
||||
response.error = Error::ZNONODE;
|
||||
}
|
||||
|
||||
return std::make_shared<ExistsResponse>(response);
|
||||
}
|
||||
|
||||
ResponsePtr TestKeeperGetRequest::process(TestKeeper::Container & container, int64_t) const
|
||||
{
|
||||
GetResponse response;
|
||||
|
||||
auto it = container.find(path);
|
||||
if (it == container.end())
|
||||
{
|
||||
response.error = Error::ZNONODE;
|
||||
}
|
||||
else
|
||||
{
|
||||
response.stat = it->second.stat;
|
||||
response.data = it->second.data;
|
||||
response.error = Error::ZOK;
|
||||
}
|
||||
|
||||
return std::make_shared<GetResponse>(response);
|
||||
}
|
||||
|
||||
ResponsePtr TestKeeperSetRequest::process(TestKeeper::Container & container, int64_t zxid) const
|
||||
{
|
||||
SetResponse response;
|
||||
|
||||
auto it = container.find(path);
|
||||
if (it == container.end())
|
||||
{
|
||||
response.error = Error::ZNONODE;
|
||||
}
|
||||
else if (version == -1 || version == it->second.stat.version)
|
||||
{
|
||||
it->second.data = data;
|
||||
++it->second.stat.version;
|
||||
it->second.stat.mzxid = zxid;
|
||||
it->second.stat.mtime = std::chrono::system_clock::now().time_since_epoch() / std::chrono::milliseconds(1);
|
||||
it->second.data = data;
|
||||
++container.at(parentPath(path)).stat.cversion;
|
||||
response.stat = it->second.stat;
|
||||
response.error = Error::ZOK;
|
||||
}
|
||||
else
|
||||
{
|
||||
response.error = Error::ZBADVERSION;
|
||||
}
|
||||
|
||||
return std::make_shared<SetResponse>(response);
|
||||
}
|
||||
|
||||
ResponsePtr TestKeeperListRequest::process(TestKeeper::Container & container, int64_t) const
|
||||
{
|
||||
ListResponse response;
|
||||
|
||||
auto it = container.find(path);
|
||||
if (it == container.end())
|
||||
{
|
||||
response.error = Error::ZNONODE;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto path_prefix = path;
|
||||
if (path_prefix.empty())
|
||||
throw Exception("Logical error: path cannot be empty", ZSESSIONEXPIRED);
|
||||
|
||||
if (path_prefix.back() != '/')
|
||||
path_prefix += '/';
|
||||
|
||||
/// Fairly inefficient.
|
||||
for (auto child_it = container.upper_bound(path_prefix); child_it != container.end() && startsWith(child_it->first, path_prefix); ++child_it)
|
||||
if (parentPath(child_it->first) == path)
|
||||
response.names.emplace_back(baseName(child_it->first));
|
||||
|
||||
response.stat = it->second.stat;
|
||||
response.error = Error::ZOK;
|
||||
}
|
||||
|
||||
return std::make_shared<ListResponse>(response);
|
||||
}
|
||||
|
||||
ResponsePtr TestKeeperCheckRequest::process(TestKeeper::Container & container, int64_t) const
|
||||
{
|
||||
CheckResponse response;
|
||||
auto it = container.find(path);
|
||||
if (it == container.end())
|
||||
{
|
||||
response.error = Error::ZNONODE;
|
||||
}
|
||||
else if (version != -1 && version != it->second.stat.version)
|
||||
{
|
||||
response.error = Error::ZBADVERSION;
|
||||
}
|
||||
else
|
||||
{
|
||||
response.error = Error::ZOK;
|
||||
}
|
||||
|
||||
return std::make_shared<CheckResponse>(response);
|
||||
}
|
||||
|
||||
ResponsePtr TestKeeperMultiRequest::process(TestKeeper::Container & container, int64_t zxid) const
|
||||
{
|
||||
MultiResponse response;
|
||||
response.responses.reserve(requests.size());
|
||||
|
||||
/// Fairly inefficient.
|
||||
auto container_copy = container;
|
||||
|
||||
try
|
||||
{
|
||||
for (const auto & request : requests)
|
||||
{
|
||||
const TestKeeperRequest & concrete_request = dynamic_cast<const TestKeeperRequest &>(*request);
|
||||
auto cur_response = concrete_request.process(container, zxid);
|
||||
response.responses.emplace_back(cur_response);
|
||||
if (cur_response->error != Error::ZOK)
|
||||
{
|
||||
response.error = cur_response->error;
|
||||
container = container_copy;
|
||||
return std::make_shared<MultiResponse>(response);
|
||||
}
|
||||
}
|
||||
|
||||
response.error = Error::ZOK;
|
||||
return std::make_shared<MultiResponse>(response);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
container = container_copy;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
ResponsePtr TestKeeperCreateRequest::createResponse() const { return std::make_shared<CreateResponse>(); }
|
||||
ResponsePtr TestKeeperRemoveRequest::createResponse() const { return std::make_shared<RemoveResponse>(); }
|
||||
ResponsePtr TestKeeperExistsRequest::createResponse() const { return std::make_shared<ExistsResponse>(); }
|
||||
ResponsePtr TestKeeperGetRequest::createResponse() const { return std::make_shared<GetResponse>(); }
|
||||
ResponsePtr TestKeeperSetRequest::createResponse() const { return std::make_shared<SetResponse>(); }
|
||||
ResponsePtr TestKeeperListRequest::createResponse() const { return std::make_shared<ListResponse>(); }
|
||||
ResponsePtr TestKeeperCheckRequest::createResponse() const { return std::make_shared<CheckResponse>(); }
|
||||
ResponsePtr TestKeeperMultiRequest::createResponse() const { return std::make_shared<MultiResponse>(); }
|
||||
|
||||
|
||||
TestKeeper::TestKeeper(const String & root_path_, Poco::Timespan operation_timeout)
|
||||
: root_path(root_path_), operation_timeout(operation_timeout)
|
||||
{
|
||||
container.emplace("/", Node());
|
||||
|
||||
if (!root_path.empty())
|
||||
{
|
||||
if (root_path.back() == '/')
|
||||
root_path.pop_back();
|
||||
}
|
||||
|
||||
processing_thread = ThreadFromGlobalPool([this] { processingThread(); });
|
||||
}
|
||||
|
||||
|
||||
TestKeeper::~TestKeeper()
|
||||
{
|
||||
try
|
||||
{
|
||||
finalize();
|
||||
if (processing_thread.joinable())
|
||||
processing_thread.join();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void TestKeeper::processingThread()
|
||||
{
|
||||
setThreadName("TestKeeperProc");
|
||||
|
||||
try
|
||||
{
|
||||
while (!expired)
|
||||
{
|
||||
RequestInfo info;
|
||||
|
||||
UInt64 max_wait = UInt64(operation_timeout.totalMilliseconds());
|
||||
if (requests_queue.tryPop(info, max_wait))
|
||||
{
|
||||
if (expired)
|
||||
break;
|
||||
|
||||
if (info.watch)
|
||||
{
|
||||
auto & watches_type = dynamic_cast<const ListRequest *>(info.request.get())
|
||||
? list_watches
|
||||
: watches;
|
||||
|
||||
watches_type[info.request->getPath()].emplace_back(std::move(info.watch));
|
||||
}
|
||||
|
||||
++zxid;
|
||||
|
||||
info.request->addRootPath(root_path);
|
||||
ResponsePtr response = info.request->process(container, zxid);
|
||||
if (response->error == Error::ZOK)
|
||||
info.request->processWatches(watches, list_watches);
|
||||
|
||||
response->removeRootPath(root_path);
|
||||
if (info.callback)
|
||||
info.callback(*response);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
finalize();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void TestKeeper::finalize()
|
||||
{
|
||||
{
|
||||
std::lock_guard lock(push_request_mutex);
|
||||
|
||||
if (expired)
|
||||
return;
|
||||
expired = true;
|
||||
}
|
||||
|
||||
processing_thread.join();
|
||||
|
||||
try
|
||||
{
|
||||
{
|
||||
for (auto & path_watch : watches)
|
||||
{
|
||||
WatchResponse response;
|
||||
response.type = SESSION;
|
||||
response.state = EXPIRED_SESSION;
|
||||
response.error = ZSESSIONEXPIRED;
|
||||
|
||||
for (auto & callback : path_watch.second)
|
||||
{
|
||||
if (callback)
|
||||
{
|
||||
try
|
||||
{
|
||||
callback(response);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
watches.clear();
|
||||
}
|
||||
|
||||
RequestInfo info;
|
||||
while (requests_queue.tryPop(info))
|
||||
{
|
||||
if (info.callback)
|
||||
{
|
||||
ResponsePtr response = info.request->createResponse();
|
||||
response->error = ZSESSIONEXPIRED;
|
||||
try
|
||||
{
|
||||
info.callback(*response);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
if (info.watch)
|
||||
{
|
||||
WatchResponse response;
|
||||
response.type = SESSION;
|
||||
response.state = EXPIRED_SESSION;
|
||||
response.error = ZSESSIONEXPIRED;
|
||||
try
|
||||
{
|
||||
info.watch(response);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
void TestKeeper::pushRequest(RequestInfo && info)
|
||||
{
|
||||
try
|
||||
{
|
||||
info.time = clock::now();
|
||||
|
||||
/// We must serialize 'pushRequest' and 'finalize' (from processingThread) calls
|
||||
/// to avoid forgotten operations in the queue when session is expired.
|
||||
/// Invariant: when expired, no new operations will be pushed to the queue in 'pushRequest'
|
||||
/// and the queue will be drained in 'finalize'.
|
||||
std::lock_guard lock(push_request_mutex);
|
||||
|
||||
if (expired)
|
||||
throw Exception("Session expired", ZSESSIONEXPIRED);
|
||||
|
||||
if (!requests_queue.tryPush(std::move(info), operation_timeout.totalMilliseconds()))
|
||||
throw Exception("Cannot push request to queue within operation timeout", ZOPERATIONTIMEOUT);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
finalize();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void TestKeeper::create(
|
||||
const String & path,
|
||||
const String & data,
|
||||
bool is_ephemeral,
|
||||
bool is_sequential,
|
||||
const ACLs &,
|
||||
CreateCallback callback)
|
||||
{
|
||||
TestKeeperCreateRequest request;
|
||||
request.path = path;
|
||||
request.data = data;
|
||||
request.is_ephemeral = is_ephemeral;
|
||||
request.is_sequential = is_sequential;
|
||||
|
||||
RequestInfo request_info;
|
||||
request_info.request = std::make_shared<TestKeeperCreateRequest>(std::move(request));
|
||||
request_info.callback = [callback](const Response & response) { callback(dynamic_cast<const CreateResponse &>(response)); };
|
||||
pushRequest(std::move(request_info));
|
||||
}
|
||||
|
||||
void TestKeeper::remove(
|
||||
const String & path,
|
||||
int32_t version,
|
||||
RemoveCallback callback)
|
||||
{
|
||||
TestKeeperRemoveRequest request;
|
||||
request.path = path;
|
||||
request.version = version;
|
||||
|
||||
RequestInfo request_info;
|
||||
request_info.request = std::make_shared<TestKeeperRemoveRequest>(std::move(request));
|
||||
request_info.callback = [callback](const Response & response) { callback(dynamic_cast<const RemoveResponse &>(response)); };
|
||||
pushRequest(std::move(request_info));
|
||||
}
|
||||
|
||||
void TestKeeper::exists(
|
||||
const String & path,
|
||||
ExistsCallback callback,
|
||||
WatchCallback watch)
|
||||
{
|
||||
TestKeeperExistsRequest request;
|
||||
request.path = path;
|
||||
|
||||
RequestInfo request_info;
|
||||
request_info.request = std::make_shared<TestKeeperExistsRequest>(std::move(request));
|
||||
request_info.callback = [callback](const Response & response) { callback(dynamic_cast<const ExistsResponse &>(response)); };
|
||||
request_info.watch = watch;
|
||||
pushRequest(std::move(request_info));
|
||||
}
|
||||
|
||||
void TestKeeper::get(
|
||||
const String & path,
|
||||
GetCallback callback,
|
||||
WatchCallback watch)
|
||||
{
|
||||
TestKeeperGetRequest request;
|
||||
request.path = path;
|
||||
|
||||
RequestInfo request_info;
|
||||
request_info.request = std::make_shared<TestKeeperGetRequest>(std::move(request));
|
||||
request_info.callback = [callback](const Response & response) { callback(dynamic_cast<const GetResponse &>(response)); };
|
||||
request_info.watch = watch;
|
||||
pushRequest(std::move(request_info));
|
||||
}
|
||||
|
||||
void TestKeeper::set(
|
||||
const String & path,
|
||||
const String & data,
|
||||
int32_t version,
|
||||
SetCallback callback)
|
||||
{
|
||||
TestKeeperSetRequest request;
|
||||
request.path = path;
|
||||
request.data = data;
|
||||
request.version = version;
|
||||
|
||||
RequestInfo request_info;
|
||||
request_info.request = std::make_shared<TestKeeperSetRequest>(std::move(request));
|
||||
request_info.callback = [callback](const Response & response) { callback(dynamic_cast<const SetResponse &>(response)); };
|
||||
pushRequest(std::move(request_info));
|
||||
}
|
||||
|
||||
void TestKeeper::list(
|
||||
const String & path,
|
||||
ListCallback callback,
|
||||
WatchCallback watch)
|
||||
{
|
||||
TestKeeperListRequest request;
|
||||
request.path = path;
|
||||
|
||||
RequestInfo request_info;
|
||||
request_info.request = std::make_shared<TestKeeperListRequest>(std::move(request));
|
||||
request_info.callback = [callback](const Response & response) { callback(dynamic_cast<const ListResponse &>(response)); };
|
||||
request_info.watch = watch;
|
||||
pushRequest(std::move(request_info));
|
||||
}
|
||||
|
||||
void TestKeeper::check(
|
||||
const String & path,
|
||||
int32_t version,
|
||||
CheckCallback callback)
|
||||
{
|
||||
TestKeeperCheckRequest request;
|
||||
request.path = path;
|
||||
request.version = version;
|
||||
|
||||
RequestInfo request_info;
|
||||
request_info.request = std::make_shared<TestKeeperCheckRequest>(std::move(request));
|
||||
request_info.callback = [callback](const Response & response) { callback(dynamic_cast<const CheckResponse &>(response)); };
|
||||
pushRequest(std::move(request_info));
|
||||
}
|
||||
|
||||
void TestKeeper::multi(
|
||||
const Requests & requests,
|
||||
MultiCallback callback)
|
||||
{
|
||||
TestKeeperMultiRequest request(requests);
|
||||
|
||||
RequestInfo request_info;
|
||||
request_info.request = std::make_shared<TestKeeperMultiRequest>(std::move(request));
|
||||
request_info.callback = [callback](const Response & response) { callback(dynamic_cast<const MultiResponse &>(response)); };
|
||||
pushRequest(std::move(request_info));
|
||||
}
|
||||
|
||||
}
|
||||
143
dbms/src/Common/ZooKeeper/TestKeeper.h
Normal file
143
dbms/src/Common/ZooKeeper/TestKeeper.h
Normal file
@ -0,0 +1,143 @@
|
||||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
#include <map>
|
||||
#include <atomic>
|
||||
#include <thread>
|
||||
#include <chrono>
|
||||
|
||||
#include <Poco/Timespan.h>
|
||||
#include <Common/ZooKeeper/IKeeper.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Common/ConcurrentBoundedQueue.h>
|
||||
|
||||
|
||||
namespace Coordination
|
||||
{
|
||||
|
||||
struct TestKeeperRequest;
|
||||
using TestKeeperRequestPtr = std::shared_ptr<TestKeeperRequest>;
|
||||
|
||||
|
||||
/** Looks like ZooKeeper but stores all data in memory of server process.
|
||||
* All data is not shared between different servers and is lost after server restart.
|
||||
*
|
||||
* The only purpose is to more simple testing for interaction with ZooKeeper within a single server.
|
||||
* This still makes sense, because multiple replicas of a single table can be created on a single server,
|
||||
* and it is used to test replication logic.
|
||||
*
|
||||
* Does not support ACLs. Does not support NULL node values.
|
||||
*
|
||||
* NOTE: You can add various failure modes for better testing.
|
||||
*/
|
||||
class TestKeeper : public IKeeper
|
||||
{
|
||||
public:
|
||||
TestKeeper(const String & root_path, Poco::Timespan operation_timeout);
|
||||
~TestKeeper() override;
|
||||
|
||||
bool isExpired() const override { return expired; }
|
||||
int64_t getSessionID() const override { return 0; }
|
||||
|
||||
|
||||
void create(
|
||||
const String & path,
|
||||
const String & data,
|
||||
bool is_ephemeral,
|
||||
bool is_sequential,
|
||||
const ACLs & acls,
|
||||
CreateCallback callback) override;
|
||||
|
||||
void remove(
|
||||
const String & path,
|
||||
int32_t version,
|
||||
RemoveCallback callback) override;
|
||||
|
||||
void exists(
|
||||
const String & path,
|
||||
ExistsCallback callback,
|
||||
WatchCallback watch) override;
|
||||
|
||||
void get(
|
||||
const String & path,
|
||||
GetCallback callback,
|
||||
WatchCallback watch) override;
|
||||
|
||||
void set(
|
||||
const String & path,
|
||||
const String & data,
|
||||
int32_t version,
|
||||
SetCallback callback) override;
|
||||
|
||||
void list(
|
||||
const String & path,
|
||||
ListCallback callback,
|
||||
WatchCallback watch) override;
|
||||
|
||||
void check(
|
||||
const String & path,
|
||||
int32_t version,
|
||||
CheckCallback callback) override;
|
||||
|
||||
void multi(
|
||||
const Requests & requests,
|
||||
MultiCallback callback) override;
|
||||
|
||||
|
||||
struct Node
|
||||
{
|
||||
String data;
|
||||
ACLs acls;
|
||||
bool is_ephemeral = false;
|
||||
bool is_sequental = false;
|
||||
Stat stat{};
|
||||
int32_t seq_num = 0;
|
||||
};
|
||||
|
||||
using Container = std::map<std::string, Node>;
|
||||
|
||||
using WatchCallbacks = std::vector<WatchCallback>;
|
||||
using Watches = std::map<String /* path, relative of root_path */, WatchCallbacks>;
|
||||
|
||||
private:
|
||||
using clock = std::chrono::steady_clock;
|
||||
|
||||
struct RequestInfo
|
||||
{
|
||||
TestKeeperRequestPtr request;
|
||||
ResponseCallback callback;
|
||||
WatchCallback watch;
|
||||
clock::time_point time;
|
||||
};
|
||||
|
||||
Container container;
|
||||
|
||||
String root_path;
|
||||
ACLs default_acls;
|
||||
|
||||
Poco::Timespan operation_timeout;
|
||||
|
||||
std::mutex push_request_mutex;
|
||||
std::atomic<bool> expired{false};
|
||||
|
||||
int64_t zxid = 0;
|
||||
|
||||
Watches watches;
|
||||
Watches list_watches; /// Watches for 'list' request (watches on children).
|
||||
|
||||
void createWatchCallBack(const String & path);
|
||||
|
||||
using RequestsQueue = ConcurrentBoundedQueue<RequestInfo>;
|
||||
RequestsQueue requests_queue{1};
|
||||
|
||||
void pushRequest(RequestInfo && request);
|
||||
|
||||
void finalize();
|
||||
|
||||
ThreadFromGlobalPool processing_thread;
|
||||
|
||||
void processingThread();
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#include "ZooKeeper.h"
|
||||
#include "ZooKeeperImpl.h"
|
||||
#include "KeeperException.h"
|
||||
#include "TestKeeper.h"
|
||||
|
||||
#include <random>
|
||||
#include <pcg_random.hpp>
|
||||
@ -24,6 +25,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
}
|
||||
|
||||
@ -44,7 +46,7 @@ static void check(int32_t code, const std::string & path)
|
||||
}
|
||||
|
||||
|
||||
void ZooKeeper::init(const std::string & hosts_, const std::string & identity_,
|
||||
void ZooKeeper::init(const std::string & implementation, const std::string & hosts_, const std::string & identity_,
|
||||
int32_t session_timeout_ms_, int32_t operation_timeout_ms_, const std::string & chroot_)
|
||||
{
|
||||
log = &Logger::get("ZooKeeper");
|
||||
@ -54,6 +56,8 @@ void ZooKeeper::init(const std::string & hosts_, const std::string & identity_,
|
||||
operation_timeout_ms = operation_timeout_ms_;
|
||||
chroot = chroot_;
|
||||
|
||||
if (implementation == "zookeeper")
|
||||
{
|
||||
if (hosts.empty())
|
||||
throw KeeperException("No addresses passed to ZooKeeper constructor.", Coordination::ZBADARGUMENTS);
|
||||
|
||||
@ -62,13 +66,13 @@ void ZooKeeper::init(const std::string & hosts_, const std::string & identity_,
|
||||
Coordination::ZooKeeper::Addresses addresses;
|
||||
addresses.reserve(addresses_strings.size());
|
||||
|
||||
for (const auto & address_string : addresses_strings)
|
||||
for (const auto &address_string : addresses_strings)
|
||||
{
|
||||
try
|
||||
{
|
||||
addresses.emplace_back(address_string);
|
||||
}
|
||||
catch (const Poco::Net::DNSException & e)
|
||||
catch (const Poco::Net::DNSException &e)
|
||||
{
|
||||
LOG_ERROR(log, "Cannot use ZooKeeper address " << address_string << ", reason: " << e.displayText());
|
||||
}
|
||||
@ -87,15 +91,26 @@ void ZooKeeper::init(const std::string & hosts_, const std::string & identity_,
|
||||
Poco::Timespan(0, operation_timeout_ms_ * 1000));
|
||||
|
||||
LOG_TRACE(log, "initialized, hosts: " << hosts << (chroot.empty() ? "" : ", chroot: " + chroot));
|
||||
}
|
||||
else if (implementation == "testkeeper")
|
||||
{
|
||||
impl = std::make_unique<Coordination::TestKeeper>(
|
||||
chroot,
|
||||
Poco::Timespan(0, operation_timeout_ms_ * 1000));
|
||||
}
|
||||
else
|
||||
{
|
||||
throw DB::Exception("Unknown implementation of coordination service: " + implementation, DB::ErrorCodes::NOT_IMPLEMENTED);
|
||||
}
|
||||
|
||||
if (!chroot.empty() && !exists("/"))
|
||||
throw KeeperException("Zookeeper root doesn't exist. You should create root node " + chroot + " before start.", Coordination::ZNONODE);
|
||||
}
|
||||
|
||||
ZooKeeper::ZooKeeper(const std::string & hosts, const std::string & identity,
|
||||
int32_t session_timeout_ms, int32_t operation_timeout_ms, const std::string & chroot)
|
||||
ZooKeeper::ZooKeeper(const std::string & hosts, const std::string & identity, int32_t session_timeout_ms,
|
||||
int32_t operation_timeout_ms, const std::string & chroot, const std::string & implementation)
|
||||
{
|
||||
init(hosts, identity, session_timeout_ms, operation_timeout_ms, chroot);
|
||||
init(implementation, hosts, identity, session_timeout_ms, operation_timeout_ms, chroot);
|
||||
}
|
||||
|
||||
struct ZooKeeperArgs
|
||||
@ -109,6 +124,7 @@ struct ZooKeeperArgs
|
||||
|
||||
session_timeout_ms = DEFAULT_SESSION_TIMEOUT;
|
||||
operation_timeout_ms = DEFAULT_OPERATION_TIMEOUT;
|
||||
implementation = "zookeeper";
|
||||
for (const auto & key : keys)
|
||||
{
|
||||
if (startsWith(key, "node"))
|
||||
@ -134,6 +150,10 @@ struct ZooKeeperArgs
|
||||
{
|
||||
chroot = config.getString(config_name + "." + key);
|
||||
}
|
||||
else if (key == "implementation")
|
||||
{
|
||||
implementation = config.getString(config_name + "." + key);
|
||||
}
|
||||
else
|
||||
throw KeeperException(std::string("Unknown key ") + key + " in config file", Coordination::ZBADARGUMENTS);
|
||||
}
|
||||
@ -163,12 +183,13 @@ struct ZooKeeperArgs
|
||||
int session_timeout_ms;
|
||||
int operation_timeout_ms;
|
||||
std::string chroot;
|
||||
std::string implementation;
|
||||
};
|
||||
|
||||
ZooKeeper::ZooKeeper(const Poco::Util::AbstractConfiguration & config, const std::string & config_name)
|
||||
{
|
||||
ZooKeeperArgs args(config, config_name);
|
||||
init(args.hosts, args.identity, args.session_timeout_ms, args.operation_timeout_ms, args.chroot);
|
||||
init(args.implementation, args.hosts, args.identity, args.session_timeout_ms, args.operation_timeout_ms, args.chroot);
|
||||
}
|
||||
|
||||
|
||||
@ -931,5 +952,4 @@ Coordination::RequestPtr makeCheckRequest(const std::string & path, int version)
|
||||
request->version = version;
|
||||
return request;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -55,7 +55,8 @@ public:
|
||||
ZooKeeper(const std::string & hosts, const std::string & identity = "",
|
||||
int32_t session_timeout_ms = DEFAULT_SESSION_TIMEOUT,
|
||||
int32_t operation_timeout_ms = DEFAULT_OPERATION_TIMEOUT,
|
||||
const std::string & chroot = "");
|
||||
const std::string & chroot = "",
|
||||
const std::string & implementation = "zookeeper");
|
||||
|
||||
/** Config of the form:
|
||||
<zookeeper>
|
||||
@ -235,7 +236,7 @@ public:
|
||||
private:
|
||||
friend class EphemeralNodeHolder;
|
||||
|
||||
void init(const std::string & hosts_, const std::string & identity_,
|
||||
void init(const std::string & implementation_, const std::string & hosts_, const std::string & identity_,
|
||||
int32_t session_timeout_ms_, int32_t operation_timeout_ms_, const std::string & chroot_);
|
||||
|
||||
void removeChildrenRecursive(const std::string & path);
|
||||
@ -320,5 +321,4 @@ private:
|
||||
};
|
||||
|
||||
using EphemeralNodeHolderPtr = EphemeralNodeHolder::Ptr;
|
||||
|
||||
}
|
||||
|
||||
@ -1,16 +1,5 @@
|
||||
<yandex>
|
||||
<zookeeper>
|
||||
<node>
|
||||
<host>localhost</host>
|
||||
<port>2181</port>
|
||||
</node>
|
||||
<node>
|
||||
<host>yandex.ru</host>
|
||||
<port>2181</port>
|
||||
</node>
|
||||
<node>
|
||||
<host>111.0.1.2</host>
|
||||
<port>2181</port>
|
||||
</node>
|
||||
<implementation>testkeeper</implementation>
|
||||
</zookeeper>
|
||||
</yandex>
|
||||
|
||||
@ -79,6 +79,7 @@ def test_mysql_client(mysql_client, server_address):
|
||||
-e "INSERT INTO table1 VALUES (0), (1), (5);"
|
||||
-e "INSERT INTO table1 VALUES (0), (1), (5);"
|
||||
-e "SELECT * FROM table1 ORDER BY a;"
|
||||
-e "DROP DATABASE x;"
|
||||
'''.format(host=server_address, port=server_port), demux=True)
|
||||
|
||||
assert stdout == 'a\n0\n0\n1\n1\n5\n5\n'
|
||||
@ -108,9 +109,10 @@ def test_python_client(server_address):
|
||||
|
||||
assert exc_info.value.args == (81, "Database system2 doesn't exist")
|
||||
|
||||
client.select_db('x')
|
||||
cursor = client.cursor(pymysql.cursors.DictCursor)
|
||||
cursor.execute("TRUNCATE TABLE table1")
|
||||
cursor.execute('CREATE DATABASE x')
|
||||
client.select_db('x')
|
||||
cursor.execute("CREATE TABLE table1 (a UInt32) ENGINE = Memory")
|
||||
cursor.execute("INSERT INTO table1 VALUES (1), (3)")
|
||||
cursor.execute("INSERT INTO table1 VALUES (1), (4)")
|
||||
cursor.execute("SELECT * FROM table1 ORDER BY a")
|
||||
|
||||
@ -1,9 +1,5 @@
|
||||
<test>
|
||||
<name>Benchmark</name>
|
||||
|
||||
<tags>
|
||||
<tag>columns_hashing</tag>
|
||||
</tags>
|
||||
<name>columns_hashing</name>
|
||||
|
||||
<preconditions>
|
||||
<table_exists>hits_100m_single</table_exists>
|
||||
@ -25,12 +21,12 @@
|
||||
|
||||
<!--
|
||||
<query><![CDATA[select count() from hits_100m_single any left join hits_100m_single using (UserID, RegionID)]]></query>
|
||||
<query><![CDATA[select count() from hits_100m_single any left join hits_100m_single using (UserID)]]></query>
|
||||
<query><![CDATA[select count() from hits_100m_single any left join hits_100m_single using UserID]]></query>
|
||||
<query><![CDATA[select count() from hits_100m_single any left join hits_100m_single using URL where URL != '']]></query>
|
||||
<query><![CDATA[select count() from hits_1000m_single any left join hits_1000m_single using MobilePhoneModel where MobilePhoneModel != '']]></query>
|
||||
<query><![CDATA[select count() from hits_100m_single any left join hits_100m_single using (MobilePhoneModel, UserID) where MobilePhoneModel != '']]></query>
|
||||
|
||||
<query><![CDATA[select count() from (select count() from hits_1000m_single group by (UserID))]]></query>
|
||||
<query><![CDATA[select count() from (select count() from hits_1000m_single group by UserID)]]></query>
|
||||
<query><![CDATA[select count() from (select count() from hits_100m_single group by (UserID, RegionID))]]></query>
|
||||
<query><![CDATA[select count() from (select count() from hits_100m_single where URL != '' group by URL)]]></query>
|
||||
<query><![CDATA[select count() from (select count() from hits_1000m_single where MobilePhoneModel != '' group by MobilePhoneModel)]]></query>
|
||||
|
||||
@ -11,6 +11,10 @@
|
||||
|
||||
<type>loop</type>
|
||||
|
||||
<settings>
|
||||
<max_memory_usage>20000000000</max_memory_usage>
|
||||
</settings>
|
||||
|
||||
<stop_conditions>
|
||||
<all_of>
|
||||
<iterations>5</iterations>
|
||||
@ -21,6 +25,7 @@
|
||||
<total_time_ms>60000</total_time_ms>
|
||||
</any_of>
|
||||
</stop_conditions>
|
||||
|
||||
<query>SELECT DISTINCT URL,Title, ngramDistance(Title, URL) AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50</query>
|
||||
<query>SELECT DISTINCT SearchPhrase,Title, ngramDistance(Title, SearchPhrase) AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50</query>
|
||||
<query>SELECT DISTINCT Title, ngramDistance(Title, 'what is love') AS distance FROM hits_100m_single ORDER BY distance ASC LIMIT 50</query>
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
<name>number_formatting_formats</name>
|
||||
<type>loop</type>
|
||||
|
||||
<create_query>CREATE TABLE IF NOT EXISTS table_{format} (x UInt64) ENGINE = File({format})</create_query>
|
||||
<create_query>CREATE TABLE IF NOT EXISTS table_{format} (x UInt64) ENGINE = File(`{format}`)</create_query>
|
||||
|
||||
<stop_conditions>
|
||||
<all_of>
|
||||
|
||||
@ -205,7 +205,21 @@ For information about other parameters, see the section "SET".
|
||||
|
||||
Similarly, you can use ClickHouse sessions in the HTTP protocol. To do this, you need to add the `session_id` GET parameter to the request. You can use any string as the session ID. By default, the session is terminated after 60 seconds of inactivity. To change this timeout, modify the `default_session_timeout` setting in the server configuration, or add the `session_timeout` GET parameter to the request. To check the session status, use the `session_check=1` parameter. Only one query at a time can be executed within a single session.
|
||||
|
||||
You have the option to receive information about the progress of query execution in X-ClickHouse-Progress headers. To do this, enable the setting send_progress_in_http_headers.
|
||||
You have the option to receive information about the progress of query execution in `X-ClickHouse-Progress` response headers. To do this, enable the setting [send_progress_in_http_headers](../operations/settings/settings.md#settings-send_progress_in_http_headers). Example of the header sequence:
|
||||
|
||||
```
|
||||
X-ClickHouse-Progress: {"read_rows":"2752512","read_bytes":"240570816","total_rows_to_read":"8880128"}
|
||||
X-ClickHouse-Progress: {"read_rows":"5439488","read_bytes":"482285394","total_rows_to_read":"8880128"}
|
||||
X-ClickHouse-Progress: {"read_rows":"8783786","read_bytes":"819092887","total_rows_to_read":"8880128"}
|
||||
```
|
||||
|
||||
Possible fields in the header:
|
||||
|
||||
- `read_rows` — Number of read rows.
|
||||
- `read_bytes` — Volume of read data in bytes.
|
||||
- `total_rows_to_read` — Total number of rows to be read.
|
||||
- `written_rows` — Number of written rows.
|
||||
- `written_bytes` — Volume of written data in bytes.
|
||||
|
||||
Running requests don't stop automatically if the HTTP connection is lost. Parsing and data formatting are performed on the server side, and using the network might be ineffective.
|
||||
The optional 'query_id' parameter can be passed as the query ID (any string). For more information, see the section "Settings, replace_running_query".
|
||||
|
||||
@ -64,7 +64,7 @@ Maximum number of bytes (uncompressed data) that can be read from a table when r
|
||||
|
||||
What to do when the volume of data read exceeds one of the limits: 'throw' or 'break'. By default, throw.
|
||||
|
||||
## max_rows_to_group_by
|
||||
## max_rows_to_group_by {#settings-max_rows_to_group_by}
|
||||
|
||||
Maximum number of unique keys received from aggregation. This setting lets you limit memory consumption when aggregating.
|
||||
|
||||
@ -73,6 +73,17 @@ Maximum number of unique keys received from aggregation. This setting lets you l
|
||||
What to do when the number of unique keys for aggregation exceeds the limit: 'throw', 'break', or 'any'. By default, throw.
|
||||
Using the 'any' value lets you run an approximation of GROUP BY. The quality of this approximation depends on the statistical nature of the data.
|
||||
|
||||
## max_bytes_before_external_group_by {#settings-max_bytes_before_external_group_by}
|
||||
|
||||
Enables or disables execution of `GROUP BY` clause in external memory. See [GROUP BY in external memory](../../query_language/select.md#select-group-by-in-external-memory).
|
||||
|
||||
Possible values:
|
||||
|
||||
- Maximum volume or RAM (in bytes) that can be used by [GROUP BY](../../query_language/select.md#select-group-by-clause) operation.
|
||||
- 0 — `GROUP BY` in external memory disabled.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## max_rows_to_sort
|
||||
|
||||
Maximum number of rows before sorting. This allows you to limit memory consumption when sorting.
|
||||
@ -194,12 +205,64 @@ Maximum number of bytes (uncompressed data) that can be passed to a remote serve
|
||||
|
||||
What to do when the amount of data exceeds one of the limits: 'throw' or 'break'. By default, throw.
|
||||
|
||||
## max_rows_in_join {#settings-max_rows_in_join}
|
||||
|
||||
Limits the number of rows in the hash table that is used when joining tables.
|
||||
|
||||
This settings applies to [SELECT ... JOIN](../../query_language/select.md#select-join) operations and the [Join table engine](../table_engines/join.md).
|
||||
|
||||
If a query contains multiple joins, ClickHouse checks this setting for every intermediate result.
|
||||
|
||||
ClickHouse can proceed with different actions when the limit is reached. Use the [join_overflow_mode](#settings-join_overflow_mode) settings to choose the action.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Unlimited number of rows.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## max_bytes_in_join {#settings-max_bytes_in_join}
|
||||
|
||||
Limits hash table size in bytes that is used when joining tables.
|
||||
|
||||
This settings applies to [SELECT ... JOIN](../../query_language/select.md#select-join) operations and the [Join table engine](../table_engines/join.md).
|
||||
|
||||
If query contains some joins, ClickHouse checks this setting for every intermediate result.
|
||||
|
||||
ClickHouse can proceed with different actions when the limit is reached. Use the [join_overflow_mode](#settings-join_overflow_mode) settings to choose the action.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Memory control is disabled.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## join_overflow_mode {#settings-join_overflow_mode}
|
||||
|
||||
Defines an action that ClickHouse performs, when any of the following join limits is reached:
|
||||
|
||||
- [max_bytes_in_join](#settings-max_bytes_in_join)
|
||||
- [max_rows_in_join](#settings-max_rows_in_join)
|
||||
|
||||
Possible values:
|
||||
|
||||
- `THROW` — ClickHouse throws an exception and breaks operation.
|
||||
- `BREAK` — ClickHouse breaks operation and doesn't throw an exception.
|
||||
|
||||
Default value: `THROW`.
|
||||
|
||||
**See Also**
|
||||
|
||||
- [JOIN clause](../../query_language/select.md#select-join)
|
||||
- [Join table engine](../table_engines/join.md)
|
||||
|
||||
|
||||
## max_partitions_per_insert_block
|
||||
|
||||
Limits the maximum number of partitions in a single inserted block.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Unlimited number of partitions.
|
||||
|
||||
@ -211,6 +274,4 @@ When inserting data, ClickHouse calculates the number of partitions in the inser
|
||||
|
||||
> "Too many partitions for single INSERT block (more than " + toString(max_parts) + "). The limit is controlled by 'max_partitions_per_insert_block' setting. Large number of partitions is a common misconception. It will lead to severe negative performance impact, including slow server startup, slow INSERT queries and slow SELECT queries. Recommended total number of partitions for a table is under 1000..10000. Please note, that partitioning is not intended to speed up SELECT queries (ORDER BY key is sufficient to make range queries fast). Partitions are intended for data manipulation (DROP PARTITION, etc)."
|
||||
|
||||
|
||||
|
||||
[Original article](https://clickhouse.yandex/docs/en/operations/settings/query_complexity/) <!--hide-->
|
||||
|
||||
@ -114,6 +114,19 @@ Possible values:
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## send_progress_in_http_headers {#settings-send_progress_in_http_headers}
|
||||
|
||||
Enables or disables sending of the `X-ClickHouse-Progress` HTTP response headers in an answer from `clickhouse-server`.
|
||||
|
||||
For more information, read the [HTTP interface description](../../interfaces/http.md).
|
||||
|
||||
Possible values:
|
||||
|
||||
- 0 — Disabled.
|
||||
- 1 — Enabled.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## input_format_allow_errors_num
|
||||
|
||||
Sets the maximum number of acceptable errors when reading from text formats (CSV, TSV, etc.).
|
||||
@ -258,6 +271,25 @@ Sets default strictness for [JOIN clauses](../../query_language/select.md#select
|
||||
|
||||
**Default value**: `ALL`
|
||||
|
||||
## join_any_take_last_row {#settings-join_any_take_last_row}
|
||||
|
||||
Changes behavior of join operations with `ANY` strictness.
|
||||
|
||||
!!! note "Attention"
|
||||
This setting applies only for the [Join](../table_engines/join.md) table engine.
|
||||
|
||||
Possible values:
|
||||
|
||||
- 0 — If the right table has more than one matching row, only the first one found is joined.
|
||||
- 1 — If the right table has more than one matching row, only the last one found is joined.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
**See Also**
|
||||
|
||||
- [JOIN clause](../../query_language/select.md#select-join)
|
||||
- [Join table engine](../table_engines/join.md)
|
||||
- [join_default_strictness](#settings-join_default_strictness)
|
||||
|
||||
## join_use_nulls {#settings-join_use_nulls}
|
||||
|
||||
@ -674,6 +706,49 @@ When sequential consistency is enabled, ClickHouse allows the client to execute
|
||||
- [insert_quorum](#settings-insert_quorum)
|
||||
- [insert_quorum_timeout](#settings-insert_quorum_timeout)
|
||||
|
||||
## max_network_bytes {#settings-max_network_bytes}
|
||||
Limits the data volume (in bytes) that is received or transmitted over the network when executing a query. This setting applies for every individual query.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Data volume control is disabled.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## max_network_bandwidth {#settings-max_network_bandwidth}
|
||||
|
||||
Limits speed of data exchange over the network in bytes per second. This setting applies for every individual query.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Bandwidth control is disabled.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## max_network_bandwidth_for_user {#settings-max_network_bandwidth_for_user}
|
||||
|
||||
Limits speed of data exchange over the network in bytes per second. This setting applies for all concurrently running queries performed by a single user.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Control of the data speed is disabled.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## max_network_bandwidth_for_all_users {#settings-max_network_bandwidth_for_all_users}
|
||||
|
||||
Limits speed of data exchange over the network in bytes per second. This setting applies for all concurrently running queries on the server.
|
||||
|
||||
Possible values:
|
||||
|
||||
- Positive integer.
|
||||
- 0 — Control of the data speed is disabled.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## allow_experimental_cross_to_join_conversion {#settings-allow_experimental_cross_to_join_conversion}
|
||||
|
||||
Enables or disables:
|
||||
|
||||
@ -8,9 +8,39 @@ They are located in the 'system' database.
|
||||
|
||||
## system.asynchronous_metrics {#system_tables-asynchronous_metrics}
|
||||
|
||||
Contain metrics used for profiling and monitoring.
|
||||
They usually reflect the number of events currently in the system, or the total resources consumed by the system.
|
||||
Example: The number of SELECT queries currently running; the amount of memory in use.`system.asynchronous_metrics`and`system.metrics` differ in their sets of metrics and how they are calculated.
|
||||
Contains metrics which are calculated periodically in background. For example, the amount of RAM in use.
|
||||
|
||||
Columns:
|
||||
|
||||
- `metric` ([String](../data_types/string.md)) — Metric's name.
|
||||
- `value` ([Float64](../data_types/float.md)) — Metric's value.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT * FROM system.asynchronous_metrics LIMIT 10
|
||||
```
|
||||
|
||||
```text
|
||||
┌─metric──────────────────────────────────┬──────value─┐
|
||||
│ jemalloc.background_thread.run_interval │ 0 │
|
||||
│ jemalloc.background_thread.num_runs │ 0 │
|
||||
│ jemalloc.background_thread.num_threads │ 0 │
|
||||
│ jemalloc.retained │ 422551552 │
|
||||
│ jemalloc.mapped │ 1682989056 │
|
||||
│ jemalloc.resident │ 1656446976 │
|
||||
│ jemalloc.metadata_thp │ 0 │
|
||||
│ jemalloc.metadata │ 10226856 │
|
||||
│ UncompressedCacheCells │ 0 │
|
||||
│ MarkCacheFiles │ 0 │
|
||||
└─────────────────────────────────────────┴────────────┘
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
- [Monitoring](monitoring.md) — Base concepts of ClickHouse monitoring.
|
||||
- [system.metrics](#system_tables-metrics) — Contains instantly calculated metrics.
|
||||
- [system.events](#system_tables-events) — Contains a number of happened events.
|
||||
|
||||
## system.clusters
|
||||
|
||||
@ -89,9 +119,35 @@ Note that the amount of memory used by the dictionary is not proportional to the
|
||||
|
||||
## system.events {#system_tables-events}
|
||||
|
||||
Contains information about the number of events that have occurred in the system. This is used for profiling and monitoring purposes.
|
||||
Example: The number of processed SELECT queries.
|
||||
Columns: 'event String' – the event name, and 'value UInt64' – the quantity.
|
||||
Contains information about the number of events that have occurred in the system. For example, in the table, you can find how many `SELECT` queries are processed from the moment of ClickHouse server start.
|
||||
|
||||
Columns:
|
||||
|
||||
- `event` ([String](../data_types/string.md)) — Event name.
|
||||
- `value` ([UInt64](../data_types/int_uint.md)) — Count of events occurred.
|
||||
- `description` ([String](../data_types/string.md)) — Description of an event.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT * FROM system.events LIMIT 5
|
||||
```
|
||||
|
||||
```text
|
||||
┌─event─────────────────────────────────┬─value─┬─description────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ Query │ 12 │ Number of queries started to be interpreted and maybe executed. Does not include queries that are failed to parse, that are rejected due to AST size limits; rejected due to quota limits or limits on number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries. │
|
||||
│ SelectQuery │ 8 │ Same as Query, but only for SELECT queries. │
|
||||
│ FileOpen │ 73 │ Number of files opened. │
|
||||
│ ReadBufferFromFileDescriptorRead │ 155 │ Number of reads (read/pread) from a file descriptor. Does not include sockets. │
|
||||
│ ReadBufferFromFileDescriptorReadBytes │ 9931 │ Number of bytes read from file descriptors. If the file is compressed, this will show compressed data size. │
|
||||
└───────────────────────────────────────┴───────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
- [system.asynchronous_metrics](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics.
|
||||
- [system.metrics](#system_tables-metrics) — Contains instantly calculated metrics.
|
||||
- [Monitoring](monitoring.md) — Base concepts of ClickHouse monitoring.
|
||||
|
||||
## system.functions
|
||||
|
||||
@ -133,7 +189,7 @@ Columns:
|
||||
- `result_part_name String` — The name of the part that will be formed as the result of merging.
|
||||
- `is_mutation UInt8` - 1 if this process is a part mutation.
|
||||
- `total_size_bytes_compressed UInt64` — The total size of the compressed data in the merged chunks.
|
||||
- `total_size_marks UInt64` — The total number of marks in the merged partss.
|
||||
- `total_size_marks UInt64` — The total number of marks in the merged parts.
|
||||
- `bytes_read_uncompressed UInt64` — Number of bytes read, uncompressed.
|
||||
- `rows_read UInt64` — Number of rows read.
|
||||
- `bytes_written_uncompressed UInt64` — Number of bytes written, uncompressed.
|
||||
@ -141,6 +197,40 @@ Columns:
|
||||
|
||||
## system.metrics {#system_tables-metrics}
|
||||
|
||||
Contains metrics which can be calculated instantly, or have an current value. For example, a number of simultaneously processed queries, the current value for replica delay. This table is always up to date.
|
||||
|
||||
Columns:
|
||||
|
||||
- `metric` ([String](../data_types/string.md)) — Metric's name.
|
||||
- `value` ([Int64](../data_types/int_uint.md)) — Metric's value.
|
||||
- `description` ([String](../data_types/string.md)) — Description of the metric.
|
||||
|
||||
**Example**
|
||||
|
||||
```sql
|
||||
SELECT * FROM system.metrics LIMIT 10
|
||||
```
|
||||
|
||||
```text
|
||||
┌─metric─────────────────────┬─value─┬─description──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ Query │ 1 │ Number of executing queries │
|
||||
│ Merge │ 0 │ Number of executing background merges │
|
||||
│ PartMutation │ 0 │ Number of mutations (ALTER DELETE/UPDATE) │
|
||||
│ ReplicatedFetch │ 0 │ Number of data parts fetching from replica │
|
||||
│ ReplicatedSend │ 0 │ Number of data parts sending to replicas │
|
||||
│ ReplicatedChecks │ 0 │ Number of data parts checking for consistency │
|
||||
│ BackgroundPoolTask │ 0 │ Number of active tasks in BackgroundProcessingPool (merges, mutations, fetches or replication queue bookkeeping) │
|
||||
│ BackgroundSchedulePoolTask │ 0 │ Number of active tasks in BackgroundSchedulePool. This pool is used for periodic tasks of ReplicatedMergeTree like cleaning old data parts, altering data parts, replica re-initialization, etc. │
|
||||
│ DiskSpaceReservedForMerge │ 0 │ Disk space reserved for currently running background merges. It is slightly more than total size of currently merging parts. │
|
||||
│ DistributedSend │ 0 │ Number of connections sending data, that was INSERTed to Distributed tables, to remote servers. Both synchronous and asynchronous mode. │
|
||||
└────────────────────────────┴───────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
**See Also**
|
||||
|
||||
- [system.asynchronous_metrics](#system_tables-asynchronous_metrics) — Contains periodically calculated metrics.
|
||||
- [system.events](#system_tables-events) — Contains a umber of happened events.
|
||||
- [Monitoring](monitoring.md) — Base concepts of ClickHouse monitoring.
|
||||
|
||||
## system.numbers
|
||||
|
||||
This table contains a single UInt64 column named 'number' that contains almost all the natural numbers starting from zero.
|
||||
|
||||
@ -26,12 +26,12 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
|
||||
For a description of request parameters, see [request description](../../query_language/create.md).
|
||||
|
||||
A table for the Graphite date should have the following columns:
|
||||
A table for the Graphite data should have the following columns for the following data:
|
||||
|
||||
- Column with the metric name (Graphite sensor). Data type: `String`.
|
||||
- Column with the time of measuring the metric. Data type: `DateTime`.
|
||||
- Column with the value of the metric. Data type: any numeric.
|
||||
- Column with the version of the metric. Data type: any numeric.
|
||||
- Metric name (Graphite sensor). Data type: `String`.
|
||||
- Time of measuring the metric. Data type: `DateTime`.
|
||||
- Value of the metric. Data type: any numeric.
|
||||
- Version of the metric. Data type: any numeric.
|
||||
|
||||
ClickHouse saves the rows with the highest version or the last written if versions are the same. Other rows are deleted during the merge of data parts.
|
||||
|
||||
@ -75,6 +75,23 @@ Rollup configuration structure:
|
||||
|
||||
```
|
||||
required-columns
|
||||
patterns
|
||||
```
|
||||
|
||||
### Required Columns
|
||||
|
||||
Set the following parameters only if the name of columns differs from their default values:
|
||||
|
||||
- `path_column_name` — The name of the column storing the metric name (Graphite sensor). Default value: `Path`.
|
||||
- `time_column_name` — The name of the column storing the time of measuring the metric. Default value: `Time`.
|
||||
- `value_column_name` — The name of the column storing the value of the metric at the time set in `time_column_name`. Default value: `Value`.
|
||||
- `version_column_name` — The name of the column storing the version of the metric. Default value: `Timestamp`.
|
||||
|
||||
### Patterns
|
||||
|
||||
Structure of the `patterns` section:
|
||||
|
||||
```
|
||||
pattern
|
||||
regexp
|
||||
function
|
||||
@ -95,14 +112,14 @@ default
|
||||
...
|
||||
```
|
||||
|
||||
**Important:** The order of patterns should be next:
|
||||
!!! warning "Attention"
|
||||
Patterns must be strictly ordered:
|
||||
|
||||
1. Patterns *without* `function` *or* `retention`.
|
||||
1. Patterns *with* both `function` *and* `retention`.
|
||||
1. Pattern `dafault`.
|
||||
1. Patterns without `function` or `retention`.
|
||||
1. Patterns with both `function` and `retention`.
|
||||
1. Pattern `default`.
|
||||
|
||||
|
||||
When processing a row, ClickHouse checks the rules in the `pattern` sections. Each of `pattern` (including `default`) sections could contain `function` parameter for aggregation, `retention` parameters or both. If the metric name matches the `regexp`, the rules from the `pattern` section (or sections) are applied; otherwise, the rules from the `default` section are used.
|
||||
When processing a row, ClickHouse checks the rules in the `pattern` sections. Each of `pattern` (including `default`) sections can contain `function` parameter for aggregation, `retention` parameters or both. If the metric name matches the `regexp`, the rules from the `pattern` section (or sections) are applied; otherwise, the rules from the `default` section are used.
|
||||
|
||||
Fields for `pattern` and `default` sections:
|
||||
|
||||
@ -111,20 +128,11 @@ Fields for `pattern` and `default` sections:
|
||||
- `precision`– How precisely to define the age of the data in seconds. Should be a divisor for 86400 (seconds in a day).
|
||||
- `function` – The name of the aggregating function to apply to data whose age falls within the range `[age, age + precision]`.
|
||||
|
||||
The `required-columns`:
|
||||
|
||||
- `path_column_name` — Column with the metric name (Graphite sensor).
|
||||
- `time_column_name` — Column with the time of measuring the metric.
|
||||
- `value_column_name` — Column with the value of the metric at the time set in `time_column_name`.
|
||||
- `version_column_name` — Column with the version of the metric.
|
||||
|
||||
Example of settings:
|
||||
### Configuration Example
|
||||
|
||||
```xml
|
||||
<graphite_rollup>
|
||||
<path_column_name>Path</path_column_name>
|
||||
<time_column_name>Time</time_column_name>
|
||||
<value_column_name>Value</value_column_name>
|
||||
<version_column_name>Version</version_column_name>
|
||||
<pattern>
|
||||
<regexp>click_cost</regexp>
|
||||
|
||||
@ -16,12 +16,12 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
) ENGINE = MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']);
|
||||
```
|
||||
|
||||
See the detailed description of the [CREATE TABLE](../../query_language/create.md#create-table-query) query.
|
||||
See a detailed description of the [CREATE TABLE](../../query_language/create.md#create-table-query) query.
|
||||
|
||||
The table structure can be not the same as the original MySQL table structure:
|
||||
The table structure can differ from the original MySQL table structure:
|
||||
|
||||
- Names of columns should be the same as in the original MySQL table, but you can use just some of these columns in any order.
|
||||
- Types of columns may differ from the types in the original MySQL table. ClickHouse tries to [cast](../../query_language/functions/type_conversion_functions.md#type_conversion_function-cast) values into the ClickHouse data types.
|
||||
- Column names should be the same as in the original MySQL table, but you can use just some of these columns and in any order.
|
||||
- Column types may differ from those in the original MySQL table. ClickHouse tries to [cast](../../query_language/functions/type_conversion_functions.md#type_conversion_function-cast) values to the ClickHouse data types.
|
||||
|
||||
**Engine Parameters**
|
||||
|
||||
@ -30,14 +30,14 @@ The table structure can be not the same as the original MySQL table structure:
|
||||
- `table` — Remote table name.
|
||||
- `user` — MySQL user.
|
||||
- `password` — User password.
|
||||
- `replace_query` — Flag that sets query substitution `INSERT INTO` to `REPLACE INTO`. If `replace_query=1`, the query is replaced.
|
||||
- `replace_query` — Flag that converts `INSERT INTO` queries to `REPLACE INTO`. If `replace_query=1`, the query is substituted.
|
||||
- `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` expression that is added to the `INSERT` query.
|
||||
|
||||
Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, where `on_duplicate_clause` is `UPDATE c2 = c2 + 1`. See MySQL documentation to find which `on_duplicate_clause` you can use with `ON DUPLICATE KEY` clause.
|
||||
Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, where `on_duplicate_clause` is `UPDATE c2 = c2 + 1`. See the [MySQL documentation](https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html) to find which `on_duplicate_clause` you can use with the `ON DUPLICATE KEY` clause.
|
||||
|
||||
To specify `on_duplicate_clause` you need to pass `0` to the `replace_query` parameter. If you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception.
|
||||
|
||||
At this time, simple `WHERE` clauses such as ` =, !=, >, >=, <, <=` are executed on the MySQL server.
|
||||
Simple `WHERE` clauses such as ` =, !=, >, >=, <, <=` are executed on the MySQL server.
|
||||
|
||||
The rest of the conditions and the `LIMIT` sampling constraint are executed in ClickHouse only after the query to MySQL finishes.
|
||||
|
||||
@ -66,7 +66,7 @@ mysql> select * from test;
|
||||
1 row in set (0,00 sec)
|
||||
```
|
||||
|
||||
Table in ClickHouse, getting data from the MySQL table:
|
||||
Table in ClickHouse, retrieving data from the MySQL table:
|
||||
|
||||
```sql
|
||||
CREATE TABLE mysql_table
|
||||
@ -88,6 +88,6 @@ SELECT * FROM mysql_table6
|
||||
## See Also
|
||||
|
||||
- [The 'mysql' table function](../../query_language/table_functions/mysql.md)
|
||||
- [Using MySQL as a source of extenal dictionary](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql)
|
||||
- [Using MySQL as a source of external dictionary](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql)
|
||||
|
||||
[Original article](https://clickhouse.yandex/docs/en/operations/table_engines/mysql/) <!--hide-->
|
||||
|
||||
@ -237,7 +237,7 @@ binary decimal
|
||||
```
|
||||
|
||||
|
||||
##groupBitmap
|
||||
## groupBitmap
|
||||
|
||||
Bitmap or Aggregate calculations from a unsigned integer column, return cardinality of type UInt64, if add suffix -State, then return [bitmap object](../functions/bitmap_functions.md).
|
||||
|
||||
@ -677,7 +677,7 @@ Optional parameters:
|
||||
Creates an array from different argument values. Memory consumption is the same as for the `uniqExact` function.
|
||||
|
||||
The second version (with the `max_size` parameter) limits the size of the resulting array to `max_size` elements.
|
||||
For example, `groupUniqArray (1) (x)` is equivalent to `[any (x)]`.
|
||||
For example, `groupUniqArray(1)(x)` is equivalent to `[any(x)]`.
|
||||
|
||||
## quantile(level)(x)
|
||||
|
||||
|
||||
@ -23,8 +23,13 @@ bitmapBuild(array)
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res
|
||||
```sql
|
||||
SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res)
|
||||
```
|
||||
```text
|
||||
┌─res─┬─toTypeName(bitmapBuild([1, 2, 3, 4, 5]))─────┐
|
||||
│ │ AggregateFunction(groupBitmap, UInt8) │
|
||||
└─────┴──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## bitmapToArray
|
||||
@ -53,16 +58,20 @@ SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5])) AS res
|
||||
|
||||
## bitmapHasAny
|
||||
|
||||
Analogous to `hasAny(array, array)` returns 1 if bitmaps have any common elements, 0 otherwise.
|
||||
For empty bitmaps returns 0.
|
||||
Checks whether two bitmaps have intersection by some elements.
|
||||
|
||||
```
|
||||
bitmapHasAny(bitmap,bitmap)
|
||||
bitmapHasAny(bitmap1, bitmap2)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `bitmap` – bitmap object.
|
||||
- `bitmap*` – bitmap object.
|
||||
|
||||
**Return values**
|
||||
|
||||
- `1`, if `bitmap1` and `bitmap2` have one similar element at least.
|
||||
- `0`, otherwise.
|
||||
|
||||
**Example**
|
||||
|
||||
|
||||
@ -115,7 +115,7 @@ HEX can be uppercase or lowercase.
|
||||
|
||||
## IPv4ToIPv6(x)
|
||||
|
||||
Takes a UInt32 number. Interprets it as an IPv4 address in big endian. Returns a FixedString(16) value containing the IPv6 address in binary format. Examples:
|
||||
Takes a `UInt32` number. Interprets it as an IPv4 address in [big endian](https://en.wikipedia.org/wiki/Endianness). Returns a `FixedString(16)` value containing the IPv6 address in binary format. Examples:
|
||||
|
||||
``` sql
|
||||
SELECT IPv6NumToString(IPv4ToIPv6(IPv4StringToNum('192.168.0.1'))) AS addr
|
||||
@ -149,7 +149,7 @@ SELECT
|
||||
|
||||
## IPv4CIDRToRange(ipv4, cidr),
|
||||
|
||||
Accepts an IPv4 and an UInt8 value containing the CIDR. Return a tuple with two IPv4 containing the lower range and the higher range of the subnet.
|
||||
Accepts an IPv4 and an UInt8 value containing the [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). Return a tuple with two IPv4 containing the lower range and the higher range of the subnet.
|
||||
|
||||
|
||||
```sql
|
||||
|
||||
@ -428,7 +428,6 @@ Joins the data in the normal [SQL JOIN](https://en.wikipedia.org/wiki/Join_(SQL)
|
||||
!!! info "Note"
|
||||
Not related to [ARRAY JOIN](#select-array-join-clause).
|
||||
|
||||
|
||||
``` sql
|
||||
SELECT <expr_list>
|
||||
FROM <left_subquery>
|
||||
@ -485,8 +484,6 @@ Be careful when using `GLOBAL`. For more information, see the section [Distribut
|
||||
|
||||
#### Usage Recommendations
|
||||
|
||||
All columns that are not needed for the `JOIN` are deleted from the subquery.
|
||||
|
||||
When running a `JOIN`, there is no optimization of the order of execution in relation to other stages of the query. The join (a search in the right table) is run before filtering in `WHERE` and before aggregation. In order to explicitly set the processing order, we recommend running a `JOIN` subquery with a subquery.
|
||||
|
||||
Example:
|
||||
@ -542,7 +539,16 @@ Each time a query is run with the same `JOIN`, the subquery is run again because
|
||||
In some cases, it is more efficient to use `IN` instead of `JOIN`.
|
||||
Among the various types of `JOIN`, the most efficient is `ANY LEFT JOIN`, then `ANY INNER JOIN`. The least efficient are `ALL LEFT JOIN` and `ALL INNER JOIN`.
|
||||
|
||||
If you need a `JOIN` for joining with dimension tables (these are relatively small tables that contain dimension properties, such as names for advertising campaigns), a `JOIN` might not be very convenient due to the bulky syntax and the fact that the right table is re-accessed for every query. For such cases, there is an "external dictionaries" feature that you should use instead of `JOIN`. For more information, see the section [External dictionaries](dicts/external_dicts.md).
|
||||
If you need a `JOIN` for joining with dimension tables (these are relatively small tables that contain dimension properties, such as names for advertising campaigns), a `JOIN` might not be very convenient due to the fact that the right table is re-accessed for every query. For such cases, there is an "external dictionaries" feature that you should use instead of `JOIN`. For more information, see the section [External dictionaries](dicts/external_dicts.md).
|
||||
|
||||
**Memory Limitations**
|
||||
|
||||
ClickHouse uses the [hash join](https://en.wikipedia.org/wiki/Hash_join) algorithm. ClickHouse takes the `<right_subquery>` and creates a hash table for it in RAM. If you need to restrict join operation memory consumption use the following settings:
|
||||
|
||||
- [max_rows_in_join](../operations/settings/query_complexity.md#settings-max_rows_in_join) — Limits number of rows in the hash table.
|
||||
- [max_bytes_in_join](../operations/settings/query_complexity.md#settings-max_bytes_in_join) — Limits size of the hash table.
|
||||
|
||||
When any of these limits is reached, ClickHouse acts as the [join_overflow_mode](../operations/settings/query_complexity.md#settings-join_overflow_mode) setting instructs.
|
||||
|
||||
#### Processing of Empty or NULL Cells
|
||||
|
||||
@ -683,21 +689,21 @@ You can use WITH TOTALS in subqueries, including subqueries in the JOIN clause (
|
||||
|
||||
#### GROUP BY in External Memory {#select-group-by-in-external-memory}
|
||||
|
||||
You can enable dumping temporary data to the disk to restrict memory usage during GROUP BY.
|
||||
The `max_bytes_before_external_group_by` setting determines the threshold RAM consumption for dumping GROUP BY temporary data to the file system. If set to 0 (the default), it is disabled.
|
||||
You can enable dumping temporary data to the disk to restrict memory usage during `GROUP BY`.
|
||||
The [max_bytes_before_external_group_by](../operations/settings/settings.md#settings-max_bytes_before_external_group_by) setting determines the threshold RAM consumption for dumping `GROUP BY` temporary data to the file system. If set to 0 (the default), it is disabled.
|
||||
|
||||
When using `max_bytes_before_external_group_by`, we recommend that you set max_memory_usage about twice as high. This is necessary because there are two stages to aggregation: reading the date and forming intermediate data (1) and merging the intermediate data (2). Dumping data to the file system can only occur during stage 1. If the temporary data wasn't dumped, then stage 2 might require up to the same amount of memory as in stage 1.
|
||||
When using `max_bytes_before_external_group_by`, we recommend that you set `max_memory_usage` about twice as high. This is necessary because there are two stages to aggregation: reading the date and forming intermediate data (1) and merging the intermediate data (2). Dumping data to the file system can only occur during stage 1. If the temporary data wasn't dumped, then stage 2 might require up to the same amount of memory as in stage 1.
|
||||
|
||||
For example, if `max_memory_usage` was set to 10000000000 and you want to use external aggregation, it makes sense to set `max_bytes_before_external_group_by` to 10000000000, and max_memory_usage to 20000000000. When external aggregation is triggered (if there was at least one dump of temporary data), maximum consumption of RAM is only slightly more than ` max_bytes_before_external_group_by`.
|
||||
For example, if [max_memory_usage](../operations/settings/settings.md#settings_max_memory_usage) was set to 10000000000 and you want to use external aggregation, it makes sense to set `max_bytes_before_external_group_by` to 10000000000, and max_memory_usage to 20000000000. When external aggregation is triggered (if there was at least one dump of temporary data), maximum consumption of RAM is only slightly more than `max_bytes_before_external_group_by`.
|
||||
|
||||
With distributed query processing, external aggregation is performed on remote servers. In order for the requestor server to use only a small amount of RAM, set ` distributed_aggregation_memory_efficient` to 1.
|
||||
With distributed query processing, external aggregation is performed on remote servers. In order for the requester server to use only a small amount of RAM, set `distributed_aggregation_memory_efficient` to 1.
|
||||
|
||||
When merging data flushed to the disk, as well as when merging results from remote servers when the ` distributed_aggregation_memory_efficient` setting is enabled, consumes up to 1/256 \* the number of threads from the total amount of RAM.
|
||||
When merging data flushed to the disk, as well as when merging results from remote servers when the `distributed_aggregation_memory_efficient` setting is enabled, consumes up to `1/256 * the_number_of_threads` from the total amount of RAM.
|
||||
|
||||
When external aggregation is enabled, if there was less than ` max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times).
|
||||
When external aggregation is enabled, if there was less than `max_bytes_before_external_group_by` of data (i.e. data was not flushed), the query runs just as fast as without external aggregation. If any temporary data was flushed, the run time will be several times longer (approximately three times).
|
||||
|
||||
If you have an ORDER BY with a small LIMIT after GROUP BY, then the ORDER BY CLAUSE will not use significant amounts of RAM.
|
||||
But if the ORDER BY doesn't have LIMIT, don't forget to enable external sorting (`max_bytes_before_external_sort`).
|
||||
If you have an `ORDER BY` with a small `LIMIT` after `GROUP BY`, then the `ORDER BY` clause will not use significant amounts of RAM.
|
||||
But if the `ORDER BY` doesn't have `LIMIT`, don't forget to enable external sorting (`max_bytes_before_external_sort`).
|
||||
|
||||
### LIMIT BY Clause
|
||||
|
||||
@ -854,7 +860,7 @@ DISTINCT is not supported if SELECT has at least one array column.
|
||||
|
||||
ClickHouse supports using the `DISTINCT` and `ORDER BY` clauses for different columns in one query. The `DISTINCT` clause is executed before the `ORDER BY` clause.
|
||||
|
||||
The sample table:
|
||||
Example table:
|
||||
|
||||
```text
|
||||
┌─a─┬─b─┐
|
||||
@ -865,7 +871,7 @@ The sample table:
|
||||
└───┴───┘
|
||||
```
|
||||
|
||||
When selecting data by the `SELECT DISTINCT a FROM t1 ORDER BY b ASC` query, we get the following result:
|
||||
When selecting data with the `SELECT DISTINCT a FROM t1 ORDER BY b ASC` query, we get the following result:
|
||||
|
||||
```text
|
||||
┌─a─┐
|
||||
@ -875,7 +881,7 @@ When selecting data by the `SELECT DISTINCT a FROM t1 ORDER BY b ASC` query, we
|
||||
└───┘
|
||||
```
|
||||
|
||||
If we change the direction of ordering `SELECT DISTINCT a FROM t1 ORDER BY b DESC`, we get the following result:
|
||||
If we change the sorting direction `SELECT DISTINCT a FROM t1 ORDER BY b DESC`, we get the following result:
|
||||
|
||||
```text
|
||||
┌─a─┐
|
||||
@ -887,7 +893,7 @@ If we change the direction of ordering `SELECT DISTINCT a FROM t1 ORDER BY b DES
|
||||
|
||||
Row `2, 4` was cut before sorting.
|
||||
|
||||
Take into account this implementation specificity when programming queries.
|
||||
Take this implementation specificity into account when programming queries.
|
||||
|
||||
### LIMIT Clause
|
||||
|
||||
@ -1145,11 +1151,11 @@ It also makes sense to specify a local table in the `GLOBAL IN` clause, in case
|
||||
|
||||
In addition to results, you can also get minimum and maximum values for the results columns. To do this, set the **extremes** setting to 1. Minimums and maximums are calculated for numeric types, dates, and dates with times. For other columns, the default values are output.
|
||||
|
||||
An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in JSON\*, TabSeparated\*, and Pretty\* formats, separate from the other rows. They are not output for other formats.
|
||||
An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`, and `Pretty*` formats, separate from the other rows. They are not output for other formats.
|
||||
|
||||
In JSON\* formats, the extreme values are output in a separate 'extremes' field. In TabSeparated\* formats, the row comes after the main result, and after 'totals' if present. It is preceded by an empty row (after the other data). In Pretty\* formats, the row is output as a separate table after the main result, and after 'totals' if present.
|
||||
In `JSON*` formats, the extreme values are output in a separate 'extremes' field. In `TabSeparated*` formats, the row comes after the main result, and after 'totals' if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after 'totals' if present.
|
||||
|
||||
Extreme values are calculated for rows that have passed through LIMIT. However, when using 'LIMIT offset, size', the rows before 'offset' are included in 'extremes'. In stream requests, the result may also include a small number of rows that passed through LIMIT.
|
||||
Extreme values are calculated for rows before `LIMIT`, but after `LIMIT BY`. However, when using `LIMIT offset, size`, the rows before `offset` are included in `extremes`. In stream requests, the result may also include a small number of rows that passed through `LIMIT`.
|
||||
|
||||
### Notes
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# mysql
|
||||
|
||||
Allows to perform `SELECT` queries on data that is stored on a remote MySQL server.
|
||||
Allows `SELECT` queries to be performed on data that is stored on a remote MySQL server.
|
||||
|
||||
```
|
||||
mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']);
|
||||
@ -13,14 +13,14 @@ mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_
|
||||
- `table` — Remote table name.
|
||||
- `user` — MySQL user.
|
||||
- `password` — User password.
|
||||
- `replace_query` — If `replace_query=1`, the `REPLACE` query will be performed instead of `INSERT`.
|
||||
- `replace_query` — Flag that converts `INSERT INTO` queries to `REPLACE INTO`. If `replace_query=1`, the query is replaced.
|
||||
- `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` expression that is added to the `INSERT` query.
|
||||
|
||||
Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, where `on_duplicate_clause` is `UPDATE c2 = c2 + 1`. See MySQL documentation to find which `on_duplicate_clause` you can use with `ON DUPLICATE KEY` clause.
|
||||
Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, where `on_duplicate_clause` is `UPDATE c2 = c2 + 1`. See the MySQL documentation to find which `on_duplicate_clause` you can use with the `ON DUPLICATE KEY` clause.
|
||||
|
||||
To specify `on_duplicate_clause` you need to pass `0` to the `replace_query` parameter. If you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception.
|
||||
|
||||
At this time, simple `WHERE` clauses such as ` =, !=, >, >=, <, <=` are executed on the MySQL server.
|
||||
Simple `WHERE` clauses such as ` =, !=, >, >=, <, <=` are currently executed on the MySQL server.
|
||||
|
||||
The rest of the conditions and the `LIMIT` sampling constraint are executed in ClickHouse only after the query to MySQL finishes.
|
||||
|
||||
@ -53,7 +53,7 @@ mysql> select * from test;
|
||||
1 row in set (0,00 sec)
|
||||
```
|
||||
|
||||
Selection of the data from ClickHouse:
|
||||
Selecting data from ClickHouse:
|
||||
|
||||
```sql
|
||||
SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123')
|
||||
@ -67,6 +67,6 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123')
|
||||
## See Also
|
||||
|
||||
- [The 'MySQL' table engine](../../operations/table_engines/mysql.md)
|
||||
- [Using MySQL as a source of extenal dictionary](../dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql)
|
||||
- [Using MySQL as a source of external dictionary](../dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql)
|
||||
|
||||
[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/mysql/) <!--hide-->
|
||||
|
||||
@ -688,7 +688,7 @@ message MessageType {
|
||||
ClickHouse попытается найти столбец с именем `x.y.z` (или `x_y_z`, или `X.y_Z` и т.п.).
|
||||
Вложенные сообщения удобно использовать в качестве соответствия для [вложенной структуры данных](../data_types/nested_data_structures/nested.md).
|
||||
|
||||
Значения по умолчанию, определенные в схеме, например,
|
||||
Значения по умолчанию, определённые в схеме `proto2`, например,
|
||||
|
||||
```
|
||||
syntax = "proto2";
|
||||
|
||||
@ -1,32 +1,94 @@
|
||||
|
||||
# MySQL
|
||||
|
||||
Движок MySQL позволяет выполнять `SELECT` запросы над данными, хранящимися на удалённом MySQL сервере.
|
||||
Движок MySQL позволяет выполнять запросы `SELECT` над данными, хранящимися на удалённом MySQL сервере.
|
||||
|
||||
Формат вызова:
|
||||
## Создание таблицы
|
||||
|
||||
```
|
||||
MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']);
|
||||
```sql
|
||||
CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
(
|
||||
name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1],
|
||||
name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2],
|
||||
...
|
||||
INDEX index_name1 expr1 TYPE type1(...) GRANULARITY value1,
|
||||
INDEX index_name2 expr2 TYPE type2(...) GRANULARITY value2
|
||||
) ENGINE = MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']);
|
||||
```
|
||||
|
||||
**Параметры вызова**
|
||||
Смотрите подробное описание запроса [CREATE TABLE](../../query_language/create.md#create-table-query).
|
||||
|
||||
- `host:port` — Адрес сервера MySQL.
|
||||
- `database` — Имя базы данных на сервере MySQL.
|
||||
- `table` — Имя таблицы.
|
||||
- `user` — Пользователь MySQL.
|
||||
- `password` — Пароль пользователя.
|
||||
- `replace_query` — Флаг, устанавливающий замену запроса `INSERT INTO` на `REPLACE INTO`. Если `replace_query=1`, то запрос заменяется.
|
||||
- `on_duplicate_clause` — Добавляет выражение `ON DUPLICATE KEY 'on_duplicate_clause'` в запрос `INSERT`.
|
||||
Структура таблицы может отличаться от исходной структуры таблицы MySQL:
|
||||
|
||||
Например, `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, где `on_duplicate_clause` — `UPDATE c2 = c2 + 1`. Какие выражения `on_duplicate_clause` вы можете использовать с `ON DUPLICATE KEY`, смотрите в документации MySQL.
|
||||
- Имена столбцов должны быть такими же, как в исходной таблице MySQL, но вы можете использовать только некоторые из этих столбцов и в любом порядке.
|
||||
- Типы столбцов могут отличаться от типов в исходной таблице MySQL. ClickHouse пытается [приводить](../../query_language/functions/type_conversion_functions.md#type_conversion_function-cast) значения к типам данных ClickHouse.
|
||||
|
||||
**Параметры движка**
|
||||
|
||||
- `host:port` — адрес сервера MySQL.
|
||||
- `database` — имя базы данных на удалённом сервере.
|
||||
- `table` — имя таблицы на удалённом сервере.
|
||||
- `user` — пользователь MySQL.
|
||||
- `password` — пароль пользователя.
|
||||
- `replace_query` — флаг, отвечающий за преобразование запросов `INSERT INTO` в `REPLACE INTO`. Если `replace_query=1`, то запрос заменяется.
|
||||
- `on_duplicate_clause` — выражение `ON DUPLICATE KEY on_duplicate_clause`, добавляемое к запросу `INSERT`.
|
||||
|
||||
Пример: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, где `on_duplicate_clause` это `UPDATE c2 = c2 + 1`. Чтобы узнать какие `on_duplicate_clause` можно использовать с секцией `ON DUPLICATE KEY` обратитесь к [документации MySQL](https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html).
|
||||
|
||||
Чтобы указать `on_duplicate_clause` необходимо передать `0` в параметр `replace_query`. Если одновременно передать `replace_query = 1` и `on_duplicate_clause`, то ClickHouse сгенерирует исключение.
|
||||
|
||||
На данный момент простые условия `WHERE`, такие как `=, !=, >, >=, <, <=` будут выполняться на стороне сервера MySQL.
|
||||
Простые условия `WHERE` такие как ` =, !=, >, >=, <, =` выполняются на стороне сервера MySQL.
|
||||
|
||||
Остальные условия и ограничение выборки `LIMIT` будут выполнены в ClickHouse только после выполнения запроса к MySQL.
|
||||
|
||||
Движок `MySQL` не поддерживает тип данных [Nullable](../../data_types/nullable.md), поэтому при чтении данных из таблиц MySQL `NULL` преобразуются в значения по умолчанию для заданного типа столбца, обычно это 0 или пустая строка.
|
||||
## Пример использования
|
||||
|
||||
Таблица в MySQL:
|
||||
|
||||
```
|
||||
mysql> CREATE TABLE `test`.`test` (
|
||||
-> `int_id` INT NOT NULL AUTO_INCREMENT,
|
||||
-> `int_nullable` INT NULL DEFAULT NULL,
|
||||
-> `float` FLOAT NOT NULL,
|
||||
-> `float_nullable` FLOAT NULL DEFAULT NULL,
|
||||
-> PRIMARY KEY (`int_id`));
|
||||
Query OK, 0 rows affected (0,09 sec)
|
||||
|
||||
mysql> insert into test (`int_id`, `float`) VALUES (1,2);
|
||||
Query OK, 1 row affected (0,00 sec)
|
||||
|
||||
mysql> select * from test;
|
||||
+--------+--------------+-------+----------------+
|
||||
| int_id | int_nullable | float | float_nullable |
|
||||
+--------+--------------+-------+----------------+
|
||||
| 1 | NULL | 2 | NULL |
|
||||
+--------+--------------+-------+----------------+
|
||||
1 row in set (0,00 sec)
|
||||
```
|
||||
|
||||
Таблица в ClickHouse, которая получает данные из таблицы MySQL:
|
||||
|
||||
```sql
|
||||
CREATE TABLE mysql_table
|
||||
(
|
||||
`float_nullable` Nullable(Float32),
|
||||
`int_id` Int32
|
||||
)
|
||||
ENGINE = MySQL('localhost:3306', 'test', 'test', 'bayonet', '123')
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT * FROM mysql_table6
|
||||
```
|
||||
|
||||
```text
|
||||
┌─float_nullable─┬─int_id─┐
|
||||
│ ᴺᵁᴸᴸ │ 1 │
|
||||
└────────────────┴────────┘
|
||||
```
|
||||
|
||||
## Смотрите также
|
||||
|
||||
- [Табличная функция 'mysql'](../../query_language/table_functions/mysql.md)
|
||||
- [Использование MySQL в качестве источника для внешнего словаря](../../query_language/dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql)
|
||||
|
||||
[Оригинальная статья](https://clickhouse.yandex/docs/ru/operations/table_engines/mysql/) <!--hide-->
|
||||
|
||||
@ -178,6 +178,47 @@ binary decimal
|
||||
01101000 = 104
|
||||
```
|
||||
|
||||
## groupBitmap
|
||||
|
||||
Bitmap или агрегатные вычисления для столбца с типом данных `UInt*`, возвращают кардинальность в виде значения типа UInt64, если добавить суффикс -State, то возвращают [объект bitmap](../functions/bitmap_functions.md).
|
||||
|
||||
```
|
||||
groupBitmap(expr)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
`expr` – выражение, возвращающее тип данных `UInt*`.
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
Значение типа `UInt64`.
|
||||
|
||||
**Пример**
|
||||
|
||||
Тестовые данные:
|
||||
|
||||
```
|
||||
userid
|
||||
1
|
||||
1
|
||||
2
|
||||
3
|
||||
```
|
||||
|
||||
Запрос:
|
||||
|
||||
```
|
||||
SELECT groupBitmap(userid) as num FROM t
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```
|
||||
num
|
||||
3
|
||||
```
|
||||
|
||||
## min(x) {#agg_function-min}
|
||||
|
||||
Вычисляет минимум.
|
||||
@ -390,9 +431,11 @@ FROM (
|
||||
- Значение по умолчанию для подстановки на пустые позиции.
|
||||
- Длина результирующего массива. Например, если вы хотите получать массивы одинакового размера для всех агрегатных ключей. При использовании этого параметра значение по умолчанию задавать обязательно.
|
||||
|
||||
## groupUniqArray(x)
|
||||
## groupUniqArray(x), groupUniqArray(max_size)(x)
|
||||
|
||||
Составляет массив из различных значений аргумента. Расход оперативки такой же, как у функции `uniqExact`.
|
||||
Составляет массив из различных значений аргумента. Расход оперативной памяти такой же, как у функции `uniqExact`.
|
||||
|
||||
Функция `groupUniqArray(max_size)(x)` ограничивает размер результирующего массива до `max_size` элементов. Например, `groupUniqArray(1)(x)` равнозначно `[any(x)]`.
|
||||
|
||||
## quantile(level)(x)
|
||||
|
||||
|
||||
321
docs/ru/query_language/functions/bitmap_functions.md
Normal file
321
docs/ru/query_language/functions/bitmap_functions.md
Normal file
@ -0,0 +1,321 @@
|
||||
# Функции для битмапов
|
||||
|
||||
## bitmapBuild
|
||||
|
||||
Создаёт битмап из массива целочисленных значений.
|
||||
|
||||
```
|
||||
bitmapBuild(array)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `array` – массив типа `UInt*`.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res)
|
||||
```
|
||||
```text
|
||||
┌─res─┬─toTypeName(bitmapBuild([1, 2, 3, 4, 5]))─────┐
|
||||
│ │ AggregateFunction(groupBitmap, UInt8) │
|
||||
└─────┴──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
## bitmapToArray
|
||||
|
||||
Преобразует битмап в массив целочисленных значений.
|
||||
|
||||
```
|
||||
bitmapToArray(bitmap)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap` – битмап.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5])) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─────────┐
|
||||
│ [1,2,3,4,5] │
|
||||
└─────────────┘
|
||||
```
|
||||
|
||||
## bitmapHasAny
|
||||
|
||||
Проверяет, имеют ли два битмапа хотя бы один общий элемент.
|
||||
|
||||
```
|
||||
bitmapHasAny(bitmap1, bitmap2)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap*` – массив любого типа с набором элементов.
|
||||
|
||||
**Возвращаемые значения**
|
||||
|
||||
- `1`, если `bitmap1` и `bitmap2` имеют хотя бы один одинаковый элемент.
|
||||
- `0`, в противном случае.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapHasAny(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 1 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## bitmapHasAll
|
||||
|
||||
Аналогично функции `hasAll(array, array)` возвращает 1 если первый битмап содержит все элементы второго, 0 в противном случае.
|
||||
Если второй аргумент является пустым битмапом, то возвращает 1.
|
||||
|
||||
```
|
||||
bitmapHasAll(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap` – битмап.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapHasAll(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 0 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## bitmapAnd
|
||||
|
||||
Логическое И для двух битмапов. Результат — новый битмап.
|
||||
|
||||
```
|
||||
bitmapAnd(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap` – битмап.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ [3] │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## bitmapOr
|
||||
|
||||
Логическое ИЛИ для двух битмапов. Результат — новый битмап.
|
||||
|
||||
```
|
||||
bitmapOr(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap` – битмап.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapToArray(bitmapOr(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─────────┐
|
||||
│ [1,2,3,4,5] │
|
||||
└─────────────┘
|
||||
```
|
||||
|
||||
## bitmapXor
|
||||
|
||||
Логическое исключающее ИЛИ для двух битмапов. Результат — новый битмап.
|
||||
|
||||
```
|
||||
bitmapXor(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap` – битмап.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapToArray(bitmapXor(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res───────┐
|
||||
│ [1,2,4,5] │
|
||||
└───────────┘
|
||||
```
|
||||
|
||||
## bitmapAndnot
|
||||
|
||||
Логическое отрицание И для двух битмапов. Результат — новый битмап.
|
||||
|
||||
```
|
||||
bitmapAndnot(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap` – битмап.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res───┐
|
||||
│ [1,2] │
|
||||
└───────┘
|
||||
```
|
||||
|
||||
## bitmapCardinality
|
||||
|
||||
Возвращает кардинальность битмапа в виде значения типа `UInt64`.
|
||||
|
||||
```
|
||||
bitmapCardinality(bitmap)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap` – битмап.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5])) AS res
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 5 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## bitmapAndCardinality
|
||||
|
||||
Выполняет логическое И и возвращает кардинальность (`UInt64`) результирующего битмапа.
|
||||
|
||||
```
|
||||
bitmapAndCardinality(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap` – битмап.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapAndCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res;
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 1 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## bitmapOrCardinality
|
||||
|
||||
Выполняет логическое ИЛИ и возвращает кардинальность (`UInt64`) результирующего битмапа.
|
||||
|
||||
```
|
||||
bitmapOrCardinality(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap` – битмап.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapOrCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res;
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 5 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## bitmapXorCardinality
|
||||
|
||||
Выполняет логическое исключающее ИЛИ и возвращает кардинальность (`UInt64`) результирующего битмапа.
|
||||
|
||||
```
|
||||
bitmapXorCardinality(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap` – битмап.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapXorCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res;
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 4 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
## bitmapAndnotCardinality
|
||||
|
||||
Выполняет логическое отрицание И и возвращает кардинальность (`UInt64`) результирующего битмапа.
|
||||
|
||||
```
|
||||
bitmapAndnotCardinality(bitmap,bitmap)
|
||||
```
|
||||
|
||||
**Параметры**
|
||||
|
||||
- `bitmap` – битмап.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT bitmapAndnotCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res;
|
||||
```
|
||||
|
||||
```
|
||||
┌─res─┐
|
||||
│ 2 │
|
||||
└─────┘
|
||||
```
|
||||
|
||||
[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/bitmap_functions/) <!--hide-->
|
||||
@ -1,17 +1,20 @@
|
||||
# Функции для работы с IP-адресами
|
||||
|
||||
## IPv4NumToString(num)
|
||||
|
||||
Принимает число типа UInt32. Интерпретирует его, как IPv4-адрес в big endian. Возвращает строку, содержащую соответствующий IPv4-адрес в формате A.B.C.D (числа в десятичной форме через точки).
|
||||
|
||||
## IPv4StringToNum(s)
|
||||
|
||||
Функция, обратная к IPv4NumToString. Если IPv4 адрес в неправильном формате, то возвращает 0.
|
||||
|
||||
## IPv4NumToStringClassC(num)
|
||||
|
||||
Похоже на IPv4NumToString, но вместо последнего октета используется xxx.
|
||||
|
||||
Пример:
|
||||
|
||||
``` sql
|
||||
```sql
|
||||
SELECT
|
||||
IPv4NumToStringClassC(ClientIP) AS k,
|
||||
count() AS c
|
||||
@ -36,13 +39,14 @@ LIMIT 10
|
||||
└────────────────┴───────┘
|
||||
```
|
||||
|
||||
В связи с тем, что использование xxx весьма необычно, это может быть изменено в дальнейшем, и вам не следует полагаться на конкретный вид этого фрагмента.
|
||||
В связи с тем, что использование xxx весьма необычно, это может быть изменено в дальнейшем. Вам не следует полагаться на конкретный вид этого фрагмента.
|
||||
|
||||
### IPv6NumToString(x)
|
||||
|
||||
Принимает значение типа FixedString(16), содержащее IPv6-адрес в бинарном виде. Возвращает строку, содержащую этот адрес в текстовом виде.
|
||||
IPv6-mapped IPv4 адреса выводится в формате ::ffff:111.222.33.44. Примеры:
|
||||
|
||||
``` sql
|
||||
```sql
|
||||
SELECT IPv6NumToString(toFixedString(unhex('2A0206B8000000000000000000000011'), 16)) AS addr
|
||||
```
|
||||
|
||||
@ -52,7 +56,7 @@ SELECT IPv6NumToString(toFixedString(unhex('2A0206B8000000000000000000000011'),
|
||||
└──────────────┘
|
||||
```
|
||||
|
||||
``` sql
|
||||
```sql
|
||||
SELECT
|
||||
IPv6NumToString(ClientIP6 AS k),
|
||||
count() AS c
|
||||
@ -78,7 +82,7 @@ LIMIT 10
|
||||
└─────────────────────────────────────────┴───────┘
|
||||
```
|
||||
|
||||
``` sql
|
||||
```sql
|
||||
SELECT
|
||||
IPv6NumToString(ClientIP6 AS k),
|
||||
count() AS c
|
||||
@ -105,7 +109,133 @@ LIMIT 10
|
||||
```
|
||||
|
||||
## IPv6StringToNum(s)
|
||||
|
||||
Функция, обратная к IPv6NumToString. Если IPv6 адрес в неправильном формате, то возвращает строку из нулевых байт.
|
||||
HEX может быть в любом регистре.
|
||||
|
||||
## IPv4ToIPv6(x)
|
||||
|
||||
Принимает число типа `UInt32`. Интерпретирует его, как IPv4-адрес в [big endian](https://en.wikipedia.org/wiki/Endianness). Возвращает значение `FixedString(16)`, содержащее адрес IPv6 в двоичном формате. Примеры:
|
||||
|
||||
```sql
|
||||
SELECT IPv6NumToString(IPv4ToIPv6(IPv4StringToNum('192.168.0.1'))) AS addr
|
||||
```
|
||||
|
||||
```
|
||||
┌─addr───────────────┐
|
||||
│ ::ffff:192.168.0.1 │
|
||||
└────────────────────┘
|
||||
```
|
||||
|
||||
## cutIPv6(x, bitsToCutForIPv6, bitsToCutForIPv4)
|
||||
|
||||
Принимает значение типа FixedString(16), содержащее IPv6-адрес в бинарном виде. Возвращает строку, содержащую адрес из указанного количества битов, удаленных в текстовом формате. Например:
|
||||
|
||||
```sql
|
||||
WITH
|
||||
IPv6StringToNum('2001:0DB8:AC10:FE01:FEED:BABE:CAFE:F00D') AS ipv6,
|
||||
IPv4ToIPv6(IPv4StringToNum('192.168.0.1')) AS ipv4
|
||||
SELECT
|
||||
cutIPv6(ipv6, 2, 0),
|
||||
cutIPv6(ipv4, 0, 2)
|
||||
```
|
||||
|
||||
```
|
||||
┌─cutIPv6(ipv6, 2, 0)─────────────────┬─cutIPv6(ipv4, 0, 2)─┐
|
||||
│ 2001:db8:ac10:fe01:feed:babe:cafe:0 │ ::ffff:192.168.0.0 │
|
||||
└─────────────────────────────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
## IPv4CIDRtoIPv4Range(ipv4, cidr),
|
||||
|
||||
Принимает на вход IPv4 и значение `UInt8`, содержащее [CIDR](https://ru.wikipedia.org/wiki/Бесклассовая_адресация). Возвращает кортеж с двумя IPv4, содержащими нижний и более высокий диапазон подсети.
|
||||
|
||||
```sql
|
||||
SELECT IPv4CIDRtoIPv4Range(toIPv4('192.168.5.2'), 16)
|
||||
```
|
||||
|
||||
```
|
||||
┌─IPv4CIDRtoIPv4Range(toIPv4('192.168.5.2'), 16)─┐
|
||||
│ ('192.168.0.0','192.168.255.255') │
|
||||
└────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## IPv6CIDRtoIPv6Range(ipv6, cidr),
|
||||
|
||||
Принимает на вход IPv6 и значение `UInt8`, содержащее CIDR. Возвращает кортеж с двумя IPv6, содержащими нижний и более высокий диапазон подсети.
|
||||
|
||||
```sql
|
||||
SELECT IPv6CIDRtoIPv6Range(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32);
|
||||
```
|
||||
|
||||
```
|
||||
┌─IPv6CIDRtoIPv6Range(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32)─┐
|
||||
│ ('2001:db8::','2001:db8:ffff:ffff:ffff:ffff:ffff:ffff') │
|
||||
└────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toIPv4(string)
|
||||
|
||||
Псевдоним функции `IPv4StringToNum()` которая принимает строку с адресом IPv4 и возвращает значение типа [IPv4](../../data_types/domains/ipv4.md), которое равно значению, возвращаемому функцией `IPv4StringToNum()`.
|
||||
|
||||
```sql
|
||||
WITH
|
||||
'171.225.130.45' as IPv4_string
|
||||
SELECT
|
||||
toTypeName(IPv4StringToNum(IPv4_string)),
|
||||
toTypeName(toIPv4(IPv4_string))
|
||||
```
|
||||
|
||||
```
|
||||
┌─toTypeName(IPv4StringToNum(IPv4_string))─┬─toTypeName(toIPv4(IPv4_string))─┐
|
||||
│ UInt32 │ IPv4 │
|
||||
└──────────────────────────────────────────┴─────────────────────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
WITH
|
||||
'171.225.130.45' as IPv4_string
|
||||
SELECT
|
||||
hex(IPv4StringToNum(IPv4_string)),
|
||||
hex(toIPv4(IPv4_string))
|
||||
```
|
||||
|
||||
```
|
||||
┌─hex(IPv4StringToNum(IPv4_string))─┬─hex(toIPv4(IPv4_string))─┐
|
||||
│ ABE1822D │ ABE1822D │
|
||||
└───────────────────────────────────┴──────────────────────────┘
|
||||
```
|
||||
|
||||
## toIPv6(string)
|
||||
|
||||
Псевдоним функции `IPv6StringToNum()` которая принимает строку с адресом IPv6 и возвращает значение типа [IPv6](../../data_types/domains/ipv6.md), которое равно значению, возвращаемому функцией `IPv6StringToNum()`.
|
||||
|
||||
```sql
|
||||
WITH
|
||||
'2001:438:ffff::407d:1bc1' as IPv6_string
|
||||
SELECT
|
||||
toTypeName(IPv6StringToNum(IPv6_string)),
|
||||
toTypeName(toIPv6(IPv6_string))
|
||||
```
|
||||
|
||||
```
|
||||
┌─toTypeName(IPv6StringToNum(IPv6_string))─┬─toTypeName(toIPv6(IPv6_string))─┐
|
||||
│ FixedString(16) │ IPv6 │
|
||||
└──────────────────────────────────────────┴─────────────────────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
WITH
|
||||
'2001:438:ffff::407d:1bc1' as IPv6_string
|
||||
SELECT
|
||||
hex(IPv6StringToNum(IPv6_string)),
|
||||
hex(toIPv6(IPv6_string))
|
||||
```
|
||||
|
||||
```
|
||||
┌─hex(IPv6StringToNum(IPv6_string))─┬─hex(toIPv6(IPv6_string))─────────┐
|
||||
│ 20010438FFFF000000000000407D1BC1 │ 20010438FFFF000000000000407D1BC1 │
|
||||
└───────────────────────────────────┴──────────────────────────────────┘
|
||||
```
|
||||
|
||||
[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/ip_address_functions/) <!--hide-->
|
||||
|
||||
@ -7,7 +7,7 @@ SELECT [DISTINCT] expr_list
|
||||
[FROM [db.]table | (subquery) | table_function] [FINAL]
|
||||
[SAMPLE sample_coeff]
|
||||
[ARRAY JOIN ...]
|
||||
[[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list
|
||||
[GLOBAL] [ANY|ALL] [INNER|LEFT|RIGHT|FULL|CROSS] [OUTER] JOIN (subquery)|table USING columns_list
|
||||
[PREWHERE expr]
|
||||
[WHERE expr]
|
||||
[GROUP BY expr_list] [WITH TOTALS]
|
||||
@ -849,6 +849,43 @@ WHERE и HAVING отличаются тем, что WHERE выполняется
|
||||
|
||||
`DISTINCT` работает с [NULL](syntax.md) как если бы `NULL` был конкретным значением, причём `NULL=NULL`. Т.е. в результате `DISTINCT` разные комбинации с `NULL` встретятся только по одному разу.
|
||||
|
||||
ClickHouse поддерживает использование в одном запросе секций `DISTINCT` и `ORDER BY` для разных столбцов. Секция `DISTINCT` исполняется перед секцией `ORDER BY`.
|
||||
|
||||
Таблица для примера:
|
||||
|
||||
```text
|
||||
┌─a─┬─b─┐
|
||||
│ 2 │ 1 │
|
||||
│ 1 │ 2 │
|
||||
│ 3 │ 3 │
|
||||
│ 2 │ 4 │
|
||||
└───┴───┘
|
||||
```
|
||||
|
||||
При выборке данных запросом `SELECT DISTINCT a FROM t1 ORDER BY b ASC`, мы получаем следующий результат:
|
||||
|
||||
```text
|
||||
┌─a─┐
|
||||
│ 2 │
|
||||
│ 1 │
|
||||
│ 3 │
|
||||
└───┘
|
||||
```
|
||||
|
||||
Если изменить направление сортировки `SELECT DISTINCT a FROM t1 ORDER BY b DESC`, то результат получается следующий:
|
||||
|
||||
```text
|
||||
┌─a─┐
|
||||
│ 3 │
|
||||
│ 1 │
|
||||
│ 2 │
|
||||
└───┘
|
||||
```
|
||||
|
||||
Строка `2, 4` была удалена перед сортировкой.
|
||||
|
||||
Учитывайте эту особенность реализации при программировании запросов.
|
||||
|
||||
### Секция LIMIT
|
||||
|
||||
`LIMIT m` позволяет выбрать из результата первые `m` строк.
|
||||
|
||||
@ -1,36 +1,36 @@
|
||||
# mysql
|
||||
|
||||
Allows to perform `SELECT` queries on data that is stored on a remote MySQL server.
|
||||
Позволяет выполнять запросы `SELECT` над данными, хранящимися на удалённом MySQL сервере.
|
||||
|
||||
```
|
||||
mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']);
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
**Параметры**
|
||||
|
||||
- `host:port` — MySQL server address.
|
||||
- `database` — Remote database name.
|
||||
- `table` — Remote table name.
|
||||
- `user` — MySQL user.
|
||||
- `password` — User password.
|
||||
- `replace_query` — Flag that sets query substitution `INSERT INTO` to `REPLACE INTO`. If `replace_query=1`, the query is replaced.
|
||||
- `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` expression that is added to the `INSERT` query.
|
||||
- `host:port` — адрес сервера MySQL.
|
||||
- `database` — имя базы данных на удалённом сервере.
|
||||
- `table` — имя таблицы на удалённом сервере.
|
||||
- `user` — пользователь MySQL.
|
||||
- `password` — пароль пользователя.
|
||||
- `replace_query` — флаг, отвечающий за преобразование запросов `INSERT INTO` в `REPLACE INTO`. Если `replace_query=1`, то запрос заменяется.
|
||||
- `on_duplicate_clause` — выражение `ON DUPLICATE KEY on_duplicate_clause`, добавляемое в запрос `INSERT`.
|
||||
|
||||
Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, where `on_duplicate_clause` is `UPDATE c2 = c2 + 1`. See MySQL documentation to find which `on_duplicate_clause` you can use with `ON DUPLICATE KEY` clause.
|
||||
Пример: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, где `on_duplicate_clause` это `UPDATE c2 = c2 + 1`. Чтобы узнать какие `on_duplicate_clause` можно использовать с секцией `ON DUPLICATE KEY` обратитесь к документации MySQL.
|
||||
|
||||
To specify `on_duplicate_clause` you need to pass `0` to the `replace_query` parameter. If you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception.
|
||||
Чтобы указать `'on_duplicate_clause'` необходимо передать `0` в параметр `replace_query`. Если одновременно передать `replace_query = 1` и `'on_duplicate_clause'`, то ClickHouse сгенерирует исключение.
|
||||
|
||||
At this time, simple `WHERE` clauses and elements of `AND` chain in `WHERE` clause such as `=, !=, >, >=, <, <=, LIKE, IN` with compatible functions are also pushed down for execution on the MySQL server.
|
||||
Простые условия `WHERE` такие как ` =, !=, >, >=, <, =` выполняются на стороне сервера MySQL.
|
||||
|
||||
The rest of the conditions and the `LIMIT` constraint are executed in ClickHouse only after the query to MySQL finishes.
|
||||
Остальные условия и ограничение выборки `LIMIT` будут выполнены в ClickHouse только после выполнения запроса к MySQL.
|
||||
|
||||
**Returned Value**
|
||||
**Возвращаемое значение**
|
||||
|
||||
A table object with the same columns as the original MySQL table.
|
||||
Объект таблицы с теми же столбцами, что и в исходной таблице MySQL.
|
||||
|
||||
## Usage Example
|
||||
## Пример использования
|
||||
|
||||
Table in MySQL:
|
||||
Таблица в MySQL:
|
||||
|
||||
```
|
||||
mysql> CREATE TABLE `test`.`test` (
|
||||
@ -53,20 +53,21 @@ mysql> select * from test;
|
||||
1 row in set (0,00 sec)
|
||||
```
|
||||
|
||||
Selection of the data from ClickHouse:
|
||||
Получение данных в ClickHouse:
|
||||
|
||||
```sql
|
||||
SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123')
|
||||
```
|
||||
|
||||
```text
|
||||
┌─int_id─┬─int_nullable─┬─float─┬─float_nullable─┐
|
||||
│ 1 │ ᴺᵁᴸᴸ │ 2 │ ᴺᵁᴸᴸ │
|
||||
└────────┴──────────────┴───────┴────────────────┘
|
||||
```
|
||||
|
||||
## See Also
|
||||
## Смотрите также
|
||||
|
||||
- [The 'MySQL' table engine](../../operations/table_engines/mysql.md)
|
||||
- [Using MySQL as a source of extenal dictionary](../dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql)
|
||||
- [Движок таблиц 'MySQL'](../../operations/table_engines/mysql.md)
|
||||
- [Использование MySQL как источника данных для внешнего словаря](../dicts/external_dicts_dict_sources.md#dicts-external_dicts_dict_sources-mysql)
|
||||
|
||||
[Original article](https://clickhouse.yandex/docs/en/query_language/table_functions/mysql/) <!--hide-->
|
||||
[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/table_functions/mysql/) <!--hide-->
|
||||
|
||||
@ -122,6 +122,7 @@ nav:
|
||||
- 'Функции по работе с массивами': 'query_language/functions/array_functions.md'
|
||||
- 'Функции разбиения и слияния строк и массивов': 'query_language/functions/splitting_merging_functions.md'
|
||||
- 'Битовые функции': 'query_language/functions/bit_functions.md'
|
||||
- 'Функции для битмапов': 'query_language/functions/bitmap_functions.md'
|
||||
- 'Функции хэширования': 'query_language/functions/hash_functions.md'
|
||||
- 'Функции генерации псевдослучайных чисел': 'query_language/functions/random_functions.md'
|
||||
- 'Функции для работы с UUID': 'query_language/functions/uuid_functions.md'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user