From dbd136672f074404530fabd917f8b7e6eb65f86d Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 26 Nov 2020 10:07:04 +0300 Subject: [PATCH 1/5] Fix error handling in grpc-client.py --- utils/grpc-client/clickhouse-grpc-client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/grpc-client/clickhouse-grpc-client.py b/utils/grpc-client/clickhouse-grpc-client.py index ef5f6d1f154..691917cc757 100755 --- a/utils/grpc-client/clickhouse-grpc-client.py +++ b/utils/grpc-client/clickhouse-grpc-client.py @@ -10,7 +10,9 @@ # Most of the command line options are the same, for more information type # ./clickhouse_grpc_client.py --help -import argparse, cmd, os, signal, subprocess, sys, threading, time, uuid, grpc +import grpc # pip3 install grpcio +import grpc_tools # pip3 install grpcio-tools +import argparse, cmd, os, signal, subprocess, sys, threading, time, uuid default_host = 'localhost' default_port = 9001 @@ -196,7 +198,7 @@ class ClickHouseGRPCClient(cmd.Cmd): errors = p.stderr.read().decode().strip('\n').split('\n') only_warnings = all(('Warning' in error) for error in errors) if not only_warnings: - error_print(errors.join('\n')) + error_print('\n'.join(errors)) # Import the generated *pb2.py files. @staticmethod From 49cf98076112818984a6eaf447a92e964c6fa6aa Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 26 Nov 2020 10:14:27 +0300 Subject: [PATCH 2/5] Use port 9100 for grpc by default. --- programs/server/config.xml | 2 +- tests/integration/test_grpc_protocol/configs/grpc_port.xml | 2 +- tests/integration/test_grpc_protocol/test.py | 2 +- .../test_grpc_protocol_ssl/configs/grpc_config.xml | 2 +- tests/integration/test_grpc_protocol_ssl/test.py | 2 +- utils/grpc-client/clickhouse-grpc-client.py | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index cef191a71f1..b1b185057e9 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -135,7 +135,7 @@ 3 diff --git a/tests/integration/test_grpc_protocol/configs/grpc_port.xml b/tests/integration/test_grpc_protocol/configs/grpc_port.xml index 31b6229a3c6..61922140bd3 100644 --- a/tests/integration/test_grpc_protocol/configs/grpc_port.xml +++ b/tests/integration/test_grpc_protocol/configs/grpc_port.xml @@ -1,3 +1,3 @@ - 9001 + 9100 diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index 6589e2336d7..5ff1d543f07 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -29,7 +29,7 @@ import clickhouse_grpc_pb2_grpc config_dir = os.path.join(SCRIPT_DIR, './configs') cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=['configs/grpc_port.xml']) -grpc_port = 9001 +grpc_port = 9100 main_channel = None def create_channel(): diff --git a/tests/integration/test_grpc_protocol_ssl/configs/grpc_config.xml b/tests/integration/test_grpc_protocol_ssl/configs/grpc_config.xml index e9754b7e79b..76ef4d8c952 100644 --- a/tests/integration/test_grpc_protocol_ssl/configs/grpc_config.xml +++ b/tests/integration/test_grpc_protocol_ssl/configs/grpc_config.xml @@ -1,5 +1,5 @@ - 9001 + 9100 true diff --git a/tests/integration/test_grpc_protocol_ssl/test.py b/tests/integration/test_grpc_protocol_ssl/test.py index 339a41c799d..a311dee8781 100644 --- a/tests/integration/test_grpc_protocol_ssl/test.py +++ b/tests/integration/test_grpc_protocol_ssl/test.py @@ -25,7 +25,7 @@ import clickhouse_grpc_pb2_grpc # Utilities node_ip = '10.5.172.77' # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). -grpc_port = 9001 +grpc_port = 9100 node_ip_with_grpc_port = node_ip + ':' + str(grpc_port) config_dir = os.path.join(SCRIPT_DIR, './configs') cluster = ClickHouseCluster(__file__) diff --git a/utils/grpc-client/clickhouse-grpc-client.py b/utils/grpc-client/clickhouse-grpc-client.py index 691917cc757..19d213a8e3f 100755 --- a/utils/grpc-client/clickhouse-grpc-client.py +++ b/utils/grpc-client/clickhouse-grpc-client.py @@ -15,7 +15,7 @@ import grpc_tools # pip3 install grpcio-tools import argparse, cmd, os, signal, subprocess, sys, threading, time, uuid default_host = 'localhost' -default_port = 9001 +default_port = 9100 default_user_name = 'default' default_output_format_for_interactive_mode = 'PrettyCompact' history_filename = '~/.clickhouse_grpc_history' @@ -263,7 +263,7 @@ def main(args): parser = argparse.ArgumentParser(description='ClickHouse client accessing server through gRPC protocol.', add_help=False) parser.add_argument('--help', help='Show this help message and exit', action='store_true') parser.add_argument('--host', '-h', help='The server name, ‘localhost’ by default. You can use either the name or the IPv4 or IPv6 address.', default='localhost') - parser.add_argument('--port', help='The port to connect to. This port should be enabled on the ClickHouse server (see grpc_port in the config).', default=9001) + parser.add_argument('--port', help='The port to connect to. This port should be enabled on the ClickHouse server (see grpc_port in the config).', default=9100) parser.add_argument('--user', '-u', dest='user_name', help='The username. Default value: ‘default’.', default='default') parser.add_argument('--password', help='The password. Default value: empty string.', default='') parser.add_argument('--query', '-q', help='The query to process when using non-interactive mode.', default='') From 3d00cc36f39398ce77e56700200e89d25b430185 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 26 Nov 2020 10:18:21 +0300 Subject: [PATCH 3/5] Remove accidently committed src/Core/include/config_core.h --- src/Core/include/config_core.h | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 src/Core/include/config_core.h diff --git a/src/Core/include/config_core.h b/src/Core/include/config_core.h deleted file mode 100644 index 725f9146413..00000000000 --- a/src/Core/include/config_core.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -// .h autogenerated by cmake! - -#define USE_ICU 1 -#define USE_MYSQL 1 -#define USE_RDKAFKA 1 -#define USE_AMQPCPP 1 -#define USE_EMBEDDED_COMPILER 0 -#define USE_INTERNAL_LLVM_LIBRARY 0 -#define USE_SSL 1 -#define USE_OPENCL 0 -#define USE_LDAP 1 From 2063cbae0efa8657448c7b9758c77f49beae90b9 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 26 Nov 2020 10:23:01 +0300 Subject: [PATCH 4/5] Remove old and broken C++ grpc-client. --- programs/CMakeLists.txt | 2 - programs/grpc-client/CMakeLists.txt | 7 -- programs/grpc-client/grpc_client.cpp | 173 --------------------------- 3 files changed, 182 deletions(-) delete mode 100644 programs/grpc-client/CMakeLists.txt delete mode 100644 programs/grpc-client/grpc_client.cpp diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index d9c5dc78fe4..3817bc62bcb 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -112,8 +112,6 @@ add_subdirectory (obfuscator) add_subdirectory (install) add_subdirectory (git-import) -#add_subdirectory (grpc-client) - if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) add_subdirectory (odbc-bridge) endif () diff --git a/programs/grpc-client/CMakeLists.txt b/programs/grpc-client/CMakeLists.txt deleted file mode 100644 index d848434e918..00000000000 --- a/programs/grpc-client/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -get_filename_component(rpc_proto "${CMAKE_CURRENT_SOURCE_DIR}/../server/grpc_protos/GrpcConnection.proto" ABSOLUTE) -protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${rpc_proto}) -PROTOBUF_GENERATE_GRPC_CPP(GRPC_SRCS GRPC_HDRS ${rpc_proto}) - -add_executable(grpc-client grpc_client.cpp ${PROTO_SRCS} ${PROTO_HDRS} ${GRPC_SRCS} ${GRPC_HDRS}) -target_link_libraries(grpc-client PUBLIC grpc++ PUBLIC libprotobuf PUBLIC daemon) \ No newline at end of file diff --git a/programs/grpc-client/grpc_client.cpp b/programs/grpc-client/grpc_client.cpp deleted file mode 100644 index 5345b3e7d33..00000000000 --- a/programs/grpc-client/grpc_client.cpp +++ /dev/null @@ -1,173 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "GrpcConnection.grpc.pb.h" - -class GRPCClient -{ - public: - explicit GRPCClient(std::shared_ptr channel) - : stub_(GRPCConnection::GRPC::NewStub(channel)) - {} - std::string Query(const GRPCConnection::User& userInfo, - const std::string& query, - std::vector insert_data = {}) - { - GRPCConnection::QueryRequest request; - grpc::Status status; - GRPCConnection::QueryResponse reply; - grpc::ClientContext context; - auto deadline = std::chrono::system_clock::now() + std::chrono::milliseconds(10000); - context.set_deadline(deadline); - - auto user = std::make_unique(userInfo); - auto querySettigs = std::make_unique(); - int id = rand(); - request.set_allocated_user_info(user.release()); - // interactive_delay in miliseconds - request.set_interactive_delay(1000); - - querySettigs->set_query(query); - querySettigs->set_format("Values"); - querySettigs->set_query_id(std::to_string(id)); - querySettigs->set_data_stream((insert_data.size() != 0)); - (*querySettigs->mutable_settings())["max_query_size"] ="100"; - - - request.set_allocated_query_info(querySettigs.release()); - - void* got_tag = (void*)1; - bool ok = false; - - std::unique_ptr > reader(stub_->Query(&context)); - reader->Write(request); - - auto write = [&reply, &reader, &insert_data]() - { - GRPCConnection::QueryRequest request_insert; - for (const auto& data : insert_data) - { - request_insert.set_insert_data(data); - if (reply.exception_occured().empty()) - { - reader->Write(request_insert); - } - else - { - break; - } - } - request_insert.set_insert_data(""); - if (reply.exception_occured().empty()) - { - reader->Write(request_insert); - } - // reader->WritesDone(); - }; - std::thread write_thread(write); - write_thread.detach(); - - while (reader->Read(&reply)) - { - - if (!reply.output().empty()) - { - std::cout << "Query Part:\n " << id<< reply.output()<<'\n'; - } - else if (reply.progress().read_rows() - || reply.progress().read_bytes() - || reply.progress().total_rows_to_read() - || reply.progress().written_rows() - || reply.progress().written_bytes()) - { - std::cout << "Progress " << id<< ":{\n" << "read_rows: " << reply.progress().read_rows() << '\n' - << "read_bytes: " << reply.progress().read_bytes() << '\n' - << "total_rows_to_read: " << reply.progress().total_rows_to_read() << '\n' - << "written_rows: " << reply.progress().written_rows() << '\n' - << "written_bytes: " << reply.progress().written_bytes() << '\n'; - - - } - else if (!reply.totals().empty()) - { - std::cout << "Totals:\n " << id << " " << reply.totals() <<'\n'; - } - else if (!reply.extremes().empty()) - { - std::cout << "Extremes:\n " << id << " " << reply.extremes() <<'\n'; - } - } - - if (status.ok() && reply.exception_occured().empty()) - { - return ""; - } - else if (status.ok() && !reply.exception_occured().empty()) - { - return reply.exception_occured(); - } - else - { - return "RPC failed"; - } - } - - private: - std::unique_ptr stub_; -}; - -int main(int argc, char** argv) -{ - GRPCConnection::User userInfo1; - userInfo1.set_user("default"); - userInfo1.set_password(""); - userInfo1.set_quota("default"); - - std::cout << "Try: " << argv[1] << std::endl; - grpc::ChannelArguments ch_args; - ch_args.SetMaxReceiveMessageSize(-1); - GRPCClient client( - grpc::CreateCustomChannel(argv[1], grpc::InsecureChannelCredentials(), ch_args)); - { - std::cout << client.Query(userInfo1, "CREATE TABLE t (a UInt8) ENGINE = Memory") << std::endl; - std::cout << client.Query(userInfo1, "CREATE TABLE t (a UInt8) ENGINE = Memory") << std::endl; - std::cout << client.Query(userInfo1, "INSERT INTO t VALUES", {"(1),(2),(3)", "(4),(6),(5)"}) << std::endl; - std::cout << client.Query(userInfo1, "INSERT INTO t_not_defined VALUES", {"(1),(2),(3)", "(4),(6),(5)"}) << std::endl; - std::cout << client.Query(userInfo1, "SELECT a FROM t ORDER BY a") << std::endl; - std::cout << client.Query(userInfo1, "DROP TABLE t") << std::endl; - } - { - std::cout << client.Query(userInfo1, "SELECT count() FROM numbers(1)") << std::endl; - std::cout << client.Query(userInfo1, "SELECT 100") << std::endl; - std::cout << client.Query(userInfo1, "SELECT count() FROM numbers(10000000000)") << std::endl; - std::cout << client.Query(userInfo1, "SELECT count() FROM numbers(100)") << std::endl; - } - { - std::cout << client.Query(userInfo1, "CREATE TABLE arrays_test (s String, arr Array(UInt8)) ENGINE = Memory;") << std::endl; - std::cout << client.Query(userInfo1, "INSERT INTO arrays_test VALUES ('Hello', [1,2]), ('World', [3,4,5]), ('Goodbye', []);") << std::endl; - std::cout << client.Query(userInfo1, "SELECT s FROM arrays_test") << std::endl; - std::cout << client.Query(userInfo1, "DROP TABLE arrays_test") << std::endl; - std::cout << client.Query(userInfo1, "") << std::endl; - } - - {//Check null return from pipe - std::cout << client.Query(userInfo1, "CREATE TABLE table2 (x UInt8, y UInt8) ENGINE = Memory;") << std::endl; - std::cout << client.Query(userInfo1, "SELECT x FROM table2") << std::endl; - std::cout << client.Query(userInfo1, "DROP TABLE table2") << std::endl; - } - {//Check Totals - std::cout << client.Query(userInfo1, "CREATE TABLE tabl (x UInt8, y UInt8) ENGINE = Memory;") << std::endl; - std::cout << client.Query(userInfo1, "INSERT INTO tabl VALUES (1, 2), (2, 4), (3, 2), (3, 3), (3, 4);") << std::endl; - std::cout << client.Query(userInfo1, "SELECT sum(x), y FROM tabl GROUP BY y WITH TOTALS") << std::endl; - std::cout << client.Query(userInfo1, "DROP TABLE tabl") << std::endl; - } - - return 0; -} From 1fc43b3c9324d7cde7003e4f0e1d53fffe3c143d Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 26 Nov 2020 10:27:04 +0300 Subject: [PATCH 5/5] Remove trailing whitespaces in config.xml --- programs/server/config.xml | 8 ++++---- .../test_grpc_protocol_ssl/configs/grpc_config.xml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index b1b185057e9..872b0182788 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -138,17 +138,17 @@ 9100 true --> - + - + - + - + diff --git a/tests/integration/test_grpc_protocol_ssl/configs/grpc_config.xml b/tests/integration/test_grpc_protocol_ssl/configs/grpc_config.xml index 76ef4d8c952..f40e73c5d99 100644 --- a/tests/integration/test_grpc_protocol_ssl/configs/grpc_config.xml +++ b/tests/integration/test_grpc_protocol_ssl/configs/grpc_config.xml @@ -2,17 +2,17 @@ 9100 true - + /etc/clickhouse-server/config.d/server-cert.pem /etc/clickhouse-server/config.d/server-key.pem - + true - + /etc/clickhouse-server/config.d/ca-cert.pem - + gzip