diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a42ba9a2ce..847c0085515 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,7 +89,6 @@ string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES) option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF) -option (ENABLE_FUZZER_TEST "Build testing fuzzers in order to test libFuzzer functionality" OFF) if (ENABLE_FUZZING) # Also set WITH_COVERAGE=1 for better fuzzing process diff --git a/base/harmful/harmful.c b/base/harmful/harmful.c index 19bb962999f..3b20b7f946f 100644 --- a/base/harmful/harmful.c +++ b/base/harmful/harmful.c @@ -65,7 +65,6 @@ TRAP(gethostbyaddr) TRAP(gethostbyname) TRAP(gethostbyname2) TRAP(gethostent) -TRAP(getlogin) TRAP(getnetbyaddr) TRAP(getnetbyname) TRAP(getnetent) @@ -291,6 +290,7 @@ TRAP(putenv) TRAP(setlogmask) TRAP(rand) TRAP(getmntent) +TRAP(getlogin) #endif #endif diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 3add371b30f..d6d92674a30 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -119,20 +119,26 @@ macro(clickhouse_program_add name) clickhouse_program_add_library(${name}) endmacro() -add_subdirectory (server) -add_subdirectory (client) -add_subdirectory (local) -add_subdirectory (benchmark) -add_subdirectory (extract-from-config) -add_subdirectory (compressor) -add_subdirectory (format) -add_subdirectory (obfuscator) -add_subdirectory (install) -add_subdirectory (git-import) add_subdirectory (bash-completion) +add_subdirectory (benchmark) +add_subdirectory (check-marks) +add_subdirectory (checksum-for-compressed-block) +add_subdirectory (client) +add_subdirectory (compressor) +add_subdirectory (disks) +add_subdirectory (extract-from-config) +add_subdirectory (format) +add_subdirectory (git-import) +add_subdirectory (install) +add_subdirectory (keeper-bench) +add_subdirectory (keeper-data-dumper) +add_subdirectory (local) +add_subdirectory (obfuscator) +add_subdirectory (server) add_subdirectory (static-files-disk-uploader) add_subdirectory (su) -add_subdirectory (disks) +add_subdirectory (zookeeper-dump-tree) +add_subdirectory (zookeeper-remove-by-list) if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER) add_subdirectory (keeper-converter) @@ -198,18 +204,30 @@ if (ENABLE_CLICKHOUSE_SELF_EXTRACTING) list(APPEND CLICKHOUSE_BUNDLE self-extracting) endif () -clickhouse_program_install(clickhouse-server server) -clickhouse_program_install(clickhouse-client client chc) -clickhouse_program_install(clickhouse-local local chl ch) clickhouse_program_install(clickhouse-benchmark benchmark) -clickhouse_program_install(clickhouse-extract-from-config extract-from-config) +clickhouse_program_install(clickhouse-check-marks check-marks) +clickhouse_program_install(clickhouse-checksum-for-compressed-block checksum-for-compressed-block) +clickhouse_program_install(clickhouse-client client chc) clickhouse_program_install(clickhouse-compressor compressor) -clickhouse_program_install(clickhouse-format format) -clickhouse_program_install(clickhouse-obfuscator obfuscator) -clickhouse_program_install(clickhouse-git-import git-import) -clickhouse_program_install(clickhouse-static-files-disk-uploader static-files-disk-uploader) clickhouse_program_install(clickhouse-disks disks) +clickhouse_program_install(clickhouse-extract-from-config extract-from-config) +clickhouse_program_install(clickhouse-format format) +clickhouse_program_install(clickhouse-git-import git-import) +clickhouse_program_install(clickhouse-local local chl ch) +clickhouse_program_install(clickhouse-obfuscator obfuscator) +clickhouse_program_install(clickhouse-server server) +clickhouse_program_install(clickhouse-static-files-disk-uploader static-files-disk-uploader) clickhouse_program_install(clickhouse-su su) +clickhouse_program_install(clickhouse-zookeeper-dump-tree zookeeper-dump-tree) +clickhouse_program_install(clickhouse-zookeeper-remove-by-list zookeeper-remove-by-list) + +if (TARGET ch_contrib::nuraft) + clickhouse_program_install(clickhouse-keeper-data-dumper keeper-data-dumper) +endif () + +if (TARGET ch_contrib::rapidjson AND TARGET ch_contrib::nuraft) + clickhouse_program_install(clickhouse-keeper-bench keeper-bench) +endif () if (ENABLE_CLICKHOUSE_KEEPER) if (NOT BUILD_STANDALONE_KEEPER AND CREATE_KEEPER_SYMLINK) diff --git a/programs/check-marks/CMakeLists.txt b/programs/check-marks/CMakeLists.txt new file mode 100644 index 00000000000..875dff8e976 --- /dev/null +++ b/programs/check-marks/CMakeLists.txt @@ -0,0 +1,11 @@ +set (CLICKHOUSE_CHECK_MARKS_SOURCES CheckMarks.cpp) + +set (CLICKHOUSE_CHECK_MARKS_LINK + PRIVATE + boost::program_options + clickhouse_aggregate_functions + clickhouse_common_config + dbms +) + +clickhouse_program_add(check-marks) diff --git a/utils/check-marks/main.cpp b/programs/check-marks/CheckMarks.cpp similarity index 97% rename from utils/check-marks/main.cpp rename to programs/check-marks/CheckMarks.cpp index 28ce0121f79..a3995449bf2 100644 --- a/utils/check-marks/main.cpp +++ b/programs/check-marks/CheckMarks.cpp @@ -54,7 +54,7 @@ static void checkByCompressedReadBuffer(const std::string & mrk_path, const std: } -int main(int argc, char ** argv) +int mainEntryClickHouseCheckMarks(int argc, char ** argv) { boost::program_options::options_description desc("Allowed options"); desc.add_options() diff --git a/programs/check-marks/check-marks.cpp b/programs/check-marks/check-marks.cpp new file mode 100644 index 00000000000..52b5bc8e85c --- /dev/null +++ b/programs/check-marks/check-marks.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseCheckMarks(int argc, char ** argv); diff --git a/programs/checksum-for-compressed-block/CMakeLists.txt b/programs/checksum-for-compressed-block/CMakeLists.txt new file mode 100644 index 00000000000..7ab5a4e6e38 --- /dev/null +++ b/programs/checksum-for-compressed-block/CMakeLists.txt @@ -0,0 +1,9 @@ +set (CLICKHOUSE_CHECKSUM_FOR_COMPRESSED_BLOCK_SOURCES ChecksumForCompressedBlock.cpp) + +set (CLICKHOUSE_CHECKSUM_FOR_COMPRESSED_BLOCK_LINK + PRIVATE + clickhouse_functions + dbms +) + +clickhouse_program_add(checksum-for-compressed-block) diff --git a/utils/checksum-for-compressed-block/main.cpp b/programs/checksum-for-compressed-block/ChecksumForCompressedBlock.cpp similarity index 97% rename from utils/checksum-for-compressed-block/main.cpp rename to programs/checksum-for-compressed-block/ChecksumForCompressedBlock.cpp index 4ae06a78ab4..adf74569b6b 100644 --- a/utils/checksum-for-compressed-block/main.cpp +++ b/programs/checksum-for-compressed-block/ChecksumForCompressedBlock.cpp @@ -34,7 +34,7 @@ std::string flipBit(std::string s, size_t pos) } -int main(int, char **) +int mainEntryClickHouseChecksumForCompressedBlock(int, char **) { using namespace DB; ReadBufferFromFileDescriptor in(STDIN_FILENO); diff --git a/programs/checksum-for-compressed-block/checksum-for-compressed-block.cpp b/programs/checksum-for-compressed-block/checksum-for-compressed-block.cpp new file mode 100644 index 00000000000..a3b72c42ca8 --- /dev/null +++ b/programs/checksum-for-compressed-block/checksum-for-compressed-block.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseChecksumForCompressedBlock(int, char **); diff --git a/programs/keeper-bench/CMakeLists.txt b/programs/keeper-bench/CMakeLists.txt new file mode 100644 index 00000000000..76aff16b667 --- /dev/null +++ b/programs/keeper-bench/CMakeLists.txt @@ -0,0 +1,20 @@ +if (NOT TARGET ch_contrib::rapidjson OR NOT TARGET ch_contrib::nuraft) + message (WARNING "Not building keeper-bench due to rapidjson or nuraft is disabled") + return() +endif () + +set (CLICKHOUSE_KEEPER_BENCH_SOURCES + Generator.cpp + Runner.cpp + Stats.cpp + KeeperBench.cpp +) + +set (CLICKHOUSE_KEEPER_BENCH_LINK + PRIVATE + dbms + clickhouse_functions + ch_contrib::rapidjson +) + +clickhouse_program_add(keeper-bench) diff --git a/utils/keeper-bench/Generator.cpp b/programs/keeper-bench/Generator.cpp similarity index 100% rename from utils/keeper-bench/Generator.cpp rename to programs/keeper-bench/Generator.cpp diff --git a/utils/keeper-bench/Generator.h b/programs/keeper-bench/Generator.h similarity index 100% rename from utils/keeper-bench/Generator.h rename to programs/keeper-bench/Generator.h diff --git a/utils/keeper-bench/main.cpp b/programs/keeper-bench/KeeperBench.cpp similarity index 98% rename from utils/keeper-bench/main.cpp rename to programs/keeper-bench/KeeperBench.cpp index 0b963abf406..62f961fdc04 100644 --- a/utils/keeper-bench/main.cpp +++ b/programs/keeper-bench/KeeperBench.cpp @@ -20,7 +20,7 @@ std::optional valueToOptional(const boost::program_options::variable_value & } -int main(int argc, char *argv[]) +int mainEntryClickHouseKeeperBench(int argc, char ** argv) { bool print_stacktrace = true; diff --git a/utils/keeper-bench/README.md b/programs/keeper-bench/README.md similarity index 100% rename from utils/keeper-bench/README.md rename to programs/keeper-bench/README.md diff --git a/utils/keeper-bench/Runner.cpp b/programs/keeper-bench/Runner.cpp similarity index 100% rename from utils/keeper-bench/Runner.cpp rename to programs/keeper-bench/Runner.cpp diff --git a/utils/keeper-bench/Runner.h b/programs/keeper-bench/Runner.h similarity index 100% rename from utils/keeper-bench/Runner.h rename to programs/keeper-bench/Runner.h diff --git a/utils/keeper-bench/Stats.cpp b/programs/keeper-bench/Stats.cpp similarity index 100% rename from utils/keeper-bench/Stats.cpp rename to programs/keeper-bench/Stats.cpp diff --git a/utils/keeper-bench/Stats.h b/programs/keeper-bench/Stats.h similarity index 100% rename from utils/keeper-bench/Stats.h rename to programs/keeper-bench/Stats.h diff --git a/utils/keeper-bench/example.yaml b/programs/keeper-bench/example.yaml similarity index 100% rename from utils/keeper-bench/example.yaml rename to programs/keeper-bench/example.yaml diff --git a/programs/keeper-bench/keeper-bench.cpp b/programs/keeper-bench/keeper-bench.cpp new file mode 100644 index 00000000000..b9d9c12a2a5 --- /dev/null +++ b/programs/keeper-bench/keeper-bench.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseKeeperBench(int argc, char ** argv); diff --git a/programs/keeper-data-dumper/CMakeLists.txt b/programs/keeper-data-dumper/CMakeLists.txt new file mode 100644 index 00000000000..024645a1543 --- /dev/null +++ b/programs/keeper-data-dumper/CMakeLists.txt @@ -0,0 +1,14 @@ +if (NOT TARGET ch_contrib::nuraft) + message (WARNING "Not building keeper-data-dumper due to nuraft is disabled") + return () +endif () + +set (CLICKHOUSE_KEEPER_DATA_DUMPER_SOURCES KeeperDataDumper.cpp) + +set (CLICKHOUSE_KEEPER_DATA_DUMPER_LINK + PRIVATE + clickhouse_functions + dbms +) + +clickhouse_program_add(keeper-data-dumper) diff --git a/utils/keeper-data-dumper/main.cpp b/programs/keeper-data-dumper/KeeperDataDumper.cpp similarity index 98% rename from utils/keeper-data-dumper/main.cpp rename to programs/keeper-data-dumper/KeeperDataDumper.cpp index 161115c44f6..71614a4972f 100644 --- a/utils/keeper-data-dumper/main.cpp +++ b/programs/keeper-data-dumper/KeeperDataDumper.cpp @@ -54,7 +54,7 @@ void dumpMachine(std::shared_ptr> ma std::cout << std::flush; } -int main(int argc, char *argv[]) +int mainEntryClickHouseKeeperDataDumper(int argc, char ** argv) { if (argc != 3) { diff --git a/programs/keeper-data-dumper/keeper-data-dumper.cpp b/programs/keeper-data-dumper/keeper-data-dumper.cpp new file mode 100644 index 00000000000..6c839764e14 --- /dev/null +++ b/programs/keeper-data-dumper/keeper-data-dumper.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseKeeperDataDumper(int argc, char ** argv); diff --git a/programs/main.cpp b/programs/main.cpp index 77edad28160..2a7a2fde3cc 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -20,18 +20,22 @@ #include /// Universal executable for various clickhouse applications -int mainEntryClickHouseServer(int argc, char ** argv); -int mainEntryClickHouseClient(int argc, char ** argv); -int mainEntryClickHouseLocal(int argc, char ** argv); int mainEntryClickHouseBenchmark(int argc, char ** argv); -int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); +int mainEntryClickHouseCheckMarks(int argc, char ** argv); +int mainEntryClickHouseChecksumForCompressedBlock(int, char **); +int mainEntryClickHouseClient(int argc, char ** argv); int mainEntryClickHouseCompressor(int argc, char ** argv); -int mainEntryClickHouseFormat(int argc, char ** argv); -int mainEntryClickHouseObfuscator(int argc, char ** argv); -int mainEntryClickHouseGitImport(int argc, char ** argv); -int mainEntryClickHouseStaticFilesDiskUploader(int argc, char ** argv); -int mainEntryClickHouseSU(int argc, char ** argv); int mainEntryClickHouseDisks(int argc, char ** argv); +int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); +int mainEntryClickHouseFormat(int argc, char ** argv); +int mainEntryClickHouseGitImport(int argc, char ** argv); +int mainEntryClickHouseLocal(int argc, char ** argv); +int mainEntryClickHouseObfuscator(int argc, char ** argv); +int mainEntryClickHouseSU(int argc, char ** argv); +int mainEntryClickHouseServer(int argc, char ** argv); +int mainEntryClickHouseStaticFilesDiskUploader(int argc, char ** argv); +int mainEntryClickHouseZooKeeperDumpTree(int argc, char ** argv); +int mainEntryClickHouseZooKeeperRemoveByList(int argc, char ** argv); int mainEntryClickHouseHashBinary(int, char **) { @@ -50,6 +54,12 @@ int mainEntryClickHouseKeeperConverter(int argc, char ** argv); #if ENABLE_CLICKHOUSE_KEEPER_CLIENT int mainEntryClickHouseKeeperClient(int argc, char ** argv); #endif +#if USE_RAPIDJSON && USE_NURAFT +int mainEntryClickHouseKeeperBench(int argc, char ** argv); +#endif +#if USE_NURAFT +int mainEntryClickHouseKeeperDataDumper(int argc, char ** argv); +#endif // install int mainEntryClickHouseInstall(int argc, char ** argv); @@ -63,7 +73,10 @@ namespace using MainFunc = int (*)(int, char**); -/// Add an item here to register new application +/// Add an item here to register new application. +/// This list has a "priority" - e.g. we need to disambiguate clickhouse --format being +/// either clickouse-format or clickhouse-{local, client} --format. +/// Currently we will prefer the latter option. std::pair clickhouse_applications[] = { {"local", mainEntryClickHouseLocal}, @@ -79,6 +92,10 @@ std::pair clickhouse_applications[] = {"su", mainEntryClickHouseSU}, {"hash-binary", mainEntryClickHouseHashBinary}, {"disks", mainEntryClickHouseDisks}, + {"check-marks", mainEntryClickHouseCheckMarks}, + {"checksum-for-compressed-block", mainEntryClickHouseChecksumForCompressedBlock}, + {"zookeeper-dump-tree", mainEntryClickHouseZooKeeperDumpTree}, + {"zookeeper-remove-by-list", mainEntryClickHouseZooKeeperRemoveByList}, // keeper #if ENABLE_CLICKHOUSE_KEEPER @@ -90,7 +107,12 @@ std::pair clickhouse_applications[] = #if ENABLE_CLICKHOUSE_KEEPER_CLIENT {"keeper-client", mainEntryClickHouseKeeperClient}, #endif - +#if USE_RAPIDJSON && USE_NURAFT + {"keeper-bench", mainEntryClickHouseKeeperBench}, +#endif +#if USE_NURAFT + {"keeper-data-dumper", mainEntryClickHouseKeeperDataDumper}, +#endif // install {"install", mainEntryClickHouseInstall}, {"start", mainEntryClickHouseStart}, diff --git a/programs/zookeeper-dump-tree/CMakeLists.txt b/programs/zookeeper-dump-tree/CMakeLists.txt new file mode 100644 index 00000000000..7f4164fca25 --- /dev/null +++ b/programs/zookeeper-dump-tree/CMakeLists.txt @@ -0,0 +1,10 @@ +set (CLICKHOUSE_ZOOKEEPER_DUMP_TREE_SOURCES ZooKeeperDumpTree.cpp) + +set (CLICKHOUSE_ZOOKEEPER_DUMP_TREE_LINK + PRIVATE + dbms + clickhouse_functions + boost::program_options +) + +clickhouse_program_add(zookeeper-dump-tree) diff --git a/utils/zookeeper-dump-tree/main.cpp b/programs/zookeeper-dump-tree/ZooKeeperDumpTree.cpp similarity index 98% rename from utils/zookeeper-dump-tree/main.cpp rename to programs/zookeeper-dump-tree/ZooKeeperDumpTree.cpp index 2d997f877b2..eb659b5abb6 100644 --- a/utils/zookeeper-dump-tree/main.cpp +++ b/programs/zookeeper-dump-tree/ZooKeeperDumpTree.cpp @@ -6,7 +6,7 @@ #include -int main(int argc, char ** argv) +int mainEntryClickHouseZooKeeperDumpTree(int argc, char ** argv) { try { diff --git a/programs/zookeeper-dump-tree/zookeeper-dump-tree.cpp b/programs/zookeeper-dump-tree/zookeeper-dump-tree.cpp new file mode 100644 index 00000000000..5de0ace40c4 --- /dev/null +++ b/programs/zookeeper-dump-tree/zookeeper-dump-tree.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseZooKeeperDumpTree(int argc, char ** argv); diff --git a/programs/zookeeper-remove-by-list/CMakeLists.txt b/programs/zookeeper-remove-by-list/CMakeLists.txt new file mode 100644 index 00000000000..5cc3488182c --- /dev/null +++ b/programs/zookeeper-remove-by-list/CMakeLists.txt @@ -0,0 +1,10 @@ +set (CLICKHOUSE_ZOOKEEPER_REMOVE_BY_LIST_SOURCES ZooKeeperRemoveByList.cpp) + +set (CLICKHOUSE_ZOOKEEPER_REMOVE_BY_LIST_LINK + PRIVATE + dbms + clickhouse_functions + boost::program_options +) + +clickhouse_program_add(zookeeper-remove-by-list) diff --git a/utils/zookeeper-remove-by-list/main.cpp b/programs/zookeeper-remove-by-list/ZooKeeperRemoveByList.cpp similarity index 96% rename from utils/zookeeper-remove-by-list/main.cpp rename to programs/zookeeper-remove-by-list/ZooKeeperRemoveByList.cpp index 4de300ca98a..05e55022bd0 100644 --- a/utils/zookeeper-remove-by-list/main.cpp +++ b/programs/zookeeper-remove-by-list/ZooKeeperRemoveByList.cpp @@ -5,7 +5,7 @@ #include -int main(int argc, char ** argv) +int mainEntryClickHouseZooKeeperRemoveByList(int argc, char ** argv) try { boost::program_options::options_description desc("Allowed options"); diff --git a/programs/zookeeper-remove-by-list/zookeeper-remove-by-list.cpp b/programs/zookeeper-remove-by-list/zookeeper-remove-by-list.cpp new file mode 100644 index 00000000000..f1f62d24098 --- /dev/null +++ b/programs/zookeeper-remove-by-list/zookeeper-remove-by-list.cpp @@ -0,0 +1 @@ +extern int mainEntryClickHouseZooKeeperRemoveByList(int argc, char ** argv); diff --git a/src/Compression/examples/CMakeLists.txt b/src/Compression/examples/CMakeLists.txt index 039eb203ad5..0d522ffd892 100644 --- a/src/Compression/examples/CMakeLists.txt +++ b/src/Compression/examples/CMakeLists.txt @@ -1,2 +1,5 @@ clickhouse_add_executable (compressed_buffer compressed_buffer.cpp) target_link_libraries (compressed_buffer PRIVATE dbms clickhouse_common_io clickhouse_common_config clickhouse_compression) + +clickhouse_add_executable (decompress_perf decompress_perf.cpp) +target_link_libraries(decompress_perf PRIVATE dbms clickhouse_functions ch_contrib::lz4) diff --git a/utils/compressor/decompress_perf.cpp b/src/Compression/examples/decompress_perf.cpp similarity index 97% rename from utils/compressor/decompress_perf.cpp rename to src/Compression/examples/decompress_perf.cpp index cb98121c024..b9a33105224 100644 --- a/utils/compressor/decompress_perf.cpp +++ b/src/Compression/examples/decompress_perf.cpp @@ -164,6 +164,17 @@ public: } +/* Usage example: + +#!/bin/sh + +./clickhouse-compressor < clickhouse-compressor > compressed +./clickhouse-compressor -d < compressed > clickhouse-compressor2 +cmp clickhouse-compressor clickhouse-compressor2 && echo "Ok." || echo "Fail." + +*/ + + int main(int argc, char ** argv) try { diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 2373a98239a..52fd77b271e 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -11,19 +11,6 @@ endif () # Not used in packages if (ENABLE_UTILS) - add_subdirectory (compressor) add_subdirectory (corrector_utf8) - add_subdirectory (zookeeper-cli) - add_subdirectory (zookeeper-dump-tree) - add_subdirectory (zookeeper-remove-by-list) - add_subdirectory (check-marks) - add_subdirectory (checksum-for-compressed-block) - add_subdirectory (check-mysql-binlog) - add_subdirectory (keeper-bench) - add_subdirectory (keeper-data-dumper) add_subdirectory (memcpy-bench) endif () - -if (ENABLE_FUZZING AND ENABLE_FUZZER_TEST) - add_subdirectory (libfuzzer-test) -endif () diff --git a/utils/check-marks/CMakeLists.txt b/utils/check-marks/CMakeLists.txt deleted file mode 100644 index 857dc51d40a..00000000000 --- a/utils/check-marks/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -clickhouse_add_executable (check-marks main.cpp) -target_link_libraries(check-marks PRIVATE dbms clickhouse_functions boost::program_options) diff --git a/utils/check-mysql-binlog/CMakeLists.txt b/utils/check-mysql-binlog/CMakeLists.txt deleted file mode 100644 index 0ece1684e6b..00000000000 --- a/utils/check-mysql-binlog/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -clickhouse_add_executable(check-mysql-binlog main.cpp) -target_link_libraries(check-mysql-binlog PRIVATE dbms clickhouse_functions boost::program_options) diff --git a/utils/check-mysql-binlog/main.cpp b/utils/check-mysql-binlog/main.cpp deleted file mode 100644 index 484dd46a90c..00000000000 --- a/utils/check-mysql-binlog/main.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include -#include -#include -#include - -bool quit = false; -void signal_handler(int) -{ - quit = true; -} - -static void processBinlogFromFile(const std::string & bin_path, bool disable_checksum) -{ - DB::MySQLReplication::BinlogFromFile binlog; - binlog.open(bin_path); - binlog.setChecksum(disable_checksum ? DB::MySQLReplication::IBinlog::NONE : DB::MySQLReplication::IBinlog::CRC32); - - DB::MySQLReplication::BinlogEventPtr event; - while (binlog.tryReadEvent(event, /*timeout*/ 0) && !quit) - { - DB::WriteBufferFromOStream cout(std::cout); - event->dump(cout); - binlog.getPosition().dump(cout); - cout.finalize(); - } -} - -static void processBinlogFromSocket(const std::string & host, int port, const std::string & user, const std::string & password, const std::string & executed_gtid_set, bool disable_checksum) -{ - DB::MySQLReplication::BinlogFromSocket binlog; - binlog.setChecksum(disable_checksum ? DB::MySQLReplication::IBinlog::NONE : DB::MySQLReplication::IBinlog::CRC32); - - binlog.connect(host, port, user, password); - binlog.start(/*unique number*/ 42, executed_gtid_set); - DB::MySQLReplication::BinlogEventPtr event; - - while (!quit) - { - if (binlog.tryReadEvent(event, /*timeout*/ 100)) - { - if (event->header.type != DB::MySQLReplication::HEARTBEAT_EVENT) - { - DB::WriteBufferFromOStream cout(std::cout); - event->dump(cout); - binlog.getPosition().dump(cout); - cout.finalize(); - } - } - } -} - -int main(int argc, char ** argv) -{ - (void)signal(SIGINT, signal_handler); - boost::program_options::options_description desc("Allowed options"); - - std::string host = "127.0.0.1"; - int port = 3306; - std::string user = "root"; - std::string password; - std::string gtid; - - desc.add_options() - ("help", "Produce help message") - ("disable_checksum", "Disable checksums in binlog files.") - ("binlog", boost::program_options::value(), "Binlog file") - ("host", boost::program_options::value(&host)->default_value(host), "Host to connect") - ("port", boost::program_options::value(&port)->default_value(port), "Port number to connect") - ("user", boost::program_options::value(&user)->default_value(user), "User") - ("password", boost::program_options::value(&password), "Password") - ("gtid", boost::program_options::value(>id), "Executed gtid set"); - - try - { - boost::program_options::variables_map options; - boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); - boost::program_options::notify(options); - - if (options.count("help") || (!options.count("binlog") && !options.count("gtid"))) - { - std::cout << "Usage: " << argv[0] << std::endl; - std::cout << desc << std::endl; - return EXIT_FAILURE; - } - - if (options.count("binlog")) - processBinlogFromFile(options["binlog"].as(), options.count("disable_checksum")); - else - processBinlogFromSocket(host, port, user, password, gtid, options.count("disable_checksum")); - } - catch (std::exception & ex) - { - std::cerr << ex.what() << std::endl; - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; -} diff --git a/utils/check-style/check-style b/utils/check-style/check-style index d86ab8e029d..901330e0015 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -14,7 +14,7 @@ LC_ALL="en_US.UTF-8" ROOT_PATH=$(git rev-parse --show-toplevel) -EXCLUDE='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/.*.cpp|utils/keeper-bench/example.yaml' +EXCLUDE='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/.*.cpp|programs/keeper-bench/example.yaml' EXCLUDE_DOCS='Settings\.cpp|FormatFactorySettings\.h' # From [1]: diff --git a/utils/checksum-for-compressed-block/CMakeLists.txt b/utils/checksum-for-compressed-block/CMakeLists.txt deleted file mode 100644 index 84378f6c1b7..00000000000 --- a/utils/checksum-for-compressed-block/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -clickhouse_add_executable (checksum-for-compressed-block-find-bit-flips main.cpp) -target_link_libraries(checksum-for-compressed-block-find-bit-flips PRIVATE dbms clickhouse_functions) diff --git a/utils/compressor/CMakeLists.txt b/utils/compressor/CMakeLists.txt deleted file mode 100644 index 81f981f8126..00000000000 --- a/utils/compressor/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -clickhouse_add_executable (decompress_perf decompress_perf.cpp) -target_link_libraries(decompress_perf PRIVATE dbms clickhouse_functions ch_contrib::lz4) diff --git a/utils/compressor/test.sh b/utils/compressor/test.sh deleted file mode 100755 index fec9102f072..00000000000 --- a/utils/compressor/test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -./clickhouse-compressor < clickhouse-compressor > compressed -./clickhouse-compressor -d < compressed > clickhouse-compressor2 -cmp clickhouse-compressor clickhouse-compressor2 && echo "Ok." || echo "Fail." diff --git a/utils/keeper-bench/CMakeLists.txt b/utils/keeper-bench/CMakeLists.txt deleted file mode 100644 index 4fe0d852fd2..00000000000 --- a/utils/keeper-bench/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -if (NOT TARGET ch_contrib::rapidjson) - message (${RECONFIGURE_MESSAGE_LEVEL} "Not building keeper-bench due to rapidjson is disabled") - return() -endif () - -clickhouse_add_executable(keeper-bench Generator.cpp Runner.cpp Stats.cpp main.cpp) -target_link_libraries(keeper-bench PRIVATE dbms clickhouse_functions ch_contrib::rapidjson) diff --git a/utils/keeper-data-dumper/CMakeLists.txt b/utils/keeper-data-dumper/CMakeLists.txt deleted file mode 100644 index 71f29781b73..00000000000 --- a/utils/keeper-data-dumper/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -if (NOT TARGET ch_contrib::nuraft) - message (WARNING "Not building keeper-data-dumper due to nuraft is disabled") - return () -endif () - -clickhouse_add_executable(keeper-data-dumper main.cpp) -target_link_libraries(keeper-data-dumper PRIVATE dbms clickhouse_functions) diff --git a/utils/libfuzzer-test/CMakeLists.txt b/utils/libfuzzer-test/CMakeLists.txt deleted file mode 100644 index 8765787ff8a..00000000000 --- a/utils/libfuzzer-test/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory (test_basic_fuzzer) diff --git a/utils/libfuzzer-test/README.md b/utils/libfuzzer-test/README.md deleted file mode 100644 index 5598cbdb961..00000000000 --- a/utils/libfuzzer-test/README.md +++ /dev/null @@ -1 +0,0 @@ -This folder contains various stuff intended to test libfuzzer functionality. diff --git a/utils/libfuzzer-test/test_basic_fuzzer/CMakeLists.txt b/utils/libfuzzer-test/test_basic_fuzzer/CMakeLists.txt deleted file mode 100644 index dc927f35a4b..00000000000 --- a/utils/libfuzzer-test/test_basic_fuzzer/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_executable (test_basic_fuzzer main.cpp) diff --git a/utils/libfuzzer-test/test_basic_fuzzer/main.cpp b/utils/libfuzzer-test/test_basic_fuzzer/main.cpp deleted file mode 100644 index 7ccad63273d..00000000000 --- a/utils/libfuzzer-test/test_basic_fuzzer/main.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include -#include - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) -{ - if (size > 0 && data[0] == 'H') - if (size > 1 && data[1] == 'I') - if (size > 2 && data[2] == '!') - __builtin_trap(); - return 0; -} diff --git a/utils/zookeeper-cli/CMakeLists.txt b/utils/zookeeper-cli/CMakeLists.txt deleted file mode 100644 index 2d0769b7bf2..00000000000 --- a/utils/zookeeper-cli/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -clickhouse_add_executable(clickhouse-zookeeper-cli - zookeeper-cli.cpp - ${ClickHouse_SOURCE_DIR}/src/Client/LineReader.cpp) -target_link_libraries(clickhouse-zookeeper-cli PRIVATE - clickhouse_common_zookeeper_base - clickhouse_common_zookeeper_no_log - dbms - clickhouse_functions -) diff --git a/utils/zookeeper-cli/zookeeper-cli.cpp b/utils/zookeeper-cli/zookeeper-cli.cpp deleted file mode 100644 index 4aef29847ce..00000000000 --- a/utils/zookeeper-cli/zookeeper-cli.cpp +++ /dev/null @@ -1,223 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include -#include - - -void printStat(const Coordination::Stat & s) -{ - std::cout << "Stat:\n"; - std::cout << " czxid: " << s.czxid << '\n'; - std::cout << " mzxid: " << s.mzxid << '\n'; - std::cout << " ctime: " << s.ctime << '\n'; - std::cout << " mtime: " << s.mtime << '\n'; - std::cout << " version: " << s.version << '\n'; - std::cout << " cversion: " << s.cversion << '\n'; - std::cout << " aversion: " << s.aversion << '\n'; - std::cout << " ephemeralOwner: " << s.ephemeralOwner << '\n'; - std::cout << " dataLength: " << s.dataLength << '\n'; - std::cout << " numChildren: " << s.numChildren << '\n'; - std::cout << " pzxid: " << s.pzxid << std::endl; -} - -void waitForWatch(const zkutil::EventPtr & event) -{ - std::cout << "waiting for watch" << std::endl; - event->wait(); - std::cout << "watch event was signalled" << std::endl; -} - - -void readUntilSpace(std::string & s, DB::ReadBuffer & buf) -{ - s = ""; - while (!buf.eof()) - { - if (isspace(*buf.position())) - return; - s.push_back(*buf.position()); - ++buf.position(); - } -} - -void readMaybeQuoted(std::string & s, DB::ReadBuffer & buf) -{ - if (!buf.eof() && *buf.position() == '\'') - DB::readQuotedString(s, buf); - else - readUntilSpace(s, buf); -} - - -int main(int argc, char ** argv) -{ - try - { - if (argc != 2) - { - std::cerr << "usage: " << argv[0] << " hosts" << std::endl; - return 2; - } - - Poco::AutoPtr channel = new Poco::ConsoleChannel(std::cerr); - Poco::Logger::root().setChannel(channel); - Poco::Logger::root().setLevel("trace"); - - auto zk = zkutil::ZooKeeper::createWithoutKillingPreviousSessions(zkutil::ZooKeeperArgs(argv[1])); - DB::LineReader lr({}, false, {"\\"}, {}); - - do - { - const auto & line = lr.readLine(":3 ", ":3 "); - if (line.empty()) - break; - - try - { - std::stringstream ss(line); // STYLE_CHECK_ALLOW_STD_STRING_STREAM - - std::string cmd; - ss >> cmd; - - if (cmd == "q" || cmd == "quit" || cmd == "exit" || cmd == ":q") - break; - - std::string path; - ss >> path; - if (cmd == "ls") - { - std::string w; - ss >> w; - bool watch = w == "w"; - zkutil::EventPtr event = watch ? std::make_shared() : nullptr; - std::vector v = zk->getChildren(path, nullptr, event); - for (const auto & child : v) - std::cout << child << std::endl; - if (watch) - waitForWatch(event); - } - else if (cmd == "create") - { - DB::ReadBufferFromString in(line); - - std::string path_ignored; - std::string data; - std::string mode; - - DB::assertString("create", in); - DB::skipWhitespaceIfAny(in); - readMaybeQuoted(path_ignored, in); - DB::skipWhitespaceIfAny(in); - readMaybeQuoted(data, in); - DB::skipWhitespaceIfAny(in); - readUntilSpace(mode, in); - - int32_t m; - if (mode == "p") - m = zkutil::CreateMode::Persistent; - else if (mode == "ps") - m = zkutil::CreateMode::PersistentSequential; - else if (mode == "e") - m = zkutil::CreateMode::Ephemeral; - else if (mode == "es") - m = zkutil::CreateMode::EphemeralSequential; - else - { - std::cout << "Bad create mode" << std::endl; - continue; - } - std::cout << zk->create(path, data, m) << std::endl; - } - else if (cmd == "remove") - { - zk->remove(path); - } - else if (cmd == "rmr") - { - zk->removeRecursive(path); - } - else if (cmd == "exists") - { - std::string w; - ss >> w; - bool watch = w == "w"; - zkutil::EventPtr event = watch ? std::make_shared() : nullptr; - Coordination::Stat stat; - bool e = zk->exists(path, &stat, event); - if (e) - printStat(stat); - else - std::cout << path << " does not exist" << std::endl; - if (watch) - waitForWatch(event); - } - else if (cmd == "get") - { - std::string w; - ss >> w; - bool watch = w == "w"; - zkutil::EventPtr event = watch ? std::make_shared() : nullptr; - Coordination::Stat stat; - std::string data = zk->get(path, &stat, event); - std::cout << "Data: " << data << std::endl; - printStat(stat); - if (watch) - waitForWatch(event); - } - else if (cmd == "set") - { - DB::ReadBufferFromString in(line); - - std::string data; - int version = -1; - - DB::assertString("set", in); - DB::skipWhitespaceIfAny(in); - DB::assertString(path, in); - DB::skipWhitespaceIfAny(in); - readMaybeQuoted(data, in); - DB::skipWhitespaceIfAny(in); - - if (!in.eof()) - DB::readText(version, in); - - Coordination::Stat stat; - zk->set(path, data, version, &stat); - printStat(stat); - } - else if (!cmd.empty()) - { - std::cout << "commands:\n"; - std::cout << " q\n"; - std::cout << " ls path [w]\n"; - std::cout << " create path data (p|ps|e|es)\n"; - std::cout << " remove path\n"; - std::cout << " rmr path\n"; - std::cout << " exists path [w]\n"; - std::cout << " get path [w]\n"; - std::cout << " set path data [version]" << std::endl; - continue; - } - - } - catch (const Coordination::Exception & e) - { - std::cerr << "KeeperException: " << e.displayText() << std::endl; - } - } - while (true); - } - catch (const Coordination::Exception & e) - { - std::cerr << "KeeperException: " << e.displayText() << std::endl; - return 1; - } - - return 0; -} diff --git a/utils/zookeeper-dump-tree/CMakeLists.txt b/utils/zookeeper-dump-tree/CMakeLists.txt deleted file mode 100644 index 835d37bd1cd..00000000000 --- a/utils/zookeeper-dump-tree/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -clickhouse_add_executable (zookeeper-dump-tree main.cpp ${SRCS}) -target_link_libraries(zookeeper-dump-tree PRIVATE - clickhouse_common_zookeeper_base - clickhouse_common_zookeeper_no_log - clickhouse_common_io - dbms - clickhouse_functions - boost::program_options) diff --git a/utils/zookeeper-remove-by-list/CMakeLists.txt b/utils/zookeeper-remove-by-list/CMakeLists.txt deleted file mode 100644 index 4365c716596..00000000000 --- a/utils/zookeeper-remove-by-list/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -clickhouse_add_executable (zookeeper-remove-by-list main.cpp ${SRCS}) -target_link_libraries(zookeeper-remove-by-list PRIVATE - clickhouse_common_zookeeper_base - clickhouse_common_zookeeper_no_log - dbms - clickhouse_functions - boost::program_options)