mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 00:30:49 +00:00
Merge pull request #42972 from ClickHouse/remove-some-utils
Remove some utils
This commit is contained in:
Sema Checherinda
GitHub
commit
8860550e87
@ -20,17 +20,13 @@ add_subdirectory (report)
|
||||
# Not used in package
|
||||
if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
|
||||
add_subdirectory (compressor)
|
||||
add_subdirectory (iotest)
|
||||
add_subdirectory (corrector_utf8)
|
||||
add_subdirectory (zookeeper-cli)
|
||||
add_subdirectory (zookeeper-dump-tree)
|
||||
add_subdirectory (zookeeper-remove-by-list)
|
||||
add_subdirectory (zookeeper-create-entry-to-download-part)
|
||||
add_subdirectory (zookeeper-adjust-block-numbers-to-parts)
|
||||
add_subdirectory (wikistat-loader)
|
||||
add_subdirectory (check-marks)
|
||||
add_subdirectory (checksum-for-compressed-block)
|
||||
add_subdirectory (db-generator)
|
||||
add_subdirectory (wal-dump)
|
||||
add_subdirectory (check-mysql-binlog)
|
||||
add_subdirectory (keeper-bench)
|
||||
|
||||
@ -1,2 +0,0 @@
|
||||
clickhouse_add_executable (query_db_generator query_db_generator.cpp)
|
||||
target_link_libraries(query_db_generator PRIVATE clickhouse_parsers boost::program_options)
|
||||
@ -1,35 +0,0 @@
|
||||
# Clickhouse query analysis
|
||||
|
||||
Here we will consider only `SELECT` queries, i.e. those queries that get data from the table.
|
||||
The built-in Clickhouse parser accepts a string as input, which is a query. Among 14 main clauses of `SELECT` statement: `WITH`, `SELECT`, `TABLES`, `PREWHERE`, `WHERE`, `GROUP_BY`, `HAVING`, `ORDER_BY`, `LIMIT_BY_OFFSET`, `LIMIT_BY_LENGTH`, `LIMIT_BY`, `LIMIT_OFFSET`, `LIMIT_LENGTH`, `SETTINGS`, we will analyze the `SELECT`, `TABLES`, `WHERE`, `GROUP_BY`, `HAVING`, `ORDER_BY` clauses because the most of data is there. We need this data to analyze the structure and to identify values. The parser issues a tree structure after parsing a query, where each node is a specific query execution operation, a function over values, a constant, a designation, etc. Nodes also have subtrees where their arguments or suboperations are located. We will try to reveal the data we need by avoiding this tree.
|
||||
|
||||
## Scheme analysis
|
||||
|
||||
It is necessary to determine possible tables by a query. Having a query string, you can understand which parts of it represent the names of the tables, so you can determine their number in our database.
|
||||
In the Clickhouse parser, `TABLES` (Figure 1) is a query subtree responsible for tables where we get data. It contains the main table where the columns come from, as well as the `JOIN` operations that are performed in the query. Avoiding all nodes in the subtree, we use the names of the tables and databases where they are located, as well as their alias, i.e. the shortened names chosen by the query author. We may need these names to determine the ownership of the column in the future.
|
||||
Thus, we get a set of databases for the query, as well as tables and their aliases, with the help of them a query is made.
|
||||
|
||||
Then we need to define the set of columns that are in the query and the tables they can refer to. The set of columns in each table is already known during the query execution. Therefore, the program automatically links the column and table at runtime. However, in our case, it is impossible to unambiguously interpret the belonging of a column to a specific table, for example, in the following query `SELECT column1, column2, column3 FROM table1 JOIN table2 on table1.column2 = table2.column3`. In this case, we can say which table `column2` and `column3` belong to. However, `column1` can belong to either the first or the second table. We will refer undefined columns to the main table, on which a query is made, for unambiguous interpretation of such cases. For example, in this case, it will be `table1`.
|
||||
All columns in the tree are in `IDENTIFIER` type nodes, which are in the `SELECT`, `TABLES`, `WHERE`, `GROUP_BY`, `HAVING`, `ORDER_BY` subtrees. We form a set of all tables recursively avoiding the subtrees, then we split the column into constituents such as the table (if it is explicitly specified with a dot) and the name. Then, since the table can be an alias, we replace the alias with the original table name. We now have a list of all the columns and tables they belong to. We define the main query table for non-table columns.
|
||||
|
||||
## Column analysis
|
||||
|
||||
Then we need to exactly define data types for columns that have a value in the query. An example is the boolean `WHERE` clause where we test boolean expressions in its attributes. If the query specifies `column > 5`, then we can conclude that this column contains a numeric value, or if the `LIKE` expression is applied to the attribute, then the attribute has a string type.
|
||||
In this part, you need to learn how to extract such expressions from a query and match data types for columns, where it is possible. At the same time, it is clear that it is not always possible to make an unambiguous decision about the type of a particular attribute from the available values. For example, `column > 5` can mean many numeric types such as `UINT8`, `UINT32`, `INT32`, `INT64`, etc. It is necessary to determine the interpretation of certain values since searching through all possible values can be quite large and long.
|
||||
It can take a long time to iterate over all possible values, so we use `INT64` and `FLOAT64` types for numeric values, `STRING` for strings, `DATE` and `DATETIME` for dates, and `ARRAY`.
|
||||
We can determine column values using boolean, arithmetic and other functions on the column values that are specified in the query. Such functions are in the `SELECT` and `WHERE` subtrees. The function parameter can be a constant, a column or another function (Figure 2). Thus, the following parameters can help to understand the type of the column:
|
||||
- The types of arguments that a function can take, for example, the `TOSTARTOFMINUTE` function (truncate time up to a multiple of 5 minutes down) can only accept `DATETIME`, so if the argument of this function is a column, then this column has `DATETIME` type.
|
||||
- The types of the remaining arguments in this function. For example, the `EQUALS` function means equality of its argument types, so if a constant and a column are present in this function, then we can define the type of the column as the type of the constant.
|
||||
|
||||
Thus, we define the possible argument types, the return type, the parameter for each function, and the function arguments of the identical type. The recursive function handler will determine the possible types of columns used in these functions by the values of the arguments, and then return the possible types of the function's result.
|
||||
Now, for each column, we have many possible types of values. We will choose one specific type from this set to interpret the query unambiguously.
|
||||
|
||||
## Column values definition
|
||||
|
||||
At this stage, we already have a certain structure of the database tables, we need to fill this table with values. We should understand which columns depend on each other when executing the function (for example, the join is done according to two columns, which means that they must have the same values). We also need to understand what values the columns must have to fulfill various conditions during execution.
|
||||
We search for all comparison operations in our query to achieve the goal. If the arguments of the operation are two columns, then we consider them linked. If the arguments are the column and the value, then we assign that value to the possible column value and add the value with some noise. A random number is a noise for a numeric type, it is a random number of days for a date, etc. In this case, a handler for this operation is required for each comparison operation, which generates at least two values, one of them is the operation condition, and the other is not. For example, a value greater than 5 and less than or equal to 5 must be assigned for the operation `column1 > 5`, `column1`, for the operation `column2 LIKE some% string` the same is true. The satisfying and not satisfying expression must be assigned to `column2`.
|
||||
Now we have many associated columns and many values. We know that the connectivity of columns is symmetric, but we need to add transitivity for a complete definition, because if `column1 = column2` and `column2 = column3`, then `column1 = column3`, but this does not follow from the construction. Accordingly, we need to extend the connectivity across all columns. We combine multiple values for each column with the values associated with it. If we have columns with no values, then we generate random values.
|
||||
|
||||
## Generation
|
||||
|
||||
We have a complete view of the database schema as well as many values for each table now. We will generate data by cartesian product of the value set of each column for a specific table. Thus, we get a set for each table, consisting of sets of values for each column. We start generating queries that create this table and fill it with data. We generate the `CREATE QUERY` that creates this table based on the structure of the table and the types of its columns, and then we generate the `INSERT QUERY` over the set of values, which fills the table with data.
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,9 +0,0 @@
|
||||
|
||||
clickhouse_add_executable (iotest iotest.cpp ${SRCS})
|
||||
target_link_libraries (iotest PRIVATE clickhouse_common_io)
|
||||
|
||||
clickhouse_add_executable (iotest_nonblock iotest_nonblock.cpp ${SRCS})
|
||||
target_link_libraries (iotest_nonblock PRIVATE clickhouse_common_io)
|
||||
|
||||
clickhouse_add_executable (iotest_aio iotest_aio.cpp ${SRCS})
|
||||
target_link_libraries (iotest_aio PRIVATE clickhouse_common_io)
|
||||
@ -1,197 +0,0 @@
|
||||
#include <IO/BufferWithOwnMemory.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <pcg_random.hpp>
|
||||
#include <Poco/Exception.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <base/getPageSize.h>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <ctime>
|
||||
#include <unistd.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_OPEN_FILE;
|
||||
extern const int CANNOT_CLOSE_FILE;
|
||||
extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;
|
||||
extern const int CANNOT_WRITE_TO_FILE_DESCRIPTOR;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
enum Mode
|
||||
{
|
||||
MODE_NONE = 0,
|
||||
MODE_READ = 1,
|
||||
MODE_WRITE = 2,
|
||||
MODE_ALIGNED = 4,
|
||||
MODE_DIRECT = 8,
|
||||
MODE_SYNC = 16,
|
||||
};
|
||||
|
||||
|
||||
void thread(int fd, int mode, size_t min_offset, size_t max_offset, size_t block_size, size_t count)
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
Memory<> direct_buf(block_size, ::getPageSize());
|
||||
std::vector<char> simple_buf(block_size);
|
||||
|
||||
char * buf;
|
||||
if ((mode & MODE_DIRECT))
|
||||
buf = direct_buf.data();
|
||||
else
|
||||
buf = simple_buf.data();
|
||||
|
||||
pcg64 rng(randomSeed());
|
||||
|
||||
for (size_t i = 0; i < count; ++i)
|
||||
{
|
||||
uint64_t rand_result1 = rng();
|
||||
uint64_t rand_result2 = rng();
|
||||
uint64_t rand_result3 = rng();
|
||||
|
||||
size_t rand_result = rand_result1 ^ (rand_result2 << 22) ^ (rand_result3 << 43);
|
||||
size_t offset;
|
||||
if ((mode & MODE_DIRECT) || (mode & MODE_ALIGNED))
|
||||
offset = min_offset + rand_result % ((max_offset - min_offset) / block_size) * block_size;
|
||||
else
|
||||
offset = min_offset + rand_result % (max_offset - min_offset - block_size + 1);
|
||||
|
||||
if (mode & MODE_READ)
|
||||
{
|
||||
if (static_cast<int>(block_size) != pread(fd, buf, block_size, offset))
|
||||
throwFromErrno("Cannot read", ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (static_cast<int>(block_size) != pwrite(fd, buf, block_size, offset))
|
||||
throwFromErrno("Cannot write", ErrorCodes::CANNOT_WRITE_TO_FILE_DESCRIPTOR);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int mainImpl(int argc, char ** argv)
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
const char * file_name = nullptr;
|
||||
int mode = MODE_NONE;
|
||||
UInt64 min_offset = 0;
|
||||
UInt64 max_offset = 0;
|
||||
UInt64 block_size = 0;
|
||||
UInt64 threads = 0;
|
||||
UInt64 count = 0;
|
||||
|
||||
if (argc != 8)
|
||||
{
|
||||
std::cerr << "Usage: " << argv[0] << " file_name (r|w)[a][d][s] min_offset max_offset block_size threads count" << std::endl <<
|
||||
"a - aligned, d - direct, s - sync" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
file_name = argv[1];
|
||||
min_offset = parse<UInt64>(argv[3]);
|
||||
max_offset = parse<UInt64>(argv[4]);
|
||||
block_size = parse<UInt64>(argv[5]);
|
||||
threads = parse<UInt64>(argv[6]);
|
||||
count = parse<UInt64>(argv[7]);
|
||||
|
||||
for (int i = 0; argv[2][i]; ++i)
|
||||
{
|
||||
char c = argv[2][i];
|
||||
switch (c)
|
||||
{
|
||||
case 'r':
|
||||
mode |= MODE_READ;
|
||||
break;
|
||||
case 'w':
|
||||
mode |= MODE_WRITE;
|
||||
break;
|
||||
case 'a':
|
||||
mode |= MODE_ALIGNED;
|
||||
break;
|
||||
case 'd':
|
||||
mode |= MODE_DIRECT;
|
||||
break;
|
||||
case 's':
|
||||
mode |= MODE_SYNC;
|
||||
break;
|
||||
default:
|
||||
throw Poco::Exception("Invalid mode");
|
||||
}
|
||||
}
|
||||
|
||||
ThreadPool pool(threads);
|
||||
|
||||
#ifndef OS_DARWIN
|
||||
int fd = open(file_name, ((mode & MODE_READ) ? O_RDONLY : O_WRONLY) | ((mode & MODE_DIRECT) ? O_DIRECT : 0) | ((mode & MODE_SYNC) ? O_SYNC : 0));
|
||||
#else
|
||||
int fd = open(file_name, ((mode & MODE_READ) ? O_RDONLY : O_WRONLY) | ((mode & MODE_SYNC) ? O_SYNC : 0));
|
||||
#endif
|
||||
if (-1 == fd)
|
||||
throwFromErrno("Cannot open file", ErrorCodes::CANNOT_OPEN_FILE);
|
||||
#ifdef OS_DARWIN
|
||||
if (mode & MODE_DIRECT)
|
||||
if (fcntl(fd, F_NOCACHE, 1) == -1)
|
||||
throwFromErrno("Cannot open file", ErrorCodes::CANNOT_CLOSE_FILE);
|
||||
#endif
|
||||
Stopwatch watch;
|
||||
|
||||
for (size_t i = 0; i < threads; ++i)
|
||||
pool.scheduleOrThrowOnError([=]{ thread(fd, mode, min_offset, max_offset, block_size, count); });
|
||||
pool.wait();
|
||||
|
||||
#if defined(OS_DARWIN)
|
||||
fsync(fd);
|
||||
#else
|
||||
fdatasync(fd);
|
||||
#endif
|
||||
|
||||
watch.stop();
|
||||
|
||||
if (0 != close(fd))
|
||||
throwFromErrno("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE);
|
||||
|
||||
std::cout << std::fixed << std::setprecision(2)
|
||||
<< "Done " << count << " * " << threads << " ops";
|
||||
if (mode & MODE_ALIGNED)
|
||||
std::cout << " (aligned)";
|
||||
if (mode & MODE_DIRECT)
|
||||
std::cout << " (direct)";
|
||||
if (mode & MODE_SYNC)
|
||||
std::cout << " (sync)";
|
||||
std::cout << " in " << watch.elapsedSeconds() << " sec."
|
||||
<< ", " << count * threads / watch.elapsedSeconds() << " ops/sec."
|
||||
<< ", " << count * threads * block_size / watch.elapsedSeconds() / 1000000 << " MB/sec."
|
||||
<< std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
try
|
||||
{
|
||||
return mainImpl(argc, argv);
|
||||
}
|
||||
catch (const Poco::Exception & e)
|
||||
{
|
||||
std::cerr << e.what() << ", " << e.message() << std::endl;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@ -1,203 +0,0 @@
|
||||
#if !defined(OS_LINUX)
|
||||
int main(int, char **) { return 0; }
|
||||
#else
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <cstdlib>
|
||||
#include <ctime>
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
#include <vector>
|
||||
#include <Poco/Exception.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <base/getPageSize.h>
|
||||
#include <pcg_random.hpp>
|
||||
#include <IO/BufferWithOwnMemory.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <cstdio>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <IO/AIO.h>
|
||||
#include <malloc.h>
|
||||
#include <sys/syscall.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_OPEN_FILE;
|
||||
extern const int CANNOT_CLOSE_FILE;
|
||||
extern const int CANNOT_IO_SUBMIT;
|
||||
extern const int CANNOT_IO_GETEVENTS;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
enum Mode
|
||||
{
|
||||
MODE_READ = 1,
|
||||
MODE_WRITE = 2,
|
||||
};
|
||||
|
||||
|
||||
void thread(int fd, int mode, size_t min_offset, size_t max_offset, size_t block_size, size_t buffers_count, size_t count)
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
AIOContext ctx;
|
||||
|
||||
std::vector<Memory<>> buffers(buffers_count);
|
||||
for (size_t i = 0; i < buffers_count; ++i)
|
||||
buffers[i] = Memory<>(block_size, ::getPageSize());
|
||||
|
||||
pcg64_fast rng(randomSeed());
|
||||
|
||||
size_t in_progress = 0;
|
||||
size_t blocks_sent = 0;
|
||||
std::vector<bool> buffer_used(buffers_count, false);
|
||||
std::vector<iocb> iocbs(buffers_count);
|
||||
std::vector<iocb*> query_cbs;
|
||||
std::vector<io_event> events(buffers_count);
|
||||
|
||||
while (blocks_sent < count || in_progress > 0)
|
||||
{
|
||||
/// Prepare queries.
|
||||
query_cbs.clear();
|
||||
for (size_t i = 0; i < buffers_count; ++i)
|
||||
{
|
||||
if (blocks_sent >= count || in_progress >= buffers_count)
|
||||
break;
|
||||
|
||||
if (buffer_used[i])
|
||||
continue;
|
||||
|
||||
buffer_used[i] = true;
|
||||
++blocks_sent;
|
||||
++in_progress;
|
||||
|
||||
char * buf = buffers[i].data();
|
||||
|
||||
uint64_t rand_result1 = rng();
|
||||
uint64_t rand_result2 = rng();
|
||||
uint64_t rand_result3 = rng();
|
||||
|
||||
size_t rand_result = rand_result1 ^ (rand_result2 << 22) ^ (rand_result3 << 43);
|
||||
size_t offset = min_offset + rand_result % ((max_offset - min_offset) / block_size) * block_size;
|
||||
|
||||
iocb & cb = iocbs[i];
|
||||
memset(&cb, 0, sizeof(cb));
|
||||
cb.aio_buf = reinterpret_cast<UInt64>(buf);
|
||||
cb.aio_fildes = fd;
|
||||
cb.aio_nbytes = block_size;
|
||||
cb.aio_offset = offset;
|
||||
cb.aio_data = static_cast<UInt64>(i);
|
||||
|
||||
if (mode == MODE_READ)
|
||||
{
|
||||
cb.aio_lio_opcode = IOCB_CMD_PREAD;
|
||||
}
|
||||
else
|
||||
{
|
||||
cb.aio_lio_opcode = IOCB_CMD_PWRITE;
|
||||
}
|
||||
|
||||
query_cbs.push_back(&cb);
|
||||
}
|
||||
|
||||
/// Send queries.
|
||||
if (io_submit(ctx.ctx, query_cbs.size(), query_cbs.data()) < 0)
|
||||
throwFromErrno("io_submit failed", ErrorCodes::CANNOT_IO_SUBMIT);
|
||||
|
||||
/// Receive answers. If we have something else to send, then receive at least one answer (after that send them), otherwise wait all answers.
|
||||
memset(events.data(), 0, buffers_count * sizeof(events[0]));
|
||||
int evs = io_getevents(ctx.ctx, (blocks_sent < count ? 1 : in_progress), buffers_count, events.data(), nullptr);
|
||||
if (evs < 0)
|
||||
throwFromErrno("io_getevents failed", ErrorCodes::CANNOT_IO_GETEVENTS);
|
||||
|
||||
for (int i = 0; i < evs; ++i)
|
||||
{
|
||||
int b = static_cast<int>(events[i].data);
|
||||
if (events[i].res != static_cast<int>(block_size))
|
||||
throw Poco::Exception("read/write error");
|
||||
--in_progress;
|
||||
buffer_used[b] = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int mainImpl(int argc, char ** argv)
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
const char * file_name = nullptr;
|
||||
int mode = MODE_READ;
|
||||
UInt64 min_offset = 0;
|
||||
UInt64 max_offset = 0;
|
||||
UInt64 block_size = 0;
|
||||
UInt64 buffers_count = 0;
|
||||
UInt64 threads_count = 0;
|
||||
UInt64 count = 0;
|
||||
|
||||
if (argc != 9)
|
||||
{
|
||||
std::cerr << "Usage: " << argv[0] << " file_name r|w min_offset max_offset block_size threads buffers count" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
file_name = argv[1];
|
||||
if (argv[2][0] == 'w')
|
||||
mode = MODE_WRITE;
|
||||
min_offset = parse<UInt64>(argv[3]);
|
||||
max_offset = parse<UInt64>(argv[4]);
|
||||
block_size = parse<UInt64>(argv[5]);
|
||||
threads_count = parse<UInt64>(argv[6]);
|
||||
buffers_count = parse<UInt64>(argv[7]);
|
||||
count = parse<UInt64>(argv[8]);
|
||||
|
||||
int fd = open(file_name, ((mode == MODE_READ) ? O_RDONLY : O_WRONLY) | O_DIRECT);
|
||||
if (-1 == fd)
|
||||
throwFromErrno("Cannot open file", ErrorCodes::CANNOT_OPEN_FILE);
|
||||
|
||||
ThreadPool pool(threads_count);
|
||||
|
||||
Stopwatch watch;
|
||||
|
||||
for (size_t i = 0; i < threads_count; ++i)
|
||||
pool.scheduleOrThrowOnError([=]{ thread(fd, mode, min_offset, max_offset, block_size, buffers_count, count); });
|
||||
pool.wait();
|
||||
|
||||
watch.stop();
|
||||
|
||||
if (0 != close(fd))
|
||||
throwFromErrno("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE);
|
||||
|
||||
std::cout << std::fixed << std::setprecision(2)
|
||||
<< "Done " << count << " * " << threads_count << " ops";
|
||||
std::cout << " in " << watch.elapsedSeconds() << " sec."
|
||||
<< ", " << count * threads_count / watch.elapsedSeconds() << " ops/sec."
|
||||
<< ", " << count * threads_count * block_size / watch.elapsedSeconds() / 1000000 << " MB/sec."
|
||||
<< std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
try
|
||||
{
|
||||
return mainImpl(argc, argv);
|
||||
}
|
||||
catch (const Poco::Exception & e)
|
||||
{
|
||||
std::cerr << e.what() << ", " << e.message() << std::endl;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@ -1,177 +0,0 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <pcg_random.hpp>
|
||||
#include <Poco/Exception.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
#include <Common/randomSeed.h>
|
||||
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <poll.h>
|
||||
#include <cstdlib>
|
||||
#include <ctime>
|
||||
#include <unistd.h>
|
||||
|
||||
#if defined (OS_LINUX)
|
||||
# include <malloc.h>
|
||||
#endif
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int CANNOT_OPEN_FILE;
|
||||
extern const int CANNOT_CLOSE_FILE;
|
||||
extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;
|
||||
extern const int CANNOT_WRITE_TO_FILE_DESCRIPTOR;
|
||||
extern const int CANNOT_FSYNC;
|
||||
extern const int SYSTEM_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
enum Mode
|
||||
{
|
||||
MODE_READ,
|
||||
MODE_WRITE,
|
||||
};
|
||||
|
||||
|
||||
int mainImpl(int argc, char ** argv)
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
const char * file_name = nullptr;
|
||||
Mode mode = MODE_READ;
|
||||
UInt64 min_offset = 0;
|
||||
UInt64 max_offset = 0;
|
||||
UInt64 block_size = 0;
|
||||
UInt64 descriptors = 0;
|
||||
UInt64 count = 0;
|
||||
|
||||
if (argc != 8)
|
||||
{
|
||||
std::cerr << "Usage: " << argv[0] << " file_name r|w min_offset max_offset block_size descriptors count" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
file_name = argv[1];
|
||||
min_offset = parse<UInt64>(argv[3]);
|
||||
max_offset = parse<UInt64>(argv[4]);
|
||||
block_size = parse<UInt64>(argv[5]);
|
||||
descriptors = parse<UInt64>(argv[6]);
|
||||
count = parse<UInt64>(argv[7]);
|
||||
|
||||
if (!strcmp(argv[2], "r"))
|
||||
mode = MODE_READ;
|
||||
else if (!strcmp(argv[2], "w"))
|
||||
mode = MODE_WRITE;
|
||||
else
|
||||
throw Poco::Exception("Invalid mode");
|
||||
|
||||
std::vector<int> fds(descriptors);
|
||||
for (size_t i = 0; i < descriptors; ++i)
|
||||
{
|
||||
fds[i] = open(file_name, O_SYNC | ((mode == MODE_READ) ? O_RDONLY : O_WRONLY));
|
||||
if (-1 == fds[i])
|
||||
throwFromErrno("Cannot open file", ErrorCodes::CANNOT_OPEN_FILE);
|
||||
}
|
||||
|
||||
std::vector<char> buf(block_size);
|
||||
|
||||
pcg64 rng(randomSeed());
|
||||
|
||||
Stopwatch watch;
|
||||
|
||||
std::vector<pollfd> polls(descriptors);
|
||||
|
||||
for (size_t i = 0; i < descriptors; ++i)
|
||||
{
|
||||
polls[i].fd = fds[i];
|
||||
polls[i].events = (mode == MODE_READ) ? POLLIN : POLLOUT;
|
||||
polls[i].revents = 0;
|
||||
}
|
||||
|
||||
size_t ops = 0;
|
||||
while (ops < count)
|
||||
{
|
||||
if (poll(polls.data(), static_cast<nfds_t>(descriptors), -1) <= 0)
|
||||
throwFromErrno("poll failed", ErrorCodes::SYSTEM_ERROR);
|
||||
for (size_t i = 0; i < descriptors; ++i)
|
||||
{
|
||||
if (!polls[i].revents)
|
||||
continue;
|
||||
|
||||
if (polls[i].revents != polls[i].events)
|
||||
throw Poco::Exception("revents indicates error");
|
||||
polls[i].revents = 0;
|
||||
++ops;
|
||||
|
||||
uint64_t rand_result1 = rng();
|
||||
uint64_t rand_result2 = rng();
|
||||
uint64_t rand_result3 = rng();
|
||||
|
||||
size_t rand_result = rand_result1 ^ (rand_result2 << 22) ^ (rand_result3 << 43);
|
||||
size_t offset;
|
||||
offset = min_offset + rand_result % ((max_offset - min_offset) / block_size) * block_size;
|
||||
|
||||
if (mode == MODE_READ)
|
||||
{
|
||||
if (static_cast<int>(block_size) != pread(fds[i], buf.data(), block_size, offset))
|
||||
throwFromErrno("Cannot read", ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (static_cast<int>(block_size) != pwrite(fds[i], buf.data(), block_size, offset))
|
||||
throwFromErrno("Cannot write", ErrorCodes::CANNOT_WRITE_TO_FILE_DESCRIPTOR);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < descriptors; ++i)
|
||||
{
|
||||
#if defined(OS_DARWIN)
|
||||
if (fsync(fds[i]))
|
||||
throwFromErrno("Cannot fsync", ErrorCodes::CANNOT_FSYNC);
|
||||
#else
|
||||
if (fdatasync(fds[i]))
|
||||
throwFromErrno("Cannot fdatasync", ErrorCodes::CANNOT_FSYNC);
|
||||
#endif
|
||||
}
|
||||
|
||||
watch.stop();
|
||||
|
||||
for (size_t i = 0; i < descriptors; ++i)
|
||||
{
|
||||
if (0 != close(fds[i]))
|
||||
throwFromErrno("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE);
|
||||
}
|
||||
|
||||
std::cout << std::fixed << std::setprecision(2)
|
||||
<< "Done " << count << " ops" << " in " << watch.elapsedSeconds() << " sec."
|
||||
<< ", " << count / watch.elapsedSeconds() << " ops/sec."
|
||||
<< ", " << count * block_size / watch.elapsedSeconds() / 1000000 << " MB/sec."
|
||||
<< std::endl;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
{
|
||||
try
|
||||
{
|
||||
return mainImpl(argc, argv);
|
||||
}
|
||||
catch (const Poco::Exception & e)
|
||||
{
|
||||
std::cerr << e.what() << ", " << e.message() << std::endl;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
clickhouse_add_executable (zookeeper-adjust-block-numbers-to-parts main.cpp ${SRCS})
|
||||
target_compile_options(zookeeper-adjust-block-numbers-to-parts PRIVATE -Wno-format)
|
||||
target_link_libraries (zookeeper-adjust-block-numbers-to-parts PRIVATE clickhouse_aggregate_functions dbms clickhouse_common_zookeeper boost::program_options)
|
||||
@ -1,286 +0,0 @@
|
||||
#include <Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h>
|
||||
#include <Storages/MergeTree/MergeTreePartInfo.h>
|
||||
#include <Common/ZooKeeper/ZooKeeper.h>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
#include <boost/program_options.hpp>
|
||||
#include <IO/ReadHelpers.h>
|
||||
|
||||
#include <unordered_map>
|
||||
#include <cmath>
|
||||
|
||||
|
||||
std::vector<std::string> getAllShards(zkutil::ZooKeeper & zk, const std::string & root)
|
||||
{
|
||||
return zk.getChildren(root);
|
||||
}
|
||||
|
||||
|
||||
std::vector<std::string> removeNotExistingShards(zkutil::ZooKeeper & zk, const std::string & root, const std::vector<std::string> & shards)
|
||||
{
|
||||
auto existing_shards = getAllShards(zk, root);
|
||||
std::vector<std::string> filtered_shards;
|
||||
filtered_shards.reserve(shards.size());
|
||||
for (const auto & shard : shards)
|
||||
if (std::find(existing_shards.begin(), existing_shards.end(), shard) == existing_shards.end())
|
||||
std::cerr << "Shard " << shard << " not found." << std::endl;
|
||||
else
|
||||
filtered_shards.emplace_back(shard);
|
||||
return filtered_shards;
|
||||
}
|
||||
|
||||
|
||||
std::vector<std::string> getAllTables(zkutil::ZooKeeper & zk, const std::string & root, const std::string & shard)
|
||||
{
|
||||
return zk.getChildren(root + "/" + shard);
|
||||
}
|
||||
|
||||
|
||||
std::vector<std::string> removeNotExistingTables(zkutil::ZooKeeper & zk, const std::string & root, const std::string & shard, const std::vector<std::string> & tables)
|
||||
{
|
||||
auto existing_tables = getAllTables(zk, root, shard);
|
||||
std::vector<std::string> filtered_tables;
|
||||
filtered_tables.reserve(tables.size());
|
||||
for (const auto & table : tables)
|
||||
if (std::find(existing_tables.begin(), existing_tables.end(), table) == existing_tables.end())
|
||||
std::cerr << "\tTable " << table << " not found on shard " << shard << "." << std::endl;
|
||||
else
|
||||
filtered_tables.emplace_back(table);
|
||||
return filtered_tables;
|
||||
}
|
||||
|
||||
|
||||
Int64 getMaxBlockNumberForPartition(zkutil::ZooKeeper & zk,
|
||||
const std::string & replica_path,
|
||||
const std::string & partition_name,
|
||||
const DB::MergeTreeDataFormatVersion & format_version)
|
||||
{
|
||||
auto replicas_path = replica_path + "/replicas";
|
||||
auto replica_hosts = zk.getChildren(replicas_path);
|
||||
Int64 max_block_num = 0;
|
||||
for (const auto & replica_host : replica_hosts)
|
||||
{
|
||||
auto parts = zk.getChildren(replicas_path + "/" + replica_host + "/parts");
|
||||
for (const auto & part : parts)
|
||||
{
|
||||
try
|
||||
{
|
||||
auto info = DB::MergeTreePartInfo::fromPartName(part, format_version);
|
||||
if (info.partition_id == partition_name)
|
||||
max_block_num = std::max<Int64>(info.max_block, max_block_num);
|
||||
}
|
||||
catch (const DB::Exception & ex)
|
||||
{
|
||||
std::cerr << ex.displayText() << ", Part " << part << "skipped." << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
return max_block_num;
|
||||
}
|
||||
|
||||
|
||||
Int64 getCurrentBlockNumberForPartition(zkutil::ZooKeeper & zk, const std::string & part_path)
|
||||
{
|
||||
Coordination::Stat stat;
|
||||
zk.get(part_path, &stat);
|
||||
|
||||
/// References:
|
||||
/// https://stackoverflow.com/a/10347910
|
||||
/// https://bowenli86.github.io/2016/07/07/distributed%20system/zookeeper/How-does-ZooKeeper-s-persistent-sequential-id-work/
|
||||
return (stat.cversion + stat.numChildren) / 2;
|
||||
}
|
||||
|
||||
|
||||
std::unordered_map<std::string, Int64> getPartitionsNeedAdjustingBlockNumbers(
|
||||
zkutil::ZooKeeper & zk, const std::string & root, const std::vector<std::string> & shards, const std::vector<std::string> & tables)
|
||||
{
|
||||
std::unordered_map<std::string, Int64> result;
|
||||
|
||||
std::vector<std::string> use_shards = shards.empty() ? getAllShards(zk, root) : removeNotExistingShards(zk, root, shards);
|
||||
|
||||
for (const auto & shard : use_shards)
|
||||
{
|
||||
std::cout << "Shard: " << shard << std::endl;
|
||||
std::vector<std::string> use_tables = tables.empty() ? getAllTables(zk, root, shard) : removeNotExistingTables(zk, root, shard, tables);
|
||||
|
||||
for (const auto & table : use_tables)
|
||||
{
|
||||
std::cout << "\tTable: " << table << std::endl;
|
||||
std::string table_path = root + "/" + shard + "/" + table;
|
||||
std::string blocks_path = table_path + "/block_numbers";
|
||||
|
||||
std::vector<std::string> partitions;
|
||||
DB::MergeTreeDataFormatVersion format_version;
|
||||
try
|
||||
{
|
||||
format_version = DB::ReplicatedMergeTreeTableMetadata::parse(zk.get(table_path + "/metadata")).data_format_version;
|
||||
partitions = zk.getChildren(blocks_path);
|
||||
}
|
||||
catch (const DB::Exception & ex)
|
||||
{
|
||||
std::cerr << ex.displayText() << ", table " << table << " skipped." << std::endl;
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const auto & partition : partitions)
|
||||
{
|
||||
try
|
||||
{
|
||||
std::string part_path = blocks_path + "/" + partition;
|
||||
Int64 partition_max_block = getMaxBlockNumberForPartition(zk, table_path, partition, format_version);
|
||||
Int64 current_block_number = getCurrentBlockNumberForPartition(zk, part_path);
|
||||
if (current_block_number < partition_max_block + 1)
|
||||
{
|
||||
std::cout << "\t\tPartition: " << partition << ": current block_number: " << current_block_number
|
||||
<< ", max block number: " << partition_max_block << ". Adjusting is required." << std::endl;
|
||||
result.emplace(part_path, partition_max_block);
|
||||
}
|
||||
}
|
||||
catch (const DB::Exception & ex)
|
||||
{
|
||||
std::cerr << ex.displayText() << ", partition " << partition << " skipped." << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
void setCurrentBlockNumber(zkutil::ZooKeeper & zk, const std::string & path, Int64 new_current_block_number)
|
||||
{
|
||||
Int64 current_block_number = getCurrentBlockNumberForPartition(zk, path);
|
||||
|
||||
auto create_ephemeral_nodes = [&](size_t count)
|
||||
{
|
||||
std::string block_prefix = path + "/block-";
|
||||
Coordination::Requests requests;
|
||||
requests.reserve(count);
|
||||
for (size_t i = 0; i != count; ++i)
|
||||
requests.emplace_back(zkutil::makeCreateRequest(block_prefix, "", zkutil::CreateMode::EphemeralSequential));
|
||||
auto responses = zk.multi(requests);
|
||||
|
||||
std::vector<std::string> paths_created;
|
||||
paths_created.reserve(responses.size());
|
||||
for (const auto & response : responses)
|
||||
{
|
||||
const auto * create_response = dynamic_cast<Coordination::CreateResponse*>(response.get());
|
||||
if (!create_response)
|
||||
{
|
||||
std::cerr << "\tCould not create ephemeral node " << block_prefix << std::endl;
|
||||
return false;
|
||||
}
|
||||
paths_created.emplace_back(create_response->path_created);
|
||||
}
|
||||
|
||||
std::sort(paths_created.begin(), paths_created.end());
|
||||
for (const auto & path_created : paths_created)
|
||||
{
|
||||
Int64 number = DB::parse<Int64>(path_created.c_str() + block_prefix.size(), path_created.size() - block_prefix.size());
|
||||
if (number != current_block_number)
|
||||
{
|
||||
char suffix[11] = "";
|
||||
size_t size = snprintf(suffix, sizeof(suffix), "%010lld", current_block_number);
|
||||
std::string expected_path = block_prefix + std::string(suffix, size);
|
||||
std::cerr << "\t" << path_created << ": Ephemeral node has been created with an unexpected path (expected something like "
|
||||
<< expected_path << ")." << std::endl;
|
||||
return false;
|
||||
}
|
||||
std::cout << "\t" << path_created << std::endl;
|
||||
++current_block_number;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
if (current_block_number >= new_current_block_number)
|
||||
return;
|
||||
|
||||
std::cout << "Creating ephemeral sequential nodes:" << std::endl;
|
||||
create_ephemeral_nodes(1); /// Firstly try to create just a single node.
|
||||
|
||||
/// Create other nodes in batches of 50 nodes.
|
||||
while (current_block_number + 50 <= new_current_block_number) // NOLINT: clang-tidy thinks that the loop is infinite
|
||||
create_ephemeral_nodes(50);
|
||||
|
||||
create_ephemeral_nodes(new_current_block_number - current_block_number);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
try
|
||||
{
|
||||
/// Parse the command line.
|
||||
namespace po = boost::program_options;
|
||||
po::options_description desc("Allowed options");
|
||||
desc.add_options()
|
||||
("help,h", "show help")
|
||||
("zookeeper,z", po::value<std::string>(), "Addresses of ZooKeeper instances, comma-separated. Example: example01e.clickhouse.com:2181")
|
||||
("path,p", po::value<std::string>(), "[optional] Path of replica queue to insert node (without trailing slash). By default it's /clickhouse/tables")
|
||||
("shard,s", po::value<std::string>(), "[optional] Shards to process, comma-separated. If not specified then the utility will process all the shards.")
|
||||
("table,t", po::value<std::string>(), "[optional] Tables to process, comma-separated. If not specified then the utility will process all the tables.")
|
||||
("dry-run", "[optional] Specify if you want this utility just to analyze block numbers without any changes.");
|
||||
|
||||
po::variables_map options;
|
||||
po::store(po::parse_command_line(argc, argv, desc), options);
|
||||
|
||||
auto show_usage = [&]
|
||||
{
|
||||
std::cout << "Usage: " << std::endl;
|
||||
std::cout << " " << argv[0] << " [options]" << std::endl;
|
||||
std::cout << desc << std::endl;
|
||||
};
|
||||
|
||||
if (options.count("help") || (argc == 1))
|
||||
{
|
||||
std::cout << "This utility adjusts the /block_numbers zookeeper nodes to the correct block number in partition." << std::endl;
|
||||
std::cout << "It might be useful when incorrect block numbers stored in zookeeper don't allow you to insert data into a table or drop/detach a partition." << std::endl;
|
||||
show_usage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!options.count("zookeeper"))
|
||||
{
|
||||
std::cerr << "Option --zookeeper should be set." << std::endl;
|
||||
show_usage();
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::string root = options.count("path") ? options.at("path").as<std::string>() : "/clickhouse/tables";
|
||||
|
||||
std::vector<std::string> shards, tables;
|
||||
if (options.count("shard"))
|
||||
boost::split(shards, options.at("shard").as<std::string>(), boost::algorithm::is_any_of(","));
|
||||
if (options.count("table"))
|
||||
boost::split(tables, options.at("table").as<std::string>(), boost::algorithm::is_any_of(","));
|
||||
|
||||
/// Check if the adjusting of the block numbers is required.
|
||||
std::cout << "Checking if adjusting of the block numbers is required:" << std::endl;
|
||||
zkutil::ZooKeeper zookeeper(options.at("zookeeper").as<std::string>());
|
||||
auto part_paths_with_max_block_numbers = getPartitionsNeedAdjustingBlockNumbers(zookeeper, root, shards, tables);
|
||||
|
||||
if (part_paths_with_max_block_numbers.empty())
|
||||
{
|
||||
std::cout << "No adjusting required." << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::cout << "Required adjusting of " << part_paths_with_max_block_numbers.size() << " block numbers." << std::endl;
|
||||
|
||||
/// Adjust the block numbers.
|
||||
if (options.count("dry-run"))
|
||||
{
|
||||
std::cout << "This is a dry-run, exiting." << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::cout << std::endl << "Adjusting the block numbers:" << std::endl;
|
||||
for (const auto & [part_path, max_block_number] : part_paths_with_max_block_numbers)
|
||||
setCurrentBlockNumber(zookeeper, part_path, max_block_number + 1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
|
||||
throw;
|
||||
}
|
||||
@ -1,2 +0,0 @@
|
||||
clickhouse_add_executable (zookeeper-create-entry-to-download-part main.cpp ${SRCS})
|
||||
target_link_libraries (zookeeper-create-entry-to-download-part PRIVATE dbms clickhouse_common_zookeeper boost::program_options)
|
||||
@ -1,47 +0,0 @@
|
||||
#include <list>
|
||||
#include <Storages/MergeTree/ReplicatedMergeTreeLogEntry.h>
|
||||
#include <Common/ZooKeeper/ZooKeeper.h>
|
||||
#include <boost/program_options.hpp>
|
||||
|
||||
|
||||
int main(int argc, char ** argv)
|
||||
try
|
||||
{
|
||||
boost::program_options::options_description desc("Allowed options");
|
||||
desc.add_options()
|
||||
("help,h", "produce help message")
|
||||
("address,a", boost::program_options::value<std::string>()->required(),
|
||||
"addresses of ZooKeeper instances, comma separated. Example: example01e.clickhouse.com:2181")
|
||||
("path,p", boost::program_options::value<std::string>()->required(), "path of replica queue to insert node (without trailing slash)")
|
||||
("name,n", boost::program_options::value<std::string>()->required(), "name of part to download")
|
||||
;
|
||||
|
||||
boost::program_options::variables_map options;
|
||||
boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options);
|
||||
|
||||
if (options.count("help"))
|
||||
{
|
||||
std::cout << "Insert log entry to replication queue to download part from any replica." << std::endl;
|
||||
std::cout << "Usage: " << argv[0] << " [options]" << std::endl;
|
||||
std::cout << desc << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::string path = options.at("path").as<std::string>();
|
||||
std::string name = options.at("name").as<std::string>();
|
||||
|
||||
zkutil::ZooKeeper zookeeper(options.at("address").as<std::string>());
|
||||
|
||||
DB::ReplicatedMergeTreeLogEntry entry;
|
||||
entry.type = DB::ReplicatedMergeTreeLogEntry::MERGE_PARTS;
|
||||
entry.source_parts = {name};
|
||||
entry.new_part_name = name;
|
||||
|
||||
zookeeper.create(path + "/queue-", entry.toString(), zkutil::CreateMode::PersistentSequential);
|
||||
return 0;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cerr << DB::getCurrentExceptionMessage(true) << '\n';
|
||||
throw;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user