mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-30 05:30:51 +00:00
Merge branch 'master' into feature/create-simple-lambda-function
This commit is contained in:
Realist007
GitHub
commit
673a813cfa
2
.gitmodules
vendored
2
.gitmodules
vendored
@ -103,7 +103,7 @@
|
||||
url = https://github.com/ClickHouse-Extras/fastops
|
||||
[submodule "contrib/orc"]
|
||||
path = contrib/orc
|
||||
url = https://github.com/apache/orc
|
||||
url = https://github.com/ClickHouse-Extras/orc
|
||||
[submodule "contrib/sparsehash-c11"]
|
||||
path = contrib/sparsehash-c11
|
||||
url = https://github.com/sparsehash/sparsehash-c11.git
|
||||
|
||||
@ -8,13 +8,6 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <pthread.h>
|
||||
|
||||
size_t __pthread_get_minstack(const pthread_attr_t * attr)
|
||||
{
|
||||
return 1048576; /// This is a guess. Don't sure it is correct.
|
||||
}
|
||||
|
||||
#include <signal.h>
|
||||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
@ -141,6 +134,8 @@ int __open_2(const char *path, int oflag)
|
||||
}
|
||||
|
||||
|
||||
#include <pthread.h>
|
||||
|
||||
/// No-ops.
|
||||
int pthread_setname_np(pthread_t thread, const char *name) { return 0; }
|
||||
int pthread_getname_np(pthread_t thread, char *name, size_t len) { name[0] = '\0'; return 0; };
|
||||
|
||||
2
contrib/arrow
vendored
2
contrib/arrow
vendored
@ -1 +1 @@
|
||||
Subproject commit 616b3dc76a0c8450b4027ded8a78e9619d7c845f
|
||||
Subproject commit debf751a129bdda9ff4d1e895e08957ff77000a1
|
||||
@ -188,6 +188,7 @@ set(ARROW_SRCS
|
||||
"${LIBRARY_DIR}/array/util.cc"
|
||||
"${LIBRARY_DIR}/array/validate.cc"
|
||||
|
||||
"${LIBRARY_DIR}/compute/api_aggregate.cc"
|
||||
"${LIBRARY_DIR}/compute/api_scalar.cc"
|
||||
"${LIBRARY_DIR}/compute/api_vector.cc"
|
||||
"${LIBRARY_DIR}/compute/cast.cc"
|
||||
@ -198,8 +199,11 @@ set(ARROW_SRCS
|
||||
|
||||
"${LIBRARY_DIR}/compute/kernels/aggregate_basic.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/aggregate_mode.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/aggregate_quantile.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/aggregate_tdigest.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/aggregate_var_std.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/codegen_internal.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/hash_aggregate.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/scalar_arithmetic.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/scalar_boolean.cc"
|
||||
"${LIBRARY_DIR}/compute/kernels/scalar_cast_boolean.cc"
|
||||
@ -243,6 +247,7 @@ set(ARROW_SRCS
|
||||
"${LIBRARY_DIR}/io/interfaces.cc"
|
||||
"${LIBRARY_DIR}/io/memory.cc"
|
||||
"${LIBRARY_DIR}/io/slow.cc"
|
||||
"${LIBRARY_DIR}/io/transform.cc"
|
||||
|
||||
"${LIBRARY_DIR}/tensor/coo_converter.cc"
|
||||
"${LIBRARY_DIR}/tensor/csf_converter.cc"
|
||||
@ -256,11 +261,8 @@ set(ARROW_SRCS
|
||||
"${LIBRARY_DIR}/util/bitmap_builders.cc"
|
||||
"${LIBRARY_DIR}/util/bitmap_ops.cc"
|
||||
"${LIBRARY_DIR}/util/bpacking.cc"
|
||||
"${LIBRARY_DIR}/util/cancel.cc"
|
||||
"${LIBRARY_DIR}/util/compression.cc"
|
||||
"${LIBRARY_DIR}/util/compression_lz4.cc"
|
||||
"${LIBRARY_DIR}/util/compression_snappy.cc"
|
||||
"${LIBRARY_DIR}/util/compression_zlib.cc"
|
||||
"${LIBRARY_DIR}/util/compression_zstd.cc"
|
||||
"${LIBRARY_DIR}/util/cpu_info.cc"
|
||||
"${LIBRARY_DIR}/util/decimal.cc"
|
||||
"${LIBRARY_DIR}/util/delimiting.cc"
|
||||
@ -268,13 +270,14 @@ set(ARROW_SRCS
|
||||
"${LIBRARY_DIR}/util/future.cc"
|
||||
"${LIBRARY_DIR}/util/int_util.cc"
|
||||
"${LIBRARY_DIR}/util/io_util.cc"
|
||||
"${LIBRARY_DIR}/util/iterator.cc"
|
||||
"${LIBRARY_DIR}/util/key_value_metadata.cc"
|
||||
"${LIBRARY_DIR}/util/logging.cc"
|
||||
"${LIBRARY_DIR}/util/memory.cc"
|
||||
"${LIBRARY_DIR}/util/mutex.cc"
|
||||
"${LIBRARY_DIR}/util/string_builder.cc"
|
||||
"${LIBRARY_DIR}/util/string.cc"
|
||||
"${LIBRARY_DIR}/util/task_group.cc"
|
||||
"${LIBRARY_DIR}/util/tdigest.cc"
|
||||
"${LIBRARY_DIR}/util/thread_pool.cc"
|
||||
"${LIBRARY_DIR}/util/time.cc"
|
||||
"${LIBRARY_DIR}/util/trie.cc"
|
||||
@ -368,14 +371,14 @@ set(PARQUET_SRCS
|
||||
"${LIBRARY_DIR}/column_reader.cc"
|
||||
"${LIBRARY_DIR}/column_scanner.cc"
|
||||
"${LIBRARY_DIR}/column_writer.cc"
|
||||
"${LIBRARY_DIR}/deprecated_io.cc"
|
||||
"${LIBRARY_DIR}/encoding.cc"
|
||||
"${LIBRARY_DIR}/encryption.cc"
|
||||
"${LIBRARY_DIR}/encryption_internal.cc"
|
||||
"${LIBRARY_DIR}/encryption/encryption.cc"
|
||||
"${LIBRARY_DIR}/encryption/encryption_internal.cc"
|
||||
"${LIBRARY_DIR}/encryption/internal_file_decryptor.cc"
|
||||
"${LIBRARY_DIR}/encryption/internal_file_encryptor.cc"
|
||||
"${LIBRARY_DIR}/exception.cc"
|
||||
"${LIBRARY_DIR}/file_reader.cc"
|
||||
"${LIBRARY_DIR}/file_writer.cc"
|
||||
"${LIBRARY_DIR}/internal_file_decryptor.cc"
|
||||
"${LIBRARY_DIR}/internal_file_encryptor.cc"
|
||||
"${LIBRARY_DIR}/level_conversion.cc"
|
||||
"${LIBRARY_DIR}/level_comparison.cc"
|
||||
"${LIBRARY_DIR}/metadata.cc"
|
||||
@ -385,6 +388,8 @@ set(PARQUET_SRCS
|
||||
"${LIBRARY_DIR}/properties.cc"
|
||||
"${LIBRARY_DIR}/schema.cc"
|
||||
"${LIBRARY_DIR}/statistics.cc"
|
||||
"${LIBRARY_DIR}/stream_reader.cc"
|
||||
"${LIBRARY_DIR}/stream_writer.cc"
|
||||
"${LIBRARY_DIR}/types.cc"
|
||||
|
||||
"${GEN_LIBRARY_DIR}/parquet_constants.cpp"
|
||||
|
||||
2
contrib/orc
vendored
2
contrib/orc
vendored
@ -1 +1 @@
|
||||
Subproject commit 5981208e39447df84827f6a961d1da76bacb6078
|
||||
Subproject commit 0a936f6bbdb9303308973073f8623b5a8d82eae1
|
||||
2
contrib/replxx
vendored
2
contrib/replxx
vendored
@ -1 +1 @@
|
||||
Subproject commit 2b24f14594d7606792b92544bb112a6322ba34d7
|
||||
Subproject commit c81be6c68b146f15f2096b7ef80e3f21fe27004c
|
||||
@ -1591,6 +1591,18 @@ FORMAT PrettyCompactMonoBlock
|
||||
|
||||
Default value: 0
|
||||
|
||||
## distributed_push_down_limit (#distributed-push-down-limit}
|
||||
|
||||
LIMIT will be applied on each shard separatelly. Usually you don't need to use it, since this will be done automatically if it is possible, i.e. for simple query SELECT FROM LIMIT.
|
||||
|
||||
Possible values:
|
||||
|
||||
- 0 - Disabled
|
||||
- 1 - Enabled
|
||||
|
||||
!!! note "Note"
|
||||
That with this setting the result of the query may be inaccurate.
|
||||
|
||||
## optimize_skip_unused_shards_limit {#optimize-skip-unused-shards-limit}
|
||||
|
||||
Limit for number of sharding key values, turns off `optimize_skip_unused_shards` if the limit is reached.
|
||||
|
||||
@ -38,6 +38,7 @@ The list of available `SYSTEM` statements:
|
||||
- [START REPLICATION QUEUES](#query_language-system-start-replication-queues)
|
||||
- [SYNC REPLICA](#query_language-system-sync-replica)
|
||||
- [RESTART REPLICA](#query_language-system-restart-replica)
|
||||
- [RESTORE REPLICA](#query_language-system-restore-replica)
|
||||
- [RESTART REPLICAS](#query_language-system-restart-replicas)
|
||||
|
||||
## RELOAD EMBEDDED DICTIONARIES {#query_language-system-reload-emdedded-dictionaries}
|
||||
@ -290,13 +291,60 @@ After running this statement the `[db.]replicated_merge_tree_family_table_name`
|
||||
|
||||
### RESTART REPLICA {#query_language-system-restart-replica}
|
||||
|
||||
Provides possibility to reinitialize Zookeeper sessions state for `ReplicatedMergeTree` table, will compare current state with Zookeeper as source of true and add tasks to Zookeeper queue if needed
|
||||
Initialization replication quene based on ZooKeeper date happens in the same way as `ATTACH TABLE` statement. For a short time the table will be unavailable for any operations.
|
||||
Provides possibility to reinitialize Zookeeper sessions state for `ReplicatedMergeTree` table, will compare current state with Zookeeper as source of true and add tasks to Zookeeper queue if needed.
|
||||
Initialization replication queue based on ZooKeeper date happens in the same way as `ATTACH TABLE` statement. For a short time the table will be unavailable for any operations.
|
||||
|
||||
``` sql
|
||||
SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name
|
||||
```
|
||||
|
||||
### RESTORE REPLICA {#query_language-system-restore-replica}
|
||||
|
||||
Restores a replica if data is [possibly] present but Zookeeper metadata is lost.
|
||||
|
||||
Works only on readonly `ReplicatedMergeTree` tables.
|
||||
|
||||
One may execute query after:
|
||||
|
||||
- ZooKeeper root `/` loss.
|
||||
- Replicas path `/replicas` loss.
|
||||
- Individual replica path `/replicas/replica_name/` loss.
|
||||
|
||||
Replica attaches locally found parts and sends info about them to Zookeeper.
|
||||
Parts present on replica before metadata loss are not re-fetched from other replicas if not being outdated
|
||||
(so replica restoration does not mean re-downloading all data over the network).
|
||||
|
||||
Caveat: parts in all states are moved to `detached/` folder. Parts active before data loss (Committed) are attached.
|
||||
|
||||
#### Syntax
|
||||
|
||||
```sql
|
||||
SYSTEM RESTORE REPLICA [db.]replicated_merge_tree_family_table_name [ON CLUSTER cluster_name]
|
||||
```
|
||||
|
||||
Alternative syntax:
|
||||
|
||||
```sql
|
||||
SYSTEM RESTORE REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_family_table_name
|
||||
```
|
||||
|
||||
#### Example
|
||||
|
||||
```sql
|
||||
-- Creating table on multiple servers
|
||||
|
||||
CREATE TABLE test(n UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/', '{replica}')
|
||||
ORDER BY n PARTITION BY n % 10;
|
||||
|
||||
INSERT INTO test SELECT * FROM numbers(1000);
|
||||
|
||||
-- zookeeper_delete_path("/clickhouse/tables/test", recursive=True) <- root loss.
|
||||
|
||||
SYSTEM RESTART REPLICA test; -- Table will attach as readonly as metadata is missing.
|
||||
SYSTEM RESTORE REPLICA test; -- Need to execute on every replica, another way: RESTORE REPLICA test ON CLUSTER cluster
|
||||
```
|
||||
|
||||
### RESTART REPLICAS {#query_language-system-restart-replicas}
|
||||
|
||||
Provides possibility to reinitialize Zookeeper sessions state for all `ReplicatedMergeTree` tables, will compare current state with Zookeeper as source of true and add tasks to Zookeeper queue if needed
|
||||
|
||||
@ -33,6 +33,7 @@ option (ENABLE_CLICKHOUSE_OBFUSCATOR "Table data obfuscator (convert real data t
|
||||
${ENABLE_CLICKHOUSE_ALL})
|
||||
|
||||
# https://clickhouse.tech/docs/en/operations/utilities/odbc-bridge/
|
||||
# TODO Also needs NANODBC.
|
||||
if (ENABLE_ODBC)
|
||||
option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver"
|
||||
${ENABLE_CLICKHOUSE_ALL})
|
||||
|
||||
@ -20,6 +20,7 @@ CLICKHOUSE_QueryProcessingStage=(
|
||||
fetch_columns
|
||||
with_mergeable_state
|
||||
with_mergeable_state_after_aggregation
|
||||
with_mergeable_state_after_aggregation_and_limit
|
||||
)
|
||||
|
||||
CLICKHOUSE_Format=(
|
||||
|
||||
@ -580,7 +580,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
|
||||
("query", value<std::string>()->default_value(""), "query to execute")
|
||||
("concurrency,c", value<unsigned>()->default_value(1), "number of parallel queries")
|
||||
("delay,d", value<double>()->default_value(1), "delay between intermediate reports in seconds (set 0 to disable reports)")
|
||||
("stage", value<std::string>()->default_value("complete"), "request query processing up to specified stage: complete,fetch_columns,with_mergeable_state,with_mergeable_state_after_aggregation")
|
||||
("stage", value<std::string>()->default_value("complete"), "request query processing up to specified stage: complete,fetch_columns,with_mergeable_state,with_mergeable_state_after_aggregation,with_mergeable_state_after_aggregation_and_limit")
|
||||
("iterations,i", value<size_t>()->default_value(0), "amount of queries to be executed")
|
||||
("timelimit,t", value<double>()->default_value(0.), "stop launch of queries after specified time limit")
|
||||
("randomize,r", value<bool>()->default_value(false), "randomize order of execution")
|
||||
|
||||
@ -112,6 +112,7 @@ namespace ErrorCodes
|
||||
extern const int DEADLOCK_AVOIDED;
|
||||
extern const int UNRECOGNIZED_ARGUMENTS;
|
||||
extern const int SYNTAX_ERROR;
|
||||
extern const int TOO_DEEP_RECURSION;
|
||||
}
|
||||
|
||||
|
||||
@ -1267,7 +1268,8 @@ private:
|
||||
}
|
||||
catch (const Exception & e)
|
||||
{
|
||||
if (e.code() != ErrorCodes::SYNTAX_ERROR)
|
||||
if (e.code() != ErrorCodes::SYNTAX_ERROR &&
|
||||
e.code() != ErrorCodes::TOO_DEEP_RECURSION)
|
||||
throw;
|
||||
}
|
||||
|
||||
@ -1449,11 +1451,10 @@ private:
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
if (e.code() != ErrorCodes::SYNTAX_ERROR)
|
||||
{
|
||||
if (e.code() != ErrorCodes::SYNTAX_ERROR &&
|
||||
e.code() != ErrorCodes::TOO_DEEP_RECURSION)
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
if (ast_2)
|
||||
{
|
||||
@ -2471,7 +2472,7 @@ public:
|
||||
("password", po::value<std::string>()->implicit_value("\n", ""), "password")
|
||||
("ask-password", "ask-password")
|
||||
("quota_key", po::value<std::string>(), "A string to differentiate quotas when the user have keyed quotas configured on server")
|
||||
("stage", po::value<std::string>()->default_value("complete"), "Request query processing up to specified stage: complete,fetch_columns,with_mergeable_state,with_mergeable_state_after_aggregation")
|
||||
("stage", po::value<std::string>()->default_value("complete"), "Request query processing up to specified stage: complete,fetch_columns,with_mergeable_state,with_mergeable_state_after_aggregation,with_mergeable_state_after_aggregation_and_limit")
|
||||
("query_id", po::value<std::string>(), "query_id")
|
||||
("query,q", po::value<std::string>(), "query")
|
||||
("database,d", po::value<std::string>(), "database")
|
||||
|
||||
@ -154,6 +154,7 @@ enum class AccessType
|
||||
M(SYSTEM_DROP_REPLICA, "DROP REPLICA", TABLE, SYSTEM) \
|
||||
M(SYSTEM_SYNC_REPLICA, "SYNC REPLICA", TABLE, SYSTEM) \
|
||||
M(SYSTEM_RESTART_REPLICA, "RESTART REPLICA", TABLE, SYSTEM) \
|
||||
M(SYSTEM_RESTORE_REPLICA, "RESTORE REPLICA", TABLE, SYSTEM) \
|
||||
M(SYSTEM_FLUSH_DISTRIBUTED, "FLUSH DISTRIBUTED", TABLE, SYSTEM_FLUSH) \
|
||||
M(SYSTEM_FLUSH_LOGS, "FLUSH LOGS", GLOBAL, SYSTEM_FLUSH) \
|
||||
M(SYSTEM_FLUSH, "", GROUP, SYSTEM) \
|
||||
|
||||
@ -191,6 +191,7 @@ public:
|
||||
void nestedRemoveNullable() { dictionary.getColumnUnique().nestedRemoveNullable(); }
|
||||
|
||||
const IColumnUnique & getDictionary() const { return dictionary.getColumnUnique(); }
|
||||
IColumnUnique & getDictionary() { return dictionary.getColumnUnique(); }
|
||||
const ColumnPtr & getDictionaryPtr() const { return dictionary.getColumnUniquePtr(); }
|
||||
/// IColumnUnique & getUnique() { return static_cast<IColumnUnique &>(*column_unique); }
|
||||
/// ColumnPtr getUniquePtr() const { return column_unique; }
|
||||
|
||||
@ -554,8 +554,9 @@
|
||||
M(584, PROJECTION_NOT_USED) \
|
||||
M(585, CANNOT_PARSE_YAML) \
|
||||
M(586, CANNOT_CREATE_FILE) \
|
||||
M(587, FUNCTION_ALREADY_EXISTS) \
|
||||
M(588, CANNOT_DROP_SYSTEM_FUNCTION) \
|
||||
M(587, CONCURRENT_ACCESS_NOT_SUPPORTED) \
|
||||
M(588, FUNCTION_ALREADY_EXISTS) \
|
||||
M(589, CANNOT_DROP_SYSTEM_FUNCTION) \
|
||||
\
|
||||
M(998, POSTGRESQL_CONNECTION_FAILURE) \
|
||||
M(999, KEEPER_EXCEPTION) \
|
||||
|
||||
@ -39,7 +39,7 @@ public:
|
||||
void setFileProgressCallback(ContextMutablePtr context, bool write_progress_on_update = false);
|
||||
|
||||
/// How much seconds passed since query execution start.
|
||||
UInt64 elapsedSeconds() const { return watch.elapsedSeconds(); }
|
||||
double elapsedSeconds() const { return watch.elapsedSeconds(); }
|
||||
|
||||
private:
|
||||
/// This flag controls whether to show the progress bar. We start showing it after
|
||||
|
||||
@ -7,8 +7,9 @@
|
||||
|
||||
#include <Poco/Logger.h>
|
||||
#include <common/getThreadId.h>
|
||||
#include <common/getPageSize.h>
|
||||
|
||||
#include <signal.h>
|
||||
#include <csignal>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -25,7 +26,47 @@ thread_local ThreadStatus * current_thread = nullptr;
|
||||
thread_local ThreadStatus * main_thread = nullptr;
|
||||
|
||||
#if !defined(SANITIZER) && !defined(ARCADIA_BUILD)
|
||||
alignas(4096) static thread_local char alt_stack[std::max<size_t>(MINSIGSTKSZ, 4096)];
|
||||
namespace
|
||||
{
|
||||
|
||||
/// Alternative stack for signal handling.
|
||||
///
|
||||
/// This stack should not be located in the TLS (thread local storage), since:
|
||||
/// - TLS locates data on the per-thread stack
|
||||
/// - And in case of stack in the signal handler will grow too much,
|
||||
/// it will start overwriting TLS storage
|
||||
/// (and note, that it is not too small, due to StackTrace obtaining)
|
||||
/// - Plus there is no way to determine TLS block size, yes there is
|
||||
/// __pthread_get_minstack() in glibc, but it is private and hence not portable.
|
||||
///
|
||||
/// Also we should not use getStackSize() (pthread_attr_getstack()) since it
|
||||
/// will return 8MB, and this is too huge for signal stack.
|
||||
struct ThreadStack
|
||||
{
|
||||
ThreadStack()
|
||||
: data(aligned_alloc(getPageSize(), size))
|
||||
{
|
||||
/// Add a guard page
|
||||
/// (and since the stack grows downward, we need to protect the first page).
|
||||
mprotect(data, getPageSize(), PROT_NONE);
|
||||
}
|
||||
~ThreadStack()
|
||||
{
|
||||
mprotect(data, getPageSize(), PROT_WRITE|PROT_READ);
|
||||
free(data);
|
||||
}
|
||||
|
||||
static size_t getSize() { return size; }
|
||||
void * getData() const { return data; }
|
||||
|
||||
private:
|
||||
static constexpr size_t size = 16 << 10; /// 16 KiB - not too big but enough to handle error.
|
||||
void * data;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
static thread_local ThreadStack alt_stack;
|
||||
static thread_local bool has_alt_stack = false;
|
||||
#endif
|
||||
|
||||
@ -54,9 +95,9 @@ ThreadStatus::ThreadStatus()
|
||||
|
||||
/// We have to call 'sigaltstack' before first 'sigaction'. (It does not work other way, for unknown reason).
|
||||
stack_t altstack_description{};
|
||||
altstack_description.ss_sp = alt_stack;
|
||||
altstack_description.ss_sp = alt_stack.getData();
|
||||
altstack_description.ss_flags = 0;
|
||||
altstack_description.ss_size = sizeof(alt_stack);
|
||||
altstack_description.ss_size = alt_stack.getSize();
|
||||
|
||||
if (0 != sigaltstack(&altstack_description, nullptr))
|
||||
{
|
||||
|
||||
@ -22,19 +22,17 @@ namespace DB
|
||||
static thread_local void * stack_address = nullptr;
|
||||
static thread_local size_t max_stack_size = 0;
|
||||
|
||||
/** It works fine when interpreters are instantiated by ClickHouse code in properly prepared threads,
|
||||
* but there are cases when ClickHouse runs as a library inside another application.
|
||||
* If application is using user-space lightweight threads with manually allocated stacks,
|
||||
* current implementation is not reasonable, as it has no way to properly check the remaining
|
||||
* stack size without knowing the details of how stacks are allocated.
|
||||
* We mark this function as weak symbol to be able to replace it in another ClickHouse-based products.
|
||||
/**
|
||||
* @param out_address - if not nullptr, here the address of the stack will be written.
|
||||
* @return stack size
|
||||
*/
|
||||
__attribute__((__weak__)) void checkStackSize()
|
||||
size_t getStackSize(void ** out_address)
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
if (!stack_address)
|
||||
{
|
||||
size_t size;
|
||||
void * address;
|
||||
|
||||
#if defined(OS_DARWIN)
|
||||
// pthread_get_stacksize_np() returns a value too low for the main thread on
|
||||
// OSX 10.9, http://mail.openjdk.java.net/pipermail/hotspot-dev/2013-October/011369.html
|
||||
@ -43,10 +41,10 @@ __attribute__((__weak__)) void checkStackSize()
|
||||
// https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/Multithreading/CreatingThreads/CreatingThreads.html
|
||||
// Stack size for the main thread is 8MB on OSX excluding the guard page size.
|
||||
pthread_t thread = pthread_self();
|
||||
max_stack_size = pthread_main_np() ? (8 * 1024 * 1024) : pthread_get_stacksize_np(thread);
|
||||
size = pthread_main_np() ? (8 * 1024 * 1024) : pthread_get_stacksize_np(thread);
|
||||
|
||||
// stack_address points to the start of the stack, not the end how it's returned by pthread_get_stackaddr_np
|
||||
stack_address = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(pthread_get_stackaddr_np(thread)) - max_stack_size);
|
||||
// stack address points to the start of the stack, not the end how it's returned by pthread_get_stackaddr_np
|
||||
address = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(pthread_get_stackaddr_np(thread)) - max_stack_size);
|
||||
#else
|
||||
pthread_attr_t attr;
|
||||
# if defined(__FreeBSD__) || defined(OS_SUNOS)
|
||||
@ -60,11 +58,30 @@ __attribute__((__weak__)) void checkStackSize()
|
||||
|
||||
SCOPE_EXIT({ pthread_attr_destroy(&attr); });
|
||||
|
||||
if (0 != pthread_attr_getstack(&attr, &stack_address, &max_stack_size))
|
||||
if (0 != pthread_attr_getstack(&attr, &address, &size))
|
||||
throwFromErrno("Cannot pthread_getattr_np", ErrorCodes::CANNOT_PTHREAD_ATTR);
|
||||
#endif // OS_DARWIN
|
||||
|
||||
if (out_address)
|
||||
*out_address = address;
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
/** It works fine when interpreters are instantiated by ClickHouse code in properly prepared threads,
|
||||
* but there are cases when ClickHouse runs as a library inside another application.
|
||||
* If application is using user-space lightweight threads with manually allocated stacks,
|
||||
* current implementation is not reasonable, as it has no way to properly check the remaining
|
||||
* stack size without knowing the details of how stacks are allocated.
|
||||
* We mark this function as weak symbol to be able to replace it in another ClickHouse-based products.
|
||||
*/
|
||||
__attribute__((__weak__)) void checkStackSize()
|
||||
{
|
||||
using namespace DB;
|
||||
|
||||
if (!stack_address)
|
||||
max_stack_size = getStackSize(&stack_address);
|
||||
|
||||
const void * frame_address = __builtin_frame_address(0);
|
||||
uintptr_t int_frame_address = reinterpret_cast<uintptr_t>(frame_address);
|
||||
uintptr_t int_stack_address = reinterpret_cast<uintptr_t>(stack_address);
|
||||
|
||||
@ -24,6 +24,8 @@ namespace QueryProcessingStage
|
||||
stage = WithMergeableState;
|
||||
else if (stage_string == "with_mergeable_state_after_aggregation")
|
||||
stage = WithMergeableStateAfterAggregation;
|
||||
else if (stage_string == "with_mergeable_state_after_aggregation_and_limit")
|
||||
stage = WithMergeableStateAfterAggregationAndLimit;
|
||||
else
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown query processing stage: {}", stage_string);
|
||||
|
||||
|
||||
@ -26,8 +26,15 @@ namespace QueryProcessingStage
|
||||
/// It is used for auto distributed_group_by_no_merge optimization for distributed engine.
|
||||
/// (See comments in StorageDistributed).
|
||||
WithMergeableStateAfterAggregation = 3,
|
||||
/// Same as WithMergeableStateAfterAggregation but also will apply limit on each shard.
|
||||
///
|
||||
/// This query stage will be used for auto
|
||||
/// distributed_group_by_no_merge/distributed_push_down_limit
|
||||
/// optimization.
|
||||
/// (See comments in StorageDistributed).
|
||||
WithMergeableStateAfterAggregationAndLimit = 4,
|
||||
|
||||
MAX = 4,
|
||||
MAX = 5,
|
||||
};
|
||||
|
||||
inline const char * toString(UInt64 stage)
|
||||
@ -38,6 +45,7 @@ namespace QueryProcessingStage
|
||||
"WithMergeableState",
|
||||
"Complete",
|
||||
"WithMergeableStateAfterAggregation",
|
||||
"WithMergeableStateAfterAggregationAndLimit",
|
||||
};
|
||||
return stage < MAX
|
||||
? data[stage]
|
||||
|
||||
@ -118,6 +118,7 @@ class IColumn;
|
||||
\
|
||||
M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard, if 1 SELECT is executed on each shard, if 2 SELECT and INSERT is executed on each shard", 0) \
|
||||
M(UInt64, distributed_group_by_no_merge, 0, "If 1, Do not merge aggregation states from different servers for distributed queries (shards will process query up to the Complete stage, initiator just proxies the data from the shards). If 2 the initiator will apply ORDER BY and LIMIT stages (it is not in case when shard process query up to the Complete stage)", 0) \
|
||||
M(UInt64, distributed_push_down_limit, 0, "If 1, LIMIT will be applied on each shard separatelly. Usually you don't need to use it, since this will be done automatically if it is possible, i.e. for simple query SELECT FROM LIMIT.", 0) \
|
||||
M(Bool, optimize_distributed_group_by_sharding_key, false, "Optimize GROUP BY sharding_key queries (by avoiding costly aggregation on the initiator server).", 0) \
|
||||
M(UInt64, optimize_skip_unused_shards_limit, 1000, "Limit for number of sharding key values, turns off optimize_skip_unused_shards if the limit is reached", 0) \
|
||||
M(Bool, optimize_skip_unused_shards, false, "Assumes that data is distributed by sharding_key. Optimization to skip unused shards if SELECT query filters by sharding_key.", 0) \
|
||||
@ -563,7 +564,8 @@ class IColumn;
|
||||
M(Bool, output_format_pretty_row_numbers, false, "Add row numbers before each row for pretty output format", 0) \
|
||||
M(Bool, insert_distributed_one_random_shard, false, "If setting is enabled, inserting into distributed table will choose a random shard to write when there is no sharding key", 0) \
|
||||
M(Bool, cross_to_inner_join_rewrite, true, "Use inner join instead of comma/cross join if possible", 0) \
|
||||
|
||||
\
|
||||
M(Bool, output_format_arrow_low_cardinality_as_dictionary, false, "Enable output LowCardinality type as Dictionary Arrow type", 0) \
|
||||
|
||||
// End of FORMAT_FACTORY_SETTINGS
|
||||
// Please add settings non-related to formats into the COMMON_SETTINGS above.
|
||||
|
||||
@ -112,6 +112,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings)
|
||||
format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions;
|
||||
format_settings.with_names_use_header = settings.input_format_with_names_use_header;
|
||||
format_settings.write_statistics = settings.output_format_write_statistics;
|
||||
format_settings.arrow.low_cardinality_as_dictionary = settings.output_format_arrow_low_cardinality_as_dictionary;
|
||||
|
||||
/// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context
|
||||
if (format_settings.schema.is_server)
|
||||
|
||||
@ -52,6 +52,7 @@ struct FormatSettings
|
||||
struct
|
||||
{
|
||||
UInt64 row_group_size = 1000000;
|
||||
bool low_cardinality_as_dictionary = false;
|
||||
} arrow;
|
||||
|
||||
struct
|
||||
|
||||
@ -831,14 +831,17 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
|
||||
if (create.attach && !create.storage && !create.columns_list)
|
||||
{
|
||||
auto database = DatabaseCatalog::instance().getDatabase(database_name);
|
||||
|
||||
if (database->getEngineName() == "Replicated")
|
||||
{
|
||||
auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.table);
|
||||
if (typeid_cast<DatabaseReplicated *>(database.get()) && getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY)
|
||||
|
||||
if (auto* ptr = typeid_cast<DatabaseReplicated *>(database.get());
|
||||
ptr && getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY)
|
||||
{
|
||||
create.database = database_name;
|
||||
guard->releaseTableLock();
|
||||
return typeid_cast<DatabaseReplicated *>(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext());
|
||||
return ptr->tryEnqueueReplicatedDDL(query_ptr, getContext());
|
||||
}
|
||||
}
|
||||
|
||||
@ -926,11 +929,13 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
|
||||
if (need_add_to_database && database->getEngineName() == "Replicated")
|
||||
{
|
||||
auto guard = DatabaseCatalog::instance().getDDLGuard(create.database, create.table);
|
||||
if (typeid_cast<DatabaseReplicated *>(database.get()) && getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY)
|
||||
|
||||
if (auto * ptr = typeid_cast<DatabaseReplicated *>(database.get());
|
||||
ptr && getContext()->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY)
|
||||
{
|
||||
assertOrSetUUID(create, database);
|
||||
guard->releaseTableLock();
|
||||
return typeid_cast<DatabaseReplicated *>(database.get())->tryEnqueueReplicatedDDL(query_ptr, getContext());
|
||||
return ptr->tryEnqueueReplicatedDDL(query_ptr, getContext());
|
||||
}
|
||||
}
|
||||
|
||||
@ -992,8 +997,10 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
|
||||
}
|
||||
|
||||
data_path = database->getTableDataPath(create);
|
||||
|
||||
if (!create.attach && !data_path.empty() && fs::exists(fs::path{getContext()->getPath()} / data_path))
|
||||
throw Exception(storage_already_exists_error_code, "Directory for {} data {} already exists", Poco::toLower(storage_name), String(data_path));
|
||||
throw Exception(storage_already_exists_error_code,
|
||||
"Directory for {} data {} already exists", Poco::toLower(storage_name), String(data_path));
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@ -189,12 +189,11 @@ BlockIO InterpreterInsertQuery::execute()
|
||||
const auto & union_modes = select_query.list_of_modes;
|
||||
|
||||
/// ASTSelectWithUnionQuery is not normalized now, so it may pass some queries which can be Trivial select queries
|
||||
is_trivial_insert_select
|
||||
= std::all_of(
|
||||
union_modes.begin(),
|
||||
union_modes.end(),
|
||||
[](const ASTSelectWithUnionQuery::Mode & mode) { return mode == ASTSelectWithUnionQuery::Mode::ALL; })
|
||||
&& std::all_of(selects.begin(), selects.end(), [](const ASTPtr & select) { return isTrivialSelect(select); });
|
||||
const auto mode_is_all = [](const auto & mode) { return mode == ASTSelectWithUnionQuery::Mode::ALL; };
|
||||
|
||||
is_trivial_insert_select =
|
||||
std::all_of(union_modes.begin(), union_modes.end(), std::move(mode_is_all))
|
||||
&& std::all_of(selects.begin(), selects.end(), isTrivialSelect);
|
||||
}
|
||||
|
||||
if (is_trivial_insert_select)
|
||||
|
||||
@ -633,7 +633,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl()
|
||||
/// Running on the initiating server during distributed processing or if query is not distributed.
|
||||
///
|
||||
/// Also note that with distributed_group_by_no_merge=2 (i.e. when optimize_distributed_group_by_sharding_key takes place)
|
||||
/// the query on the remote server will be processed up to WithMergeableStateAfterAggregation,
|
||||
/// the query on the remote server will be processed up to WithMergeableStateAfterAggregationAndLimit,
|
||||
/// So it will do partial second stage (second_stage=true), and initiator will do the final part.
|
||||
bool second_stage = from_stage <= QueryProcessingStage::WithMergeableState
|
||||
&& options.to_stage > QueryProcessingStage::WithMergeableState;
|
||||
@ -705,7 +705,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl()
|
||||
return res;
|
||||
}
|
||||
|
||||
if (options.to_stage == QueryProcessingStage::Enum::WithMergeableStateAfterAggregation)
|
||||
if (options.to_stage >= QueryProcessingStage::Enum::WithMergeableStateAfterAggregation)
|
||||
{
|
||||
// It's different from selected_columns, see the comment above for
|
||||
// WithMergeableState stage.
|
||||
@ -1012,10 +1012,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
|
||||
|
||||
/// Support optimize_distributed_group_by_sharding_key
|
||||
/// Is running on the initiating server during distributed processing?
|
||||
if (from_stage == QueryProcessingStage::WithMergeableStateAfterAggregation)
|
||||
if (from_stage >= QueryProcessingStage::WithMergeableStateAfterAggregation)
|
||||
from_aggregation_stage = true;
|
||||
/// Is running on remote servers during distributed processing?
|
||||
if (options.to_stage == QueryProcessingStage::WithMergeableStateAfterAggregation)
|
||||
if (options.to_stage >= QueryProcessingStage::WithMergeableStateAfterAggregation)
|
||||
to_aggregation_stage = true;
|
||||
|
||||
/// Read the data from Storage. from_stage - to what stage the request was completed in Storage.
|
||||
@ -1301,7 +1301,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
|
||||
* but there is no aggregation, then on the remote servers ORDER BY was made
|
||||
* - therefore, we merge the sorted streams from remote servers.
|
||||
*
|
||||
* Also in case of remote servers was process the query up to WithMergeableStateAfterAggregation
|
||||
* Also in case of remote servers was process the query up to WithMergeableStateAfterAggregationAndLimit
|
||||
* (distributed_group_by_no_merge=2 or optimize_distributed_group_by_sharding_key=1 takes place),
|
||||
* then merge the sorted streams is enough, since remote servers already did full ORDER BY.
|
||||
*/
|
||||
@ -1335,13 +1335,15 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
|
||||
}
|
||||
}
|
||||
|
||||
bool apply_limit = options.to_stage != QueryProcessingStage::WithMergeableStateAfterAggregation;
|
||||
bool apply_offset = options.to_stage != QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit;
|
||||
bool has_prelimit = false;
|
||||
if (!to_aggregation_stage &&
|
||||
if (apply_limit &&
|
||||
query.limitLength() && !query.limit_with_ties && !hasWithTotalsInAnySubqueryInFromClause(query) &&
|
||||
!query.arrayJoinExpressionList() && !query.distinct && !expressions.hasLimitBy() && !settings.extremes &&
|
||||
!has_withfill)
|
||||
{
|
||||
executePreLimit(query_plan, false);
|
||||
executePreLimit(query_plan, /* do_not_skip_offset= */!apply_offset);
|
||||
has_prelimit = true;
|
||||
}
|
||||
|
||||
@ -1368,7 +1370,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
|
||||
}
|
||||
|
||||
/// Projection not be done on the shards, since then initiator will not find column in blocks.
|
||||
/// (significant only for WithMergeableStateAfterAggregation).
|
||||
/// (significant only for WithMergeableStateAfterAggregation/WithMergeableStateAfterAggregationAndLimit).
|
||||
if (!to_aggregation_stage)
|
||||
{
|
||||
/// We must do projection after DISTINCT because projection may remove some columns.
|
||||
@ -1379,10 +1381,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, const BlockInpu
|
||||
executeExtremes(query_plan);
|
||||
|
||||
/// Limit is no longer needed if there is prelimit.
|
||||
if (!to_aggregation_stage && !has_prelimit)
|
||||
if (apply_limit && !has_prelimit)
|
||||
executeLimit(query_plan);
|
||||
|
||||
if (!to_aggregation_stage)
|
||||
if (apply_offset)
|
||||
executeOffset(query_plan);
|
||||
}
|
||||
}
|
||||
|
||||
@ -43,11 +43,8 @@
|
||||
# include "config_core.h"
|
||||
#endif
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
@ -56,6 +53,7 @@ namespace ErrorCodes
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
extern const int TIMEOUT_EXCEEDED;
|
||||
extern const int TABLE_WAS_NOT_DROPPED;
|
||||
extern const int NO_ZOOKEEPER;
|
||||
}
|
||||
|
||||
|
||||
@ -131,6 +129,8 @@ AccessType getRequiredAccessType(StorageActionBlockType action_type)
|
||||
throw Exception("Unknown action type: " + std::to_string(action_type), ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
constexpr std::string_view table_is_not_replicated = "Table {} is not replicated";
|
||||
|
||||
}
|
||||
|
||||
/// Implements SYSTEM [START|STOP] <something action from ActionLocks>
|
||||
@ -212,11 +212,16 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
system_context->setSetting("profile", getContext()->getSystemProfileName());
|
||||
|
||||
/// Make canonical query for simpler processing
|
||||
if (!query.table.empty())
|
||||
if (query.type == Type::RELOAD_DICTIONARY)
|
||||
{
|
||||
if (!query.database.empty())
|
||||
query.table = query.database + "." + query.table;
|
||||
}
|
||||
else if (!query.table.empty())
|
||||
{
|
||||
table_id = getContext()->resolveStorageID(StorageID(query.database, query.table), Context::ResolveOrdinary);
|
||||
}
|
||||
|
||||
if (!query.target_dictionary.empty() && !query.database.empty())
|
||||
query.target_dictionary = query.database + "." + query.target_dictionary;
|
||||
|
||||
volume_ptr = {};
|
||||
if (!query.storage_policy.empty() && !query.volume.empty())
|
||||
@ -286,7 +291,7 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
getContext()->checkAccess(AccessType::SYSTEM_RELOAD_DICTIONARY);
|
||||
|
||||
auto & external_dictionaries_loader = system_context->getExternalDictionariesLoader();
|
||||
external_dictionaries_loader.reloadDictionary(query.target_dictionary, getContext());
|
||||
external_dictionaries_loader.reloadDictionary(query.table, getContext());
|
||||
|
||||
|
||||
ExternalDictionariesLoader::resetAll();
|
||||
@ -296,8 +301,8 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
{
|
||||
getContext()->checkAccess(AccessType::SYSTEM_RELOAD_DICTIONARY);
|
||||
executeCommandsAndThrowIfError(
|
||||
[&] () { system_context->getExternalDictionariesLoader().reloadAllTriedToLoad(); },
|
||||
[&] () { system_context->getEmbeddedDictionaries().reload(); }
|
||||
[&] { system_context->getExternalDictionariesLoader().reloadAllTriedToLoad(); },
|
||||
[&] { system_context->getEmbeddedDictionaries().reload(); }
|
||||
);
|
||||
ExternalDictionariesLoader::resetAll();
|
||||
break;
|
||||
@ -392,8 +397,10 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
break;
|
||||
case Type::RESTART_REPLICA:
|
||||
if (!tryRestartReplica(table_id, system_context))
|
||||
throw Exception("There is no " + query.database + "." + query.table + " replicated table",
|
||||
ErrorCodes::BAD_ARGUMENTS);
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs());
|
||||
break;
|
||||
case Type::RESTORE_REPLICA:
|
||||
restoreReplica();
|
||||
break;
|
||||
case Type::RESTART_DISK:
|
||||
restartDisk(query.disk);
|
||||
@ -402,14 +409,14 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
{
|
||||
getContext()->checkAccess(AccessType::SYSTEM_FLUSH_LOGS);
|
||||
executeCommandsAndThrowIfError(
|
||||
[&] () { if (auto query_log = getContext()->getQueryLog()) query_log->flush(true); },
|
||||
[&] () { if (auto part_log = getContext()->getPartLog("")) part_log->flush(true); },
|
||||
[&] () { if (auto query_thread_log = getContext()->getQueryThreadLog()) query_thread_log->flush(true); },
|
||||
[&] () { if (auto trace_log = getContext()->getTraceLog()) trace_log->flush(true); },
|
||||
[&] () { if (auto text_log = getContext()->getTextLog()) text_log->flush(true); },
|
||||
[&] () { if (auto metric_log = getContext()->getMetricLog()) metric_log->flush(true); },
|
||||
[&] () { if (auto asynchronous_metric_log = getContext()->getAsynchronousMetricLog()) asynchronous_metric_log->flush(true); },
|
||||
[&] () { if (auto opentelemetry_span_log = getContext()->getOpenTelemetrySpanLog()) opentelemetry_span_log->flush(true); }
|
||||
[&] { if (auto query_log = getContext()->getQueryLog()) query_log->flush(true); },
|
||||
[&] { if (auto part_log = getContext()->getPartLog("")) part_log->flush(true); },
|
||||
[&] { if (auto query_thread_log = getContext()->getQueryThreadLog()) query_thread_log->flush(true); },
|
||||
[&] { if (auto trace_log = getContext()->getTraceLog()) trace_log->flush(true); },
|
||||
[&] { if (auto text_log = getContext()->getTextLog()) text_log->flush(true); },
|
||||
[&] { if (auto metric_log = getContext()->getMetricLog()) metric_log->flush(true); },
|
||||
[&] { if (auto asynchronous_metric_log = getContext()->getAsynchronousMetricLog()) asynchronous_metric_log->flush(true); },
|
||||
[&] { if (auto opentelemetry_span_log = getContext()->getOpenTelemetrySpanLog()) opentelemetry_span_log->flush(true); }
|
||||
);
|
||||
break;
|
||||
}
|
||||
@ -423,12 +430,51 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
return BlockIO();
|
||||
}
|
||||
|
||||
void InterpreterSystemQuery::restoreReplica()
|
||||
{
|
||||
getContext()->checkAccess(AccessType::SYSTEM_RESTORE_REPLICA, table_id);
|
||||
|
||||
const zkutil::ZooKeeperPtr& zookeeper = getContext()->getZooKeeper();
|
||||
|
||||
if (zookeeper->expired())
|
||||
throw Exception(ErrorCodes::NO_ZOOKEEPER,
|
||||
"Cannot restore table metadata because ZooKeeper session has expired");
|
||||
|
||||
const StoragePtr table_ptr = DatabaseCatalog::instance().getTable(table_id, getContext());
|
||||
|
||||
auto * const table_replicated_ptr = dynamic_cast<StorageReplicatedMergeTree *>(table_ptr.get());
|
||||
|
||||
if (table_replicated_ptr == nullptr)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs());
|
||||
|
||||
auto & table_replicated = *table_replicated_ptr;
|
||||
|
||||
StorageReplicatedMergeTree::Status status;
|
||||
table_replicated.getStatus(status);
|
||||
|
||||
if (!status.is_readonly)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Replica must be readonly");
|
||||
|
||||
const String replica_name = table_replicated.getReplicaName();
|
||||
const String& zk_root_path = status.zookeeper_path;
|
||||
|
||||
if (String replica_path = zk_root_path + "replicas/" + replica_name; zookeeper->exists(replica_path))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||
"Replica path is present at {} -- nothing to restore. "
|
||||
"If you are sure that metadata it lost and replica path contain some garbage, "
|
||||
"then use SYSTEM DROP REPLICA query first.", replica_path);
|
||||
|
||||
table_replicated.restoreMetadataInZooKeeper();
|
||||
}
|
||||
|
||||
StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, ContextMutablePtr system_context, bool need_ddl_guard)
|
||||
{
|
||||
getContext()->checkAccess(AccessType::SYSTEM_RESTART_REPLICA, replica);
|
||||
|
||||
auto table_ddl_guard = need_ddl_guard ? DatabaseCatalog::instance().getDDLGuard(replica.getDatabaseName(), replica.getTableName()) : nullptr;
|
||||
auto table_ddl_guard = need_ddl_guard
|
||||
? DatabaseCatalog::instance().getDDLGuard(replica.getDatabaseName(), replica.getTableName())
|
||||
: nullptr;
|
||||
|
||||
auto [database, table] = DatabaseCatalog::instance().tryGetDatabaseAndTable(replica, getContext());
|
||||
ASTPtr create_ast;
|
||||
|
||||
@ -475,28 +521,23 @@ void InterpreterSystemQuery::restartReplicas(ContextMutablePtr system_context)
|
||||
auto & catalog = DatabaseCatalog::instance();
|
||||
|
||||
for (auto & elem : catalog.getDatabases())
|
||||
{
|
||||
DatabasePtr & database = elem.second;
|
||||
for (auto iterator = database->getTablesIterator(getContext()); iterator->isValid(); iterator->next())
|
||||
{
|
||||
if (auto table = iterator->table())
|
||||
{
|
||||
if (dynamic_cast<const StorageReplicatedMergeTree *>(table.get()))
|
||||
replica_names.emplace_back(StorageID{iterator->databaseName(), iterator->name()});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto it = elem.second->getTablesIterator(getContext()); it->isValid(); it->next())
|
||||
if (dynamic_cast<const StorageReplicatedMergeTree *>(it->table().get()))
|
||||
replica_names.emplace_back(it->databaseName(), it->name());
|
||||
|
||||
if (replica_names.empty())
|
||||
return;
|
||||
|
||||
TableGuards guards;
|
||||
|
||||
for (const auto & name : replica_names)
|
||||
guards.emplace(UniqueTableName{name.database_name, name.table_name}, nullptr);
|
||||
|
||||
for (auto & guard : guards)
|
||||
guard.second = catalog.getDDLGuard(guard.first.database_name, guard.first.table_name);
|
||||
|
||||
ThreadPool pool(std::min(size_t(getNumberOfPhysicalCPUCores()), replica_names.size()));
|
||||
|
||||
for (auto & replica : replica_names)
|
||||
{
|
||||
LOG_TRACE(log, "Restarting replica on {}", replica.getNameForLogs());
|
||||
@ -516,7 +557,7 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query)
|
||||
StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext());
|
||||
|
||||
if (!dropReplicaImpl(query, table))
|
||||
throw Exception("Table " + table_id.getNameForLogs() + " is not replicated", ErrorCodes::BAD_ARGUMENTS);
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs());
|
||||
}
|
||||
else if (!query.database.empty())
|
||||
{
|
||||
@ -628,7 +669,7 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery &)
|
||||
LOG_TRACE(log, "SYNC REPLICA {}: OK", table_id.getNameForLogs());
|
||||
}
|
||||
else
|
||||
throw Exception("Table " + table_id.getNameForLogs() + " is not replicated", ErrorCodes::BAD_ARGUMENTS);
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs());
|
||||
}
|
||||
|
||||
void InterpreterSystemQuery::flushDistributed(ASTSystemQuery &)
|
||||
@ -659,6 +700,7 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
|
||||
const auto & query = query_ptr->as<const ASTSystemQuery &>();
|
||||
using Type = ASTSystemQuery::Type;
|
||||
AccessRightsElements required_access;
|
||||
|
||||
switch (query.type)
|
||||
{
|
||||
case Type::SHUTDOWN: [[fallthrough]];
|
||||
@ -770,6 +812,11 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
|
||||
required_access.emplace_back(AccessType::SYSTEM_DROP_REPLICA, query.database, query.table);
|
||||
break;
|
||||
}
|
||||
case Type::RESTORE_REPLICA:
|
||||
{
|
||||
required_access.emplace_back(AccessType::SYSTEM_RESTORE_REPLICA, query.database, query.table);
|
||||
break;
|
||||
}
|
||||
case Type::SYNC_REPLICA:
|
||||
{
|
||||
required_access.emplace_back(AccessType::SYSTEM_SYNC_REPLICA, query.database, query.table);
|
||||
|
||||
@ -49,6 +49,9 @@ private:
|
||||
|
||||
void restartReplicas(ContextMutablePtr system_context);
|
||||
void syncReplica(ASTSystemQuery & query);
|
||||
|
||||
void restoreReplica();
|
||||
|
||||
void dropReplica(ASTSystemQuery & query);
|
||||
bool dropReplicaImpl(ASTSystemQuery & query, const StoragePtr & table);
|
||||
void flushDistributed(ASTSystemQuery & query);
|
||||
|
||||
@ -102,12 +102,10 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context,
|
||||
|
||||
/// The current database in a distributed query need to be replaced with either
|
||||
/// the local current database or a shard's default database.
|
||||
bool need_replace_current_database
|
||||
= (std::find_if(
|
||||
bool need_replace_current_database = std::any_of(
|
||||
query_requires_access.begin(),
|
||||
query_requires_access.end(),
|
||||
[](const AccessRightsElement & elem) { return elem.isEmptyDatabase(); })
|
||||
!= query_requires_access.end());
|
||||
[](const AccessRightsElement & elem) { return elem.isEmptyDatabase(); });
|
||||
|
||||
bool use_local_default_database = false;
|
||||
const String & current_database = context->getCurrentDatabase();
|
||||
|
||||
@ -113,6 +113,7 @@ Block getHeaderForProcessingStage(
|
||||
case QueryProcessingStage::WithMergeableState:
|
||||
case QueryProcessingStage::Complete:
|
||||
case QueryProcessingStage::WithMergeableStateAfterAggregation:
|
||||
case QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit:
|
||||
case QueryProcessingStage::MAX:
|
||||
{
|
||||
auto query = query_info.query->clone();
|
||||
|
||||
@ -44,6 +44,8 @@ const char * ASTSystemQuery::typeToString(Type type)
|
||||
return "RESTART REPLICAS";
|
||||
case Type::RESTART_REPLICA:
|
||||
return "RESTART REPLICA";
|
||||
case Type::RESTORE_REPLICA:
|
||||
return "RESTORE REPLICA";
|
||||
case Type::DROP_REPLICA:
|
||||
return "DROP REPLICA";
|
||||
case Type::SYNC_REPLICA:
|
||||
@ -119,18 +121,6 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &,
|
||||
<< (settings.hilite ? hilite_none : "");
|
||||
};
|
||||
|
||||
auto print_database_dictionary = [&]
|
||||
{
|
||||
settings.ostr << " ";
|
||||
if (!database.empty())
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(database)
|
||||
<< (settings.hilite ? hilite_none : "") << ".";
|
||||
}
|
||||
settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(target_dictionary)
|
||||
<< (settings.hilite ? hilite_none : "");
|
||||
};
|
||||
|
||||
auto print_drop_replica = [&]
|
||||
{
|
||||
settings.ostr << " " << quoteString(replica);
|
||||
@ -187,14 +177,14 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &,
|
||||
else if (!volume.empty())
|
||||
print_on_volume();
|
||||
}
|
||||
else if (type == Type::RESTART_REPLICA || type == Type::SYNC_REPLICA || type == Type::FLUSH_DISTRIBUTED)
|
||||
else if ( type == Type::RESTART_REPLICA
|
||||
|| type == Type::RESTORE_REPLICA
|
||||
|| type == Type::SYNC_REPLICA
|
||||
|| type == Type::FLUSH_DISTRIBUTED
|
||||
|| type == Type::RELOAD_DICTIONARY)
|
||||
{
|
||||
print_database_table();
|
||||
}
|
||||
else if (type == Type::RELOAD_DICTIONARY)
|
||||
{
|
||||
print_database_dictionary();
|
||||
}
|
||||
else if (type == Type::DROP_REPLICA)
|
||||
{
|
||||
print_drop_replica();
|
||||
|
||||
@ -32,6 +32,7 @@ public:
|
||||
START_LISTEN_QUERIES,
|
||||
RESTART_REPLICAS,
|
||||
RESTART_REPLICA,
|
||||
RESTORE_REPLICA,
|
||||
DROP_REPLICA,
|
||||
SYNC_REPLICA,
|
||||
RELOAD_DICTIONARY,
|
||||
@ -65,7 +66,6 @@ public:
|
||||
|
||||
Type type = Type::UNKNOWN;
|
||||
|
||||
String target_dictionary;
|
||||
String target_model;
|
||||
String database;
|
||||
String table;
|
||||
|
||||
@ -489,14 +489,12 @@ bool ParserPrefixUnaryOperatorExpression::parseImpl(Pos & pos, ASTPtr & node, Ex
|
||||
/** This is done, because among the unary operators there is only a minus and NOT.
|
||||
* But for a minus the chain of unary operators does not need to be supported.
|
||||
*/
|
||||
size_t count = 1;
|
||||
if (it[0] && 0 == strncmp(it[0], "NOT", 3))
|
||||
{
|
||||
/// Was there an even number of NOTs.
|
||||
bool even = false;
|
||||
|
||||
const char ** jt;
|
||||
while (true)
|
||||
{
|
||||
const char ** jt;
|
||||
for (jt = operators; *jt; jt += 2)
|
||||
if (parseOperator(pos, *jt, expected))
|
||||
break;
|
||||
@ -504,11 +502,8 @@ bool ParserPrefixUnaryOperatorExpression::parseImpl(Pos & pos, ASTPtr & node, Ex
|
||||
if (!*jt)
|
||||
break;
|
||||
|
||||
even = !even;
|
||||
++count;
|
||||
}
|
||||
|
||||
if (even)
|
||||
it = jt; /// Zero the result of parsing the first NOT. It turns out, as if there is no `NOT` chain at all.
|
||||
}
|
||||
|
||||
ASTPtr elem;
|
||||
@ -518,6 +513,8 @@ bool ParserPrefixUnaryOperatorExpression::parseImpl(Pos & pos, ASTPtr & node, Ex
|
||||
if (!*it)
|
||||
node = elem;
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < count; ++i)
|
||||
{
|
||||
/// the function corresponding to the operator
|
||||
auto function = std::make_shared<ASTFunction>();
|
||||
@ -529,10 +526,14 @@ bool ParserPrefixUnaryOperatorExpression::parseImpl(Pos & pos, ASTPtr & node, Ex
|
||||
function->arguments = exp_list;
|
||||
function->children.push_back(exp_list);
|
||||
|
||||
if (node)
|
||||
exp_list->children.push_back(node);
|
||||
else
|
||||
exp_list->children.push_back(elem);
|
||||
|
||||
node = function;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -133,7 +133,7 @@ ASTPtr SystemQuery::convertToOld() const
|
||||
{
|
||||
auto table = std::static_pointer_cast<ASTTableIdentifier>(get(TABLE)->convertToOld());
|
||||
query->database = table->getDatabaseName();
|
||||
query->target_dictionary = table->shortName();
|
||||
query->table = table->getTableId().table_name;
|
||||
}
|
||||
break;
|
||||
case QueryType::REPLICATED_SENDS:
|
||||
|
||||
@ -95,21 +95,18 @@ bool ParserRenameQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
|
||||
ASTRenameQuery::Elements elements;
|
||||
|
||||
auto ignore_delim = [&]()
|
||||
{
|
||||
return exchange ? s_and.ignore(pos) : s_to.ignore(pos);
|
||||
};
|
||||
const auto ignore_delim = [&] { return exchange ? s_and.ignore(pos) : s_to.ignore(pos); };
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (!elements.empty() && !s_comma.ignore(pos))
|
||||
break;
|
||||
|
||||
elements.push_back(ASTRenameQuery::Element());
|
||||
ASTRenameQuery::Element& ref = elements.emplace_back();
|
||||
|
||||
if (!parseDatabaseAndTable(elements.back().from, pos, expected)
|
||||
if (!parseDatabaseAndTable(ref.from, pos, expected)
|
||||
|| !ignore_delim()
|
||||
|| !parseDatabaseAndTable(elements.back().to, pos, expected))
|
||||
|| !parseDatabaseAndTable(ref.to, pos, expected))
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@ -15,6 +15,47 @@ namespace ErrorCodes
|
||||
namespace DB
|
||||
{
|
||||
|
||||
static bool parseQueryWithOnClusterAndMaybeTable(std::shared_ptr<ASTSystemQuery> & res, IParser::Pos & pos,
|
||||
Expected & expected, bool require_table, bool allow_string_literal)
|
||||
{
|
||||
/// Better form for user: SYSTEM <ACTION> table ON CLUSTER cluster
|
||||
/// Query rewritten form + form while executing on cluster: SYSTEM <ACTION> ON CLUSTER cluster table
|
||||
/// Need to support both
|
||||
String cluster;
|
||||
bool parsed_on_cluster = false;
|
||||
|
||||
if (ParserKeyword{"ON"}.ignore(pos, expected))
|
||||
{
|
||||
if (!ASTQueryWithOnCluster::parse(pos, cluster, expected))
|
||||
return false;
|
||||
parsed_on_cluster = true;
|
||||
}
|
||||
|
||||
bool parsed_table = false;
|
||||
if (allow_string_literal)
|
||||
{
|
||||
ASTPtr ast;
|
||||
if (ParserStringLiteral{}.parse(pos, ast, expected))
|
||||
{
|
||||
res->database = {};
|
||||
res->table = ast->as<ASTLiteral &>().value.safeGet<String>();
|
||||
parsed_table = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!parsed_table)
|
||||
parsed_table = parseDatabaseAndTableName(pos, expected, res->database, res->table);
|
||||
|
||||
if (!parsed_table && require_table)
|
||||
return false;
|
||||
|
||||
if (!parsed_on_cluster && ParserKeyword{"ON"}.ignore(pos, expected))
|
||||
if (!ASTQueryWithOnCluster::parse(pos, cluster, expected))
|
||||
return false;
|
||||
|
||||
res->cluster = cluster;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected)
|
||||
{
|
||||
@ -43,17 +84,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &
|
||||
{
|
||||
case Type::RELOAD_DICTIONARY:
|
||||
{
|
||||
String cluster_str;
|
||||
if (ParserKeyword{"ON"}.ignore(pos, expected))
|
||||
{
|
||||
if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected))
|
||||
return false;
|
||||
}
|
||||
res->cluster = cluster_str;
|
||||
ASTPtr ast;
|
||||
if (ParserStringLiteral{}.parse(pos, ast, expected))
|
||||
res->target_dictionary = ast->as<ASTLiteral &>().value.safeGet<String>();
|
||||
else if (!parseDatabaseAndTableName(pos, expected, res->database, res->target_dictionary))
|
||||
if (!parseQueryWithOnClusterAndMaybeTable(res, pos, expected, /* require table = */ true, /* allow_string_literal = */ true))
|
||||
return false;
|
||||
break;
|
||||
}
|
||||
@ -145,24 +176,21 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected &
|
||||
break;
|
||||
}
|
||||
|
||||
case Type::STOP_DISTRIBUTED_SENDS:
|
||||
case Type::START_DISTRIBUTED_SENDS:
|
||||
case Type::FLUSH_DISTRIBUTED:
|
||||
{
|
||||
String cluster_str;
|
||||
if (ParserKeyword{"ON"}.ignore(pos, expected))
|
||||
{
|
||||
if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected))
|
||||
return false;
|
||||
}
|
||||
res->cluster = cluster_str;
|
||||
if (!parseDatabaseAndTableName(pos, expected, res->database, res->table))
|
||||
{
|
||||
/// FLUSH DISTRIBUTED requires table
|
||||
/// START/STOP DISTRIBUTED SENDS does not require table
|
||||
if (res->type == Type::FLUSH_DISTRIBUTED)
|
||||
case Type::STOP_DISTRIBUTED_SENDS:
|
||||
case Type::START_DISTRIBUTED_SENDS:
|
||||
{
|
||||
if (!parseQueryWithOnClusterAndMaybeTable(res, pos, expected, /* require table = */ false, /* allow_string_literal = */ false))
|
||||
return false;
|
||||
break;
|
||||
}
|
||||
|
||||
case Type::FLUSH_DISTRIBUTED:
|
||||
case Type::RESTORE_REPLICA:
|
||||
{
|
||||
if (!parseQueryWithOnClusterAndMaybeTable(res, pos, expected, /* require table = */ true, /* allow_string_literal = */ false))
|
||||
return false;
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#include "ArrowBlockInputFormat.h"
|
||||
|
||||
#if USE_ARROW
|
||||
|
||||
#include <Formats/FormatFactory.h>
|
||||
@ -29,7 +30,6 @@ ArrowBlockInputFormat::ArrowBlockInputFormat(ReadBuffer & in_, const Block & hea
|
||||
Chunk ArrowBlockInputFormat::generate()
|
||||
{
|
||||
Chunk res;
|
||||
const Block & header = getPort().getHeader();
|
||||
arrow::Result<std::shared_ptr<arrow::RecordBatch>> batch_result;
|
||||
|
||||
if (stream)
|
||||
@ -63,7 +63,7 @@ Chunk ArrowBlockInputFormat::generate()
|
||||
|
||||
++record_batch_current;
|
||||
|
||||
ArrowColumnToCHColumn::arrowTableToCHChunk(res, *table_result, header, "Arrow");
|
||||
arrow_column_to_ch_column->arrowTableToCHChunk(res, *table_result);
|
||||
|
||||
return res;
|
||||
}
|
||||
@ -81,6 +81,8 @@ void ArrowBlockInputFormat::resetParser()
|
||||
|
||||
void ArrowBlockInputFormat::prepareReader()
|
||||
{
|
||||
std::shared_ptr<arrow::Schema> schema;
|
||||
|
||||
if (stream)
|
||||
{
|
||||
auto stream_reader_status = arrow::ipc::RecordBatchStreamReader::Open(std::make_unique<ArrowInputStreamFromReadBuffer>(in));
|
||||
@ -88,6 +90,7 @@ void ArrowBlockInputFormat::prepareReader()
|
||||
throw Exception(ErrorCodes::UNKNOWN_EXCEPTION,
|
||||
"Error while opening a table: {}", stream_reader_status.status().ToString());
|
||||
stream_reader = *stream_reader_status;
|
||||
schema = stream_reader->schema();
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -96,8 +99,11 @@ void ArrowBlockInputFormat::prepareReader()
|
||||
throw Exception(ErrorCodes::UNKNOWN_EXCEPTION,
|
||||
"Error while opening a table: {}", file_reader_status.status().ToString());
|
||||
file_reader = *file_reader_status;
|
||||
schema = file_reader->schema();
|
||||
}
|
||||
|
||||
arrow_column_to_ch_column = std::make_unique<ArrowColumnToCHColumn>(getPort().getHeader(), std::move(schema), "Arrow");
|
||||
|
||||
if (stream)
|
||||
record_batch_total = -1;
|
||||
else
|
||||
|
||||
@ -11,6 +11,7 @@ namespace DB
|
||||
{
|
||||
|
||||
class ReadBuffer;
|
||||
class ArrowColumnToCHColumn;
|
||||
|
||||
class ArrowBlockInputFormat : public IInputFormat
|
||||
{
|
||||
@ -32,6 +33,8 @@ private:
|
||||
// The following fields are used only for Arrow format
|
||||
std::shared_ptr<arrow::ipc::RecordBatchFileReader> file_reader;
|
||||
|
||||
std::unique_ptr<ArrowColumnToCHColumn> arrow_column_to_ch_column;
|
||||
|
||||
int record_batch_total = 0;
|
||||
int record_batch_current = 0;
|
||||
|
||||
|
||||
@ -18,17 +18,26 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
ArrowBlockOutputFormat::ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, bool stream_, const FormatSettings & format_settings_)
|
||||
: IOutputFormat(header_, out_), stream{stream_}, format_settings{format_settings_}, arrow_ostream{std::make_shared<ArrowBufferedOutputStream>(out_)}
|
||||
: IOutputFormat(header_, out_)
|
||||
, stream{stream_}
|
||||
, format_settings{format_settings_}
|
||||
, arrow_ostream{std::make_shared<ArrowBufferedOutputStream>(out_)}
|
||||
{
|
||||
}
|
||||
|
||||
void ArrowBlockOutputFormat::consume(Chunk chunk)
|
||||
{
|
||||
const Block & header = getPort(PortKind::Main).getHeader();
|
||||
const size_t columns_num = chunk.getNumColumns();
|
||||
std::shared_ptr<arrow::Table> arrow_table;
|
||||
|
||||
CHColumnToArrowColumn::chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Arrow");
|
||||
if (!ch_column_to_arrow_column)
|
||||
{
|
||||
const Block & header = getPort(PortKind::Main).getHeader();
|
||||
ch_column_to_arrow_column
|
||||
= std::make_unique<CHColumnToArrowColumn>(header, "Arrow", format_settings.arrow.low_cardinality_as_dictionary);
|
||||
}
|
||||
|
||||
ch_column_to_arrow_column->chChunkToArrowTable(arrow_table, chunk, columns_num);
|
||||
|
||||
if (!writer)
|
||||
prepareWriter(arrow_table->schema());
|
||||
|
||||
@ -12,6 +12,8 @@ namespace arrow::ipc { class RecordBatchWriter; }
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class CHColumnToArrowColumn;
|
||||
|
||||
class ArrowBlockOutputFormat : public IOutputFormat
|
||||
{
|
||||
public:
|
||||
@ -28,6 +30,7 @@ private:
|
||||
const FormatSettings format_settings;
|
||||
std::shared_ptr<ArrowBufferedOutputStream> arrow_ostream;
|
||||
std::shared_ptr<arrow::ipc::RecordBatchWriter> writer;
|
||||
std::unique_ptr<CHColumnToArrowColumn> ch_column_to_arrow_column;
|
||||
|
||||
void prepareWriter(const std::shared_ptr<arrow::Schema> & schema);
|
||||
};
|
||||
|
||||
@ -7,15 +7,22 @@
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeLowCardinality.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypeMap.h>
|
||||
#include <common/DateLUTImpl.h>
|
||||
#include <common/types.h>
|
||||
#include <Core/Block.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnNullable.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Columns/ColumnLowCardinality.h>
|
||||
#include <Columns/ColumnUnique.h>
|
||||
#include <Columns/ColumnMap.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
#include <algorithm>
|
||||
#include <DataTypes/DataTypeLowCardinality.h>
|
||||
#include <fmt/format.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -27,6 +34,7 @@ namespace DB
|
||||
extern const int CANNOT_CONVERT_TYPE;
|
||||
extern const int CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN;
|
||||
extern const int THERE_IS_NO_COLUMN;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
static const std::initializer_list<std::pair<arrow::Type::type, const char *>> arrow_type_to_internal_type =
|
||||
@ -152,11 +160,11 @@ namespace DB
|
||||
if (days_num > DATE_LUT_MAX_DAY_NUM)
|
||||
{
|
||||
// TODO: will it rollback correctly?
|
||||
throw Exception{"Input value " + std::to_string(days_num) + " of a column \"" + internal_column.getName()
|
||||
+ "\" is greater than "
|
||||
"max allowed Date value, which is "
|
||||
+ std::to_string(DATE_LUT_MAX_DAY_NUM),
|
||||
ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE};
|
||||
throw Exception
|
||||
{
|
||||
fmt::format("Input value {} of a column \"{}\" is greater than max allowed Date value, which is {}", days_num, internal_column.getName(), DATE_LUT_MAX_DAY_NUM),
|
||||
ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE
|
||||
};
|
||||
}
|
||||
|
||||
column_data.emplace_back(days_num);
|
||||
@ -263,23 +271,47 @@ namespace DB
|
||||
offsets_data.emplace_back(start + arrow_offsets.Value(i));
|
||||
}
|
||||
}
|
||||
static ColumnPtr createAndFillColumnWithIndexesData(std::shared_ptr<arrow::ChunkedArray> & arrow_column)
|
||||
{
|
||||
switch (arrow_column->type()->id())
|
||||
{
|
||||
# define DISPATCH(ARROW_NUMERIC_TYPE, CPP_NUMERIC_TYPE) \
|
||||
case ARROW_NUMERIC_TYPE: \
|
||||
{ \
|
||||
auto column = DataTypeNumber<CPP_NUMERIC_TYPE>().createColumn(); \
|
||||
fillColumnWithNumericData<CPP_NUMERIC_TYPE>(arrow_column, *column); \
|
||||
return column; \
|
||||
}
|
||||
FOR_ARROW_INDEXES_TYPES(DISPATCH)
|
||||
# undef DISPATCH
|
||||
default:
|
||||
throw Exception(fmt::format("Unsupported type for indexes in LowCardinality: {}.", arrow_column->type()->name()), ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
}
|
||||
|
||||
static void readColumnFromArrowColumn(std::shared_ptr<arrow::ChunkedArray> & arrow_column, IColumn & internal_column, const std::string & column_name, const std::string format_name, bool is_nullable)
|
||||
static void readColumnFromArrowColumn(
|
||||
std::shared_ptr<arrow::ChunkedArray> & arrow_column,
|
||||
IColumn & internal_column,
|
||||
const std::string & column_name,
|
||||
const std::string & format_name,
|
||||
bool is_nullable,
|
||||
std::unordered_map<String, ColumnPtr> dictionary_values)
|
||||
{
|
||||
if (internal_column.isNullable())
|
||||
{
|
||||
ColumnNullable & column_nullable = typeid_cast<ColumnNullable &>(internal_column);
|
||||
readColumnFromArrowColumn(arrow_column, column_nullable.getNestedColumn(), column_name, format_name, true);
|
||||
ColumnNullable & column_nullable = assert_cast<ColumnNullable &>(internal_column);
|
||||
readColumnFromArrowColumn(arrow_column, column_nullable.getNestedColumn(), column_name, format_name, true, dictionary_values);
|
||||
fillByteMapFromArrowColumn(arrow_column, column_nullable.getNullMapColumn());
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: check if a column is const?
|
||||
if (!is_nullable && !checkColumn<ColumnArray>(internal_column) && arrow_column->null_count())
|
||||
/// TODO: check if a column is const?
|
||||
if (!is_nullable && arrow_column->null_count() && arrow_column->type()->id() != arrow::Type::LIST
|
||||
&& arrow_column->type()->id() != arrow::Type::MAP && arrow_column->type()->id() != arrow::Type::STRUCT)
|
||||
{
|
||||
throw Exception
|
||||
{
|
||||
"Can not insert NULL data into non-nullable column \"" + column_name + "\"",
|
||||
fmt::format("Can not insert NULL data into non-nullable column \"{}\".", column_name),
|
||||
ErrorCodes::CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN
|
||||
};
|
||||
}
|
||||
@ -304,13 +336,11 @@ namespace DB
|
||||
fillColumnWithTimestampData(arrow_column, internal_column);
|
||||
break;
|
||||
case arrow::Type::DECIMAL:
|
||||
//fillColumnWithNumericData<Decimal128, ColumnDecimal<Decimal128>>(arrow_column, read_column); // Have problems with trash values under NULL, but faster
|
||||
fillColumnWithDecimalData(arrow_column, internal_column /*, internal_nested_type*/);
|
||||
break;
|
||||
case arrow::Type::MAP: [[fallthrough]];
|
||||
case arrow::Type::LIST:
|
||||
{
|
||||
const auto * list_type = static_cast<arrow::ListType *>(arrow_column->type().get());
|
||||
auto list_nested_type = list_type->value_type();
|
||||
arrow::ArrayVector array_vector;
|
||||
array_vector.reserve(arrow_column->num_chunks());
|
||||
for (size_t chunk_i = 0, num_chunks = static_cast<size_t>(arrow_column->num_chunks()); chunk_i < num_chunks; ++chunk_i)
|
||||
@ -321,11 +351,70 @@ namespace DB
|
||||
}
|
||||
auto arrow_nested_column = std::make_shared<arrow::ChunkedArray>(array_vector);
|
||||
|
||||
ColumnArray & column_array = typeid_cast<ColumnArray &>(internal_column);
|
||||
readColumnFromArrowColumn(arrow_nested_column, column_array.getData(), column_name, format_name, false);
|
||||
ColumnArray & column_array = arrow_column->type()->id() == arrow::Type::MAP
|
||||
? assert_cast<ColumnMap &>(internal_column).getNestedColumn()
|
||||
: assert_cast<ColumnArray &>(internal_column);
|
||||
|
||||
readColumnFromArrowColumn(arrow_nested_column, column_array.getData(), column_name, format_name, false, dictionary_values);
|
||||
fillOffsetsFromArrowListColumn(arrow_column, column_array.getOffsetsColumn());
|
||||
break;
|
||||
}
|
||||
case arrow::Type::STRUCT:
|
||||
{
|
||||
ColumnTuple & column_tuple = assert_cast<ColumnTuple &>(internal_column);
|
||||
int fields_count = column_tuple.tupleSize();
|
||||
std::vector<arrow::ArrayVector> nested_arrow_columns(fields_count);
|
||||
for (size_t chunk_i = 0, num_chunks = static_cast<size_t>(arrow_column->num_chunks()); chunk_i < num_chunks; ++chunk_i)
|
||||
{
|
||||
arrow::StructArray & struct_chunk = static_cast<arrow::StructArray &>(*(arrow_column->chunk(chunk_i)));
|
||||
for (int i = 0; i < fields_count; ++i)
|
||||
nested_arrow_columns[i].emplace_back(struct_chunk.field(i));
|
||||
}
|
||||
|
||||
for (int i = 0; i != fields_count; ++i)
|
||||
{
|
||||
auto nested_arrow_column = std::make_shared<arrow::ChunkedArray>(nested_arrow_columns[i]);
|
||||
readColumnFromArrowColumn(nested_arrow_column, column_tuple.getColumn(i), column_name, format_name, false, dictionary_values);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case arrow::Type::DICTIONARY:
|
||||
{
|
||||
ColumnLowCardinality & column_lc = assert_cast<ColumnLowCardinality &>(internal_column);
|
||||
auto & dict_values = dictionary_values[column_name];
|
||||
/// Load dictionary values only once and reuse it.
|
||||
if (!dict_values)
|
||||
{
|
||||
arrow::ArrayVector dict_array;
|
||||
for (size_t chunk_i = 0, num_chunks = static_cast<size_t>(arrow_column->num_chunks()); chunk_i < num_chunks; ++chunk_i)
|
||||
{
|
||||
arrow::DictionaryArray & dict_chunk = static_cast<arrow::DictionaryArray &>(*(arrow_column->chunk(chunk_i)));
|
||||
dict_array.emplace_back(dict_chunk.dictionary());
|
||||
}
|
||||
auto arrow_dict_column = std::make_shared<arrow::ChunkedArray>(dict_array);
|
||||
|
||||
auto dict_column = IColumn::mutate(column_lc.getDictionaryPtr());
|
||||
auto * uniq_column = static_cast<IColumnUnique *>(dict_column.get());
|
||||
auto values_column = uniq_column->getNestedColumn()->cloneEmpty();
|
||||
readColumnFromArrowColumn(arrow_dict_column, *values_column, column_name, format_name, false, dictionary_values);
|
||||
uniq_column->uniqueInsertRangeFrom(*values_column, 0, values_column->size());
|
||||
dict_values = std::move(dict_column);
|
||||
}
|
||||
|
||||
arrow::ArrayVector indexes_array;
|
||||
for (size_t chunk_i = 0, num_chunks = static_cast<size_t>(arrow_column->num_chunks()); chunk_i < num_chunks; ++chunk_i)
|
||||
{
|
||||
arrow::DictionaryArray & dict_chunk = static_cast<arrow::DictionaryArray &>(*(arrow_column->chunk(chunk_i)));
|
||||
indexes_array.emplace_back(dict_chunk.indices());
|
||||
}
|
||||
|
||||
auto arrow_indexes_column = std::make_shared<arrow::ChunkedArray>(indexes_array);
|
||||
auto indexes_column = createAndFillColumnWithIndexesData(arrow_indexes_column);
|
||||
|
||||
auto new_column_lc = ColumnLowCardinality::create(dict_values, std::move(indexes_column));
|
||||
column_lc = std::move(*new_column_lc);
|
||||
break;
|
||||
}
|
||||
# define DISPATCH(ARROW_NUMERIC_TYPE, CPP_NUMERIC_TYPE) \
|
||||
case ARROW_NUMERIC_TYPE: \
|
||||
fillColumnWithNumericData<CPP_NUMERIC_TYPE>(arrow_column, internal_column); \
|
||||
@ -339,8 +428,7 @@ namespace DB
|
||||
default:
|
||||
throw Exception
|
||||
{
|
||||
"Unsupported " + format_name + " type \"" + arrow_column->type()->name() + "\" of an input column \""
|
||||
+ column_name + "\"",
|
||||
fmt::format(R"(Unsupported {} type "{}" of an input column "{}".)", format_name, arrow_column->type()->name(), column_name),
|
||||
ErrorCodes::UNKNOWN_TYPE
|
||||
};
|
||||
}
|
||||
@ -350,7 +438,7 @@ namespace DB
|
||||
{
|
||||
if (column_type->isNullable())
|
||||
{
|
||||
DataTypePtr nested_type = typeid_cast<const DataTypeNullable *>(column_type.get())->getNestedType();
|
||||
DataTypePtr nested_type = assert_cast<const DataTypeNullable *>(column_type.get())->getNestedType();
|
||||
return makeNullable(getInternalType(arrow_type, nested_type, column_name, format_name));
|
||||
}
|
||||
|
||||
@ -367,11 +455,61 @@ namespace DB
|
||||
|
||||
const DataTypeArray * array_type = typeid_cast<const DataTypeArray *>(column_type.get());
|
||||
if (!array_type)
|
||||
throw Exception{"Cannot convert arrow LIST type to a not Array ClickHouse type " + column_type->getName(), ErrorCodes::CANNOT_CONVERT_TYPE};
|
||||
throw Exception{fmt::format("Cannot convert arrow LIST type to a not Array ClickHouse type {}.", column_type->getName()), ErrorCodes::CANNOT_CONVERT_TYPE};
|
||||
|
||||
return std::make_shared<DataTypeArray>(getInternalType(list_nested_type, array_type->getNestedType(), column_name, format_name));
|
||||
}
|
||||
|
||||
if (arrow_type->id() == arrow::Type::STRUCT)
|
||||
{
|
||||
const auto * struct_type = static_cast<arrow::StructType *>(arrow_type.get());
|
||||
const DataTypeTuple * tuple_type = typeid_cast<const DataTypeTuple *>(column_type.get());
|
||||
if (!tuple_type)
|
||||
throw Exception{fmt::format("Cannot convert arrow STRUCT type to a not Tuple ClickHouse type {}.", column_type->getName()), ErrorCodes::CANNOT_CONVERT_TYPE};
|
||||
|
||||
const DataTypes & tuple_nested_types = tuple_type->getElements();
|
||||
int internal_fields_num = tuple_nested_types.size();
|
||||
/// If internal column has less elements then arrow struct, we will select only first internal_fields_num columns.
|
||||
if (internal_fields_num > struct_type->num_fields())
|
||||
throw Exception
|
||||
{
|
||||
fmt::format(
|
||||
"Cannot convert arrow STRUCT with {} fields to a ClickHouse Tuple with {} elements: {}.",
|
||||
struct_type->num_fields(),
|
||||
internal_fields_num,
|
||||
column_type->getName()),
|
||||
ErrorCodes::CANNOT_CONVERT_TYPE
|
||||
};
|
||||
|
||||
DataTypes nested_types;
|
||||
for (int i = 0; i < internal_fields_num; ++i)
|
||||
nested_types.push_back(getInternalType(struct_type->field(i)->type(), tuple_nested_types[i], column_name, format_name));
|
||||
|
||||
return std::make_shared<DataTypeTuple>(std::move(nested_types));
|
||||
}
|
||||
|
||||
if (arrow_type->id() == arrow::Type::DICTIONARY)
|
||||
{
|
||||
const auto * arrow_dict_type = static_cast<arrow::DictionaryType *>(arrow_type.get());
|
||||
const auto * lc_type = typeid_cast<const DataTypeLowCardinality *>(column_type.get());
|
||||
/// We allow to insert arrow dictionary into a non-LowCardinality column.
|
||||
const auto & dict_type = lc_type ? lc_type->getDictionaryType() : column_type;
|
||||
return std::make_shared<DataTypeLowCardinality>(getInternalType(arrow_dict_type->value_type(), dict_type, column_name, format_name));
|
||||
}
|
||||
|
||||
if (arrow_type->id() == arrow::Type::MAP)
|
||||
{
|
||||
const auto * arrow_map_type = typeid_cast<arrow::MapType *>(arrow_type.get());
|
||||
const auto * map_type = typeid_cast<const DataTypeMap *>(column_type.get());
|
||||
if (!map_type)
|
||||
throw Exception{fmt::format("Cannot convert arrow MAP type to a not Map ClickHouse type {}.", column_type->getName()), ErrorCodes::CANNOT_CONVERT_TYPE};
|
||||
|
||||
return std::make_shared<DataTypeMap>(
|
||||
getInternalType(arrow_map_type->key_type(), map_type->getKeyType(), column_name, format_name),
|
||||
getInternalType(arrow_map_type->item_type(), map_type->getValueType(), column_name, format_name)
|
||||
);
|
||||
}
|
||||
|
||||
if (const auto * internal_type_it = std::find_if(arrow_type_to_internal_type.begin(), arrow_type_to_internal_type.end(),
|
||||
[=](auto && elem) { return elem.first == arrow_type->id(); });
|
||||
internal_type_it != arrow_type_to_internal_type.end())
|
||||
@ -380,13 +518,24 @@ namespace DB
|
||||
}
|
||||
throw Exception
|
||||
{
|
||||
"The type \"" + arrow_type->name() + "\" of an input column \"" + column_name + "\" is not supported for conversion from a " + format_name + " data format",
|
||||
fmt::format(R"(The type "{}" of an input column "{}" is not supported for conversion from a {} data format.)", arrow_type->name(), column_name, format_name),
|
||||
ErrorCodes::CANNOT_CONVERT_TYPE
|
||||
};
|
||||
}
|
||||
|
||||
void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk & res, std::shared_ptr<arrow::Table> & table,
|
||||
const Block & header, std::string format_name)
|
||||
ArrowColumnToCHColumn::ArrowColumnToCHColumn(const Block & header_, std::shared_ptr<arrow::Schema> schema_, const std::string & format_name_) : header(header_), format_name(format_name_)
|
||||
{
|
||||
for (const auto & field : schema_->fields())
|
||||
{
|
||||
if (header.has(field->name()))
|
||||
{
|
||||
const auto column_type = recursiveRemoveLowCardinality(header.getByName(field->name()).type);
|
||||
name_to_internal_type[field->name()] = getInternalType(field->type(), column_type, field->name(), format_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk & res, std::shared_ptr<arrow::Table> & table)
|
||||
{
|
||||
Columns columns_list;
|
||||
UInt64 num_rows = 0;
|
||||
@ -404,20 +553,18 @@ namespace DB
|
||||
|
||||
for (size_t column_i = 0, columns = header.columns(); column_i < columns; ++column_i)
|
||||
{
|
||||
ColumnWithTypeAndName header_column = header.getByPosition(column_i);
|
||||
const auto column_type = recursiveRemoveLowCardinality(header_column.type);
|
||||
const ColumnWithTypeAndName & header_column = header.getByPosition(column_i);
|
||||
|
||||
if (name_to_column_ptr.find(header_column.name) == name_to_column_ptr.end())
|
||||
// TODO: What if some columns were not presented? Insert NULLs? What if a column is not nullable?
|
||||
throw Exception{"Column \"" + header_column.name + "\" is not presented in input data",
|
||||
throw Exception{fmt::format("Column \"{}\" is not presented in input data.", header_column.name),
|
||||
ErrorCodes::THERE_IS_NO_COLUMN};
|
||||
|
||||
std::shared_ptr<arrow::ChunkedArray> arrow_column = name_to_column_ptr[header_column.name];
|
||||
|
||||
DataTypePtr internal_type = getInternalType(arrow_column->type(), column_type, header_column.name, format_name);
|
||||
|
||||
DataTypePtr & internal_type = name_to_internal_type[header_column.name];
|
||||
MutableColumnPtr read_column = internal_type->createColumn();
|
||||
readColumnFromArrowColumn(arrow_column, *read_column, header_column.name, format_name, false);
|
||||
readColumnFromArrowColumn(arrow_column, *read_column, header_column.name, format_name, false, dictionary_values);
|
||||
|
||||
ColumnWithTypeAndName column;
|
||||
column.name = header_column.name;
|
||||
|
||||
@ -21,8 +21,12 @@ namespace DB
|
||||
|
||||
class ArrowColumnToCHColumn
|
||||
{
|
||||
private:
|
||||
public:
|
||||
ArrowColumnToCHColumn(const Block & header_, std::shared_ptr<arrow::Schema> schema_, const std::string & format_name_);
|
||||
|
||||
void arrowTableToCHChunk(Chunk & res, std::shared_ptr<arrow::Table> & table);
|
||||
|
||||
private:
|
||||
#define FOR_ARROW_NUMERIC_TYPES(M) \
|
||||
M(arrow::Type::UINT8, DB::UInt8) \
|
||||
M(arrow::Type::INT8, DB::Int8) \
|
||||
@ -36,11 +40,24 @@ namespace DB
|
||||
M(arrow::Type::FLOAT, DB::Float32) \
|
||||
M(arrow::Type::DOUBLE, DB::Float64)
|
||||
|
||||
#define FOR_ARROW_INDEXES_TYPES(M) \
|
||||
M(arrow::Type::UINT8, DB::UInt8) \
|
||||
M(arrow::Type::INT8, DB::UInt8) \
|
||||
M(arrow::Type::UINT16, DB::UInt16) \
|
||||
M(arrow::Type::INT16, DB::UInt16) \
|
||||
M(arrow::Type::UINT32, DB::UInt32) \
|
||||
M(arrow::Type::INT32, DB::UInt32) \
|
||||
M(arrow::Type::UINT64, DB::UInt64) \
|
||||
M(arrow::Type::INT64, DB::UInt64)
|
||||
|
||||
public:
|
||||
|
||||
static void arrowTableToCHChunk(Chunk & res, std::shared_ptr<arrow::Table> & table,
|
||||
const Block & header, std::string format_name);
|
||||
const Block & header;
|
||||
std::unordered_map<std::string, DataTypePtr> name_to_internal_type;
|
||||
const std::string format_name;
|
||||
/// Map {column name : dictionary column}.
|
||||
/// To avoid converting dictionary from Arrow Dictionary
|
||||
/// to LowCardinality every chunk we save it and reuse.
|
||||
std::unordered_map<std::string, ColumnPtr> dictionary_values;
|
||||
};
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -6,17 +6,22 @@
|
||||
#include <Columns/ColumnNullable.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Columns/ColumnLowCardinality.h>
|
||||
#include <Columns/ColumnMap.h>
|
||||
#include <Core/callOnTypeIndex.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypeLowCardinality.h>
|
||||
#include <DataTypes/DataTypeMap.h>
|
||||
#include <Processors/Formats/IOutputFormat.h>
|
||||
#include <arrow/api.h>
|
||||
#include <arrow/builder.h>
|
||||
#include <arrow/type.h>
|
||||
#include <arrow/util/decimal.h>
|
||||
#include <DataTypes/DataTypeLowCardinality.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -25,6 +30,7 @@ namespace DB
|
||||
{
|
||||
extern const int UNKNOWN_EXCEPTION;
|
||||
extern const int UNKNOWN_TYPE;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
static const std::initializer_list<std::pair<String, std::shared_ptr<arrow::DataType>>> internal_type_to_arrow_type =
|
||||
@ -46,16 +52,15 @@ namespace DB
|
||||
//{"DateTime", arrow::date64()}, // BUG! saves as date32
|
||||
{"DateTime", arrow::uint32()},
|
||||
|
||||
// TODO: ClickHouse can actually store non-utf8 strings!
|
||||
{"String", arrow::utf8()},
|
||||
{"FixedString", arrow::utf8()},
|
||||
{"String", arrow::binary()},
|
||||
{"FixedString", arrow::binary()},
|
||||
};
|
||||
|
||||
|
||||
static void checkStatus(const arrow::Status & status, const String & column_name, const String & format_name)
|
||||
{
|
||||
if (!status.ok())
|
||||
throw Exception{"Error with a " + format_name + " column \"" + column_name + "\": " + status.ToString(), ErrorCodes::UNKNOWN_EXCEPTION};
|
||||
throw Exception{fmt::format("Error with a {} column \"{}\": {}.", format_name, column_name, status.ToString()), ErrorCodes::UNKNOWN_EXCEPTION};
|
||||
}
|
||||
|
||||
template <typename NumericType, typename ArrowBuilderType>
|
||||
@ -101,8 +106,10 @@ namespace DB
|
||||
arrow::ArrayBuilder * array_builder,
|
||||
String format_name,
|
||||
size_t start,
|
||||
size_t end);
|
||||
size_t end,
|
||||
std::unordered_map<String, std::shared_ptr<arrow::Array>> & dictionary_values);
|
||||
|
||||
template <typename Builder>
|
||||
static void fillArrowArrayWithArrayColumnData(
|
||||
const String & column_name,
|
||||
ColumnPtr & column,
|
||||
@ -111,26 +118,164 @@ namespace DB
|
||||
arrow::ArrayBuilder * array_builder,
|
||||
String format_name,
|
||||
size_t start,
|
||||
size_t end)
|
||||
size_t end,
|
||||
std::unordered_map<String, std::shared_ptr<arrow::Array>> & dictionary_values)
|
||||
{
|
||||
const auto * column_array = static_cast<const ColumnArray *>(column.get());
|
||||
const auto * column_array = assert_cast<const ColumnArray *>(column.get());
|
||||
ColumnPtr nested_column = column_array->getDataPtr();
|
||||
DataTypePtr nested_type = typeid_cast<const DataTypeArray *>(column_type.get())->getNestedType();
|
||||
DataTypePtr nested_type = assert_cast<const DataTypeArray *>(column_type.get())->getNestedType();
|
||||
const auto & offsets = column_array->getOffsets();
|
||||
|
||||
arrow::ListBuilder & builder = assert_cast<arrow::ListBuilder &>(*array_builder);
|
||||
Builder & builder = assert_cast<Builder &>(*array_builder);
|
||||
arrow::ArrayBuilder * value_builder = builder.value_builder();
|
||||
arrow::Status components_status;
|
||||
|
||||
for (size_t array_idx = start; array_idx < end; ++array_idx)
|
||||
{
|
||||
/// Start new array
|
||||
/// Start new array.
|
||||
components_status = builder.Append();
|
||||
checkStatus(components_status, nested_column->getName(), format_name);
|
||||
fillArrowArray(column_name, nested_column, nested_type, null_bytemap, value_builder, format_name, offsets[array_idx - 1], offsets[array_idx]);
|
||||
fillArrowArray(column_name, nested_column, nested_type, null_bytemap, value_builder, format_name, offsets[array_idx - 1], offsets[array_idx], dictionary_values);
|
||||
}
|
||||
}
|
||||
|
||||
static void fillArrowArrayWithTupleColumnData(
|
||||
const String & column_name,
|
||||
ColumnPtr & column,
|
||||
const std::shared_ptr<const IDataType> & column_type,
|
||||
const PaddedPODArray<UInt8> * null_bytemap,
|
||||
arrow::ArrayBuilder * array_builder,
|
||||
String format_name,
|
||||
size_t start,
|
||||
size_t end,
|
||||
std::unordered_map<String, std::shared_ptr<arrow::Array>> & dictionary_values)
|
||||
{
|
||||
const auto * column_tuple = assert_cast<const ColumnTuple *>(column.get());
|
||||
const auto & nested_types = assert_cast<const DataTypeTuple *>(column_type.get())->getElements();
|
||||
|
||||
arrow::StructBuilder & builder = assert_cast<arrow::StructBuilder &>(*array_builder);
|
||||
|
||||
for (size_t i = 0; i != column_tuple->tupleSize(); ++i)
|
||||
{
|
||||
ColumnPtr nested_column = column_tuple->getColumnPtr(i);
|
||||
fillArrowArray(column_name + "." + std::to_string(i), nested_column, nested_types[i], null_bytemap, builder.field_builder(i), format_name, start, end, dictionary_values);
|
||||
}
|
||||
|
||||
for (size_t i = start; i != end; ++i)
|
||||
{
|
||||
auto status = builder.Append();
|
||||
checkStatus(status, column->getName(), format_name);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static PaddedPODArray<Int64> extractIndexesImpl(ColumnPtr column, size_t start, size_t end)
|
||||
{
|
||||
const PaddedPODArray<T> & data = assert_cast<const ColumnVector<T> *>(column.get())->getData();
|
||||
PaddedPODArray<Int64> result;
|
||||
result.reserve(end - start);
|
||||
std::transform(data.begin() + start, data.begin() + end, std::back_inserter(result), [](T value) { return Int64(value); });
|
||||
return result;
|
||||
}
|
||||
|
||||
static PaddedPODArray<Int64> extractIndexesImpl(ColumnPtr column, size_t start, size_t end)
|
||||
{
|
||||
switch (column->getDataType())
|
||||
{
|
||||
case TypeIndex::UInt8:
|
||||
return extractIndexesImpl<UInt8>(column, start, end);
|
||||
case TypeIndex::UInt16:
|
||||
return extractIndexesImpl<UInt16>(column, start, end);
|
||||
case TypeIndex::UInt32:
|
||||
return extractIndexesImpl<UInt32>(column, start, end);
|
||||
case TypeIndex::UInt64:
|
||||
return extractIndexesImpl<UInt64>(column, start, end);
|
||||
default:
|
||||
throw Exception(fmt::format("Indexes column must be ColumnUInt, got {}.", column->getName()),
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename ValueType>
|
||||
static void fillArrowArrayWithLowCardinalityColumnDataImpl(
|
||||
const String & column_name,
|
||||
ColumnPtr & column,
|
||||
const std::shared_ptr<const IDataType> & column_type,
|
||||
const PaddedPODArray<UInt8> * null_bytemap,
|
||||
arrow::ArrayBuilder * array_builder,
|
||||
String format_name,
|
||||
size_t start,
|
||||
size_t end,
|
||||
std::unordered_map<String, std::shared_ptr<arrow::Array>> & dictionary_values)
|
||||
{
|
||||
const auto * column_lc = assert_cast<const ColumnLowCardinality *>(column.get());
|
||||
arrow::DictionaryBuilder<ValueType> * builder = assert_cast<arrow::DictionaryBuilder<ValueType> *>(array_builder);
|
||||
auto & dict_values = dictionary_values[column_name];
|
||||
|
||||
/// Convert dictionary from LowCardinality to Arrow dictionary only once and then reuse it.
|
||||
if (!dict_values)
|
||||
{
|
||||
auto value_type = assert_cast<arrow::DictionaryType *>(builder->type().get())->value_type();
|
||||
std::unique_ptr<arrow::ArrayBuilder> values_builder;
|
||||
arrow::MemoryPool* pool = arrow::default_memory_pool();
|
||||
arrow::Status status = MakeBuilder(pool, value_type, &values_builder);
|
||||
checkStatus(status, column->getName(), format_name);
|
||||
|
||||
auto dict_column = column_lc->getDictionary().getNestedColumn();
|
||||
const auto & dict_type = assert_cast<const DataTypeLowCardinality *>(column_type.get())->getDictionaryType();
|
||||
fillArrowArray(column_name, dict_column, dict_type, nullptr, values_builder.get(), format_name, 0, dict_column->size(), dictionary_values);
|
||||
status = values_builder->Finish(&dict_values);
|
||||
checkStatus(status, column->getName(), format_name);
|
||||
}
|
||||
|
||||
arrow::Status status = builder->InsertMemoValues(*dict_values);
|
||||
checkStatus(status, column->getName(), format_name);
|
||||
|
||||
/// AppendIndices in DictionaryBuilder works only with int64_t data, so we cannot use
|
||||
/// fillArrowArray here and should copy all indexes to int64_t container.
|
||||
auto indexes = extractIndexesImpl(column_lc->getIndexesPtr(), start, end);
|
||||
const uint8_t * arrow_null_bytemap_raw_ptr = nullptr;
|
||||
PaddedPODArray<uint8_t> arrow_null_bytemap;
|
||||
if (null_bytemap)
|
||||
{
|
||||
/// Invert values since Arrow interprets 1 as a non-null value, while CH as a null
|
||||
arrow_null_bytemap.reserve(end - start);
|
||||
for (size_t i = start; i < end; ++i)
|
||||
arrow_null_bytemap.emplace_back(!(*null_bytemap)[i]);
|
||||
|
||||
arrow_null_bytemap_raw_ptr = arrow_null_bytemap.data();
|
||||
}
|
||||
|
||||
status = builder->AppendIndices(indexes.data(), indexes.size(), arrow_null_bytemap_raw_ptr);
|
||||
checkStatus(status, column->getName(), format_name);
|
||||
}
|
||||
|
||||
|
||||
static void fillArrowArrayWithLowCardinalityColumnData(
|
||||
const String & column_name,
|
||||
ColumnPtr & column,
|
||||
const std::shared_ptr<const IDataType> & column_type,
|
||||
const PaddedPODArray<UInt8> * null_bytemap,
|
||||
arrow::ArrayBuilder * array_builder,
|
||||
String format_name,
|
||||
size_t start,
|
||||
size_t end,
|
||||
std::unordered_map<String, std::shared_ptr<arrow::Array>> & dictionary_values)
|
||||
{
|
||||
auto value_type = assert_cast<arrow::DictionaryType *>(array_builder->type().get())->value_type();
|
||||
|
||||
#define DISPATCH(ARROW_TYPE_ID, ARROW_TYPE) \
|
||||
if (arrow::Type::ARROW_TYPE_ID == value_type->id()) \
|
||||
{ \
|
||||
fillArrowArrayWithLowCardinalityColumnDataImpl<ARROW_TYPE>(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, dictionary_values); \
|
||||
return; \
|
||||
}
|
||||
|
||||
FOR_ARROW_TYPES(DISPATCH)
|
||||
#undef DISPATCH
|
||||
|
||||
}
|
||||
|
||||
template <typename ColumnType>
|
||||
static void fillArrowArrayWithStringColumnData(
|
||||
ColumnPtr write_column,
|
||||
@ -141,7 +286,7 @@ namespace DB
|
||||
size_t end)
|
||||
{
|
||||
const auto & internal_column = assert_cast<const ColumnType &>(*write_column);
|
||||
arrow::StringBuilder & builder = assert_cast<arrow::StringBuilder &>(*array_builder);
|
||||
arrow::BinaryBuilder & builder = assert_cast<arrow::BinaryBuilder &>(*array_builder);
|
||||
arrow::Status status;
|
||||
|
||||
for (size_t string_i = start; string_i < end; ++string_i)
|
||||
@ -155,7 +300,6 @@ namespace DB
|
||||
StringRef string_ref = internal_column.getDataAt(string_i);
|
||||
status = builder.Append(string_ref.data, string_ref.size);
|
||||
}
|
||||
|
||||
checkStatus(status, write_column->getName(), format_name);
|
||||
}
|
||||
}
|
||||
@ -218,18 +362,19 @@ namespace DB
|
||||
arrow::ArrayBuilder * array_builder,
|
||||
String format_name,
|
||||
size_t start,
|
||||
size_t end)
|
||||
size_t end,
|
||||
std::unordered_map<String, std::shared_ptr<arrow::Array>> & dictionary_values)
|
||||
{
|
||||
const String column_type_name = column_type->getFamilyName();
|
||||
|
||||
if ("Nullable" == column_type_name)
|
||||
{
|
||||
const ColumnNullable * column_nullable = checkAndGetColumn<ColumnNullable>(column.get());
|
||||
const ColumnNullable * column_nullable = assert_cast<const ColumnNullable *>(column.get());
|
||||
ColumnPtr nested_column = column_nullable->getNestedColumnPtr();
|
||||
DataTypePtr nested_type = typeid_cast<const DataTypeNullable *>(column_type.get())->getNestedType();
|
||||
DataTypePtr nested_type = assert_cast<const DataTypeNullable *>(column_type.get())->getNestedType();
|
||||
const ColumnPtr & null_column = column_nullable->getNullMapColumnPtr();
|
||||
const PaddedPODArray<UInt8> & bytemap = assert_cast<const ColumnVector<UInt8> &>(*null_column).getData();
|
||||
fillArrowArray(column_name, nested_column, nested_type, &bytemap, array_builder, format_name, start, end);
|
||||
fillArrowArray(column_name, nested_column, nested_type, &bytemap, array_builder, format_name, start, end, dictionary_values);
|
||||
}
|
||||
else if ("String" == column_type_name)
|
||||
{
|
||||
@ -249,7 +394,21 @@ namespace DB
|
||||
}
|
||||
else if ("Array" == column_type_name)
|
||||
{
|
||||
fillArrowArrayWithArrayColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end);
|
||||
fillArrowArrayWithArrayColumnData<arrow::ListBuilder>(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, dictionary_values);
|
||||
}
|
||||
else if ("Tuple" == column_type_name)
|
||||
{
|
||||
fillArrowArrayWithTupleColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, dictionary_values);
|
||||
}
|
||||
else if ("LowCardinality" == column_type_name)
|
||||
{
|
||||
fillArrowArrayWithLowCardinalityColumnData(column_name, column, column_type, null_bytemap, array_builder, format_name, start, end, dictionary_values);
|
||||
}
|
||||
else if ("Map" == column_type_name)
|
||||
{
|
||||
ColumnPtr column_array = assert_cast<const ColumnMap *>(column.get())->getNestedColumnPtr();
|
||||
DataTypePtr array_type = assert_cast<const DataTypeMap *>(column_type.get())->getNestedType();
|
||||
fillArrowArrayWithArrayColumnData<arrow::MapBuilder>(column_name, column_array, array_type, null_bytemap, array_builder, format_name, start, end, dictionary_values);
|
||||
}
|
||||
else if (isDecimal(column_type))
|
||||
{
|
||||
@ -280,7 +439,7 @@ namespace DB
|
||||
{
|
||||
throw Exception
|
||||
{
|
||||
"Internal type \"" + column_type_name + "\" of a column \"" + column_name + "\" is not supported for conversion into a " + format_name + " data format",
|
||||
fmt::format(R"(Internal type "{}" of a column "{}" is not supported for conversion into a {} data format.)", column_type_name, column_name, format_name),
|
||||
ErrorCodes::UNKNOWN_TYPE
|
||||
};
|
||||
}
|
||||
@ -295,7 +454,7 @@ namespace DB
|
||||
size_t start,
|
||||
size_t end)
|
||||
{
|
||||
const auto & column = static_cast<const typename DataType::ColumnType &>(*write_column);
|
||||
const auto & column = assert_cast<const typename DataType::ColumnType &>(*write_column);
|
||||
arrow::DecimalBuilder & builder = assert_cast<arrow::DecimalBuilder &>(*array_builder);
|
||||
arrow::Status status;
|
||||
|
||||
@ -312,12 +471,33 @@ namespace DB
|
||||
checkStatus(status, write_column->getName(), format_name);
|
||||
}
|
||||
|
||||
static std::shared_ptr<arrow::DataType> getArrowType(DataTypePtr column_type, const std::string & column_name, const std::string & format_name, bool * is_column_nullable)
|
||||
static std::shared_ptr<arrow::DataType> getArrowTypeForLowCardinalityIndexes(ColumnPtr indexes_column)
|
||||
{
|
||||
/// Arrow docs recommend preferring signed integers over unsigned integers for representing dictionary indices.
|
||||
/// https://arrow.apache.org/docs/format/Columnar.html#dictionary-encoded-layout
|
||||
switch (indexes_column->getDataType())
|
||||
{
|
||||
case TypeIndex::UInt8:
|
||||
return arrow::int8();
|
||||
case TypeIndex::UInt16:
|
||||
return arrow::int16();
|
||||
case TypeIndex::UInt32:
|
||||
return arrow::int32();
|
||||
case TypeIndex::UInt64:
|
||||
return arrow::int64();
|
||||
default:
|
||||
throw Exception(fmt::format("Indexes column for getUniqueIndex must be ColumnUInt, got {}.", indexes_column->getName()),
|
||||
ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
static std::shared_ptr<arrow::DataType> getArrowType(DataTypePtr column_type, ColumnPtr column, const std::string & column_name, const std::string & format_name, bool * is_column_nullable)
|
||||
{
|
||||
if (column_type->isNullable())
|
||||
{
|
||||
DataTypePtr nested_type = typeid_cast<const DataTypeNullable *>(column_type.get())->getNestedType();
|
||||
auto arrow_type = getArrowType(nested_type, column_name, format_name, is_column_nullable);
|
||||
DataTypePtr nested_type = assert_cast<const DataTypeNullable *>(column_type.get())->getNestedType();
|
||||
ColumnPtr nested_column = assert_cast<const ColumnNullable *>(column.get())->getNestedColumnPtr();
|
||||
auto arrow_type = getArrowType(nested_type, nested_column, column_name, format_name, is_column_nullable);
|
||||
*is_column_nullable = true;
|
||||
return arrow_type;
|
||||
}
|
||||
@ -334,7 +514,7 @@ namespace DB
|
||||
|| std::is_same_v<ToDataType, DataTypeDecimal<Decimal64>>
|
||||
|| std::is_same_v<ToDataType, DataTypeDecimal<Decimal128>>)
|
||||
{
|
||||
const auto & decimal_type = static_cast<const ToDataType *>(column_type.get());
|
||||
const auto & decimal_type = assert_cast<const ToDataType *>(column_type.get());
|
||||
arrow_type = arrow::decimal(decimal_type->getPrecision(), decimal_type->getScale());
|
||||
}
|
||||
|
||||
@ -346,11 +526,50 @@ namespace DB
|
||||
|
||||
if (isArray(column_type))
|
||||
{
|
||||
auto nested_type = typeid_cast<const DataTypeArray *>(column_type.get())->getNestedType();
|
||||
auto nested_arrow_type = getArrowType(nested_type, column_name, format_name, is_column_nullable);
|
||||
auto nested_type = assert_cast<const DataTypeArray *>(column_type.get())->getNestedType();
|
||||
auto nested_column = assert_cast<const ColumnArray *>(column.get())->getDataPtr();
|
||||
auto nested_arrow_type = getArrowType(nested_type, nested_column, column_name, format_name, is_column_nullable);
|
||||
return arrow::list(nested_arrow_type);
|
||||
}
|
||||
|
||||
if (isTuple(column_type))
|
||||
{
|
||||
const auto & nested_types = assert_cast<const DataTypeTuple *>(column_type.get())->getElements();
|
||||
const auto * tuple_column = assert_cast<const ColumnTuple *>(column.get());
|
||||
std::vector<std::shared_ptr<arrow::Field>> nested_fields;
|
||||
for (size_t i = 0; i != nested_types.size(); ++i)
|
||||
{
|
||||
String name = column_name + "." + std::to_string(i);
|
||||
auto nested_arrow_type = getArrowType(nested_types[i], tuple_column->getColumnPtr(i), name, format_name, is_column_nullable);
|
||||
nested_fields.push_back(std::make_shared<arrow::Field>(name, nested_arrow_type, *is_column_nullable));
|
||||
}
|
||||
return arrow::struct_(std::move(nested_fields));
|
||||
}
|
||||
|
||||
if (column_type->lowCardinality())
|
||||
{
|
||||
auto nested_type = assert_cast<const DataTypeLowCardinality *>(column_type.get())->getDictionaryType();
|
||||
const auto * lc_column = assert_cast<const ColumnLowCardinality *>(column.get());
|
||||
const auto & nested_column = lc_column->getDictionaryPtr();
|
||||
const auto & indexes_column = lc_column->getIndexesPtr();
|
||||
return arrow::dictionary(
|
||||
getArrowTypeForLowCardinalityIndexes(indexes_column),
|
||||
getArrowType(nested_type, nested_column, column_name, format_name, is_column_nullable));
|
||||
}
|
||||
|
||||
if (isMap(column_type))
|
||||
{
|
||||
const auto * map_type = assert_cast<const DataTypeMap *>(column_type.get());
|
||||
const auto & key_type = map_type->getKeyType();
|
||||
const auto & val_type = map_type->getValueType();
|
||||
|
||||
const auto & columns = assert_cast<const ColumnMap *>(column.get())->getNestedData().getColumns();
|
||||
return arrow::map(
|
||||
getArrowType(key_type, columns[0], column_name, format_name, is_column_nullable),
|
||||
getArrowType(val_type, columns[1], column_name, format_name, is_column_nullable)
|
||||
);
|
||||
}
|
||||
|
||||
const std::string type_name = column_type->getFamilyName();
|
||||
if (const auto * arrow_type_it = std::find_if(
|
||||
internal_type_to_arrow_type.begin(),
|
||||
@ -361,49 +580,59 @@ namespace DB
|
||||
return arrow_type_it->second;
|
||||
}
|
||||
|
||||
throw Exception{"The type \"" + column_name + "\" of a column \"" + column_name + "\""
|
||||
" is not supported for conversion into a " + format_name + " data format",
|
||||
throw Exception{fmt::format(R"(The type "{}" of a column "{}" is not supported for conversion into a {} data format.)", column_type->getName(), column_name, format_name),
|
||||
ErrorCodes::UNKNOWN_TYPE};
|
||||
}
|
||||
|
||||
CHColumnToArrowColumn::CHColumnToArrowColumn(const Block & header, const std::string & format_name_, bool low_cardinality_as_dictionary_)
|
||||
: format_name(format_name_), low_cardinality_as_dictionary(low_cardinality_as_dictionary_)
|
||||
{
|
||||
arrow_fields.reserve(header.columns());
|
||||
header_columns.reserve(header.columns());
|
||||
for (auto column : header.getColumnsWithTypeAndName())
|
||||
{
|
||||
if (!low_cardinality_as_dictionary)
|
||||
{
|
||||
column.type = recursiveRemoveLowCardinality(column.type);
|
||||
column.column = recursiveRemoveLowCardinality(column.column);
|
||||
}
|
||||
bool is_column_nullable = false;
|
||||
auto arrow_type = getArrowType(column.type, column.column, column.name, format_name, &is_column_nullable);
|
||||
arrow_fields.emplace_back(std::make_shared<arrow::Field>(column.name, arrow_type, is_column_nullable));
|
||||
header_columns.emplace_back(std::move(column));
|
||||
}
|
||||
}
|
||||
|
||||
void CHColumnToArrowColumn::chChunkToArrowTable(
|
||||
std::shared_ptr<arrow::Table> & res,
|
||||
const Block & header,
|
||||
const Chunk & chunk,
|
||||
size_t columns_num,
|
||||
String format_name)
|
||||
size_t columns_num)
|
||||
{
|
||||
/// For arrow::Schema and arrow::Table creation
|
||||
std::vector<std::shared_ptr<arrow::Field>> arrow_fields;
|
||||
std::vector<std::shared_ptr<arrow::Array>> arrow_arrays;
|
||||
arrow_fields.reserve(columns_num);
|
||||
arrow_arrays.reserve(columns_num);
|
||||
|
||||
for (size_t column_i = 0; column_i < columns_num; ++column_i)
|
||||
{
|
||||
// TODO: constructed every iteration
|
||||
ColumnWithTypeAndName column = header.safeGetByPosition(column_i);
|
||||
column.column = recursiveRemoveLowCardinality(chunk.getColumns()[column_i]);
|
||||
column.type = recursiveRemoveLowCardinality(column.type);
|
||||
const ColumnWithTypeAndName & header_column = header_columns[column_i];
|
||||
auto column = chunk.getColumns()[column_i];
|
||||
|
||||
bool is_column_nullable = false;
|
||||
auto arrow_type = getArrowType(column.type, column.name, format_name, &is_column_nullable);
|
||||
arrow_fields.emplace_back(std::make_shared<arrow::Field>(column.name, arrow_type, is_column_nullable));
|
||||
if (!low_cardinality_as_dictionary)
|
||||
column = recursiveRemoveLowCardinality(column);
|
||||
|
||||
arrow::MemoryPool* pool = arrow::default_memory_pool();
|
||||
std::unique_ptr<arrow::ArrayBuilder> array_builder;
|
||||
arrow::Status status = MakeBuilder(pool, arrow_fields[column_i]->type(), &array_builder);
|
||||
checkStatus(status, column.column->getName(), format_name);
|
||||
checkStatus(status, column->getName(), format_name);
|
||||
|
||||
fillArrowArray(column.name, column.column, column.type, nullptr, array_builder.get(), format_name, 0, column.column->size());
|
||||
fillArrowArray(header_column.name, column, header_column.type, nullptr, array_builder.get(), format_name, 0, column->size(), dictionary_values);
|
||||
|
||||
std::shared_ptr<arrow::Array> arrow_array;
|
||||
status = array_builder->Finish(&arrow_array);
|
||||
checkStatus(status, column.column->getName(), format_name);
|
||||
checkStatus(status, column->getName(), format_name);
|
||||
arrow_arrays.emplace_back(std::move(arrow_array));
|
||||
}
|
||||
|
||||
std::shared_ptr<arrow::Schema> arrow_schema = std::make_shared<arrow::Schema>(std::move(arrow_fields));
|
||||
std::shared_ptr<arrow::Schema> arrow_schema = std::make_shared<arrow::Schema>(arrow_fields);
|
||||
|
||||
res = arrow::Table::Make(arrow_schema, arrow_arrays);
|
||||
}
|
||||
|
||||
@ -12,6 +12,10 @@ namespace DB
|
||||
|
||||
class CHColumnToArrowColumn
|
||||
{
|
||||
public:
|
||||
CHColumnToArrowColumn(const Block & header, const std::string & format_name_, bool low_cardinality_as_dictionary_ = false);
|
||||
|
||||
void chChunkToArrowTable(std::shared_ptr<arrow::Table> & res, const Chunk & chunk, size_t columns_num);
|
||||
private:
|
||||
|
||||
#define FOR_INTERNAL_NUMERIC_TYPES(M) \
|
||||
@ -26,10 +30,27 @@ private:
|
||||
M(Float32, arrow::FloatBuilder) \
|
||||
M(Float64, arrow::DoubleBuilder)
|
||||
|
||||
#define FOR_ARROW_TYPES(M) \
|
||||
M(UINT8, arrow::UInt8Type) \
|
||||
M(INT8, arrow::Int8Type) \
|
||||
M(UINT16, arrow::UInt16Type) \
|
||||
M(INT16, arrow::Int16Type) \
|
||||
M(UINT32, arrow::UInt32Type) \
|
||||
M(INT32, arrow::Int32Type) \
|
||||
M(UINT64, arrow::UInt64Type) \
|
||||
M(INT64, arrow::Int64Type) \
|
||||
M(FLOAT, arrow::FloatType) \
|
||||
M(DOUBLE, arrow::DoubleType) \
|
||||
M(STRING, arrow::StringType)
|
||||
|
||||
public:
|
||||
static void chChunkToArrowTable(std::shared_ptr<arrow::Table> & res, const Block & header, const Chunk & chunk,
|
||||
size_t columns_num, String format_name);
|
||||
ColumnsWithTypeAndName header_columns;
|
||||
std::vector<std::shared_ptr<arrow::Field>> arrow_fields;
|
||||
const std::string format_name;
|
||||
bool low_cardinality_as_dictionary;
|
||||
/// Map {column name : arrow dictionary}.
|
||||
/// To avoid converting dictionary from LowCardinality to Arrow
|
||||
/// Dictionary every chunk we save it and reuse.
|
||||
std::unordered_map<std::string, std::shared_ptr<arrow::Array>> dictionary_values;
|
||||
};
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -33,7 +33,6 @@ ORCBlockInputFormat::ORCBlockInputFormat(ReadBuffer & in_, Block header_) : IInp
|
||||
Chunk ORCBlockInputFormat::generate()
|
||||
{
|
||||
Chunk res;
|
||||
const Block & header = getPort().getHeader();
|
||||
|
||||
if (!file_reader)
|
||||
prepareReader();
|
||||
@ -54,7 +53,7 @@ Chunk ORCBlockInputFormat::generate()
|
||||
|
||||
++stripe_current;
|
||||
|
||||
ArrowColumnToCHColumn::arrowTableToCHChunk(res, *table_result, header, "ORC");
|
||||
arrow_column_to_ch_column->arrowTableToCHChunk(res, *table_result);
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -67,11 +66,26 @@ void ORCBlockInputFormat::resetParser()
|
||||
stripe_current = 0;
|
||||
}
|
||||
|
||||
size_t countIndicesForType(std::shared_ptr<arrow::DataType> type)
|
||||
static size_t countIndicesForType(std::shared_ptr<arrow::DataType> type)
|
||||
{
|
||||
if (type->id() == arrow::Type::LIST)
|
||||
return countIndicesForType(static_cast<arrow::ListType *>(type.get())->value_type()) + 1;
|
||||
|
||||
if (type->id() == arrow::Type::STRUCT)
|
||||
{
|
||||
int indices = 1;
|
||||
auto * struct_type = static_cast<arrow::StructType *>(type.get());
|
||||
for (int i = 0; i != struct_type->num_fields(); ++i)
|
||||
indices += countIndicesForType(struct_type->field(i)->type());
|
||||
return indices;
|
||||
}
|
||||
|
||||
if (type->id() == arrow::Type::MAP)
|
||||
{
|
||||
auto * map_type = static_cast<arrow::MapType *>(type.get());
|
||||
return countIndicesForType(map_type->key_type()) + countIndicesForType(map_type->item_type());
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
@ -84,17 +98,22 @@ void ORCBlockInputFormat::prepareReader()
|
||||
std::shared_ptr<arrow::Schema> schema;
|
||||
THROW_ARROW_NOT_OK(file_reader->ReadSchema(&schema));
|
||||
|
||||
int index = 0;
|
||||
arrow_column_to_ch_column = std::make_unique<ArrowColumnToCHColumn>(getPort().getHeader(), schema, "ORC");
|
||||
|
||||
/// In ReadStripe column indices should be started from 1,
|
||||
/// because 0 indicates to select all columns.
|
||||
int index = 1;
|
||||
for (int i = 0; i < schema->num_fields(); ++i)
|
||||
{
|
||||
/// LIST type require 2 indices, STRUCT - the number of elements + 1,
|
||||
/// so we should recursively count the number of indices we need for this type.
|
||||
int indexes_count = countIndicesForType(schema->field(i)->type());
|
||||
if (getPort().getHeader().has(schema->field(i)->name()))
|
||||
{
|
||||
/// LIST type require 2 indices, so we should recursively
|
||||
/// count the number of indices we need for this type.
|
||||
int indexes_count = countIndicesForType(schema->field(i)->type());
|
||||
for (int j = 0; j != indexes_count; ++j)
|
||||
include_indices.push_back(index++);
|
||||
include_indices.push_back(index + j);
|
||||
}
|
||||
index += indexes_count;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -8,6 +8,9 @@ namespace arrow::adapters::orc { class ORCFileReader; }
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class ArrowColumnToCHColumn;
|
||||
|
||||
class ORCBlockInputFormat : public IInputFormat
|
||||
{
|
||||
public:
|
||||
@ -26,6 +29,8 @@ private:
|
||||
|
||||
std::unique_ptr<arrow::adapters::orc::ORCFileReader> file_reader;
|
||||
|
||||
std::unique_ptr<ArrowColumnToCHColumn> arrow_column_to_ch_column;
|
||||
|
||||
int stripe_total = 0;
|
||||
|
||||
int stripe_current = 0;
|
||||
|
||||
@ -10,12 +10,16 @@
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Columns/ColumnMap.h>
|
||||
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypeMap.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -46,15 +50,9 @@ void ORCOutputStream::write(const void* buf, size_t length)
|
||||
ORCBlockOutputFormat::ORCBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_)
|
||||
: IOutputFormat(header_, out_), format_settings{format_settings_}, output_stream(out_), data_types(header_.getDataTypes())
|
||||
{
|
||||
schema = orc::createStructType();
|
||||
options.setCompression(orc::CompressionKind::CompressionKind_NONE);
|
||||
size_t columns_count = header_.columns();
|
||||
for (size_t i = 0; i != columns_count; ++i)
|
||||
schema->addStructField(header_.safeGetByPosition(i).name, getORCType(data_types[i]));
|
||||
writer = orc::createWriter(*schema, &output_stream, options);
|
||||
}
|
||||
|
||||
ORC_UNIQUE_PTR<orc::Type> ORCBlockOutputFormat::getORCType(const DataTypePtr & type)
|
||||
ORC_UNIQUE_PTR<orc::Type> ORCBlockOutputFormat::getORCType(const DataTypePtr & type, const std::string & column_name)
|
||||
{
|
||||
switch (type->getTypeId())
|
||||
{
|
||||
@ -102,28 +100,48 @@ ORC_UNIQUE_PTR<orc::Type> ORCBlockOutputFormat::getORCType(const DataTypePtr & t
|
||||
}
|
||||
case TypeIndex::Nullable:
|
||||
{
|
||||
return getORCType(removeNullable(type));
|
||||
return getORCType(removeNullable(type), column_name);
|
||||
}
|
||||
case TypeIndex::Array:
|
||||
{
|
||||
const auto * array_type = typeid_cast<const DataTypeArray *>(type.get());
|
||||
return orc::createListType(getORCType(array_type->getNestedType()));
|
||||
const auto * array_type = assert_cast<const DataTypeArray *>(type.get());
|
||||
return orc::createListType(getORCType(array_type->getNestedType(), column_name));
|
||||
}
|
||||
case TypeIndex::Decimal32:
|
||||
{
|
||||
const auto * decimal_type = typeid_cast<const DataTypeDecimal<Decimal32> *>(type.get());
|
||||
const auto * decimal_type = assert_cast<const DataTypeDecimal<Decimal32> *>(type.get());
|
||||
return orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale());
|
||||
}
|
||||
case TypeIndex::Decimal64:
|
||||
{
|
||||
const auto * decimal_type = typeid_cast<const DataTypeDecimal<Decimal64> *>(type.get());
|
||||
const auto * decimal_type = assert_cast<const DataTypeDecimal<Decimal64> *>(type.get());
|
||||
return orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale());
|
||||
}
|
||||
case TypeIndex::Decimal128:
|
||||
{
|
||||
const auto * decimal_type = typeid_cast<const DataTypeDecimal<Decimal128> *>(type.get());
|
||||
const auto * decimal_type = assert_cast<const DataTypeDecimal<Decimal128> *>(type.get());
|
||||
return orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale());
|
||||
}
|
||||
case TypeIndex::Tuple:
|
||||
{
|
||||
const auto * tuple_type = assert_cast<const DataTypeTuple *>(type.get());
|
||||
const auto & nested_types = tuple_type->getElements();
|
||||
auto struct_type = orc::createStructType();
|
||||
for (size_t i = 0; i < nested_types.size(); ++i)
|
||||
{
|
||||
String name = column_name + "." + std::to_string(i);
|
||||
struct_type->addStructField(name, getORCType(nested_types[i], name));
|
||||
}
|
||||
return struct_type;
|
||||
}
|
||||
case TypeIndex::Map:
|
||||
{
|
||||
const auto * map_type = assert_cast<const DataTypeMap *>(type.get());
|
||||
return orc::createMapType(
|
||||
getORCType(map_type->getKeyType(), column_name),
|
||||
getORCType(map_type->getValueType(), column_name)
|
||||
);
|
||||
}
|
||||
default:
|
||||
{
|
||||
throw Exception("Type " + type->getName() + " is not supported for ORC output format", ErrorCodes::ILLEGAL_COLUMN);
|
||||
@ -149,6 +167,8 @@ void ORCBlockOutputFormat::writeNumbers(
|
||||
number_orc_column.notNull[i] = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
number_orc_column.notNull[i] = 1;
|
||||
number_orc_column.data[i] = convert(number_column.getElement(i));
|
||||
}
|
||||
number_orc_column.numElements = number_column.size();
|
||||
@ -164,7 +184,7 @@ void ORCBlockOutputFormat::writeDecimals(
|
||||
{
|
||||
DecimalVectorBatch & decimal_orc_column = dynamic_cast<DecimalVectorBatch &>(orc_column);
|
||||
const auto & decimal_column = assert_cast<const ColumnDecimal<Decimal> &>(column);
|
||||
const auto * decimal_type = typeid_cast<const DataTypeDecimal<Decimal> *>(type.get());
|
||||
const auto * decimal_type = assert_cast<const DataTypeDecimal<Decimal> *>(type.get());
|
||||
decimal_orc_column.precision = decimal_type->getPrecision();
|
||||
decimal_orc_column.scale = decimal_type->getScale();
|
||||
decimal_orc_column.resize(decimal_column.size());
|
||||
@ -175,6 +195,8 @@ void ORCBlockOutputFormat::writeDecimals(
|
||||
decimal_orc_column.notNull[i] = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
decimal_orc_column.notNull[i] = 1;
|
||||
decimal_orc_column.values[i] = convert(decimal_column.getElement(i).value);
|
||||
}
|
||||
decimal_orc_column.numElements = decimal_column.size();
|
||||
@ -197,6 +219,8 @@ void ORCBlockOutputFormat::writeStrings(
|
||||
string_orc_column.notNull[i] = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
string_orc_column.notNull[i] = 1;
|
||||
const StringRef & string = string_column.getDataAt(i);
|
||||
string_orc_column.data[i] = const_cast<char *>(string.data);
|
||||
string_orc_column.length[i] = string.size;
|
||||
@ -223,6 +247,8 @@ void ORCBlockOutputFormat::writeDateTimes(
|
||||
timestamp_orc_column.notNull[i] = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
timestamp_orc_column.notNull[i] = 1;
|
||||
timestamp_orc_column.data[i] = get_seconds(timestamp_column.getElement(i));
|
||||
timestamp_orc_column.nanoseconds[i] = get_nanoseconds(timestamp_column.getElement(i));
|
||||
}
|
||||
@ -235,11 +261,10 @@ void ORCBlockOutputFormat::writeColumn(
|
||||
DataTypePtr & type,
|
||||
const PaddedPODArray<UInt8> * null_bytemap)
|
||||
{
|
||||
if (null_bytemap)
|
||||
{
|
||||
orc_column.hasNulls = true;
|
||||
orc_column.notNull.resize(column.size());
|
||||
}
|
||||
if (null_bytemap)
|
||||
orc_column.hasNulls = true;
|
||||
|
||||
switch (type->getTypeId())
|
||||
{
|
||||
case TypeIndex::Int8:
|
||||
@ -374,12 +399,52 @@ void ORCBlockOutputFormat::writeColumn(
|
||||
for (size_t i = 0; i != list_column.size(); ++i)
|
||||
{
|
||||
list_orc_column.offsets[i + 1] = offsets[i];
|
||||
list_orc_column.notNull[i] = 1;
|
||||
}
|
||||
orc::ColumnVectorBatch & nested_orc_column = *list_orc_column.elements;
|
||||
writeColumn(nested_orc_column, list_column.getData(), nested_type, null_bytemap);
|
||||
list_orc_column.numElements = list_column.size();
|
||||
break;
|
||||
}
|
||||
case TypeIndex::Tuple:
|
||||
{
|
||||
orc::StructVectorBatch & struct_orc_column = dynamic_cast<orc::StructVectorBatch &>(orc_column);
|
||||
const auto & tuple_column = assert_cast<const ColumnTuple &>(column);
|
||||
auto nested_types = assert_cast<const DataTypeTuple *>(type.get())->getElements();
|
||||
for (size_t i = 0; i != tuple_column.size(); ++i)
|
||||
struct_orc_column.notNull[i] = 1;
|
||||
for (size_t i = 0; i != tuple_column.tupleSize(); ++i)
|
||||
writeColumn(*struct_orc_column.fields[i], tuple_column.getColumn(i), nested_types[i], null_bytemap);
|
||||
break;
|
||||
}
|
||||
case TypeIndex::Map:
|
||||
{
|
||||
orc::MapVectorBatch & map_orc_column = dynamic_cast<orc::MapVectorBatch &>(orc_column);
|
||||
const auto & list_column = assert_cast<const ColumnMap &>(column).getNestedColumn();
|
||||
const auto & map_type = assert_cast<const DataTypeMap &>(*type);
|
||||
const ColumnArray::Offsets & offsets = list_column.getOffsets();
|
||||
|
||||
map_orc_column.resize(list_column.size());
|
||||
/// The length of list i in ListVectorBatch is offsets[i+1] - offsets[i].
|
||||
map_orc_column.offsets[0] = 0;
|
||||
for (size_t i = 0; i != list_column.size(); ++i)
|
||||
{
|
||||
map_orc_column.offsets[i + 1] = offsets[i];
|
||||
map_orc_column.notNull[i] = 1;
|
||||
}
|
||||
const auto nested_columns = assert_cast<const ColumnTuple *>(list_column.getDataPtr().get())->getColumns();
|
||||
|
||||
orc::ColumnVectorBatch & keys_orc_column = *map_orc_column.keys;
|
||||
auto key_type = map_type.getKeyType();
|
||||
writeColumn(keys_orc_column, *nested_columns[0], key_type, null_bytemap);
|
||||
|
||||
orc::ColumnVectorBatch & values_orc_column = *map_orc_column.elements;
|
||||
auto value_type = map_type.getValueType();
|
||||
writeColumn(values_orc_column, *nested_columns[1], value_type, null_bytemap);
|
||||
|
||||
map_orc_column.numElements = list_column.size();
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw Exception("Type " + type->getName() + " is not supported for ORC output format", ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
@ -409,6 +474,8 @@ size_t ORCBlockOutputFormat::getMaxColumnSize(Chunk & chunk)
|
||||
|
||||
void ORCBlockOutputFormat::consume(Chunk chunk)
|
||||
{
|
||||
if (!writer)
|
||||
prepareWriter();
|
||||
size_t columns_num = chunk.getNumColumns();
|
||||
size_t rows_num = chunk.getNumRows();
|
||||
/// getMaxColumnSize is needed to write arrays.
|
||||
@ -425,9 +492,23 @@ void ORCBlockOutputFormat::consume(Chunk chunk)
|
||||
|
||||
void ORCBlockOutputFormat::finalize()
|
||||
{
|
||||
if (!writer)
|
||||
prepareWriter();
|
||||
|
||||
writer->close();
|
||||
}
|
||||
|
||||
void ORCBlockOutputFormat::prepareWriter()
|
||||
{
|
||||
const Block & header = getPort(PortKind::Main).getHeader();
|
||||
schema = orc::createStructType();
|
||||
options.setCompression(orc::CompressionKind::CompressionKind_NONE);
|
||||
size_t columns_count = header.columns();
|
||||
for (size_t i = 0; i != columns_count; ++i)
|
||||
schema->addStructField(header.safeGetByPosition(i).name, getORCType(data_types[i], header.safeGetByPosition(i).name));
|
||||
writer = orc::createWriter(*schema, &output_stream, options);
|
||||
}
|
||||
|
||||
void registerOutputFormatProcessorORC(FormatFactory & factory)
|
||||
{
|
||||
factory.registerOutputFormatProcessor("ORC", [](
|
||||
|
||||
@ -43,7 +43,7 @@ public:
|
||||
void finalize() override;
|
||||
|
||||
private:
|
||||
ORC_UNIQUE_PTR<orc::Type> getORCType(const DataTypePtr & type);
|
||||
ORC_UNIQUE_PTR<orc::Type> getORCType(const DataTypePtr & type, const std::string & column_name);
|
||||
|
||||
/// ConvertFunc is needed for type UInt8, because firstly UInt8 (char8_t) must be
|
||||
/// converted to unsigned char (bugprone-signed-char-misuse in clang).
|
||||
@ -71,6 +71,8 @@ private:
|
||||
size_t getColumnSize(const IColumn & column, DataTypePtr & type);
|
||||
size_t getMaxColumnSize(Chunk & chunk);
|
||||
|
||||
void prepareWriter();
|
||||
|
||||
const FormatSettings format_settings;
|
||||
ORCOutputStream output_stream;
|
||||
DataTypes data_types;
|
||||
|
||||
@ -38,7 +38,6 @@ ParquetBlockInputFormat::ParquetBlockInputFormat(ReadBuffer & in_, Block header_
|
||||
Chunk ParquetBlockInputFormat::generate()
|
||||
{
|
||||
Chunk res;
|
||||
const Block & header = getPort().getHeader();
|
||||
|
||||
if (!file_reader)
|
||||
prepareReader();
|
||||
@ -54,7 +53,7 @@ Chunk ParquetBlockInputFormat::generate()
|
||||
|
||||
++row_group_current;
|
||||
|
||||
ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, header, "Parquet");
|
||||
arrow_column_to_ch_column->arrowTableToCHChunk(res, table);
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -67,6 +66,29 @@ void ParquetBlockInputFormat::resetParser()
|
||||
row_group_current = 0;
|
||||
}
|
||||
|
||||
static size_t countIndicesForType(std::shared_ptr<arrow::DataType> type)
|
||||
{
|
||||
if (type->id() == arrow::Type::LIST)
|
||||
return countIndicesForType(static_cast<arrow::ListType *>(type.get())->value_type());
|
||||
|
||||
if (type->id() == arrow::Type::STRUCT)
|
||||
{
|
||||
int indices = 0;
|
||||
auto * struct_type = static_cast<arrow::StructType *>(type.get());
|
||||
for (int i = 0; i != struct_type->num_fields(); ++i)
|
||||
indices += countIndicesForType(struct_type->field(i)->type());
|
||||
return indices;
|
||||
}
|
||||
|
||||
if (type->id() == arrow::Type::MAP)
|
||||
{
|
||||
auto * map_type = static_cast<arrow::MapType *>(type.get());
|
||||
return countIndicesForType(map_type->key_type()) + countIndicesForType(map_type->item_type());
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
void ParquetBlockInputFormat::prepareReader()
|
||||
{
|
||||
THROW_ARROW_NOT_OK(parquet::arrow::OpenFile(asArrowFile(in), arrow::default_memory_pool(), &file_reader));
|
||||
@ -76,12 +98,21 @@ void ParquetBlockInputFormat::prepareReader()
|
||||
std::shared_ptr<arrow::Schema> schema;
|
||||
THROW_ARROW_NOT_OK(file_reader->GetSchema(&schema));
|
||||
|
||||
arrow_column_to_ch_column = std::make_unique<ArrowColumnToCHColumn>(getPort().getHeader(), schema, "Parquet");
|
||||
|
||||
int index = 0;
|
||||
for (int i = 0; i < schema->num_fields(); ++i)
|
||||
{
|
||||
/// STRUCT type require the number of indexes equal to the number of
|
||||
/// nested elements, so we should recursively
|
||||
/// count the number of indices we need for this type.
|
||||
int indexes_count = countIndicesForType(schema->field(i)->type());
|
||||
if (getPort().getHeader().has(schema->field(i)->name()))
|
||||
{
|
||||
column_indices.push_back(i);
|
||||
for (int j = 0; j != indexes_count; ++j)
|
||||
column_indices.push_back(index + j);
|
||||
}
|
||||
index += indexes_count;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -12,6 +12,8 @@ namespace arrow { class Buffer; }
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class ArrowColumnToCHColumn;
|
||||
|
||||
class ParquetBlockInputFormat : public IInputFormat
|
||||
{
|
||||
public:
|
||||
@ -32,6 +34,7 @@ private:
|
||||
int row_group_total = 0;
|
||||
// indices of columns to read from Parquet file
|
||||
std::vector<int> column_indices;
|
||||
std::unique_ptr<ArrowColumnToCHColumn> arrow_column_to_ch_column;
|
||||
int row_group_current = 0;
|
||||
};
|
||||
|
||||
|
||||
@ -13,7 +13,6 @@
|
||||
#include <arrow/api.h>
|
||||
#include <arrow/util/memory.h>
|
||||
#include <parquet/arrow/writer.h>
|
||||
#include <parquet/deprecated_io.h>
|
||||
#include "ArrowBufferedStreams.h"
|
||||
#include "CHColumnToArrowColumn.h"
|
||||
|
||||
@ -32,11 +31,16 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, const Blo
|
||||
|
||||
void ParquetBlockOutputFormat::consume(Chunk chunk)
|
||||
{
|
||||
const Block & header = getPort(PortKind::Main).getHeader();
|
||||
const size_t columns_num = chunk.getNumColumns();
|
||||
std::shared_ptr<arrow::Table> arrow_table;
|
||||
|
||||
CHColumnToArrowColumn::chChunkToArrowTable(arrow_table, header, chunk, columns_num, "Parquet");
|
||||
if (!ch_column_to_arrow_column)
|
||||
{
|
||||
const Block & header = getPort(PortKind::Main).getHeader();
|
||||
ch_column_to_arrow_column = std::make_unique<CHColumnToArrowColumn>(header, "Parquet");
|
||||
}
|
||||
|
||||
ch_column_to_arrow_column->chChunkToArrowTable(arrow_table, chunk, columns_num);
|
||||
|
||||
if (!file_writer)
|
||||
{
|
||||
|
||||
@ -21,6 +21,9 @@ namespace arrow
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class CHColumnToArrowColumn;
|
||||
|
||||
class ParquetBlockOutputFormat : public IOutputFormat
|
||||
{
|
||||
public:
|
||||
@ -36,6 +39,7 @@ private:
|
||||
const FormatSettings format_settings;
|
||||
|
||||
std::unique_ptr<parquet::arrow::FileWriter> file_writer;
|
||||
std::unique_ptr<CHColumnToArrowColumn> ch_column_to_arrow_column;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -190,9 +190,10 @@ protected:
|
||||
/// Initially reserved virtual column name may be shadowed by real column.
|
||||
bool isVirtualColumn(const String & column_name, const StorageMetadataPtr & metadata_snapshot) const;
|
||||
|
||||
|
||||
private:
|
||||
|
||||
StorageID storage_id;
|
||||
|
||||
mutable std::mutex id_mutex;
|
||||
|
||||
/// Multiversion storage metadata. Allows to read/write storage metadata
|
||||
|
||||
@ -223,6 +223,12 @@ public:
|
||||
DeleteOnDestroy, /// part was moved to another disk and should be deleted in own destructor
|
||||
};
|
||||
|
||||
static constexpr auto all_part_states =
|
||||
{
|
||||
State::Temporary, State::PreCommitted, State::Committed, State::Outdated, State::Deleting,
|
||||
State::DeleteOnDestroy
|
||||
};
|
||||
|
||||
using TTLInfo = MergeTreeDataPartTTLInfo;
|
||||
using TTLInfos = MergeTreeDataPartTTLInfos;
|
||||
|
||||
|
||||
@ -2156,8 +2156,7 @@ bool MergeTreeData::renameTempPartAndReplace(
|
||||
|
||||
LOG_TRACE(log, "Renaming temporary part {} to {}.", part->relative_path, part_name);
|
||||
|
||||
auto it_duplicate = data_parts_by_info.find(part_info);
|
||||
if (it_duplicate != data_parts_by_info.end())
|
||||
if (auto it_duplicate = data_parts_by_info.find(part_info); it_duplicate != data_parts_by_info.end())
|
||||
{
|
||||
String message = "Part " + (*it_duplicate)->getNameWithState() + " already exists";
|
||||
|
||||
|
||||
@ -402,6 +402,7 @@ public:
|
||||
|
||||
/// Returns a copy of the list so that the caller shouldn't worry about locks.
|
||||
DataParts getDataParts(const DataPartStates & affordable_states) const;
|
||||
|
||||
/// Returns sorted list of the parts with specified states
|
||||
/// out_states will contain snapshot of each part state
|
||||
DataPartsVector getDataPartsVector(
|
||||
|
||||
@ -262,8 +262,8 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(
|
||||
{
|
||||
log_entry.type = StorageReplicatedMergeTree::LogEntry::ATTACH_PART;
|
||||
|
||||
/// We don't need to involve ZooKeeper to obtain the checksums as by the time we get
|
||||
/// the MutableDataPartPtr here, we already have the data thus being able to
|
||||
/// We don't need to involve ZooKeeper to obtain checksums as by the time we get
|
||||
/// MutableDataPartPtr here, we already have the data thus being able to
|
||||
/// calculate the checksums.
|
||||
log_entry.part_checksum = part->checksums.getTotalChecksumHex();
|
||||
}
|
||||
@ -384,6 +384,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(
|
||||
|
||||
MergeTreeData::Transaction transaction(storage); /// If you can not add a part to ZK, we'll remove it back from the working set.
|
||||
bool renamed = false;
|
||||
|
||||
try
|
||||
{
|
||||
renamed = storage.renameTempPartAndAdd(part, nullptr, &transaction);
|
||||
@ -394,6 +395,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(
|
||||
&& e.code() != ErrorCodes::PART_IS_TEMPORARILY_LOCKED)
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!renamed)
|
||||
{
|
||||
if (is_already_existing_part)
|
||||
|
||||
@ -257,7 +257,7 @@ If you use the Replicated version of engines, see https://clickhouse.tech/docs/e
|
||||
|
||||
static StoragePtr create(const StorageFactory::Arguments & args)
|
||||
{
|
||||
/** [Replicated][|Summing|Collapsing|Aggregating|Replacing|Graphite]MergeTree (2 * 7 combinations) engines
|
||||
/** [Replicated][|Summing|VersionedCollapsing|Collapsing|Aggregating|Replacing|Graphite]MergeTree (2 * 7 combinations) engines
|
||||
* The argument for the engine should be:
|
||||
* - (for Replicated) The path to the table in ZooKeeper
|
||||
* - (for Replicated) Replica name in ZooKeeper
|
||||
|
||||
@ -288,6 +288,7 @@ void replaceConstantExpressions(
|
||||
/// is one of the following:
|
||||
/// - QueryProcessingStage::Complete
|
||||
/// - QueryProcessingStage::WithMergeableStateAfterAggregation
|
||||
/// - QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit
|
||||
/// - none (in this case regular WithMergeableState should be used)
|
||||
std::optional<QueryProcessingStage::Enum> getOptimizedQueryProcessingStage(const SelectQueryInfo & query_info, bool extremes, const Block & sharding_key_block)
|
||||
{
|
||||
@ -349,13 +350,13 @@ std::optional<QueryProcessingStage::Enum> getOptimizedQueryProcessingStage(const
|
||||
// ORDER BY
|
||||
const ASTPtr order_by = select.orderBy();
|
||||
if (order_by)
|
||||
return QueryProcessingStage::WithMergeableStateAfterAggregation;
|
||||
return QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit;
|
||||
|
||||
// LIMIT BY
|
||||
// LIMIT
|
||||
// OFFSET
|
||||
if (select.limitBy() || select.limitLength() || select.limitOffset())
|
||||
return QueryProcessingStage::WithMergeableStateAfterAggregation;
|
||||
return QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit;
|
||||
|
||||
// Only simple SELECT FROM GROUP BY sharding_key can use Complete state.
|
||||
return QueryProcessingStage::Complete;
|
||||
@ -514,10 +515,22 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(
|
||||
if (settings.distributed_group_by_no_merge)
|
||||
{
|
||||
if (settings.distributed_group_by_no_merge == DISTRIBUTED_GROUP_BY_NO_MERGE_AFTER_AGGREGATION)
|
||||
return QueryProcessingStage::WithMergeableStateAfterAggregation;
|
||||
{
|
||||
if (settings.distributed_push_down_limit)
|
||||
return QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit;
|
||||
else
|
||||
return QueryProcessingStage::WithMergeableStateAfterAggregation;
|
||||
}
|
||||
else
|
||||
{
|
||||
/// NOTE: distributed_group_by_no_merge=1 does not respect distributed_push_down_limit
|
||||
/// (since in this case queries processed separatelly and the initiator is just a proxy in this case).
|
||||
return QueryProcessingStage::Complete;
|
||||
}
|
||||
}
|
||||
|
||||
if (settings.distributed_push_down_limit)
|
||||
return QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit;
|
||||
|
||||
/// Nested distributed query cannot return Complete stage,
|
||||
/// since the parent query need to aggregate the results after.
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <Core/Defines.h>
|
||||
|
||||
#include "Common/hex.h"
|
||||
#include <Common/Macros.h>
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
#include <Common/ThreadPool.h>
|
||||
@ -63,10 +64,13 @@
|
||||
#include <common/scope_guard.h>
|
||||
#include <common/scope_guard_safe.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <ctime>
|
||||
#include <filesystem>
|
||||
#include <iterator>
|
||||
#include <numeric>
|
||||
#include <thread>
|
||||
#include <future>
|
||||
#include <filesystem>
|
||||
|
||||
#include <boost/algorithm/string/join.hpp>
|
||||
|
||||
@ -135,6 +139,7 @@ namespace ErrorCodes
|
||||
extern const int INTERSERVER_SCHEME_DOESNT_MATCH;
|
||||
extern const int DUPLICATE_DATA_PART;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int CONCURRENT_ACCESS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
namespace ActionLocks
|
||||
@ -153,10 +158,6 @@ static const auto MERGE_SELECTING_SLEEP_MS = 5 * 1000;
|
||||
static const auto MUTATIONS_FINALIZING_SLEEP_MS = 1 * 1000;
|
||||
static const auto MUTATIONS_FINALIZING_IDLE_SLEEP_MS = 5 * 1000;
|
||||
|
||||
|
||||
std::atomic_uint StorageReplicatedMergeTree::total_fetches {0};
|
||||
|
||||
|
||||
void StorageReplicatedMergeTree::setZooKeeper()
|
||||
{
|
||||
/// Every ReplicatedMergeTree table is using only one ZooKeeper session.
|
||||
@ -376,7 +377,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
|
||||
|
||||
if (attach && !current_zookeeper->exists(zookeeper_path + "/metadata"))
|
||||
{
|
||||
LOG_WARNING(log, "No metadata in ZooKeeper: table will be in readonly mode.");
|
||||
LOG_WARNING(log, "No metadata in ZooKeeper for {}: table will be in readonly mode.", zookeeper_path);
|
||||
is_readonly = true;
|
||||
has_metadata_in_zookeeper = false;
|
||||
return;
|
||||
@ -384,10 +385,20 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
|
||||
|
||||
auto metadata_snapshot = getInMemoryMetadataPtr();
|
||||
|
||||
/// May it be ZK lost not the whole root, so the upper check passed, but only the /replicas/replica
|
||||
/// folder.
|
||||
if (attach && !current_zookeeper->exists(replica_path))
|
||||
{
|
||||
LOG_WARNING(log, "No metadata in ZooKeeper for {}: table will be in readonly mode", replica_path);
|
||||
is_readonly = true;
|
||||
has_metadata_in_zookeeper = false;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!attach)
|
||||
{
|
||||
if (!getDataParts().empty())
|
||||
throw Exception("Data directory for table already containing data parts"
|
||||
throw Exception("Data directory for table already contains data parts"
|
||||
" - probably it was unclean DROP table or manual intervention."
|
||||
" You must either clear directory by hand or use ATTACH TABLE"
|
||||
" instead of CREATE TABLE if you need to use that parts.", ErrorCodes::INCORRECT_DATA);
|
||||
@ -433,13 +444,17 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
|
||||
{
|
||||
/// In old tables this node may missing or be empty
|
||||
String replica_metadata;
|
||||
bool replica_metadata_exists = current_zookeeper->tryGet(replica_path + "/metadata", replica_metadata);
|
||||
const bool replica_metadata_exists = current_zookeeper->tryGet(replica_path + "/metadata", replica_metadata);
|
||||
|
||||
if (!replica_metadata_exists || replica_metadata.empty())
|
||||
{
|
||||
/// We have to check shared node granularity before we create ours.
|
||||
other_replicas_fixed_granularity = checkFixedGranualrityInZookeeper();
|
||||
|
||||
ReplicatedMergeTreeTableMetadata current_metadata(*this, metadata_snapshot);
|
||||
current_zookeeper->createOrUpdate(replica_path + "/metadata", current_metadata.toString(), zkutil::CreateMode::Persistent);
|
||||
|
||||
current_zookeeper->createOrUpdate(replica_path + "/metadata", current_metadata.toString(),
|
||||
zkutil::CreateMode::Persistent);
|
||||
}
|
||||
|
||||
checkTableStructure(replica_path, metadata_snapshot);
|
||||
@ -460,8 +475,8 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
|
||||
current_zookeeper->get(zookeeper_path + "/metadata", &metadata_stat);
|
||||
metadata_version = metadata_stat.version;
|
||||
}
|
||||
/// Temporary directories contain untinalized results of Merges or Fetches (after forced restart)
|
||||
/// and don't allow to reinitialize them, so delete each of them immediately
|
||||
/// Temporary directories contain uninitialized results of Merges or Fetches (after forced restart),
|
||||
/// don't allow to reinitialize them, delete each of them immediately.
|
||||
clearOldTemporaryDirectories(0);
|
||||
clearOldWriteAheadLogs();
|
||||
}
|
||||
@ -727,12 +742,13 @@ void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metada
|
||||
String replicas_value;
|
||||
|
||||
if (!zookeeper->tryGet(zookeeper_path + "/replicas", replicas_value, &replicas_stat))
|
||||
throw Exception(fmt::format("Cannot create a replica of the table {}, because the last replica of the table was dropped right now",
|
||||
zookeeper_path), ErrorCodes::ALL_REPLICAS_LOST);
|
||||
throw Exception(ErrorCodes::ALL_REPLICAS_LOST,
|
||||
"Cannot create a replica of the table {}, because the last replica of the table was dropped right now",
|
||||
zookeeper_path);
|
||||
|
||||
/// It is not the first replica, we will mark it as "lost", to immediately repair (clone) from existing replica.
|
||||
/// By the way, it's possible that the replica will be first, if all previous replicas were removed concurrently.
|
||||
String is_lost_value = replicas_stat.numChildren ? "1" : "0";
|
||||
const String is_lost_value = replicas_stat.numChildren ? "1" : "0";
|
||||
|
||||
Coordination::Requests ops;
|
||||
ops.emplace_back(zkutil::makeCreateRequest(replica_path, "",
|
||||
@ -761,20 +777,17 @@ void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metada
|
||||
|
||||
Coordination::Responses responses;
|
||||
code = zookeeper->tryMulti(ops, responses);
|
||||
if (code == Coordination::Error::ZNODEEXISTS)
|
||||
{
|
||||
throw Exception("Replica " + replica_path + " already exists.", ErrorCodes::REPLICA_IS_ALREADY_EXIST);
|
||||
}
|
||||
else if (code == Coordination::Error::ZBADVERSION)
|
||||
|
||||
switch (code)
|
||||
{
|
||||
case Coordination::Error::ZNODEEXISTS:
|
||||
throw Exception(ErrorCodes::REPLICA_IS_ALREADY_EXIST, "Replica {} already exists", replica_path);
|
||||
case Coordination::Error::ZBADVERSION:
|
||||
LOG_ERROR(log, "Retrying createReplica(), because some other replicas were created at the same time");
|
||||
}
|
||||
else if (code == Coordination::Error::ZNONODE)
|
||||
{
|
||||
throw Exception("Table " + zookeeper_path + " was suddenly removed.", ErrorCodes::ALL_REPLICAS_LOST);
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
case Coordination::Error::ZNONODE:
|
||||
throw Exception(ErrorCodes::ALL_REPLICAS_LOST, "Table {} was suddenly removed", zookeeper_path);
|
||||
default:
|
||||
zkutil::KeeperMultiException::check(code, ops, responses);
|
||||
}
|
||||
} while (code == Coordination::Error::ZBADVERSION);
|
||||
@ -1123,6 +1136,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks)
|
||||
size_t unexpected_parts_nonnew = 0;
|
||||
UInt64 unexpected_parts_nonnew_rows = 0;
|
||||
UInt64 unexpected_parts_rows = 0;
|
||||
|
||||
for (const auto & part : unexpected_parts)
|
||||
{
|
||||
if (part->info.level > 0)
|
||||
@ -1134,20 +1148,17 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks)
|
||||
unexpected_parts_rows += part->rows_count;
|
||||
}
|
||||
|
||||
/// Additional helpful statistics
|
||||
auto get_blocks_count_in_data_part = [&] (const String & part_name) -> UInt64
|
||||
const UInt64 parts_to_fetch_blocks = std::accumulate(parts_to_fetch.cbegin(), parts_to_fetch.cend(), 0,
|
||||
[&](UInt64 acc, const String& part_name)
|
||||
{
|
||||
MergeTreePartInfo part_info;
|
||||
|
||||
if (MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version))
|
||||
return part_info.getBlocksCount();
|
||||
return acc + part_info.getBlocksCount();
|
||||
|
||||
LOG_ERROR(log, "Unexpected part name: {}", part_name);
|
||||
return 0;
|
||||
};
|
||||
|
||||
UInt64 parts_to_fetch_blocks = 0;
|
||||
for (const String & name : parts_to_fetch)
|
||||
parts_to_fetch_blocks += get_blocks_count_in_data_part(name);
|
||||
return acc;
|
||||
});
|
||||
|
||||
/** We can automatically synchronize data,
|
||||
* if the ratio of the total number of errors to the total number of parts (minimum - on the local filesystem or in ZK)
|
||||
@ -1499,7 +1510,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry)
|
||||
{
|
||||
if (MutableDataPartPtr part = attachPartHelperFoundValidPart(entry); part)
|
||||
{
|
||||
LOG_TRACE(log, "Found valid part to attach from local data, preparing the transaction");
|
||||
LOG_TRACE(log, "Found valid local part for {}, preparing the transaction", part->name);
|
||||
|
||||
Transaction transaction(*this);
|
||||
|
||||
@ -1512,7 +1523,9 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry)
|
||||
return true;
|
||||
}
|
||||
|
||||
LOG_TRACE(log, "Didn't find part with the correct checksums, will fetch it from other replica");
|
||||
LOG_TRACE(log, "Didn't find valid local part for {} ({}), will fetch it from other replica",
|
||||
entry.new_part_name,
|
||||
entry.actual_new_part_name);
|
||||
}
|
||||
|
||||
if (is_get_or_attach && entry.source_replica == replica_name)
|
||||
@ -2732,6 +2745,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo
|
||||
/// Remove local parts if source replica does not have them, because such parts will never be fetched by other replicas.
|
||||
Strings local_parts_in_zk = zookeeper->getChildren(fs::path(replica_path) / "parts");
|
||||
Strings parts_to_remove_from_zk;
|
||||
|
||||
for (const auto & part : local_parts_in_zk)
|
||||
{
|
||||
if (active_parts_set.getContainingPart(part).empty())
|
||||
@ -2740,10 +2754,13 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo
|
||||
LOG_WARNING(log, "Source replica does not have part {}. Removing it from ZooKeeper.", part);
|
||||
}
|
||||
}
|
||||
|
||||
tryRemovePartsFromZooKeeperWithRetries(parts_to_remove_from_zk);
|
||||
|
||||
auto local_active_parts = getDataParts();
|
||||
|
||||
DataPartsVector parts_to_remove_from_working_set;
|
||||
|
||||
for (const auto & part : local_active_parts)
|
||||
{
|
||||
if (active_parts_set.getContainingPart(part->name).empty())
|
||||
@ -2756,6 +2773,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo
|
||||
if (getSettings()->detach_old_local_parts_when_cloning_replica)
|
||||
{
|
||||
auto metadata_snapshot = getInMemoryMetadataPtr();
|
||||
|
||||
for (const auto & part : parts_to_remove_from_working_set)
|
||||
{
|
||||
LOG_INFO(log, "Detaching {}", part->relative_path);
|
||||
@ -2768,7 +2786,35 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo
|
||||
for (const String & name : active_parts)
|
||||
{
|
||||
LogEntry log_entry;
|
||||
|
||||
if (!are_restoring_replica)
|
||||
log_entry.type = LogEntry::GET_PART;
|
||||
else
|
||||
{
|
||||
LOG_DEBUG(log, "Obtaining checksum for path {}", name);
|
||||
|
||||
// The part we want to fetch is probably present in detached/ folder.
|
||||
// However, we need to get part's checksum to check if it's not corrupt.
|
||||
log_entry.type = LogEntry::ATTACH_PART;
|
||||
|
||||
MinimalisticDataPartChecksums desired_checksums;
|
||||
|
||||
const fs::path part_path = fs::path(source_path) / "parts" / name;
|
||||
|
||||
const String part_znode = zookeeper->get(part_path);
|
||||
|
||||
if (!part_znode.empty())
|
||||
desired_checksums = ReplicatedMergeTreePartHeader::fromString(part_znode).getChecksums();
|
||||
else
|
||||
{
|
||||
String desired_checksums_str = zookeeper->get(part_path / "checksums");
|
||||
desired_checksums = MinimalisticDataPartChecksums::deserializeFrom(desired_checksums_str);
|
||||
}
|
||||
|
||||
const auto [lo, hi] = desired_checksums.hash_of_all_files;
|
||||
log_entry.part_checksum = getHexUIntUppercase(hi) + getHexUIntUppercase(lo);
|
||||
}
|
||||
|
||||
log_entry.source_replica = "";
|
||||
log_entry.new_part_name = name;
|
||||
log_entry.create_time = tryGetPartCreateTime(zookeeper, source_path, name);
|
||||
@ -2868,6 +2914,7 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke
|
||||
Coordination::Stat is_lost_stat;
|
||||
bool is_new_replica = true;
|
||||
String res;
|
||||
|
||||
if (zookeeper->tryGet(fs::path(replica_path) / "is_lost", res, &is_lost_stat))
|
||||
{
|
||||
if (res == "0")
|
||||
@ -3968,6 +4015,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora
|
||||
MinimalisticDataPartChecksums desired_checksums;
|
||||
String part_path = fs::path(source_replica_path) / "parts" / part_name;
|
||||
String part_znode = zookeeper->get(part_path);
|
||||
|
||||
if (!part_znode.empty())
|
||||
desired_checksums = ReplicatedMergeTreePartHeader::fromString(part_znode).getChecksums();
|
||||
else
|
||||
@ -5030,6 +5078,59 @@ bool StorageReplicatedMergeTree::getFakePartCoveringAllPartsInPartition(const St
|
||||
return true;
|
||||
}
|
||||
|
||||
void StorageReplicatedMergeTree::restoreMetadataInZooKeeper()
|
||||
{
|
||||
LOG_INFO(log, "Restoring replica metadata");
|
||||
|
||||
if (!is_readonly || has_metadata_in_zookeeper)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "It's a bug: replica is not readonly");
|
||||
|
||||
if (are_restoring_replica.exchange(true))
|
||||
throw Exception(ErrorCodes::CONCURRENT_ACCESS_NOT_SUPPORTED, "Replica restoration in progress");
|
||||
|
||||
auto metadata_snapshot = getInMemoryMetadataPtr();
|
||||
|
||||
const DataPartsVector all_parts = getDataPartsVector(IMergeTreeDataPart::all_part_states);
|
||||
Strings active_parts_names;
|
||||
|
||||
/// Why all parts (not only Committed) are moved to detached/:
|
||||
/// After ZK metadata restoration ZK resets sequential counters (including block number counters), so one may
|
||||
/// potentially encounter a situation that a part we want to attach already exists.
|
||||
for (const auto & part : all_parts)
|
||||
{
|
||||
if (part->getState() == DataPartState::Committed)
|
||||
active_parts_names.push_back(part->name);
|
||||
|
||||
forgetPartAndMoveToDetached(part);
|
||||
}
|
||||
|
||||
LOG_INFO(log, "Moved all parts to detached/");
|
||||
|
||||
const bool is_first_replica = createTableIfNotExists(metadata_snapshot);
|
||||
|
||||
LOG_INFO(log, "Created initial ZK nodes, replica is first: {}", is_first_replica);
|
||||
|
||||
if (!is_first_replica)
|
||||
createReplica(metadata_snapshot);
|
||||
|
||||
createNewZooKeeperNodes();
|
||||
|
||||
LOG_INFO(log, "Created ZK nodes for table");
|
||||
|
||||
is_readonly = false;
|
||||
has_metadata_in_zookeeper = true;
|
||||
|
||||
if (is_first_replica)
|
||||
for (const String& part_name : active_parts_names)
|
||||
attachPartition(std::make_shared<ASTLiteral>(part_name), metadata_snapshot, true, getContext());
|
||||
|
||||
LOG_INFO(log, "Attached all partitions, starting table");
|
||||
|
||||
startup();
|
||||
|
||||
are_restoring_replica.store(false);
|
||||
}
|
||||
|
||||
void StorageReplicatedMergeTree::dropPartNoWaitNoThrow(const String & part_name)
|
||||
{
|
||||
assertNotReadonly();
|
||||
@ -6938,8 +7039,10 @@ bool StorageReplicatedMergeTree::dropAllPartsInPartition(
|
||||
zookeeper.get(alter_partition_version_path, &alter_partition_version_stat);
|
||||
|
||||
MergeTreePartInfo drop_range_info;
|
||||
/// It prevent other replicas from assigning merges which intersect locked block number.
|
||||
|
||||
/// It would prevent other replicas from assigning merges which intersect locked block number.
|
||||
std::optional<EphemeralLockInZooKeeper> delimiting_block_lock;
|
||||
|
||||
if (!getFakePartCoveringAllPartsInPartition(partition_id, drop_range_info, delimiting_block_lock))
|
||||
{
|
||||
LOG_INFO(log, "Will not drop partition {}, it is empty.", partition_id);
|
||||
@ -6960,23 +7063,31 @@ bool StorageReplicatedMergeTree::dropAllPartsInPartition(
|
||||
entry.create_time = time(nullptr);
|
||||
|
||||
Coordination::Requests ops;
|
||||
ops.emplace_back(zkutil::makeCreateRequest(fs::path(zookeeper_path) / "log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential));
|
||||
|
||||
ops.emplace_back(zkutil::makeCreateRequest(fs::path(zookeeper_path) / "log/log-", entry.toString(),
|
||||
zkutil::CreateMode::PersistentSequential));
|
||||
|
||||
/// Check and update version to avoid race with REPLACE_RANGE.
|
||||
/// Otherwise new parts covered by drop_range_info may appear after execution of current DROP_RANGE entry
|
||||
/// as a result of execution of concurrently created REPLACE_RANGE entry.
|
||||
ops.emplace_back(zkutil::makeCheckRequest(alter_partition_version_path, alter_partition_version_stat.version));
|
||||
ops.emplace_back(zkutil::makeSetRequest(alter_partition_version_path, "", -1));
|
||||
|
||||
/// Just update version, because merges assignment relies on it
|
||||
ops.emplace_back(zkutil::makeSetRequest(fs::path(zookeeper_path) / "log", "", -1));
|
||||
delimiting_block_lock->getUnlockOps(ops);
|
||||
|
||||
if (auto txn = query_context->getZooKeeperMetadataTransaction())
|
||||
txn->moveOpsTo(ops);
|
||||
|
||||
Coordination::Responses responses;
|
||||
Coordination::Error code = zookeeper.tryMulti(ops, responses);
|
||||
|
||||
if (code == Coordination::Error::ZOK)
|
||||
delimiting_block_lock->assumeUnlocked();
|
||||
else if (code == Coordination::Error::ZBADVERSION)
|
||||
throw Exception(ErrorCodes::CANNOT_ASSIGN_ALTER, "Cannot assign ALTER PARTITION, because another ALTER PARTITION query was concurrently executed");
|
||||
throw Exception(ErrorCodes::CANNOT_ASSIGN_ALTER,
|
||||
"Cannot assign ALTER PARTITION because another ALTER PARTITION query was concurrently executed");
|
||||
else
|
||||
zkutil::KeeperMultiException::check(code, ops, responses);
|
||||
|
||||
|
||||
@ -35,7 +35,7 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/** The engine that uses the merge tree (see MergeTreeData) and replicated through ZooKeeper.
|
||||
/** The engine that uses the merge tree (see MergeTreeData) and is replicated through ZooKeeper.
|
||||
*
|
||||
* ZooKeeper is used for the following things:
|
||||
* - the structure of the table (/metadata, /columns)
|
||||
@ -57,6 +57,7 @@ namespace DB
|
||||
* Log - a sequence of entries (LogEntry) about what to do.
|
||||
* Each entry is one of:
|
||||
* - normal data insertion (GET),
|
||||
* - data insertion with a possible attach from local data (ATTACH),
|
||||
* - merge (MERGE),
|
||||
* - delete the partition (DROP).
|
||||
*
|
||||
@ -65,10 +66,8 @@ namespace DB
|
||||
* Despite the name of the "queue", execution can be reordered, if necessary (shouldExecuteLogEntry, executeLogEntry).
|
||||
* In addition, the records in the queue can be generated independently (not from the log), in the following cases:
|
||||
* - when creating a new replica, actions are put on GET from other replicas (createReplica);
|
||||
* - if the part is corrupt (removePartAndEnqueueFetch) or absent during the check (at start - checkParts, while running - searchForMissingPart),
|
||||
* actions are put on GET from other replicas;
|
||||
*
|
||||
* TODO Update the GET part after rewriting the code (search locally).
|
||||
* - if the part is corrupt (removePartAndEnqueueFetch) or absent during the check
|
||||
* (at start - checkParts, while running - searchForMissingPart), actions are put on GET from other replicas;
|
||||
*
|
||||
* The replica to which INSERT was made in the queue will also have an entry of the GET of this data.
|
||||
* Such an entry is considered to be executed as soon as the queue handler sees it.
|
||||
@ -240,6 +239,13 @@ public:
|
||||
/// Get best replica having this partition on S3
|
||||
String getSharedDataReplica(const IMergeTreeDataPart & part) const;
|
||||
|
||||
inline String getReplicaName() const { return replica_name; }
|
||||
|
||||
/// Restores table metadata if ZooKeeper lost it.
|
||||
/// Used only on restarted readonly replicas (not checked). All active (Committed) parts are moved to detached/
|
||||
/// folder and attached. Parts in all other states are just moved to detached/ folder.
|
||||
void restoreMetadataInZooKeeper();
|
||||
|
||||
/// Get throttler for replicated fetches
|
||||
ThrottlerPtr getFetchesThrottler() const
|
||||
{
|
||||
@ -253,6 +259,8 @@ public:
|
||||
}
|
||||
|
||||
private:
|
||||
std::atomic_bool are_restoring_replica {false};
|
||||
|
||||
/// Get a sequential consistent view of current parts.
|
||||
ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock getMaxAddedBlocks() const;
|
||||
|
||||
@ -332,7 +340,7 @@ private:
|
||||
Poco::Event partial_shutdown_event {false}; /// Poco::Event::EVENT_MANUALRESET
|
||||
|
||||
/// Limiting parallel fetches per node
|
||||
static std::atomic_uint total_fetches;
|
||||
static inline std::atomic_uint total_fetches {0};
|
||||
|
||||
/// Limiting parallel fetches per one table
|
||||
std::atomic_uint current_table_fetches {0};
|
||||
@ -389,7 +397,8 @@ private:
|
||||
*/
|
||||
bool createTableIfNotExists(const StorageMetadataPtr & metadata_snapshot);
|
||||
|
||||
/** Creates a replica in ZooKeeper and adds to the queue all that it takes to catch up with the rest of the replicas.
|
||||
/**
|
||||
* Creates a replica in ZooKeeper and adds to the queue all that it takes to catch up with the rest of the replicas.
|
||||
*/
|
||||
void createReplica(const StorageMetadataPtr & metadata_snapshot);
|
||||
|
||||
|
||||
@ -16,11 +16,10 @@ def fill_node(node):
|
||||
'''.format(replica=node.name))
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
configs =["configs/remote_servers.xml"]
|
||||
|
||||
node_1 = cluster.add_instance('replica1', with_zookeeper=True, main_configs=configs)
|
||||
node_2 = cluster.add_instance('replica2', with_zookeeper=True, main_configs=configs)
|
||||
node_3 = cluster.add_instance('replica3', with_zookeeper=True, main_configs=configs)
|
||||
node_1 = cluster.add_instance('replica1', with_zookeeper=True)
|
||||
node_2 = cluster.add_instance('replica2', with_zookeeper=True)
|
||||
node_3 = cluster.add_instance('replica3', with_zookeeper=True)
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def start_cluster():
|
||||
|
||||
@ -151,7 +151,7 @@ def test_grant_all_on_table():
|
||||
instance.query("GRANT ALL ON test.table TO A WITH GRANT OPTION")
|
||||
instance.query("GRANT ALL ON test.table TO B", user='A')
|
||||
assert instance.query(
|
||||
"SHOW GRANTS FOR B") == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n"
|
||||
"SHOW GRANTS FOR B") == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n"
|
||||
instance.query("REVOKE ALL ON test.table FROM B", user='A')
|
||||
assert instance.query("SHOW GRANTS FOR B") == ""
|
||||
|
||||
|
||||
0
tests/integration/test_restore_replica/__init__.py
Normal file
0
tests/integration/test_restore_replica/__init__.py
Normal file
@ -4,15 +4,15 @@
|
||||
<shard>
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<host>node_1_1</host>
|
||||
<host>replica1</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<host>node_1_2</host>
|
||||
<host>replica2</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<host>node_1_3</host>
|
||||
<host>replica3</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
156
tests/integration/test_restore_replica/test.py
Normal file
156
tests/integration/test_restore_replica/test.py
Normal file
@ -0,0 +1,156 @@
|
||||
import time
|
||||
import pytest
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.cluster import ClickHouseKiller
|
||||
from helpers.test_tools import assert_eq_with_retry
|
||||
from helpers.network import PartitionManager
|
||||
|
||||
def fill_nodes(nodes):
|
||||
for node in nodes:
|
||||
node.query(
|
||||
'''
|
||||
CREATE TABLE test(n UInt32)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/', '{replica}')
|
||||
ORDER BY n PARTITION BY n % 10;
|
||||
'''.format(replica=node.name))
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
configs =["configs/remote_servers.xml"]
|
||||
|
||||
node_1 = cluster.add_instance('replica1', with_zookeeper=True, main_configs=configs)
|
||||
node_2 = cluster.add_instance('replica2', with_zookeeper=True, main_configs=configs)
|
||||
node_3 = cluster.add_instance('replica3', with_zookeeper=True, main_configs=configs)
|
||||
nodes = [node_1, node_2, node_3]
|
||||
|
||||
def fill_table():
|
||||
node_1.query("TRUNCATE TABLE test")
|
||||
|
||||
for node in nodes:
|
||||
node.query("SYSTEM SYNC REPLICA test")
|
||||
|
||||
check_data(0, 0)
|
||||
|
||||
# it will create multiple parts in each partition and probably cause merges
|
||||
node_1.query("INSERT INTO test SELECT number + 0 FROM numbers(200)")
|
||||
node_1.query("INSERT INTO test SELECT number + 200 FROM numbers(200)")
|
||||
node_1.query("INSERT INTO test SELECT number + 400 FROM numbers(200)")
|
||||
node_1.query("INSERT INTO test SELECT number + 600 FROM numbers(200)")
|
||||
node_1.query("INSERT INTO test SELECT number + 800 FROM numbers(200)")
|
||||
check_data(499500, 1000)
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def start_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
fill_nodes(nodes)
|
||||
yield cluster
|
||||
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
def check_data(_sum: int, count: int) -> None:
|
||||
res = "{}\t{}\n".format(_sum, count)
|
||||
assert_eq_with_retry(node_1, "SELECT sum(n), count() FROM test", res)
|
||||
assert_eq_with_retry(node_2, "SELECT sum(n), count() FROM test", res)
|
||||
assert_eq_with_retry(node_3, "SELECT sum(n), count() FROM test", res)
|
||||
|
||||
def check_after_restoration():
|
||||
check_data(1999000, 2000)
|
||||
|
||||
for node in nodes:
|
||||
node.query_and_get_error("SYSTEM RESTORE REPLICA test")
|
||||
|
||||
def test_restore_replica_invalid_tables(start_cluster):
|
||||
print("Checking the invocation on non-existent and non-replicated tables")
|
||||
node_1.query_and_get_error("SYSTEM RESTORE REPLICA i_dont_exist_42")
|
||||
node_1.query_and_get_error("SYSTEM RESTORE REPLICA no_db.i_dont_exist_42")
|
||||
node_1.query_and_get_error("SYSTEM RESTORE REPLICA system.numbers")
|
||||
|
||||
def test_restore_replica_sequential(start_cluster):
|
||||
zk = cluster.get_kazoo_client('zoo1')
|
||||
fill_table()
|
||||
|
||||
print("Deleting root ZK path metadata")
|
||||
zk.delete("/clickhouse/tables/test", recursive=True)
|
||||
assert zk.exists("/clickhouse/tables/test") is None
|
||||
|
||||
node_1.query("SYSTEM RESTART REPLICA test")
|
||||
node_1.query_and_get_error("INSERT INTO test SELECT number AS num FROM numbers(1000,2000) WHERE num % 2 = 0")
|
||||
|
||||
print("Restoring replica1")
|
||||
|
||||
node_1.query("SYSTEM RESTORE REPLICA test")
|
||||
assert zk.exists("/clickhouse/tables/test")
|
||||
check_data(499500, 1000)
|
||||
|
||||
node_1.query("INSERT INTO test SELECT number + 1000 FROM numbers(1000)")
|
||||
|
||||
print("Restoring other replicas")
|
||||
|
||||
node_2.query("SYSTEM RESTART REPLICA test")
|
||||
node_2.query("SYSTEM RESTORE REPLICA test")
|
||||
|
||||
node_3.query("SYSTEM RESTART REPLICA test")
|
||||
node_3.query("SYSTEM RESTORE REPLICA test")
|
||||
|
||||
node_2.query("SYSTEM SYNC REPLICA test")
|
||||
node_3.query("SYSTEM SYNC REPLICA test")
|
||||
|
||||
check_after_restoration()
|
||||
|
||||
def test_restore_replica_parallel(start_cluster):
|
||||
zk = cluster.get_kazoo_client('zoo1')
|
||||
fill_table()
|
||||
|
||||
print("Deleting root ZK path metadata")
|
||||
zk.delete("/clickhouse/tables/test", recursive=True)
|
||||
assert zk.exists("/clickhouse/tables/test") is None
|
||||
|
||||
node_1.query("SYSTEM RESTART REPLICA test")
|
||||
node_1.query_and_get_error("INSERT INTO test SELECT number AS num FROM numbers(1000,2000) WHERE num % 2 = 0")
|
||||
|
||||
print("Restoring replicas in parallel")
|
||||
|
||||
node_2.query("SYSTEM RESTART REPLICA test")
|
||||
node_3.query("SYSTEM RESTART REPLICA test")
|
||||
|
||||
node_1.query("SYSTEM RESTORE REPLICA test ON CLUSTER test_cluster")
|
||||
|
||||
assert zk.exists("/clickhouse/tables/test")
|
||||
check_data(499500, 1000)
|
||||
|
||||
node_1.query("INSERT INTO test SELECT number + 1000 FROM numbers(1000)")
|
||||
|
||||
check_after_restoration()
|
||||
|
||||
def test_restore_replica_alive_replicas(start_cluster):
|
||||
zk = cluster.get_kazoo_client('zoo1')
|
||||
fill_table()
|
||||
|
||||
print("Deleting replica2 path, trying to restore replica1")
|
||||
zk.delete("/clickhouse/tables/test/replicas/replica2", recursive=True)
|
||||
assert zk.exists("/clickhouse/tables/test/replicas/replica2") is None
|
||||
node_1.query_and_get_error("SYSTEM RESTORE REPLICA test")
|
||||
|
||||
print("Deleting replica1 path, trying to restore replica1")
|
||||
zk.delete("/clickhouse/tables/test/replicas/replica1", recursive=True)
|
||||
assert zk.exists("/clickhouse/tables/test/replicas/replica1") is None
|
||||
|
||||
node_1.query("SYSTEM RESTART REPLICA test")
|
||||
node_1.query("SYSTEM RESTORE REPLICA test")
|
||||
|
||||
node_2.query("SYSTEM RESTART REPLICA test")
|
||||
node_2.query("SYSTEM RESTORE REPLICA test")
|
||||
|
||||
check_data(499500, 1000)
|
||||
|
||||
node_1.query("INSERT INTO test SELECT number + 1000 FROM numbers(1000)")
|
||||
|
||||
node_2.query("SYSTEM SYNC REPLICA test")
|
||||
node_3.query("SYSTEM SYNC REPLICA test")
|
||||
|
||||
check_after_restoration()
|
||||
@ -22,6 +22,9 @@
|
||||
<value>Native</value>
|
||||
<value>Avro</value>
|
||||
<value>MsgPack</value>
|
||||
<value>ORC</value>
|
||||
<value>Parquet</value>
|
||||
<value>Arrow</value>
|
||||
</values>
|
||||
</substitution>
|
||||
</substitutions>
|
||||
|
||||
@ -36,6 +36,8 @@
|
||||
<value>Avro</value>
|
||||
<value>MsgPack</value>
|
||||
<value>ORC</value>
|
||||
<value>Parquet</value>
|
||||
<value>Arrow</value>
|
||||
</values>
|
||||
</substitution>
|
||||
</substitutions>
|
||||
|
||||
@ -298,24 +298,40 @@ Code: 33. DB::ParsingEx---tion: Error while reading Parquet data: IOError: Not y
|
||||
[[['a','b'],['c','d']],[[],['e']]] 1
|
||||
[[['a','b'],['c','d'],['e']],[[],['f']]] 1
|
||||
=== Try load data from nested_maps.snappy.parquet
|
||||
Code: 70. DB::Ex---tion: The type "map" of an input column "a" is not supported for conversion from a Parquet data format: data for INSERT was parsed from stdin
|
||||
|
||||
{'a':{1:1,2:0}} 1 1
|
||||
{'b':{1:1}} 1 1
|
||||
{'c':{}} 1 1
|
||||
{'d':{}} 1 1
|
||||
{'e':{1:1}} 1 1
|
||||
{'f':{3:1,4:0,5:1}} 1 1
|
||||
=== Try load data from non_hadoop_lz4_compressed.parquet
|
||||
1593604800 abc 42
|
||||
1593604800 def 7.7
|
||||
1593604801 abc 42.125
|
||||
1593604801 def 7.7
|
||||
=== Try load data from nonnullable.impala.parquet
|
||||
../contrib/arrow/cpp/src/arrow/array/array_nested.cc:192: Check failed: (self->list_type_->value_type()->id()) == (data->child_data[0]->type->id())
|
||||
8 [-1] [[-1,-2],[]] {'k1':-1} [{},{'k1':1},{},{}] (-1,[-1],([[(-1)]]),{})
|
||||
=== Try load data from nullable.impala.parquet
|
||||
../contrib/arrow/cpp/src/arrow/array/array_nested.cc:192: Check failed: (self->list_type_->value_type()->id()) == (data->child_data[0]->type->id())
|
||||
1 [1,2,3] [[1,2],[3,4]] {'k1':1,'k2':100} [{'k1':1}] (1,[1],([[(10),(-10)],[(11)]]),{'foo':(([1.1]))})
|
||||
2 [NULL,1,2,NULL,3,NULL] [[NULL,1,2,NULL],[3,NULL,4],[],[]] {'k1':2,'k2':NULL} [{'k3':NULL,'k1':1},{},{}] (NULL,[NULL],([[(NULL),(10),(NULL),(-10),(NULL)],[(11),(NULL)],[],[]]),{'g1':(([2.2,NULL])),'g2':(([])),'g3':(([])),'g4':(([])),'g5':(([]))})
|
||||
3 [] [[]] {} [{},{}] (NULL,[],([]),{})
|
||||
4 [] [] {} [] (NULL,[],([]),{})
|
||||
5 [] [] {} [] (NULL,[],([]),{'foo':(([2.2,3.3]))})
|
||||
6 [] [] {} [] (NULL,[],([]),{})
|
||||
7 [] [[],[5,6]] {'k1':NULL,'k3':NULL} [] (7,[2,3,NULL],([[],[(NULL)],[]]),{})
|
||||
=== Try load data from nullable_list.parquet
|
||||
[1,NULL,2] [NULL,'Some string',NULL] [0.00,NULL,42.42]
|
||||
[NULL] [NULL] [NULL]
|
||||
[] [] []
|
||||
=== Try load data from nulls.snappy.parquet
|
||||
Code: 70. DB::Ex---tion: The type "struct" of an input column "b_struct" is not supported for conversion from a Parquet data format: data for INSERT was parsed from stdin
|
||||
|
||||
(NULL)
|
||||
(NULL)
|
||||
(NULL)
|
||||
(NULL)
|
||||
(NULL)
|
||||
(NULL)
|
||||
(NULL)
|
||||
(NULL)
|
||||
=== Try load data from single_nan.parquet
|
||||
\N
|
||||
=== Try load data from userdata1.parquet
|
||||
|
||||
@ -55,7 +55,10 @@ for NAME in $(find "$DATA_DIR"/*.parquet -print0 | xargs -0 -n 1 basename | LC_A
|
||||
COLUMNS=$(cat "$COLUMNS_FILE") || continue
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_load"
|
||||
${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_load ($COLUMNS) ENGINE = Memory"
|
||||
$CLICKHOUSE_CLIENT --multiquery <<EOF
|
||||
SET allow_experimental_map_type = 1;
|
||||
CREATE TABLE parquet_load ($COLUMNS) ENGINE = Memory;
|
||||
EOF
|
||||
|
||||
# Some files contain unsupported data structures, exception is ok.
|
||||
cat "$DATA_DIR"/"$NAME" | ${CLICKHOUSE_CLIENT} --query="INSERT INTO parquet_load FORMAT Parquet" 2>&1 | sed 's/Exception/Ex---tion/'
|
||||
|
||||
@ -0,0 +1,6 @@
|
||||
Arrow
|
||||
{1:2,2:3} {'1':'a','2':'b'} {1:(1,2),2:(3,4)} {1:[1,2],2:[3,4]} [{1:2,2:3},{3:4,4:5}] ({1:2,2:3},{'a':'b','c':'d'}) [{1:[({1:2},(1)),({2:3},(2))]},{2:[({3:4},(3)),({4:5},(4))]}]
|
||||
Parquet
|
||||
{1:2,2:3} {'1':'a','2':'b'} {1:(1,2),2:(3,4)} {1:[1,2],2:[3,4]} [{1:2,2:3},{3:4,4:5}] ({1:2,2:3},{'a':'b','c':'d'}) [{1:[({1:2},(1)),({2:3},(2))]},{2:[({3:4},(3)),({4:5},(4))]}]
|
||||
ORC
|
||||
{1:2,2:3} {'1':'a','2':'b'} {1:(1,2),2:(3,4)} {1:[1,2],2:[3,4]} [{1:2,2:3},{3:4,4:5}] ({1:2,2:3},{'a':'b','c':'d'}) [{1:[({1:2},(1)),({2:3},(2))]},{2:[({3:4},(3)),({4:5},(4))]}]
|
||||
28
tests/queries/0_stateless/00900_orc_arrow_parquet_maps.sh
Executable file
28
tests/queries/0_stateless/00900_orc_arrow_parquet_maps.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS maps"
|
||||
${CLICKHOUSE_CLIENT} --multiquery <<EOF
|
||||
SET allow_experimental_map_type = 1;
|
||||
CREATE TABLE maps (m1 Map(UInt32, UInt32), m2 Map(String, String), m3 Map(UInt32, Tuple(UInt32, UInt32)), m4 Map(UInt32, Array(UInt32)), m5 Array(Map(UInt32, UInt32)), m6 Tuple(Map(UInt32, UInt32), Map(String, String)), m7 Array(Map(UInt32, Array(Tuple(Map(UInt32, UInt32), Tuple(UInt32)))))) ENGINE=Memory();
|
||||
EOF
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="INSERT INTO maps VALUES ({1 : 2, 2 : 3}, {'1' : 'a', '2' : 'b'}, {1 : (1, 2), 2 : (3, 4)}, {1 : [1, 2], 2 : [3, 4]}, [{1 : 2, 2 : 3}, {3 : 4, 4 : 5}], ({1 : 2, 2 : 3}, {'a' : 'b', 'c' : 'd'}), [{1 : [({1 : 2}, (1)), ({2 : 3}, (2))]}, {2 : [({3 : 4}, (3)), ({4 : 5}, (4))]}])"
|
||||
|
||||
|
||||
formats="Arrow Parquet ORC";
|
||||
|
||||
|
||||
for format in ${formats}; do
|
||||
echo $format
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="SELECT * FROM maps FORMAT Parquet" > "${CLICKHOUSE_TMP}"/maps
|
||||
${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE maps"
|
||||
cat "${CLICKHOUSE_TMP}"/maps | ${CLICKHOUSE_CLIENT} -q "INSERT INTO maps FORMAT Parquet"
|
||||
${CLICKHOUSE_CLIENT} --query="SELECT * FROM maps"
|
||||
done
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="DROP TABLE maps"
|
||||
@ -0,0 +1,6 @@
|
||||
Arrow
|
||||
(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))]
|
||||
Parquet
|
||||
(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))]
|
||||
ORC
|
||||
(1,2) ('1','2') ((1,'1'),1) ((1,2),('1','2')) ([1,2,3],1) (([1,2,3],[1,2,3]),([[1,2,3],[1,2,3]],1)) [([[1,2,3],[1,2,3]],([(1,2),(1,2)],1))]
|
||||
24
tests/queries/0_stateless/00900_orc_arrow_parquet_tuples.sh
Executable file
24
tests/queries/0_stateless/00900_orc_arrow_parquet_tuples.sh
Executable file
@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS tuples";
|
||||
${CLICKHOUSE_CLIENT} --query="CREATE TABLE tuples (t1 Tuple(UInt32, UInt32), t2 Tuple(String, String), t3 Tuple(Tuple(UInt32, String), UInt32), t4 Tuple(Tuple(UInt32, UInt32), Tuple(String, String)), t5 Tuple(Array(UInt32), UInt32), t6 Tuple(Tuple(Array(UInt32), Array(UInt32)), Tuple(Array(Array(UInt32)), UInt32)), t7 Array(Tuple(Array(Array(UInt32)), Tuple(Array(Tuple(UInt32, UInt32)), UInt32)))) ENGINE=Memory()"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="INSERT INTO tuples VALUES ((1, 2), ('1', '2'), ((1, '1'), 1), ((1, 2), ('1', '2')), ([1,2,3], 1), (([1,2,3], [1,2,3]), ([[1,2,3], [1,2,3]], 1)), [([[1,2,3], [1,2,3]], ([(1, 2), (1, 2)], 1))])"
|
||||
|
||||
formats="Arrow Parquet ORC";
|
||||
|
||||
for format in ${formats}; do
|
||||
echo $format
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="SELECT * FROM tuples FORMAT $format" > "${CLICKHOUSE_TMP}"/tuples
|
||||
${CLICKHOUSE_CLIENT} --query="TRUNCATE TABLE tuples"
|
||||
cat "${CLICKHOUSE_TMP}"/tuples | ${CLICKHOUSE_CLIENT} -q "INSERT INTO tuples FORMAT $format"
|
||||
${CLICKHOUSE_CLIENT} --query="SELECT * FROM tuples"
|
||||
done
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="DROP TABLE tuples"
|
||||
@ -103,6 +103,7 @@ SYSTEM REPLICATION QUEUES ['SYSTEM STOP REPLICATION QUEUES','SYSTEM START REPLIC
|
||||
SYSTEM DROP REPLICA ['DROP REPLICA'] TABLE SYSTEM
|
||||
SYSTEM SYNC REPLICA ['SYNC REPLICA'] TABLE SYSTEM
|
||||
SYSTEM RESTART REPLICA ['RESTART REPLICA'] TABLE SYSTEM
|
||||
SYSTEM RESTORE REPLICA ['RESTORE REPLICA'] TABLE SYSTEM
|
||||
SYSTEM FLUSH DISTRIBUTED ['FLUSH DISTRIBUTED'] TABLE SYSTEM FLUSH
|
||||
SYSTEM FLUSH LOGS ['FLUSH LOGS'] GLOBAL SYSTEM FLUSH
|
||||
SYSTEM FLUSH [] \N SYSTEM
|
||||
|
||||
@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_arrays"
|
||||
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_arrays"
|
||||
${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_arrays (arr1 Array(Int8), arr2 Array(UInt8), arr3 Array(Int16), arr4 Array(UInt16), arr5 Array(Int32), arr6 Array(UInt32), arr7 Array(Int64), arr8 Array(UInt64), arr9 Array(String), arr10 Array(FixedString(4)), arr11 Array(Float32), arr12 Array(Float64), arr13 Array(Date), arr14 Array(Datetime)) ENGINE=Memory()"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_arrays VALUES ([1,-2,3],[1,2,3],[100,-200,300],[100,200,300],[10000000,-20000000,30000000],[10000000,2000000,3000000],[100000000000000,-200000000000,3000000000000],[100000000000000,20000000000000,3000000000000],['Some string','Some string','Some string'],['0000','1111','2222'],[42.42,424.2,0.4242],[424242.424242,4242042420.242424,42],['2000-01-01','2001-01-01','2002-01-01'],['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00']),([],[],[],[],[],[],[],[],[],[],[],[],[],[])"
|
||||
|
||||
@ -0,0 +1,2 @@
|
||||
1 ['a','b','c'] ('z','6')
|
||||
2 ['d','e'] ('x','9')
|
||||
24
tests/queries/0_stateless/01273_arrow_dictionaries_load.sh
Executable file
24
tests/queries/0_stateless/01273_arrow_dictionaries_load.sh
Executable file
@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS arrow_dicts"
|
||||
${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_dicts (a LowCardinality(String), b Array(LowCardinality(String)), c Tuple(LowCardinality(String), LowCardinality(String))) ENGINE=Memory()"
|
||||
${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_dicts VALUES ('1', ['a', 'b', 'c'], ('z', '6')), ('2', ['d', 'e'], ('x', '9'))"
|
||||
${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_dicts FORMAT Arrow SETTINGS output_format_arrow_low_cardinality_as_dictionary=1" > "${CLICKHOUSE_TMP}"/dicts.arrow
|
||||
|
||||
cat "${CLICKHOUSE_TMP}"/dicts.arrow | ${CLICKHOUSE_CLIENT} -q "INSERT INTO arrow_dicts FORMAT Arrow"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_dicts"
|
||||
${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_dicts"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="CREATE TABLE arrow_dicts (a LowCardinality(String)) ENGINE=Memory()"
|
||||
${CLICKHOUSE_CLIENT} --query="INSERT INTO arrow_dicts SELECT toString(number % 500) from numbers(10000000)"
|
||||
${CLICKHOUSE_CLIENT} --query="SELECT * FROM arrow_dicts FORMAT Arrow SETTINGS output_format_arrow_low_cardinality_as_dictionary=1" > "${CLICKHOUSE_TMP}"/dicts.arrow
|
||||
|
||||
cat "${CLICKHOUSE_TMP}"/dicts.arrow | ${CLICKHOUSE_CLIENT} -q "INSERT INTO arrow_dicts FORMAT Arrow"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="DROP TABLE arrow_dicts"
|
||||
|
||||
@ -0,0 +1,37 @@
|
||||
distributed_push_down_limit=0
|
||||
100 100
|
||||
distributed_push_down_limit=1
|
||||
0
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
40 40
|
||||
auto-distributed_push_down_limit
|
||||
0
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
40 40
|
||||
distributed_push_down_limit=1 with OFFSET
|
||||
97
|
||||
96
|
||||
96
|
||||
95
|
||||
95
|
||||
94
|
||||
94
|
||||
93
|
||||
93
|
||||
92
|
||||
167
tests/queries/0_stateless/01814_distributed_push_down_limit.sh
Executable file
167
tests/queries/0_stateless/01814_distributed_push_down_limit.sh
Executable file
@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=SC2206
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
# -- NOTE: this test cannot use 'current_database = $CLICKHOUSE_DATABASE',
|
||||
# -- because it does not propagated via remote queries,
|
||||
# -- hence it uses query_id/initial_query_id.
|
||||
|
||||
function setup()
|
||||
{
|
||||
$CLICKHOUSE_CLIENT -nm -q "
|
||||
drop table if exists data_01814;
|
||||
drop table if exists dist_01814;
|
||||
|
||||
create table data_01814 (key Int) Engine=MergeTree() order by key settings index_granularity=10 as select * from numbers(100);
|
||||
create table dist_01814 as data_01814 engine=Distributed('test_cluster_two_shards', $CLICKHOUSE_DATABASE, data_01814, key);
|
||||
"
|
||||
}
|
||||
|
||||
function cleanup()
|
||||
{
|
||||
$CLICKHOUSE_CLIENT -nm -q "
|
||||
drop table data_01814;
|
||||
drop table dist_01814;
|
||||
"
|
||||
}
|
||||
|
||||
function make_query_id()
|
||||
{
|
||||
echo "$(tr -cd '[:lower:]' < /dev/urandom | head -c10)-$CLICKHOUSE_DATABASE"
|
||||
}
|
||||
|
||||
function test_distributed_push_down_limit_with_query_log()
|
||||
{
|
||||
local table=$1 && shift
|
||||
local offset=$1 && shift
|
||||
local query_id
|
||||
|
||||
query_id="$(make_query_id)"
|
||||
|
||||
# NOTES:
|
||||
# - max_rows_to_read_leaf cannot be used since it does not know anything
|
||||
# about optimize_aggregation_in_order,
|
||||
# - limit push down can be checked only with optimize_aggregation_in_order,
|
||||
# since otherwise the query will be canceled too early, and read_rows will be
|
||||
# small.
|
||||
local settings_and_opts=(
|
||||
--query_id "$query_id"
|
||||
|
||||
--max_block_size 20
|
||||
--optimize_aggregation_in_order 1
|
||||
--log_queries 1
|
||||
--log_queries_min_type 'QUERY_FINISH'
|
||||
|
||||
# disable hedged requests to avoid excessive log entries
|
||||
--use_hedged_requests 0
|
||||
|
||||
"$@"
|
||||
)
|
||||
|
||||
$CLICKHOUSE_CLIENT "${settings_and_opts[@]}" -q "select * from $table group by key limit $offset, 10"
|
||||
|
||||
$CLICKHOUSE_CLIENT -nm -q "
|
||||
system flush logs;
|
||||
select read_rows from system.query_log
|
||||
where
|
||||
event_date = today()
|
||||
and query_kind = 'Select' /* exclude DESC TABLE */
|
||||
and initial_query_id = '$query_id' and initial_query_id != query_id;
|
||||
" | xargs # convert new lines to spaces
|
||||
}
|
||||
|
||||
function test_distributed_push_down_limit_0()
|
||||
{
|
||||
local args=(
|
||||
"remote('127.{2,3}', $CLICKHOUSE_DATABASE, data_01814)"
|
||||
0 # offset
|
||||
--distributed_push_down_limit 0
|
||||
)
|
||||
test_distributed_push_down_limit_with_query_log "${args[@]}" "$@"
|
||||
}
|
||||
|
||||
function test_distributed_push_down_limit_1()
|
||||
{
|
||||
local args=(
|
||||
"remote('127.{2,3}', $CLICKHOUSE_DATABASE, data_01814)"
|
||||
0 # offset
|
||||
--distributed_push_down_limit 1
|
||||
)
|
||||
test_distributed_push_down_limit_with_query_log "${args[@]}"
|
||||
}
|
||||
|
||||
function test_distributed_push_down_limit_1_offset()
|
||||
{
|
||||
local settings_and_opts=(
|
||||
--distributed_push_down_limit 1
|
||||
)
|
||||
|
||||
$CLICKHOUSE_CLIENT "${settings_and_opts[@]}" -q "select * from remote('127.{2,3}', $CLICKHOUSE_DATABASE, data_01814) group by key order by key desc limit 5, 10"
|
||||
}
|
||||
|
||||
function test_auto_distributed_push_down_limit()
|
||||
{
|
||||
local args=(
|
||||
dist_01814
|
||||
0 # offset
|
||||
--optimize_skip_unused_shards 1
|
||||
--optimize_distributed_group_by_sharding_key 1
|
||||
--prefer_localhost_replica 0
|
||||
--distributed_push_down_limit 0
|
||||
)
|
||||
test_distributed_push_down_limit_with_query_log "${args[@]}"
|
||||
}
|
||||
|
||||
function main()
|
||||
{
|
||||
setup
|
||||
trap cleanup EXIT
|
||||
|
||||
echo 'distributed_push_down_limit=0'
|
||||
test_distributed_push_down_limit_0 --format Null
|
||||
|
||||
#
|
||||
# The following tests (tests with distributed_push_down_limit=1) requires
|
||||
# retries, since the query may be canceled earlier due to LIMIT, and so
|
||||
# only one shard will be processed, and it will get not 40 but 20 rows:
|
||||
#
|
||||
# 1.160920 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} <Debug> executeQuery: (from [::ffff:127.0.0.1]:42778, initial_query_id: 66cf643c-b1b4-4f7e-942a-c4c3493029f6, using production parser) (comment: /usr/share/clickhouse-test/queries/0_stateless/01814_distributed_push_down_limit.sql) WITH CAST('test_31uut9', 'String') AS id_distributed_push_down_limit_1 SELECT key FROM test_31uut9.data_01814 GROUP BY key LIMIT 10
|
||||
# 1.214964 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} <Trace> ContextAccess (default): Access granted: SELECT(key) ON test_31uut9.data_01814
|
||||
# 1.216790 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} <Debug> test_31uut9.data_01814 (b484ad2e-0591-4faf-8110-1dcbd7cdd0db) (SelectExecutor): Key condition: unknown
|
||||
# 1.227245 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} <Debug> test_31uut9.data_01814 (b484ad2e-0591-4faf-8110-1dcbd7cdd0db) (SelectExecutor): Selected 1/1 parts by partition key, 1 parts by primary key, 10/11 marks by primary key, 10 marks to read from 1 ranges
|
||||
# 1.228452 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} <Trace> MergeTreeSelectProcessor: Reading 3 ranges from part all_1_1_0, approx. 100 rows starting from 0
|
||||
# 1.229104 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} <Trace> InterpreterSelectQuery: FetchColumns -> WithMergeableStateAfterAggregationAndLimit
|
||||
# 1.339085 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} <Information> TCPHandler: Query was cancelled.
|
||||
# 1.416573 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} <Information> executeQuery: Read 20 rows, 80.00 B in 0.254374666 sec., 78 rows/sec., 314.50 B/sec.
|
||||
# 1.419006 [ 291 ] {7ac5de70-c26c-4e3b-bdee-3873ad1b84f1} <Debug> MemoryTracker: Peak memory usage (for query): 0.00 B.
|
||||
#
|
||||
|
||||
local out out_lines max_tries=20
|
||||
|
||||
echo 'distributed_push_down_limit=1'
|
||||
for ((i = 0; i < max_tries; ++i)); do
|
||||
out=$(test_distributed_push_down_limit_1)
|
||||
out_lines=( $out )
|
||||
if [[ ${#out_lines[@]} -gt 2 ]] && [[ ${out_lines[-1]} = 40 ]] && [[ ${out_lines[-2]} = 40 ]]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
echo "$out"
|
||||
|
||||
echo 'auto-distributed_push_down_limit'
|
||||
for ((i = 0; i < max_tries; ++i)); do
|
||||
out=$(test_auto_distributed_push_down_limit)
|
||||
out_lines=( $out )
|
||||
if [[ ${#out_lines[@]} -gt 2 ]] && [[ ${out_lines[-1]} = 40 ]] && [[ ${out_lines[-2]} = 40 ]]; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
echo "$out"
|
||||
|
||||
echo 'distributed_push_down_limit=1 with OFFSET'
|
||||
test_distributed_push_down_limit_1_offset
|
||||
}
|
||||
main "$@"
|
||||
@ -0,0 +1 @@
|
||||
0
|
||||
@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
# with_mergeable_state_after_aggregation will not stop after 1 row, while with_mergeable_state_after_aggregation_and_limit should
|
||||
$CLICKHOUSE_CLIENT -q 'select * from system.numbers limit 1' --stage with_mergeable_state_after_aggregation_and_limit
|
||||
34
tests/queries/0_stateless/01910_client_replxx_container_overflow_long.expect
Executable file
34
tests/queries/0_stateless/01910_client_replxx_container_overflow_long.expect
Executable file
@ -0,0 +1,34 @@
|
||||
#!/usr/bin/expect -f
|
||||
|
||||
log_user 0
|
||||
set timeout 60
|
||||
match_max 100000
|
||||
# A default timeout action is to do nothing, change it to fail
|
||||
expect_after {
|
||||
timeout {
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
set basedir [file dirname $argv0]
|
||||
|
||||
# history file is not required, in-memory history is enough
|
||||
spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$basedir/01910_client_replxx_container_overflow_long.history.log"
|
||||
expect ":) "
|
||||
|
||||
# Make a query.
|
||||
send -- "SELECT 1\r"
|
||||
expect "1"
|
||||
expect ":) "
|
||||
|
||||
# Do reverse-search.
|
||||
send -- ""
|
||||
expect "(reverse-i-search)"
|
||||
send -- "1"
|
||||
expect "(reverse-i-search)"
|
||||
# This will trigger the container-overflow under ASAN before the fix.
|
||||
send -- ""
|
||||
expect "(reverse-i-search)"
|
||||
|
||||
# Exit.
|
||||
send -- "\4"
|
||||
expect eof
|
||||
@ -0,0 +1,5 @@
|
||||
-- { echo }
|
||||
EXPLAIN SYNTAX SELECT NOT NOT (NOT (NOT (NULL)));
|
||||
SELECT NOT (NOT (NOT NOT NULL))
|
||||
EXPLAIN SYNTAX SELECT NOT (NOT (NOT NOT NULL));
|
||||
SELECT NOT (NOT (NOT NOT NULL))
|
||||
3
tests/queries/0_stateless/01920_not_chain_format.sql
Normal file
3
tests/queries/0_stateless/01920_not_chain_format.sql
Normal file
@ -0,0 +1,3 @@
|
||||
-- { echo }
|
||||
EXPLAIN SYNTAX SELECT NOT NOT (NOT (NOT (NULL)));
|
||||
EXPLAIN SYNTAX SELECT NOT (NOT (NOT NOT NULL));
|
||||
@ -234,6 +234,7 @@
|
||||
01801_distinct_group_by_shard
|
||||
01804_dictionary_decimal256_type
|
||||
01801_s3_distributed
|
||||
01814_distributed_push_down_limit
|
||||
01833_test_collation_alvarotuso
|
||||
01850_dist_INSERT_preserve_error
|
||||
01870_modulo_partition_key
|
||||
|
||||
@ -1 +1 @@
|
||||
`a` Nullable(String), `b` Nullable(Int32), `c` Nullable(Float64), `d` Nullable(UInt8), `e` Nullable(Int32)
|
||||
`a` Nullable(String), `b` Array(Nullable(Int32)), `c` Nullable(Float64), `d` Nullable(UInt8), `e` Array(Nullable(Int32))
|
||||
|
||||
@ -1 +1 @@
|
||||
`a` Tuple(Nullable(String), Nullable(Int32), Nullable(UInt8)), `b` Nullable(Int32), `c` Nullable(Float64)
|
||||
`a` Map(String, Map(Int32, Nullable(UInt8))), `b` Nullable(Int32), `c` Nullable(Float64)
|
||||
|
||||
@ -1 +1 @@
|
||||
`ID` Nullable(Int64), `Int_Array` Nullable(Int32), `int_array_array` Nullable(Int32), `Int_Map` Tuple(Nullable(String), Nullable(Int32)), `int_map_array` Tuple(Nullable(String), Nullable(Int32)), `nested_Struct` Tuple(Nullable(Int32), Nullable(Int32), Nullable(Int32), Nullable(String), Nullable(String), Nullable(Float64))
|
||||
`ID` Nullable(Int64), `Int_Array` Array(Nullable(Int32)), `int_array_array` Array(Array(Nullable(Int32))), `Int_Map` Map(String, Nullable(Int32)), `int_map_array` Array(Map(String, Nullable(Int32))), `nested_Struct` Tuple(Nullable(Int32), Array(Nullable(Int32)), Tuple(Array(Array(Tuple(Nullable(Int32))))), Map(String, Tuple(Tuple(Array(Nullable(Float64))))))
|
||||
|
||||
@ -1 +1 @@
|
||||
`id` Nullable(Int64), `int_array` Nullable(Int32), `int_array_Array` Nullable(Int32), `int_map` Tuple(Nullable(String), Nullable(Int32)), `int_Map_Array` Tuple(Nullable(String), Nullable(Int32)), `nested_struct` Tuple(Nullable(Int32), Nullable(Int32), Nullable(Int32), Nullable(String), Nullable(String), Nullable(Float64))
|
||||
`id` Nullable(Int64), `int_array` Array(Nullable(Int32)), `int_array_Array` Array(Array(Nullable(Int32))), `int_map` Map(String, Nullable(Int32)), `int_Map_Array` Array(Map(String, Nullable(Int32))), `nested_struct` Tuple(Nullable(Int32), Array(Nullable(Int32)), Tuple(Array(Array(Tuple(Nullable(Int32))))), Map(String, Tuple(Tuple(Array(Nullable(Float64))))))
|
||||
|
||||
@ -1 +1 @@
|
||||
`b_struct` Nullable(Int32)
|
||||
`b_struct` Tuple(Nullable(Int32))
|
||||
|
||||
@ -22,7 +22,7 @@ xflags = {
|
||||
@XFlags(xflags)
|
||||
@Name("extended precision data types")
|
||||
@Specifications(
|
||||
QA_SRS020_ClickHouse_Extended_Precision_Data_Types
|
||||
SRS020_ClickHouse_Extended_Precision_Data_Types
|
||||
)
|
||||
@Requirements(
|
||||
RQ_SRS_020_ClickHouse_Extended_Precision("1.0"),
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
# QA-SRS020 ClickHouse Extended Precision Data Types
|
||||
# SRS020 ClickHouse Extended Precision Data Types
|
||||
# Software Requirements Specification
|
||||
|
||||
## Table of Contents
|
||||
|
||||
@ -754,8 +754,8 @@ RQ_SRS_020_ClickHouse_Extended_Precision_Create_Table = Requirement(
|
||||
level=3,
|
||||
num='4.13.1')
|
||||
|
||||
QA_SRS020_ClickHouse_Extended_Precision_Data_Types = Specification(
|
||||
name='QA-SRS020 ClickHouse Extended Precision Data Types',
|
||||
SRS020_ClickHouse_Extended_Precision_Data_Types = Specification(
|
||||
name='SRS020 ClickHouse Extended Precision Data Types',
|
||||
description=None,
|
||||
author=None,
|
||||
date=None,
|
||||
@ -855,7 +855,7 @@ QA_SRS020_ClickHouse_Extended_Precision_Data_Types = Specification(
|
||||
RQ_SRS_020_ClickHouse_Extended_Precision_Create_Table,
|
||||
),
|
||||
content='''
|
||||
# QA-SRS020 ClickHouse Extended Precision Data Types
|
||||
# SRS020 ClickHouse Extended Precision Data Types
|
||||
# Software Requirements Specification
|
||||
|
||||
## Table of Contents
|
||||
|
||||
@ -30,6 +30,7 @@ issue_18110 = "https://github.com/ClickHouse/ClickHouse/issues/18110"
|
||||
issue_18206 = "https://github.com/ClickHouse/ClickHouse/issues/18206"
|
||||
issue_21083 = "https://github.com/ClickHouse/ClickHouse/issues/21083"
|
||||
issue_21084 = "https://github.com/ClickHouse/ClickHouse/issues/21084"
|
||||
issue_25413 = "https://github.com/ClickHouse/ClickHouse/issues/25413"
|
||||
|
||||
xfails = {
|
||||
"syntax/show create quota/I show create quota current":
|
||||
@ -144,6 +145,12 @@ xfails = {
|
||||
[(Fail, "new bug")],
|
||||
"privileges/show dictionaries/:/check privilege/check privilege=DROP DICTIONARY/show dict/SHOW DICTIONARIES with privilege":
|
||||
[(Fail, "new bug")],
|
||||
"privileges/kill mutation/:/:/KILL ALTER : without privilege":
|
||||
[(Fail, issue_25413)],
|
||||
"privileges/kill mutation/:/:/KILL ALTER : with revoked privilege":
|
||||
[(Fail, issue_25413)],
|
||||
"privileges/kill mutation/:/:/KILL ALTER : with revoked ALL privilege":
|
||||
[(Fail, issue_25413)]
|
||||
}
|
||||
|
||||
xflags = {
|
||||
|
||||
@ -202,7 +202,7 @@ def check_modify_column_when_privilege_is_granted(table, user, node, column=None
|
||||
column = 'modify'
|
||||
|
||||
with Given(f"I add the column {column}"):
|
||||
node.query(f"ALTER TABLE {table} ADD COLUMN {column} String")
|
||||
node.query(f"ALTER TABLE {table} ADD COLUMN {column} String DEFAULT '0'")
|
||||
|
||||
with When(f"I insert some data into column {column}"):
|
||||
node.query(f"INSERT INTO {table} ({column}) VALUES ('3.4')")
|
||||
|
||||
@ -22,15 +22,15 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None):
|
||||
tasks = []
|
||||
with Pool(8) as pool:
|
||||
try:
|
||||
run_scenario(pool, tasks, Feature(test=load("example.regression", "regression")), args)
|
||||
run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args)
|
||||
#run_scenario(pool, tasks, Feature(test=load("example.regression", "regression")), args)
|
||||
#run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args)
|
||||
#run_scenario(pool, tasks, Feature(test=load("rbac.regression", "regression")), args)
|
||||
run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args)
|
||||
run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args)
|
||||
run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args)
|
||||
run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args)
|
||||
#run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args)
|
||||
#run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args)
|
||||
#run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args)
|
||||
#run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args)
|
||||
#run_scenario(pool, tasks, Feature(test=load("kerberos.regression", "regression")), args)
|
||||
run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args)
|
||||
#run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args)
|
||||
finally:
|
||||
join(tasks)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user