thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 09:32:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:ClickHouse/ClickHouse

Browse Source
This commit is contained in:
Sergei Shtykov 2019-12-26 13:19:24 +03:00
commit fe187029f2
122 changed files with 1182 additions and 713 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/base64 vendored

@ -1 +1 @@
Subproject commit a27c565d1b6c676beaf297fe503c4518185666f7
Subproject commit 32a87227aa254e2f02332c0ed8bc375522414fa3

27
contrib/base64-cmake/CMakeLists.txt
View File

@ -25,19 +25,20 @@ endif ()
configure_file(config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h)
add_library(base64
${LIBRARY_DIR}/lib/lib.c
${LIBRARY_DIR}/lib/codec_choose.c
${LIBRARY_DIR}/lib/arch/avx/codec.c
${LIBRARY_DIR}/lib/arch/avx2/codec.c
${LIBRARY_DIR}/lib/arch/generic/codec.c
${LIBRARY_DIR}/lib/arch/neon32/codec.c
${LIBRARY_DIR}/lib/arch/neon64/codec.c
${LIBRARY_DIR}/lib/arch/sse41/codec.c
${LIBRARY_DIR}/lib/arch/sse42/codec.c
${LIBRARY_DIR}/lib/arch/ssse3/codec.c
${LIBRARY_DIR}/lib/lib.c
${LIBRARY_DIR}/lib/codec_choose.c
${LIBRARY_DIR}/lib/arch/avx/codec.c
${LIBRARY_DIR}/lib/arch/avx2/codec.c
${LIBRARY_DIR}/lib/arch/generic/codec.c
${LIBRARY_DIR}/lib/arch/neon32/codec.c
${LIBRARY_DIR}/lib/arch/neon64/codec.c
${LIBRARY_DIR}/lib/arch/sse41/codec.c
${LIBRARY_DIR}/lib/arch/sse42/codec.c
${LIBRARY_DIR}/lib/arch/ssse3/codec.c
${LIBRARY_DIR}/lib/codecs.h
${CMAKE_CURRENT_BINARY_DIR}/config.h)
${LIBRARY_DIR}/lib/codecs.h
${CMAKE_CURRENT_BINARY_DIR}/config.h
${CMAKE_CURRENT_SOURCE_DIR}/tables.h)
if(HAVE_AVX)
set_source_files_properties(${LIBRARY_DIR}/lib/arch/avx/codec.c PROPERTIES COMPILE_FLAGS -mavx)
@ -55,4 +56,4 @@ if(HAVE_SSSE3)
set_source_files_properties(${LIBRARY_DIR}/lib/arch/ssse3/codec.c PROPERTIES COMPILE_FLAGS -mssse3)
endif()
target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include ${CMAKE_CURRENT_BINARY_DIR})
target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})

5
contrib/base64-cmake/README.md Normal file
View File

@ -0,0 +1,5 @@
`tables.h` file was generated by the base64 library with it's makefile:
```
cc -std=c99 -O3 -Wall -Wextra -pedantic -o lib/table_generator lib/table_generator.c
./lib/table_generator > lib/tables.h
```

393
contrib/base64-cmake/tables.h Normal file
View File

@ -0,0 +1,393 @@
#include <stdint.h>
#define CHAR62 '+'
#define CHAR63 '/'
#define CHARPAD '='
#if BASE64_LITTLE_ENDIAN
/* SPECIAL DECODE TABLES FOR LITTLE ENDIAN (INTEL) CPUS */
const uint32_t base64_table_dec_d0[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x000000f8, 0xffffffff, 0xffffffff, 0xffffffff, 0x000000fc,
0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4,
0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018,
0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030,
0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048,
0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060,
0x00000064, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078,
0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090,
0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8,
0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0,
0x000000c4, 0x000000c8, 0x000000cc, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d1[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x0000e003, 0xffffffff, 0xffffffff, 0xffffffff, 0x0000f003,
0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003,
0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000,
0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000,
0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001,
0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001,
0x00009001, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001,
0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002,
0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002,
0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003,
0x00001003, 0x00002003, 0x00003003, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d2[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00800f00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00c00f00,
0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00,
0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100,
0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300,
0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400,
0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600,
0x00400600, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700,
0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900,
0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00,
0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00,
0x00400c00, 0x00800c00, 0x00c00c00, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d3[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x003e0000, 0xffffffff, 0xffffffff, 0xffffffff, 0x003f0000,
0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000,
0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000,
0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000,
0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000,
0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000,
0x00190000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000,
0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000,
0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000,
0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000,
0x00310000, 0x00320000, 0x00330000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
#else
/* SPECIAL DECODE TABLES FOR BIG ENDIAN (IBM/MOTOROLA/SUN) CPUS */
const uint32_t base64_table_dec_d0[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xf8000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xfc000000,
0xd0000000, 0xd4000000, 0xd8000000, 0xdc000000, 0xe0000000, 0xe4000000,
0xe8000000, 0xec000000, 0xf0000000, 0xf4000000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x04000000, 0x08000000, 0x0c000000, 0x10000000, 0x14000000, 0x18000000,
0x1c000000, 0x20000000, 0x24000000, 0x28000000, 0x2c000000, 0x30000000,
0x34000000, 0x38000000, 0x3c000000, 0x40000000, 0x44000000, 0x48000000,
0x4c000000, 0x50000000, 0x54000000, 0x58000000, 0x5c000000, 0x60000000,
0x64000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x68000000, 0x6c000000, 0x70000000, 0x74000000, 0x78000000,
0x7c000000, 0x80000000, 0x84000000, 0x88000000, 0x8c000000, 0x90000000,
0x94000000, 0x98000000, 0x9c000000, 0xa0000000, 0xa4000000, 0xa8000000,
0xac000000, 0xb0000000, 0xb4000000, 0xb8000000, 0xbc000000, 0xc0000000,
0xc4000000, 0xc8000000, 0xcc000000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d1[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x03e00000, 0xffffffff, 0xffffffff, 0xffffffff, 0x03f00000,
0x03400000, 0x03500000, 0x03600000, 0x03700000, 0x03800000, 0x03900000,
0x03a00000, 0x03b00000, 0x03c00000, 0x03d00000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00100000, 0x00200000, 0x00300000, 0x00400000, 0x00500000, 0x00600000,
0x00700000, 0x00800000, 0x00900000, 0x00a00000, 0x00b00000, 0x00c00000,
0x00d00000, 0x00e00000, 0x00f00000, 0x01000000, 0x01100000, 0x01200000,
0x01300000, 0x01400000, 0x01500000, 0x01600000, 0x01700000, 0x01800000,
0x01900000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x01a00000, 0x01b00000, 0x01c00000, 0x01d00000, 0x01e00000,
0x01f00000, 0x02000000, 0x02100000, 0x02200000, 0x02300000, 0x02400000,
0x02500000, 0x02600000, 0x02700000, 0x02800000, 0x02900000, 0x02a00000,
0x02b00000, 0x02c00000, 0x02d00000, 0x02e00000, 0x02f00000, 0x03000000,
0x03100000, 0x03200000, 0x03300000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d2[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x000f8000, 0xffffffff, 0xffffffff, 0xffffffff, 0x000fc000,
0x000d0000, 0x000d4000, 0x000d8000, 0x000dc000, 0x000e0000, 0x000e4000,
0x000e8000, 0x000ec000, 0x000f0000, 0x000f4000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00004000, 0x00008000, 0x0000c000, 0x00010000, 0x00014000, 0x00018000,
0x0001c000, 0x00020000, 0x00024000, 0x00028000, 0x0002c000, 0x00030000,
0x00034000, 0x00038000, 0x0003c000, 0x00040000, 0x00044000, 0x00048000,
0x0004c000, 0x00050000, 0x00054000, 0x00058000, 0x0005c000, 0x00060000,
0x00064000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00068000, 0x0006c000, 0x00070000, 0x00074000, 0x00078000,
0x0007c000, 0x00080000, 0x00084000, 0x00088000, 0x0008c000, 0x00090000,
0x00094000, 0x00098000, 0x0009c000, 0x000a0000, 0x000a4000, 0x000a8000,
0x000ac000, 0x000b0000, 0x000b4000, 0x000b8000, 0x000bc000, 0x000c0000,
0x000c4000, 0x000c8000, 0x000cc000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d3[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00003e00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00003f00,
0x00003400, 0x00003500, 0x00003600, 0x00003700, 0x00003800, 0x00003900,
0x00003a00, 0x00003b00, 0x00003c00, 0x00003d00, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00000100, 0x00000200, 0x00000300, 0x00000400, 0x00000500, 0x00000600,
0x00000700, 0x00000800, 0x00000900, 0x00000a00, 0x00000b00, 0x00000c00,
0x00000d00, 0x00000e00, 0x00000f00, 0x00001000, 0x00001100, 0x00001200,
0x00001300, 0x00001400, 0x00001500, 0x00001600, 0x00001700, 0x00001800,
0x00001900, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00001a00, 0x00001b00, 0x00001c00, 0x00001d00, 0x00001e00,
0x00001f00, 0x00002000, 0x00002100, 0x00002200, 0x00002300, 0x00002400,
0x00002500, 0x00002600, 0x00002700, 0x00002800, 0x00002900, 0x00002a00,
0x00002b00, 0x00002c00, 0x00002d00, 0x00002e00, 0x00002f00, 0x00003000,
0x00003100, 0x00003200, 0x00003300, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
#endif

4
dbms/CMakeLists.txt
View File

@ -186,8 +186,8 @@ endif()
list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})
list (APPEND dbms_sources src/Functions/IFunction.cpp src/Functions/FunctionFactory.cpp src/Functions/FunctionHelpers.cpp)
list (APPEND dbms_headers src/Functions/IFunctionImpl.h src/Functions/FunctionFactory.h src/Functions/FunctionHelpers.h)
list (APPEND dbms_sources src/Functions/IFunction.cpp src/Functions/FunctionFactory.cpp src/Functions/FunctionHelpers.cpp src/Functions/extractTimeZoneFromFunctionArguments.cpp)
list (APPEND dbms_headers src/Functions/IFunctionImpl.h src/Functions/FunctionFactory.h src/Functions/FunctionHelpers.h src/Functions/extractTimeZoneFromFunctionArguments.h)
list (APPEND dbms_sources
src/AggregateFunctions/AggregateFunctionFactory.cpp

1
dbms/programs/server/config.d/log_to_console.xml
View File

@ -1,5 +1,6 @@
<yandex>
<logger>
<console>true</console>
<log remove="remove"/>
<errorlog remove="remove"/>
</logger>

5
dbms/src/Compression/CompressedReadBufferBase.cpp
View File

@ -32,6 +32,7 @@ namespace ErrorCodes
extern const int TOO_LARGE_SIZE_COMPRESSED;
extern const int CHECKSUM_DOESNT_MATCH;
extern const int CANNOT_DECOMPRESS;
extern const int CORRUPTED_DATA;
}
using Checksum = CityHash_v1_0_2::uint128;
@ -124,6 +125,10 @@ size_t CompressedReadBufferBase::readCompressedData(size_t & size_decompressed,
+ ". Most likely corrupted data.",
ErrorCodes::TOO_LARGE_SIZE_COMPRESSED);
if (size_compressed_without_checksum < header_size)
throw Exception("Can't decompress data: the compressed data size (" + toString(size_compressed_without_checksum)
+ ", this should include header size) is less than the header size (" + toString(header_size) + ")", ErrorCodes::CORRUPTED_DATA);
ProfileEvents::increment(ProfileEvents::ReadCompressedBytes, size_compressed_without_checksum + sizeof(Checksum));
/// Is whole compressed block located in 'compressed_in->' buffer?

14
dbms/src/Compression/CompressionCodecMultiple.cpp
View File

@ -5,6 +5,8 @@
#include <Compression/CompressionFactory.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
#include <Common/hex.h>
#include <sstream>
@ -31,16 +33,15 @@ UInt8 CompressionCodecMultiple::getMethodByte() const
String CompressionCodecMultiple::getCodecDesc() const
{
std::ostringstream ss;
for (size_t idx = 0; idx < codecs.size(); idx++)
WriteBufferFromOwnString out;
for (size_t idx = 0; idx < codecs.size(); ++idx)
{
if (idx != 0)
ss << ',' << ' ';
out << ", ";
const auto codec = codecs[idx];
ss << codec->getCodecDesc();
out << codecs[idx]->getCodecDesc();
}
return ss.str();
return out.str();
}
UInt32 CompressionCodecMultiple::getMaxCompressedDataSize(UInt32 uncompressed_size) const
@ -55,7 +56,6 @@ UInt32 CompressionCodecMultiple::getMaxCompressedDataSize(UInt32 uncompressed_si
UInt32 CompressionCodecMultiple::doCompressData(const char * source, UInt32 source_size, char * dest) const
{
PODArray<char> compressed_buf;
PODArray<char> uncompressed_buf(source, source + source_size);

9
dbms/src/Compression/ICompressionCodec.cpp
View File

@ -8,6 +8,7 @@
#include <Common/typeid_cast.h>
#include <Compression/CompressionFactory.h>
namespace ProfileEvents
{
extern const Event ReadCompressedBytes;
@ -25,6 +26,7 @@ namespace ErrorCodes
extern const int UNKNOWN_COMPRESSION_METHOD;
extern const int CANNOT_DECOMPRESS;
extern const int SEEK_POSITION_OUT_OF_BOUND;
extern const int CORRUPTED_DATA;
}
@ -42,11 +44,16 @@ UInt32 ICompressionCodec::compress(const char * source, UInt32 source_size, char
UInt32 ICompressionCodec::decompress(const char * source, UInt32 source_size, char * dest) const
{
UInt8 header_size = getHeaderSize();
if (source_size < header_size)
throw Exception("Can't decompress data: the compressed data size (" + toString(source_size)
+ ", this should include header size) is less than the header size (" + toString(header_size) + ")", ErrorCodes::CORRUPTED_DATA);
UInt8 method = source[0];
if (method != getMethodByte())
throw Exception("Can't decompress data with codec byte " + toString(method) + " from codec with byte " + toString(method), ErrorCodes::CANNOT_DECOMPRESS);
UInt8 header_size = getHeaderSize();
UInt32 decompressed_size = readDecompressedBlockSize(source);
doDecompressData(&source[header_size], source_size - header_size, dest, decompressed_size);

6
dbms/src/Compression/ICompressionCodec.h
View File

@ -33,10 +33,10 @@ public:
virtual String getCodecDesc() const = 0;
/// Compressed bytes from uncompressed source to dest. Dest should preallocate memory
virtual UInt32 compress(const char * source, UInt32 source_size, char * dest) const;
UInt32 compress(const char * source, UInt32 source_size, char * dest) const;
/// Decompress bytes from compressed source to dest. Dest should preallocate memory
virtual UInt32 decompress(const char * source, UInt32 source_size, char * dest) const;
UInt32 decompress(const char * source, UInt32 source_size, char * dest) const;
/// Number of bytes, that will be used to compress uncompressed_size bytes with current codec
virtual UInt32 getCompressedReserveSize(UInt32 uncompressed_size) const { return getHeaderSize() + getMaxCompressedDataSize(uncompressed_size); }
@ -45,7 +45,7 @@ public:
virtual UInt32 getAdditionalSizeAtTheEndOfBuffer() const { return 0; }
/// Size of header in compressed data on disk
static UInt8 getHeaderSize() { return COMPRESSED_BLOCK_HEADER_SIZE; }
static constexpr UInt8 getHeaderSize() { return COMPRESSED_BLOCK_HEADER_SIZE; }
/// Read size of compressed block from compressed source
static UInt32 readCompressedBlockSize(const char * source);

2
dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp
View File

@ -65,7 +65,7 @@ void ParallelParsingBlockInputStream::parserThreadFunction(size_t current_unit_n
*/
ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0);
auto parser = std::make_unique<InputStreamFromInputFormat>(
input_processor_creator(read_buffer, header, context,
input_processor_creator(read_buffer, header,
row_input_format_params, format_settings));
unit.block_ext.block.clear();

12
dbms/src/DataStreams/ParallelParsingBlockInputStream.h
View File

@ -55,23 +55,21 @@ private:
using InputProcessorCreator = std::function<InputFormatPtr(
ReadBuffer & buf,
const Block & header,
const Context & context,
const RowInputFormatParams & params,
const FormatSettings & settings)>;
public:
struct InputCreatorParams
{
const Block &sample;
const Context &context;
const RowInputFormatParams& row_input_format_params;
const Block & sample;
const RowInputFormatParams & row_input_format_params;
const FormatSettings &settings;
};
struct Params
{
ReadBuffer & read_buffer;
const InputProcessorCreator &input_processor_creator;
const InputCreatorParams &input_creator_params;
const InputProcessorCreator & input_processor_creator;
const InputCreatorParams & input_creator_params;
FormatFactory::FileSegmentationEngine file_segmentation_engine;
int max_threads;
size_t min_chunk_bytes;
@ -79,7 +77,6 @@ public:
explicit ParallelParsingBlockInputStream(const Params & params)
: header(params.input_creator_params.sample),
context(params.input_creator_params.context),
row_input_format_params(params.input_creator_params.row_input_format_params),
format_settings(params.input_creator_params.settings),
input_processor_creator(params.input_processor_creator),
@ -149,7 +146,6 @@ protected:
private:
const Block header;
const Context context;
const RowInputFormatParams row_input_format_params;
const FormatSettings format_settings;
const InputProcessorCreator input_processor_creator;

14
dbms/src/Databases/DatabaseLazy.cpp
View File

@ -30,22 +30,22 @@ namespace ErrorCodes
DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context)
DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context_)
: name(name_)
, metadata_path(metadata_path_)
, data_path(context.getPath() + "data/" + escapeForFileName(name) + "/")
, data_path("data/" + escapeForFileName(name) + "/")
, expiration_time(expiration_time_)
, log(&Logger::get("DatabaseLazy (" + name + ")"))
{
Poco::File(getDataPath()).createDirectories();
Poco::File(context_.getPath() + getDataPath()).createDirectories();
}
void DatabaseLazy::loadStoredObjects(
Context & /* context */,
Context & context,
bool /* has_force_restore_data_flag */)
{
DatabaseOnDisk::iterateMetadataFiles(*this, log, [this](const String & file_name)
DatabaseOnDisk::iterateMetadataFiles(*this, log, context, [this](const String & file_name)
{
const std::string table_name = file_name.substr(0, file_name.size() - 4);
attachTable(table_name, nullptr);
@ -185,9 +185,9 @@ void DatabaseLazy::alterTable(
}
void DatabaseLazy::drop()
void DatabaseLazy::drop(const Context & context)
{
DatabaseOnDisk::drop(*this);
DatabaseOnDisk::drop(*this, context);
}
bool DatabaseLazy::isTableExist(

4
dbms/src/Databases/DatabaseLazy.h
View File

@ -18,7 +18,7 @@ class DatabaseLazyIterator;
class DatabaseLazy : public IDatabase
{
public:
DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context);
DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context_);
String getEngineName() const override { return "Lazy"; }
@ -87,7 +87,7 @@ public:
String getMetadataPath() const override;
String getObjectMetadataPath(const String & table_name) const override;
void drop() override;
void drop(const Context & context) override;
bool isTableExist(
const Context & context,

2
dbms/src/Databases/DatabaseMySQL.cpp
View File

@ -332,7 +332,7 @@ void DatabaseMySQL::shutdown()
local_tables_cache.clear();
}
void DatabaseMySQL::drop()
void DatabaseMySQL::drop(const Context & /*context*/)
{
Poco::File(getMetadataPath()).remove(true);
}

2
dbms/src/Databases/DatabaseMySQL.h
View File

@ -60,7 +60,7 @@ public:
void shutdown() override;
void drop() override;
void drop(const Context & /*context*/) override;
String getMetadataPath() const override;

13
dbms/src/Databases/DatabaseOnDisk.cpp
View File

@ -129,7 +129,7 @@ ASTPtr parseCreateQueryFromMetadataFile(const String & filepath, Poco::Logger *
std::pair<String, StoragePtr> createTableFromAST(
ASTCreateQuery ast_create_query,
const String & database_name,
const String & database_data_path,
const String & database_data_path_relative,
Context & context,
bool has_force_restore_data_flag)
{
@ -152,12 +152,13 @@ std::pair<String, StoragePtr> createTableFromAST(
ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context);
ConstraintsDescription constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints);
String table_data_path_relative = database_data_path_relative + escapeForFileName(ast_create_query.table) + '/';
return
{
ast_create_query.table,
StorageFactory::instance().get(
ast_create_query,
database_data_path, ast_create_query.table, database_name, context, context.getGlobalContext(),
table_data_path_relative, ast_create_query.table, database_name, context, context.getGlobalContext(),
columns, constraints,
true, has_force_restore_data_flag)
};
@ -495,9 +496,9 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery(const IDatabase & database, const
return ast;
}
void DatabaseOnDisk::drop(const IDatabase & database)
void DatabaseOnDisk::drop(const IDatabase & database, const Context & context)
{
Poco::File(database.getDataPath()).remove(false);
Poco::File(context.getPath() + database.getDataPath()).remove(false);
Poco::File(database.getMetadataPath()).remove(false);
}
@ -519,7 +520,7 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(
return static_cast<time_t>(0);
}
void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const IteratingFunction & iterating_function)
void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const Context & context, const IteratingFunction & iterating_function)
{
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator dir_it(database.getMetadataPath()); dir_it != dir_end; ++dir_it)
@ -537,7 +538,7 @@ void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logg
if (endsWith(dir_it.name(), tmp_drop_ext))
{
const std::string object_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
if (Poco::File(database.getDataPath() + '/' + object_name).exists())
if (Poco::File(context.getPath() + database.getDataPath() + '/' + object_name).exists())
{
/// TODO maybe complete table drop and remove all table data (including data on other volumes and metadata in ZK)
Poco::File(dir_it->path()).renameTo(database.getMetadataPath() + object_name + ".sql");

8
dbms/src/Databases/DatabaseOnDisk.h
View File

@ -24,7 +24,7 @@ ASTPtr parseCreateQueryFromMetadataFile(const String & filepath, Poco::Logger *
std::pair<String, StoragePtr> createTableFromAST(
ASTCreateQuery ast_create_query,
const String & database_name,
const String & database_data_path,
const String & database_data_path_relative,
Context & context,
bool has_force_restore_data_flag);
@ -98,7 +98,7 @@ public:
const IDatabase & database,
const Context & context);
static void drop(const IDatabase & database);
static void drop(const IDatabase & database, const Context & context);
static String getObjectMetadataPath(
const IDatabase & database,
@ -110,7 +110,7 @@ public:
using IteratingFunction = std::function<void(const String &)>;
static void iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const IteratingFunction & iterating_function);
static void iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const Context & context, const IteratingFunction & iterating_function);
private:
static ASTPtr getCreateTableQueryImpl(
@ -156,7 +156,7 @@ void DatabaseOnDisk::renameTable(
/// Notify the table that it is renamed. If the table does not support renaming, exception is thrown.
try
{
table->rename(context.getPath() + "/data/" + escapeForFileName(to_database_concrete->getDatabaseName()) + "/",
table->rename("/data/" + escapeForFileName(to_database_concrete->getDatabaseName()) + "/" + escapeForFileName(to_table_name) + '/',
to_database_concrete->getDatabaseName(),
to_table_name, lock);
}

17
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -59,7 +59,6 @@ namespace
Context & context,
const ASTCreateQuery & query,
DatabaseOrdinary & database,
const String database_data_path,
const String & database_name,
bool has_force_restore_data_flag)
{
@ -69,7 +68,7 @@ namespace
String table_name;
StoragePtr table;
std::tie(table_name, table)
= createTableFromAST(query, database_name, database_data_path, context, has_force_restore_data_flag);
= createTableFromAST(query, database_name, database.getDataPath(), context, has_force_restore_data_flag);
database.attachTable(table_name, table);
}
catch (const Exception & e)
@ -115,13 +114,13 @@ namespace
}
DatabaseOrdinary::DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context)
DatabaseOrdinary::DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context_)
: DatabaseWithOwnTablesBase(std::move(name_))
, metadata_path(metadata_path_)
, data_path(context.getPath() + "data/" + escapeForFileName(name) + "/")
, data_path("data/" + escapeForFileName(name) + "/")
, log(&Logger::get("DatabaseOrdinary (" + name + ")"))
{
Poco::File(getDataPath()).createDirectories();
Poco::File(context_.getPath() + getDataPath()).createDirectories();
}
@ -138,7 +137,7 @@ void DatabaseOrdinary::loadStoredObjects(
FileNames file_names;
size_t total_dictionaries = 0;
DatabaseOnDisk::iterateMetadataFiles(*this, log, [&file_names, &total_dictionaries, this](const String & file_name)
DatabaseOnDisk::iterateMetadataFiles(*this, log, context, [&file_names, &total_dictionaries, this](const String & file_name)
{
String full_path = metadata_path + "/" + file_name;
try
@ -176,7 +175,7 @@ void DatabaseOrdinary::loadStoredObjects(
if (!create_query.is_dictionary)
pool.scheduleOrThrowOnError([&]()
{
tryAttachTable(context, create_query, *this, getDataPath(), getDatabaseName(), has_force_restore_data_flag);
tryAttachTable(context, create_query, *this, getDatabaseName(), has_force_restore_data_flag);
/// Messages, so that it's not boring to wait for the server to load for a long time.
logAboutProgress(log, ++tables_processed, total_tables, watch);
@ -374,9 +373,9 @@ void DatabaseOrdinary::alterTable(
}
void DatabaseOrdinary::drop()
void DatabaseOrdinary::drop(const Context & context)
{
DatabaseOnDisk::drop(*this);
DatabaseOnDisk::drop(*this, context);
}

2
dbms/src/Databases/DatabaseOrdinary.h
View File

@ -83,7 +83,7 @@ public:
String getMetadataPath() const override;
String getObjectMetadataPath(const String & table_name) const override;
void drop() override;
void drop(const Context & context) override;
private:
const String metadata_path;

9
dbms/src/Databases/DatabasesCommon.cpp
View File

@ -1,22 +1,13 @@
#include <Databases/DatabasesCommon.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/ExternalLoaderDatabaseConfigRepository.h>
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/ParserDictionary.h>
#include <Parsers/formatAST.h>
#include <Parsers/parseQuery.h>
#include <Storages/IStorage.h>
#include <Storages/StorageDictionary.h>
#include <Storages/StorageFactory.h>
#include <Common/typeid_cast.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Dictionaries/DictionaryFactory.h>
#include <sstream>
namespace DB

2
dbms/src/Databases/IDatabase.h
View File

@ -237,7 +237,7 @@ public:
virtual void shutdown() = 0;
/// Delete data and metadata stored inside the database, if exists.
virtual void drop() {}
virtual void drop(const Context & /*context*/) {}
virtual ~IDatabase() {}
};

62
dbms/src/Formats/FormatFactory.cpp
View File

@ -12,6 +12,8 @@
#include <Processors/Formats/OutputStreamToOutputFormat.h>
#include <DataStreams/SquashingBlockOutputStream.h>
#include <DataStreams/NativeBlockInputStream.h>
#include <Processors/Formats/Impl/ValuesBlockInputFormat.h>
#include <Processors/Formats/Impl/MySQLOutputFormat.h>
namespace DB
@ -34,7 +36,7 @@ const FormatFactory::Creators & FormatFactory::getCreators(const String & name)
}
static FormatSettings getInputFormatSetting(const Settings & settings)
static FormatSettings getInputFormatSetting(const Settings & settings, const Context & context)
{
FormatSettings format_settings;
format_settings.csv.delimiter = settings.format_csv_delimiter;
@ -56,11 +58,21 @@ static FormatSettings getInputFormatSetting(const Settings & settings)
format_settings.template_settings.row_format = settings.format_template_row;
format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default;
format_settings.schema.format_schema = settings.format_schema;
format_settings.schema.format_schema_path = context.getFormatSchemaPath();
format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);
format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter;
format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter;
format_settings.custom.escaping_rule = settings.format_custom_escaping_rule;
format_settings.custom.field_delimiter = settings.format_custom_field_delimiter;
format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter;
format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter;
format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter;
return format_settings;
}
static FormatSettings getOutputFormatSetting(const Settings & settings)
static FormatSettings getOutputFormatSetting(const Settings & settings, const Context & context)
{
FormatSettings format_settings;
format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers;
@ -77,6 +89,16 @@ static FormatSettings getOutputFormatSetting(const Settings & settings)
format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
format_settings.write_statistics = settings.output_format_write_statistics;
format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size;
format_settings.schema.format_schema = settings.format_schema;
format_settings.schema.format_schema_path = context.getFormatSchemaPath();
format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);
format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter;
format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter;
format_settings.custom.escaping_rule = settings.format_custom_escaping_rule;
format_settings.custom.field_delimiter = settings.format_custom_field_delimiter;
format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter;
format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter;
format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter;
return format_settings;
}
@ -100,9 +122,9 @@ BlockInputStreamPtr FormatFactory::getInput(
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
const Settings & settings = context.getSettingsRef();
FormatSettings format_settings = getInputFormatSetting(settings);
FormatSettings format_settings = getInputFormatSetting(settings, context);
return input_getter(buf, sample, context, max_block_size, callback ? callback : ReadCallback(), format_settings);
return input_getter(buf, sample, max_block_size, callback ? callback : ReadCallback(), format_settings);
}
const Settings & settings = context.getSettingsRef();
@ -118,7 +140,7 @@ BlockInputStreamPtr FormatFactory::getInput(
if (!input_getter)
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
FormatSettings format_settings = getInputFormatSetting(settings);
FormatSettings format_settings = getInputFormatSetting(settings, context);
RowInputFormatParams row_input_format_params;
row_input_format_params.max_block_size = max_block_size;
@ -128,7 +150,7 @@ BlockInputStreamPtr FormatFactory::getInput(
row_input_format_params.max_execution_time = settings.max_execution_time;
row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode;
auto input_creator_params = ParallelParsingBlockInputStream::InputCreatorParams{sample, context, row_input_format_params, format_settings};
auto input_creator_params = ParallelParsingBlockInputStream::InputCreatorParams{sample, row_input_format_params, format_settings};
ParallelParsingBlockInputStream::Params params{buf, input_getter,
input_creator_params, file_segmentation_engine,
static_cast<int>(settings.max_threads),
@ -164,16 +186,16 @@ BlockOutputStreamPtr FormatFactory::getOutput(
throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
const Settings & settings = context.getSettingsRef();
FormatSettings format_settings = getOutputFormatSetting(settings);
FormatSettings format_settings = getOutputFormatSetting(settings, context);
/** Materialization is needed, because formats can use the functions `IDataType`,
* which only work with full columns.
*/
return std::make_shared<MaterializingBlockOutputStream>(
output_getter(buf, sample, context, callback, format_settings), sample);
output_getter(buf, sample, std::move(callback), format_settings), sample);
}
auto format = getOutputFormat(name, buf, sample, context, callback);
auto format = getOutputFormat(name, buf, sample, context, std::move(callback));
return std::make_shared<MaterializingBlockOutputStream>(std::make_shared<OutputStreamToOutputFormat>(format), sample);
}
@ -191,7 +213,7 @@ InputFormatPtr FormatFactory::getInputFormat(
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
const Settings & settings = context.getSettingsRef();
FormatSettings format_settings = getInputFormatSetting(settings);
FormatSettings format_settings = getInputFormatSetting(settings, context);
RowInputFormatParams params;
params.max_block_size = max_block_size;
@ -201,7 +223,13 @@ InputFormatPtr FormatFactory::getInputFormat(
params.max_execution_time = settings.max_execution_time;
params.timeout_overflow_mode = settings.timeout_overflow_mode;
return input_getter(buf, sample, context, params, format_settings);
auto format = input_getter(buf, sample, params, format_settings);
/// It's a kludge. Because I cannot remove context from values format.
if (auto * values = typeid_cast<ValuesBlockInputFormat *>(format.get()))
values->setContext(context);
return format;
}
@ -213,12 +241,18 @@ OutputFormatPtr FormatFactory::getOutputFormat(
throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
const Settings & settings = context.getSettingsRef();
FormatSettings format_settings = getOutputFormatSetting(settings);
FormatSettings format_settings = getOutputFormatSetting(settings, context);
/** TODO: Materialization is needed, because formats can use the functions `IDataType`,
* which only work with full columns.
*/
return output_getter(buf, sample, context, callback, format_settings);
auto format = output_getter(buf, sample, std::move(callback), format_settings);
/// It's a kludge. Because I cannot remove context from MySQL format.
if (auto * mysql = typeid_cast<MySQLOutputFormat *>(format.get()))
mysql->setContext(context);
return format;
}
@ -259,7 +293,7 @@ void FormatFactory::registerFileSegmentationEngine(const String & name, FileSegm
auto & target = dict[name].file_segmentation_engine;
if (target)
throw Exception("FormatFactory: File segmentation engine " + name + " is already registered", ErrorCodes::LOGICAL_ERROR);
target = file_segmentation_engine;
target = std::move(file_segmentation_engine);
}
FormatFactory::FormatFactory()

4
dbms/src/Formats/FormatFactory.h
View File

@ -59,7 +59,6 @@ private:
using InputCreator = std::function<BlockInputStreamPtr(
ReadBuffer & buf,
const Block & sample,
const Context & context,
UInt64 max_block_size,
ReadCallback callback,
const FormatSettings & settings)>;
@ -67,21 +66,18 @@ private:
using OutputCreator = std::function<BlockOutputStreamPtr(
WriteBuffer & buf,
const Block & sample,
const Context & context,
WriteCallback callback,
const FormatSettings & settings)>;
using InputProcessorCreator = std::function<InputFormatPtr(
ReadBuffer & buf,
const Block & header,
const Context & context,
const RowInputFormatParams & params,
const FormatSettings & settings)>;
using OutputProcessorCreator = std::function<OutputFormatPtr(
WriteBuffer & buf,
const Block & sample,
const Context & context,
WriteCallback callback,
const FormatSettings & settings)>;

14
dbms/src/Formats/FormatSchemaInfo.cpp
View File

@ -26,7 +26,7 @@ namespace
}
FormatSchemaInfo::FormatSchemaInfo(const Context & context, const String & format_schema, const String & format, bool require_message)
FormatSchemaInfo::FormatSchemaInfo(const String & format_schema, const String & format, bool require_message, bool is_server, const std::string & format_schema_path)
{
if (format_schema.empty())
throw Exception(
@ -54,29 +54,25 @@ FormatSchemaInfo::FormatSchemaInfo(const Context & context, const String & forma
else
path.assign(format_schema).makeFile().getFileName();
auto default_schema_directory = [&context]()
auto default_schema_directory = [&format_schema_path]()
{
static const String str = Poco::Path(context.getFormatSchemaPath()).makeAbsolute().makeDirectory().toString();
static const String str = Poco::Path(format_schema_path).makeAbsolute().makeDirectory().toString();
return str;
};
auto is_server = [&context]()
{
return context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);
};
if (path.getExtension().empty() && !default_file_extension.empty())
path.setExtension(default_file_extension);
if (path.isAbsolute())
{
if (is_server())
if (is_server)
throw Exception("Absolute path in the 'format_schema' setting is prohibited: " + path.toString(), ErrorCodes::BAD_ARGUMENTS);
schema_path = path.getFileName();
schema_directory = path.makeParent().toString();
}
else if (path.depth() >= 1 && path.directory(0) == "..")
{
if (is_server())
if (is_server)
throw Exception(
"Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: " + path.toString(),
ErrorCodes::BAD_ARGUMENTS);

2
dbms/src/Formats/FormatSchemaInfo.h
View File

@ -10,7 +10,7 @@ class Context;
class FormatSchemaInfo
{
public:
FormatSchemaInfo(const Context & context, const String & format_schema, const String & format, bool require_message);
FormatSchemaInfo(const String & format_schema, const String & format, bool require_message, bool is_server, const std::string & format_schema_path);
/// Returns path to the schema file.
const String & schemaPath() const { return schema_path; }

21
dbms/src/Formats/FormatSettings.h
View File

@ -89,6 +89,27 @@ struct FormatSettings
UInt64 row_group_size = 1000000;
} parquet;
struct Schema
{
std::string format_schema;
std::string format_schema_path;
bool is_server = false;
};
Schema schema;
struct Custom
{
std::string result_before_delimiter;
std::string result_after_delimiter;
std::string row_before_delimiter;
std::string row_after_delimiter;
std::string row_between_delimiter;
std::string field_delimiter;
std::string escaping_rule;
};
Custom custom;
};
}

2
dbms/src/Formats/NativeFormat.cpp
View File

@ -11,7 +11,6 @@ void registerInputFormatNative(FormatFactory & factory)
factory.registerInputFormat("Native", [](
ReadBuffer & buf,
const Block & sample,
const Context &,
UInt64 /* max_block_size */,
FormatFactory::ReadCallback /* callback */,
const FormatSettings &)
@ -25,7 +24,6 @@ void registerOutputFormatNative(FormatFactory & factory)
factory.registerOutputFormat("Native", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings &)
{

1
dbms/src/Formats/NullFormat.cpp
View File

@ -10,7 +10,6 @@ void registerOutputFormatNull(FormatFactory & factory)
factory.registerOutputFormat("Null", [](
WriteBuffer &,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings &)
{

18
dbms/src/Formats/ParsedTemplateFormatString.cpp
View File

@ -234,36 +234,32 @@ void ParsedTemplateFormatString::throwInvalidFormat(const String & message, size
ErrorCodes::INVALID_TEMPLATE_FORMAT);
}
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(const Context & context)
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(const FormatSettings::Custom & settings)
{
const Settings & settings = context.getSettingsRef();
/// Set resultset format to "result_before_delimiter ${data} result_after_delimiter"
ParsedTemplateFormatString resultset_format;
resultset_format.delimiters.emplace_back(settings.format_custom_result_before_delimiter);
resultset_format.delimiters.emplace_back(settings.format_custom_result_after_delimiter);
resultset_format.delimiters.emplace_back(settings.result_before_delimiter);
resultset_format.delimiters.emplace_back(settings.result_after_delimiter);
resultset_format.formats.emplace_back(ParsedTemplateFormatString::ColumnFormat::None);
resultset_format.format_idx_to_column_idx.emplace_back(0);
resultset_format.column_names.emplace_back("data");
return resultset_format;
}
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedRowFormat(const Context & context, const Block & sample)
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedRowFormat(const FormatSettings::Custom & settings, const Block & sample)
{
const Settings & settings = context.getSettingsRef();
/// Set row format to
/// "row_before_delimiter ${Col0:escaping} field_delimiter ${Col1:escaping} field_delimiter ... ${ColN:escaping} row_after_delimiter"
ParsedTemplateFormatString::ColumnFormat escaping = ParsedTemplateFormatString::stringToFormat(settings.format_custom_escaping_rule);
ParsedTemplateFormatString::ColumnFormat escaping = ParsedTemplateFormatString::stringToFormat(settings.escaping_rule);
ParsedTemplateFormatString row_format;
row_format.delimiters.emplace_back(settings.format_custom_row_before_delimiter);
row_format.delimiters.emplace_back(settings.row_before_delimiter);
for (size_t i = 0; i < sample.columns(); ++i)
{
row_format.formats.emplace_back(escaping);
row_format.format_idx_to_column_idx.emplace_back(i);
row_format.column_names.emplace_back(sample.getByPosition(i).name);
bool last_column = i == sample.columns() - 1;
row_format.delimiters.emplace_back(last_column ? settings.format_custom_row_after_delimiter : settings.format_custom_field_delimiter);
row_format.delimiters.emplace_back(last_column ? settings.row_after_delimiter : settings.field_delimiter);
}
return row_format;
}

5
dbms/src/Formats/ParsedTemplateFormatString.h
View File

@ -4,6 +4,7 @@
#include <functional>
#include <optional>
#include <Formats/FormatSchemaInfo.h>
#include <Formats/FormatSettings.h>
namespace DB
{
@ -49,8 +50,8 @@ struct ParsedTemplateFormatString
String dump() const;
[[noreturn]] void throwInvalidFormat(const String & message, size_t column) const;
static ParsedTemplateFormatString setupCustomSeparatedResultsetFormat(const Context & context);
static ParsedTemplateFormatString setupCustomSeparatedRowFormat(const Context & context, const Block & sample);
static ParsedTemplateFormatString setupCustomSeparatedResultsetFormat(const FormatSettings::Custom & settings);
static ParsedTemplateFormatString setupCustomSeparatedRowFormat(const FormatSettings::Custom & settings, const Block & sample);
};
}

10
dbms/src/Functions/FunctionBase64Conversion.h
View File

@ -166,4 +166,14 @@ private:
}
};
}
/** We must call it in advance from a single thread
* to avoid thread sanitizer report about data race in "codec_choose" function.
*/
inline void initializeBase64()
{
size_t outlen = 0;
base64_encode(nullptr, 0, nullptr, &outlen, 0);
}
#endif

23
dbms/src/Functions/FunctionsConversion.cpp
View File

@ -5,29 +5,6 @@
namespace DB
{
void throwExceptionForIncompletelyParsedValue(
ReadBuffer & read_buffer, Block & block, size_t result)
{
const IDataType & to_type = *block.getByPosition(result).type;
WriteBufferFromOwnString message_buf;
message_buf << "Cannot parse string " << quote << String(read_buffer.buffer().begin(), read_buffer.buffer().size())
<< " as " << to_type.getName()
<< ": syntax error";
if (read_buffer.offset())
message_buf << " at position " << read_buffer.offset()
<< " (parsed just " << quote << String(read_buffer.buffer().begin(), read_buffer.offset()) << ")";
else
message_buf << " at begin of string";
if (isNativeNumber(to_type))
message_buf << ". Note: there are to" << to_type.getName() << "OrZero and to" << to_type.getName() << "OrNull functions, which returns zero/NULL instead of throwing exception.";
throw Exception(message_buf.str(), ErrorCodes::CANNOT_PARSE_TEXT);
}
void registerFunctionsConversion(FunctionFactory & factory)
{
factory.registerFunction<FunctionToUInt8>();

48
dbms/src/Functions/FunctionsConversion.h
View File

@ -501,7 +501,26 @@ inline bool tryParseImpl<DataTypeDateTime>(DataTypeDateTime::FieldType & x, Read
/** Throw exception with verbose message when string value is not parsed completely.
*/
[[noreturn]] void throwExceptionForIncompletelyParsedValue(ReadBuffer & read_buffer, Block & block, size_t result);
[[noreturn]] inline void throwExceptionForIncompletelyParsedValue(ReadBuffer & read_buffer, Block & block, size_t result)
{
const IDataType & to_type = *block.getByPosition(result).type;
WriteBufferFromOwnString message_buf;
message_buf << "Cannot parse string " << quote << String(read_buffer.buffer().begin(), read_buffer.buffer().size())
<< " as " << to_type.getName()
<< ": syntax error";
if (read_buffer.offset())
message_buf << " at position " << read_buffer.offset()
<< " (parsed just " << quote << String(read_buffer.buffer().begin(), read_buffer.offset()) << ")";
else
message_buf << " at begin of string";
if (isNativeNumber(to_type))
message_buf << ". Note: there are to" << to_type.getName() << "OrZero and to" << to_type.getName() << "OrNull functions, which returns zero/NULL instead of throwing exception.";
throw Exception(message_buf.str(), ErrorCodes::CANNOT_PARSE_TEXT);
}
enum class ConvertFromStringExceptionMode
@ -886,6 +905,7 @@ public:
static constexpr bool to_datetime64 = std::is_same_v<ToDataType, DataTypeDateTime64>;
static FunctionPtr create(const Context &) { return std::make_shared<FunctionConvert>(); }
static FunctionPtr create() { return std::make_shared<FunctionConvert>(); }
String getName() const override
{
@ -1083,6 +1103,7 @@ public:
std::is_same_v<ToDataType, DataTypeDecimal<Decimal128>>;
static FunctionPtr create(const Context &) { return std::make_shared<FunctionConvertFromString>(); }
static FunctionPtr create() { return std::make_shared<FunctionConvertFromString>(); }
String getName() const override
{
@ -1231,6 +1252,7 @@ class FunctionToFixedString : public IFunction
public:
static constexpr auto name = "toFixedString";
static FunctionPtr create(const Context &) { return std::make_shared<FunctionToFixedString>(); }
static FunctionPtr create() { return std::make_shared<FunctionToFixedString>(); }
String getName() const override
{
@ -1686,9 +1708,9 @@ public:
using WrapperType = std::function<void(Block &, const ColumnNumbers &, size_t, size_t)>;
using MonotonicityForRange = std::function<Monotonicity(const IDataType &, const Field &, const Field &)>;
FunctionCast(const Context & context_, const char * name_, MonotonicityForRange && monotonicity_for_range_
FunctionCast(const char * name_, MonotonicityForRange && monotonicity_for_range_
, const DataTypes & argument_types_, const DataTypePtr & return_type_)
: context(context_), name(name_), monotonicity_for_range(monotonicity_for_range_)
: name(name_), monotonicity_for_range(monotonicity_for_range_)
, argument_types(argument_types_), return_type(return_type_)
{
}
@ -1719,7 +1741,6 @@ public:
private:
const Context & context;
const char * name;
MonotonicityForRange monotonicity_for_range;
@ -1735,10 +1756,10 @@ private:
{
/// In case when converting to Nullable type, we apply different parsing rule,
/// that will not throw an exception but return NULL in case of malformed input.
function = FunctionConvertFromString<DataType, NameCast, ConvertFromStringExceptionMode::Null>::create(context);
function = FunctionConvertFromString<DataType, NameCast, ConvertFromStringExceptionMode::Null>::create();
}
else
function = FunctionTo<DataType>::Type::create(context);
function = FunctionTo<DataType>::Type::create();
auto function_adaptor =
FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
@ -1752,7 +1773,7 @@ private:
WrapperType createStringWrapper(const DataTypePtr & from_type) const
{
FunctionPtr function = FunctionToString::create(context);
FunctionPtr function = FunctionToString::create();
auto function_adaptor =
FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
@ -1780,7 +1801,7 @@ private:
if (requested_result_is_nullable)
throw Exception{"CAST AS Nullable(UUID) is not implemented", ErrorCodes::NOT_IMPLEMENTED};
FunctionPtr function = FunctionTo<DataTypeUUID>::Type::create(context);
FunctionPtr function = FunctionTo<DataTypeUUID>::Type::create();
auto function_adaptor =
FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
@ -1985,7 +2006,7 @@ private:
return createStringToEnumWrapper<ColumnFixedString, EnumType>();
else if (isNativeNumber(from_type) || isEnum(from_type))
{
auto function = Function::create(context);
auto function = Function::create();
auto func_or_adaptor = FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
.build(ColumnsWithTypeAndName{{nullptr, from_type, "" }});
@ -2337,9 +2358,10 @@ public:
using MonotonicityForRange = FunctionCast::MonotonicityForRange;
static constexpr auto name = "CAST";
static FunctionOverloadResolverImplPtr create(const Context & context) { return std::make_unique<CastOverloadResolver>(context); }
static FunctionOverloadResolverImplPtr create(const Context &) { return createImpl(); }
static FunctionOverloadResolverImplPtr createImpl() { return std::make_unique<CastOverloadResolver>(); }
CastOverloadResolver(const Context & context_) : context(context_) {}
CastOverloadResolver() {}
String getName() const override { return name; }
@ -2357,7 +2379,7 @@ protected:
data_types[i] = arguments[i].type;
auto monotonicity = getMonotonicityInformation(arguments.front().type, return_type.get());
return std::make_unique<FunctionCast>(context, name, std::move(monotonicity), data_types, return_type);
return std::make_unique<FunctionCast>(name, std::move(monotonicity), data_types, return_type);
}
DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments) const override
@ -2418,8 +2440,6 @@ private:
/// other types like Null, FixedString, Array and Tuple have no monotonicity defined
return {};
}
const Context & context;
};
}

1
dbms/src/Functions/base64Decode.cpp
View File

@ -8,6 +8,7 @@ namespace DB
{
void registerFunctionBase64Decode(FunctionFactory & factory)
{
initializeBase64();
factory.registerFunction<FunctionBase64Conversion<Base64Decode>>();
}
}

1
dbms/src/Functions/base64Encode.cpp
View File

@ -9,6 +9,7 @@ namespace DB
{
void registerFunctionBase64Encode(FunctionFactory & factory)
{
initializeBase64();
factory.registerFunction<FunctionBase64Conversion<Base64Encode>>();
}
}

2
dbms/src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -632,7 +632,7 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create,
else
{
res = StorageFactory::instance().get(create,
data_path,
data_path + escapeForFileName(table_name) + "/",
table_name,
create.database,
context,

4
dbms/src/Interpreters/InterpreterDropQuery.cpp
View File

@ -137,7 +137,7 @@ BlockIO InterpreterDropQuery::executeToTable(
/// If it is not virtual database like Dictionary then drop remaining data dir
if (!database_data_path.empty())
{
String table_data_path = database_data_path + "/" + escapeForFileName(database_and_table.second->getTableName());
String table_data_path = context.getPath() + database_data_path + "/" + escapeForFileName(table_name);
if (Poco::File(table_data_path).exists())
Poco::File(table_data_path).remove(true);
@ -269,7 +269,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(String & database_name, ASTDropQ
database->shutdown();
/// Delete the database.
database->drop();
database->drop(context);
/// Old ClickHouse versions did not store database.sql files
Poco::File database_metadata_file(context.getPath() + "metadata/" + escapeForFileName(database_name) + ".sql");

3
dbms/src/Interpreters/InterpreterSystemQuery.cpp
View File

@ -5,6 +5,7 @@
#include <Common/typeid_cast.h>
#include <Common/getNumberOfPhysicalCPUCores.h>
#include <Common/ThreadPool.h>
#include <Common/escapeForFileName.h>
#include <Interpreters/Context.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/EmbeddedDictionaries.h>
@ -294,7 +295,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_nam
auto constraints = InterpreterCreateQuery::getConstraintsDescription(create.columns_list->constraints);
StoragePtr table = StorageFactory::instance().get(create,
data_path,
data_path + escapeForFileName(table_name) + "/",
table_name,
database_name,
system_context,

2
dbms/src/Interpreters/MutationsInterpreter.cpp
View File

@ -339,6 +339,8 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
affected_materialized.emplace(mat_column);
}
/// Just to be sure, that we don't change type
/// after update expression execution.
const auto & update_expr = kv.second;
auto updated_column = makeASTFunction("CAST",
makeASTFunction("if",

7
dbms/src/Interpreters/MutationsInterpreter.h
View File

@ -65,8 +65,9 @@ private:
/// Each stage has output_columns that contain columns that are changed at the end of that stage
/// plus columns needed for the next mutations.
///
/// First stage is special: it can contain only DELETEs and is executed using InterpreterSelectQuery
/// to take advantage of table indexes (if there are any).
/// First stage is special: it can contain only filters and is executed using InterpreterSelectQuery
/// to take advantage of table indexes (if there are any). It's necessary because all mutations have
/// `WHERE clause` part.
struct Stage
{
@ -83,7 +84,7 @@ private:
/// A chain of actions needed to execute this stage.
/// First steps calculate filter columns for DELETEs (in the same order as in `filter_column_names`),
/// then there is (possibly) an UPDATE stage, and finally a projection stage.
/// then there is (possibly) an UPDATE step, and finally a projection step.
ExpressionActionsChain expressions_chain;
Names filter_column_names;
};

12
dbms/src/Interpreters/castColumn.cpp
View File

@ -2,13 +2,14 @@
#include <Interpreters/castColumn.h>
#include <Interpreters/ExpressionActions.h>
#include <DataTypes/DataTypeString.h>
#include <Functions/FunctionFactory.h>
#include <Functions/IFunctionAdaptors.h>
#include <Functions/FunctionsConversion.h>
namespace DB
{
ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type, const Context & context)
ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type)
{
if (arg.type->equals(*type))
return arg.column;
@ -28,7 +29,7 @@ ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type
}
};
FunctionOverloadResolverPtr func_builder_cast = FunctionFactory::instance().get("CAST", context);
FunctionOverloadResolverPtr func_builder_cast = std::make_shared<FunctionOverloadResolverAdaptor>(CastOverloadResolver::createImpl());
ColumnsWithTypeAndName arguments{ temporary_block.getByPosition(0), temporary_block.getByPosition(1) };
auto func_cast = func_builder_cast->build(arguments);
@ -37,4 +38,9 @@ ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type
return temporary_block.getByPosition(2).column;
}
ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type, const Context &)
{
return castColumn(arg, type);
}
}

2
dbms/src/Interpreters/castColumn.h
View File

@ -6,7 +6,7 @@
namespace DB
{
ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type);
ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type, const Context & context);
}

15
dbms/src/Processors/Executors/PipelineExecutor.cpp
View File

@ -11,6 +11,10 @@
#include <Processors/ISource.h>
#include <Common/setThreadName.h>
#if !defined(__APPLE__) && !defined(__FreeBSD__)
#include <sched.h>
#endif
namespace DB
{
@ -452,6 +456,17 @@ void PipelineExecutor::execute(size_t num_threads)
void PipelineExecutor::executeSingleThread(size_t thread_num, size_t num_threads)
{
#if !defined(__APPLE__) && !defined(__FreeBSD__)
/// Specify CPU core for thread if can.
/// It may reduce the number of context swithches.
cpu_set_t cpu_set;
CPU_ZERO(&cpu_set);
CPU_SET(thread_num, &cpu_set);
if (sched_setaffinity(0, sizeof(cpu_set_t), &cpu_set) == -1)
LOG_TRACE(log, "Cannot set affinity for thread " << num_threads);
#endif
UInt64 total_time_ns = 0;
UInt64 execution_time_ns = 0;
UInt64 processing_time_ns = 0;

4
dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp
View File

@ -248,7 +248,7 @@ namespace DB
void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk &res, std::shared_ptr<arrow::Table> &table,
arrow::Status &read_status, const Block &header,
int &row_group_current, const Context &context, std::string format_name)
int &row_group_current, std::string format_name)
{
Columns columns_list;
UInt64 num_rows = 0;
@ -389,7 +389,7 @@ namespace DB
else
column.column = std::move(read_column);
column.column = castColumn(column, column_type, context);
column.column = castColumn(column, column_type);
column.type = column_type;
num_rows = column.column->size();
columns_list.push_back(std::move(column.column));

2
dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h
View File

@ -39,7 +39,7 @@ namespace DB
static void arrowTableToCHChunk(Chunk &res, std::shared_ptr<arrow::Table> &table,
arrow::Status &read_status, const Block &header,
int &row_group_current, const Context &context, std::string format_name);
int &row_group_current, std::string format_name);
};
}
#endif

2
dbms/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp
View File

@ -61,7 +61,6 @@ void registerInputFormatProcessorRowBinary(FormatFactory & factory)
factory.registerInputFormatProcessor("RowBinary", [](
ReadBuffer & buf,
const Block & sample,
const Context &,
const IRowInputFormat::Params & params,
const FormatSettings &)
{
@ -71,7 +70,6 @@ void registerInputFormatProcessorRowBinary(FormatFactory & factory)
factory.registerInputFormatProcessor("RowBinaryWithNamesAndTypes", [](
ReadBuffer & buf,
const Block & sample,
const Context &,
const IRowInputFormat::Params & params,
const FormatSettings &)
{

2
dbms/src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp
View File

@ -52,7 +52,6 @@ void registerOutputFormatProcessorRowBinary(FormatFactory & factory)
factory.registerOutputFormatProcessor("RowBinary", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings &)
{
@ -62,7 +61,6 @@ void registerOutputFormatProcessorRowBinary(FormatFactory & factory)
factory.registerOutputFormatProcessor("RowBinaryWithNamesAndTypes", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings &)
{

1
dbms/src/Processors/Formats/Impl/CSVRowInputFormat.cpp
View File

@ -421,7 +421,6 @@ void registerInputFormatProcessorCSV(FormatFactory & factory)
factory.registerInputFormatProcessor(with_names ? "CSVWithNames" : "CSV", [=](
ReadBuffer & buf,
const Block & sample,
const Context &,
IRowInputFormat::Params params,
const FormatSettings & settings)
{

1
dbms/src/Processors/Formats/Impl/CSVRowOutputFormat.cpp
View File

@ -76,7 +76,6 @@ void registerOutputFormatProcessorCSV(FormatFactory & factory)
factory.registerOutputFormatProcessor(with_names ? "CSVWithNames" : "CSV", [=](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & format_settings)
{

5
dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp
View File

@ -301,10 +301,11 @@ void registerInputFormatProcessorCapnProto(FormatFactory & factory)
{
factory.registerInputFormatProcessor(
"CapnProto",
[](ReadBuffer & buf, const Block & sample, const Context & context, IRowInputFormat::Params params, const FormatSettings &)
[](ReadBuffer & buf, const Block & sample, IRowInputFormat::Params params, const FormatSettings & settings)
{
return std::make_shared<CapnProtoRowInputFormat>(buf, sample, std::move(params),
FormatSchemaInfo(context, context.getSettingsRef().format_schema, "CapnProto", true));
FormatSchemaInfo(settings.schema.format_schema, "CapnProto", true,
settings.schema.is_server, settings.schema.format_schema_path));
});
}

2
dbms/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp
View File

@ -217,7 +217,6 @@ void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
factory.registerInputFormatProcessor("JSONCompactEachRow", [](
ReadBuffer & buf,
const Block & sample,
const Context &,
IRowInputFormat::Params params,
const FormatSettings & settings)
{
@ -227,7 +226,6 @@ void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
factory.registerInputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", [](
ReadBuffer & buf,
const Block & sample,
const Context &,
IRowInputFormat::Params params,
const FormatSettings & settings)
{

2
dbms/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp
View File

@ -94,7 +94,6 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
factory.registerOutputFormatProcessor("JSONCompactEachRow", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & format_settings)
{
@ -104,7 +103,6 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
factory.registerOutputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", [](
WriteBuffer &buf,
const Block &sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings &format_settings)
{

1
dbms/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.cpp
View File

@ -80,7 +80,6 @@ void registerOutputFormatProcessorJSONCompact(FormatFactory & factory)
factory.registerOutputFormatProcessor("JSONCompact", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & format_settings)
{

1
dbms/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
View File

@ -271,7 +271,6 @@ void registerInputFormatProcessorJSONEachRow(FormatFactory & factory)
factory.registerInputFormatProcessor("JSONEachRow", [](
ReadBuffer & buf,
const Block & sample,
const Context &,
IRowInputFormat::Params params,
const FormatSettings & settings)
{

1
dbms/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp
View File

@ -56,7 +56,6 @@ void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory)
factory.registerOutputFormatProcessor("JSONEachRow", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & format_settings)
{

1
dbms/src/Processors/Formats/Impl/JSONEachRowWithProgressRowOutputFormat.cpp
View File

@ -33,7 +33,6 @@ void registerOutputFormatProcessorJSONEachRowWithProgress(FormatFactory & factor
factory.registerOutputFormatProcessor("JSONEachRowWithProgress", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & format_settings)
{

1
dbms/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp
View File

@ -246,7 +246,6 @@ void registerOutputFormatProcessorJSON(FormatFactory & factory)
factory.registerOutputFormatProcessor("JSON", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & format_settings)
{

33
dbms/src/Processors/Formats/Impl/MySQLOutputFormat.cpp
View File

@ -12,13 +12,10 @@ namespace DB
using namespace MySQLProtocol;
MySQLOutputFormat::MySQLOutputFormat(WriteBuffer & out_, const Block & header_, const Context & context_, const FormatSettings & settings_)
MySQLOutputFormat::MySQLOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & settings_)
: IOutputFormat(header_, out_)
, context(context_)
, packet_sender(out, const_cast<uint8_t &>(context_.mysql.sequence_id)) /// TODO: fix it
, format_settings(settings_)
{
packet_sender.max_packet_size = context_.mysql.max_packet_size;
}
void MySQLOutputFormat::initialize()
@ -32,17 +29,17 @@ void MySQLOutputFormat::initialize()
if (header.columns())
{
packet_sender.sendPacket(LengthEncodedNumber(header.columns()));
packet_sender->sendPacket(LengthEncodedNumber(header.columns()));
for (size_t i = 0; i < header.columns(); i++)
{
const auto & column_name = header.getColumnsWithTypeAndName()[i].name;
packet_sender.sendPacket(getColumnDefinition(column_name, data_types[i]->getTypeId()));
packet_sender->sendPacket(getColumnDefinition(column_name, data_types[i]->getTypeId()));
}
if (!(context.mysql.client_capabilities & Capability::CLIENT_DEPRECATE_EOF))
if (!(context->mysql.client_capabilities & Capability::CLIENT_DEPRECATE_EOF))
{
packet_sender.sendPacket(EOF_Packet(0, 0));
packet_sender->sendPacket(EOF_Packet(0, 0));
}
}
}
@ -50,10 +47,13 @@ void MySQLOutputFormat::initialize()
void MySQLOutputFormat::consume(Chunk chunk)
{
initialize();
for (size_t i = 0; i < chunk.getNumRows(); i++)
{
ProtocolText::ResultsetRow row_packet(data_types, chunk.getColumns(), i);
packet_sender.sendPacket(row_packet);
packet_sender->sendPacket(row_packet);
}
}
@ -61,7 +61,7 @@ void MySQLOutputFormat::finalize()
{
size_t affected_rows = 0;
std::stringstream human_readable_info;
if (QueryStatus * process_list_elem = context.getProcessListElement())
if (QueryStatus * process_list_elem = context->getProcessListElement())
{
CurrentThread::finalizePerformanceCounters();
QueryStatusInfo info = process_list_elem->getInfo();
@ -74,17 +74,17 @@ void MySQLOutputFormat::finalize()
const auto & header = getPort(PortKind::Main).getHeader();
if (header.columns() == 0)
packet_sender.sendPacket(OK_Packet(0x0, context.mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info.str()), true);
packet_sender->sendPacket(OK_Packet(0x0, context->mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info.str()), true);
else
if (context.mysql.client_capabilities & CLIENT_DEPRECATE_EOF)
packet_sender.sendPacket(OK_Packet(0xfe, context.mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info.str()), true);
if (context->mysql.client_capabilities & CLIENT_DEPRECATE_EOF)
packet_sender->sendPacket(OK_Packet(0xfe, context->mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info.str()), true);
else
packet_sender.sendPacket(EOF_Packet(0, 0), true);
packet_sender->sendPacket(EOF_Packet(0, 0), true);
}
void MySQLOutputFormat::flush()
{
packet_sender.out->next();
packet_sender->out->next();
}
void registerOutputFormatProcessorMySQLWrite(FormatFactory & factory)
@ -93,9 +93,8 @@ void registerOutputFormatProcessorMySQLWrite(FormatFactory & factory)
"MySQLWire",
[](WriteBuffer & buf,
const Block & sample,
const Context & context,
FormatFactory::WriteCallback,
const FormatSettings & settings) { return std::make_shared<MySQLOutputFormat>(buf, sample, context, settings); });
const FormatSettings & settings) { return std::make_shared<MySQLOutputFormat>(buf, sample, settings); });
}
}

15
dbms/src/Processors/Formats/Impl/MySQLOutputFormat.h
View File

@ -16,13 +16,20 @@ class Context;
/** A stream for outputting data in a binary line-by-line format.
*/
class MySQLOutputFormat: public IOutputFormat
class MySQLOutputFormat final : public IOutputFormat
{
public:
MySQLOutputFormat(WriteBuffer & out_, const Block & header_, const Context & context_, const FormatSettings & settings_);
MySQLOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & settings_);
String getName() const override { return "MySQLOutputFormat"; }
void setContext(const Context & context_)
{
context = &context_;
packet_sender = std::make_unique<MySQLProtocol::PacketSender>(out, const_cast<uint8_t &>(context_.mysql.sequence_id)); /// TODO: fix it
packet_sender->max_packet_size = context_.mysql.max_packet_size;
}
void consume(Chunk) override;
void finalize() override;
void flush() override;
@ -34,8 +41,8 @@ private:
bool initialized = false;
const Context & context;
MySQLProtocol::PacketSender packet_sender;
const Context * context = nullptr;
std::unique_ptr<MySQLProtocol::PacketSender> packet_sender;
FormatSettings format_settings;
DataTypes data_types;
};

2
dbms/src/Processors/Formats/Impl/NativeFormat.cpp
View File

@ -156,7 +156,6 @@ void registerInputFormatProcessorNative(FormatFactory & factory)
factory.registerInputFormatProcessor("Native", [](
ReadBuffer & buf,
const Block & sample,
const Context &,
const RowInputFormatParams &,
const FormatSettings &)
{
@ -169,7 +168,6 @@ void registerOutputFormatProcessorNative(FormatFactory & factory)
factory.registerOutputFormatProcessor("Native", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings &)
{

1
dbms/src/Processors/Formats/Impl/NullFormat.cpp
View File

@ -21,7 +21,6 @@ void registerOutputFormatProcessorNull(FormatFactory & factory)
factory.registerOutputFormatProcessor("Null", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings &)
{

2
dbms/src/Processors/Formats/Impl/ODBCDriver2BlockOutputFormat.cpp
View File

@ -107,7 +107,7 @@ void ODBCDriver2BlockOutputFormat::writePrefix()
void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory)
{
factory.registerOutputFormatProcessor(
"ODBCDriver2", [](WriteBuffer & buf, const Block & sample, const Context &, FormatFactory::WriteCallback, const FormatSettings & format_settings)
"ODBCDriver2", [](WriteBuffer & buf, const Block & sample, FormatFactory::WriteCallback, const FormatSettings & format_settings)
{
return std::make_shared<ODBCDriver2BlockOutputFormat>(buf, sample, format_settings);
});

1
dbms/src/Processors/Formats/Impl/ODBCDriverBlockOutputFormat.cpp
View File

@ -69,7 +69,6 @@ void registerOutputFormatProcessorODBCDriver(FormatFactory & factory)
factory.registerOutputFormatProcessor("ODBCDriver", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings & format_settings)
{

9
dbms/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp
View File

@ -12,8 +12,8 @@
namespace DB
{
ORCBlockInputFormat::ORCBlockInputFormat(ReadBuffer &in_, Block header_, const Context &context_)
: IInputFormat(std::move(header_), in_), context{context_} {
ORCBlockInputFormat::ORCBlockInputFormat(ReadBuffer &in_, Block header_) : IInputFormat(std::move(header_), in_)
{
}
Chunk ORCBlockInputFormat::generate()
@ -57,7 +57,7 @@ namespace DB
arrow::Status read_status = file_reader->Read(&table);
ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, context, "ORC");
ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, "ORC");
return res;
}
@ -78,11 +78,10 @@ namespace DB
"ORC",
[](ReadBuffer &buf,
const Block &sample,
const Context &context,
const RowInputFormatParams &,
const FormatSettings & /* settings */)
{
return std::make_shared<ORCBlockInputFormat>(buf, sample, context);
return std::make_shared<ORCBlockInputFormat>(buf, sample);
});
}

4
dbms/src/Processors/Formats/Impl/ORCBlockInputFormat.h
View File

@ -17,7 +17,7 @@ class Context;
class ORCBlockInputFormat: public IInputFormat
{
public:
ORCBlockInputFormat(ReadBuffer & in_, Block header_, const Context & context_);
ORCBlockInputFormat(ReadBuffer & in_, Block header_);
String getName() const override { return "ORCBlockInputFormat"; }
@ -30,8 +30,6 @@ private:
// TODO: check that this class implements every part of its parent
const Context & context;
std::unique_ptr<arrow::adapters::orc::ORCFileReader> file_reader;
std::string file_data;
int row_group_total = 0;

10
dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp
View File

@ -15,8 +15,9 @@
namespace DB
{
ParquetBlockInputFormat::ParquetBlockInputFormat(ReadBuffer &in_, Block header_, const Context &context_)
: IInputFormat(std::move(header_), in_), context{context_} {
ParquetBlockInputFormat::ParquetBlockInputFormat(ReadBuffer & in_, Block header_)
: IInputFormat(std::move(header_), in_)
{
}
Chunk ParquetBlockInputFormat::generate()
@ -59,7 +60,7 @@ namespace DB
std::shared_ptr<arrow::Table> table;
arrow::Status read_status = file_reader->ReadRowGroup(row_group_current, &table);
ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, context, "Parquet");
ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, "Parquet");
return res;
}
@ -80,11 +81,10 @@ namespace DB
"Parquet",
[](ReadBuffer &buf,
const Block &sample,
const Context &context,
const RowInputFormatParams &,
const FormatSettings & /* settings */)
{
return std::make_shared<ParquetBlockInputFormat>(buf, sample, context);
return std::make_shared<ParquetBlockInputFormat>(buf, sample);
});
}

4
dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.h
View File

@ -16,7 +16,7 @@ class Context;
class ParquetBlockInputFormat: public IInputFormat
{
public:
ParquetBlockInputFormat(ReadBuffer & in_, Block header_, const Context & context_);
ParquetBlockInputFormat(ReadBuffer & in_, Block header_);
void resetParser() override;
@ -30,8 +30,6 @@ private:
// TODO: check that this class implements every part of its parent
const Context & context;
std::unique_ptr<parquet::arrow::FileReader> file_reader;
std::string file_data;
std::unique_ptr<arrow::Buffer> buffer;

1
dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp
View File

@ -426,7 +426,6 @@ void registerOutputFormatProcessorParquet(FormatFactory & factory)
"Parquet",
[](WriteBuffer & buf,
const Block & sample,
const Context & /*context*/,
FormatFactory::WriteCallback,
const FormatSettings & format_settings)
{

2
dbms/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp
View File

@ -260,7 +260,6 @@ void registerOutputFormatProcessorPretty(FormatFactory & factory)
factory.registerOutputFormatProcessor("Pretty", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings & format_settings)
{
@ -270,7 +269,6 @@ void registerOutputFormatProcessorPretty(FormatFactory & factory)
factory.registerOutputFormatProcessor("PrettyNoEscapes", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings & format_settings)
{

3
dbms/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp
View File

@ -134,7 +134,6 @@ void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory)
factory.registerOutputFormatProcessor("PrettyCompact", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings & format_settings)
{
@ -144,7 +143,6 @@ void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory)
factory.registerOutputFormatProcessor("PrettyCompactNoEscapes", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings & format_settings)
{
@ -157,7 +155,6 @@ void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory)
// factory.registerOutputFormat("PrettyCompactMonoBlock", [](
// WriteBuffer & buf,
// const Block & sample,
// const Context &,
// const FormatSettings & format_settings)
// {
// BlockOutputStreamPtr impl = std::make_shared<PrettyCompactBlockOutputFormat>(buf, sample, format_settings);

2
dbms/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp
View File

@ -97,7 +97,6 @@ void registerOutputFormatProcessorPrettySpace(FormatFactory & factory)
factory.registerOutputFormatProcessor("PrettySpace", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings & format_settings)
{
@ -107,7 +106,6 @@ void registerOutputFormatProcessorPrettySpace(FormatFactory & factory)
factory.registerOutputFormatProcessor("PrettySpaceNoEscapes", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings & format_settings)
{

6
dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp
View File

@ -70,12 +70,12 @@ void registerInputFormatProcessorProtobuf(FormatFactory & factory)
factory.registerInputFormatProcessor("Protobuf", [](
ReadBuffer & buf,
const Block & sample,
const Context & context,
IRowInputFormat::Params params,
const FormatSettings &)
const FormatSettings & settings)
{
return std::make_shared<ProtobufRowInputFormat>(buf, sample, std::move(params),
FormatSchemaInfo(context, context.getSettingsRef().format_schema, "Protobuf", true));
FormatSchemaInfo(settings.schema.format_schema, "Protobuf", true,
settings.schema.is_server, settings.schema.format_schema_path));
});
}

8
dbms/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp
View File

@ -50,12 +50,12 @@ void registerOutputFormatProcessorProtobuf(FormatFactory & factory)
"Protobuf",
[](WriteBuffer & buf,
const Block & header,
const Context & context,
FormatFactory::WriteCallback callback,
const FormatSettings &)
const FormatSettings & settings)
{
return std::make_shared<ProtobufRowOutputFormat>(buf, header, callback,
FormatSchemaInfo(context, context.getSettingsRef().format_schema, "Protobuf", true));
return std::make_shared<ProtobufRowOutputFormat>(buf, header, std::move(callback),
FormatSchemaInfo(settings.schema.format_schema, "Protobuf", true,
settings.schema.is_server, settings.schema.format_schema_path));
});
}

1
dbms/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp
View File

@ -210,7 +210,6 @@ void registerInputFormatProcessorTSKV(FormatFactory & factory)
factory.registerInputFormatProcessor("TSKV", [](
ReadBuffer & buf,
const Block & sample,
const Context &,
IRowInputFormat::Params params,
const FormatSettings & settings)
{

1
dbms/src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp
View File

@ -45,7 +45,6 @@ void registerOutputFormatProcessorTSKV(FormatFactory & factory)
factory.registerOutputFormatProcessor("TSKV", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & settings)
{

3
dbms/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp
View File

@ -356,7 +356,6 @@ void registerInputFormatProcessorTabSeparated(FormatFactory & factory)
factory.registerInputFormatProcessor(name, [](
ReadBuffer & buf,
const Block & sample,
const Context &,
IRowInputFormat::Params params,
const FormatSettings & settings)
{
@ -369,7 +368,6 @@ void registerInputFormatProcessorTabSeparated(FormatFactory & factory)
factory.registerInputFormatProcessor(name, [](
ReadBuffer & buf,
const Block & sample,
const Context &,
IRowInputFormat::Params params,
const FormatSettings & settings)
{
@ -382,7 +380,6 @@ void registerInputFormatProcessorTabSeparated(FormatFactory & factory)
factory.registerInputFormatProcessor(name, [](
ReadBuffer & buf,
const Block & sample,
const Context &,
IRowInputFormat::Params params,
const FormatSettings & settings)
{

4
dbms/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp
View File

@ -78,7 +78,6 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory)
factory.registerOutputFormatProcessor(name, [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & settings)
{
@ -91,7 +90,6 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory)
factory.registerOutputFormatProcessor(name, [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & settings)
{
@ -104,7 +102,6 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory)
factory.registerOutputFormatProcessor(name, [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & settings)
{
@ -117,7 +114,6 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory)
factory.registerOutputFormatProcessor(name, [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & settings)
{

26
dbms/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp
View File

@ -14,8 +14,10 @@ namespace ErrorCodes
}
TemplateBlockOutputFormat::TemplateBlockOutputFormat(const Block & header_, WriteBuffer & out_, const FormatSettings & settings_,
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_)
: IOutputFormat(header_, out_), settings(settings_), format(std::move(format_)), row_format(std::move(row_format_))
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_,
std::string row_between_delimiter_)
: IOutputFormat(header_, out_), settings(settings_), format(std::move(format_))
, row_format(std::move(row_format_)), row_between_delimiter(std::move(row_between_delimiter_))
{
auto & sample = getPort(PortKind::Main).getHeader();
size_t columns = sample.columns();
@ -152,7 +154,7 @@ void TemplateBlockOutputFormat::consume(Chunk chunk)
for (size_t i = 0; i < rows; ++i)
{
if (row_count)
writeString(settings.template_settings.row_between_delimiter, out);
writeString(row_between_delimiter, out);
writeRow(chunk, i);
++row_count;
@ -230,7 +232,6 @@ void registerOutputFormatProcessorTemplate(FormatFactory & factory)
factory.registerOutputFormatProcessor("Template", [](
WriteBuffer & buf,
const Block & sample,
const Context & context,
FormatFactory::WriteCallback,
const FormatSettings & settings)
{
@ -247,7 +248,8 @@ void registerOutputFormatProcessorTemplate(FormatFactory & factory)
{
/// Read format string from file
resultset_format = ParsedTemplateFormatString(
FormatSchemaInfo(context, settings.template_settings.resultset_format, "Template", false),
FormatSchemaInfo(settings.template_settings.resultset_format, "Template", false,
settings.schema.is_server, settings.schema.format_schema_path),
[&](const String & partName)
{
return static_cast<size_t>(TemplateBlockOutputFormat::stringToResultsetPart(partName));
@ -255,28 +257,26 @@ void registerOutputFormatProcessorTemplate(FormatFactory & factory)
}
ParsedTemplateFormatString row_format = ParsedTemplateFormatString(
FormatSchemaInfo(context, settings.template_settings.row_format, "Template", false),
FormatSchemaInfo(settings.template_settings.row_format, "Template", false,
settings.schema.is_server, settings.schema.format_schema_path),
[&](const String & colName)
{
return sample.getPositionByName(colName);
});
return std::make_shared<TemplateBlockOutputFormat>(sample, buf, settings, resultset_format, row_format);
return std::make_shared<TemplateBlockOutputFormat>(sample, buf, settings, resultset_format, row_format, settings.template_settings.row_between_delimiter);
});
factory.registerOutputFormatProcessor("CustomSeparated", [](
WriteBuffer & buf,
const Block & sample,
const Context & context,
FormatFactory::WriteCallback,
const FormatSettings & settings)
{
ParsedTemplateFormatString resultset_format = ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(context);
ParsedTemplateFormatString row_format = ParsedTemplateFormatString::setupCustomSeparatedRowFormat(context, sample);
FormatSettings format_settings = settings;
format_settings.template_settings.row_between_delimiter = context.getSettingsRef().format_custom_row_between_delimiter;
ParsedTemplateFormatString resultset_format = ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(settings.custom);
ParsedTemplateFormatString row_format = ParsedTemplateFormatString::setupCustomSeparatedRowFormat(settings.custom, sample);
return std::make_shared<TemplateBlockOutputFormat>(sample, buf, format_settings, resultset_format, row_format);
return std::make_shared<TemplateBlockOutputFormat>(sample, buf, settings, resultset_format, row_format, settings.custom.row_between_delimiter);
});
}
}

5
dbms/src/Processors/Formats/Impl/TemplateBlockOutputFormat.h
View File

@ -15,7 +15,8 @@ class TemplateBlockOutputFormat : public IOutputFormat
using ColumnFormat = ParsedTemplateFormatString::ColumnFormat;
public:
TemplateBlockOutputFormat(const Block & header_, WriteBuffer & out_, const FormatSettings & settings_,
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_);
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_,
std::string row_between_delimiter_);
String getName() const override { return "TemplateBlockOutputFormat"; }
@ -65,6 +66,8 @@ protected:
size_t row_count = 0;
bool need_write_prefix = true;
std::string row_between_delimiter;
};
}

39
dbms/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp
View File

@ -20,12 +20,13 @@ extern const int SYNTAX_ERROR;
TemplateRowInputFormat::TemplateRowInputFormat(const Block & header_, ReadBuffer & in_, const Params & params_,
const FormatSettings & settings_, bool ignore_spaces_,
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_)
FormatSettings settings_, bool ignore_spaces_,
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_,
std::string row_between_delimiter_)
: RowInputFormatWithDiagnosticInfo(header_, buf, params_), buf(in_), data_types(header_.getDataTypes()),
settings(settings_), ignore_spaces(ignore_spaces_),
settings(std::move(settings_)), ignore_spaces(ignore_spaces_),
format(std::move(format_)), row_format(std::move(row_format_)),
default_csv_delimiter(settings.csv.delimiter)
default_csv_delimiter(settings.csv.delimiter), row_between_delimiter(std::move(row_between_delimiter_))
{
/// Validate format string for result set
bool has_data = false;
@ -160,7 +161,7 @@ bool TemplateRowInputFormat::readRow(MutableColumns & columns, RowReadExtension
updateDiagnosticInfo();
if (likely(row_num != 1))
assertString(settings.template_settings.row_between_delimiter, buf);
assertString(row_between_delimiter, buf);
extra.read_columns.assign(columns.size(), false);
@ -339,11 +340,11 @@ bool TemplateRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & col
try
{
if (likely(row_num != 1))
assertString(settings.template_settings.row_between_delimiter, buf);
assertString(row_between_delimiter, buf);
}
catch (const DB::Exception &)
{
writeErrorStringForWrongDelimiter(out, "delimiter between rows", settings.template_settings.row_between_delimiter);
writeErrorStringForWrongDelimiter(out, "delimiter between rows", row_between_delimiter);
return false;
}
@ -428,7 +429,7 @@ bool TemplateRowInputFormat::isGarbageAfterField(size_t, ReadBuffer::Position)
bool TemplateRowInputFormat::allowSyncAfterError() const
{
return !row_format.delimiters.back().empty() || !settings.template_settings.row_between_delimiter.empty();
return !row_format.delimiters.back().empty() || !row_between_delimiter.empty();
}
void TemplateRowInputFormat::syncAfterError()
@ -450,10 +451,10 @@ void TemplateRowInputFormat::syncAfterError()
bool last_delimiter_in_row_found = !row_format.delimiters.back().empty();
if (last_delimiter_in_row_found && checkString(settings.template_settings.row_between_delimiter, buf))
if (last_delimiter_in_row_found && checkString(row_between_delimiter, buf))
at_beginning_of_row_or_eof = true;
else
skipToNextDelimiterOrEof(settings.template_settings.row_between_delimiter);
skipToNextDelimiterOrEof(row_between_delimiter);
if (buf.eof())
at_beginning_of_row_or_eof = end_of_stream = true;
@ -509,7 +510,6 @@ void registerInputFormatProcessorTemplate(FormatFactory & factory)
factory.registerInputFormatProcessor(ignore_spaces ? "TemplateIgnoreSpaces" : "Template", [=](
ReadBuffer & buf,
const Block & sample,
const Context & context,
IRowInputFormat::Params params,
const FormatSettings & settings)
{
@ -526,7 +526,8 @@ void registerInputFormatProcessorTemplate(FormatFactory & factory)
{
/// Read format string from file
resultset_format = ParsedTemplateFormatString(
FormatSchemaInfo(context, settings.template_settings.resultset_format, "Template", false),
FormatSchemaInfo(settings.template_settings.resultset_format, "Template", false,
settings.schema.is_server, settings.schema.format_schema_path),
[&](const String & partName) -> std::optional<size_t>
{
if (partName == "data")
@ -537,13 +538,14 @@ void registerInputFormatProcessorTemplate(FormatFactory & factory)
}
ParsedTemplateFormatString row_format = ParsedTemplateFormatString(
FormatSchemaInfo(context, settings.template_settings.row_format, "Template", false),
FormatSchemaInfo(settings.template_settings.row_format, "Template", false,
settings.schema.is_server, settings.schema.format_schema_path),
[&](const String & colName) -> std::optional<size_t>
{
return sample.getPositionByName(colName);
});
return std::make_shared<TemplateRowInputFormat>(sample, buf, params, settings, ignore_spaces, resultset_format, row_format);
return std::make_shared<TemplateRowInputFormat>(sample, buf, params, settings, ignore_spaces, resultset_format, row_format, settings.template_settings.row_between_delimiter);
});
}
@ -552,16 +554,13 @@ void registerInputFormatProcessorTemplate(FormatFactory & factory)
factory.registerInputFormatProcessor(ignore_spaces ? "CustomSeparatedIgnoreSpaces" : "CustomSeparated", [=](
ReadBuffer & buf,
const Block & sample,
const Context & context,
IRowInputFormat::Params params,
const FormatSettings & settings)
{
ParsedTemplateFormatString resultset_format = ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(context);
ParsedTemplateFormatString row_format = ParsedTemplateFormatString::setupCustomSeparatedRowFormat(context, sample);
FormatSettings format_settings = settings;
format_settings.template_settings.row_between_delimiter = context.getSettingsRef().format_custom_row_between_delimiter;
ParsedTemplateFormatString resultset_format = ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(settings.custom);
ParsedTemplateFormatString row_format = ParsedTemplateFormatString::setupCustomSeparatedRowFormat(settings.custom, sample);
return std::make_shared<TemplateRowInputFormat>(sample, buf, params, format_settings, ignore_spaces, resultset_format, row_format);
return std::make_shared<TemplateRowInputFormat>(sample, buf, params, settings, ignore_spaces, resultset_format, row_format, settings.custom.row_between_delimiter);
});
}
}

7
dbms/src/Processors/Formats/Impl/TemplateRowInputFormat.h
View File

@ -16,8 +16,9 @@ class TemplateRowInputFormat : public RowInputFormatWithDiagnosticInfo
using ColumnFormat = ParsedTemplateFormatString::ColumnFormat;
public:
TemplateRowInputFormat(const Block & header_, ReadBuffer & in_, const Params & params_,
const FormatSettings & settings_, bool ignore_spaces_,
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_);
FormatSettings settings_, bool ignore_spaces_,
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_,
std::string row_between_delimiter);
String getName() const override { return "TemplateRowInputFormat"; }
@ -61,6 +62,8 @@ private:
bool end_of_stream = false;
std::vector<size_t> always_default_columns;
char default_csv_delimiter;
std::string row_between_delimiter;
};
}

7
dbms/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp
View File

@ -33,8 +33,8 @@ namespace ErrorCodes
ValuesBlockInputFormat::ValuesBlockInputFormat(ReadBuffer & in_, const Block & header_, const RowInputFormatParams & params_,
const Context & context_, const FormatSettings & format_settings_)
: IInputFormat(header_, buf), buf(in_), params(params_), context(std::make_unique<Context>(context_)),
const FormatSettings & format_settings_)
: IInputFormat(header_, buf), buf(in_), params(params_),
format_settings(format_settings_), num_columns(header_.columns()),
parser_type_for_column(num_columns, ParserType::Streaming),
attempts_to_deduce_template(num_columns), attempts_to_deduce_template_cached(num_columns),
@ -424,11 +424,10 @@ void registerInputFormatProcessorValues(FormatFactory & factory)
factory.registerInputFormatProcessor("Values", [](
ReadBuffer & buf,
const Block & header,
const Context & context,
const RowInputFormatParams & params,
const FormatSettings & settings)
{
return std::make_shared<ValuesBlockInputFormat>(buf, header, params, context, settings);
return std::make_shared<ValuesBlockInputFormat>(buf, header, params, settings);
});
}

7
dbms/src/Processors/Formats/Impl/ValuesBlockInputFormat.h
View File

@ -18,7 +18,7 @@ class ReadBuffer;
/** Stream to read data in VALUES format (as in INSERT query).
*/
class ValuesBlockInputFormat : public IInputFormat
class ValuesBlockInputFormat final : public IInputFormat
{
public:
/** Data is parsed using fast, streaming parser.
@ -29,12 +29,15 @@ public:
* than interpreting expressions in each row separately, but it's still slower than streaming parsing)
*/
ValuesBlockInputFormat(ReadBuffer & in_, const Block & header_, const RowInputFormatParams & params_,
const Context & context_, const FormatSettings & format_settings_);
const FormatSettings & format_settings_);
String getName() const override { return "ValuesBlockInputFormat"; }
void resetParser() override;
/// TODO: remove context somehow.
void setContext(const Context & context_) { context = std::make_unique<Context>(context_); }
const BlockMissingValues & getMissingValues() const override { return block_missing_values; }
private:

1
dbms/src/Processors/Formats/Impl/ValuesRowOutputFormat.cpp
View File

@ -46,7 +46,6 @@ void registerOutputFormatProcessorValues(FormatFactory & factory)
factory.registerOutputFormatProcessor("Values", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & settings)
{

1
dbms/src/Processors/Formats/Impl/VerticalRowOutputFormat.cpp
View File

@ -168,7 +168,6 @@ void registerOutputFormatProcessorVertical(FormatFactory & factory)
factory.registerOutputFormatProcessor("Vertical", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & settings)
{

1
dbms/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp
View File

@ -245,7 +245,6 @@ void registerOutputFormatProcessorXML(FormatFactory & factory)
factory.registerOutputFormatProcessor("XML", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback callback,
const FormatSettings & settings)
{

43
dbms/src/Storages/AlterCommands.cpp
View File

@ -109,7 +109,7 @@ std::optional<AlterCommand> AlterCommand::parse(const ASTAlterCommand * command_
if (ast_col_decl.comment)
{
const auto & ast_comment = ast_col_decl.comment->as<ASTLiteral &>();
command.comment = ast_comment.value.get<String>();
command.comment.emplace(ast_comment.value.get<String>());
}
if (ast_col_decl.ttl)
@ -225,7 +225,9 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri
column.default_desc.kind = default_kind;
column.default_desc.expression = default_expression;
}
column.comment = comment;
if (comment)
column.comment = *comment;
column.codec = codec;
column.ttl = ttl;
@ -251,19 +253,22 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri
column.codec = codec;
}
if (!isMutable())
{
column.comment = comment;
return;
}
if (comment)
column.comment = *comment;
if (ttl)
column.ttl = ttl;
column.type = data_type;
if (data_type)
column.type = data_type;
column.default_desc.kind = default_kind;
column.default_desc.expression = default_expression;
/// User specified default expression or changed
/// datatype. We have to replace default.
if (default_expression || data_type)
{
column.default_desc.kind = default_kind;
column.default_desc.expression = default_expression;
}
});
}
else if (type == MODIFY_ORDER_BY)
@ -279,7 +284,7 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri
}
else if (type == COMMENT_COLUMN)
{
columns_description.modify(column_name, [&](ColumnDescription & column) { column.comment = comment; });
columns_description.modify(column_name, [&](ColumnDescription & column) { column.comment = *comment; });
}
else if (type == ADD_INDEX)
{
@ -390,13 +395,15 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri
throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR);
}
bool AlterCommand::isMutable() const
bool AlterCommand::isModifyingData() const
{
if (type == COMMENT_COLUMN || type == MODIFY_SETTING)
return false;
/// Possible change data representation on disk
if (type == MODIFY_COLUMN)
return data_type.get() || default_expression;
return true;
return data_type != nullptr;
return type == ADD_COLUMN /// We need to change columns.txt in each part for MergeTree
|| type == DROP_COLUMN /// We need to change columns.txt in each part for MergeTree
|| type == DROP_INDEX; /// We need to remove file from filesystem for MergeTree
}
bool AlterCommand::isSettingsAlter() const
@ -666,11 +673,11 @@ void AlterCommands::applyForSettingsOnly(SettingsChanges & changes) const
changes = std::move(out_changes);
}
bool AlterCommands::isMutable() const
bool AlterCommands::isModifyingData() const
{
for (const auto & param : *this)
{
if (param.isMutable())
if (param.isModifyingData())
return true;
}

11
dbms/src/Storages/AlterCommands.h
View File

@ -47,7 +47,7 @@ struct AlterCommand
ColumnDefaultKind default_kind{};
ASTPtr default_expression{};
String comment;
std::optional<String> comment;
/// For ADD - after which column to add a new one. If an empty string, add to the end. To add to the beginning now it is impossible.
String after_column;
@ -102,8 +102,11 @@ struct AlterCommand
ConstraintsDescription & constraints_description, ASTPtr & order_by_ast,
ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast, SettingsChanges & changes) const;
/// Checks that not only metadata touched by that command
bool isMutable() const;
/// Checks that alter query changes data. For MergeTree:
/// * column files (data and marks)
/// * each part meta (columns.txt)
/// in each part on disk (it's not lightweight alter).
bool isModifyingData() const;
/// checks that only settings changed by alter
bool isSettingsAlter() const;
@ -124,7 +127,7 @@ public:
void applyForSettingsOnly(SettingsChanges & changes) const;
void validate(const IStorage & table, const Context & context);
bool isMutable() const;
bool isModifyingData() const;
bool isSettingsAlter() const;
};

2
dbms/src/Storages/IStorage.cpp
View File

@ -402,7 +402,7 @@ void IStorage::alter(
const Context & context,
TableStructureWriteLockHolder & table_lock_holder)
{
if (params.isMutable())
if (params.isModifyingData())
throw Exception("Method alter supports only change comment of column for storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
const String database_name = getDatabaseName();

2
dbms/src/Storages/IStorage.h
View File

@ -304,7 +304,7 @@ public:
* In this function, you need to rename the directory with the data, if any.
* Called when the table structure is locked for write.
*/
virtual void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/,
virtual void rename(const String & /*new_path_to_table_data*/, const String & /*new_database_name*/, const String & /*new_table_name*/,
TableStructureWriteLockHolder &)
{
throw Exception("Method rename is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);

111
dbms/src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -112,6 +112,7 @@ namespace
MergeTreeData::MergeTreeData(
const String & database_,
const String & table_,
const String & relative_data_path_,
const ColumnsDescription & columns_,
const IndicesDescription & indices_,
const ConstraintsDescription & constraints_,
@ -134,6 +135,7 @@ MergeTreeData::MergeTreeData(
, require_part_metadata(require_part_metadata_)
, database_name(database_)
, table_name(table_)
, relative_data_path(relative_data_path_)
, broken_part_callback(broken_part_callback_)
, log_name(database_name + "." + table_name)
, log(&Logger::get(log_name))
@ -1213,13 +1215,9 @@ void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_re
}
void MergeTreeData::rename(
const String & /*new_path_to_db*/, const String & new_database_name,
const String & new_table_path, const String & new_database_name,
const String & new_table_name, TableStructureWriteLockHolder &)
{
auto old_table_path = "data/" + escapeForFileName(database_name) + '/' + escapeForFileName(table_name) + '/';
auto new_db_path = "data/" + escapeForFileName(new_database_name) + '/';
auto new_table_path = new_db_path + escapeForFileName(new_table_name) + '/';
auto disks = storage_policy->getDisks();
for (const auto & disk : disks)
@ -1230,15 +1228,16 @@ void MergeTreeData::rename(
for (const auto & disk : disks)
{
disk->createDirectory(new_db_path);
disk->moveFile(old_table_path, new_table_path);
auto new_table_path_parent = Poco::Path(new_table_path).makeParent().toString();
disk->createDirectory(new_table_path_parent);
disk->moveFile(relative_data_path, new_table_path);
}
global_context.dropCaches();
database_name = new_database_name;
relative_data_path = new_table_path;
table_name = new_table_name;
database_name = new_database_name;
}
void MergeTreeData::dropAllData()
@ -1342,11 +1341,11 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
"before using data skipping indices.", ErrorCodes::BAD_ARGUMENTS);
/// Set of columns that shouldn't be altered.
NameSet columns_alter_forbidden;
NameSet columns_alter_type_forbidden;
/// Primary key columns can be ALTERed only if they are used in the key as-is
/// (and not as a part of some expression) and if the ALTER only affects column metadata.
NameSet columns_alter_metadata_only;
NameSet columns_alter_type_metadata_only;
if (partition_key_expr)
{
@ -1354,13 +1353,13 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
/// TODO: in some cases (e.g. adding an Enum value) a partition key column can still be ALTERed.
/// We should allow it.
for (const String & col : partition_key_expr->getRequiredColumns())
columns_alter_forbidden.insert(col);
columns_alter_type_forbidden.insert(col);
}
for (const auto & index : skip_indices)
{
for (const String & col : index->expr->getRequiredColumns())
columns_alter_forbidden.insert(col);
columns_alter_type_forbidden.insert(col);
}
if (sorting_key_expr)
@ -1368,17 +1367,16 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
for (const ExpressionAction & action : sorting_key_expr->getActions())
{
auto action_columns = action.getNeededColumns();
columns_alter_forbidden.insert(action_columns.begin(), action_columns.end());
columns_alter_type_forbidden.insert(action_columns.begin(), action_columns.end());
}
for (const String & col : sorting_key_expr->getRequiredColumns())
columns_alter_metadata_only.insert(col);
columns_alter_type_metadata_only.insert(col);
/// We don't process sample_by_ast separately because it must be among the primary key columns
/// and we don't process primary_key_expr separately because it is a prefix of sorting_key_expr.
}
if (!merging_params.sign_column.empty())
columns_alter_forbidden.insert(merging_params.sign_column);
columns_alter_type_forbidden.insert(merging_params.sign_column);
std::map<String, const IDataType *> old_types;
for (const auto & column : getColumns().getAllPhysical())
@ -1386,34 +1384,26 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
for (const AlterCommand & command : commands)
{
if (!command.isMutable())
if (command.type == AlterCommand::MODIFY_ORDER_BY && !is_custom_partitioned)
{
continue;
}
if (columns_alter_forbidden.count(command.column_name))
throw Exception("Trying to ALTER key column " + command.column_name, ErrorCodes::ILLEGAL_COLUMN);
if (columns_alter_metadata_only.count(command.column_name))
{
if (command.type == AlterCommand::MODIFY_COLUMN)
{
auto it = old_types.find(command.column_name);
if (it != old_types.end() && isMetadataOnlyConversion(it->second, command.data_type.get()))
continue;
}
throw Exception(
"ALTER of key column " + command.column_name + " must be metadata-only",
ErrorCodes::ILLEGAL_COLUMN);
"ALTER MODIFY ORDER BY is not supported for default-partitioned tables created with the old syntax",
ErrorCodes::BAD_ARGUMENTS);
}
if (command.type == AlterCommand::MODIFY_ORDER_BY)
else if (command.isModifyingData())
{
if (!is_custom_partitioned)
throw Exception(
"ALTER MODIFY ORDER BY is not supported for default-partitioned tables created with the old syntax",
ErrorCodes::BAD_ARGUMENTS);
if (columns_alter_type_forbidden.count(command.column_name))
throw Exception("Trying to ALTER key column " + command.column_name, ErrorCodes::ILLEGAL_COLUMN);
if (columns_alter_type_metadata_only.count(command.column_name))
{
if (command.type == AlterCommand::MODIFY_COLUMN)
{
auto it = old_types.find(command.column_name);
if (it == old_types.end() || !isMetadataOnlyConversion(it->second, command.data_type.get()))
throw Exception("ALTER of key column " + command.column_name + " must be metadata-only", ErrorCodes::ILLEGAL_COLUMN);
}
}
}
}
@ -1425,17 +1415,20 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
for (const auto & setting : new_changes)
checkSettingCanBeChanged(setting.name);
/// Check that type conversions are possible.
ExpressionActionsPtr unused_expression;
NameToNameMap unused_map;
bool unused_bool;
createConvertExpression(nullptr, getColumns().getAllPhysical(), new_columns.getAllPhysical(),
getIndices().indices, new_indices.indices, unused_expression, unused_map, unused_bool);
if (commands.isModifyingData())
{
/// Check that type conversions are possible.
ExpressionActionsPtr unused_expression;
NameToNameMap unused_map;
bool unused_bool;
createConvertExpression(nullptr, getColumns().getAllPhysical(), new_columns.getAllPhysical(),
getIndices().indices, new_indices.indices, unused_expression, unused_map, unused_bool);
}
}
void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns,
const IndicesASTs & old_indices, const IndicesASTs & new_indices, ExpressionActionsPtr & out_expression,
NameToNameMap & out_rename_map, bool & out_force_update_metadata) const
void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns,
const NamesAndTypesList & new_columns, const IndicesASTs & old_indices, const IndicesASTs & new_indices,
ExpressionActionsPtr & out_expression, NameToNameMap & out_rename_map, bool & out_force_update_metadata) const
{
const auto settings = getSettings();
out_expression = nullptr;
@ -1457,7 +1450,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name
/// Remove old indices
std::set<String> new_indices_set;
std::unordered_set<String> new_indices_set;
for (const auto & index_decl : new_indices)
new_indices_set.emplace(index_decl->as<ASTIndexDeclaration &>().name);
for (const auto & index_decl : old_indices)
@ -1465,8 +1458,8 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name
const auto & index = index_decl->as<ASTIndexDeclaration &>();
if (!new_indices_set.count(index.name))
{
out_rename_map["skp_idx_" + index.name + ".idx"] = "";
out_rename_map["skp_idx_" + index.name + part_mrk_file_extension] = "";
out_rename_map["skp_idx_" + index.name + ".idx"] = ""; /// drop this file
out_rename_map["skp_idx_" + index.name + part_mrk_file_extension] = ""; /// and this one
}
}
@ -1494,8 +1487,8 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name
/// Delete files if they are no longer shared with another column.
if (--stream_counts[file_name] == 0)
{
out_rename_map[file_name + ".bin"] = "";
out_rename_map[file_name + part_mrk_file_extension] = "";
out_rename_map[file_name + ".bin"] = ""; /// drop this file
out_rename_map[file_name + part_mrk_file_extension] = ""; /// and this one
}
}, {});
}
@ -1847,7 +1840,7 @@ void MergeTreeData::AlterDataPartTransaction::commit()
mutable_part.checksums = new_checksums;
mutable_part.columns = new_columns;
/// 3) Delete the old files.
/// 3) Delete the old files and drop required columns (DROP COLUMN)
for (const auto & from_to : rename_map)
{
String name = from_to.second.empty() ? from_to.first : from_to.second;
@ -3457,7 +3450,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPartOnSameDisk(
String MergeTreeData::getFullPathOnDisk(const DiskPtr & disk) const
{
return disk->getPath() + "data/" + escapeForFileName(database_name) + '/' + escapeForFileName(table_name) + '/';
return disk->getPath() + relative_data_path;
}
@ -3530,9 +3523,7 @@ void MergeTreeData::freezePartitionsByMatcher(MatcherFn matcher, const String &
String part_absolute_path = Poco::Path(part->getFullPath()).absolute().toString();
String backup_part_absolute_path = backup_path
+ "data/"
+ escapeForFileName(getDatabaseName()) + "/"
+ escapeForFileName(getTableName()) + "/"
+ relative_data_path
+ part->relative_path;
localBackup(part_absolute_path, backup_part_absolute_path);
part->is_frozen.store(true, std::memory_order_relaxed);

8
dbms/src/Storages/MergeTree/MergeTreeData.h
View File

@ -234,7 +234,7 @@ public:
const NamesAndTypesList & getNewColumns() const { return new_columns; }
const DataPart::Checksums & getNewChecksums() const { return new_checksums; }
AlterDataPartTransaction(DataPartPtr data_part_) : data_part(data_part_), alter_lock(data_part->alter_mutex) {}
AlterDataPartTransaction(DataPartPtr data_part_) : data_part(data_part_) {}
const DataPartPtr & getDataPart() const { return data_part; }
bool isValid() const;
@ -244,9 +244,7 @@ public:
bool valid = true;
//don't interchange order of data_part & alter_lock
DataPartPtr data_part;
DataPartsLock alter_lock;
DataPart::Checksums new_checksums;
NamesAndTypesList new_columns;
@ -333,6 +331,7 @@ public:
/// require_part_metadata - should checksums.txt and columns.txt exist in the part directory.
/// attach - whether the existing table is attached or the new table is created.
MergeTreeData(const String & database_, const String & table_,
const String & relative_data_path_,
const ColumnsDescription & columns_,
const IndicesDescription & indices_,
const ConstraintsDescription & constraints_,
@ -538,7 +537,7 @@ public:
/// Moves the entire data directory.
/// Flushes the uncompressed blocks cache and the marks cache.
/// Must be called with locked lockStructureForAlter().
void rename(const String & new_path_to_db, const String & new_database_name,
void rename(const String & new_path_to_table_data, const String & new_database_name,
const String & new_table_name, TableStructureWriteLockHolder &) override;
/// Check if the ALTER can be performed:
@ -793,6 +792,7 @@ protected:
String database_name;
String table_name;
String relative_data_path;
/// Current column sizes in compressed and uncompressed form.

10
dbms/src/Storages/MergeTree/MergeTreeDataPart.h
View File

@ -227,16 +227,6 @@ struct MergeTreeDataPart
*/
mutable std::shared_mutex columns_lock;
/** It is taken for the whole time ALTER a part: from the beginning of the recording of the temporary files to their renaming to permanent.
* It is taken with unlocked `columns_lock`.
*
* NOTE: "You can" do without this mutex if you could turn ReadRWLock into WriteRWLock without removing the lock.
* This transformation is impossible, because it would create a deadlock, if you do it from two threads at once.
* Taking this mutex means that we want to lock columns_lock on read with intention then, not
* unblocking, block it for writing.
*/
mutable std::mutex alter_mutex;
MergeTreeIndexGranularityInfo index_granularity_info;
~MergeTreeDataPart();

4
dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp
View File

@ -639,14 +639,14 @@ static StoragePtr create(const StorageFactory::Arguments & args)
if (replicated)
return StorageReplicatedMergeTree::create(
zookeeper_path, replica_name, args.attach, args.database_name, args.table_name,
zookeeper_path, replica_name, args.attach, args.database_name, args.table_name, args.relative_data_path,
args.columns, indices_description, args.constraints,
args.context, date_column_name, partition_by_ast, order_by_ast, primary_key_ast,
sample_by_ast, ttl_table_ast, merging_params, std::move(storage_settings),
args.has_force_restore_data_flag);
else
return StorageMergeTree::create(
args.database_name, args.table_name, args.columns, indices_description,
args.database_name, args.table_name, args.relative_data_path, args.columns, indices_description,
args.constraints, args.attach, args.context, date_column_name, partition_by_ast, order_by_ast,
primary_key_ast, sample_by_ast, ttl_table_ast, merging_params, std::move(storage_settings),
args.has_force_restore_data_flag);

19
dbms/src/Storages/StorageDistributed.cpp
View File

@ -214,7 +214,7 @@ StorageDistributed::StorageDistributed(
const String & cluster_name_,
const Context & context_,
const ASTPtr & sharding_key_,
const String & data_path_,
const String & relative_data_path_,
bool attach_)
: IStorage(ColumnsDescription({
{"_shard_num", std::make_shared<DataTypeUInt32>()},
@ -222,7 +222,7 @@ StorageDistributed::StorageDistributed(
table_name(table_name_), database_name(database_name_),
remote_database(remote_database_), remote_table(remote_table_),
global_context(context_), cluster_name(global_context.getMacros()->expand(cluster_name_)), has_sharding_key(sharding_key_),
path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/'))
path(relative_data_path_.empty() ? "" : (context_.getPath() + relative_data_path_))
{
setColumns(columns_);
setConstraints(constraints_);
@ -252,9 +252,9 @@ StorageDistributed::StorageDistributed(
const String & cluster_name_,
const Context & context_,
const ASTPtr & sharding_key_,
const String & data_path_,
const String & relative_data_path_,
bool attach)
: StorageDistributed(database_name_, table_name_, columns_, constraints_, String{}, String{}, cluster_name_, context_, sharding_key_, data_path_, attach)
: StorageDistributed(database_name_, table_name_, columns_, constraints_, String{}, String{}, cluster_name_, context_, sharding_key_, relative_data_path_, attach)
{
remote_table_function_ptr = remote_table_function_ptr_;
}
@ -472,7 +472,7 @@ void StorageDistributed::createDirectoryMonitors()
if (path.empty())
return;
Poco::File{path}.createDirectory();
Poco::File{path}.createDirectories();
std::filesystem::directory_iterator begin(path);
std::filesystem::directory_iterator end;
@ -596,15 +596,16 @@ void StorageDistributed::flushClusterNodesAllData()
it->second.flushAllData();
}
void StorageDistributed::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name,
void StorageDistributed::rename(const String & new_path_to_table_data, const String & new_database_name, const String & new_table_name,
TableStructureWriteLockHolder &)
{
table_name = new_table_name;
database_name = new_database_name;
if (!path.empty())
{
Poco::File(path).renameTo(new_path_to_db + escapeForFileName(new_table_name));
path = new_path_to_db + escapeForFileName(new_table_name) + '/';
auto new_path = global_context.getPath() + new_path_to_table_data;
Poco::File(path).renameTo(new_path);
path = new_path;
std::lock_guard lock(cluster_nodes_mutex);
for (auto & node : cluster_nodes_data)
node.second.directory_monitor->updatePath();
@ -664,7 +665,7 @@ void registerStorageDistributed(StorageFactory & factory)
return StorageDistributed::create(
args.database_name, args.table_name, args.columns, args.constraints,
remote_database, remote_table, cluster_name,
args.context, sharding_key, args.data_path,
args.context, sharding_key, args.relative_data_path,
args.attach);
});
}

6
dbms/src/Storages/StorageDistributed.h
View File

@ -82,7 +82,7 @@ public:
/// Removes temporary data in local filesystem.
void truncate(const ASTPtr &, const Context &, TableStructureWriteLockHolder &) override;
void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
void rename(const String & new_path_to_table_data, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &) override;
/// in the sub-tables, you need to manually add and delete columns
/// the structure of the sub-table is not checked
@ -168,7 +168,7 @@ protected:
const String & cluster_name_,
const Context & context_,
const ASTPtr & sharding_key_,
const String & data_path_,
const String & relative_data_path_,
bool attach_);
StorageDistributed(
@ -180,7 +180,7 @@ protected:
const String & cluster_name_,
const Context & context_,
const ASTPtr & sharding_key_,
const String & data_path_,
const String & relative_data_path_,
bool attach);
ClusterPtr skipUnusedShards(ClusterPtr cluster, const SelectQueryInfo & query_info);

4
dbms/src/Storages/StorageFactory.cpp
View File

@ -40,7 +40,7 @@ void StorageFactory::registerStorage(const std::string & name, Creator creator)
StoragePtr StorageFactory::get(
const ASTCreateQuery & query,
const String & data_path,
const String & relative_data_path,
const String & table_name,
const String & database_name,
Context & local_context,
@ -150,7 +150,7 @@ StoragePtr StorageFactory::get(
.engine_args = args,
.storage_def = storage_def,
.query = query,
.data_path = data_path,
.relative_data_path = relative_data_path,
.table_name = table_name,
.database_name = database_name,
.local_context = local_context,

6
dbms/src/Storages/StorageFactory.h
View File

@ -33,7 +33,9 @@ public:
ASTs & engine_args;
ASTStorage * storage_def;
const ASTCreateQuery & query;
const String & data_path;
/// Path to table data.
/// Relative to <path> from server config (possibly <path> of some <disk> of some <volume> for *MergeTree)
const String & relative_data_path;
const String & table_name;
const String & database_name;
Context & local_context;
@ -48,7 +50,7 @@ public:
StoragePtr get(
const ASTCreateQuery & query,
const String & data_path,
const String & relative_data_path,
const String & table_name,
const String & database_name,
Context & local_context,

210
dbms/src/Storages/StorageFile.cpp
View File

@ -28,7 +28,6 @@
#include <Poco/File.h>
#include <re2/re2.h>
#include <re2/stringpiece.h>
#include <filesystem>
namespace fs = std::filesystem;
@ -54,7 +53,7 @@ namespace
/* Recursive directory listing with matched paths as a result.
* Have the same method in StorageHDFS.
*/
std::vector<std::string> listFilesWithRegexpMatching(const std::string & path_for_ls, const std::string & for_match)
static std::vector<std::string> listFilesWithRegexpMatching(const std::string & path_for_ls, const std::string & for_match)
{
const size_t first_glob = for_match.find_first_of("*?{");
@ -98,13 +97,13 @@ std::vector<std::string> listFilesWithRegexpMatching(const std::string & path_fo
return result;
}
std::string getTablePath(const std::string & db_dir_path, const std::string & table_name, const std::string & format_name)
static std::string getTablePath(const std::string & table_dir_path, const std::string & format_name)
{
return db_dir_path + escapeForFileName(table_name) + "/data." + escapeForFileName(format_name);
return table_dir_path + "/data." + escapeForFileName(format_name);
}
/// Both db_dir_path and table_path must be converted to absolute paths (in particular, path cannot contain '..').
void checkCreationIsAllowed(Context & context_global, const std::string & db_dir_path, const std::string & table_path)
static void checkCreationIsAllowed(const Context & context_global, const std::string & db_dir_path, const std::string & table_path)
{
if (context_global.getApplicationType() != Context::ApplicationType::SERVER)
return;
@ -118,70 +117,60 @@ void checkCreationIsAllowed(Context & context_global, const std::string & db_dir
}
}
StorageFile::StorageFile(
const std::string & table_path_,
int table_fd_,
const std::string & db_dir_path,
const std::string & database_name_,
const std::string & table_name_,
const std::string & format_name_,
const ColumnsDescription & columns_,
const ConstraintsDescription & constraints_,
Context & context_,
const String & compression_method_ = "")
:
table_name(table_name_), database_name(database_name_), format_name(format_name_), context_global(context_), table_fd(table_fd_), compression_method(compression_method_)
StorageFile::StorageFile(int table_fd_, CommonArguments args)
: StorageFile(args)
{
setColumns(columns_);
setConstraints(constraints_);
if (args.context.getApplicationType() == Context::ApplicationType::SERVER)
throw Exception("Using file descriptor as source of storage isn't allowed for server daemons", ErrorCodes::DATABASE_ACCESS_DENIED);
std::string db_dir_path_abs = Poco::Path(db_dir_path).makeAbsolute().makeDirectory().toString();
is_db_table = false;
use_table_fd = true;
table_fd = table_fd_;
if (table_fd < 0) /// Will use file
{
use_table_fd = false;
if (!table_path_.empty()) /// Is user's file
{
Poco::Path poco_path = Poco::Path(table_path_);
if (poco_path.isRelative())
poco_path = Poco::Path(db_dir_path_abs, poco_path);
const std::string path = poco_path.absolute().toString();
if (path.find_first_of("*?{") == std::string::npos)
{
paths.push_back(path);
}
else
paths = listFilesWithRegexpMatching("/", path);
for (const auto & cur_path : paths)
checkCreationIsAllowed(context_global, db_dir_path_abs, cur_path);
is_db_table = false;
}
else /// Is DB's file
{
if (db_dir_path_abs.empty())
throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME);
paths = {getTablePath(db_dir_path_abs, table_name, format_name)};
is_db_table = true;
Poco::File(Poco::Path(paths.back()).parent()).createDirectories();
}
}
else /// Will use FD
{
if (context_global.getApplicationType() == Context::ApplicationType::SERVER)
throw Exception("Using file descriptor as source of storage isn't allowed for server daemons", ErrorCodes::DATABASE_ACCESS_DENIED);
is_db_table = false;
use_table_fd = true;
/// Save initial offset, it will be used for repeating SELECTs
/// If FD isn't seekable (lseek returns -1), then the second and subsequent SELECTs will fail.
table_fd_init_offset = lseek(table_fd, 0, SEEK_CUR);
}
/// Save initial offset, it will be used for repeating SELECTs
/// If FD isn't seekable (lseek returns -1), then the second and subsequent SELECTs will fail.
table_fd_init_offset = lseek(table_fd, 0, SEEK_CUR);
}
StorageFile::StorageFile(const std::string & table_path_, const std::string & user_files_path, CommonArguments args)
: StorageFile(args)
{
is_db_table = false;
std::string user_files_absolute_path = Poco::Path(user_files_path).makeAbsolute().makeDirectory().toString();
Poco::Path poco_path = Poco::Path(table_path_);
if (poco_path.isRelative())
poco_path = Poco::Path(user_files_absolute_path, poco_path);
const std::string path = poco_path.absolute().toString();
if (path.find_first_of("*?{") == std::string::npos)
{
paths.push_back(path);
}
else
paths = listFilesWithRegexpMatching("/", path);
for (const auto & cur_path : paths)
checkCreationIsAllowed(args.context, user_files_absolute_path, cur_path);
}
StorageFile::StorageFile(const std::string & relative_table_dir_path, CommonArguments args)
: StorageFile(args)
{
if (relative_table_dir_path.empty())
throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME);
String table_dir_path = base_path + relative_table_dir_path + "/";
Poco::File(table_dir_path).createDirectories();
paths = {getTablePath(table_dir_path, format_name)};
}
StorageFile::StorageFile(CommonArguments args)
: table_name(args.table_name), database_name(args.database_name), format_name(args.format_name)
, compression_method(args.compression_method), base_path(args.context.getPath())
{
setColumns(args.columns);
setConstraints(args.constraints);
}
class StorageFileBlockInputStream : public IBlockInputStream
{
@ -287,7 +276,8 @@ class StorageFileBlockOutputStream : public IBlockOutputStream
{
public:
explicit StorageFileBlockOutputStream(StorageFile & storage_,
const CompressionMethod compression_method)
const CompressionMethod compression_method,
const Context & context)
: storage(storage_), lock(storage.rwlock)
{
if (storage.use_table_fd)
@ -306,7 +296,7 @@ public:
write_buf = getWriteBuffer<WriteBufferFromFile>(compression_method, storage.paths[0], DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT);
}
writer = FormatFactory::instance().getOutput(storage.format_name, *write_buf, storage.getSampleBlock(), storage.context_global);
writer = FormatFactory::instance().getOutput(storage.format_name, *write_buf, storage.getSampleBlock(), context);
}
Block getHeader() const override { return storage.getSampleBlock(); }
@ -340,10 +330,10 @@ private:
BlockOutputStreamPtr StorageFile::write(
const ASTPtr & /*query*/,
const Context & /*context*/)
const Context & context)
{
return std::make_shared<StorageFileBlockOutputStream>(*this,
IStorage::chooseCompressionMethod(paths[0], compression_method));
IStorage::chooseCompressionMethod(paths[0], compression_method), context);
}
Strings StorageFile::getDataPaths() const
@ -353,7 +343,7 @@ Strings StorageFile::getDataPaths() const
return paths;
}
void StorageFile::rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
void StorageFile::rename(const String & new_path_to_table_data, const String & new_database_name, const String & new_table_name, TableStructureWriteLockHolder &)
{
if (!is_db_table)
throw Exception("Can't rename table '" + table_name + "' binded to user-defined file (or FD)", ErrorCodes::DATABASE_ACCESS_DENIED);
@ -363,7 +353,7 @@ void StorageFile::rename(const String & new_path_to_db, const String & new_datab
std::unique_lock<std::shared_mutex> lock(rwlock);
std::string path_new = getTablePath(new_path_to_db, new_table_name, format_name);
std::string path_new = getTablePath(base_path + new_path_to_table_data, format_name);
Poco::File(Poco::Path(path_new).parent()).createDirectories();
Poco::File(paths[0]).renameTo(path_new);
@ -387,48 +377,54 @@ void registerStorageFile(StorageFactory & factory)
engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.local_context);
String format_name = engine_args[0]->as<ASTLiteral &>().value.safeGet<String>();
String compression_method;
StorageFile::CommonArguments common_args{args.database_name, args.table_name, format_name, compression_method,
args.columns, args.constraints, args.context};
if (engine_args.size() == 1) /// Table in database
return StorageFile::create(args.relative_data_path, common_args);
/// Will use FD if engine_args[1] is int literal or identifier with std* name
int source_fd = -1;
String source_path;
String compression_method;
if (engine_args.size() >= 2)
{
/// Will use FD if engine_args[1] is int literal or identifier with std* name
if (auto opt_name = tryGetIdentifierName(engine_args[1]))
{
if (*opt_name == "stdin")
source_fd = STDIN_FILENO;
else if (*opt_name == "stdout")
source_fd = STDOUT_FILENO;
else if (*opt_name == "stderr")
source_fd = STDERR_FILENO;
else
throw Exception("Unknown identifier '" + *opt_name + "' in second arg of File storage constructor",
ErrorCodes::UNKNOWN_IDENTIFIER);
}
else if (const auto * literal = engine_args[1]->as<ASTLiteral>())
{
auto type = literal->value.getType();
if (type == Field::Types::Int64)
source_fd = static_cast<int>(literal->value.get<Int64>());
else if (type == Field::Types::UInt64)
source_fd = static_cast<int>(literal->value.get<UInt64>());
else if (type == Field::Types::String)
source_path = literal->value.get<String>();
}
if (engine_args.size() == 3)
{
engine_args[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[2], args.local_context);
compression_method = engine_args[2]->as<ASTLiteral &>().value.safeGet<String>();
} else compression_method = "auto";
if (auto opt_name = tryGetIdentifierName(engine_args[1]))
{
if (*opt_name == "stdin")
source_fd = STDIN_FILENO;
else if (*opt_name == "stdout")
source_fd = STDOUT_FILENO;
else if (*opt_name == "stderr")
source_fd = STDERR_FILENO;
else
throw Exception("Unknown identifier '" + *opt_name + "' in second arg of File storage constructor",
ErrorCodes::UNKNOWN_IDENTIFIER);
}
else if (const auto * literal = engine_args[1]->as<ASTLiteral>())
{
auto type = literal->value.getType();
if (type == Field::Types::Int64)
source_fd = static_cast<int>(literal->value.get<Int64>());
else if (type == Field::Types::UInt64)
source_fd = static_cast<int>(literal->value.get<UInt64>());
else if (type == Field::Types::String)
source_path = literal->value.get<String>();
else
throw Exception("Second argument must be path or file descriptor", ErrorCodes::BAD_ARGUMENTS);
}
return StorageFile::create(
source_path, source_fd,
args.data_path,
args.database_name, args.table_name, format_name, args.columns, args.constraints,
args.context,
compression_method);
if (engine_args.size() == 3)
{
engine_args[2] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[2], args.local_context);
compression_method = engine_args[2]->as<ASTLiteral &>().value.safeGet<String>();
}
else
compression_method = "auto";
if (0 <= source_fd) /// File descriptor
return StorageFile::create(source_fd, common_args);
else /// User's file
return StorageFile::create(source_path, args.context.getUserFilesPath(), common_args);
});
}

Some files were not shown because too many files have changed in this diff Show More