Merge branch 'master' of github.com:ClickHouse/ClickHouse into quorum_bug

This commit is contained in:
Nikita Mikhaylov 2020-04-07 13:42:48 +03:00
commit 70c0646942
61 changed files with 426 additions and 364 deletions

View File

@ -228,7 +228,7 @@ else ()
set(NOT_UNBUNDLED 1)
endif ()
if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN) OR ARCH_32)
if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN))
# Using system libs can cause a lot of warnings in includes (on macro expansion).
option (WERROR "Enable -Werror compiler option" OFF)
else ()
@ -251,6 +251,8 @@ if (OS_LINUX)
include(cmake/linux/default_libs.cmake)
elseif (OS_DARWIN)
include(cmake/darwin/default_libs.cmake)
elseif (OS_FREEBSD)
include(cmake/freebsd/default_libs.cmake)
endif ()
######################################
@ -316,7 +318,6 @@ include (cmake/find/poco.cmake)
include (cmake/find/lz4.cmake)
include (cmake/find/xxhash.cmake)
include (cmake/find/sparsehash.cmake)
include (cmake/find/execinfo.cmake)
include (cmake/find/re2.cmake)
include (cmake/find/libgsasl.cmake)
include (cmake/find/rdkafka.cmake)

View File

@ -4,4 +4,6 @@
#if defined (OS_DARWIN)
# define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC
#elif defined (OS_FREEBSD)
# define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC_FAST
#endif

View File

@ -11,7 +11,6 @@ if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "i386")
set (ARCH_I386 1)
endif ()
if ((ARCH_ARM AND NOT ARCH_AARCH64) OR ARCH_I386)
set (ARCH_32 1)
message (FATAL_ERROR "32bit platforms are not supported")
endif ()

View File

@ -1,8 +0,0 @@
if (OS_FREEBSD)
find_library (EXECINFO_LIBRARY execinfo)
find_library (ELF_LIBRARY elf)
set (EXECINFO_LIBRARIES ${EXECINFO_LIBRARY} ${ELF_LIBRARY})
message (STATUS "Using execinfo: ${EXECINFO_LIBRARIES}")
else ()
set (EXECINFO_LIBRARIES "")
endif ()

View File

@ -1,6 +1,4 @@
if (NOT ARCH_32)
option (USE_INTERNAL_LIBGSASL_LIBRARY "Set to FALSE to use system libgsasl library instead of bundled" ${NOT_UNBUNDLED})
endif ()
option (USE_INTERNAL_LIBGSASL_LIBRARY "Set to FALSE to use system libgsasl library instead of bundled" ${NOT_UNBUNDLED})
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src/gsasl.h")
if (USE_INTERNAL_LIBGSASL_LIBRARY)
@ -16,7 +14,7 @@ if (NOT USE_INTERNAL_LIBGSASL_LIBRARY)
endif ()
if (LIBGSASL_LIBRARY AND LIBGSASL_INCLUDE_DIR)
elseif (NOT MISSING_INTERNAL_LIBGSASL_LIBRARY AND NOT ARCH_32)
elseif (NOT MISSING_INTERNAL_LIBGSASL_LIBRARY)
set (LIBGSASL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src ${ClickHouse_SOURCE_DIR}/contrib/libgsasl/linux_x86_64/include)
set (USE_INTERNAL_LIBGSASL_LIBRARY 1)
set (LIBGSASL_LIBRARY libgsasl)

View File

@ -1,5 +1,5 @@
# Freebsd: contrib/cppkafka/include/cppkafka/detail/endianness.h:53:23: error: 'betoh16' was not declared in this scope
if (NOT ARCH_ARM AND NOT ARCH_32 AND NOT OS_FREEBSD AND OPENSSL_FOUND)
if (NOT ARCH_ARM AND NOT OS_FREEBSD AND OPENSSL_FOUND)
option (ENABLE_RDKAFKA "Enable kafka" ${ENABLE_LIBRARIES})
endif ()

View File

@ -2,9 +2,7 @@ option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES})
if(ENABLE_SSL)
if(NOT ARCH_32)
option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${NOT_UNBUNDLED})
endif()
option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${NOT_UNBUNDLED})
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openssl/README")
if(USE_INTERNAL_SSL_LIBRARY)

View File

@ -1,14 +1,5 @@
option (USE_UNWIND "Enable libunwind (better stacktraces)" ${ENABLE_LIBRARIES})
if (NOT CMAKE_SYSTEM MATCHES "Linux" OR ARCH_ARM OR ARCH_32)
set (USE_UNWIND OFF)
endif ()
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libunwind/CMakeLists.txt")
message(WARNING "submodule contrib/libunwind is missing. to fix try run: \n git submodule update --init --recursive")
set (USE_UNWIND OFF)
endif ()
if (USE_UNWIND)
add_subdirectory(contrib/libunwind-cmake)
set (UNWIND_LIBRARIES unwind)

View File

@ -1,6 +1,4 @@
if (NOT OS_FREEBSD AND NOT ARCH_32)
option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
endif ()
option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
if (NOT MSVC)
set (INTERNAL_ZLIB_NAME "zlib-ng" CACHE INTERNAL "")

View File

@ -0,0 +1,40 @@
set (DEFAULT_LIBS "-nodefaultlibs")
if (NOT COMPILER_CLANG)
message (FATAL_ERROR "FreeBSD build is supported only for Clang")
endif ()
execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread")
message(STATUS "Default libraries: ${DEFAULT_LIBS}")
set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
# Global libraries
add_library(global-libs INTERFACE)
# Unfortunately '-pthread' doesn't work with '-nodefaultlibs'.
# Just make sure we have pthreads at all.
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
include (cmake/find/unwind.cmake)
include (cmake/find/cxx.cmake)
add_library(global-group INTERFACE)
target_link_libraries(global-group INTERFACE
$<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
)
link_libraries(global-group)
# FIXME: remove when all contribs will get custom cmake lists
install(
TARGETS global-group global-libs
EXPORT global
)

View File

@ -0,0 +1,19 @@
set (CMAKE_SYSTEM_NAME "FreeBSD")
set (CMAKE_SYSTEM_PROCESSOR "x86_64")
set (CMAKE_C_COMPILER_TARGET "x86_64-pc-freebsd12.1")
set (CMAKE_CXX_COMPILER_TARGET "x86_64-pc-freebsd12.1")
set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd12.1")
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/freebsd-x86_64")
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake
set (LINKER_NAME "lld" CACHE STRING "" FORCE)
set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld")
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

View File

@ -24,6 +24,9 @@ if (CMAKE_CROSSCOMPILING)
set (ENABLE_PARQUET OFF CACHE INTERNAL "")
set (ENABLE_MYSQL OFF CACHE INTERNAL "")
endif ()
elseif (OS_FREEBSD)
# FIXME: broken dependencies
set (ENABLE_PROTOBUF OFF CACHE INTERNAL "")
else ()
message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!")
endif ()

View File

@ -65,5 +65,8 @@ RUN wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/M
# It contains all required headers and libraries. Note that it's named as "gcc" but actually we are using clang for cross compiling.
RUN wget "https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en" -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz
# Download toolchain for FreeBSD 12.1
RUN wget https://clickhouse-datasets.s3.yandex.net/toolchains/toolchains/freebsd-12.1-toolchain.tar.xz
COPY build.sh /
CMD ["/bin/bash", "/build.sh"]

View File

@ -8,6 +8,9 @@ tar xJf MacOSX10.14.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-co
mkdir -p build/cmake/toolchain/linux-aarch64
tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build/cmake/toolchain/linux-aarch64 --strip-components=1
mkdir -p build/cmake/toolchain/freebsd-x86_64
tar xJf freebsd-12.1-toolchain.tar.xz -C build/cmake/toolchain/freebsd-x86_64 --strip-components=1
mkdir -p build/build_docker
cd build/build_docker
ccache --show-stats ||:

View File

@ -107,6 +107,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
CLANG_PREFIX = "clang"
DARWIN_SUFFIX = "-darwin"
ARM_SUFFIX = "-aarch64"
FREEBSD_SUFFIX = "-freebsd"
result = []
cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1']
@ -114,7 +115,8 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
is_clang = compiler.startswith(CLANG_PREFIX)
is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
is_cross_arm = compiler.endswith(ARM_SUFFIX)
is_cross_compile = is_cross_darwin or is_cross_arm
is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
is_cross_compile = is_cross_darwin or is_cross_arm or is_cross_freebsd
# Explicitly use LLD with Clang by default.
# Don't force linker for cross-compilation.
@ -131,6 +133,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
elif is_cross_arm:
cc = compiler[:-len(ARM_SUFFIX)]
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake")
elif is_cross_freebsd:
cc = compiler[:-len(FREEBSD_SUFFIX)]
cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake")
else:
cc = compiler

View File

@ -148,7 +148,7 @@ function run_tests
TIMEFORMAT=$(printf "$test_name\t%%3R\t%%3U\t%%3S\n")
# the grep is to filter out set -x output and keep only time output
{ time "$script_dir/perf.py" "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" ; } 2>&1 >/dev/null | grep -v ^+ >> "wall-clock-times.tsv" || continue
{ time "$script_dir/perf.py" --host=localhost --port=9001 --host=localhost --port=9002 "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" ; } 2>&1 >/dev/null | grep -v ^+ >> "wall-clock-times.tsv" || continue
# The test completed with zero status, so we treat stderr as warnings
mv "$test_name-err.log" "$test_name-warn.log"

View File

@ -23,8 +23,8 @@ report_stage_end('start')
parser = argparse.ArgumentParser(description='Run performance test.')
# Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set.
parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file')
parser.add_argument('--host', nargs='*', default=['127.0.0.1', '127.0.0.1'], help="Server hostname. Parallel to '--port'.")
parser.add_argument('--port', nargs='*', default=[9001, 9002], help="Server port. Parallel to '--host'.")
parser.add_argument('--host', nargs='*', default=['localhost'], help="Server hostname(s). Corresponds to '--port' options.")
parser.add_argument('--port', nargs='*', default=[9000], help="Server port(s). Corresponds to '--host' options.")
parser.add_argument('--runs', type=int, default=int(os.environ.get('CHPC_RUNS', 7)), help='Number of query runs per server. Defaults to CHPC_RUNS environment variable.')
parser.add_argument('--no-long', type=bool, default=True, help='Skip the tests tagged as long.')
args = parser.parse_args()

View File

@ -34,6 +34,7 @@ toc_title: Client Libraries
- [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse)
- Ruby
- [ClickHouse (Ruby)](https://github.com/shlima/click_house)
- [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord)
- R
- [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r)
- [RClickhouse](https://github.com/IMSMWU/RClickhouse)

View File

@ -36,6 +36,7 @@ toc_title: Bibliotecas de clientes
- [Cualquier evento-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse)
- Rubí
- [Haga clic en Casa (Ruby)](https://github.com/shlima/click_house)
- [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord)
- R
- [Sistema abierto.](https://github.com/hannesmuehleisen/clickhouse-r)
- [Bienvenidos al Portal de Licitación Electrónica de Licitación Electrónica](https://github.com/IMSMWU/RClickhouse)

View File

@ -37,6 +37,7 @@ toc_title: "\u06A9\u062A\u0627\u0628\u062E\u0627\u0646\u0647 \u0647\u0627\u06CC
- [هرفنت-کلیکهاوس](https://metacpan.org/release/AnyEvent-ClickHouse)
- روبی
- [تاتر (روبی)](https://github.com/shlima/click_house)
- [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord)
- R
- [کلیک تحقیق](https://github.com/hannesmuehleisen/clickhouse-r)
- [خانه روستایی](https://github.com/IMSMWU/RClickhouse)

View File

@ -36,6 +36,7 @@ toc_title: "Biblioth\xE8ques Clientes"
- [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse)
- Rubis
- [ClickHouse (Ruby)](https://github.com/shlima/click_house)
- [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord)
- R
- [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r)
- [RClickhouse](https://github.com/IMSMWU/RClickhouse)

View File

@ -36,6 +36,7 @@ toc_title: "\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8"
- [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse)
- Ruby
- [クリックハウス(ruby)](https://github.com/shlima/click_house)
- [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord)
- R
- [クリックハウス-r](https://github.com/hannesmuehleisen/clickhouse-r)
- [Rクリックハウス](https://github.com/IMSMWU/RClickhouse)

View File

@ -29,6 +29,7 @@
- [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse)
- Ruby
- [ClickHouse (Ruby)](https://github.com/shlima/click_house)
- [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord)
- R
- [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r)
- [RClickhouse](https://github.com/IMSMWU/RClickhouse)

View File

@ -28,6 +28,7 @@
- [AnyEvent-ClickHouse](https://metacpan.org/release/AnyEvent-ClickHouse)
- Ruby
- [ClickHouse (Ruby)](https://github.com/shlima/click_house)
- [clickhouse-activerecord](https://github.com/PNixx/clickhouse-activerecord)
- R
- [clickhouse-r](https://github.com/hannesmuehleisen/clickhouse-r)
- [RClickhouse](https://github.com/IMSMWU/RClickhouse)

View File

@ -274,10 +274,6 @@ set_source_files_properties(Common/malloc.cpp PROPERTIES COMPILE_FLAGS "-fno-bui
add_library (clickhouse_new_delete STATIC Common/new_delete.cpp)
target_link_libraries (clickhouse_new_delete PRIVATE clickhouse_common_io jemalloc)
if (OS_FREEBSD)
target_compile_definitions (clickhouse_common_io PUBLIC CLOCK_MONOTONIC_COARSE=CLOCK_MONOTONIC_FAST)
endif ()
add_subdirectory(Common/ZooKeeper)
add_subdirectory(Common/Config)

View File

@ -23,7 +23,9 @@ namespace DB
namespace
{
#if defined(OS_LINUX)
thread_local size_t write_trace_iteration = 0;
#endif
void writeTraceInfo(TraceType trace_type, int /* sig */, siginfo_t * info, void * context)
{
@ -53,7 +55,6 @@ namespace
}
#else
UNUSED(info);
UNUSED(write_trace_iteration);
#endif
const auto signal_context = *reinterpret_cast<ucontext_t *>(context);
@ -110,7 +111,7 @@ QueryProfilerBase<ProfilerImpl>::QueryProfilerBase(const UInt64 thread_id, const
sev.sigev_notify = SIGEV_THREAD_ID;
sev.sigev_signo = pause_signal;
# if defined(__FreeBSD__)
# if defined(OS_FREEBSD)
sev._sigev_un._threadid = thread_id;
# else
sev._sigev_un._tid = thread_id;

View File

@ -2,7 +2,7 @@
#include <time.h>
#include <sys/time.h>
#if defined(OS_LINUX)
#include <sys/sysinfo.h>
# include <sys/sysinfo.h>
#endif
#include <sched.h>
@ -20,9 +20,9 @@
/// We will also wrap some thread synchronization functions to inject sleep/migration before or after.
#if defined(OS_LINUX)
#define FOR_EACH_WRAPPED_FUNCTION(M) \
M(int, pthread_mutex_lock, pthread_mutex_t * arg) \
M(int, pthread_mutex_unlock, pthread_mutex_t * arg)
# define FOR_EACH_WRAPPED_FUNCTION(M) \
M(int, pthread_mutex_lock, pthread_mutex_t * arg) \
M(int, pthread_mutex_unlock, pthread_mutex_t * arg)
#endif
namespace DB
@ -67,20 +67,20 @@ static void initFromEnv(std::atomic<T> & what, const char * name)
static std::atomic<int> num_cpus = 0;
#if defined(OS_LINUX)
#define DEFINE_WRAPPER_PARAMS(RET, NAME, ...) \
static std::atomic<double> NAME ## _before_yield_probability = 0; \
static std::atomic<double> NAME ## _before_migrate_probability = 0; \
static std::atomic<double> NAME ## _before_sleep_probability = 0; \
static std::atomic<double> NAME ## _before_sleep_time_us = 0; \
\
static std::atomic<double> NAME ## _after_yield_probability = 0; \
static std::atomic<double> NAME ## _after_migrate_probability = 0; \
static std::atomic<double> NAME ## _after_sleep_probability = 0; \
static std::atomic<double> NAME ## _after_sleep_time_us = 0; \
# define DEFINE_WRAPPER_PARAMS(RET, NAME, ...) \
static std::atomic<double> NAME##_before_yield_probability = 0; \
static std::atomic<double> NAME##_before_migrate_probability = 0; \
static std::atomic<double> NAME##_before_sleep_probability = 0; \
static std::atomic<double> NAME##_before_sleep_time_us = 0; \
\
static std::atomic<double> NAME##_after_yield_probability = 0; \
static std::atomic<double> NAME##_after_migrate_probability = 0; \
static std::atomic<double> NAME##_after_sleep_probability = 0; \
static std::atomic<double> NAME##_after_sleep_time_us = 0;
FOR_EACH_WRAPPED_FUNCTION(DEFINE_WRAPPER_PARAMS)
#undef DEFINE_WRAPPER_PARAMS
# undef DEFINE_WRAPPER_PARAMS
#endif
void ThreadFuzzer::initConfiguration()
@ -98,20 +98,20 @@ void ThreadFuzzer::initConfiguration()
initFromEnv(sleep_time_us, "THREAD_FUZZER_SLEEP_TIME_US");
#if defined(OS_LINUX)
#define INIT_WRAPPER_PARAMS(RET, NAME, ...) \
initFromEnv(NAME ## _before_yield_probability, "THREAD_FUZZER_" #NAME "_BEFORE_YIELD_PROBABILITY"); \
initFromEnv(NAME ## _before_migrate_probability, "THREAD_FUZZER_" #NAME "_BEFORE_MIGRATE_PROBABILITY"); \
initFromEnv(NAME ## _before_sleep_probability, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_PROBABILITY"); \
initFromEnv(NAME ## _before_sleep_time_us, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US"); \
\
initFromEnv(NAME ## _after_yield_probability, "THREAD_FUZZER_" #NAME "_AFTER_YIELD_PROBABILITY"); \
initFromEnv(NAME ## _after_migrate_probability, "THREAD_FUZZER_" #NAME "_AFTER_MIGRATE_PROBABILITY"); \
initFromEnv(NAME ## _after_sleep_probability, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_PROBABILITY"); \
initFromEnv(NAME ## _after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US"); \
# define INIT_WRAPPER_PARAMS(RET, NAME, ...) \
initFromEnv(NAME##_before_yield_probability, "THREAD_FUZZER_" #NAME "_BEFORE_YIELD_PROBABILITY"); \
initFromEnv(NAME##_before_migrate_probability, "THREAD_FUZZER_" #NAME "_BEFORE_MIGRATE_PROBABILITY"); \
initFromEnv(NAME##_before_sleep_probability, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_PROBABILITY"); \
initFromEnv(NAME##_before_sleep_time_us, "THREAD_FUZZER_" #NAME "_BEFORE_SLEEP_TIME_US"); \
\
initFromEnv(NAME##_after_yield_probability, "THREAD_FUZZER_" #NAME "_AFTER_YIELD_PROBABILITY"); \
initFromEnv(NAME##_after_migrate_probability, "THREAD_FUZZER_" #NAME "_AFTER_MIGRATE_PROBABILITY"); \
initFromEnv(NAME##_after_sleep_probability, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_PROBABILITY"); \
initFromEnv(NAME##_after_sleep_time_us, "THREAD_FUZZER_" #NAME "_AFTER_SLEEP_TIME_US");
FOR_EACH_WRAPPED_FUNCTION(INIT_WRAPPER_PARAMS)
#undef INIT_WRAPPER_PARAMS
# undef INIT_WRAPPER_PARAMS
#endif
}
@ -119,20 +119,28 @@ void ThreadFuzzer::initConfiguration()
bool ThreadFuzzer::isEffective() const
{
#if defined(OS_LINUX)
#define CHECK_WRAPPER_PARAMS(RET, NAME, ...) \
if (NAME ## _before_yield_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _before_migrate_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _before_sleep_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _before_sleep_time_us.load(std::memory_order_relaxed)) return true; \
\
if (NAME ## _after_yield_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _after_migrate_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _after_sleep_probability.load(std::memory_order_relaxed)) return true; \
if (NAME ## _after_sleep_time_us.load(std::memory_order_relaxed)) return true; \
# define CHECK_WRAPPER_PARAMS(RET, NAME, ...) \
if (NAME##_before_yield_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_before_migrate_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_before_sleep_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_before_sleep_time_us.load(std::memory_order_relaxed)) \
return true; \
\
if (NAME##_after_yield_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_after_migrate_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_after_sleep_probability.load(std::memory_order_relaxed)) \
return true; \
if (NAME##_after_sleep_time_us.load(std::memory_order_relaxed)) \
return true;
FOR_EACH_WRAPPED_FUNCTION(CHECK_WRAPPER_PARAMS)
#undef INIT_WRAPPER_PARAMS
# undef INIT_WRAPPER_PARAMS
#endif
return cpu_time_period_us != 0
@ -229,30 +237,29 @@ void ThreadFuzzer::setup()
/// NOTE We cannot use dlsym(... RTLD_NEXT), because it will call pthread_mutex_lock and it will lead to infinite recursion.
#if defined(OS_LINUX)
#define MAKE_WRAPPER(RET, NAME, ...) \
extern "C" RET __ ## NAME(__VA_ARGS__); /* NOLINT */ \
extern "C" RET NAME(__VA_ARGS__) /* NOLINT */ \
{ \
injection( \
NAME ## _before_yield_probability.load(std::memory_order_relaxed), \
NAME ## _before_migrate_probability.load(std::memory_order_relaxed), \
NAME ## _before_sleep_probability.load(std::memory_order_relaxed), \
NAME ## _before_sleep_time_us.load(std::memory_order_relaxed)); \
\
auto && ret{__ ## NAME(arg)}; \
\
injection( \
NAME ## _after_yield_probability.load(std::memory_order_relaxed), \
NAME ## _after_migrate_probability.load(std::memory_order_relaxed), \
NAME ## _after_sleep_probability.load(std::memory_order_relaxed), \
NAME ## _after_sleep_time_us.load(std::memory_order_relaxed)); \
\
return ret; \
} \
# define MAKE_WRAPPER(RET, NAME, ...) \
extern "C" RET __##NAME(__VA_ARGS__); /* NOLINT */ \
extern "C" RET NAME(__VA_ARGS__) /* NOLINT */ \
{ \
injection( \
NAME##_before_yield_probability.load(std::memory_order_relaxed), \
NAME##_before_migrate_probability.load(std::memory_order_relaxed), \
NAME##_before_sleep_probability.load(std::memory_order_relaxed), \
NAME##_before_sleep_time_us.load(std::memory_order_relaxed)); \
\
auto && ret{__##NAME(arg)}; \
\
injection( \
NAME##_after_yield_probability.load(std::memory_order_relaxed), \
NAME##_after_migrate_probability.load(std::memory_order_relaxed), \
NAME##_after_sleep_probability.load(std::memory_order_relaxed), \
NAME##_after_sleep_time_us.load(std::memory_order_relaxed)); \
\
return ret; \
}
FOR_EACH_WRAPPED_FUNCTION(MAKE_WRAPPER)
FOR_EACH_WRAPPED_FUNCTION(MAKE_WRAPPER)
#undef MAKE_WRAPPER
# undef MAKE_WRAPPER
#endif
}

View File

@ -29,11 +29,10 @@ void setThreadName(const char * name)
throw DB::Exception("Thread name cannot be longer than 15 bytes", DB::ErrorCodes::PTHREAD_ERROR);
#endif
#if defined(__FreeBSD__)
#if defined(OS_FREEBSD)
pthread_set_name_np(pthread_self(), name);
return;
#elif defined(__APPLE__)
if ((false))
#elif defined(OS_DARWIN)
if (0 != pthread_setname_np(name))
#else
if (0 != prctl(PR_SET_NAME, name, 0, 0, 0))

View File

@ -3,7 +3,8 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionHelpers.h>
#include <Interpreters/Context.h>
#include <Interpreters/Join.h>
#include <Interpreters/HashJoin.h>
#include <Columns/ColumnString.h>
#include <Storages/StorageJoin.h>

View File

@ -6,8 +6,8 @@ namespace DB
{
class Context;
class Join;
using HashJoinPtr = std::shared_ptr<Join>;
class HashJoin;
using HashJoinPtr = std::shared_ptr<HashJoin>;
class ExecutableFunctionJoinGet final : public IExecutableFunctionImpl
{

View File

@ -1,12 +1,12 @@
#if defined(__linux__)
#include <boost/noncopyable.hpp>
#include <Common/Exception.h>
#include <sys/syscall.h>
#include <unistd.h>
#include <IO/AIO.h>
#if defined(OS_LINUX)
# include <Common/Exception.h>
# include <sys/syscall.h>
# include <unistd.h>
/** Small wrappers for asynchronous I/O.
*/
@ -53,17 +53,10 @@ AIOContext::~AIOContext()
io_destroy(ctx);
}
#elif defined(__FreeBSD__)
#elif defined(OS_FREEBSD)
# include <aio.h>
# include <boost/noncopyable.hpp>
# include <sys/event.h>
# include <sys/time.h>
# include <sys/types.h>
# include <Common/Exception.h>
# include <IO/AIO.h>
/** Small wrappers for asynchronous I/O.
*/
@ -123,7 +116,7 @@ int io_submit(int ctx, long nr, struct iocb * iocbpp[])
int io_getevents(int ctx, long, long max_nr, struct kevent * events, struct timespec * timeout)
{
return kevent(ctx, NULL, 0, events, max_nr, timeout);
return kevent(ctx, nullptr, 0, events, max_nr, timeout);
}

View File

@ -2,20 +2,20 @@
#include <boost/noncopyable.hpp>
#if defined(__linux__)
#if defined(OS_LINUX)
/// https://stackoverflow.com/questions/20759750/resolving-redefinition-of-timespec-in-time-h
#define timespec linux_timespec
#define timeval linux_timeval
#define itimerspec linux_itimerspec
#define sigset_t linux_sigset_t
# define timespec linux_timespec
# define timeval linux_timeval
# define itimerspec linux_itimerspec
# define sigset_t linux_sigset_t
#include <linux/aio_abi.h>
# include <linux/aio_abi.h>
#undef timespec
#undef timeval
#undef itimerspec
#undef sigset_t
# undef timespec
# undef timeval
# undef itimerspec
# undef sigset_t
/** Small wrappers for asynchronous I/O.
@ -39,12 +39,12 @@ struct AIOContext : private boost::noncopyable
~AIOContext();
};
#elif defined(__FreeBSD__)
#elif defined(OS_FREEBSD)
#include <aio.h>
#include <sys/types.h>
#include <sys/event.h>
#include <sys/time.h>
# include <aio.h>
# include <sys/event.h>
# include <sys/time.h>
# include <sys/types.h>
typedef struct kevent io_event;
typedef int aio_context_t;

View File

@ -2,7 +2,7 @@
#include <Interpreters/CollectJoinOnKeysVisitor.h>
#include <Interpreters/IdentifierSemantic.h>
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/TableJoin.h>
namespace DB
{

View File

@ -11,7 +11,7 @@ namespace DB
{
class ASTIdentifier;
class AnalyzedJoin;
class TableJoin;
namespace ASOF
{
@ -25,7 +25,7 @@ public:
struct Data
{
AnalyzedJoin & analyzed_join;
TableJoin & analyzed_join;
const TableWithColumnNames & left_table;
const TableWithColumnNames & right_table;
const Aliases & aliases;

View File

@ -4,7 +4,7 @@
#include <Common/SipHash.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/ExpressionJIT.h>
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/TableJoin.h>
#include <Columns/ColumnsNumber.h>
#include <Common/typeid_cast.h>
#include <DataTypes/DataTypeArray.h>
@ -147,7 +147,7 @@ ExpressionAction ExpressionAction::arrayJoin(const NameSet & array_joined_column
return a;
}
ExpressionAction ExpressionAction::ordinaryJoin(std::shared_ptr<AnalyzedJoin> table_join, JoinPtr join)
ExpressionAction ExpressionAction::ordinaryJoin(std::shared_ptr<TableJoin> table_join, JoinPtr join)
{
ExpressionAction a;
a.type = JOIN;
@ -1206,7 +1206,7 @@ bool ExpressionAction::operator==(const ExpressionAction & other) const
&& result_name == other.result_name
&& argument_names == other.argument_names
&& same_array_join
&& AnalyzedJoin::sameJoin(table_join.get(), other.table_join.get())
&& TableJoin::sameJoin(table_join.get(), other.table_join.get())
&& projection == other.projection
&& is_function_compiled == other.is_function_compiled;
}

View File

@ -22,7 +22,7 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
class AnalyzedJoin;
class TableJoin;
class IJoin;
using JoinPtr = std::shared_ptr<IJoin>;
@ -97,7 +97,7 @@ public:
std::shared_ptr<ArrayJoinAction> array_join;
/// For JOIN
std::shared_ptr<const AnalyzedJoin> table_join;
std::shared_ptr<const TableJoin> table_join;
JoinPtr join;
/// For PROJECT.
@ -114,7 +114,7 @@ public:
static ExpressionAction project(const Names & projected_columns_);
static ExpressionAction addAliases(const NamesWithAliases & aliased_columns_);
static ExpressionAction arrayJoin(const NameSet & array_joined_columns, bool array_join_is_left, const Context & context);
static ExpressionAction ordinaryJoin(std::shared_ptr<AnalyzedJoin> table_join, JoinPtr join);
static ExpressionAction ordinaryJoin(std::shared_ptr<TableJoin> table_join, JoinPtr join);
/// Which columns necessary to perform this action.
Names getNeededColumns() const;

View File

@ -27,9 +27,9 @@
#include <Interpreters/LogicalExpressionsOptimizer.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/Set.h>
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/TableJoin.h>
#include <Interpreters/JoinSwitcher.h>
#include <Interpreters/Join.h>
#include <Interpreters/HashJoin.h>
#include <Interpreters/MergeJoin.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
@ -502,7 +502,7 @@ bool SelectQueryExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain, b
return true;
}
static JoinPtr tryGetStorageJoin(const ASTTablesInSelectQueryElement & join_element, std::shared_ptr<AnalyzedJoin> analyzed_join,
static JoinPtr tryGetStorageJoin(const ASTTablesInSelectQueryElement & join_element, std::shared_ptr<TableJoin> analyzed_join,
const Context & context)
{
const auto & table_to_join = join_element.table_expression->as<ASTTableExpression &>();
@ -524,19 +524,19 @@ static JoinPtr tryGetStorageJoin(const ASTTablesInSelectQueryElement & join_elem
return {};
}
static ExpressionActionsPtr createJoinedBlockActions(const Context & context, const AnalyzedJoin & analyzed_join)
static ExpressionActionsPtr createJoinedBlockActions(const Context & context, const TableJoin & analyzed_join)
{
ASTPtr expression_list = analyzed_join.rightKeysList();
auto syntax_result = SyntaxAnalyzer(context).analyze(expression_list, analyzed_join.columnsFromJoinedTable());
return ExpressionAnalyzer(expression_list, syntax_result, context).getActions(true, false);
}
static std::shared_ptr<IJoin> makeJoin(std::shared_ptr<AnalyzedJoin> analyzed_join, const Block & sample_block)
static std::shared_ptr<IJoin> makeJoin(std::shared_ptr<TableJoin> analyzed_join, const Block & sample_block)
{
bool allow_merge_join = analyzed_join->allowMergeJoin();
if (analyzed_join->forceHashJoin() || (analyzed_join->preferMergeJoin() && !allow_merge_join))
return std::make_shared<Join>(analyzed_join, sample_block);
return std::make_shared<HashJoin>(analyzed_join, sample_block);
else if (analyzed_join->forceMergeJoin() || (analyzed_join->preferMergeJoin() && allow_merge_join))
return std::make_shared<MergeJoin>(analyzed_join, sample_block);
return std::make_shared<JoinSwitcher>(analyzed_join, sample_block);

View File

@ -123,7 +123,7 @@ protected:
SyntaxAnalyzerResultPtr syntax;
const ConstStoragePtr & storage() const { return syntax->storage; } /// The main table in FROM clause, if exists.
const AnalyzedJoin & analyzedJoin() const { return *syntax->analyzed_join; }
const TableJoin & analyzedJoin() const { return *syntax->analyzed_join; }
const NamesAndTypesList & sourceColumns() const { return syntax->required_source_columns; }
const std::vector<const ASTFunction *> & aggregates() const { return syntax->aggregates; }
NamesAndTypesList sourceWithJoinedColumns() const;

View File

@ -9,9 +9,9 @@
#include <DataTypes/DataTypeNullable.h>
#include <Interpreters/Join.h>
#include <Interpreters/HashJoin.h>
#include <Interpreters/join_common.h>
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/TableJoin.h>
#include <Interpreters/joinDispatch.h>
#include <Interpreters/NullableUtils.h>
@ -189,7 +189,7 @@ static void changeColumnRepresentation(const ColumnPtr & src_column, ColumnPtr &
}
Join::Join(std::shared_ptr<AnalyzedJoin> table_join_, const Block & right_sample_block, bool any_take_last_row_)
HashJoin::HashJoin(std::shared_ptr<TableJoin> table_join_, const Block & right_sample_block, bool any_take_last_row_)
: table_join(table_join_)
, kind(table_join->kind())
, strictness(table_join->strictness())
@ -199,13 +199,13 @@ Join::Join(std::shared_ptr<AnalyzedJoin> table_join_, const Block & right_sample
, any_take_last_row(any_take_last_row_)
, asof_inequality(table_join->getAsofInequality())
, data(std::make_shared<RightTableData>())
, log(&Logger::get("Join"))
, log(&Logger::get("HashJoin"))
{
setSampleBlock(right_sample_block);
}
Join::Type Join::chooseMethod(const ColumnRawPtrs & key_columns, Sizes & key_sizes)
HashJoin::Type HashJoin::chooseMethod(const ColumnRawPtrs & key_columns, Sizes & key_sizes)
{
size_t keys_size = key_columns.size();
@ -282,47 +282,47 @@ static KeyGetter createKeyGetter(const ColumnRawPtrs & key_columns, const Sizes
return KeyGetter(key_columns, key_sizes, nullptr);
}
template <Join::Type type, typename Value, typename Mapped>
template <HashJoin::Type type, typename Value, typename Mapped>
struct KeyGetterForTypeImpl;
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<Join::Type::key8, Value, Mapped>
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::key8, Value, Mapped>
{
using Type = ColumnsHashing::HashMethodOneNumber<Value, Mapped, UInt8, false>;
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<Join::Type::key16, Value, Mapped>
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::key16, Value, Mapped>
{
using Type = ColumnsHashing::HashMethodOneNumber<Value, Mapped, UInt16, false>;
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<Join::Type::key32, Value, Mapped>
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::key32, Value, Mapped>
{
using Type = ColumnsHashing::HashMethodOneNumber<Value, Mapped, UInt32, false>;
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<Join::Type::key64, Value, Mapped>
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::key64, Value, Mapped>
{
using Type = ColumnsHashing::HashMethodOneNumber<Value, Mapped, UInt64, false>;
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<Join::Type::key_string, Value, Mapped>
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::key_string, Value, Mapped>
{
using Type = ColumnsHashing::HashMethodString<Value, Mapped, true, false>;
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<Join::Type::key_fixed_string, Value, Mapped>
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::key_fixed_string, Value, Mapped>
{
using Type = ColumnsHashing::HashMethodFixedString<Value, Mapped, true, false>;
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<Join::Type::keys128, Value, Mapped>
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::keys128, Value, Mapped>
{
using Type = ColumnsHashing::HashMethodKeysFixed<Value, UInt128, Mapped, false, false, false>;
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<Join::Type::keys256, Value, Mapped>
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::keys256, Value, Mapped>
{
using Type = ColumnsHashing::HashMethodKeysFixed<Value, UInt256, Mapped, false, false, false>;
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<Join::Type::hashed, Value, Mapped>
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::hashed, Value, Mapped>
{
using Type = ColumnsHashing::HashMethodHashed<Value, Mapped, false>;
};
template <Join::Type type, typename Data>
template <HashJoin::Type type, typename Data>
struct KeyGetterForType
{
using Value = typename Data::value_type;
@ -332,7 +332,7 @@ struct KeyGetterForType
};
void Join::init(Type type_)
void HashJoin::init(Type type_)
{
data->type = type_;
@ -342,7 +342,7 @@ void Join::init(Type type_)
joinDispatch(kind, strictness, data->maps, [&](auto, auto, auto & map) { map.create(data->type); });
}
size_t Join::getTotalRowCount() const
size_t HashJoin::getTotalRowCount() const
{
size_t res = 0;
@ -359,7 +359,7 @@ size_t Join::getTotalRowCount() const
return res;
}
size_t Join::getTotalByteCount() const
size_t HashJoin::getTotalByteCount() const
{
size_t res = 0;
@ -377,7 +377,7 @@ size_t Join::getTotalByteCount() const
return res;
}
void Join::setSampleBlock(const Block & block)
void HashJoin::setSampleBlock(const Block & block)
{
/// You have to restore this lock if you call the function outside of ctor.
//std::unique_lock lock(rwlock);
@ -441,7 +441,7 @@ namespace
template <typename Map, typename KeyGetter>
struct Inserter
{
static ALWAYS_INLINE void insertOne(const Join & join, Map & map, KeyGetter & key_getter, Block * stored_block, size_t i,
static ALWAYS_INLINE void insertOne(const HashJoin & join, Map & map, KeyGetter & key_getter, Block * stored_block, size_t i,
Arena & pool)
{
auto emplace_result = key_getter.emplaceKey(map, i, pool);
@ -450,7 +450,7 @@ namespace
new (&emplace_result.getMapped()) typename Map::mapped_type(stored_block, i);
}
static ALWAYS_INLINE void insertAll(const Join &, Map & map, KeyGetter & key_getter, Block * stored_block, size_t i, Arena & pool)
static ALWAYS_INLINE void insertAll(const HashJoin &, Map & map, KeyGetter & key_getter, Block * stored_block, size_t i, Arena & pool)
{
auto emplace_result = key_getter.emplaceKey(map, i, pool);
@ -463,7 +463,7 @@ namespace
}
}
static ALWAYS_INLINE void insertAsof(Join & join, Map & map, KeyGetter & key_getter, Block * stored_block, size_t i, Arena & pool,
static ALWAYS_INLINE void insertAsof(HashJoin & join, Map & map, KeyGetter & key_getter, Block * stored_block, size_t i, Arena & pool,
const IColumn * asof_column)
{
auto emplace_result = key_getter.emplaceKey(map, i, pool);
@ -478,7 +478,7 @@ namespace
template <ASTTableJoin::Strictness STRICTNESS, typename KeyGetter, typename Map, bool has_null_map>
void NO_INLINE insertFromBlockImplTypeCase(
Join & join, Map & map, size_t rows, const ColumnRawPtrs & key_columns,
HashJoin & join, Map & map, size_t rows, const ColumnRawPtrs & key_columns,
const Sizes & key_sizes, Block * stored_block, ConstNullMapPtr null_map, Arena & pool)
{
[[maybe_unused]] constexpr bool mapped_one = std::is_same_v<typename Map::mapped_type, JoinStuff::MappedOne> ||
@ -508,7 +508,7 @@ namespace
template <ASTTableJoin::Strictness STRICTNESS, typename KeyGetter, typename Map>
void insertFromBlockImplType(
Join & join, Map & map, size_t rows, const ColumnRawPtrs & key_columns,
HashJoin & join, Map & map, size_t rows, const ColumnRawPtrs & key_columns,
const Sizes & key_sizes, Block * stored_block, ConstNullMapPtr null_map, Arena & pool)
{
if (null_map)
@ -520,17 +520,17 @@ namespace
template <ASTTableJoin::Strictness STRICTNESS, typename Maps>
void insertFromBlockImpl(
Join & join, Join::Type type, Maps & maps, size_t rows, const ColumnRawPtrs & key_columns,
HashJoin & join, HashJoin::Type type, Maps & maps, size_t rows, const ColumnRawPtrs & key_columns,
const Sizes & key_sizes, Block * stored_block, ConstNullMapPtr null_map, Arena & pool)
{
switch (type)
{
case Join::Type::EMPTY: break;
case Join::Type::CROSS: break; /// Do nothing. We have already saved block, and it is enough.
case HashJoin::Type::EMPTY: break;
case HashJoin::Type::CROSS: break; /// Do nothing. We have already saved block, and it is enough.
#define M(TYPE) \
case Join::Type::TYPE: \
insertFromBlockImplType<STRICTNESS, typename KeyGetterForType<Join::Type::TYPE, std::remove_reference_t<decltype(*maps.TYPE)>>::Type>(\
case HashJoin::Type::TYPE: \
insertFromBlockImplType<STRICTNESS, typename KeyGetterForType<HashJoin::Type::TYPE, std::remove_reference_t<decltype(*maps.TYPE)>>::Type>(\
join, *maps.TYPE, rows, key_columns, key_sizes, stored_block, null_map, pool); \
break;
APPLY_FOR_JOIN_VARIANTS(M)
@ -539,7 +539,7 @@ namespace
}
}
void Join::initRequiredRightKeys()
void HashJoin::initRequiredRightKeys()
{
const Names & left_keys = table_join->keyNamesLeft();
const Names & right_keys = table_join->keyNamesRight();
@ -558,7 +558,7 @@ void Join::initRequiredRightKeys()
}
}
void Join::initRightBlockStructure(Block & saved_block_sample)
void HashJoin::initRightBlockStructure(Block & saved_block_sample)
{
/// We could remove key columns for LEFT | INNER HashJoin but we should keep them for JoinSwitcher (if any).
bool save_key_columns = !table_join->forceHashJoin() || isRightOrFull(kind);
@ -580,7 +580,7 @@ void Join::initRightBlockStructure(Block & saved_block_sample)
JoinCommon::convertColumnsToNullable(saved_block_sample, (isFull(kind) ? right_table_keys.columns() : 0));
}
Block Join::structureRightBlock(const Block & block) const
Block HashJoin::structureRightBlock(const Block & block) const
{
Block structured_block;
for (auto & sample_column : savedBlockSample().getColumnsWithTypeAndName())
@ -594,10 +594,10 @@ Block Join::structureRightBlock(const Block & block) const
return structured_block;
}
bool Join::addJoinedBlock(const Block & source_block, bool check_limits)
bool HashJoin::addJoinedBlock(const Block & source_block, bool check_limits)
{
if (empty())
throw Exception("Logical error: Join was not initialized", ErrorCodes::LOGICAL_ERROR);
throw Exception("Logical error: HashJoin was not initialized", ErrorCodes::LOGICAL_ERROR);
/// There's no optimization for right side const columns. Remove constness if any.
Block block = materializeBlock(source_block);
@ -666,7 +666,7 @@ public:
const Block & block,
const Block & saved_block_sample,
const ColumnsWithTypeAndName & extras,
const Join & join_,
const HashJoin & join_,
const ColumnRawPtrs & key_columns_,
const Sizes & key_sizes_)
: join(join_)
@ -729,7 +729,7 @@ public:
}
}
const Join & join;
const HashJoin & join;
const ColumnRawPtrs & key_columns;
const Sizes & key_sizes;
size_t rows_to_add;
@ -839,7 +839,7 @@ NO_INLINE IColumn::Filter joinRightColumns(const Map & map, AddedColumns & added
if constexpr (is_asof_join)
{
const Join & join = added_columns.join;
const HashJoin & join = added_columns.join;
if (const RowRef * found = mapped.findAsof(join.getAsofType(), join.getAsofInequality(), asof_column, i))
{
setUsed<need_filter>(filter, i);
@ -924,14 +924,14 @@ IColumn::Filter joinRightColumnsSwitchNullability(const Map & map, AddedColumns
}
template <ASTTableJoin::Kind KIND, ASTTableJoin::Strictness STRICTNESS, typename Maps>
IColumn::Filter switchJoinRightColumns(const Maps & maps_, AddedColumns & added_columns, Join::Type type, const ConstNullMapPtr & null_map)
IColumn::Filter switchJoinRightColumns(const Maps & maps_, AddedColumns & added_columns, HashJoin::Type type, const ConstNullMapPtr & null_map)
{
switch (type)
{
#define M(TYPE) \
case Join::Type::TYPE: \
case HashJoin::Type::TYPE: \
return joinRightColumnsSwitchNullability<KIND, STRICTNESS,\
typename KeyGetterForType<Join::Type::TYPE, const std::remove_reference_t<decltype(*maps_.TYPE)>>::Type>(\
typename KeyGetterForType<HashJoin::Type::TYPE, const std::remove_reference_t<decltype(*maps_.TYPE)>>::Type>(\
*maps_.TYPE, added_columns, null_map);\
break;
APPLY_FOR_JOIN_VARIANTS(M)
@ -946,7 +946,7 @@ IColumn::Filter switchJoinRightColumns(const Maps & maps_, AddedColumns & added_
template <ASTTableJoin::Kind KIND, ASTTableJoin::Strictness STRICTNESS, typename Maps>
void Join::joinBlockImpl(
void HashJoin::joinBlockImpl(
Block & block,
const Names & key_names_left,
const Block & block_with_columns_to_add,
@ -1065,7 +1065,7 @@ void Join::joinBlockImpl(
}
}
void Join::joinBlockImplCross(Block & block, ExtraBlockPtr & not_processed) const
void HashJoin::joinBlockImplCross(Block & block, ExtraBlockPtr & not_processed) const
{
size_t max_joined_block_rows = table_join->maxJoinedBlockRows();
size_t start_left_row = 0;
@ -1158,7 +1158,7 @@ static void checkTypeOfKey(const Block & block_left, const Block & block_right)
}
DataTypePtr Join::joinGetReturnType(const String & column_name) const
DataTypePtr HashJoin::joinGetReturnType(const String & column_name) const
{
std::shared_lock lock(data->rwlock);
@ -1169,7 +1169,7 @@ DataTypePtr Join::joinGetReturnType(const String & column_name) const
template <typename Maps>
void Join::joinGetImpl(Block & block, const String & column_name, const Maps & maps_) const
void HashJoin::joinGetImpl(Block & block, const String & column_name, const Maps & maps_) const
{
joinBlockImpl<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::RightAny>(
block, {block.getByPosition(0).name}, {sample_block_with_columns_to_add.getByName(column_name)}, maps_);
@ -1179,7 +1179,7 @@ void Join::joinGetImpl(Block & block, const String & column_name, const Maps & m
// TODO: support composite key
// TODO: return multiple columns as named tuple
// TODO: return array of values when strictness == ASTTableJoin::Strictness::All
void Join::joinGet(Block & block, const String & column_name) const
void HashJoin::joinGet(Block & block, const String & column_name) const
{
std::shared_lock lock(data->rwlock);
@ -1198,7 +1198,7 @@ void Join::joinGet(Block & block, const String & column_name) const
}
void Join::joinBlock(Block & block, ExtraBlockPtr & not_processed)
void HashJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed)
{
std::shared_lock lock(data->rwlock);
@ -1219,7 +1219,7 @@ void Join::joinBlock(Block & block, ExtraBlockPtr & not_processed)
}
void Join::joinTotals(Block & block) const
void HashJoin::joinTotals(Block & block) const
{
JoinCommon::joinTotals(totals, sample_block_with_columns_to_add, key_names_right, block);
}
@ -1268,7 +1268,7 @@ struct AdderNonJoined
class NonJoinedBlockInputStream : public IBlockInputStream
{
public:
NonJoinedBlockInputStream(const Join & parent_, const Block & result_sample_block_, UInt64 max_block_size_)
NonJoinedBlockInputStream(const HashJoin & parent_, const Block & result_sample_block_, UInt64 max_block_size_)
: parent(parent_)
, max_block_size(max_block_size_)
, result_sample_block(materializeBlock(result_sample_block_))
@ -1342,7 +1342,7 @@ protected:
}
private:
const Join & parent;
const HashJoin & parent;
UInt64 max_block_size;
Block result_sample_block;
@ -1359,7 +1359,7 @@ private:
std::vector<std::pair<size_t, ColumnPtr>> right_lowcard_changes;
std::any position;
std::optional<Join::BlockNullmapList::const_iterator> nulls_position;
std::optional<HashJoin::BlockNullmapList::const_iterator> nulls_position;
void setRightIndex(size_t right_pos, size_t result_position)
{
@ -1452,7 +1452,7 @@ private:
switch (parent.data->type)
{
#define M(TYPE) \
case Join::Type::TYPE: \
case HashJoin::Type::TYPE: \
return fillColumns<STRICTNESS>(*maps.TYPE, columns_keys_and_right);
APPLY_FOR_JOIN_VARIANTS(M)
#undef M
@ -1523,7 +1523,7 @@ private:
};
BlockInputStreamPtr Join::createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const
BlockInputStreamPtr HashJoin::createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const
{
if (table_join->strictness() == ASTTableJoin::Strictness::Asof ||
table_join->strictness() == ASTTableJoin::Strictness::Semi)
@ -1535,7 +1535,7 @@ BlockInputStreamPtr Join::createStreamWithNonJoinedRows(const Block & result_sam
}
bool Join::hasStreamWithNonJoinedRows() const
bool HashJoin::hasStreamWithNonJoinedRows() const
{
if (table_join->strictness() == ASTTableJoin::Strictness::Asof ||
table_join->strictness() == ASTTableJoin::Strictness::Semi)

View File

@ -26,7 +26,7 @@
namespace DB
{
class AnalyzedJoin;
class TableJoin;
namespace JoinStuff
{
@ -143,10 +143,10 @@ using MappedAsof = WithFlags<AsofRowRefs, false>;
* If it is true, we always generate Nullable column and substitute NULLs for non-joined rows,
* as in standard SQL.
*/
class Join : public IJoin
class HashJoin : public IJoin
{
public:
Join(std::shared_ptr<AnalyzedJoin> table_join_, const Block & right_sample_block, bool any_take_last_row_ = false);
HashJoin(std::shared_ptr<TableJoin> table_join_, const Block & right_sample_block, bool any_take_last_row_ = false);
bool empty() { return data->type == Type::EMPTY; }
@ -315,7 +315,7 @@ public:
Arena pool;
};
void reuseJoinedData(const Join & join)
void reuseJoinedData(const HashJoin & join)
{
data = join.data;
}
@ -329,7 +329,7 @@ private:
friend class NonJoinedBlockInputStream;
friend class JoinSource;
std::shared_ptr<AnalyzedJoin> table_join;
std::shared_ptr<TableJoin> table_join;
ASTTableJoin::Kind kind;
ASTTableJoin::Strictness strictness;

View File

@ -48,8 +48,8 @@
#include <Interpreters/getTableExpressions.h>
#include <Interpreters/JoinToSubqueryTransformVisitor.h>
#include <Interpreters/CrossToInnerJoinVisitor.h>
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/Join.h>
#include <Interpreters/TableJoin.h>
#include <Interpreters/HashJoin.h>
#include <Interpreters/JoinedTables.h>
#include <Interpreters/QueryAliasesVisitor.h>
@ -897,7 +897,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS
if (join)
{
inflating_join = true;
if (auto * hash_join = typeid_cast<Join *>(join.get()))
if (auto * hash_join = typeid_cast<HashJoin *>(join.get()))
inflating_join = isCross(hash_join->getKind());
}

View File

@ -1,6 +1,6 @@
#include <Common/typeid_cast.h>
#include <Interpreters/JoinSwitcher.h>
#include <Interpreters/Join.h>
#include <Interpreters/HashJoin.h>
#include <Interpreters/MergeJoin.h>
#include <Interpreters/join_common.h>
@ -17,13 +17,13 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column,
return std::move(column);
}
JoinSwitcher::JoinSwitcher(std::shared_ptr<AnalyzedJoin> table_join_, const Block & right_sample_block_)
JoinSwitcher::JoinSwitcher(std::shared_ptr<TableJoin> table_join_, const Block & right_sample_block_)
: limits(table_join_->sizeLimits())
, switched(false)
, table_join(table_join_)
, right_sample_block(right_sample_block_.cloneEmpty())
{
join = std::make_shared<Join>(table_join, right_sample_block);
join = std::make_shared<HashJoin>(table_join, right_sample_block);
if (!limits.hasLimits())
limits.max_bytes = table_join->defaultMaxBytes();
@ -50,7 +50,7 @@ bool JoinSwitcher::addJoinedBlock(const Block & block, bool)
void JoinSwitcher::switchJoin()
{
std::shared_ptr<Join::RightTableData> joined_data = static_cast<const Join &>(*join).getJoinedData();
std::shared_ptr<HashJoin::RightTableData> joined_data = static_cast<const HashJoin &>(*join).getJoinedData();
BlocksList right_blocks = std::move(joined_data->blocks);
/// Destroy old join & create new one. Early destroy for memory saving.

View File

@ -4,7 +4,7 @@
#include <Core/Block.h>
#include <Interpreters/IJoin.h>
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/TableJoin.h>
namespace DB
{
@ -15,7 +15,7 @@ namespace DB
class JoinSwitcher : public IJoin
{
public:
JoinSwitcher(std::shared_ptr<AnalyzedJoin> table_join_, const Block & right_sample_block_);
JoinSwitcher(std::shared_ptr<TableJoin> table_join_, const Block & right_sample_block_);
/// Add block of data from right hand of JOIN into current join object.
/// If join-in-memory memory limit exceeded switches to join-on-disk and continue with it.
@ -72,7 +72,7 @@ private:
SizeLimits limits;
bool switched;
mutable std::mutex switch_mutex;
std::shared_ptr<AnalyzedJoin> table_join;
std::shared_ptr<TableJoin> table_join;
const Block right_sample_block;
/// Change join-in-memory to join-on-disk moving right hand JOIN data from one to another.

View File

@ -4,7 +4,7 @@
#include <Core/SortCursor.h>
#include <Columns/ColumnNullable.h>
#include <Interpreters/MergeJoin.h>
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/TableJoin.h>
#include <Interpreters/sortBlock.h>
#include <Interpreters/join_common.h>
#include <DataStreams/materializeBlock.h>
@ -445,7 +445,7 @@ void MiniLSM::merge(std::function<void(const Block &)> callback)
}
MergeJoin::MergeJoin(std::shared_ptr<AnalyzedJoin> table_join_, const Block & right_sample_block_)
MergeJoin::MergeJoin(std::shared_ptr<TableJoin> table_join_, const Block & right_sample_block_)
: table_join(table_join_)
, size_limits(table_join->sizeLimits())
, right_sample_block(right_sample_block_)

View File

@ -13,7 +13,7 @@
namespace DB
{
class AnalyzedJoin;
class TableJoin;
class MergeJoinCursor;
struct MergeJoinEqualRange;
@ -48,7 +48,7 @@ struct MiniLSM
class MergeJoin : public IJoin
{
public:
MergeJoin(std::shared_ptr<AnalyzedJoin> table_join_, const Block & right_sample_block);
MergeJoin(std::shared_ptr<TableJoin> table_join_, const Block & right_sample_block);
bool addJoinedBlock(const Block & block, bool check_limits) override;
void joinBlock(Block &, ExtraBlockPtr & not_processed) override;
@ -76,7 +76,7 @@ private:
using Cache = LRUCache<size_t, Block, std::hash<size_t>, BlockByteWeight>;
mutable std::shared_mutex rwlock;
std::shared_ptr<AnalyzedJoin> table_join;
std::shared_ptr<TableJoin> table_join;
SizeLimits size_limits;
SortDescription left_sort_description;
SortDescription right_sort_description;

View File

@ -1,6 +1,6 @@
#include <Interpreters/SubqueryForSet.h>
#include <Interpreters/InterpreterSelectWithUnionQuery.h>
#include <Interpreters/Join.h>
#include <Interpreters/IJoin.h>
#include <Interpreters/MergeJoin.h>
#include <Interpreters/ExpressionActions.h>
#include <DataStreams/LazyBlockInputStream.h>

View File

@ -17,7 +17,7 @@
#include <Interpreters/OptimizeIfWithConstantConditionVisitor.h>
#include <Interpreters/RequiredSourceColumnsVisitor.h>
#include <Interpreters/GetAggregatesVisitor.h>
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/TableJoin.h>
#include <Interpreters/ExpressionActions.h> /// getSmallestColumn()
#include <Interpreters/getTableExpressions.h>
#include <Interpreters/OptimizeIfChains.h>
@ -520,7 +520,7 @@ void setJoinStrictness(ASTSelectQuery & select_query, JoinStrictness join_defaul
}
/// Find the columns that are obtained by JOIN.
void collectJoinedColumns(AnalyzedJoin & analyzed_join, const ASTSelectQuery & select_query,
void collectJoinedColumns(TableJoin & analyzed_join, const ASTSelectQuery & select_query,
const std::vector<TableWithColumnNames> & tables, const Aliases & aliases)
{
const ASTTablesInSelectQueryElement * node = select_query.join();
@ -795,7 +795,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyzeSelect(
const auto & settings = context.getSettingsRef();
const NameSet & source_columns_set = result.source_columns_set;
result.analyzed_join = std::make_shared<AnalyzedJoin>(settings, context.getTemporaryVolume());
result.analyzed_join = std::make_shared<TableJoin>(settings, context.getTemporaryVolume());
if (remove_duplicates)
renameDuplicatedColumns(select_query);

View File

@ -11,7 +11,7 @@ namespace DB
{
class ASTFunction;
class AnalyzedJoin;
class TableJoin;
class Context;
struct Settings;
struct SelectQueryOptions;
@ -20,7 +20,7 @@ using Scalars = std::map<String, Block>;
struct SyntaxAnalyzerResult
{
ConstStoragePtr storage;
std::shared_ptr<AnalyzedJoin> analyzed_join;
std::shared_ptr<TableJoin> analyzed_join;
NamesAndTypesList source_columns;
NameSet source_columns_set; /// Set of names of source_columns.

View File

@ -1,4 +1,4 @@
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/TableJoin.h>
#include <Parsers/ASTExpressionList.h>
@ -15,7 +15,7 @@ namespace ErrorCodes
{
}
AnalyzedJoin::AnalyzedJoin(const Settings & settings, VolumePtr tmp_volume_)
TableJoin::TableJoin(const Settings & settings, VolumePtr tmp_volume_)
: size_limits(SizeLimits{settings.max_rows_in_join, settings.max_bytes_in_join, settings.join_overflow_mode})
, default_max_bytes(settings.default_max_bytes_in_join)
, join_use_nulls(settings.join_use_nulls)
@ -29,7 +29,7 @@ AnalyzedJoin::AnalyzedJoin(const Settings & settings, VolumePtr tmp_volume_)
join_algorithm = JoinAlgorithm::PREFER_PARTIAL_MERGE;
}
void AnalyzedJoin::addUsingKey(const ASTPtr & ast)
void TableJoin::addUsingKey(const ASTPtr & ast)
{
key_names_left.push_back(ast->getColumnName());
key_names_right.push_back(ast->getAliasOrColumnName());
@ -42,7 +42,7 @@ void AnalyzedJoin::addUsingKey(const ASTPtr & ast)
right_key = renames[right_key];
}
void AnalyzedJoin::addOnKeys(ASTPtr & left_table_ast, ASTPtr & right_table_ast)
void TableJoin::addOnKeys(ASTPtr & left_table_ast, ASTPtr & right_table_ast)
{
key_names_left.push_back(left_table_ast->getColumnName());
key_names_right.push_back(right_table_ast->getAliasOrColumnName());
@ -52,7 +52,7 @@ void AnalyzedJoin::addOnKeys(ASTPtr & left_table_ast, ASTPtr & right_table_ast)
}
/// @return how many times right key appears in ON section.
size_t AnalyzedJoin::rightKeyInclusion(const String & name) const
size_t TableJoin::rightKeyInclusion(const String & name) const
{
if (hasUsing())
return 0;
@ -64,7 +64,7 @@ size_t AnalyzedJoin::rightKeyInclusion(const String & name) const
return count;
}
void AnalyzedJoin::deduplicateAndQualifyColumnNames(const NameSet & left_table_columns, const String & right_table_prefix)
void TableJoin::deduplicateAndQualifyColumnNames(const NameSet & left_table_columns, const String & right_table_prefix)
{
NameSet joined_columns;
NamesAndTypesList dedup_columns;
@ -90,7 +90,7 @@ void AnalyzedJoin::deduplicateAndQualifyColumnNames(const NameSet & left_table_c
columns_from_joined_table.swap(dedup_columns);
}
NameSet AnalyzedJoin::getQualifiedColumnsSet() const
NameSet TableJoin::getQualifiedColumnsSet() const
{
NameSet out;
for (const auto & names : original_names)
@ -98,7 +98,7 @@ NameSet AnalyzedJoin::getQualifiedColumnsSet() const
return out;
}
NamesWithAliases AnalyzedJoin::getNamesWithAliases(const NameSet & required_columns) const
NamesWithAliases TableJoin::getNamesWithAliases(const NameSet & required_columns) const
{
NamesWithAliases out;
for (const auto & column : required_columns)
@ -110,14 +110,14 @@ NamesWithAliases AnalyzedJoin::getNamesWithAliases(const NameSet & required_colu
return out;
}
ASTPtr AnalyzedJoin::leftKeysList() const
ASTPtr TableJoin::leftKeysList() const
{
ASTPtr keys_list = std::make_shared<ASTExpressionList>();
keys_list->children = key_asts_left;
return keys_list;
}
ASTPtr AnalyzedJoin::rightKeysList() const
ASTPtr TableJoin::rightKeysList() const
{
ASTPtr keys_list = std::make_shared<ASTExpressionList>();
if (hasOn())
@ -125,7 +125,7 @@ ASTPtr AnalyzedJoin::rightKeysList() const
return keys_list;
}
Names AnalyzedJoin::requiredJoinedNames() const
Names TableJoin::requiredJoinedNames() const
{
NameSet required_columns_set(key_names_right.begin(), key_names_right.end());
for (const auto & joined_column : columns_added_by_join)
@ -134,7 +134,7 @@ Names AnalyzedJoin::requiredJoinedNames() const
return Names(required_columns_set.begin(), required_columns_set.end());
}
NameSet AnalyzedJoin::requiredRightKeys() const
NameSet TableJoin::requiredRightKeys() const
{
NameSet required;
for (const auto & name : key_names_right)
@ -144,7 +144,7 @@ NameSet AnalyzedJoin::requiredRightKeys() const
return required;
}
NamesWithAliases AnalyzedJoin::getRequiredColumns(const Block & sample, const Names & action_required_columns) const
NamesWithAliases TableJoin::getRequiredColumns(const Block & sample, const Names & action_required_columns) const
{
NameSet required_columns(action_required_columns.begin(), action_required_columns.end());
@ -155,7 +155,7 @@ NamesWithAliases AnalyzedJoin::getRequiredColumns(const Block & sample, const Na
return getNamesWithAliases(required_columns);
}
void AnalyzedJoin::addJoinedColumn(const NameAndTypePair & joined_column)
void TableJoin::addJoinedColumn(const NameAndTypePair & joined_column)
{
if (join_use_nulls && isLeftOrFull(table_join.kind))
{
@ -166,7 +166,7 @@ void AnalyzedJoin::addJoinedColumn(const NameAndTypePair & joined_column)
columns_added_by_join.push_back(joined_column);
}
void AnalyzedJoin::addJoinedColumnsAndCorrectNullability(Block & sample_block) const
void TableJoin::addJoinedColumnsAndCorrectNullability(Block & sample_block) const
{
bool right_or_full_join = isRightOrFull(table_join.kind);
bool left_or_full_join = isLeftOrFull(table_join.kind);
@ -198,7 +198,7 @@ void AnalyzedJoin::addJoinedColumnsAndCorrectNullability(Block & sample_block) c
}
}
bool AnalyzedJoin::sameJoin(const AnalyzedJoin * x, const AnalyzedJoin * y)
bool TableJoin::sameJoin(const TableJoin * x, const TableJoin * y)
{
if (!x && !y)
return true;
@ -212,7 +212,7 @@ bool AnalyzedJoin::sameJoin(const AnalyzedJoin * x, const AnalyzedJoin * y)
&& x->columns_added_by_join == y->columns_added_by_join;
}
bool AnalyzedJoin::sameStrictnessAndKind(ASTTableJoin::Strictness strictness_, ASTTableJoin::Kind kind_) const
bool TableJoin::sameStrictnessAndKind(ASTTableJoin::Strictness strictness_, ASTTableJoin::Kind kind_) const
{
if (strictness_ == strictness() && kind_ == kind())
return true;
@ -228,7 +228,7 @@ bool AnalyzedJoin::sameStrictnessAndKind(ASTTableJoin::Strictness strictness_, A
return false;
}
bool AnalyzedJoin::allowMergeJoin() const
bool TableJoin::allowMergeJoin() const
{
bool is_any = (strictness() == ASTTableJoin::Strictness::Any);
bool is_all = (strictness() == ASTTableJoin::Strictness::All);

View File

@ -25,7 +25,7 @@ struct Settings;
class Volume;
using VolumePtr = std::shared_ptr<Volume>;
class AnalyzedJoin
class TableJoin
{
/** Query of the form `SELECT expr(x) AS k FROM t1 ANY LEFT JOIN (SELECT expr(x) AS k FROM t2) USING k`
* The join is made by column k.
@ -69,10 +69,10 @@ class AnalyzedJoin
VolumePtr tmp_volume;
public:
AnalyzedJoin(const Settings &, VolumePtr tmp_volume);
TableJoin(const Settings &, VolumePtr tmp_volume);
/// for StorageJoin
AnalyzedJoin(SizeLimits limits, bool use_nulls, ASTTableJoin::Kind kind, ASTTableJoin::Strictness strictness,
TableJoin(SizeLimits limits, bool use_nulls, ASTTableJoin::Kind kind, ASTTableJoin::Strictness strictness,
const Names & key_names_right_)
: size_limits(limits)
, default_max_bytes(0)
@ -133,7 +133,7 @@ public:
/// StorageJoin overrides key names (cause of different names qualification)
void setRightKeys(const Names & keys) { key_names_right = keys; }
static bool sameJoin(const AnalyzedJoin * x, const AnalyzedJoin * y);
static bool sameJoin(const TableJoin * x, const TableJoin * y);
};
}

View File

@ -3,7 +3,7 @@
#include <array>
#include <common/constexpr_helpers.h>
#include <Interpreters/Join.h>
#include <Interpreters/HashJoin.h>
/** Used in implementation of Join to process different data structures.
@ -15,37 +15,37 @@ namespace DB
template <ASTTableJoin::Kind kind, typename ASTTableJoin::Strictness>
struct MapGetter;
template <> struct MapGetter<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::RightAny> { using Map = Join::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Inner, ASTTableJoin::Strictness::RightAny> { using Map = Join::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Right, ASTTableJoin::Strictness::RightAny> { using Map = Join::MapsOneFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Full, ASTTableJoin::Strictness::RightAny> { using Map = Join::MapsOneFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::RightAny> { using Map = HashJoin::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Inner, ASTTableJoin::Strictness::RightAny> { using Map = HashJoin::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Right, ASTTableJoin::Strictness::RightAny> { using Map = HashJoin::MapsOneFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Full, ASTTableJoin::Strictness::RightAny> { using Map = HashJoin::MapsOneFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::Any> { using Map = Join::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Inner, ASTTableJoin::Strictness::Any> { using Map = Join::MapsOneFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Right, ASTTableJoin::Strictness::Any> { using Map = Join::MapsAllFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Full, ASTTableJoin::Strictness::Any> { using Map = Join::MapsAllFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::Any> { using Map = HashJoin::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Inner, ASTTableJoin::Strictness::Any> { using Map = HashJoin::MapsOneFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Right, ASTTableJoin::Strictness::Any> { using Map = HashJoin::MapsAllFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Full, ASTTableJoin::Strictness::Any> { using Map = HashJoin::MapsAllFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::All> { using Map = Join::MapsAll; };
template <> struct MapGetter<ASTTableJoin::Kind::Inner, ASTTableJoin::Strictness::All> { using Map = Join::MapsAll; };
template <> struct MapGetter<ASTTableJoin::Kind::Right, ASTTableJoin::Strictness::All> { using Map = Join::MapsAllFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Full, ASTTableJoin::Strictness::All> { using Map = Join::MapsAllFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::All> { using Map = HashJoin::MapsAll; };
template <> struct MapGetter<ASTTableJoin::Kind::Inner, ASTTableJoin::Strictness::All> { using Map = HashJoin::MapsAll; };
template <> struct MapGetter<ASTTableJoin::Kind::Right, ASTTableJoin::Strictness::All> { using Map = HashJoin::MapsAllFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Full, ASTTableJoin::Strictness::All> { using Map = HashJoin::MapsAllFlagged; };
/// Only SEMI LEFT and SEMI RIGHT are valid. INNER and FULL are here for templates instantiation.
template <> struct MapGetter<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::Semi> { using Map = Join::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Inner, ASTTableJoin::Strictness::Semi> { using Map = Join::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Right, ASTTableJoin::Strictness::Semi> { using Map = Join::MapsAllFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Full, ASTTableJoin::Strictness::Semi> { using Map = Join::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::Semi> { using Map = HashJoin::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Inner, ASTTableJoin::Strictness::Semi> { using Map = HashJoin::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Right, ASTTableJoin::Strictness::Semi> { using Map = HashJoin::MapsAllFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Full, ASTTableJoin::Strictness::Semi> { using Map = HashJoin::MapsOne; };
/// Only SEMI LEFT and SEMI RIGHT are valid. INNER and FULL are here for templates instantiation.
template <> struct MapGetter<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::Anti> { using Map = Join::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Inner, ASTTableJoin::Strictness::Anti> { using Map = Join::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Right, ASTTableJoin::Strictness::Anti> { using Map = Join::MapsAllFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Full, ASTTableJoin::Strictness::Anti> { using Map = Join::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Left, ASTTableJoin::Strictness::Anti> { using Map = HashJoin::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Inner, ASTTableJoin::Strictness::Anti> { using Map = HashJoin::MapsOne; };
template <> struct MapGetter<ASTTableJoin::Kind::Right, ASTTableJoin::Strictness::Anti> { using Map = HashJoin::MapsAllFlagged; };
template <> struct MapGetter<ASTTableJoin::Kind::Full, ASTTableJoin::Strictness::Anti> { using Map = HashJoin::MapsOne; };
template <ASTTableJoin::Kind kind>
struct MapGetter<kind, ASTTableJoin::Strictness::Asof>
{
using Map = Join::MapsAsof;
using Map = HashJoin::MapsAsof;
};
@ -66,7 +66,7 @@ static constexpr std::array<ASTTableJoin::Kind, 4> KINDS = {
};
/// Init specified join map
inline bool joinDispatchInit(ASTTableJoin::Kind kind, ASTTableJoin::Strictness strictness, Join::MapsVariant & maps)
inline bool joinDispatchInit(ASTTableJoin::Kind kind, ASTTableJoin::Strictness strictness, HashJoin::MapsVariant & maps)
{
return static_for<0, KINDS.size() * STRICTNESSES.size()>([&](auto ij)
{

View File

@ -5,7 +5,7 @@
#include <DataStreams/IBlockOutputStream.h>
#include <Interpreters/Set.h>
#include <Interpreters/Join.h>
#include <Interpreters/IJoin.h>
#include <Storages/IStorage.h>
#include <iomanip>

View File

@ -1,7 +1,7 @@
#include <Storages/ReadInOrderOptimizer.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/MergeTree/StorageFromMergeTreeDataPart.h>
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/TableJoin.h>
#include <Functions/IFunction.h>
namespace DB

View File

@ -1,6 +1,6 @@
#include <Storages/StorageJoin.h>
#include <Storages/StorageFactory.h>
#include <Interpreters/Join.h>
#include <Interpreters/HashJoin.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTSetQuery.h>
@ -9,7 +9,7 @@
#include <DataStreams/IBlockInputStream.h>
#include <DataTypes/NestedUtils.h>
#include <Interpreters/joinDispatch.h>
#include <Interpreters/AnalyzedJoin.h>
#include <Interpreters/TableJoin.h>
#include <Common/assert_cast.h>
#include <Common/quoteString.h>
@ -57,8 +57,8 @@ StorageJoin::StorageJoin(
if (!getColumns().hasPhysical(key))
throw Exception{"Key column (" + key + ") does not exist in table declaration.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE};
table_join = std::make_shared<AnalyzedJoin>(limits, use_nulls, kind, strictness, key_names);
join = std::make_shared<Join>(table_join, getSampleBlock().sortColumns(), overwrite);
table_join = std::make_shared<TableJoin>(limits, use_nulls, kind, strictness, key_names);
join = std::make_shared<HashJoin>(table_join, getSampleBlock().sortColumns(), overwrite);
restore();
}
@ -70,11 +70,11 @@ void StorageJoin::truncate(const ASTPtr &, const Context &, TableStructureWriteL
Poco::File(path + "tmp/").createDirectories();
increment = 0;
join = std::make_shared<Join>(table_join, getSampleBlock().sortColumns(), overwrite);
join = std::make_shared<HashJoin>(table_join, getSampleBlock().sortColumns(), overwrite);
}
HashJoinPtr StorageJoin::getJoin(std::shared_ptr<AnalyzedJoin> analyzed_join) const
HashJoinPtr StorageJoin::getJoin(std::shared_ptr<TableJoin> analyzed_join) const
{
if (!analyzed_join->sameStrictnessAndKind(strictness, kind))
throw Exception("Table " + getStorageID().getNameForLogs() + " has incompatible type of JOIN.", ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN);
@ -89,7 +89,7 @@ HashJoinPtr StorageJoin::getJoin(std::shared_ptr<AnalyzedJoin> analyzed_join) co
/// Some HACK to remove wrong names qualifiers: table.column -> column.
analyzed_join->setRightKeys(key_names);
HashJoinPtr join_clone = std::make_shared<Join>(analyzed_join, getSampleBlock().sortColumns());
HashJoinPtr join_clone = std::make_shared<HashJoin>(analyzed_join, getSampleBlock().sortColumns());
join_clone->reuseJoinedData(*join);
return join_clone;
}
@ -244,7 +244,7 @@ size_t rawSize(const StringRef & t)
class JoinSource : public SourceWithProgress
{
public:
JoinSource(const Join & parent_, UInt64 max_block_size_, Block sample_block_)
JoinSource(const HashJoin & parent_, UInt64 max_block_size_, Block sample_block_)
: SourceWithProgress(sample_block_)
, parent(parent_)
, lock(parent.data->rwlock)
@ -287,7 +287,7 @@ protected:
}
private:
const Join & parent;
const HashJoin & parent;
std::shared_lock<std::shared_mutex> lock;
UInt64 max_block_size;
Block sample_block;
@ -326,7 +326,7 @@ private:
switch (parent.data->type)
{
#define M(TYPE) \
case Join::Type::TYPE: \
case HashJoin::Type::TYPE: \
rows_added = fillColumns<KIND, STRICTNESS>(*maps.TYPE); \
break;
APPLY_FOR_JOIN_VARIANTS_LIMITED(M)

View File

@ -9,9 +9,9 @@
namespace DB
{
class AnalyzedJoin;
class Join;
using HashJoinPtr = std::shared_ptr<Join>;
class TableJoin;
class HashJoin;
using HashJoinPtr = std::shared_ptr<HashJoin>;
/** Allows you save the state for later use on the right side of the JOIN.
@ -31,7 +31,7 @@ public:
/// Access the innards.
HashJoinPtr & getJoin() { return join; }
HashJoinPtr getJoin(std::shared_ptr<AnalyzedJoin> analyzed_join) const;
HashJoinPtr getJoin(std::shared_ptr<TableJoin> analyzed_join) const;
/// Verify that the data structure is suitable for implementing this type of JOIN.
void assertCompatible(ASTTableJoin::Kind kind_, ASTTableJoin::Strictness strictness_) const;
@ -53,7 +53,7 @@ private:
ASTTableJoin::Strictness strictness; /// ANY | ALL
bool overwrite;
std::shared_ptr<AnalyzedJoin> table_join;
std::shared_ptr<TableJoin> table_join;
HashJoinPtr join;
void insertBlock(const Block & block) override;

View File

@ -7,7 +7,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>decimals</table>
</clickhouse>
</source>
@ -45,7 +45,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>decimals</table>
</clickhouse>
</source>
@ -83,7 +83,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>decimals</table>
</clickhouse>
</source>
@ -121,7 +121,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>decimals</table>
</clickhouse>
</source>
@ -162,7 +162,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>decimals</table>
</clickhouse>
</source>

View File

@ -7,7 +7,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>ints</table>
</clickhouse>
</source>
@ -70,7 +70,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>ints</table>
</clickhouse>
</source>
@ -133,7 +133,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>ints</table>
</clickhouse>
</source>
@ -196,7 +196,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>ints</table>
</clickhouse>
</source>
@ -259,7 +259,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>ints</table>
</clickhouse>
</source>
@ -325,7 +325,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>ints</table>
</clickhouse>
</source>

View File

@ -7,7 +7,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>strings</table>
</clickhouse>
</source>
@ -35,7 +35,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>strings</table>
</clickhouse>
</source>
@ -63,7 +63,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>strings</table>
</clickhouse>
</source>
@ -91,7 +91,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>strings</table>
</clickhouse>
</source>
@ -122,7 +122,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>strings</table>
</clickhouse>
</source>
@ -153,7 +153,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>strings</table>
</clickhouse>
</source>
@ -184,7 +184,7 @@
<port>9000</port>
<user>default</user>
<password></password>
<db>test_00950</db>
<db>system</db>
<table>strings</table>
</clickhouse>
</source>

View File

@ -1,6 +1,5 @@
-- Must use `test_00950` database and these tables - they're configured in tests/*_dictionary.xml
create database if not exists test_00950;
use test_00950;
-- Must use `system` database and these tables - they're configured in tests/*_dictionary.xml
use system;
drop table if exists ints;
drop table if exists strings;
drop table if exists decimals;
@ -270,7 +269,14 @@ select 'dictGetOrDefault', 'complex_cache_decimals' as dict_name, tuple(toUInt64
dictGetOrDefault(dict_name, 'd64', k, toDecimal64(42, 6)),
dictGetOrDefault(dict_name, 'd128', k, toDecimal128(42, 1));
drop table ints;
drop table strings;
drop table decimals;
drop database test_00950;
--
-- Keep the tables, so that the dictionaries can be reloaded correctly and
-- SYSTEM RELOAD DICTIONARIES doesn't break.
-- We could also:
-- * drop the dictionaries -- not possible, they are configured in a .xml;
-- * switch dictionaries to DDL syntax so that they can be dropped -- tedious,
-- because there are a couple dozens of them, and also we need to have some
-- .xml dictionaries in tests so that we test backward compatibility with this
-- format;
-- * unload dictionaries -- no command for that.
--

View File

@ -1,44 +1,44 @@
EXISTS database_for_dict.t;
EXISTS TABLE database_for_dict.t;
EXISTS DICTIONARY database_for_dict.t;
EXISTS db_01048.t_01048;
EXISTS TABLE db_01048.t_01048;
EXISTS DICTIONARY db_01048.t_01048;
DROP DATABASE IF EXISTS database_for_dict;
CREATE DATABASE database_for_dict Engine = Ordinary;
DROP DATABASE IF EXISTS db_01048;
CREATE DATABASE db_01048 Engine = Ordinary;
DROP TABLE IF EXISTS database_for_dict.t;
EXISTS database_for_dict.t;
EXISTS TABLE database_for_dict.t;
EXISTS DICTIONARY database_for_dict.t;
DROP TABLE IF EXISTS db_01048.t_01048;
EXISTS db_01048.t_01048;
EXISTS TABLE db_01048.t_01048;
EXISTS DICTIONARY db_01048.t_01048;
CREATE TABLE database_for_dict.t (x UInt8) ENGINE = Memory;
EXISTS database_for_dict.t;
EXISTS TABLE database_for_dict.t;
EXISTS DICTIONARY database_for_dict.t;
CREATE TABLE db_01048.t_01048 (x UInt8) ENGINE = Memory;
EXISTS db_01048.t_01048;
EXISTS TABLE db_01048.t_01048;
EXISTS DICTIONARY db_01048.t_01048;
DROP TABLE database_for_dict.t;
EXISTS database_for_dict.t;
EXISTS TABLE database_for_dict.t;
EXISTS DICTIONARY database_for_dict.t;
DROP TABLE db_01048.t_01048;
EXISTS db_01048.t_01048;
EXISTS TABLE db_01048.t_01048;
EXISTS DICTIONARY db_01048.t_01048;
DROP DICTIONARY IF EXISTS t;
CREATE TEMPORARY TABLE t (x UInt8);
EXISTS t; -- Does not work for temporary tables. Maybe have to fix.
EXISTS TABLE t;
EXISTS DICTIONARY t;
DROP DICTIONARY IF EXISTS t_01048;
CREATE TEMPORARY TABLE t_01048 (x UInt8);
EXISTS t_01048; -- Does not work for temporary tables. Maybe have to fix.
EXISTS TABLE t_01048;
EXISTS DICTIONARY t_01048;
CREATE DICTIONARY database_for_dict.t (k UInt64, v String) PRIMARY KEY k LAYOUT(FLAT()) SOURCE(HTTP(URL 'http://example.test/' FORMAT TSV)) LIFETIME(1000);
EXISTS database_for_dict.t;
EXISTS TABLE database_for_dict.t; -- Dictionaries are tables as well. But not all tables are dictionaries.
EXISTS DICTIONARY database_for_dict.t;
CREATE DICTIONARY db_01048.t_01048 (k UInt64, v String) PRIMARY KEY k LAYOUT(FLAT()) SOURCE(HTTP(URL 'http://example.test/' FORMAT TSV)) LIFETIME(1000);
EXISTS db_01048.t_01048;
EXISTS TABLE db_01048.t_01048; -- Dictionaries are tables as well. But not all tables are dictionaries.
EXISTS DICTIONARY db_01048.t_01048;
-- But dictionary-tables cannot be dropped as usual tables.
DROP TABLE database_for_dict.t; -- { serverError 60 }
DROP DICTIONARY database_for_dict.t;
EXISTS database_for_dict.t;
EXISTS TABLE database_for_dict.t;
EXISTS DICTIONARY database_for_dict.t;
DROP TABLE db_01048.t_01048; -- { serverError 60 }
DROP DICTIONARY db_01048.t_01048;
EXISTS db_01048.t_01048;
EXISTS TABLE db_01048.t_01048;
EXISTS DICTIONARY db_01048.t_01048;
DROP DATABASE database_for_dict;
EXISTS database_for_dict.t;
EXISTS TABLE database_for_dict.t;
EXISTS DICTIONARY database_for_dict.t;
DROP DATABASE db_01048;
EXISTS db_01048.t_01048;
EXISTS TABLE db_01048.t_01048;
EXISTS DICTIONARY db_01048.t_01048;

View File

@ -1,7 +1,7 @@
CREATE TABLE t (a Int) ENGINE = Log;
ATTACH TABLE t; -- { serverError 57 }
ATTACH TABLE IF NOT EXISTS t;
DETACH TABLE t;
ATTACH TABLE IF NOT EXISTS t;
EXISTS TABLE t;
DROP TABLE t;
CREATE TABLE aine (a Int) ENGINE = Log;
ATTACH TABLE aine; -- { serverError 57 }
ATTACH TABLE IF NOT EXISTS aine;
DETACH TABLE aine;
ATTACH TABLE IF NOT EXISTS aine;
EXISTS TABLE aine;
DROP TABLE aine;