diff --git a/README.md b/README.md index 1c6a021c00c..3329a98877f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,3 @@ ClickHouse® is an open-source column-oriented database management system that a * [Code Browser](https://clickhouse.tech/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation. * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any. * You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person. - -## Upcoming Events -* [Chinese ClickHouse Meetup (online)](http://hdxu.cn/8KxZE) on 6 February 2021. diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 18449dad855..491ffe6a775 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -152,7 +152,7 @@ static void signalHandler(int sig, siginfo_t * info, void * context) if (sig != SIGTSTP) /// This signal is used for debugging. { /// The time that is usually enough for separate thread to print info into log. - sleepForSeconds(10); + sleepForSeconds(20); /// FIXME: use some feedback from threads that process stacktrace call_default_signal_handler(sig); } @@ -311,7 +311,8 @@ private: if (stack_trace.getSize()) { /// Write bare stack trace (addresses) just in case if we will fail to print symbolized stack trace. - /// NOTE This still require memory allocations and mutex lock inside logger. BTW we can also print it to stderr using write syscalls. + /// NOTE: This still require memory allocations and mutex lock inside logger. + /// BTW we can also print it to stderr using write syscalls. std::stringstream bare_stacktrace; bare_stacktrace << "Stack trace:"; @@ -324,7 +325,7 @@ private: /// Write symbolized stack trace line by line for better grep-ability. stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); -#if defined(__linux__) +#if defined(OS_LINUX) /// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace. String calculated_binary_hash = getHashOfLoadedBinaryHex(); if (daemon.stored_binary_hash.empty()) diff --git a/docker/server/alpine-build.sh b/docker/server/alpine-build.sh index 0142149b5bd..329888f2fcb 100755 --- a/docker/server/alpine-build.sh +++ b/docker/server/alpine-build.sh @@ -54,8 +54,10 @@ docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libm.so.6 "${CONTAIN docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libpthread.so.0 "${CONTAINER_ROOT_FOLDER}/lib" docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/librt.so.1 "${CONTAINER_ROOT_FOLDER}/lib" docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libnss_dns.so.2 "${CONTAINER_ROOT_FOLDER}/lib" +docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libnss_files.so.2 "${CONTAINER_ROOT_FOLDER}/lib" docker cp -L "${ubuntu20image}":/lib/x86_64-linux-gnu/libresolv.so.2 "${CONTAINER_ROOT_FOLDER}/lib" docker cp -L "${ubuntu20image}":/lib64/ld-linux-x86-64.so.2 "${CONTAINER_ROOT_FOLDER}/lib64" +docker cp -L "${ubuntu20image}":/etc/nsswitch.conf "${CONTAINER_ROOT_FOLDER}/etc" docker build "$DOCKER_BUILD_FOLDER" -f Dockerfile.alpine -t "${DOCKER_IMAGE}:${VERSION}-alpine" --pull rm -rf "$CONTAINER_ROOT_FOLDER" diff --git a/docs/en/sql-reference/window-functions/index.md b/docs/en/sql-reference/window-functions/index.md index 5a6f13226a5..46f7ed3824e 100644 --- a/docs/en/sql-reference/window-functions/index.md +++ b/docs/en/sql-reference/window-functions/index.md @@ -10,33 +10,49 @@ This is an experimental feature that is currently in development and is not read for general use. It will change in unpredictable backwards-incompatible ways in the future releases. Set `allow_experimental_window_functions = 1` to enable it. -ClickHouse currently supports calculation of aggregate functions over a window. -Pure window functions such as `rank`, `lag`, `lead` and so on are not yet supported. +ClickHouse supports the standard grammar for defining windows and window functions. The following features are currently supported: -The window can be specified either with an `OVER` clause or with a separate -`WINDOW` clause. - -Only two variants of frame are supported, `ROWS` and `RANGE`. Offsets for the `RANGE` frame are not yet supported. +| Feature | Support or workaround | +| --------| ----------| +| ad hoc window specification (`count(*) over (partition by id order by time desc)`) | yes | +| `WINDOW` clause (`select ... from table window w as (partiton by id)`) | yes | +| `ROWS` frame | yes | +| `RANGE` frame | yes, it is the default | +| `GROUPS` frame | no | +| Calculating aggregate functions over a frame (`sum(value) over (order by time)`) | all aggregate functions are supported | +| `rank()`, `dense_rank()`, `row_number()` | yes | +| `lag/lead(value, offset)` | no, replace with `any(value) over (.... rows between preceding and preceding)`, or `following` for `lead`| ## References ### GitHub Issues + The roadmap for the initial support of window functions is [in this issue](https://github.com/ClickHouse/ClickHouse/issues/18097). All GitHub issues related to window funtions have the [comp-window-functions](https://github.com/ClickHouse/ClickHouse/labels/comp-window-functions) tag. ### Tests + These tests contain the examples of the currently supported grammar: + https://github.com/ClickHouse/ClickHouse/blob/master/tests/performance/window_functions.xml + https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/01591_window_functions.sql ### Postgres Docs + https://www.postgresql.org/docs/current/sql-select.html#SQL-WINDOW + https://www.postgresql.org/docs/devel/sql-expressions.html#SYNTAX-WINDOW-FUNCTIONS + https://www.postgresql.org/docs/devel/functions-window.html + https://www.postgresql.org/docs/devel/tutorial-window.html ### MySQL Docs + https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html + https://dev.mysql.com/doc/refman/8.0/en/window-functions-usage.html + https://dev.mysql.com/doc/refman/8.0/en/window-functions-frames.html diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index f6261079287..ddc971032b6 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -30,6 +30,12 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +template class DecimalPaddedPODArray; +template class DecimalPaddedPODArray; +template class DecimalPaddedPODArray; +template class DecimalPaddedPODArray; +template class DecimalPaddedPODArray; + template int ColumnDecimal::compareAt(size_t n, size_t m, const IColumn & rhs_, int) const { @@ -370,4 +376,5 @@ template class ColumnDecimal; template class ColumnDecimal; template class ColumnDecimal; template class ColumnDecimal; + } diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 1578633c13d..ef841292a7d 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -50,6 +50,14 @@ private: UInt32 scale; }; +/// Prevent implicit template instantiation of DecimalPaddedPODArray for common decimal types + +extern template class DecimalPaddedPODArray; +extern template class DecimalPaddedPODArray; +extern template class DecimalPaddedPODArray; +extern template class DecimalPaddedPODArray; +extern template class DecimalPaddedPODArray; + /// A ColumnVector for Decimals template class ColumnDecimal final : public COWHelper> @@ -215,4 +223,14 @@ ColumnPtr ColumnDecimal::indexImpl(const PaddedPODArray & indexes, size return res; } + +/// Prevent implicit template instantiation of ColumnDecimal for common decimal types + +extern template class ColumnDecimal; +extern template class ColumnDecimal; +extern template class ColumnDecimal; +extern template class ColumnDecimal; +extern template class ColumnDecimal; + + } diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 1374b049ccf..b8bfef7258e 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -563,4 +563,5 @@ template class ColumnVector; template class ColumnVector; template class ColumnVector; template class ColumnVector; + } diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 623a828a110..f0aa4a3bab5 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -347,4 +347,21 @@ ColumnPtr ColumnVector::indexImpl(const PaddedPODArray & indexes, size_ return res; } +/// Prevent implicit template instantiation of ColumnVector for common types + +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; +extern template class ColumnVector; + } diff --git a/src/Common/Allocator.cpp b/src/Common/Allocator.cpp index 08c275abfc2..5a66ddb63a2 100644 --- a/src/Common/Allocator.cpp +++ b/src/Common/Allocator.cpp @@ -19,3 +19,8 @@ */ __attribute__((__weak__)) extern const size_t MMAP_THRESHOLD = 16384; #endif + +template class Allocator; +template class Allocator; +template class Allocator; +template class Allocator; diff --git a/src/Common/Allocator.h b/src/Common/Allocator.h index a499f4a442b..e3c6ddf9ff4 100644 --- a/src/Common/Allocator.h +++ b/src/Common/Allocator.h @@ -352,6 +352,12 @@ template constexpr size_t allocatorInitialBytes> = initial_bytes; +/// Prevent implicit template instantiation of Allocator + +extern template class Allocator; +extern template class Allocator; +extern template class Allocator; +extern template class Allocator; #if !__clang__ #pragma GCC diagnostic pop diff --git a/src/Common/Dwarf.cpp b/src/Common/Dwarf.cpp index 7a697a2c9ef..d0b3244dac2 100644 --- a/src/Common/Dwarf.cpp +++ b/src/Common/Dwarf.cpp @@ -19,8 +19,6 @@ /** This file was edited for ClickHouse. */ -#include - #include #include @@ -43,6 +41,7 @@ #define DW_FORM_ref4 0x13 #define DW_FORM_data8 0x07 #define DW_FORM_ref8 0x14 +#define DW_FORM_ref_sig8 0x20 #define DW_FORM_sdata 0x0d #define DW_FORM_udata 0x0f #define DW_FORM_ref_udata 0x15 @@ -54,9 +53,24 @@ #define DW_FORM_strp 0x0e #define DW_FORM_indirect 0x16 #define DW_TAG_compile_unit 0x11 +#define DW_TAG_subprogram 0x2e +#define DW_TAG_try_block 0x32 +#define DW_TAG_catch_block 0x25 +#define DW_TAG_entry_point 0x03 +#define DW_TAG_common_block 0x1a +#define DW_TAG_lexical_block 0x0b #define DW_AT_stmt_list 0x10 #define DW_AT_comp_dir 0x1b #define DW_AT_name 0x03 +#define DW_AT_high_pc 0x12 +#define DW_AT_low_pc 0x11 +#define DW_AT_entry_pc 0x52 +#define DW_AT_ranges 0x55 +#define DW_AT_abstract_origin 0x31 +#define DW_AT_call_line 0x59 +#define DW_AT_call_file 0x58 +#define DW_AT_linkage_name 0x6e +#define DW_AT_specification 0x47 #define DW_LNE_define_file 0x03 #define DW_LNS_copy 0x01 #define DW_LNS_advance_pc 0x02 @@ -84,7 +98,7 @@ namespace ErrorCodes } -Dwarf::Dwarf(const Elf & elf) : elf_(&elf) +Dwarf::Dwarf(const std::shared_ptr & elf) : elf_(elf) { init(); } @@ -99,6 +113,10 @@ Dwarf::Section::Section(std::string_view d) : is64Bit_(false), data_(d) namespace { +// Maximum number of DIEAbbreviation to cache in a compilation unit. Used to +// speed up inline function lookup. +const uint32_t kMaxAbbreviationEntries = 1000; + // All following read* functions read from a std::string_view, advancing the // std::string_view, and aborting if there's not enough room. @@ -158,7 +176,7 @@ uint64_t readOffset(std::string_view & sp, bool is64Bit) // Read "len" bytes std::string_view readBytes(std::string_view & sp, uint64_t len) { - SAFE_CHECK(len >= sp.size(), "invalid string length"); + SAFE_CHECK(len <= sp.size(), "invalid string length: " + std::to_string(len) + " vs. " + std::to_string(sp.size())); std::string_view ret(sp.data(), len); sp.remove_prefix(len); return ret; @@ -364,15 +382,18 @@ void Dwarf::init() || !getSection(".debug_line", &line_) || !getSection(".debug_str", &strings_)) { - elf_ = nullptr; + elf_.reset(); return; } // Optional: fast address range lookup. If missing .debug_info can // be used - but it's much slower (linear scan). getSection(".debug_aranges", &aranges_); + + getSection(".debug_ranges", &ranges_); } +// static bool Dwarf::readAbbreviation(std::string_view & section, DIEAbbreviation & abbr) { // abbreviation code @@ -384,14 +405,14 @@ bool Dwarf::readAbbreviation(std::string_view & section, DIEAbbreviation & abbr) abbr.tag = readULEB(section); // does this entry have children? - abbr.hasChildren = (read(section) != DW_CHILDREN_no); + abbr.has_children = (read(section) != DW_CHILDREN_no); // attributes const char * attribute_begin = section.data(); for (;;) { SAFE_CHECK(!section.empty(), "invalid attribute section"); - auto attr = readAttribute(section); + auto attr = readAttributeSpec(section); if (attr.name == 0 && attr.form == 0) break; } @@ -400,11 +421,161 @@ bool Dwarf::readAbbreviation(std::string_view & section, DIEAbbreviation & abbr) return true; } -Dwarf::DIEAbbreviation::Attribute Dwarf::readAttribute(std::string_view & sp) +// static +void Dwarf::readCompilationUnitAbbrs(std::string_view abbrev, CompilationUnit & cu) +{ + abbrev.remove_prefix(cu.abbrev_offset); + + DIEAbbreviation abbr; + while (readAbbreviation(abbrev, abbr)) + { + // Abbreviation code 0 is reserved for null debugging information entries. + if (abbr.code != 0 && abbr.code <= kMaxAbbreviationEntries) + { + cu.abbr_cache[abbr.code - 1] = abbr; + } + } +} + +size_t Dwarf::forEachChild(const CompilationUnit & cu, const Die & die, std::function f) const +{ + size_t next_die_offset = forEachAttribute(cu, die, [&](const Attribute &) { return true; }); + if (!die.abbr.has_children) + { + return next_die_offset; + } + + auto child_die = getDieAtOffset(cu, next_die_offset); + while (child_die.code != 0) + { + if (!f(child_die)) + { + return child_die.offset; + } + + // NOTE: Don't run `f` over grandchildren, just skip over them. + size_t sibling_offset = forEachChild(cu, child_die, [](const Die &) { return true; }); + child_die = getDieAtOffset(cu, sibling_offset); + } + + // childDie is now a dummy die whose offset is to the code 0 marking the + // end of the children. Need to add one to get the offset of the next die. + return child_die.offset + 1; +} + +/* + * Iterate over all attributes of the given DIE, calling the given callable + * for each. Iteration is stopped early if any of the calls return false. + */ +size_t Dwarf::forEachAttribute(const CompilationUnit & cu, const Die & die, std::function f) const +{ + auto attrs = die.abbr.attributes; + auto values = std::string_view{info_.data() + die.offset + die.attr_offset, cu.offset + cu.size - die.offset - die.attr_offset}; + while (auto spec = readAttributeSpec(attrs)) + { + auto attr = readAttribute(die, spec, values); + if (!f(attr)) + { + return static_cast(-1); + } + } + return values.data() - info_.data(); +} + +Dwarf::Attribute Dwarf::readAttribute(const Die & die, AttributeSpec spec, std::string_view & info) const +{ + switch (spec.form) + { + case DW_FORM_addr: + return {spec, die, read(info)}; + case DW_FORM_block1: + return {spec, die, readBytes(info, read(info))}; + case DW_FORM_block2: + return {spec, die, readBytes(info, read(info))}; + case DW_FORM_block4: + return {spec, die, readBytes(info, read(info))}; + case DW_FORM_block: + [[fallthrough]]; + case DW_FORM_exprloc: + return {spec, die, readBytes(info, readULEB(info))}; + case DW_FORM_data1: + [[fallthrough]]; + case DW_FORM_ref1: + return {spec, die, read(info)}; + case DW_FORM_data2: + [[fallthrough]]; + case DW_FORM_ref2: + return {spec, die, read(info)}; + case DW_FORM_data4: + [[fallthrough]]; + case DW_FORM_ref4: + return {spec, die, read(info)}; + case DW_FORM_data8: + [[fallthrough]]; + case DW_FORM_ref8: + [[fallthrough]]; + case DW_FORM_ref_sig8: + return {spec, die, read(info)}; + case DW_FORM_sdata: + return {spec, die, uint64_t(readSLEB(info))}; + case DW_FORM_udata: + [[fallthrough]]; + case DW_FORM_ref_udata: + return {spec, die, readULEB(info)}; + case DW_FORM_flag: + return {spec, die, read(info)}; + case DW_FORM_flag_present: + return {spec, die, 1u}; + case DW_FORM_sec_offset: + [[fallthrough]]; + case DW_FORM_ref_addr: + return {spec, die, readOffset(info, die.is64Bit)}; + case DW_FORM_string: + return {spec, die, readNullTerminated(info)}; + case DW_FORM_strp: + return {spec, die, getStringFromStringSection(readOffset(info, die.is64Bit))}; + case DW_FORM_indirect: // form is explicitly specified + // Update spec with the actual FORM. + spec.form = readULEB(info); + return readAttribute(die, spec, info); + default: + SAFE_CHECK(false, "invalid attribute form"); + } + + return {spec, die, 0u}; +} + +// static +Dwarf::AttributeSpec Dwarf::readAttributeSpec(std::string_view & sp) { return {readULEB(sp), readULEB(sp)}; } +// static +Dwarf::CompilationUnit Dwarf::getCompilationUnit(std::string_view info, uint64_t offset) +{ + SAFE_CHECK(offset < info.size(), "unexpected offset"); + CompilationUnit cu; + std::string_view chunk(info); + cu.offset = offset; + chunk.remove_prefix(offset); + + auto initial_length = read(chunk); + cu.is64Bit = (initial_length == uint32_t(-1)); + cu.size = cu.is64Bit ? read(chunk) : initial_length; + SAFE_CHECK(cu.size <= chunk.size(), "invalid chunk size"); + cu.size += cu.is64Bit ? 12 : 4; + + cu.version = read(chunk); + SAFE_CHECK(cu.version >= 2 && cu.version <= 4, "invalid info version"); + cu.abbrev_offset = readOffset(chunk, cu.is64Bit); + cu.addr_size = read(chunk); + SAFE_CHECK(cu.addr_size == sizeof(uintptr_t), "invalid address size"); + + cu.first_die = chunk.data() - info.data(); + return cu; +} + Dwarf::DIEAbbreviation Dwarf::getAbbreviation(uint64_t code, uint64_t offset) const { // Linear search in the .debug_abbrev section, starting at offset @@ -516,104 +687,411 @@ bool Dwarf::findDebugInfoOffset(uintptr_t address, std::string_view aranges, uin return false; } +Dwarf::Die Dwarf::getDieAtOffset(const CompilationUnit & cu, uint64_t offset) const +{ + SAFE_CHECK(offset < info_.size(), "unexpected offset"); + Die die; + std::string_view sp{info_.data() + offset, cu.offset + cu.size - offset}; + die.offset = offset; + die.is64Bit = cu.is64Bit; + auto code = readULEB(sp); + die.code = code; + if (code == 0) + { + return die; + } + die.attr_offset = sp.data() - info_.data() - offset; + die.abbr = !cu.abbr_cache.empty() && die.code < kMaxAbbreviationEntries ? cu.abbr_cache[die.code - 1] + : getAbbreviation(die.code, cu.abbrev_offset); + + return die; +} + +Dwarf::Die Dwarf::findDefinitionDie(const CompilationUnit & cu, const Die & die) const +{ + // Find the real definition instead of declaration. + // DW_AT_specification: Incomplete, non-defining, or separate declaration + // corresponding to a declaration + auto offset = getAttribute(cu, die, DW_AT_specification); + if (!offset) + { + return die; + } + return getDieAtOffset(cu, cu.offset + offset.value()); +} + /** * Find the @locationInfo for @address in the compilation unit represented * by the @sp .debug_info entry. * Returns whether the address was found. * Advances @sp to the next entry in .debug_info. */ -bool Dwarf::findLocation(uintptr_t address, std::string_view & infoEntry, LocationInfo & locationInfo) const +bool Dwarf::findLocation( + uintptr_t address, + const LocationInfoMode mode, + CompilationUnit & cu, + LocationInfo & info, + std::vector & inline_frames) const { - // For each compilation unit compiled with a DWARF producer, a - // contribution is made to the .debug_info section of the object - // file. Each such contribution consists of a compilation unit - // header (see Section 7.5.1.1) followed by a single - // DW_TAG_compile_unit or DW_TAG_partial_unit debugging information - // entry, together with its children. - - // 7.5.1.1 Compilation Unit Header - // 1. unit_length (4B or 12B): read by Section::next - // 2. version (2B) - // 3. debug_abbrev_offset (4B or 8B): offset into the .debug_abbrev section - // 4. address_size (1B) - - Section debug_info_section(infoEntry); - std::string_view chunk; - SAFE_CHECK(debug_info_section.next(chunk), "invalid debug info"); - - auto version = read(chunk); - SAFE_CHECK(version >= 2 && version <= 4, "invalid info version"); - uint64_t abbrev_offset = readOffset(chunk, debug_info_section.is64Bit()); - auto address_size = read(chunk); - SAFE_CHECK(address_size == sizeof(uintptr_t), "invalid address size"); - - // We survived so far. The first (and only) DIE should be DW_TAG_compile_unit - // NOTE: - binutils <= 2.25 does not issue DW_TAG_partial_unit. - // - dwarf compression tools like `dwz` may generate it. - // TODO(tudorb): Handle DW_TAG_partial_unit? - auto code = readULEB(chunk); - SAFE_CHECK(code != 0, "invalid code"); - auto abbr = getAbbreviation(code, abbrev_offset); - SAFE_CHECK(abbr.tag == DW_TAG_compile_unit, "expecting compile unit entry"); - // Skip children entries, remove_prefix to the next compilation unit entry. - infoEntry.remove_prefix(chunk.end() - infoEntry.begin()); + Die die = getDieAtOffset(cu, cu.first_die); + // Partial compilation unit (DW_TAG_partial_unit) is not supported. + SAFE_CHECK(die.abbr.tag == DW_TAG_compile_unit, "expecting compile unit entry"); // Read attributes, extracting the few we care about - bool found_line_offset = false; - uint64_t line_offset = 0; + std::optional line_offset = 0; std::string_view compilation_directory; - std::string_view main_file_name; + std::optional main_file_name; + std::optional base_addr_cu; - DIEAbbreviation::Attribute attr; - std::string_view attributes = abbr.attributes; - for (;;) + forEachAttribute(cu, die, [&](const Attribute & attr) { - attr = readAttribute(attributes); - if (attr.name == 0 && attr.form == 0) - { - break; - } - auto val = readAttributeValue(chunk, attr.form, debug_info_section.is64Bit()); - switch (attr.name) + switch (attr.spec.name) { case DW_AT_stmt_list: // Offset in .debug_line for the line number VM program for this // compilation unit - line_offset = std::get(val); - found_line_offset = true; + line_offset = std::get(attr.attr_value); break; case DW_AT_comp_dir: // Compilation directory - compilation_directory = std::get(val); + compilation_directory = std::get(attr.attr_value); break; case DW_AT_name: // File name of main file being compiled - main_file_name = std::get(val); + main_file_name = std::get(attr.attr_value); + break; + case DW_AT_low_pc: + case DW_AT_entry_pc: + // 2.17.1: historically DW_AT_low_pc was used. DW_AT_entry_pc was + // introduced in DWARF3. Support either to determine the base address of + // the CU. + base_addr_cu = std::get(attr.attr_value); break; } - } + // Iterate through all attributes until find all above. + return true; + }); - if (!main_file_name.empty()) + if (main_file_name) { - locationInfo.hasMainFile = true; - locationInfo.mainFile = Path(compilation_directory, "", main_file_name); + info.has_main_file = true; + info.main_file = Path(compilation_directory, "", *main_file_name); } - if (!found_line_offset) + if (!line_offset) { return false; } std::string_view line_section(line_); - line_section.remove_prefix(line_offset); + line_section.remove_prefix(*line_offset); LineNumberVM line_vm(line_section, compilation_directory); // Execute line number VM program to find file and line - locationInfo.hasFileAndLine = line_vm.findAddress(address, locationInfo.file, locationInfo.line); - return locationInfo.hasFileAndLine; + info.has_file_and_line = line_vm.findAddress(address, info.file, info.line); + + bool check_inline = (mode == LocationInfoMode::FULL_WITH_INLINE); + + if (info.has_file_and_line && check_inline) + { + // Re-get the compilation unit with abbreviation cached. + cu.abbr_cache.clear(); + cu.abbr_cache.resize(kMaxAbbreviationEntries); + readCompilationUnitAbbrs(abbrev_, cu); + + // Find the subprogram that matches the given address. + Die subprogram; + findSubProgramDieForAddress(cu, die, address, base_addr_cu, subprogram); + + // Subprogram is the DIE of caller function. + if (check_inline && subprogram.abbr.has_children) + { + // Use an extra location and get its call file and call line, so that + // they can be used for the second last location when we don't have + // enough inline frames for all inline functions call stack. + const size_t max_size = Dwarf::kMaxInlineLocationInfoPerFrame + 1; + std::vector call_locations; + call_locations.reserve(Dwarf::kMaxInlineLocationInfoPerFrame + 1); + + findInlinedSubroutineDieForAddress(cu, subprogram, line_vm, address, base_addr_cu, call_locations, max_size); + size_t num_found = call_locations.size(); + + if (num_found > 0) + { + const auto inner_most_file = info.file; + const auto inner_most_line = info.line; + + // Earlier we filled in locationInfo: + // - mainFile: the path to the CU -- the file where the non-inlined + // call is made from. + // - file + line: the location of the inner-most inlined call. + // Here we already find inlined info so mainFile would be redundant. + info.has_main_file = false; + info.main_file = Path{}; + // @findInlinedSubroutineDieForAddress fills inlineLocations[0] with the + // file+line of the non-inlined outer function making the call. + // locationInfo.name is already set by the caller by looking up the + // non-inlined function @address belongs to. + info.has_file_and_line = true; + info.file = call_locations[0].file; + info.line = call_locations[0].line; + + // The next inlined subroutine's call file and call line is the current + // caller's location. + for (size_t i = 0; i < num_found - 1; i++) + { + call_locations[i].file = call_locations[i + 1].file; + call_locations[i].line = call_locations[i + 1].line; + } + // CallLocation for the inner-most inlined function: + // - will be computed if enough space was available in the passed + // buffer. + // - will have a .name, but no !.file && !.line + // - its corresponding file+line is the one returned by LineVM based + // on @address. + // Use the inner-most inlined file+line info we got from the LineVM. + call_locations[num_found - 1].file = inner_most_file; + call_locations[num_found - 1].line = inner_most_line; + + // Fill in inline frames in reverse order (as expected by the caller). + std::reverse(call_locations.begin(), call_locations.end()); + for (const auto & call_location : call_locations) + { + SymbolizedFrame inline_frame; + inline_frame.found = true; + inline_frame.addr = address; + inline_frame.name = call_location.name.data(); + inline_frame.location.has_file_and_line = true; + inline_frame.location.file = call_location.file; + inline_frame.location.line = call_location.line; + inline_frames.push_back(inline_frame); + } + } + } + } + + return info.has_file_and_line; } -bool Dwarf::findAddress(uintptr_t address, LocationInfo & locationInfo, LocationInfoMode mode) const +void Dwarf::findSubProgramDieForAddress( + const CompilationUnit & cu, const Die & die, uint64_t address, std::optional base_addr_cu, Die & subprogram) const +{ + forEachChild(cu, die, [&](const Die & child_die) + { + if (child_die.abbr.tag == DW_TAG_subprogram) + { + std::optional low_pc; + std::optional high_pc; + std::optional is_high_pc_addr; + std::optional range_offset; + forEachAttribute(cu, child_die, [&](const Attribute & attr) + { + switch (attr.spec.name) + { + case DW_AT_ranges: + range_offset = std::get(attr.attr_value); + break; + case DW_AT_low_pc: + low_pc = std::get(attr.attr_value); + break; + case DW_AT_high_pc: + // Value of DW_AT_high_pc attribute can be an address + // (DW_FORM_addr) or an offset (DW_FORM_data). + is_high_pc_addr = (attr.spec.form == DW_FORM_addr); + high_pc = std::get(attr.attr_value); + break; + } + // Iterate through all attributes until find all above. + return true; + }); + bool pc_match = low_pc && high_pc && is_high_pc_addr && address >= *low_pc + && (address < (*is_high_pc_addr ? *high_pc : *low_pc + *high_pc)); + bool range_match = range_offset && isAddrInRangeList(address, base_addr_cu, range_offset.value(), cu.addr_size); + if (pc_match || range_match) + { + subprogram = child_die; + return false; + } + } + + findSubProgramDieForAddress(cu, child_die, address, base_addr_cu, subprogram); + + // Iterates through children until find the inline subprogram. + return true; + }); +} + +/** + * Find DW_TAG_inlined_subroutine child DIEs that contain @address and + * then extract: + * - Where was it called from (DW_AT_call_file & DW_AT_call_line): + * the statement or expression that caused the inline expansion. + * - The inlined function's name. As a function may be inlined multiple + * times, common attributes like DW_AT_linkage_name or DW_AT_name + * are only stored in its "concrete out-of-line instance" (a + * DW_TAG_subprogram) which we find using DW_AT_abstract_origin. + */ +void Dwarf::findInlinedSubroutineDieForAddress( + const CompilationUnit & cu, + const Die & die, + const LineNumberVM & line_vm, + uint64_t address, + std::optional base_addr_cu, + std::vector & locations, + const size_t max_size) const +{ + if (locations.size() >= max_size) + { + return; + } + + forEachChild(cu, die, [&](const Die & child_die) + { + // Between a DW_TAG_subprogram and and DW_TAG_inlined_subroutine we might + // have arbitrary intermediary "nodes", including DW_TAG_common_block, + // DW_TAG_lexical_block, DW_TAG_try_block, DW_TAG_catch_block and + // DW_TAG_with_stmt, etc. + // We can't filter with locationhere since its range may be not specified. + // See section 2.6.2: A location list containing only an end of list entry + // describes an object that exists in the source code but not in the + // executable program. + if (child_die.abbr.tag == DW_TAG_try_block || child_die.abbr.tag == DW_TAG_catch_block || child_die.abbr.tag == DW_TAG_entry_point + || child_die.abbr.tag == DW_TAG_common_block || child_die.abbr.tag == DW_TAG_lexical_block) + { + findInlinedSubroutineDieForAddress(cu, child_die, line_vm, address, base_addr_cu, locations, max_size); + return true; + } + + std::optional low_pc; + std::optional high_pc; + std::optional is_high_pc_addr; + std::optional abstract_origin; + std::optional abstract_origin_ref_type; + std::optional call_file; + std::optional call_line; + std::optional range_offset; + forEachAttribute(cu, child_die, [&](const Attribute & attr) + { + switch (attr.spec.name) + { + case DW_AT_ranges: + range_offset = std::get(attr.attr_value); + break; + case DW_AT_low_pc: + low_pc = std::get(attr.attr_value); + break; + case DW_AT_high_pc: + // Value of DW_AT_high_pc attribute can be an address + // (DW_FORM_addr) or an offset (DW_FORM_data). + is_high_pc_addr = (attr.spec.form == DW_FORM_addr); + high_pc = std::get(attr.attr_value); + break; + case DW_AT_abstract_origin: + abstract_origin_ref_type = attr.spec.form; + abstract_origin = std::get(attr.attr_value); + break; + case DW_AT_call_line: + call_line = std::get(attr.attr_value); + break; + case DW_AT_call_file: + call_file = std::get(attr.attr_value); + break; + } + // Iterate through all until find all above attributes. + return true; + }); + + // 2.17 Code Addresses and Ranges + // Any debugging information entry describing an entity that has a + // machine code address or range of machine code addresses, + // which includes compilation units, module initialization, subroutines, + // ordinary blocks, try/catch blocks, labels and the like, may have + // - A DW_AT_low_pc attribute for a single address, + // - A DW_AT_low_pc and DW_AT_high_pc pair of attributes for a + // single contiguous range of addresses, or + // - A DW_AT_ranges attribute for a non-contiguous range of addresses. + // TODO: Support DW_TAG_entry_point and DW_TAG_common_block that don't + // have DW_AT_low_pc/DW_AT_high_pc pairs and DW_AT_ranges. + // TODO: Support relocated address which requires lookup in relocation map. + bool pc_match + = low_pc && high_pc && is_high_pc_addr && address >= *low_pc && (address < (*is_high_pc_addr ? *high_pc : *low_pc + *high_pc)); + bool range_match = range_offset && isAddrInRangeList(address, base_addr_cu, range_offset.value(), cu.addr_size); + if (!pc_match && !range_match) + { + // Address doesn't match. Keep searching other children. + return true; + } + + if (!abstract_origin || !abstract_origin_ref_type || !call_line || !call_file) + { + // We expect a single sibling DIE to match on addr, but it's missing + // required fields. Stop searching for other DIEs. + return false; + } + + CallLocation location; + location.file = line_vm.getFullFileName(*call_file); + location.line = *call_line; + + auto get_function_name = [&](const CompilationUnit & srcu, uint64_t die_offset) + { + auto decl_die = getDieAtOffset(srcu, die_offset); + // Jump to the actual function definition instead of declaration for name + // and line info. + auto def_die = findDefinitionDie(srcu, decl_die); + + std::string_view name; + // The file and line will be set in the next inline subroutine based on + // its DW_AT_call_file and DW_AT_call_line. + forEachAttribute(srcu, def_die, [&](const Attribute & attr) + { + switch (attr.spec.name) + { + case DW_AT_linkage_name: + name = std::get(attr.attr_value); + break; + case DW_AT_name: + // NOTE: when DW_AT_linkage_name and DW_AT_name match, dwarf + // emitters omit DW_AT_linkage_name (to save space). If present + // DW_AT_linkage_name should always be preferred (mangled C++ name + // vs just the function name). + if (name.empty()) + { + name = std::get(attr.attr_value); + } + break; + } + return true; + }); + return name; + }; + + // DW_AT_abstract_origin is a reference. There a 3 types of references: + // - the reference can identify any debugging information entry within the + // compilation unit (DW_FORM_ref1, DW_FORM_ref2, DW_FORM_ref4, + // DW_FORM_ref8, DW_FORM_ref_udata). This type of reference is an offset + // from the first byte of the compilation header for the compilation unit + // containing the reference. + // - the reference can identify any debugging information entry within a + // .debug_info section; in particular, it may refer to an entry in a + // different compilation unit (DW_FORM_ref_addr) + // - the reference can identify any debugging information type entry that + // has been placed in its own type unit. + // Not applicable for DW_AT_abstract_origin. + location.name = (*abstract_origin_ref_type != DW_FORM_ref_addr) + ? get_function_name(cu, cu.offset + *abstract_origin) + : get_function_name(findCompilationUnit(info_, *abstract_origin), *abstract_origin); + + locations.push_back(location); + + findInlinedSubroutineDieForAddress(cu, child_die, line_vm, address, base_addr_cu, locations, max_size); + + return false; + }); +} + +bool Dwarf::findAddress( + uintptr_t address, LocationInfo & locationInfo, LocationInfoMode mode, std::vector & inline_frames) const { locationInfo = LocationInfo(); @@ -635,10 +1113,9 @@ bool Dwarf::findAddress(uintptr_t address, LocationInfo & locationInfo, Location if (findDebugInfoOffset(address, aranges_, offset)) { // Read compilation unit header from .debug_info - std::string_view info_entry(info_); - info_entry.remove_prefix(offset); - findLocation(address, info_entry, locationInfo); - return locationInfo.hasFileAndLine; + auto unit = getCompilationUnit(info_, offset); + findLocation(address, mode, unit, locationInfo, inline_frames); + return locationInfo.has_file_and_line; } else if (mode == LocationInfoMode::FAST) { @@ -650,20 +1127,92 @@ bool Dwarf::findAddress(uintptr_t address, LocationInfo & locationInfo, Location } else { - SAFE_CHECK(mode == LocationInfoMode::FULL, "unexpected mode"); + SAFE_CHECK(mode == LocationInfoMode::FULL || mode == LocationInfoMode::FULL_WITH_INLINE, "unexpected mode"); // Fall back to the linear scan. } } // Slow path (linear scan): Iterate over all .debug_info entries // and look for the address in each compilation unit. - std::string_view info_entry(info_); - while (!info_entry.empty() && !locationInfo.hasFileAndLine) - findLocation(address, info_entry, locationInfo); + uint64_t offset = 0; + while (offset < info_.size() && !locationInfo.has_file_and_line) + { + auto unit = getCompilationUnit(info_, offset); + offset += unit.size; + findLocation(address, mode, unit, locationInfo, inline_frames); + } - return locationInfo.hasFileAndLine; + return locationInfo.has_file_and_line; } +bool Dwarf::isAddrInRangeList(uint64_t address, std::optional base_addr, size_t offset, uint8_t addr_size) const +{ + SAFE_CHECK(addr_size == 4 || addr_size == 8, "wrong address size"); + if (ranges_.empty()) + { + return false; + } + + const bool is_64bit_addr = addr_size == 8; + std::string_view sp = ranges_; + sp.remove_prefix(offset); + const uint64_t max_addr = is_64bit_addr ? std::numeric_limits::max() : std::numeric_limits::max(); + while (!sp.empty()) + { + uint64_t begin = readOffset(sp, is_64bit_addr); + uint64_t end = readOffset(sp, is_64bit_addr); + // The range list entry is a base address selection entry. + if (begin == max_addr) + { + base_addr = end; + continue; + } + // The range list entry is an end of list entry. + if (begin == 0 && end == 0) + { + break; + } + // Check if the given address falls in the range list entry. + // 2.17.3 Non-Contiguous Address Ranges + // The applicable base address of a range list entry is determined by the + // closest preceding base address selection entry (see below) in the same + // range list. If there is no such selection entry, then the applicable base + // address defaults to the base address of the compilation unit. + if (base_addr && address >= begin + *base_addr && address < end + *base_addr) + { + return true; + } + } + + return false; +} + +// static +Dwarf::CompilationUnit Dwarf::findCompilationUnit(std::string_view info, uint64_t targetOffset) +{ + SAFE_CHECK(targetOffset < info.size(), "unexpected target address"); + uint64_t offset = 0; + while (offset < info.size()) + { + std::string_view chunk(info); + chunk.remove_prefix(offset); + + auto initial_length = read(chunk); + auto is_64bit = (initial_length == uint32_t(-1)); + auto size = is_64bit ? read(chunk) : initial_length; + SAFE_CHECK(size <= chunk.size(), "invalid chunk size"); + size += is_64bit ? 12 : 4; + + if (offset + size > targetOffset) + { + break; + } + offset += size; + } + return getCompilationUnit(info, offset); +} + + Dwarf::LineNumberVM::LineNumberVM(std::string_view data, std::string_view compilationDirectory) : compilationDirectory_(compilationDirectory) { diff --git a/src/Common/Dwarf.h b/src/Common/Dwarf.h index 40badc1c5a4..9ea940c3380 100644 --- a/src/Common/Dwarf.h +++ b/src/Common/Dwarf.h @@ -21,9 +21,13 @@ /** This file was edited for ClickHouse. */ +#include +#include +#include #include #include #include +#include namespace DB @@ -61,7 +65,13 @@ class Dwarf final // be live for as long as the passed-in Elf is live. public: /** Create a DWARF parser around an ELF file. */ - explicit Dwarf(const Elf & elf); + explicit Dwarf(const std::shared_ptr & elf); + + /** + * More than one location info may exist if current frame is an inline + * function call. + */ + static constexpr uint32_t kMaxInlineLocationInfoPerFrame = 10; /** * Represent a file path a s collection of three parts (base directory, @@ -70,7 +80,7 @@ public: class Path { public: - Path() {} + Path() = default; Path(std::string_view baseDir, std::string_view subDir, std::string_view file); @@ -107,6 +117,14 @@ public: std::string_view file_; }; + // Indicates inline function `name` is called at `line@file`. + struct CallLocation + { + Path file = {}; + uint64_t line; + std::string_view name; + }; + enum class LocationInfoMode { // Don't resolve location info. @@ -115,30 +133,47 @@ public: FAST, // Scan all CU in .debug_info (slow!) on .debug_aranges lookup failure. FULL, + // Scan .debug_info (super slower, use with caution) for inline functions in + // addition to FULL. + FULL_WITH_INLINE, }; struct LocationInfo { - bool hasMainFile = false; - Path mainFile; + bool has_main_file = false; + Path main_file; - bool hasFileAndLine = false; + bool has_file_and_line = false; Path file; uint64_t line = 0; }; + /** + * Frame information: symbol name and location. + */ + struct SymbolizedFrame + { + bool found = false; + uintptr_t addr = 0; + // Mangled symbol name. Use `folly::demangle()` to demangle it. + const char * name = nullptr; + LocationInfo location; + std::shared_ptr file; + + void clear() { *this = SymbolizedFrame(); } + }; + /** Find the file and line number information corresponding to address. * The address must be physical - offset in object file without offset in virtual memory where the object is loaded. */ - bool findAddress(uintptr_t address, LocationInfo & info, LocationInfoMode mode) const; + bool findAddress(uintptr_t address, LocationInfo & info, LocationInfoMode mode, std::vector & inline_frames) const; private: static bool findDebugInfoOffset(uintptr_t address, std::string_view aranges, uint64_t & offset); void init(); - bool findLocation(uintptr_t address, std::string_view & infoEntry, LocationInfo & info) const; - const Elf * elf_; + std::shared_ptr elf_; // DWARF section made up of chunks, each prefixed with a length header. // The length indicates whether the chunk is DWARF-32 or DWARF-64, which @@ -169,17 +204,81 @@ private: { uint64_t code; uint64_t tag; - bool hasChildren; - - struct Attribute - { - uint64_t name; - uint64_t form; - }; + bool has_children = false; std::string_view attributes; }; + // Debugging information entry to define a low-level representation of a + // source program. Each debugging information entry consists of an identifying + // tag and a series of attributes. An entry, or group of entries together, + // provide a description of a corresponding entity in the source program. + struct Die + { + bool is64Bit; + // Offset from start to first attribute + uint8_t attr_offset; + // Offset within debug info. + uint32_t offset; + uint64_t code; + DIEAbbreviation abbr; + }; + + struct AttributeSpec + { + uint64_t name = 0; + uint64_t form = 0; + + explicit operator bool() const { return name != 0 || form != 0; } + }; + + struct Attribute + { + AttributeSpec spec; + const Die & die; + std::variant attr_value; + }; + + struct CompilationUnit + { + bool is64Bit; + uint8_t version; + uint8_t addr_size; + // Offset in .debug_info of this compilation unit. + uint32_t offset; + uint32_t size; + // Offset in .debug_info for the first DIE in this compilation unit. + uint32_t first_die; + uint64_t abbrev_offset; + // Only the CompilationUnit that contains the caller functions needs this cache. + // Indexed by (abbr.code - 1) if (abbr.code - 1) < abbrCache.size(); + std::vector abbr_cache; + }; + + static CompilationUnit getCompilationUnit(std::string_view info, uint64_t offset); + + /** cu must exist during the life cycle of created detail::Die. */ + Die getDieAtOffset(const CompilationUnit & cu, uint64_t offset) const; + + /** + * Find the actual definition DIE instead of declaration for the given die. + */ + Die findDefinitionDie(const CompilationUnit & cu, const Die & die) const; + + bool findLocation( + uintptr_t address, + LocationInfoMode mode, + CompilationUnit & cu, + LocationInfo & info, + std::vector & inline_frames) const; + + /** + * Finds a subprogram debugging info entry that contains a given address among + * children of given die. Depth first search. + */ + void findSubProgramDieForAddress( + const CompilationUnit & cu, const Die & die, uint64_t address, std::optional base_addr_cu, Die & subprogram) const; + // Interpreter for the line number bytecode VM class LineNumberVM { @@ -188,6 +287,13 @@ private: bool findAddress(uintptr_t target, Path & file, uint64_t & line); + /** Gets full file name at given index including directory. */ + Path getFullFileName(uint64_t index) const + { + auto fn = getFileName(index); + return Path({}, getIncludeDirectory(fn.directoryIndex), fn.relativeName); + } + private: void init(); void reset(); @@ -259,18 +365,50 @@ private: uint64_t discriminator_; }; + /** + * Finds inlined subroutine DIEs and their caller lines that contains a given + * address among children of given die. Depth first search. + */ + void findInlinedSubroutineDieForAddress( + const CompilationUnit & cu, + const Die & die, + const LineNumberVM & line_vm, + uint64_t address, + std::optional base_addr_cu, + std::vector & locations, + size_t max_size) const; + // Read an abbreviation from a std::string_view, return true if at end; remove_prefix section static bool readAbbreviation(std::string_view & section, DIEAbbreviation & abbr); + static void readCompilationUnitAbbrs(std::string_view abbrev, CompilationUnit & cu); + + /** + * Iterates over all children of a debugging info entry, calling the given + * callable for each. Iteration is stopped early if any of the calls return + * false. Returns the offset of next DIE after iterations. + */ + size_t forEachChild(const CompilationUnit & cu, const Die & die, std::function f) const; + // Get abbreviation corresponding to a code, in the chunk starting at // offset in the .debug_abbrev section DIEAbbreviation getAbbreviation(uint64_t code, uint64_t offset) const; + /** + * Iterates over all attributes of a debugging info entry, calling the given + * callable for each. If all attributes are visited, then return the offset of + * next DIE, or else iteration is stopped early and return size_t(-1) if any + * of the calls return false. + */ + size_t forEachAttribute(const CompilationUnit & cu, const Die & die, std::function f) const; + + Attribute readAttribute(const Die & die, AttributeSpec spec, std::string_view & info) const; + // Read one attribute pair, remove_prefix sp; returns <0, 0> at end. - static DIEAbbreviation::Attribute readAttribute(std::string_view & sp); + static AttributeSpec readAttributeSpec(std::string_view & sp); // Read one attribute value, remove_prefix sp - typedef std::variant AttributeValue; + using AttributeValue = std::variant; AttributeValue readAttributeValue(std::string_view & sp, uint64_t form, bool is64Bit) const; // Get an ELF section by name, return true if found @@ -279,11 +417,34 @@ private: // Get a string from the .debug_str section std::string_view getStringFromStringSection(uint64_t offset) const; + template + std::optional getAttribute(const CompilationUnit & cu, const Die & die, uint64_t attr_name) const + { + std::optional result; + forEachAttribute(cu, die, [&](const Attribute & attr) + { + if (attr.spec.name == attr_name) + { + result = std::get(attr.attr_value); + return false; + } + return true; + }); + return result; + } + + // Check if the given address is in the range list at the given offset in .debug_ranges. + bool isAddrInRangeList(uint64_t address, std::optional base_addr, size_t offset, uint8_t addr_size) const; + + // Finds the Compilation Unit starting at offset. + static CompilationUnit findCompilationUnit(std::string_view info, uint64_t targetOffset); + std::string_view info_; // .debug_info std::string_view abbrev_; // .debug_abbrev std::string_view aranges_; // .debug_aranges std::string_view line_; // .debug_line std::string_view strings_; // .debug_str + std::string_view ranges_; // .debug_ranges }; } diff --git a/src/Common/PODArray.cpp b/src/Common/PODArray.cpp index e0b17c8125c..c1edc5bafad 100644 --- a/src/Common/PODArray.cpp +++ b/src/Common/PODArray.cpp @@ -6,4 +6,14 @@ namespace DB /// Used for left padding of PODArray when empty const char empty_pod_array[empty_pod_array_size]{}; +template class PODArray, 15, 16>; +template class PODArray, 15, 16>; +template class PODArray, 15, 16>; +template class PODArray, 15, 16>; + +template class PODArray, 15, 16>; +template class PODArray, 15, 16>; +template class PODArray, 15, 16>; +template class PODArray, 15, 16>; + } diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index f0cc9df11cd..8e05dfea8b3 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -725,4 +725,16 @@ void swap(PODArray & lhs, PODArray, 15, 16>; +extern template class PODArray, 15, 16>; +extern template class PODArray, 15, 16>; +extern template class PODArray, 15, 16>; + +extern template class PODArray, 15, 16>; +extern template class PODArray, 15, 16>; +extern template class PODArray, 15, 16>; +extern template class PODArray, 15, 16>; + } diff --git a/src/Common/PODArray_fwd.h b/src/Common/PODArray_fwd.h index f817d2f6dde..22f9230c01c 100644 --- a/src/Common/PODArray_fwd.h +++ b/src/Common/PODArray_fwd.h @@ -3,8 +3,8 @@ * This file contains some using-declarations that define various kinds of * PODArray. */ -#pragma once +#include #include namespace DB diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 44f6b9e5443..c4cf7f11e68 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -217,10 +217,12 @@ void StackTrace::symbolize(const StackTrace::FramePointers & frame_pointers, siz current_frame.object = object->name; if (std::filesystem::exists(current_frame.object.value())) { - auto dwarf_it = dwarfs.try_emplace(object->name, *object->elf).first; + auto dwarf_it = dwarfs.try_emplace(object->name, object->elf).first; DB::Dwarf::LocationInfo location; - if (dwarf_it->second.findAddress(uintptr_t(current_frame.physical_addr), location, DB::Dwarf::LocationInfoMode::FAST)) + std::vector inline_frames; + if (dwarf_it->second.findAddress( + uintptr_t(current_frame.physical_addr), location, DB::Dwarf::LocationInfoMode::FAST, inline_frames)) { current_frame.file = location.file.toString(); current_frame.line = location.line; @@ -314,7 +316,11 @@ const StackTrace::FramePointers & StackTrace::getFramePointers() const } static void toStringEveryLineImpl( - const StackTrace::FramePointers & frame_pointers, size_t offset, size_t size, std::function callback) + bool fatal, + const StackTrace::FramePointers & frame_pointers, + size_t offset, + size_t size, + std::function callback) { if (size == 0) return callback(""); @@ -324,11 +330,12 @@ static void toStringEveryLineImpl( const DB::SymbolIndex & symbol_index = *symbol_index_ptr; std::unordered_map dwarfs; - std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM out.exceptions(std::ios::failbit); for (size_t i = offset; i < size; ++i) { + std::vector inline_frames; const void * virtual_addr = frame_pointers[i]; const auto * object = symbol_index.findObject(virtual_addr); uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0; @@ -340,10 +347,11 @@ static void toStringEveryLineImpl( { if (std::filesystem::exists(object->name)) { - auto dwarf_it = dwarfs.try_emplace(object->name, *object->elf).first; + auto dwarf_it = dwarfs.try_emplace(object->name, object->elf).first; DB::Dwarf::LocationInfo location; - if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, DB::Dwarf::LocationInfoMode::FAST)) + auto mode = fatal ? DB::Dwarf::LocationInfoMode::FULL_WITH_INLINE : DB::Dwarf::LocationInfoMode::FAST; + if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, mode, inline_frames)) out << location.file.toString() << ":" << location.line << ": "; } } @@ -360,11 +368,20 @@ static void toStringEveryLineImpl( out << " @ " << physical_addr; out << " in " << (object ? object->name : "?"); + for (size_t j = 0; j < inline_frames.size(); ++j) + { + const auto & frame = inline_frames[j]; + int status = 0; + callback(fmt::format("{}.{}. inlined from {}:{}: {}", + i, j+1, frame.location.file.toString(), frame.location.line, demangle(frame.name, status))); + } + callback(out.str()); out.str({}); } #else - std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + UNUSED(fatal); + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM out.exceptions(std::ios::failbit); for (size_t i = offset; i < size; ++i) @@ -382,13 +399,13 @@ static std::string toStringImpl(const StackTrace::FramePointers & frame_pointers { std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM out.exceptions(std::ios::failbit); - toStringEveryLineImpl(frame_pointers, offset, size, [&](const std::string & str) { out << str << '\n'; }); + toStringEveryLineImpl(false, frame_pointers, offset, size, [&](const std::string & str) { out << str << '\n'; }); return out.str(); } void StackTrace::toStringEveryLine(std::function callback) const { - toStringEveryLineImpl(frame_pointers, offset, size, std::move(callback)); + toStringEveryLineImpl(true, frame_pointers, offset, size, std::move(callback)); } diff --git a/src/Common/StackTrace.h b/src/Common/StackTrace.h index b2e14a01f03..58660f9e4da 100644 --- a/src/Common/StackTrace.h +++ b/src/Common/StackTrace.h @@ -51,10 +51,10 @@ public: /// Tries to capture stack trace. Fallbacks on parsing caller address from /// signal context if no stack trace could be captured - StackTrace(const ucontext_t & signal_context); + explicit StackTrace(const ucontext_t & signal_context); /// Creates empty object for deferred initialization - StackTrace(NoCapture); + explicit StackTrace(NoCapture); size_t getSize() const; size_t getOffset() const; @@ -65,6 +65,7 @@ public: static void symbolize(const FramePointers & frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames); void toStringEveryLine(std::function callback) const; + protected: void tryCapture(); diff --git a/src/Common/SymbolIndex.h b/src/Common/SymbolIndex.h index b310f90988e..65e446a7fc4 100644 --- a/src/Common/SymbolIndex.h +++ b/src/Common/SymbolIndex.h @@ -36,7 +36,7 @@ public: const void * address_begin; const void * address_end; std::string name; - std::unique_ptr elf; + std::shared_ptr elf; }; /// Address in virtual memory should be passed. These addresses include offset where the object is loaded in memory. diff --git a/src/Common/tests/symbol_index.cpp b/src/Common/tests/symbol_index.cpp index 3811bbbdd71..496fa7dc3fe 100644 --- a/src/Common/tests/symbol_index.cpp +++ b/src/Common/tests/symbol_index.cpp @@ -47,10 +47,11 @@ int main(int argc, char ** argv) std::cerr << "dladdr: Not found\n"; const auto * object = symbol_index.findObject(getAddress()); - Dwarf dwarf(*object->elf); + Dwarf dwarf(object->elf); Dwarf::LocationInfo location; - if (dwarf.findAddress(uintptr_t(address) - uintptr_t(info.dli_fbase), location, Dwarf::LocationInfoMode::FAST)) + std::vector frames; + if (dwarf.findAddress(uintptr_t(address) - uintptr_t(info.dli_fbase), location, Dwarf::LocationInfoMode::FAST, frames)) std::cerr << location.file.toString() << ":" << location.line << "\n"; else std::cerr << "Dwarf: Not found\n"; diff --git a/src/DataTypes/DataTypeNumberBase.h b/src/DataTypes/DataTypeNumberBase.h index cbbc203bf4f..1491eabfbd5 100644 --- a/src/DataTypes/DataTypeNumberBase.h +++ b/src/DataTypes/DataTypeNumberBase.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -70,4 +71,21 @@ public: bool canBeInsideLowCardinality() const override { return true; } }; +/// Prevent implicit template instantiation of DataTypeNumberBase for common numeric types + +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; // base for UUID +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; +extern template class DataTypeNumberBase; + } diff --git a/src/Dictionaries/RangeDictionaryBlockInputStream.h b/src/Dictionaries/RangeDictionaryBlockInputStream.h index 3da43c85c45..ccd77d49e0f 100644 --- a/src/Dictionaries/RangeDictionaryBlockInputStream.h +++ b/src/Dictionaries/RangeDictionaryBlockInputStream.h @@ -47,7 +47,8 @@ private: const std::string & default_name, const std::unordered_set & column_names_set, const PaddedPODArray & values, - ColumnsWithTypeAndName & columns) const; + ColumnsWithTypeAndName & columns, + bool force = false) const; Block fillBlock( const PaddedPODArray & ids_to_fill, @@ -121,13 +122,14 @@ void RangeDictionaryBlockInputStream::addSpecial const std::string & default_name, const std::unordered_set & column_names_set, const PaddedPODArray & values, - ColumnsWithTypeAndName & columns) const + ColumnsWithTypeAndName & columns, + bool force) const { std::string name = default_name; if (attribute) name = attribute->name; - if (column_names_set.find(name) != column_names_set.end()) + if (force || column_names_set.find(name) != column_names_set.end()) columns.emplace_back(getColumnFromPODArray(values), type, name); } @@ -159,7 +161,7 @@ Block RangeDictionaryBlockInputStream::fillBlock std::unordered_set names(column_names.begin(), column_names.end()); - addSpecialColumn(structure.id, std::make_shared(), "ID", names, ids_to_fill, columns); + addSpecialColumn(structure.id, std::make_shared(), "ID", names, ids_to_fill, columns, true); auto ids_column = columns.back().column; addSpecialColumn(structure.range_min, structure.range_max->type, "Range Start", names, block_start_dates, columns); addSpecialColumn(structure.range_max, structure.range_max->type, "Range End", names, block_end_dates, columns); diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index f61c9c91d00..bb85ae32622 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -894,9 +894,8 @@ class FunctionBinaryArithmetic : public IFunction const NativeResultType const_b = helperGetOrConvert(col_right_const, right); const ResultType res = check_decimal_overflow - // the arguments are already scaled after conversion - ? OpImplCheck::template process(const_a, const_b, 1, 1) - : OpImpl::template process(const_a, const_b, 1, 1); + ? OpImplCheck::template process(const_a, const_b, scale_a, scale_b) + : OpImpl::template process(const_a, const_b, scale_a, scale_b); if constexpr (result_is_decimal) return ResultDataType(type.getPrecision(), type.getScale()).createColumnConst( diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index d64646ecaf1..17c28ee3343 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -70,8 +70,19 @@ ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName } else if (const auto * const_column = checkAndGetColumn(*col.column)) { - const auto & nested_col = checkAndGetColumn(const_column->getDataColumn())->getNestedColumnPtr(); - res.emplace_back(ColumnWithTypeAndName{ ColumnConst::create(nested_col, col.column->size()), nested_type, col.name}); + const auto * nullable_column = checkAndGetColumn(const_column->getDataColumn()); + + ColumnPtr nullable_res; + if (nullable_column) + { + const auto & nested_col = nullable_column->getNestedColumnPtr(); + nullable_res = ColumnConst::create(nested_col, col.column->size()); + } + else + { + nullable_res = makeNullable(col.column); + } + res.emplace_back(ColumnWithTypeAndName{ nullable_res, nested_type, col.name }); } else throw Exception("Illegal column for DataTypeNullable", ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/addressToLine.cpp b/src/Functions/addressToLine.cpp index 59e347dd348..a115b13e54a 100644 --- a/src/Functions/addressToLine.cpp +++ b/src/Functions/addressToLine.cpp @@ -111,12 +111,13 @@ private: if (const auto * object = symbol_index.findObject(reinterpret_cast(addr))) { - auto dwarf_it = cache.dwarfs.try_emplace(object->name, *object->elf).first; + auto dwarf_it = cache.dwarfs.try_emplace(object->name, object->elf).first; if (!std::filesystem::exists(object->name)) return {}; Dwarf::LocationInfo location; - if (dwarf_it->second.findAddress(addr - uintptr_t(object->address_begin), location, Dwarf::LocationInfoMode::FAST)) + std::vector frames; // NOTE: not used in FAST mode. + if (dwarf_it->second.findAddress(addr - uintptr_t(object->address_begin), location, Dwarf::LocationInfoMode::FAST, frames)) { const char * arena_begin = nullptr; WriteBufferFromArena out(cache.arena, arena_begin); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index ca4a313da62..5c99d39dc2e 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -331,7 +331,7 @@ struct ContextShared mutable std::optional external_models_loader; String default_profile_name; /// Default profile name used for default values. String system_profile_name; /// Profile used by system processes - std::unique_ptr access_control_manager; + AccessControlManager access_control_manager; mutable UncompressedCachePtr uncompressed_cache; /// The cache of decompressed blocks. mutable MarkCachePtr mark_cache; /// Cache of marks in compressed files. ProcessList process_list; /// Executing queries at the moment. @@ -388,8 +388,7 @@ struct ContextShared Context::ConfigReloadCallback config_reload_callback; ContextShared() - : access_control_manager(std::make_unique()) - , macros(std::make_unique()) + : macros(std::make_unique()) { /// TODO: make it singleton (?) static std::atomic num_calls{0}; @@ -435,7 +434,6 @@ struct ContextShared /// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference). /// TODO: Get rid of this. - access_control_manager.reset(); system_logs.reset(); embedded_dictionaries.reset(); external_dictionaries_loader.reset(); @@ -642,7 +640,7 @@ void Context::setConfig(const ConfigurationPtr & config) { auto lock = getLock(); shared->config = config; - shared->access_control_manager->setExternalAuthenticatorsConfig(*shared->config); + shared->access_control_manager.setExternalAuthenticatorsConfig(*shared->config); } const Poco::Util::AbstractConfiguration & Context::getConfigRef() const @@ -654,25 +652,25 @@ const Poco::Util::AbstractConfiguration & Context::getConfigRef() const AccessControlManager & Context::getAccessControlManager() { - return *shared->access_control_manager; + return shared->access_control_manager; } const AccessControlManager & Context::getAccessControlManager() const { - return *shared->access_control_manager; + return shared->access_control_manager; } void Context::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config) { auto lock = getLock(); - shared->access_control_manager->setExternalAuthenticatorsConfig(config); + shared->access_control_manager.setExternalAuthenticatorsConfig(config); } void Context::setUsersConfig(const ConfigurationPtr & config) { auto lock = getLock(); shared->users_config = config; - shared->access_control_manager->setUsersConfig(*shared->users_config); + shared->access_control_manager.setUsersConfig(*shared->users_config); } ConfigurationPtr Context::getUsersConfig() diff --git a/src/Storages/MergeTree/BackgroundJobsExecutor.cpp b/src/Storages/MergeTree/BackgroundJobsExecutor.cpp index 3e3f693addd..8e5a0e8a3b8 100644 --- a/src/Storages/MergeTree/BackgroundJobsExecutor.cpp +++ b/src/Storages/MergeTree/BackgroundJobsExecutor.cpp @@ -98,11 +98,21 @@ try { try /// We don't want exceptions in background pool { - job(); + bool job_success = job(); /// Job done, decrement metric and reset no_work counter CurrentMetrics::values[pool_config.tasks_metric]--; - /// Job done, new empty space in pool, schedule background task - runTaskWithoutDelay(); + + if (job_success) + { + /// Job done, new empty space in pool, schedule background task + runTaskWithoutDelay(); + } + else + { + /// Job done, but failed, schedule with backoff + scheduleTask(/* with_backoff = */ true); + } + } catch (...) { diff --git a/src/Storages/MergeTree/BackgroundJobsExecutor.h b/src/Storages/MergeTree/BackgroundJobsExecutor.h index 85067188f09..da22c752e1b 100644 --- a/src/Storages/MergeTree/BackgroundJobsExecutor.h +++ b/src/Storages/MergeTree/BackgroundJobsExecutor.h @@ -36,10 +36,12 @@ enum class PoolType FETCH, }; +using BackgroundJobFunc = std::function; + /// Result from background job providers. Function which will be executed in pool and pool type. struct JobAndPool { - ThreadPool::Job job; + BackgroundJobFunc job; PoolType pool_type; }; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 1ace383b051..a0d23b8ab22 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3836,7 +3836,7 @@ std::optional MergeTreeData::getDataMovingJob() return JobAndPool{[this, moving_tagger] () mutable { - moveParts(moving_tagger); + return moveParts(moving_tagger); }, PoolType::MOVE}; } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 11a159d4a6c..202e909af0f 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -962,9 +962,11 @@ std::optional StorageMergeTree::getDataProcessingJob() return JobAndPool{[this, metadata_snapshot, merge_entry, mutate_entry, share_lock] () mutable { if (merge_entry) - mergeSelectedParts(metadata_snapshot, false, {}, *merge_entry, share_lock); + return mergeSelectedParts(metadata_snapshot, false, {}, *merge_entry, share_lock); else if (mutate_entry) - mutateSelectedPart(metadata_snapshot, *mutate_entry, share_lock); + return mutateSelectedPart(metadata_snapshot, *mutate_entry, share_lock); + + __builtin_unreachable(); }, PoolType::MERGE_MUTATE}; } else if (auto lock = time_after_previous_cleanup.compareAndRestartDeferred(1)) @@ -978,6 +980,7 @@ std::optional StorageMergeTree::getDataProcessingJob() clearOldWriteAheadLogs(); clearOldMutations(); clearEmptyParts(); + return true; }, PoolType::MERGE_MUTATE}; } return {}; diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index be1159b1a63..09fd413af75 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -42,7 +42,6 @@ StorageMongoDB::StorageMongoDB( , collection_name(collection_name_) , username(username_) , password(password_) - , connection{std::make_shared(host, port)} { StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); @@ -51,6 +50,26 @@ StorageMongoDB::StorageMongoDB( } +void StorageMongoDB::connectIfNotConnected() +{ + std::lock_guard lock{connection_mutex}; + if (!connection) + connection = std::make_shared(host, port); + + if (!authentified) + { +# if POCO_VERSION >= 0x01070800 + Poco::MongoDB::Database poco_db(database_name); + if (!poco_db.authenticate(*connection, username, password, Poco::MongoDB::Database::AUTH_SCRAM_SHA1)) + throw Exception("Cannot authenticate in MongoDB, incorrect user or password", ErrorCodes::MONGODB_CANNOT_AUTHENTICATE); +# else + authenticate(*connection, database_name, username, password); +# endif + authentified = true; + } +} + + Pipe StorageMongoDB::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -60,15 +79,9 @@ Pipe StorageMongoDB::read( size_t max_block_size, unsigned) { - metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + connectIfNotConnected(); -#if POCO_VERSION >= 0x01070800 - Poco::MongoDB::Database poco_db(database_name); - if (!poco_db.authenticate(*connection, username, password, Poco::MongoDB::Database::AUTH_SCRAM_SHA1)) - throw Exception("Cannot authenticate in MongoDB, incorrect user or password", ErrorCodes::MONGODB_CANNOT_AUTHENTICATE); -#else - authenticate(*connection, database_name, username, password); -#endif + metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); Block sample_block; for (const String & column_name : column_names) diff --git a/src/Storages/StorageMongoDB.h b/src/Storages/StorageMongoDB.h index d7b71495574..589ab276539 100644 --- a/src/Storages/StorageMongoDB.h +++ b/src/Storages/StorageMongoDB.h @@ -40,16 +40,19 @@ public: size_t max_block_size, unsigned num_streams) override; - private: - std::string host; - short unsigned int port; - std::string database_name; - std::string collection_name; - std::string username; - std::string password; + void connectIfNotConnected(); + + const std::string host; + const short unsigned int port; + const std::string database_name; + const std::string collection_name; + const std::string username; + const std::string password; std::shared_ptr connection; + bool authentified = false; + std::mutex connection_mutex; /// Protects the variables `connection` and `authentified`. }; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 53104efeb43..097b7679899 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2682,7 +2682,7 @@ std::optional StorageReplicatedMergeTree::getDataProcessingJob() return JobAndPool{[this, selected_entry] () mutable { - processQueueEntry(selected_entry); + return processQueueEntry(selected_entry); }, pool_type}; } diff --git a/src/Storages/tests/gtest_background_executor.cpp b/src/Storages/tests/gtest_background_executor.cpp index bf9a305ccc9..0ddf2d9ea2a 100644 --- a/src/Storages/tests/gtest_background_executor.cpp +++ b/src/Storages/tests/gtest_background_executor.cpp @@ -32,7 +32,7 @@ protected: std::optional getBackgroundJob() override { - return JobAndPool{[] { std::this_thread::sleep_for(1s); counter++; }, PoolType::MERGE_MUTATE}; + return JobAndPool{[] { std::this_thread::sleep_for(1s); counter++; return true; }, PoolType::MERGE_MUTATE}; } }; diff --git a/tests/integration/test_send_crash_reports/test.py b/tests/integration/test_send_crash_reports/test.py index a3c35ca1537..65d49637b13 100644 --- a/tests/integration/test_send_crash_reports/test.py +++ b/tests/integration/test_send_crash_reports/test.py @@ -24,14 +24,17 @@ def started_node(): def test_send_segfault(started_node, ): + if started_node.is_built_with_thread_sanitizer(): + pytest.skip("doesn't fit in timeouts for stacktrace generation") + started_node.copy_file_to_container(os.path.join(SCRIPT_DIR, "fake_sentry_server.py"), "/fake_sentry_server.py") started_node.exec_in_container(["bash", "-c", "python3 /fake_sentry_server.py > /fake_sentry_server.log 2>&1"], detach=True, user="root") - time.sleep(0.5) + time.sleep(1) started_node.exec_in_container(["bash", "-c", "pkill -11 clickhouse"], user="root") result = None for attempt in range(1, 6): - time.sleep(0.25 * attempt) + time.sleep(attempt) result = started_node.exec_in_container(['cat', fake_sentry_server.RESULT_PATH], user='root') if result == 'OK': break diff --git a/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.reference b/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.reference index 1a9e5685a6a..71be9c3fb5b 100644 --- a/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.reference +++ b/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.reference @@ -1,3 +1,4 @@ 1 2019-01-05 2020-01-10 1 +1 date_table somedict diff --git a/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.sql b/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.sql index 6ad76ee5a7e..471fd7959a9 100644 --- a/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.sql +++ b/tests/queries/0_stateless/01125_dict_ddl_cannot_add_column.sql @@ -29,6 +29,9 @@ LIFETIME(MIN 300 MAX 360); SELECT * from somedict; +-- No dictionary columns +SELECT 1 FROM somedict; + SHOW TABLES; -DROP DATABASE IF EXISTS database_for_dict; +DROP DATABASE database_for_dict; diff --git a/tests/queries/0_stateless/01650_drop_part_and_deduplication_zookeeper.sql b/tests/queries/0_stateless/01650_drop_part_and_deduplication_zookeeper.sql index 50596680618..c3e459dfc49 100644 --- a/tests/queries/0_stateless/01650_drop_part_and_deduplication_zookeeper.sql +++ b/tests/queries/0_stateless/01650_drop_part_and_deduplication_zookeeper.sql @@ -5,7 +5,7 @@ CREATE TABLE partitioned_table ( partitioner UInt8, value String ) -ENGINE ReplicatedMergeTree('/clickhouse/test/01650_drop_part_and_deduplication/partitioned_table', '1') +ENGINE ReplicatedMergeTree('/clickhouse/01650_drop_part_and_deduplication_partitioned_table', '1') ORDER BY key PARTITION BY partitioner; @@ -16,24 +16,24 @@ INSERT INTO partitioned_table VALUES (11, 1, 'AA'), (22, 2, 'BB'), (33, 3, 'CC') SELECT partition_id, name FROM system.parts WHERE table = 'partitioned_table' AND database = currentDatabase() ORDER BY name; -SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/test/01650_drop_part_and_deduplication/partitioned_table/blocks/' ORDER BY value; +SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/01650_drop_part_and_deduplication_partitioned_table/blocks/' ORDER BY value; INSERT INTO partitioned_table VALUES (33, 3, 'CC'); -- must be deduplicated SELECT partition_id, name FROM system.parts WHERE table = 'partitioned_table' AND database = currentDatabase() ORDER BY name; -SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/test/01650_drop_part_and_deduplication/partitioned_table/blocks/' ORDER BY value; +SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/01650_drop_part_and_deduplication_partitioned_table/blocks/' ORDER BY value; ALTER TABLE partitioned_table DROP PART '3_1_1_0'; SELECT partition_id, name FROM system.parts WHERE table = 'partitioned_table' AND database = currentDatabase() ORDER BY name; -SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/test/01650_drop_part_and_deduplication/partitioned_table/blocks/' ORDER BY value; +SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/01650_drop_part_and_deduplication_partitioned_table/blocks/' ORDER BY value; INSERT INTO partitioned_table VALUES (33, 3, 'CC'); -- mustn't be deduplicated SELECT partition_id, name FROM system.parts WHERE table = 'partitioned_table' AND database = currentDatabase() ORDER BY name; -SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/test/01650_drop_part_and_deduplication/partitioned_table/blocks/' ORDER BY value; +SELECT substring(name, 1, 2), value FROM system.zookeeper WHERE path='/clickhouse/01650_drop_part_and_deduplication_partitioned_table/blocks/' ORDER BY value; DROP TABLE IF EXISTS partitioned_table; diff --git a/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference b/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference index 78462f9fc0e..2fc177c812e 100644 --- a/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference +++ b/tests/queries/0_stateless/01700_system_zookeeper_path_in.reference @@ -1,7 +1,16 @@ -clickhouse -task_queue -clickhouse -task_queue -clickhouse -task_queue -ddl +block_numbers +blocks +1 +======== +block_numbers +blocks +1 +======== +block_numbers +blocks +======== +1 +failed_parts +last_part +leader_election-0000000000 +parallel diff --git a/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql b/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql index a5c7488ef97..d4126098c7c 100644 --- a/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql +++ b/tests/queries/0_stateless/01700_system_zookeeper_path_in.sql @@ -1,6 +1,19 @@ -SELECT name FROM system.zookeeper WHERE path = '/'; -SELECT name FROM system.zookeeper WHERE path = 'clickhouse'; -SELECT name FROM system.zookeeper WHERE path IN ('/'); -SELECT name FROM system.zookeeper WHERE path IN ('clickhouse'); -SELECT name FROM system.zookeeper WHERE path IN ('/','/clickhouse'); -SELECT name FROM system.zookeeper WHERE path IN (SELECT concat('/clickhouse/',name) FROM system.zookeeper WHERE (path = '/clickhouse/')); \ No newline at end of file +DROP TABLE IF EXISTS sample_table; + +CREATE TABLE sample_table ( + key UInt64 +) +ENGINE ReplicatedMergeTree('/clickhouse/01700_system_zookeeper_path_in', '1') +ORDER BY tuple(); + +SELECT name FROM system.zookeeper WHERE path = '/clickhouse/01700_system_zookeeper_path_in' AND name like 'block%' ORDER BY name; +SELECT name FROM system.zookeeper WHERE path = '/clickhouse/01700_system_zookeeper_path_in/replicas' ORDER BY name; +SELECT '========'; +SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/01700_system_zookeeper_path_in') AND name LIKE 'block%' ORDER BY name; +SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/01700_system_zookeeper_path_in/replicas') ORDER BY name; +SELECT '========'; +SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/01700_system_zookeeper_path_in','/clickhouse/01700_system_zookeeper_path_in/replicas') AND name LIKE 'block%' ORDER BY name; +SELECT '========'; +SELECT name FROM system.zookeeper WHERE path IN (SELECT concat('/clickhouse/01700_system_zookeeper_path_in/', name) FROM system.zookeeper WHERE (path = '/clickhouse/01700_system_zookeeper_path_in')) ORDER BY name; + +DROP TABLE IF EXISTS sample_table; diff --git a/tests/queries/0_stateless/01710_join_use_nulls.reference b/tests/queries/0_stateless/01710_join_use_nulls.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01710_join_use_nulls.sql b/tests/queries/0_stateless/01710_join_use_nulls.sql new file mode 100644 index 00000000000..2845af8b8ed --- /dev/null +++ b/tests/queries/0_stateless/01710_join_use_nulls.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS X; +DROP TABLE IF EXISTS Y; + +CREATE TABLE X (id Int) ENGINE=Memory; +CREATE TABLE Y (id Int) ENGINE=Memory; + +-- Type mismatch of columns to JOIN by: plus(id, 1) Int64 at left, Y.id Int32 at right. +SELECT + Y.id - 1 +FROM X +RIGHT JOIN Y ON (X.id + 1) = Y.id +SETTINGS join_use_nulls=1; -- { serverError 53 } + +DROP TABLE X; +DROP TABLE Y; diff --git a/tests/queries/0_stateless/01711_decimal_multiplication.reference b/tests/queries/0_stateless/01711_decimal_multiplication.reference new file mode 100644 index 00000000000..37869329ca4 --- /dev/null +++ b/tests/queries/0_stateless/01711_decimal_multiplication.reference @@ -0,0 +1,4 @@ +2.0000 +2.0000 +2.0000 +2.0000 diff --git a/tests/queries/0_stateless/01711_decimal_multiplication.sql b/tests/queries/0_stateless/01711_decimal_multiplication.sql new file mode 100644 index 00000000000..10d23599b4d --- /dev/null +++ b/tests/queries/0_stateless/01711_decimal_multiplication.sql @@ -0,0 +1,4 @@ +SELECT materialize(toDecimal64(4,4)) - materialize(toDecimal32(2,2)); +SELECT toDecimal64(4,4) - materialize(toDecimal32(2,2)); +SELECT materialize(toDecimal64(4,4)) - toDecimal32(2,2); +SELECT toDecimal64(4,4) - toDecimal32(2,2); diff --git a/tests/queries/0_stateless/01715_background_checker_blather_zookeeper.reference b/tests/queries/0_stateless/01715_background_checker_blather_zookeeper.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01715_background_checker_blather_zookeeper.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01715_background_checker_blather_zookeeper.sql b/tests/queries/0_stateless/01715_background_checker_blather_zookeeper.sql new file mode 100644 index 00000000000..66b53369517 --- /dev/null +++ b/tests/queries/0_stateless/01715_background_checker_blather_zookeeper.sql @@ -0,0 +1,28 @@ +DROP TABLE IF EXISTS i20203_1; +DROP TABLE IF EXISTS i20203_2; + +CREATE TABLE i20203_1 (a Int8) +ENGINE = ReplicatedMergeTree('/clickhouse/01715_background_checker_i20203', 'r1') +ORDER BY tuple(); + +CREATE TABLE i20203_2 (a Int8) +ENGINE = ReplicatedMergeTree('/clickhouse/01715_background_checker_i20203', 'r2') +ORDER BY tuple(); + +DETACH TABLE i20203_2; +INSERT INTO i20203_1 VALUES (2); + +DETACH TABLE i20203_1; +ATTACH TABLE i20203_2; + +-- sleep 10 seconds +SELECT number from numbers(10) where sleepEachRow(1) Format Null; + +SELECT num_tries < 50 +FROM system.replication_queue +WHERE table = 'i20203_2' AND database = currentDatabase(); + +ATTACH TABLE i20203_1; + +DROP TABLE IF EXISTS i20203_1; +DROP TABLE IF EXISTS i20203_2; diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index d76603bf633..07250cd9c90 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -572,6 +572,8 @@ "01603_rename_overwrite_bug", "01646_system_restart_replicas_smoke", // system restart replicas is a global query "01676_dictget_in_default_expression", + "01715_background_checker_blather_zookeeper", + "01700_system_zookeeper_path_in", "attach", "ddl_dictionaries", "dictionary",