thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 09:32:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into replace-symlink

Browse Source
This commit is contained in:
Dan Roscigno 2022-08-28 20:10:23 -04:00 committed by GitHub
commit 74ce3462fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1001 additions and 305 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CMakeLists.txt
View File

@ -199,8 +199,8 @@ endif ()
option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.")
if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
# Can be lld or ld-lld.
if (LINKER_NAME MATCHES "lld$")
# Can be lld or ld-lld or lld-13 or /path/to/lld.
if (LINKER_NAME MATCHES "lld")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index")
message (STATUS "Adding .gdb-index via --gdb-index linker option.")
@ -368,8 +368,7 @@ endif()
set (COMPILER_FLAGS "${COMPILER_FLAGS}")
# Our built-in unwinder only supports DWARF version up to 4.
set (DEBUG_INFO_FLAGS "-g -gdwarf-4")
set (DEBUG_INFO_FLAGS "-g")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS}")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")

141
programs/bash-completion/completions/clickhouse-bootstrap
View File

@ -40,67 +40,98 @@ CLICKHOUSE_QueryKind=(
no_query
)
# SELECT name FROM system.formats ORDER BY name FORMAT LineAsString
CLICKHOUSE_Format=(
CapnProto
PostgreSQLWire
MySQLWire
JSONStringsEachRowWithProgress
JSONEachRowWithProgress
JSONCompact
JSON
CSV
Vertical
ODBCDriver2
PrettySpaceNoEscapes
Pretty
JSONCompactStrings
PrettyNoEscapes
ArrowStream
TabSeparatedWithNames
Parquet
Arrow
PrettyCompact
AvroConfluent
ORC
PrettyCompactNoEscapes
RawBLOB
Template
MsgPack
JSONCompactEachRow
CustomSeparated
TemplateIgnoreSpaces
Markdown
XML
ProtobufSingle
JSONCompactStringsEachRowWithNamesAndTypes
TSKV
TabSeparated
JSONStringEachRow
JSONStringsEachRow
TSVRaw
Values
TabSeparatedWithNamesAndTypes
PrettyCompactMonoBlock
TSVWithNamesAndTypes
ArrowStream
Avro
RowBinaryWithNamesAndTypes
LineAsString
Native
JSONCompactEachRowWithNamesAndTypes
PrettySpace
Regexp
TSV
JSONEachRow
CustomSeparatedIgnoreSpaces
AvroConfluent
CSV
CSVWithNames
JSONStrings
Null
TabSeparatedRaw
TSVWithNames
Protobuf
RowBinary
CSVWithNamesAndTypes
CapnProto
CustomSeparated
CustomSeparatedIgnoreSpaces
CustomSeparatedIgnoreSpacesWithNames
CustomSeparatedIgnoreSpacesWithNamesAndTypes
CustomSeparatedWithNames
CustomSeparatedWithNamesAndTypes
HiveText
JSON
JSONAsObject
JSONAsString
JSONColumns
JSONColumnsWithMetadata
JSONCompact
JSONCompactColumns
JSONCompactEachRow
JSONCompactEachRowWithNames
JSONCompactEachRowWithNamesAndTypes
JSONCompactStrings
JSONCompactStringsEachRow
JSONCompactStringsEachRowWithNames
JSONCompactStringsEachRowWithNamesAndTypes
JSONEachRow
JSONEachRowWithProgress
JSONLines
JSONStringEachRow
JSONStrings
JSONStringsEachRow
JSONStringsEachRowWithProgress
LineAsString
LineAsStringWithNames
LineAsStringWithNamesAndTypes
Markdown
MsgPack
MySQLDump
MySQLWire
NDJSON
Native
Null
ODBCDriver2
ORC
Parquet
PostgreSQLWire
Pretty
PrettyCompact
PrettyCompactMonoBlock
PrettyCompactNoEscapes
PrettyCompactNoEscapesMonoBlock
PrettyMonoBlock
PrettyNoEscapes
PrettyNoEscapesMonoBlock
PrettySpace
PrettySpaceMonoBlock
PrettySpaceNoEscapes
PrettySpaceNoEscapesMonoBlock
Prometheus
Protobuf
ProtobufList
ProtobufSingle
RawBLOB
Regexp
RowBinary
RowBinaryWithNames
RowBinaryWithNamesAndTypes
SQLInsert
TSKV
TSV
TSVRaw
TSVRawWithNames
TSVRawWithNamesAndTypes
TSVWithNames
TSVWithNamesAndTypes
TabSeparated
TabSeparatedRaw
TabSeparatedRawWithNames
TabSeparatedRawWithNamesAndTypes
TabSeparatedWithNames
TabSeparatedWithNamesAndTypes
Template
TemplateIgnoreSpaces
Values
Vertical
XML
)
function _clickhouse_bin_exist()

991
src/Common/Dwarf.cpp
View File

File diff suppressed because it is too large Load Diff

141
src/Common/Dwarf.h
View File

@ -19,6 +19,7 @@
*/
/** This file was edited for ClickHouse.
* Original is from folly library.
*/
#include <functional>
@ -113,8 +114,8 @@ public:
// seems as the same path can be represented in multiple ways
private:
std::string_view baseDir_; /// NOLINT
std::string_view subDir_; /// NOLINT
std::string_view file_; /// NOLINT
std::string_view subDir_; /// NOLINT
std::string_view file_; /// NOLINT
};
// Indicates inline function `name` is called at `line@file`.
@ -171,8 +172,6 @@ public:
private:
static bool findDebugInfoOffset(uintptr_t address, std::string_view aranges, uint64_t & offset);
void init();
std::shared_ptr<const Elf> elf_; /// NOLINT
// DWARF section made up of chunks, each prefixed with a length header.
@ -228,6 +227,7 @@ private:
{
uint64_t name = 0;
uint64_t form = 0;
int64_t implicitConst = 0; // only set when form=DW_FORM_implicit_const
explicit operator bool() const { return name != 0 || form != 0; }
};
@ -239,25 +239,43 @@ private:
std::variant<uint64_t, std::string_view> attr_value;
};
enum
{
DW_UT_compile = 0x01,
DW_UT_skeleton = 0x04,
};
struct CompilationUnit
{
bool is64Bit; /// NOLINT
uint8_t version;
uint8_t addr_size;
bool is64Bit = false; /// NOLINT
uint8_t version = 0;
uint8_t unit_type = DW_UT_compile; // DW_UT_compile or DW_UT_skeleton
uint8_t addr_size = 0;
// Offset in .debug_info of this compilation unit.
uint32_t offset;
uint32_t size;
uint32_t offset = 0;
uint32_t size = 0;
// Offset in .debug_info for the first DIE in this compilation unit.
uint32_t first_die;
uint64_t abbrev_offset;
uint32_t first_die = 0;
uint64_t abbrev_offset = 0;
// The beginning of the CU's contribution to .debug_addr
std::optional<uint64_t> addr_base; // DW_AT_addr_base (DWARF 5)
// The beginning of the offsets table (immediately following the
// header) of the CU's contribution to .debug_loclists
std::optional<uint64_t> loclists_base; // DW_AT_loclists_base (DWARF 5)
// The beginning of the offsets table (immediately following the
// header) of the CU's contribution to .debug_rnglists
std::optional<uint64_t> rnglists_base; // DW_AT_rnglists_base (DWARF 5)
// Points to the first string offset of the compilation unit’s
// contribution to the .debug_str_offsets (or .debug_str_offsets.dwo) section.
std::optional<uint64_t> str_offsets_base; // DW_AT_str_offsets_base (DWARF 5)
// Only the CompilationUnit that contains the caller functions needs this cache.
// Indexed by (abbr.code - 1) if (abbr.code - 1) < abbrCache.size();
std::vector<DIEAbbreviation> abbr_cache;
};
static CompilationUnit getCompilationUnit(std::string_view info, uint64_t offset);
/** cu must exist during the life cycle of created detail::Die. */
/** cu must exist during the life cycle of created Die. */
Die getDieAtOffset(const CompilationUnit & cu, uint64_t offset) const;
bool findLocation(
@ -278,16 +296,16 @@ private:
class LineNumberVM
{
public:
LineNumberVM(std::string_view data, std::string_view compilationDirectory);
LineNumberVM(
std::string_view data,
std::string_view compilationDirectory,
std::string_view debugStr,
std::string_view debugLineStr);
bool findAddress(uintptr_t target, Path & file, uint64_t & line);
/** Gets full file name at given index including directory. */
Path getFullFileName(uint64_t index) const
{
auto fn = getFileName(index);
return Path({}, getIncludeDirectory(fn.directoryIndex), fn.relativeName);
}
Path getFullFileName(uint64_t index) const;
private:
void init();
@ -327,24 +345,42 @@ private:
bool nextDefineFile(std::string_view & program, FileName & fn) const;
// Initialization
bool is64Bit_; /// NOLINT
std::string_view data_; /// NOLINT
std::string_view compilationDirectory_; /// NOLINT
bool is64Bit_; /// NOLINT
std::string_view data_; /// NOLINT
std::string_view compilationDirectory_; /// NOLINT
std::string_view debugStr_; // needed for DWARF 5 /// NOLINT
std::string_view debugLineStr_; // DWARF 5 /// NOLINT
// Header
uint16_t version_; /// NOLINT
uint8_t minLength_; /// NOLINT
uint16_t version_; /// NOLINT
uint8_t minLength_; /// NOLINT
bool defaultIsStmt_; /// NOLINT
int8_t lineBase_; /// NOLINT
uint8_t lineRange_; /// NOLINT
int8_t lineBase_; /// NOLINT
uint8_t lineRange_; /// NOLINT
uint8_t opcodeBase_; /// NOLINT
const uint8_t * standardOpcodeLengths_; /// NOLINT
std::string_view includeDirectories_; /// NOLINT
size_t includeDirectoryCount_; /// NOLINT
// 6.2.4 The Line Number Program Header.
struct
{
size_t includeDirectoryCount;
std::string_view includeDirectories;
size_t fileNameCount;
std::string_view fileNames;
} v4_;
std::string_view fileNames_; /// NOLINT
size_t fileNameCount_; /// NOLINT
struct
{
uint8_t directoryEntryFormatCount;
std::string_view directoryEntryFormat;
uint64_t directoriesCount;
std::string_view directories;
uint8_t fileNameEntryFormatCount;
std::string_view fileNameEntryFormat;
uint64_t fileNamesCount;
std::string_view fileNames;
} v5_;
// State machine registers
uint64_t address_; /// NOLINT
@ -397,20 +433,26 @@ private:
*/
size_t forEachAttribute(const CompilationUnit & cu, const Die & die, std::function<bool(const Attribute & die)> f) const;
Attribute readAttribute(const Die & die, AttributeSpec spec, std::string_view & info) const;
Attribute readAttribute(
const CompilationUnit & cu,
const Die & die,
AttributeSpec spec,
std::string_view & info) const;
// Read one attribute <name, form> pair, remove_prefix sp; returns <0, 0> at end.
static AttributeSpec readAttributeSpec(std::string_view & sp);
// Read one attribute value, remove_prefix sp
using AttributeValue = std::variant<uint64_t, std::string_view>;
AttributeValue readAttributeValue(std::string_view & sp, uint64_t form, bool is64Bit) const;
AttributeValue readAttributeValue(std::string_view & sp, uint64_t form, bool is64_bit) const;
// Get an ELF section by name, return true if found
bool getSection(const char * name, std::string_view * section) const;
std::string_view getSection(const char * name) const;
CompilationUnit getCompilationUnit(uint64_t offset) const;
// Finds the Compilation Unit starting at offset.
CompilationUnit findCompilationUnit(uint64_t targetOffset) const;
// Get a string from the .debug_str section
std::string_view getStringFromStringSection(uint64_t offset) const;
template <class T>
std::optional<T> getAttribute(const CompilationUnit & cu, const Die & die, uint64_t attr_name) const
@ -429,17 +471,24 @@ private:
}
// Check if the given address is in the range list at the given offset in .debug_ranges.
bool isAddrInRangeList(uint64_t address, std::optional<uint64_t> base_addr, size_t offset, uint8_t addr_size) const;
bool isAddrInRangeList(
const CompilationUnit & cu,
uint64_t address,
std::optional<uint64_t> base_addr,
size_t offset,
uint8_t addr_size) const;
// Finds the Compilation Unit starting at offset.
static CompilationUnit findCompilationUnit(std::string_view info, uint64_t targetOffset);
std::string_view info_; // .debug_info /// NOLINT
std::string_view abbrev_; // .debug_abbrev /// NOLINT
std::string_view aranges_; // .debug_aranges /// NOLINT
std::string_view line_; // .debug_line /// NOLINT
std::string_view strings_; // .debug_str /// NOLINT
std::string_view ranges_; // .debug_ranges /// NOLINT
std::string_view abbrev_; // .debug_abbrev /// NOLINT
std::string_view addr_; // .debug_addr (DWARF 5) /// NOLINT
std::string_view aranges_; // .debug_aranges /// NOLINT
std::string_view info_; // .debug_info /// NOLINT
std::string_view line_; // .debug_line /// NOLINT
std::string_view line_str_; // .debug_line_str (DWARF 5) /// NOLINT
std::string_view loclists_; // .debug_loclists (DWARF 5) /// NOLINT
std::string_view ranges_; // .debug_ranges /// NOLINT
std::string_view rnglists_; // .debug_rnglists (DWARF 5) /// NOLINT
std::string_view str_; // .debug_str /// NOLINT
std::string_view str_offsets_; // .debug_str_offsets (DWARF 5) /// NOLINT
};
}

18
tests/queries/0_stateless/02382_filesystem_cache_persistent_files.reference
View File

@ -9,7 +9,7 @@ INSERT INTO nopers SELECT number, toString(number) FROM numbers(10);
SELECT * FROM nopers FORMAT Null;
SELECT sum(size) FROM system.filesystem_cache;
194
SELECT substring(local_path, 58, 10) as file, substring(cache_path, 53, 20) as cache, size
SELECT extract(local_path, '.*/([\w.]+)') as file, extract(cache_path, '.*/([\w.]+)') as cache, size
FROM
(
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path
@ -19,8 +19,8 @@ INNER JOIN
system.filesystem_cache AS caches
ON data_paths.cache_path = caches.cache_path
ORDER BY file, cache, size;
data.bin 114
data.mrk3 80
data.bin 0 114
data.mrk3 0 80
DROP TABLE IF EXISTS test;
CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760;
SYSTEM STOP MERGES test;
@ -32,7 +32,7 @@ SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, re
4
SELECT count() FROM system.filesystem_cache;
4
SELECT substring(local_path, 58, 10) as file, substring(cache_path, 53, 20) as cache, size
SELECT extract(local_path, '.*/([\w.]+)') as file, extract(cache_path, '.*/([\w.]+)') as cache, size
FROM
(
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path
@ -42,9 +42,9 @@ INNER JOIN
system.filesystem_cache AS caches
ON data_paths.cache_path = caches.cache_path
ORDER BY file, cache, size;
data.bin 114
data.bin 0 114
data.bin 0 746
data.mrk3 80
data.mrk3 0 80
data.mrk3 0_persistent 80
DROP TABLE IF EXISTS test2;
CREATE TABLE test2 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_small', min_bytes_for_wide_part = 10485760;
@ -57,7 +57,7 @@ SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, re
4
SELECT count() FROM system.filesystem_cache;
4
SELECT substring(local_path, 58, 10) as file, substring(cache_path, 53, 20) as cache, size
SELECT extract(local_path, '.*/([\w.]+)') as file, extract(cache_path, '.*/([\w.]+)') as cache, size
FROM
(
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path
@ -67,8 +67,8 @@ INNER JOIN
system.filesystem_cache AS caches
ON data_paths.cache_path = caches.cache_path
ORDER BY file, cache, size;
data.bin 114
data.mrk3 80
data.bin 0 114
data.mrk3 0 80
data.mrk3 0_persistent 80
data.mrk3 0_persistent 520
DROP TABLE test;

8
tests/queries/0_stateless/02382_filesystem_cache_persistent_files.sql
View File

@ -1,4 +1,4 @@
-- Tags: disabled
-- Tags: no-fasttest, no-parallel, no-s3-storage, no-random-settings
-- { echo }
@ -14,7 +14,7 @@ INSERT INTO nopers SELECT number, toString(number) FROM numbers(10);
SELECT * FROM nopers FORMAT Null;
SELECT sum(size) FROM system.filesystem_cache;
SELECT substring(local_path, 58, 10) as file, substring(cache_path, 53, 20) as cache, size
SELECT extract(local_path, '.*/([\w.]+)') as file, extract(cache_path, '.*/([\w.]+)') as cache, size
FROM
(
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path
@ -37,7 +37,7 @@ SELECT sum(size) FROM system.filesystem_cache;
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path;
SELECT count() FROM system.filesystem_cache;
SELECT substring(local_path, 58, 10) as file, substring(cache_path, 53, 20) as cache, size
SELECT extract(local_path, '.*/([\w.]+)') as file, extract(cache_path, '.*/([\w.]+)') as cache, size
FROM
(
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path
@ -60,7 +60,7 @@ SELECT sum(size) FROM system.filesystem_cache;
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path;
SELECT count() FROM system.filesystem_cache;
SELECT substring(local_path, 58, 10) as file, substring(cache_path, 53, 20) as cache, size
SELECT extract(local_path, '.*/([\w.]+)') as file, extract(cache_path, '.*/([\w.]+)') as cache, size
FROM
(
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path