thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-29 19:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'pyos-llvm-jit' of github.com:yandex/ClickHouse into pyos-llvm-jit

Browse Source
This commit is contained in:
Alexey Milovidov 2018-05-07 09:53:36 +03:00
commit b5b95e8ed3
107 changed files with 2843 additions and 1447 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
.clang-format
View File

@ -1,24 +1,21 @@
---
BasedOnStyle: WebKit
Language: Cpp
BasedOnStyle: WebKit
Language: Cpp
AlignAfterOpenBracket: false
BreakBeforeBraces: Custom
BraceWrapping: {
AfterClass: 'true'
AfterControlStatement: 'true'
AfterEnum : 'true'
AfterFunction : 'true'
AfterNamespace : 'true'
AfterStruct : 'true'
AfterUnion : 'true'
BeforeCatch : 'true'
BeforeElse : 'true'
IndentBraces : 'false'
}
BraceWrapping:
AfterClass: true
AfterControlStatement: true
AfterEnum: true
AfterFunction: true
AfterNamespace: true
AfterStruct: true
AfterUnion: true
BeforeCatch: true
BeforeElse: true
IndentBraces: false
BreakConstructorInitializersBeforeComma: false
Cpp11BracedListStyle: true
ColumnLimit: 140
ColumnLimit: 140
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ExperimentalAutoDetectBinPacking: true
UseTab: Never
@ -28,36 +25,35 @@ Standard: Cpp11
PointerAlignment: Middle
MaxEmptyLinesToKeep: 2
KeepEmptyLinesAtTheStartOfBlocks: false
#AllowShortFunctionsOnASingleLine: Inline
AllowShortFunctionsOnASingleLine: Empty
AlwaysBreakTemplateDeclarations: true
IndentCaseLabels: true
#SpaceAfterTemplateKeyword: true
SpaceAfterTemplateKeyword: true
SortIncludes: true
IncludeCategories:
- Regex: '^<[a-z_]+>'
Priority: 1
- Regex: '^<[a-z_]+.h>'
Priority: 2
- Regex: '^["<](common|ext|mysqlxx|daemon|zkutil)/'
Priority: 90
- Regex: '^["<](DB)/'
Priority: 100
- Regex: '^["<](Poco)/'
Priority: 50
- Regex: '^"'
Priority: 110
- Regex: '/'
Priority: 30
- Regex: '.*'
Priority: 40
- Regex: '^<[a-z_]+>'
Priority: 1
- Regex: '^<[a-z_]+.h>'
Priority: 2
- Regex: '^["<](common|ext|mysqlxx|daemon|zkutil)/'
Priority: 90
- Regex: '^["<](DB)/'
Priority: 100
- Regex: '^["<](Poco)/'
Priority: 50
- Regex: '^"'
Priority: 110
- Regex: '/'
Priority: 30
- Regex: '.*'
Priority: 40
ReflowComments: false
AlignEscapedNewlinesLeft: true
# Not changed:
AccessModifierOffset: -4
AlignConsecutiveAssignments: false
AlignOperands: false
AlignOperands: false
AlignTrailingComments: false
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
@ -70,16 +66,15 @@ BinPackArguments: false
BinPackParameters: false
BreakBeforeBinaryOperators: All
BreakBeforeTernaryOperators: true
CommentPragmas: '^ IWYU pragma:'
CommentPragmas: '^ IWYU pragma:'
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
DerivePointerAlignment: false
DisableFormat: false
ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ]
IndentWidth: 4
DisableFormat: false
IndentWidth: 4
IndentWrappedFunctionNames: false
MacroBlockBegin: ''
MacroBlockEnd: ''
MacroBlockEnd: ''
NamespaceIndentation: Inner
ObjCBlockIndentWidth: 4
ObjCSpaceAfterProperty: true
@ -99,5 +94,3 @@ SpacesInContainerLiterals: true
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
...

13
cmake/dbms_generate_function.cmake
View File

@ -1,13 +1,10 @@
function(generate_function_register FUNCTION_AREA)
foreach(FUNCTION IN LISTS ARGN)
configure_file (registerFunction.h.in register${FUNCTION}.h)
configure_file (registerFunction.cpp.in register${FUNCTION}.cpp)
set(REGISTER_HEADERS "${REGISTER_HEADERS} #include \"register${FUNCTION}.h\"\n")
set(REGISTER_FUNCTIONS "${REGISTER_FUNCTIONS} register${FUNCTION}(factory);\n")
foreach(FUNCTION IN LISTS ARGN)
configure_file (registerFunction.h.in register${FUNCTION}.h)
configure_file (registerFunction.cpp.in register${FUNCTION}.cpp)
set(REGISTER_HEADERS "${REGISTER_HEADERS}#include \"register${FUNCTION}.h\"\n")
set(REGISTER_FUNCTIONS "${REGISTER_FUNCTIONS} register${FUNCTION}(factory);\n")
endforeach()
configure_file (registerFunctions_area.cpp.in registerFunctions${FUNCTION_AREA}.cpp)
endfunction()

3
cmake/find_rdkafka.cmake
View File

@ -24,6 +24,9 @@ if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR)
if (SASL2_LIBRARY)
list (APPEND RDKAFKA_LIBRARY ${SASL2_LIBRARY})
endif ()
if (LZ4_LIBRARY)
list (APPEND RDKAFKA_LIBRARY ${LZ4_LIBRARY})
endif ()
elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY)
set (USE_INTERNAL_RDKAFKA_LIBRARY 1)
set (RDKAFKA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src")

2
contrib/capnproto vendored

@ -1 +1 @@
Subproject commit c949a18da5f041a36cc218c5c4b79c7705999b4f
Subproject commit 7173ab638fdf144032411dc69fb1082cd473e08f

3
dbms/CMakeLists.txt
View File

@ -103,9 +103,6 @@ if (USE_EMBEDDED_COMPILER)
llvm_map_components_to_libnames(REQUIRED_LLVM_LIBRARIES all)
target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES})
target_include_directories (dbms BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
# LLVM has a bunch of unused parameters in its header files.
set_source_files_properties(src/Functions/IFunction.cpp PROPERTIES COMPILE_FLAGS "-Wno-unused-parameter")
set_source_files_properties(src/Interpreters/ExpressionJIT.cpp PROPERTIES COMPILE_FLAGS "-Wno-unused-parameter -Wno-non-virtual-dtor")
endif ()

19
dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h
View File

@ -242,9 +242,7 @@ private:
auto throw_exception = [&](const std::string & msg)
{
throw Exception{
msg + " '" + std::string(pos, end) + "' at position " + toString(pos - begin),
ErrorCodes::SYNTAX_ERROR};
throw Exception{msg + " '" + std::string(pos, end) + "' at position " + toString(pos - begin), ErrorCodes::SYNTAX_ERROR};
};
auto match = [&pos, end](const char * str) mutable
@ -286,9 +284,7 @@ private:
if (actions.back().type != PatternActionType::SpecificEvent &&
actions.back().type != PatternActionType::AnyEvent &&
actions.back().type != PatternActionType::KleeneStar)
throw Exception{
"Temporal condition should be preceeded by an event condition",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{"Temporal condition should be preceeded by an event condition", ErrorCodes::BAD_ARGUMENTS};
actions.emplace_back(type, duration);
}
@ -301,9 +297,7 @@ private:
throw_exception("Could not parse number");
if (event_number > arg_count - 1)
throw Exception{
"Event number " + toString(event_number) + " is out of range",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{"Event number " + toString(event_number) + " is out of range", ErrorCodes::BAD_ARGUMENTS};
actions.emplace_back(PatternActionType::SpecificEvent, event_number - 1);
}
@ -428,13 +422,10 @@ protected:
break;
}
else
throw Exception{
"Unknown PatternActionType",
ErrorCodes::LOGICAL_ERROR};
throw Exception{"Unknown PatternActionType", ErrorCodes::LOGICAL_ERROR};
if (++i > sequence_match_max_iterations)
throw Exception{
"Pattern application proves too difficult, exceeding max iterations (" + toString(sequence_match_max_iterations) + ")",
throw Exception{"Pattern application proves too difficult, exceeding max iterations (" + toString(sequence_match_max_iterations) + ")",
ErrorCodes::TOO_SLOW};
}

2
dbms/src/AggregateFunctions/QuantileTiming.h
View File

@ -179,7 +179,7 @@ namespace detail
? level * elems.size()
: (elems.size() - 1);
/// Sorting an array will not be considered a violation of constancy.
/// Sorting an array will not be considered a violation of constancy.
auto & array = const_cast<Array &>(elems);
std::nth_element(array.begin(), array.begin() + n, array.end());
quantile = array[n];

8
dbms/src/AggregateFunctions/ReservoirSamplerDeterministic.h
View File

@ -15,10 +15,10 @@
#include <Poco/Exception.h>
/// Implementation of Reservoir Sampling algorithm. Incrementally selects from the added objects a random subset of the `sample_count` size.
/// Can approximately get quantiles.
/// The `quantile` call takes O(sample_count log sample_count), if after the previous call `quantile` there was at least one call to insert. Otherwise, O(1).
/// That is, it makes sense to first add, then get quantiles without adding.
/// Implementation of Reservoir Sampling algorithm. Incrementally selects from the added objects a random subset of the `sample_count` size.
/// Can approximately get quantiles.
/// The `quantile` call takes O(sample_count log sample_count), if after the previous call `quantile` there was at least one call to insert. Otherwise, O(1).
/// That is, it makes sense to first add, then get quantiles without adding.
namespace DB

4
dbms/src/Columns/ColumnConst.h
View File

@ -91,9 +91,9 @@ public:
return data->getInt(0);
}
UInt8 getBoolRepresentation(size_t) const override
bool getBool(size_t) const override
{
return data->getBoolRepresentation(0);
return data->getBool(0);
}
bool isNullAt(size_t) const override

2
dbms/src/Columns/ColumnNullable.h
View File

@ -46,7 +46,7 @@ public:
bool isNullAt(size_t n) const override { return static_cast<const ColumnUInt8 &>(*null_map).getData()[n] != 0;}
Field operator[](size_t n) const override;
void get(size_t n, Field & res) const override;
UInt8 getBoolRepresentation(size_t n) const override { return isNullAt(n) ? 0 : nested_column->getBoolRepresentation(n); }
bool getBool(size_t n) const override { return isNullAt(n) ? 0 : nested_column->getBool(n); }
UInt64 get64(size_t n) const override { return nested_column->get64(n); }
StringRef getDataAt(size_t n) const override;
void insertData(const char * pos, size_t length) override;

4
dbms/src/Columns/ColumnVector.h
View File

@ -231,9 +231,9 @@ public:
return UInt64(data[n]);
}
UInt8 getBoolRepresentation(size_t n) const override
bool getBool(size_t n) const override
{
return UInt8(!!data[n]);
return bool(data[n]);
}
Int64 getInt(size_t n) const override

15
dbms/src/Columns/IColumn.h
View File

@ -88,6 +88,7 @@ public:
}
/** If column is numeric, return value of n-th element, casted to UInt64.
* For NULL values of Nullable column it is allowed to return arbitary value.
* Otherwise throw an exception.
*/
virtual UInt64 getUInt(size_t /*n*/) const
@ -95,11 +96,6 @@ public:
throw Exception("Method getUInt is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
virtual UInt8 getBoolRepresentation(size_t /*n*/) const
{
throw Exception("Method getBoolRepresentation is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
virtual Int64 getInt(size_t /*n*/) const
{
throw Exception("Method getInt is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
@ -107,6 +103,15 @@ public:
virtual bool isNullAt(size_t /*n*/) const { return false; }
/** If column is numeric, return value of n-th element, casted to bool.
* For NULL values of Nullable column returns false.
* Otherwise throw an exception.
*/
virtual bool getBool(size_t /*n*/) const
{
throw Exception("Method getBool is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Removes all elements outside of specified range.
/// Is used in LIMIT operation, for example.
virtual Ptr cut(size_t start, size_t length) const

3
dbms/src/Common/Config/ConfigProcessor.cpp
View File

@ -373,7 +373,8 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
std::vector<std::string> merge_dirs;
merge_dirs.push_back(merge_dir_path.toString());
if (merge_dir_path.getBaseName() != "conf") {
if (merge_dir_path.getBaseName() != "conf")
{
merge_dir_path.setBaseName("conf");
merge_dirs.push_back(merge_dir_path.toString());
}

2
dbms/src/Common/ExternalTable.h
View File

@ -165,7 +165,7 @@ public:
/// Parsing of external table used when sending tables via http
/// The `handlePart` function will be called for each table passed,
/// so it's also necessary to call `clean` at the end of the `handlePart`.
/// so it's also necessary to call `clean` at the end of the `handlePart`.
class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable
{
public:

2
dbms/src/Common/ProfileEvents.cpp
View File

@ -79,6 +79,8 @@
M(CompileAttempt) \
M(CompileSuccess) \
\
M(CompileFunction) \
\
M(ExternalSortWritePart) \
M(ExternalSortMerge) \
M(ExternalAggregationWritePart) \

20
dbms/src/Common/StringSearcher.h
View File

@ -121,7 +121,7 @@ public:
continue;
}
const auto src_len = DB::UTF8::seqLength(*needle_pos);
const auto src_len = UTF8::seqLength(*needle_pos);
const auto c_u32 = utf8.convert(needle_pos);
const auto c_l_u32 = Poco::Unicode::toLower(c_u32);
@ -132,9 +132,7 @@ public:
/// @note Unicode standard states it is a rare but possible occasion
if (!(dst_l_len == dst_u_len && dst_u_len == src_len))
throw DB::Exception{
"UTF8 sequences with different lowercase and uppercase lengths are not supported",
DB::ErrorCodes::UNSUPPORTED_PARAMETER};
throw Exception{"UTF8 sequences with different lowercase and uppercase lengths are not supported", ErrorCodes::UNSUPPORTED_PARAMETER};
cache_actual_len += src_len;
if (cache_actual_len < n)
@ -183,7 +181,7 @@ public:
Poco::Unicode::toLower(utf8.convert(needle_pos)))
{
/// @note assuming sequences for lowercase and uppercase have exact same length
const auto len = DB::UTF8::seqLength(*pos);
const auto len = UTF8::seqLength(*pos);
pos += len, needle_pos += len;
}
@ -207,7 +205,7 @@ public:
Poco::Unicode::toLower(utf8.convert(pos)) ==
Poco::Unicode::toLower(utf8.convert(needle_pos)))
{
const auto len = DB::UTF8::seqLength(*pos);
const auto len = UTF8::seqLength(*pos);
pos += len, needle_pos += len;
}
@ -240,7 +238,7 @@ public:
if (mask == 0)
{
haystack += n;
DB::UTF8::syncForward(haystack, haystack_end);
UTF8::syncForward(haystack, haystack_end);
continue;
}
@ -267,7 +265,7 @@ public:
Poco::Unicode::toLower(utf8.convert(needle_pos)))
{
/// @note assuming sequences for lowercase and uppercase have exact same length
const auto len = DB::UTF8::seqLength(*haystack_pos);
const auto len = UTF8::seqLength(*haystack_pos);
haystack_pos += len, needle_pos += len;
}
@ -279,7 +277,7 @@ public:
return haystack;
/// first octet was ok, but not the first 16, move to start of next sequence and reapply
haystack += DB::UTF8::seqLength(*haystack);
haystack += UTF8::seqLength(*haystack);
continue;
}
}
@ -297,7 +295,7 @@ public:
Poco::Unicode::toLower(utf8.convert(haystack_pos)) ==
Poco::Unicode::toLower(utf8.convert(needle_pos)))
{
const auto len = DB::UTF8::seqLength(*haystack_pos);
const auto len = UTF8::seqLength(*haystack_pos);
haystack_pos += len, needle_pos += len;
}
@ -306,7 +304,7 @@ public:
}
/// advance to the start of the next sequence
haystack += DB::UTF8::seqLength(*haystack);
haystack += UTF8::seqLength(*haystack);
}
return haystack_end;

1
dbms/src/Common/StringUtils/StringUtils.h
View File

@ -3,6 +3,7 @@
#include <string>
#include <cstring>
#include <cstddef>
#include <type_traits>
namespace detail

64
dbms/src/Common/tests/AvalancheTest.cpp
View File

@ -4,55 +4,57 @@
//-----------------------------------------------------------------------------
void PrintAvalancheDiagram ( int x, int y, int reps, double scale, int * bins )
void PrintAvalancheDiagram(int x, int y, int reps, double scale, int * bins)
{
const char * symbols = ".123456789X";
const char * symbols = ".123456789X";
for(int i = 0; i < y; i++)
{
printf("[");
for(int j = 0; j < x; j++)
for (int i = 0; i < y; i++)
{
int k = (y - i) -1;
printf("[");
for (int j = 0; j < x; j++)
{
int k = (y - i) - 1;
int bin = bins[k + (j*y)];
int bin = bins[k + (j * y)];
double b = double(bin) / double(reps);
b = fabs(b*2 - 1);
double b = double(bin) / double(reps);
b = fabs(b * 2 - 1);
b *= scale;
b *= scale;
int s = static_cast<int>(floor(b*10));
int s = static_cast<int>(floor(b * 10));
if(s > 10) s = 10;
if(s < 0) s = 0;
if (s > 10)
s = 10;
if (s < 0)
s = 0;
printf("%c",symbols[s]);
printf("%c", symbols[s]);
}
printf("]\n");
}
printf("]\n");
}
}
//----------------------------------------------------------------------------
double maxBias ( std::vector<int> & counts, int reps )
double maxBias(std::vector<int> & counts, int reps)
{
double worst = 0;
double worst = 0;
for(int i = 0; i < static_cast<int>(counts.size()); i++)
{
double c = static_cast<double>(counts[i]) / static_cast<double>(reps);
double d = fabs(c * 2 - 1);
if(d > worst)
for (int i = 0; i < static_cast<int>(counts.size()); i++)
{
worst = d;
}
}
double c = static_cast<double>(counts[i]) / static_cast<double>(reps);
return worst;
double d = fabs(c * 2 - 1);
if (d > worst)
{
worst = d;
}
}
return worst;
}
//-----------------------------------------------------------------------------

326
dbms/src/Common/tests/AvalancheTest.h
View File

@ -14,131 +14,134 @@
#include "Random.h"
#include <vector>
#include <stdio.h>
#include <math.h>
#include <stdio.h>
// Avalanche fails if a bit is biased by more than 1%
#define AVALANCHE_FAIL 0.01
double maxBias ( std::vector<int> & counts, int reps );
double maxBias(std::vector<int> & counts, int reps);
typedef void (*pfHash) ( const void * blob, const int len, const uint32_t seed, void * out );
typedef void (*pfHash)(const void * blob, const int len, const uint32_t seed, void * out);
inline uint32_t getbit ( const void * block, int len, uint32_t bit )
inline uint32_t getbit(const void * block, int len, uint32_t bit)
{
uint8_t * b = reinterpret_cast<uint8_t*>(const_cast<void*>(block));
uint8_t * b = reinterpret_cast<uint8_t *>(const_cast<void *>(block));
int byte = bit >> 3;
bit = bit & 0x7;
int byte = bit >> 3;
bit = bit & 0x7;
if(byte < len) return (b[byte] >> bit) & 1;
if (byte < len)
return (b[byte] >> bit) & 1;
return 0;
return 0;
}
template < typename T >
inline uint32_t getbit ( T & blob, uint32_t bit )
template <typename T>
inline uint32_t getbit(T & blob, uint32_t bit)
{
return getbit(&blob,sizeof(blob),bit);
return getbit(&blob, sizeof(blob), bit);
}
inline void flipbit ( void * block, int len, uint32_t bit )
inline void flipbit(void * block, int len, uint32_t bit)
{
uint8_t * b = reinterpret_cast<uint8_t*>(block);
uint8_t * b = reinterpret_cast<uint8_t *>(block);
int byte = bit >> 3;
bit = bit & 0x7;
int byte = bit >> 3;
bit = bit & 0x7;
if(byte < len) b[byte] ^= (1 << bit);
if (byte < len)
b[byte] ^= (1 << bit);
}
template < typename T >
inline void flipbit ( T & blob, uint32_t bit )
template <typename T>
inline void flipbit(T & blob, uint32_t bit)
{
flipbit(&blob,sizeof(blob),bit);
flipbit(&blob, sizeof(blob), bit);
}
//-----------------------------------------------------------------------------
template < typename keytype, typename hashtype >
void calcBias ( pfHash hash, std::vector<int> & counts, int reps, Rand & r )
template <typename keytype, typename hashtype>
void calcBias(pfHash hash, std::vector<int> & counts, int reps, Rand & r)
{
const int keybytes = sizeof(keytype);
const int hashbytes = sizeof(hashtype);
const int keybytes = sizeof(keytype);
const int hashbytes = sizeof(hashtype);
const int keybits = keybytes * 8;
const int hashbits = hashbytes * 8;
const int keybits = keybytes * 8;
const int hashbits = hashbytes * 8;
keytype K;
hashtype A,B;
keytype K;
hashtype A, B;
for(int irep = 0; irep < reps; irep++)
{
if(irep % (reps/10) == 0) printf(".");
r.rand_p(&K,keybytes);
hash(&K,keybytes,0,&A);
int * cursor = &counts[0];
for(int iBit = 0; iBit < keybits; iBit++)
for (int irep = 0; irep < reps; irep++)
{
flipbit(&K,keybytes,iBit);
hash(&K,keybytes,0,&B);
flipbit(&K,keybytes,iBit);
if (irep % (reps / 10) == 0)
printf(".");
for(int iOut = 0; iOut < hashbits; iOut++)
{
int bitA = getbit(&A,hashbytes,iOut);
int bitB = getbit(&B,hashbytes,iOut);
r.rand_p(&K, keybytes);
(*cursor++) += (bitA ^ bitB);
}
hash(&K, keybytes, 0, &A);
int * cursor = &counts[0];
for (int iBit = 0; iBit < keybits; iBit++)
{
flipbit(&K, keybytes, iBit);
hash(&K, keybytes, 0, &B);
flipbit(&K, keybytes, iBit);
for (int iOut = 0; iOut < hashbits; iOut++)
{
int bitA = getbit(&A, hashbytes, iOut);
int bitB = getbit(&B, hashbytes, iOut);
(*cursor++) += (bitA ^ bitB);
}
}
}
}
}
//-----------------------------------------------------------------------------
template < typename keytype, typename hashtype >
bool AvalancheTest ( pfHash hash, const int reps )
template <typename keytype, typename hashtype>
bool AvalancheTest(pfHash hash, const int reps)
{
Rand r(48273);
Rand r(48273);
const int keybytes = sizeof(keytype);
const int hashbytes = sizeof(hashtype);
const int keybytes = sizeof(keytype);
const int hashbytes = sizeof(hashtype);
const int keybits = keybytes * 8;
const int hashbits = hashbytes * 8;
const int keybits = keybytes * 8;
const int hashbits = hashbytes * 8;
printf("Testing %3d-bit keys -> %3d-bit hashes, %8d reps",keybits,hashbits,reps);
printf("Testing %3d-bit keys -> %3d-bit hashes, %8d reps", keybits, hashbits, reps);
//----------
//----------
std::vector<int> bins(keybits*hashbits,0);
std::vector<int> bins(keybits * hashbits, 0);
calcBias<keytype,hashtype>(hash,bins,reps,r);
calcBias<keytype, hashtype>(hash, bins, reps, r);
//----------
//----------
bool result = true;
bool result = true;
double b = maxBias(bins,reps);
double b = maxBias(bins, reps);
printf(" worst bias is %f%%",b * 100.0);
printf(" worst bias is %f%%", b * 100.0);
if(b > AVALANCHE_FAIL)
{
printf(" !!!!! ");
result = false;
}
if (b > AVALANCHE_FAIL)
{
printf(" !!!!! ");
result = false;
}
printf("\n");
printf("\n");
return result;
return result;
}
@ -146,107 +149,116 @@ bool AvalancheTest ( pfHash hash, const int reps )
// BIC test variant - store all intermediate data in a table, draw diagram
// afterwards (much faster)
template < typename keytype, typename hashtype >
void BicTest3 ( pfHash hash, const int reps, bool verbose = true )
template <typename keytype, typename hashtype>
void BicTest3(pfHash hash, const int reps, bool verbose = true)
{
const int keybytes = sizeof(keytype);
const int keybits = keybytes * 8;
const int hashbytes = sizeof(hashtype);
const int hashbits = hashbytes * 8;
const int pagesize = hashbits*hashbits*4;
const int keybytes = sizeof(keytype);
const int keybits = keybytes * 8;
const int hashbytes = sizeof(hashtype);
const int hashbits = hashbytes * 8;
const int pagesize = hashbits * hashbits * 4;
Rand r(11938);
Rand r(11938);
double maxBias = 0;
int maxK = 0;
int maxA = 0;
int maxB = 0;
double maxBias = 0;
int maxK = 0;
int maxA = 0;
int maxB = 0;
keytype key;
hashtype h1,h2;
keytype key;
hashtype h1, h2;
std::vector<int> bins(keybits*pagesize,0);
std::vector<int> bins(keybits * pagesize, 0);
for(int keybit = 0; keybit < keybits; keybit++)
{
if(keybit % (keybits/10) == 0) printf(".");
int * page = &bins[keybit*pagesize];
for(int irep = 0; irep < reps; irep++)
for (int keybit = 0; keybit < keybits; keybit++)
{
r.rand_p(&key,keybytes);
hash(&key,keybytes,0,&h1);
flipbit(key,keybit);
hash(&key,keybytes,0,&h2);
if (keybit % (keybits / 10) == 0)
printf(".");
hashtype d = h1 ^ h2;
int * page = &bins[keybit * pagesize];
for(int out1 = 0; out1 < hashbits-1; out1++)
for(int out2 = out1+1; out2 < hashbits; out2++)
{
int * b = &page[(out1*hashbits+out2)*4];
uint32_t x = getbit(d,out1) | (getbit(d,out2) << 1);
b[x]++;
}
}
}
printf("\n");
for(int out1 = 0; out1 < hashbits-1; out1++)
{
for(int out2 = out1+1; out2 < hashbits; out2++)
{
if(verbose) printf("(%3d,%3d) - ",out1,out2);
for(int keybit = 0; keybit < keybits; keybit++)
{
int * page = &bins[keybit*pagesize];
int * bins = &page[(out1*hashbits+out2)*4];
double bias = 0;
for(int b = 0; b < 4; b++)
for (int irep = 0; irep < reps; irep++)
{
double b2 = static_cast<double>(bins[b]) / static_cast<double>(reps / 2);
b2 = fabs(b2 * 2 - 1);
r.rand_p(&key, keybytes);
hash(&key, keybytes, 0, &h1);
flipbit(key, keybit);
hash(&key, keybytes, 0, &h2);
if(b2 > bias) bias = b2;
hashtype d = h1 ^ h2;
for (int out1 = 0; out1 < hashbits - 1; out1++)
for (int out2 = out1 + 1; out2 < hashbits; out2++)
{
int * b = &page[(out1 * hashbits + out2) * 4];
uint32_t x = getbit(d, out1) | (getbit(d, out2) << 1);
b[x]++;
}
}
if(bias > maxBias)
{
maxBias = bias;
maxK = keybit;
maxA = out1;
maxB = out2;
}
if(verbose)
{
if (bias < 0.01) printf(".");
else if(bias < 0.05) printf("o");
else if(bias < 0.33) printf("O");
else printf("X");
}
}
// Finished keybit
if(verbose) printf("\n");
}
if(verbose)
{
for(int i = 0; i < keybits+12; i++) printf("-");
printf("\n");
}
}
printf("\n");
printf("Max bias %f - (%3d : %3d,%3d)\n",maxBias,maxK,maxA,maxB);
for (int out1 = 0; out1 < hashbits - 1; out1++)
{
for (int out2 = out1 + 1; out2 < hashbits; out2++)
{
if (verbose)
printf("(%3d,%3d) - ", out1, out2);
for (int keybit = 0; keybit < keybits; keybit++)
{
int * page = &bins[keybit * pagesize];
int * bins = &page[(out1 * hashbits + out2) * 4];
double bias = 0;
for (int b = 0; b < 4; b++)
{
double b2 = static_cast<double>(bins[b]) / static_cast<double>(reps / 2);
b2 = fabs(b2 * 2 - 1);
if (b2 > bias)
bias = b2;
}
if (bias > maxBias)
{
maxBias = bias;
maxK = keybit;
maxA = out1;
maxB = out2;
}
if (verbose)
{
if (bias < 0.01)
printf(".");
else if (bias < 0.05)
printf("o");
else if (bias < 0.33)
printf("O");
else
printf("X");
}
}
// Finished keybit
if (verbose)
printf("\n");
}
if (verbose)
{
for (int i = 0; i < keybits + 12; i++)
printf("-");
printf("\n");
}
}
printf("Max bias %f - (%3d : %3d,%3d)\n", maxBias, maxK, maxA, maxB);
}
//-----------------------------------------------------------------------------

186
dbms/src/Common/tests/Random.h
View File

@ -10,110 +10,120 @@
struct Rand
{
uint32_t x;
uint32_t y;
uint32_t z;
uint32_t w;
uint32_t x;
uint32_t y;
uint32_t z;
uint32_t w;
Rand()
{
reseed(static_cast<uint32_t>(0));
}
explicit Rand( uint32_t seed )
{
reseed(seed);
}
void reseed ( uint32_t seed )
{
x = 0x498b3bc5 ^ seed;
y = 0;
z = 0;
w = 0;
for(int i = 0; i < 10; i++) mix();
}
void reseed ( uint64_t seed )
{
x = 0x498b3bc5 ^ static_cast<uint32_t>(seed >> 0);
y = 0x5a05089a ^ static_cast<uint32_t>(seed >> 32);
z = 0;
w = 0;
for(int i = 0; i < 10; i++) mix();
}
//-----------------------------------------------------------------------------
void mix ( void )
{
uint32_t t = x ^ (x << 11);
x = y; y = z; z = w;
w = w ^ (w >> 19) ^ t ^ (t >> 8);
}
uint32_t rand_u32 ( void )
{
mix();
return x;
}
uint64_t rand_u64 ( void )
{
mix();
uint64_t a = x;
uint64_t b = y;
return (a << 32) | b;
}
void rand_p ( void * blob, int bytes )
{
uint32_t * blocks = reinterpret_cast<uint32_t*>(blob);
while(bytes >= 4)
Rand()
{
blocks[0] = rand_u32();
blocks++;
bytes -= 4;
reseed(static_cast<uint32_t>(0));
}
uint8_t * tail = reinterpret_cast<uint8_t*>(blocks);
for(int i = 0; i < bytes; i++)
explicit Rand(uint32_t seed)
{
tail[i] = static_cast<uint8_t>(rand_u32());
reseed(seed);
}
void reseed(uint32_t seed)
{
x = 0x498b3bc5 ^ seed;
y = 0;
z = 0;
w = 0;
for (int i = 0; i < 10; i++)
mix();
}
void reseed(uint64_t seed)
{
x = 0x498b3bc5 ^ static_cast<uint32_t>(seed >> 0);
y = 0x5a05089a ^ static_cast<uint32_t>(seed >> 32);
z = 0;
w = 0;
for (int i = 0; i < 10; i++)
mix();
}
//-----------------------------------------------------------------------------
void mix(void)
{
uint32_t t = x ^ (x << 11);
x = y;
y = z;
z = w;
w = w ^ (w >> 19) ^ t ^ (t >> 8);
}
uint32_t rand_u32(void)
{
mix();
return x;
}
uint64_t rand_u64(void)
{
mix();
uint64_t a = x;
uint64_t b = y;
return (a << 32) | b;
}
void rand_p(void * blob, int bytes)
{
uint32_t * blocks = reinterpret_cast<uint32_t *>(blob);
while (bytes >= 4)
{
blocks[0] = rand_u32();
blocks++;
bytes -= 4;
}
uint8_t * tail = reinterpret_cast<uint8_t *>(blocks);
for (int i = 0; i < bytes; i++)
{
tail[i] = static_cast<uint8_t>(rand_u32());
}
}
}
};
//-----------------------------------------------------------------------------
extern Rand g_rand1;
inline uint32_t rand_u32 ( void ) { return g_rand1.rand_u32(); }
inline uint64_t rand_u64 ( void ) { return g_rand1.rand_u64(); }
inline void rand_p ( void * blob, int bytes )
inline uint32_t rand_u32(void)
{
uint32_t * blocks = static_cast<uint32_t*>(blob);
return g_rand1.rand_u32();
}
inline uint64_t rand_u64(void)
{
return g_rand1.rand_u64();
}
while(bytes >= 4)
{
*blocks++ = rand_u32();
bytes -= 4;
}
inline void rand_p(void * blob, int bytes)
{
uint32_t * blocks = static_cast<uint32_t *>(blob);
uint8_t * tail = reinterpret_cast<uint8_t*>(blocks);
while (bytes >= 4)
{
*blocks++ = rand_u32();
bytes -= 4;
}
for(int i = 0; i < bytes; i++)
{
tail[i] = static_cast<uint8_t>(rand_u32());
}
uint8_t * tail = reinterpret_cast<uint8_t *>(blocks);
for (int i = 0; i < bytes; i++)
{
tail[i] = static_cast<uint8_t>(rand_u32());
}
}
//-----------------------------------------------------------------------------

101
dbms/src/Common/tests/sip_hash.cpp
View File

@ -1,23 +1,23 @@
#include <stdlib.h>
#include <string.h>
#include <iostream>
#include <iomanip>
#include <iostream>
#include <Common/SipHash.h>
/// Adapted version https://www.131002.net/siphash/siphash24.c
/*
SipHash-2-4 output with
k = 00 01 02 ...
and
in = (empty string)
in = 00 (1 byte)
in = 00 01 (2 bytes)
in = 00 01 02 (3 bytes)
...
in = 00 01 02 ... 3e (63 bytes)
SipHash-2-4 output with
k = 00 01 02 ...
and
in = (empty string)
in = 00 (1 byte)
in = 00 01 (2 bytes)
in = 00 01 02 (3 bytes)
...
in = 00 01 02 ... 3e (63 bytes)
*/
uint8_t vectors[64][8] =
{
@ -91,49 +91,50 @@ uint8_t vectors[64][8] =
int test_vectors()
{
#define MAXLEN 64
char in[MAXLEN];
char in[MAXLEN];
union
{
char out[16];
uint64_t out64[2];
};
union
{
char k[16];
uint64_t k64[2];
};
int i;
int ok = 1;
for( i = 0; i < 16; ++i ) k[i] = i;
for( i = 0; i < MAXLEN; ++i )
{
in[i] = i;
size_t part = i == 0 ? 0 : (rand() % i);
SipHash hash(k64[0], k64[1]);
hash.update(in, part);
hash.update(in + part, i - part);
hash.get128(out);
uint64_t test_vector;
memcpy(&test_vector, vectors[i], 8);
if ((out64[0] ^ out64[1]) != test_vector)
union
{
std::cerr << "test vector failed for " << i << " bytes" << std::endl;
ok = 0;
}
}
char out[16];
uint64_t out64[2];
};
return ok;
union
{
char k[16];
uint64_t k64[2];
};
int i;
int ok = 1;
for (i = 0; i < 16; ++i)
k[i] = i;
for (i = 0; i < MAXLEN; ++i)
{
in[i] = i;
size_t part = i == 0 ? 0 : (rand() % i);
SipHash hash(k64[0], k64[1]);
hash.update(in, part);
hash.update(in + part, i - part);
hash.get128(out);
uint64_t test_vector;
memcpy(&test_vector, vectors[i], 8);
if ((out64[0] ^ out64[1]) != test_vector)
{
std::cerr << "test vector failed for " << i << " bytes" << std::endl;
ok = 0;
}
}
return ok;
}

2
dbms/src/Core/BlockInfo.cpp
View File

@ -20,7 +20,7 @@ namespace ErrorCodes
/// Write values in binary form. NOTE: You could use protobuf, but it would be overkill for this case.
void BlockInfo::write(WriteBuffer & out) const
{
/// Set of pairs `FIELD_NUM`, value in binary form. Then 0.
/// Set of pairs `FIELD_NUM`, value in binary form. Then 0.
#define WRITE_FIELD(TYPE, NAME, DEFAULT, FIELD_NUM) \
writeVarUInt(FIELD_NUM, out); \
writeBinary(NAME, out);

2
dbms/src/DataTypes/IDataType.cpp
View File

@ -74,7 +74,7 @@ String IDataType::getFileNameForStream(const String & column_name, const IDataTy
String nested_table_name = Nested::extractTableName(column_name);
bool is_sizes_of_nested_type =
path.size() == 1 /// Nested structure may have arrays as nested elements (so effectively we have multidimentional arrays).
path.size() == 1 /// Nested structure may have arrays as nested elements (so effectively we have multidimensional arrays).
/// Sizes of arrays are shared only at first level.
&& path[0].type == IDataType::Substream::ArraySizes
&& nested_table_name != column_name;

6
dbms/src/DataTypes/Native.h
View File

@ -13,8 +13,14 @@
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeUUID.h>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#include <llvm/IR/IRBuilder.h>
#pragma GCC diagnostic pop
namespace DB
{

41
dbms/src/Dictionaries/CacheDictionary.cpp
View File

@ -70,9 +70,7 @@ CacheDictionary::CacheDictionary(const std::string & name, const DictionaryStruc
rnd_engine(randomSeed())
{
if (!this->source_ptr->supportsSelectiveLoad())
throw Exception{
name + ": source cannot be used with CacheDictionary",
ErrorCodes::UNSUPPORTED_METHOD};
throw Exception{name + ": source cannot be used with CacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
createAttributes();
}
@ -215,9 +213,7 @@ void CacheDictionary::get##TYPE(const std::string & attribute_name, const Padded
{\
auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
const auto null_value = std::get<TYPE>(attribute.null_values);\
\
@ -240,9 +236,7 @@ void CacheDictionary::getString(const std::string & attribute_name, const Padded
{
auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
@ -256,9 +250,7 @@ void CacheDictionary::get##TYPE(\
{\
auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
getItemsNumber<TYPE>(attribute, ids, out, [&] (const size_t row) { return def[row]; });\
}
@ -281,9 +273,7 @@ void CacheDictionary::getString(
{
auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
getItemsString(attribute, ids, out, [&] (const size_t row) { return def->getDataAt(row); });
}
@ -294,9 +284,7 @@ void CacheDictionary::get##TYPE(\
{\
auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
getItemsNumber<TYPE>(attribute, ids, out, [&] (const size_t) { return def; });\
}
@ -319,9 +307,7 @@ void CacheDictionary::getString(
{
auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
getItemsString(attribute, ids, out, [&] (const size_t) { return StringRef{def}; });
}
@ -449,9 +435,7 @@ void CacheDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::UInt64)
throw Exception{
name + ": hierarchical attribute must be UInt64.",
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -798,9 +782,7 @@ void CacheDictionary::update(
{
const auto id_column = typeid_cast<const ColumnUInt64 *>(block.safeGetByPosition(0).column.get());
if (!id_column)
throw Exception{
name + ": id column has type different from UInt64.",
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": id column has type different from UInt64.", ErrorCodes::TYPE_MISMATCH};
const auto & ids = id_column->getData();
@ -973,10 +955,7 @@ CacheDictionary::Attribute & CacheDictionary::getAttribute(const std::string & a
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{
name + ": no such attribute '" + attribute_name + "'",
ErrorCodes::BAD_ARGUMENTS
};
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}

24
dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
View File

@ -59,9 +59,7 @@ ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(const std::string & name, c
rnd_engine(randomSeed())
{
if (!this->source_ptr->supportsSelectiveLoad())
throw Exception{
name + ": source cannot be used with ComplexKeyCacheDictionary",
ErrorCodes::UNSUPPORTED_METHOD};
throw Exception{name + ": source cannot be used with ComplexKeyCacheDictionary", ErrorCodes::UNSUPPORTED_METHOD};
createAttributes();
}
@ -78,9 +76,7 @@ void ComplexKeyCacheDictionary::getString(
auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
const auto null_value = StringRef{std::get<String>(attribute.null_values)};
@ -95,9 +91,7 @@ void ComplexKeyCacheDictionary::getString(
auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
getItemsString(attribute, key_columns, out, [&] (const size_t row) { return def->getDataAt(row); });
}
@ -110,9 +104,7 @@ void ComplexKeyCacheDictionary::getString(
auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
getItemsString(attribute, key_columns, out, [&] (const size_t) { return StringRef{def}; });
}
@ -248,9 +240,7 @@ void ComplexKeyCacheDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{
name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(), ErrorCodes::TYPE_MISMATCH};
}
}
@ -258,9 +248,7 @@ ComplexKeyCacheDictionary::Attribute & ComplexKeyCacheDictionary::getAttribute(c
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{
name + ": no such attribute '" + attribute_name + "'",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}

60
dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp
View File

@ -51,9 +51,7 @@ void ComplexKeyHashedDictionary::get##TYPE(\
\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
const auto null_value = std::get<TYPE>(attribute.null_values);\
\
@ -82,9 +80,7 @@ void ComplexKeyHashedDictionary::getString(
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -102,9 +98,7 @@ void ComplexKeyHashedDictionary::get##TYPE(\
\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
getItemsNumber<TYPE>(attribute, key_columns,\
[&] (const size_t row, const auto value) { out[row] = value; },\
@ -131,9 +125,7 @@ void ComplexKeyHashedDictionary::getString(
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
getItemsImpl<StringRef, StringRef>(attribute, key_columns,
[&] (const size_t, const StringRef value) { out->insertData(value.data, value.size); },
@ -149,9 +141,7 @@ void ComplexKeyHashedDictionary::get##TYPE(\
\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
getItemsNumber<TYPE>(attribute, key_columns,\
[&] (const size_t row, const auto value) { out[row] = value; },\
@ -178,9 +168,7 @@ void ComplexKeyHashedDictionary::getString(
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
getItemsImpl<StringRef, StringRef>(attribute, key_columns,
[&] (const size_t, const StringRef value) { out->insertData(value.data, value.size); },
@ -221,9 +209,7 @@ void ComplexKeyHashedDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{
name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(), ErrorCodes::TYPE_MISMATCH};
}
}
@ -238,23 +224,26 @@ void ComplexKeyHashedDictionary::blockToAttributes(const Block & block)
element_count += rows;
const auto key_column_ptrs = ext::map<Columns>(ext::range(0, keys_size),
[&](const size_t attribute_idx) {
return block.safeGetByPosition(attribute_idx).column;
});
[&](const size_t attribute_idx)
{
return block.safeGetByPosition(attribute_idx).column;
});
const auto attribute_column_ptrs = ext::map<Columns>(ext::range(0, attributes_size),
[&](const size_t attribute_idx) {
return block.safeGetByPosition(
keys_size + attribute_idx).column;
});
[&](const size_t attribute_idx)
{
return block.safeGetByPosition(keys_size + attribute_idx).column;
});
for (const auto row_idx : ext::range(0, rows)) {
for (const auto row_idx : ext::range(0, rows))
{
/// calculate key once per row
const auto key = placeKeysInPool(row_idx, key_column_ptrs, keys, keys_pool);
auto should_rollback = false;
for (const auto attribute_idx : ext::range(0, attributes_size)) {
for (const auto attribute_idx : ext::range(0, attributes_size))
{
const auto &attribute_column = *attribute_column_ptrs[attribute_idx];
auto &attribute = attributes[attribute_idx];
const auto inserted = setAttributeValue(attribute, key, attribute_column[row_idx]);
@ -354,7 +343,8 @@ void ComplexKeyHashedDictionary::updateData()
void ComplexKeyHashedDictionary::loadData()
{
if (!source_ptr->hasUpdateField()) {
if (!source_ptr->hasUpdateField())
{
auto stream = source_ptr->loadAll();
stream->readPrefix();
@ -367,9 +357,7 @@ void ComplexKeyHashedDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{
name + ": dictionary source is empty and 'require_nonempty' property is set.",
ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -546,9 +534,7 @@ const ComplexKeyHashedDictionary::Attribute & ComplexKeyHashedDictionary::getAtt
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{
name + ": no such attribute '" + attribute_name + "'",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}

24
dbms/src/Dictionaries/DictionarySourceFactory.cpp
View File

@ -53,8 +53,7 @@ Block createSampleBlock(const DictionaryStructure & dict_struct)
Block block;
if (dict_struct.id)
block.insert(ColumnWithTypeAndName{
ColumnUInt64::create(1, 0), std::make_shared<DataTypeUInt64>(), dict_struct.id->name});
block.insert(ColumnWithTypeAndName{ColumnUInt64::create(1, 0), std::make_shared<DataTypeUInt64>(), dict_struct.id->name});
if (dict_struct.key)
{
@ -109,10 +108,7 @@ DictionarySourcePtr DictionarySourceFactory::create(
Poco::Util::AbstractConfiguration::Keys keys;
config.keys(config_prefix, keys);
if (keys.size() != 1)
throw Exception{
name +": element dictionary.source should have exactly one child element",
ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG
};
throw Exception{name +": element dictionary.source should have exactly one child element", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG};
auto sample_block = createSampleBlock(dict_struct);
@ -121,9 +117,7 @@ DictionarySourcePtr DictionarySourceFactory::create(
if ("file" == source_type)
{
if (dict_struct.has_expressions)
throw Exception{
"Dictionary source of type `file` does not support attribute expressions",
ErrorCodes::LOGICAL_ERROR};
throw Exception{"Dictionary source of type `file` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
const auto filename = config.getString(config_prefix + ".file.path");
const auto format = config.getString(config_prefix + ".file.format");
@ -164,9 +158,7 @@ DictionarySourcePtr DictionarySourceFactory::create(
else if ("executable" == source_type)
{
if (dict_struct.has_expressions)
throw Exception{
"Dictionary source of type `executable` does not support attribute expressions",
ErrorCodes::LOGICAL_ERROR};
throw Exception{"Dictionary source of type `executable` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
return std::make_unique<ExecutableDictionarySource>(dict_struct, config, config_prefix + ".executable", sample_block, context);
}
@ -174,9 +166,7 @@ DictionarySourcePtr DictionarySourceFactory::create(
{
if (dict_struct.has_expressions)
throw Exception{
"Dictionary source of type `http` does not support attribute expressions",
ErrorCodes::LOGICAL_ERROR};
throw Exception{"Dictionary source of type `http` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
#if Poco_NetSSL_FOUND
// Used for https queries
@ -199,9 +189,7 @@ DictionarySourcePtr DictionarySourceFactory::create(
}
}
throw Exception{
name + ": unknown dictionary source type: " + source_type,
ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG};
throw Exception{name + ": unknown dictionary source type: " + source_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG};
}
}

38
dbms/src/Dictionaries/DictionaryStructure.cpp
View File

@ -83,9 +83,7 @@ AttributeUnderlyingType getAttributeUnderlyingType(const std::string & type)
if (it != std::end(dictionary))
return it->second;
throw Exception{
"Unknown type " + type,
ErrorCodes::UNKNOWN_TYPE};
throw Exception{"Unknown type " + type, ErrorCodes::UNKNOWN_TYPE};
}
@ -107,9 +105,7 @@ std::string toString(const AttributeUnderlyingType type)
case AttributeUnderlyingType::String: return "String";
}
throw Exception{
"Unknown attribute_type " + toString(static_cast<int>(type)),
ErrorCodes::ARGUMENT_OUT_OF_BOUND};
throw Exception{"Unknown attribute_type " + toString(static_cast<int>(type)), ErrorCodes::ARGUMENT_OUT_OF_BOUND};
}
@ -118,9 +114,7 @@ DictionarySpecialAttribute::DictionarySpecialAttribute(const Poco::Util::Abstrac
expression{config.getString(config_prefix + ".expression", "")}
{
if (name.empty() && !expression.empty())
throw Exception{
"Element " + config_prefix + ".name is empty",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{"Element " + config_prefix + ".name is empty", ErrorCodes::BAD_ARGUMENTS};
}
@ -169,9 +163,7 @@ DictionaryStructure::DictionaryStructure(const Poco::Util::AbstractConfiguration
void DictionaryStructure::validateKeyTypes(const DataTypes & key_types) const
{
if (key_types.size() != key->size())
throw Exception{
"Key structure does not match, expected " + getKeyDescription(),
ErrorCodes::TYPE_MISMATCH};
throw Exception{"Key structure does not match, expected " + getKeyDescription(), ErrorCodes::TYPE_MISMATCH};
for (const auto i : ext::range(0, key_types.size()))
{
@ -179,10 +171,8 @@ void DictionaryStructure::validateKeyTypes(const DataTypes & key_types) const
const auto & actual_type = key_types[i]->getName();
if (expected_type != actual_type)
throw Exception{
"Key type at position " + std::to_string(i) + " does not match, expected " + expected_type +
", found " + actual_type,
ErrorCodes::TYPE_MISMATCH};
throw Exception{"Key type at position " + std::to_string(i) + " does not match, expected " + expected_type +
", found " + actual_type, ErrorCodes::TYPE_MISMATCH};
}
}
@ -240,9 +230,7 @@ static void CheckAttributeKeys(const Poco::Util::AbstractConfiguration::Keys & k
for (const auto & key : keys)
{
if (valid_keys.find(key) == valid_keys.end())
throw Exception{
"Unknown key '" + key + "' inside attribute section",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{"Unknown key '" + key + "' inside attribute section", ErrorCodes::BAD_ARGUMENTS};
}
}
@ -298,19 +286,13 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
const auto injective = config.getBool(prefix + "injective", false);
const auto is_object_id = config.getBool(prefix + "is_object_id", false);
if (name.empty())
throw Exception{
"Properties 'name' and 'type' of an attribute cannot be empty",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{"Properties 'name' and 'type' of an attribute cannot be empty", ErrorCodes::BAD_ARGUMENTS};
if (has_hierarchy && !hierarchy_allowed)
throw Exception{
"Hierarchy not allowed in '" + prefix,
ErrorCodes::BAD_ARGUMENTS};
throw Exception{"Hierarchy not allowed in '" + prefix, ErrorCodes::BAD_ARGUMENTS};
if (has_hierarchy && hierarchical)
throw Exception{
"Only one hierarchical attribute supported",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{"Only one hierarchical attribute supported", ErrorCodes::BAD_ARGUMENTS};
has_hierarchy = has_hierarchy || hierarchical;

4
dbms/src/Dictionaries/ExternalResultDescription.cpp
View File

@ -56,9 +56,7 @@ void ExternalResultDescription::init(const Block & sample_block_)
else if (typeid_cast<const DataTypeDateTime *>(type))
types.push_back(ValueType::DateTime);
else
throw Exception{
"Unsupported type " + type->getName(),
ErrorCodes::UNKNOWN_TYPE};
throw Exception{"Unsupported type " + type->getName(), ErrorCodes::UNKNOWN_TYPE};
names.emplace_back(column.name);
sample_columns.emplace_back(column.column);

43
dbms/src/Dictionaries/FlatDictionary.cpp
View File

@ -117,9 +117,7 @@ void FlatDictionary::get##TYPE(const std::string & attribute_name, const PaddedP
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
const auto null_value = std::get<TYPE>(attribute.null_values);\
\
@ -144,9 +142,7 @@ void FlatDictionary::getString(const std::string & attribute_name, const PaddedP
{
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
const auto & null_value = std::get<StringRef>(attribute.null_values);
@ -162,9 +158,7 @@ void FlatDictionary::get##TYPE(\
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
getItemsNumber<TYPE>(attribute, ids,\
[&] (const size_t row, const auto value) { out[row] = value; },\
@ -189,9 +183,7 @@ void FlatDictionary::getString(
{
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
getItemsImpl<StringRef, StringRef>(attribute, ids,
[&] (const size_t, const StringRef value) { out->insertData(value.data, value.size); },
@ -205,9 +197,7 @@ void FlatDictionary::get##TYPE(\
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
getItemsNumber<TYPE>(attribute, ids,\
[&] (const size_t row, const auto value) { out[row] = value; },\
@ -232,9 +222,7 @@ void FlatDictionary::getString(
{
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
FlatDictionary::getItemsImpl<StringRef, StringRef>(attribute, ids,
[&] (const size_t, const StringRef value) { out->insertData(value.data, value.size); },
@ -279,9 +267,7 @@ void FlatDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::UInt64)
throw Exception{
name + ": hierarchical attribute must be UInt64.",
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -374,7 +360,8 @@ void FlatDictionary::updateData()
void FlatDictionary::loadData()
{
if (!source_ptr->hasUpdateField()) {
if (!source_ptr->hasUpdateField())
{
auto stream = source_ptr->loadAll();
stream->readPrefix();
@ -387,9 +374,7 @@ void FlatDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{
name + ": dictionary source is empty and 'require_nonempty' property is set.",
ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
@ -530,9 +515,7 @@ template <typename T>
void FlatDictionary::resize(Attribute & attribute, const Key id)
{
if (id >= max_array_size)
throw Exception{
name + ": identifier should be less than " + toString(max_array_size),
ErrorCodes::ARGUMENT_OUT_OF_BOUND};
throw Exception{name + ": identifier should be less than " + toString(max_array_size), ErrorCodes::ARGUMENT_OUT_OF_BOUND};
auto & array = *std::get<ContainerPtrType<T>>(attribute.arrays);
if (id >= array.size())
@ -586,9 +569,7 @@ const FlatDictionary::Attribute & FlatDictionary::getAttribute(const std::string
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{
name + ": no such attribute '" + attribute_name + "'",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}

39
dbms/src/Dictionaries/HashedDictionary.cpp
View File

@ -114,9 +114,7 @@ void HashedDictionary::get##TYPE(const std::string & attribute_name, const Padde
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
const auto null_value = std::get<TYPE>(attribute.null_values);\
\
@ -141,9 +139,7 @@ void HashedDictionary::getString(const std::string & attribute_name, const Padde
{
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -159,9 +155,7 @@ void HashedDictionary::get##TYPE(\
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
getItemsNumber<TYPE>(attribute, ids,\
[&] (const size_t row, const auto value) { out[row] = value; },\
@ -186,9 +180,7 @@ void HashedDictionary::getString(
{
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
getItemsImpl<StringRef, StringRef>(attribute, ids,
[&] (const size_t, const StringRef value) { out->insertData(value.data, value.size); },
@ -201,9 +193,7 @@ void HashedDictionary::get##TYPE(\
{\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
getItemsNumber<TYPE>(attribute, ids,\
[&] (const size_t row, const auto value) { out[row] = value; },\
@ -228,9 +218,7 @@ void HashedDictionary::getString(
{
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
getItemsImpl<StringRef, StringRef>(attribute, ids,
[&] (const size_t, const StringRef value) { out->insertData(value.data, value.size); },
@ -273,9 +261,7 @@ void HashedDictionary::createAttributes()
hierarchical_attribute = &attributes.back();
if (hierarchical_attribute->type != AttributeUnderlyingType::UInt64)
throw Exception{
name + ": hierarchical attribute must be UInt64.",
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH};
}
}
}
@ -368,7 +354,8 @@ void HashedDictionary::updateData()
void HashedDictionary::loadData()
{
if (!source_ptr->hasUpdateField()) {
if (!source_ptr->hasUpdateField())
{
auto stream = source_ptr->loadAll();
stream->readPrefix();
@ -381,9 +368,7 @@ void HashedDictionary::loadData()
updateData();
if (require_nonempty && 0 == element_count)
throw Exception{
name + ": dictionary source is empty and 'require_nonempty' property is set.",
ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -545,9 +530,7 @@ const HashedDictionary::Attribute & HashedDictionary::getAttribute(const std::st
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{
name + ": no such attribute '" + attribute_name + "'",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}

15
dbms/src/Dictionaries/MongoDBBlockInputStream.cpp
View File

@ -104,17 +104,15 @@ namespace
break;
}
throw Exception{
"Type mismatch, expected String, got type id = " + toString(value.type()) +
" for column " + name, ErrorCodes::TYPE_MISMATCH};
throw Exception{"Type mismatch, expected String, got type id = " + toString(value.type()) +
" for column " + name, ErrorCodes::TYPE_MISMATCH};
}
case ValueType::Date:
{
if (value.type() != Poco::MongoDB::ElementTraits<Poco::Timestamp>::TypeId)
throw Exception{
"Type mismatch, expected Timestamp, got type id = " + toString(value.type()) +
" for column " + name, ErrorCodes::TYPE_MISMATCH};
throw Exception{"Type mismatch, expected Timestamp, got type id = " + toString(value.type()) +
" for column " + name, ErrorCodes::TYPE_MISMATCH};
static_cast<ColumnUInt16 &>(column).getData().push_back(
UInt16{DateLUT::instance().toDayNum(
@ -125,9 +123,8 @@ namespace
case ValueType::DateTime:
{
if (value.type() != Poco::MongoDB::ElementTraits<Poco::Timestamp>::TypeId)
throw Exception{
"Type mismatch, expected Timestamp, got type id = " + toString(value.type()) +
" for column " + name, ErrorCodes::TYPE_MISMATCH};
throw Exception{"Type mismatch, expected Timestamp, got type id = " + toString(value.type()) +
" for column " + name, ErrorCodes::TYPE_MISMATCH};
static_cast<ColumnUInt32 &>(column).getData().push_back(
static_cast<const Poco::MongoDB::ConcreteElement<Poco::Timestamp> &>(value).value().epochTime());

4
dbms/src/Dictionaries/MySQLBlockInputStream.cpp
View File

@ -24,9 +24,7 @@ MySQLBlockInputStream::MySQLBlockInputStream(
max_block_size{max_block_size}
{
if (sample_block.columns() != result.getNumFields())
throw Exception{
"mysqlxx::UseQueryResult contains " + toString(result.getNumFields()) + " columns while " +
toString(sample_block.columns()) + " expected",
throw Exception{"mysqlxx::UseQueryResult contains " + toString(result.getNumFields()) + " columns while " + toString(sample_block.columns()) + " expected",
ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH};
description.init(sample_block);

6
dbms/src/Dictionaries/ODBCBlockInputStream.cpp
View File

@ -29,10 +29,8 @@ ODBCBlockInputStream::ODBCBlockInputStream(
log(&Logger::get("ODBCBlockInputStream"))
{
if (sample_block.columns() != result.columnCount())
throw Exception{
"RecordSet contains " + toString(result.columnCount()) + " columns while " +
toString(sample_block.columns()) + " expected",
ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH};
throw Exception{"RecordSet contains " + toString(result.columnCount()) + " columns while " +
toString(sample_block.columns()) + " expected", ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH};
description.init(sample_block);
}

19
dbms/src/Dictionaries/RangeHashedDictionary.cpp
View File

@ -102,9 +102,7 @@ void RangeHashedDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{
name + ": hierarchical attributes not supported by " + getName() + " dictionary.",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{name + ": hierarchical attributes not supported by " + getName() + " dictionary.", ErrorCodes::BAD_ARGUMENTS};
}
}
@ -136,9 +134,7 @@ void RangeHashedDictionary::loadData()
stream->readSuffix();
if (require_nonempty && 0 == element_count)
throw Exception{
name + ": dictionary source is empty and 'require_nonempty' property is set.",
ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -284,7 +280,8 @@ void RangeHashedDictionary::setAttributeValueImpl(Attribute & attribute, const K
auto & values = it->second;
const auto insert_it = std::lower_bound(std::begin(values), std::end(values), range,
[] (const Value<T> & lhs, const Range & range) {
[] (const Value<T> & lhs, const Range & range)
{
return lhs.range < range;
});
@ -342,9 +339,7 @@ const RangeHashedDictionary::Attribute & RangeHashedDictionary::getAttribute(con
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{
name + ": no such attribute '" + attribute_name + "'",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}
@ -353,9 +348,7 @@ const RangeHashedDictionary::Attribute & RangeHashedDictionary::getAttributeWith
{
const auto & attribute = getAttribute(name);
if (attribute.type != type)
throw Exception{
name + ": type mismatch: attribute " + name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
return attribute;
}

44
dbms/src/Dictionaries/TrieDictionary.cpp
View File

@ -68,9 +68,7 @@ void TrieDictionary::get##TYPE(\
\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
const auto null_value = std::get<TYPE>(attribute.null_values);\
\
@ -99,9 +97,7 @@ void TrieDictionary::getString(
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
const auto & null_value = StringRef{std::get<String>(attribute.null_values)};
@ -119,9 +115,7 @@ void TrieDictionary::get##TYPE(\
\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
getItemsNumber<TYPE>(attribute, key_columns,\
[&] (const size_t row, const auto value) { out[row] = value; },\
@ -148,9 +142,7 @@ void TrieDictionary::getString(
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
getItemsImpl<StringRef, StringRef>(attribute, key_columns,
[&] (const size_t, const StringRef value) { out->insertData(value.data, value.size); },
@ -166,9 +158,7 @@ void TrieDictionary::get##TYPE(\
\
const auto & attribute = getAttribute(attribute_name);\
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::TYPE))\
throw Exception{\
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),\
ErrorCodes::TYPE_MISMATCH};\
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};\
\
getItemsNumber<TYPE>(attribute, key_columns,\
[&] (const size_t row, const auto value) { out[row] = value; },\
@ -195,9 +185,7 @@ void TrieDictionary::getString(
const auto & attribute = getAttribute(attribute_name);
if (!isAttributeTypeConvertibleTo(attribute.type, AttributeUnderlyingType::String))
throw Exception{
name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": type mismatch: attribute " + attribute_name + " has type " + toString(attribute.type), ErrorCodes::TYPE_MISMATCH};
getItemsImpl<StringRef, StringRef>(attribute, key_columns,
[&] (const size_t, const StringRef value) { out->insertData(value.data, value.size); },
@ -238,9 +226,7 @@ void TrieDictionary::createAttributes()
attributes.push_back(createAttributeWithType(attribute.underlying_type, attribute.null_value));
if (attribute.hierarchical)
throw Exception{
name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(),
ErrorCodes::TYPE_MISMATCH};
throw Exception{name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(), ErrorCodes::TYPE_MISMATCH};
}
}
@ -290,9 +276,7 @@ void TrieDictionary::loadData()
stream->readSuffix();
if (require_nonempty && 0 == element_count)
throw Exception{
name + ": dictionary source is empty and 'require_nonempty' property is set.",
ErrorCodes::DICTIONARY_IS_EMPTY};
throw Exception{name + ": dictionary source is empty and 'require_nonempty' property is set.", ErrorCodes::DICTIONARY_IS_EMPTY};
}
template <typename T>
@ -338,16 +322,12 @@ void TrieDictionary::calculateBytesAllocated()
void TrieDictionary::validateKeyTypes(const DataTypes & key_types) const
{
if (key_types.size() != 1)
throw Exception{
"Expected a single IP address",
ErrorCodes::TYPE_MISMATCH};
throw Exception{"Expected a single IP address", ErrorCodes::TYPE_MISMATCH};
const auto & actual_type = key_types[0]->getName();
if (actual_type != "UInt32" && actual_type != "FixedString(16)")
throw Exception{
"Key does not match, expected either UInt32 or FixedString(16)",
ErrorCodes::TYPE_MISMATCH};
throw Exception{"Key does not match, expected either UInt32 or FixedString(16)", ErrorCodes::TYPE_MISMATCH};
}
@ -526,9 +506,7 @@ const TrieDictionary::Attribute & TrieDictionary::getAttribute(const std::string
{
const auto it = attribute_index_by_name.find(attribute_name);
if (it == std::end(attribute_index_by_name))
throw Exception{
name + ": no such attribute '" + attribute_name + "'",
ErrorCodes::BAD_ARGUMENTS};
throw Exception{name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS};
return attributes[it->second];
}

130
dbms/src/Functions/CMakeLists.txt
View File

@ -2,77 +2,77 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_generate_function.cmake)
generate_function_register(Arithmetic
FunctionPlus
FunctionMinus
FunctionMultiply
FunctionDivideFloating
FunctionDivideIntegral
FunctionDivideIntegralOrZero
FunctionModulo
FunctionNegate
FunctionAbs
FunctionBitAnd
FunctionBitOr
FunctionBitXor
FunctionBitNot
FunctionBitShiftLeft
FunctionBitShiftRight
FunctionBitRotateLeft
FunctionBitRotateRight
FunctionLeast
FunctionGreatest
FunctionBitTest
FunctionBitTestAny
FunctionBitTestAll
FunctionGCD
FunctionLCM
FunctionIntExp2
FunctionIntExp10
FunctionPlus
FunctionMinus
FunctionMultiply
FunctionDivideFloating
FunctionDivideIntegral
FunctionDivideIntegralOrZero
FunctionModulo
FunctionNegate
FunctionAbs
FunctionBitAnd
FunctionBitOr
FunctionBitXor
FunctionBitNot
FunctionBitShiftLeft
FunctionBitShiftRight
FunctionBitRotateLeft
FunctionBitRotateRight
FunctionLeast
FunctionGreatest
FunctionBitTest
FunctionBitTestAny
FunctionBitTestAll
FunctionGCD
FunctionLCM
FunctionIntExp2
FunctionIntExp10
)
generate_function_register(Array
FunctionArray
FunctionArrayElement
FunctionHas
FunctionIndexOf
FunctionCountEqual
FunctionArrayEnumerate
FunctionArrayEnumerateUniq
FunctionArrayUniq
FunctionEmptyArrayUInt8
FunctionEmptyArrayUInt16
FunctionEmptyArrayUInt32
FunctionEmptyArrayUInt64
FunctionEmptyArrayInt8
FunctionEmptyArrayInt16
FunctionEmptyArrayInt32
FunctionEmptyArrayInt64
FunctionEmptyArrayFloat32
FunctionEmptyArrayFloat64
FunctionEmptyArrayDate
FunctionEmptyArrayDateTime
FunctionEmptyArrayString
FunctionEmptyArrayToSingle
FunctionRange
FunctionArrayReduce
FunctionArrayReverse
FunctionArrayConcat
FunctionArraySlice
FunctionArrayPushBack
FunctionArrayPushFront
FunctionArrayPopBack
FunctionArrayPopFront
FunctionArrayHasAll
FunctionArrayHasAny
FunctionArrayIntersect
FunctionArrayResize
FunctionArray
FunctionArrayElement
FunctionHas
FunctionIndexOf
FunctionCountEqual
FunctionArrayEnumerate
FunctionArrayEnumerateUniq
FunctionArrayUniq
FunctionEmptyArrayUInt8
FunctionEmptyArrayUInt16
FunctionEmptyArrayUInt32
FunctionEmptyArrayUInt64
FunctionEmptyArrayInt8
FunctionEmptyArrayInt16
FunctionEmptyArrayInt32
FunctionEmptyArrayInt64
FunctionEmptyArrayFloat32
FunctionEmptyArrayFloat64
FunctionEmptyArrayDate
FunctionEmptyArrayDateTime
FunctionEmptyArrayString
FunctionEmptyArrayToSingle
FunctionRange
FunctionArrayReduce
FunctionArrayReverse
FunctionArrayConcat
FunctionArraySlice
FunctionArrayPushBack
FunctionArrayPushFront
FunctionArrayPopBack
FunctionArrayPopFront
FunctionArrayHasAll
FunctionArrayHasAny
FunctionArrayIntersect
FunctionArrayResize
)
generate_function_register(Projection
FunctionOneOrZero
FunctionProject
FunctionBuildProjectionComposition
FunctionRestoreProjection
FunctionOneOrZero
FunctionProject
FunctionBuildProjectionComposition
FunctionRestoreProjection
)

25
dbms/src/Functions/FunctionsArithmetic.h
View File

@ -21,7 +21,10 @@
#include <boost/math/common_factor.hpp>
#if USE_EMBEDDED_COMPILER
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#include <llvm/IR/IRBuilder.h>
#pragma GCC diagnostic pop
#endif
@ -1381,17 +1384,13 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (arguments.size() < 2)
throw Exception{
"Number of arguments for function " + getName() + " doesn't match: passed "
+ toString(arguments.size()) + ", should be at least 2.",
ErrorCodes::TOO_LESS_ARGUMENTS_FOR_FUNCTION};
throw Exception{"Number of arguments for function " + getName() + " doesn't match: passed "
+ toString(arguments.size()) + ", should be at least 2.", ErrorCodes::TOO_LESS_ARGUMENTS_FOR_FUNCTION};
const auto first_arg = arguments.front().get();
if (!first_arg->isInteger())
throw Exception{
"Illegal type " + first_arg->getName() + " of first argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
throw Exception{"Illegal type " + first_arg->getName() + " of first argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
for (const auto i : ext::range(1, arguments.size()))
@ -1399,9 +1398,7 @@ public:
const auto pos_arg = arguments[i].get();
if (!pos_arg->isUnsignedInteger())
throw Exception{
"Illegal type " + pos_arg->getName() + " of " + toString(i) + " argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
throw Exception{"Illegal type " + pos_arg->getName() + " of " + toString(i) + " argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
}
return std::make_shared<DataTypeUInt8>();
@ -1419,9 +1416,7 @@ public:
&& !execute<Int16>(block, arguments, result, value_col)
&& !execute<Int32>(block, arguments, result, value_col)
&& !execute<Int64>(block, arguments, result, value_col))
throw Exception{
"Illegal column " + value_col->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Illegal column " + value_col->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
}
private:
@ -1522,9 +1517,7 @@ private:
&& !addToMaskImpl<UInt16>(mask, pos_col)
&& !addToMaskImpl<UInt32>(mask, pos_col)
&& !addToMaskImpl<UInt64>(mask, pos_col))
throw Exception{
"Illegal column " + pos_col->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Illegal column " + pos_col->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
}
return mask;

15
dbms/src/Functions/FunctionsArray.cpp
View File

@ -1962,8 +1962,7 @@ DataTypePtr FunctionRange::getReturnTypeImpl(const DataTypes & arguments) const
const DataTypePtr & arg = arguments.front();
if (!arg->isUnsignedInteger())
throw Exception{
"Illegal type " + arg->getName() + " of argument of function " + getName(),
throw Exception{"Illegal type " + arg->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
return std::make_shared<DataTypeArray>(arg);
@ -1982,17 +1981,15 @@ bool FunctionRange::executeInternal(Block & block, const IColumn * arg, const si
{
const auto sum = lhs + rhs;
if (sum < lhs)
throw Exception{
"A call to function " + getName() + " overflows, investigate the values of arguments you are passing",
throw Exception{"A call to function " + getName() + " overflows, investigate the values of arguments you are passing",
ErrorCodes::ARGUMENT_OUT_OF_BOUND};
return sum;
});
if (total_values > max_elements)
throw Exception{
"A call to function " + getName() + " would produce " + std::to_string(total_values) +
" array elements, which is greater than the allowed maximum of " + std::to_string(max_elements),
throw Exception{"A call to function " + getName() + " would produce " + std::to_string(total_values) +
" array elements, which is greater than the allowed maximum of " + std::to_string(max_elements),
ErrorCodes::ARGUMENT_OUT_OF_BOUND};
auto data_col = ColumnVector<T>::create(total_values);
@ -2027,9 +2024,7 @@ void FunctionRange::executeImpl(Block & block, const ColumnNumbers & arguments,
!executeInternal<UInt32>(block, col, result) &&
!executeInternal<UInt64>(block, col, result))
{
throw Exception{
"Illegal column " + col->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Illegal column " + col->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
}
}

6
dbms/src/Functions/FunctionsArray.h
View File

@ -1166,10 +1166,8 @@ private:
|| executeConst(block, arguments, result)
|| executeString(block, arguments, result)
|| executeGeneric(block, arguments, result)))
throw Exception{
"Illegal column " + block.getByPosition(arguments[0]).column->getName()
+ " of first argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName()
+ " of first argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
}
};

6
dbms/src/Functions/FunctionsComparison.h
View File

@ -779,10 +779,8 @@ private:
const auto column_string = checkAndGetColumnConst<ColumnString>(column_string_untyped);
if (!column_string || !legal_types)
throw Exception{
"Illegal columns " + col_left_untyped->getName() + " and " + col_right_untyped->getName()
+ " of arguments of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Illegal columns " + col_left_untyped->getName() + " and " + col_right_untyped->getName()
+ " of arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
StringRef string_value = column_string->getDataAt(0);

87
dbms/src/Functions/FunctionsConsistentHashing.h
View File

@ -1,20 +1,19 @@
#pragma once
#include <Common/typeid_cast.h>
#include <DataTypes/DataTypesNumber.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnConst.h>
#include <Functions/IFunction.h>
#include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypesNumber.h>
#include <Functions/FunctionHelpers.h>
#include <Functions/IFunction.h>
#include <Common/typeid_cast.h>
#include <common/likely.h>
#include <yandex/consistent_hashing.h>
#include <mailru/sumbur.h>
#include <yandex/consistent_hashing.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
@ -42,9 +41,11 @@ struct YandexConsistentHashImpl
/// Code from https://arxiv.org/pdf/1406.2294.pdf
static inline int32_t JumpConsistentHash(uint64_t key, int32_t num_buckets) {
static inline int32_t JumpConsistentHash(uint64_t key, int32_t num_buckets)
{
int64_t b = -1, j = 0;
while (j < num_buckets) {
while (j < num_buckets)
{
b = j;
key = key * 2862933555777941757ULL + 1;
j = static_cast<int64_t>((b + 1) * (double(1LL << 31) / double((key >> 33) + 1)));
@ -86,14 +87,22 @@ template <typename Impl>
class FunctionConsistentHashImpl : public IFunction
{
public:
static constexpr auto name = Impl::name;
static FunctionPtr create(const Context &) { return std::make_shared<FunctionConsistentHashImpl<Impl>>(); };
static FunctionPtr create(const Context &)
{
return std::make_shared<FunctionConsistentHashImpl<Impl>>();
};
String getName() const override { return name; }
String getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 2; }
size_t getNumberOfArguments() const override
{
return 2;
}
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
@ -103,7 +112,8 @@ public:
if (arguments[0]->getSizeOfValueInMemory() > sizeof(HashType))
throw Exception("Function " + getName() + " accepts " + std::to_string(sizeof(HashType) * 8) + "-bit integers at most"
+ ", got " + arguments[0]->getName(), ErrorCodes::BAD_ARGUMENTS);
+ ", got " + arguments[0]->getName(),
ErrorCodes::BAD_ARGUMENTS);
if (!arguments[1]->isInteger())
throw Exception("Illegal type " + arguments[1]->getName() + " of the second argument of function " + getName(),
@ -112,19 +122,25 @@ public:
return std::make_shared<DataTypeNumber<ResultType>>();
}
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
bool useDefaultImplementationForConstants() const override
{
return true;
}
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override
{
return {1};
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
{
if (block.getByPosition(arguments[1]).column->isColumnConst())
executeConstBuckets(block, arguments, result);
else
throw Exception("The second argument of function " + getName() + " (number of buckets) must be constant", ErrorCodes::BAD_ARGUMENTS);
throw Exception(
"The second argument of function " + getName() + " (number of buckets) must be constant", ErrorCodes::BAD_ARGUMENTS);
}
private:
using HashType = typename Impl::HashType;
using ResultType = typename Impl::ResultType;
using BucketsType = typename Impl::BucketsCountType;
@ -134,12 +150,13 @@ private:
inline BucketsType checkBucketsRange(T buckets)
{
if (unlikely(buckets <= 0))
throw Exception("The second argument of function " + getName() + " (number of buckets) must be positive number",
ErrorCodes::BAD_ARGUMENTS);
throw Exception(
"The second argument of function " + getName() + " (number of buckets) must be positive number", ErrorCodes::BAD_ARGUMENTS);
if (unlikely(static_cast<UInt64>(buckets) > max_buckets))
throw Exception("The value of the second argument of function " + getName() + " (number of buckets) is not fit to " +
DataTypeNumber<BucketsType>().getName(), ErrorCodes::BAD_ARGUMENTS);
throw Exception("The value of the second argument of function " + getName() + " (number of buckets) is not fit to "
+ DataTypeNumber<BucketsType>().getName(),
ErrorCodes::BAD_ARGUMENTS);
return static_cast<BucketsType>(buckets);
}
@ -155,23 +172,31 @@ private:
num_buckets = checkBucketsRange(buckets_field.get<UInt64>());
else
throw Exception("Illegal type " + String(buckets_field.getTypeName()) + " of the second argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const auto & hash_col = block.getByPosition(arguments[0]).column;
const IDataType * hash_type = block.getByPosition(arguments[0]).type.get();
auto res_col = ColumnVector<ResultType>::create();
if (checkDataType<DataTypeUInt8>(hash_type)) executeType<UInt8>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeUInt16>(hash_type)) executeType<UInt16>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeUInt32>(hash_type)) executeType<UInt32>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeUInt64>(hash_type)) executeType<UInt64>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeInt8>(hash_type)) executeType<Int8>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeInt16>(hash_type)) executeType<Int16>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeInt32>(hash_type)) executeType<Int32>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeInt64>(hash_type)) executeType<Int64>(hash_col, num_buckets, res_col.get());
if (checkDataType<DataTypeUInt8>(hash_type))
executeType<UInt8>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeUInt16>(hash_type))
executeType<UInt16>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeUInt32>(hash_type))
executeType<UInt32>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeUInt64>(hash_type))
executeType<UInt64>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeInt8>(hash_type))
executeType<Int8>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeInt16>(hash_type))
executeType<Int16>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeInt32>(hash_type))
executeType<Int32>(hash_col, num_buckets, res_col.get());
else if (checkDataType<DataTypeInt64>(hash_type))
executeType<Int64>(hash_col, num_buckets, res_col.get());
else
throw Exception("Illegal type " + hash_type->getName() + " of the first argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
block.getByPosition(result).column = std::move(res_col);
}

4
dbms/src/Functions/FunctionsConversion.h
View File

@ -942,9 +942,7 @@ public:
{
const auto src_n = column_fixed_string->getN();
if (src_n > n)
throw Exception{
"String too long for type FixedString(" + toString(n) + ")",
ErrorCodes::TOO_LARGE_STRING_SIZE};
throw Exception{"String too long for type FixedString(" + toString(n) + ")", ErrorCodes::TOO_LARGE_STRING_SIZE};
auto column_fixed = ColumnFixedString::create(n);

19
dbms/src/Functions/FunctionsDateTime.h
View File

@ -638,20 +638,19 @@ public:
if (arguments.size() == 1)
{
if (!arguments[0].type->isDateOrDateTime())
throw Exception{
"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() +
". Should be a date or a date with time", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() +
". Should be a date or a date with time", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
else if (arguments.size() == 2)
{
if (!checkDataType<DataTypeDateTime>(arguments[0].type.get())
|| !checkDataType<DataTypeString>(arguments[1].type.get()))
throw Exception{
throw Exception(
"Function " + getName() + " supports 1 or 2 arguments. The 1st argument "
"must be of type Date or DateTime. The 2nd argument (optional) must be "
"a constant string with timezone name. The timezone argument is allowed "
"only when the 1st argument has the type DateTime",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
else
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
@ -953,21 +952,20 @@ public:
if (arguments.size() == 2)
{
if (!arguments[0].type->isDateOrDateTime())
throw Exception{
"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() +
throw Exception{"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() +
". Should be a date or a date with time", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
}
else
{
if (!checkDataType<DataTypeDateTime>(arguments[0].type.get())
|| !checkDataType<DataTypeString>(arguments[2].type.get()))
throw Exception{
throw Exception(
"Function " + getName() + " supports 2 or 3 arguments. The 1st argument "
"must be of type Date or DateTime. The 2nd argument must be number. "
"The 3rd argument (optional) must be "
"a constant string with timezone name. The timezone argument is allowed "
"only when the 1st argument has the type DateTime",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
if (checkDataType<DataTypeDate>(arguments[0].type.get()))
@ -1299,8 +1297,7 @@ public:
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
if (!checkDataType<DataTypeDateTime>(arguments[0].type.get()))
throw Exception{
"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() +
throw Exception{"Illegal type " + arguments[0].type->getName() + " of argument of function " + getName() +
". Should be DateTime", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
String time_zone_name = extractTimeZoneNameFromFunctionArguments(arguments, 1, 0);

30
dbms/src/Functions/FunctionsHashing.h
View File

@ -702,24 +702,18 @@ public:
{
const auto arg_count = arguments.size();
if (arg_count != 1 && arg_count != 2)
throw Exception{
"Number of arguments for function " + getName() + " doesn't match: passed " +
toString(arg_count) + ", should be 1 or 2.",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
throw Exception{"Number of arguments for function " + getName() + " doesn't match: passed " +
toString(arg_count) + ", should be 1 or 2.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
const auto first_arg = arguments.front().get();
if (!checkDataType<DataTypeString>(first_arg))
throw Exception{
"Illegal type " + first_arg->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
throw Exception{"Illegal type " + first_arg->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
if (arg_count == 2)
{
const auto second_arg = arguments.back().get();
if (!second_arg->isInteger())
throw Exception{
"Illegal type " + second_arg->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
throw Exception{"Illegal type " + second_arg->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
}
return std::make_shared<DataTypeUInt64>();
@ -762,19 +756,15 @@ private:
block.getByPosition(result).column = std::move(col_to);
}
else
throw Exception{
"Illegal column " + block.getByPosition(arguments[0]).column->getName() +
" of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName() +
" of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
}
void executeTwoArgs(Block & block, const ColumnNumbers & arguments, const size_t result) const
{
const auto level_col = block.getByPosition(arguments.back()).column.get();
if (!level_col->isColumnConst())
throw Exception{
"Second argument of function " + getName() + " must be an integral constant",
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Second argument of function " + getName() + " must be an integral constant", ErrorCodes::ILLEGAL_COLUMN};
const auto level = level_col->get64(0);
@ -796,10 +786,8 @@ private:
block.getByPosition(result).column = std::move(col_to);
}
else
throw Exception{
"Illegal column " + block.getByPosition(arguments[0]).column->getName() +
" of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName() +
" of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
}
};

3
dbms/src/Functions/FunctionsLogical.h
View File

@ -13,7 +13,10 @@
#include <type_traits>
#if USE_EMBEDDED_COMPILER
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#include <llvm/IR/IRBuilder.h>
#pragma GCC diagnostic pop
#endif

20
dbms/src/Functions/FunctionsMath.h
View File

@ -79,9 +79,7 @@ private:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (!arguments.front()->isNumber())
throw Exception{
"Illegal type " + arguments.front()->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
throw Exception{"Illegal type " + arguments.front()->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
return std::make_shared<DataTypeFloat64>();
}
@ -140,9 +138,7 @@ private:
!execute<Float32>(block, arg, result) &&
!execute<Float64>(block, arg, result))
{
throw Exception{
"Illegal column " + arg->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Illegal column " + arg->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
}
}
};
@ -204,8 +200,7 @@ private:
const auto check_argument_type = [this] (const IDataType * arg)
{
if (!arg->isNumber())
throw Exception{
"Illegal type " + arg->getName() + " of argument of function " + getName(),
throw Exception{"Illegal type " + arg->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
};
@ -352,8 +347,7 @@ private:
}
else
{
throw Exception{
"Illegal column " + block.getByPosition(arguments[1]).column->getName() +
throw Exception{"Illegal column " + block.getByPosition(arguments[1]).column->getName() +
" of second argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
}
@ -377,8 +371,7 @@ private:
}
else
{
throw Exception{
"Illegal column " + block.getByPosition(arguments[1]).column->getName() +
throw Exception{"Illegal column " + block.getByPosition(arguments[1]).column->getName() +
" of second argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
}
@ -402,8 +395,7 @@ private:
!executeLeft<Float32>(block, arguments, result, left_arg) &&
!executeLeft<Float64>(block, arguments, result, left_arg))
{
throw Exception{
"Illegal column " + left_arg->getName() + " of argument of function " + getName(),
throw Exception{"Illegal column " + left_arg->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
}
}

40
dbms/src/Functions/FunctionsProjection.cpp
View File

@ -1,10 +1,10 @@
#include <Functions/FunctionsProjection.h>
#include <DataTypes/DataTypesNumber.h>
#include <memory>
#include <vector>
#include <DataTypes/DataTypesNumber.h>
#include <Functions/FunctionsProjection.h>
namespace DB {
namespace DB
{
FunctionPtr FunctionOneOrZero::create(const Context &)
{
return std::make_shared<FunctionOneOrZero>();
@ -33,7 +33,7 @@ void FunctionOneOrZero::executeImpl(Block & block, const ColumnNumbers & argumen
vec_res.resize(data_column->size());
for (size_t i = 0; i < data_column->size(); ++i)
{
if (data_column->getBoolRepresentation(i))
if (data_column->getBool(i))
{
vec_res[i] = 1;
}
@ -64,8 +64,8 @@ DataTypePtr FunctionProject::getReturnTypeImpl(const DataTypes & arguments) cons
{
if (!checkAndGetDataType<DataTypeUInt8>(arguments[1].get()))
{
throw Exception("Illegal type " + arguments[1]->getName() + " of 2nd argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
throw Exception(
"Illegal type " + arguments[1]->getName() + " of 2nd argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
return arguments[0];
}
@ -80,7 +80,7 @@ void FunctionProject::executeImpl(Block & block, const ColumnNumbers & arguments
}
else if (const auto projection_column_uint8_const = checkAndGetColumnConst<ColumnUInt8>(projection_column.get()))
{
if (projection_column_uint8_const->getBoolRepresentation(0))
if (projection_column_uint8_const->getBool(0))
{
block.getByPosition(result).column = data_column->cloneResized(data_column->size());
}
@ -116,14 +116,16 @@ DataTypePtr FunctionBuildProjectionComposition::getReturnTypeImpl(const DataType
{
if (!checkAndGetDataType<DataTypeUInt8>(arguments[i].get()))
{
throw Exception("Illegal type " + arguments[i]->getName() + " of " + std::to_string(i + 1) + " argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
throw Exception(
"Illegal type " + arguments[i]->getName() + " of " + std::to_string(i + 1) + " argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
}
return std::make_shared<DataTypeUInt8>();
}
void FunctionBuildProjectionComposition::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/)
void FunctionBuildProjectionComposition::executeImpl(
Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/)
{
const auto & first_projection_column = block.getByPosition(arguments[0]).column;
const auto & second_projection_column = block.getByPosition(arguments[1]).column;
@ -133,20 +135,21 @@ void FunctionBuildProjectionComposition::executeImpl(Block & block, const Column
size_t current_reserve_index = 0;
for (size_t i = 0; i < first_projection_column->size(); ++i)
{
if (first_projection_column->getBoolRepresentation(i) == 0)
if (!first_projection_column->getBool(i))
{
vec_res[i] = 0;
}
else
{
vec_res[i] = second_projection_column->getBoolRepresentation(current_reserve_index++);
vec_res[i] = second_projection_column->getBool(current_reserve_index);
++current_reserve_index;
}
}
if (current_reserve_index != second_projection_column->size())
{
throw Exception("Second argument size is not appropriate: " + std::to_string(second_projection_column->size())
+ " instead of " + std::to_string(current_reserve_index),
ErrorCodes::BAD_ARGUMENTS);
throw Exception("Second argument size is not appropriate: " + std::to_string(second_projection_column->size()) + " instead of "
+ std::to_string(current_reserve_index),
ErrorCodes::BAD_ARGUMENTS);
}
block.getByPosition(result).column = std::move(col_res);
}
@ -161,7 +164,8 @@ String FunctionRestoreProjection::getName() const
return name;
}
bool FunctionRestoreProjection::isVariadic() const {
bool FunctionRestoreProjection::isVariadic() const
{
return true;
}
@ -190,7 +194,7 @@ void FunctionRestoreProjection::executeImpl(Block & block, const ColumnNumbers &
std::vector<size_t> override_indices(arguments.size() - 1, 0);
for (size_t i = 0; i < projection_column->size(); ++i)
{
size_t argument_index = projection_column->getBoolRepresentation(i);
size_t argument_index = projection_column->getBool(i);
col_res->insertFrom(*block.getByPosition(arguments[argument_index + 1]).column, override_indices[argument_index]++);
}
block.getByPosition(result).column = std::move(col_res);

16
dbms/src/Functions/FunctionsProjection.h
View File

@ -3,13 +3,14 @@
#include <Functions/IFunction.h>
#include "FunctionsConversion.h"
namespace DB {
namespace DB
{
/*
* This function accepts one column and converts it to UInt8, replacing values, which evaluate to true, with 1, and values,
* which evaluate to false with 0
*/
class FunctionOneOrZero final : public IFunction {
class FunctionOneOrZero final : public IFunction
{
public:
static constexpr auto name = "one_or_zero";
static FunctionPtr create(const Context &);
@ -25,7 +26,8 @@ public:
* This function builds a column of a smaller, which contains values of the data column at the positions where
* the projection column contained 1. The size of result column equals the count of ones in the projection column.
*/
class FunctionProject final : public IFunction {
class FunctionProject final : public IFunction
{
public:
static constexpr auto name = "__inner_project__";
static FunctionPtr create(const Context &);
@ -39,7 +41,8 @@ public:
* FunctionBuildProjectionComposition constructs the composition of two projection columns. The size of
* second projection column should equal the count of ones in the first input projection column.
*/
class FunctionBuildProjectionComposition final : public IFunction {
class FunctionBuildProjectionComposition final : public IFunction
{
public:
static constexpr auto name = "__inner_build_projection_composition__";
static FunctionPtr create(const Context &);
@ -53,7 +56,8 @@ public:
* Accepts mapping column with values from range [0, N) and N more columns as arguments.
* Forms a column by taking value from column, which number is in the mapping column.
*/
class FunctionRestoreProjection final : public IFunction {
class FunctionRestoreProjection final : public IFunction
{
public:
static constexpr auto name = "__inner_restore_projection__";
static FunctionPtr create(const Context &);

12
dbms/src/Functions/FunctionsString.cpp
View File

@ -715,8 +715,7 @@ public:
{
const auto arg = arguments[arg_idx].get();
if (!arg->isStringOrFixedString())
throw Exception{
"Illegal type " + arg->getName() + " of argument " + std::to_string(arg_idx + 1) + " of function " + getName(),
throw Exception{"Illegal type " + arg->getName() + " of argument " + std::to_string(arg_idx + 1) + " of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
}
@ -1017,12 +1016,10 @@ private:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (!arguments[0]->isString())
throw Exception{
"Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
throw Exception{"Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
if (!arguments[1]->isString())
throw Exception{
"Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
throw Exception{"Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
return std::make_shared<DataTypeString>();
}
@ -1081,8 +1078,7 @@ private:
block.getByPosition(result).column = std::move(col_res);
}
else
throw Exception{
"Illegal column " + block.getByPosition(arguments[0]).column->getName() + " of argument of function " + getName(),
throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
}
};

31
dbms/src/Functions/FunctionsTransform.h
View File

@ -72,9 +72,7 @@ public:
{
const auto args_size = arguments.size();
if (args_size != 3 && args_size != 4)
throw Exception{
"Number of arguments for function " + getName() + " doesn't match: passed " +
toString(args_size) + ", should be 3 or 4",
throw Exception{"Number of arguments for function " + getName() + " doesn't match: passed " + toString(args_size) + ", should be 3 or 4",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
const DataTypePtr & type_x = arguments[0];
@ -180,9 +178,7 @@ public:
&& !executeNum<Float64>(in, out, default_column)
&& !executeString(in, out, default_column))
{
throw Exception{
"Illegal column " + in->getName() + " of first argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Illegal column " + in->getName() + " of first argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
}
block.getByPosition(result).column = std::move(column_result);
@ -224,10 +220,8 @@ private:
auto out = typeid_cast<ColumnVector<T> *>(out_untyped);
if (!out)
{
throw Exception{
"Illegal column " + out_untyped->getName() + " of elements of array of third argument of function " + getName()
+ ", must be " + in->getName(),
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Illegal column " + out_untyped->getName() + " of elements of array of third argument of function " + getName()
+ ", must be " + in->getName(), ErrorCodes::ILLEGAL_COLUMN};
}
executeImplNumToNum<T>(in->getData(), out->getData());
@ -246,8 +240,7 @@ private:
&& !executeNumToNumWithConstDefault<T, Float64>(in, out_untyped)
&& !executeNumToStringWithConstDefault<T>(in, out_untyped))
{
throw Exception{
"Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(),
throw Exception{"Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
}
}
@ -265,8 +258,7 @@ private:
&& !executeNumToNumWithNonConstDefault<T, Float64>(in, out_untyped, default_untyped)
&& !executeNumToStringWithNonConstDefault<T>(in, out_untyped, default_untyped))
{
throw Exception{
"Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(),
throw Exception{"Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
}
}
@ -284,11 +276,8 @@ private:
if (!default_untyped)
{
if (!executeStringToString(in, out_untyped))
{
throw Exception{
"Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(),
throw Exception{"Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
}
}
else if (default_untyped->isColumnConst())
{
@ -304,8 +293,7 @@ private:
&& !executeStringToNumWithConstDefault<Float64>(in, out_untyped)
&& !executeStringToStringWithConstDefault(in, out_untyped))
{
throw Exception{
"Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(),
throw Exception{"Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
}
}
@ -323,8 +311,7 @@ private:
&& !executeStringToNumWithNonConstDefault<Float64>(in, out_untyped, default_untyped)
&& !executeStringToStringWithNonConstDefault(in, out_untyped, default_untyped))
{
throw Exception{
"Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(),
throw Exception{"Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN};
}
}

5
dbms/src/Functions/IFunction.cpp
View File

@ -12,9 +12,13 @@
#include <ext/collection_cast.h>
#include <cstdlib>
#include <memory>
#include <optional>
#if USE_EMBEDDED_COMPILER
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#include <llvm/IR/IRBuilder.h>
#pragma GCC diagnostic pop
#endif
@ -27,6 +31,7 @@ namespace ErrorCodes
extern const int ILLEGAL_COLUMN;
}
namespace
{

2
dbms/src/Functions/registerFunctions_area.cpp.in
View File

@ -8,9 +8,7 @@ namespace DB
void registerFunctions@FUNCTION_AREA@(FunctionFactory & factory)
{
@REGISTER_FUNCTIONS@
}
}

4
dbms/src/IO/ReadBufferAIO.cpp
View File

@ -85,8 +85,8 @@ void ReadBufferAIO::setMaxBytes(size_t max_bytes_read_)
bool ReadBufferAIO::nextImpl()
{
/// If the end of the file has already been reached by calling this function,
/// then the current call is wrong.
/// If the end of the file has already been reached by calling this function,
/// then the current call is wrong.
if (is_eof)
return false;

28
dbms/src/Interpreters/ExpressionActions.cpp
View File

@ -288,37 +288,35 @@ void ExpressionAction::prepare(Block & sample_block)
}
}
size_t ExpressionAction::getInputRowsCount(Block & block, std::unordered_map<std::string, size_t> & input_rows_counts) const {
size_t ExpressionAction::getInputRowsCount(Block & block, std::unordered_map<std::string, size_t> & input_rows_counts) const
{
auto it = input_rows_counts.find(row_projection_column);
size_t projection_space_dimention;
size_t projection_space_dimension;
if (it == input_rows_counts.end())
{
const auto & projection_column = block.getByName(row_projection_column).column;
projection_space_dimention = 0;
projection_space_dimension = 0;
for (size_t i = 0; i < projection_column->size(); ++i)
{
if (projection_column->getBoolRepresentation(i) > 0)
{
++projection_space_dimention;
}
}
if (projection_column->getBool(i))
++projection_space_dimension;
input_rows_counts[row_projection_column] = projection_space_dimention;
input_rows_counts[row_projection_column] = projection_space_dimension;
}
else
{
projection_space_dimention = it->second;
projection_space_dimension = it->second;
}
size_t parent_space_dimention;
size_t parent_space_dimension;
if (row_projection_column.empty())
{
parent_space_dimention = input_rows_counts[""];
parent_space_dimension = input_rows_counts[""];
}
else
{
parent_space_dimention = block.getByName(row_projection_column).column->size();
parent_space_dimension = block.getByName(row_projection_column).column->size();
}
return is_row_projection_complementary ? parent_space_dimention - projection_space_dimention : projection_space_dimention;
return is_row_projection_complementary ? parent_space_dimension - projection_space_dimension : projection_space_dimension;
}
void ExpressionAction::execute(Block & block, std::unordered_map<std::string, size_t> & input_rows_counts) const

121
dbms/src/Interpreters/ExpressionJIT.cpp
View File

@ -2,15 +2,23 @@
#if USE_EMBEDDED_COMPILER
#include <optional>
#include <Columns/ColumnConst.h>
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnVector.h>
#include <Common/typeid_cast.h>
#include <Common/ProfileEvents.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/Native.h>
#include <Functions/IFunction.h>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#pragma GCC diagnostic ignored "-Wnon-virtual-dtor"
#include <llvm/Config/llvm-config.h>
#include <llvm/IR/BasicBlock.h>
#include <llvm/IR/DataLayout.h>
#include <llvm/IR/DerivedTypes.h>
@ -31,6 +39,30 @@
#include <llvm/Support/TargetSelect.h>
#include <llvm/Transforms/IPO/PassManagerBuilder.h>
#pragma GCC diagnostic pop
/** HACK
* Allow to link with LLVM that was compiled without RTTI.
* This is the default option when you build LLVM from sources.
* We define fake symbols for RTTI to help linker.
* This assumes that enabling/disabling RTTI doesn't change memory layout of objects
* in any significant way and it doesn't affect the code that isn't actually using RTTI.
* Proper solution: recompile LLVM with enabled RTTI.
*/
extern "C"
{
__attribute__((__weak__)) int _ZTIN4llvm13ErrorInfoBaseE = 0;
__attribute__((__weak__)) int _ZTIN4llvm12MemoryBufferE = 0;
}
namespace ProfileEvents
{
extern const Event CompileFunction;
}
namespace DB
{
@ -38,6 +70,7 @@ namespace DB
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int CANNOT_COMPILE_CODE;
}
namespace
@ -86,17 +119,38 @@ static void applyFunction(IFunctionBase & function, Field & value)
struct LLVMContext
{
llvm::LLVMContext context;
#if LLVM_VERSION_MAJOR >= 7
llvm::orc::ExecutionSession execution_session;
std::unique_ptr<llvm::Module> module;
#else
std::shared_ptr<llvm::Module> module;
#endif
std::unique_ptr<llvm::TargetMachine> machine;
llvm::orc::RTDyldObjectLinkingLayer objectLayer;
llvm::orc::IRCompileLayer<decltype(objectLayer), llvm::orc::SimpleCompiler> compileLayer;
llvm::DataLayout layout;
llvm::IRBuilder<> builder;
std::unordered_map<std::string, void *> symbols;
LLVMContext()
#if LLVM_VERSION_MAJOR >= 7
: module(std::make_unique<llvm::Module>("jit", context))
#else
: module(std::make_shared<llvm::Module>("jit", context))
#endif
, machine(llvm::EngineBuilder().selectTarget())
#if LLVM_VERSION_MAJOR >= 7
, objectLayer(execution_session, [](llvm::orc::VModuleKey)
{
return llvm::orc::RTDyldObjectLinkingLayer::Resources
{
std::make_shared<llvm::SectionMemoryManager>(),
std::make_shared<llvm::orc::NullResolver>()
};
})
#else
, objectLayer([]() { return std::make_shared<llvm::SectionMemoryManager>(); })
#endif
, compileLayer(objectLayer, llvm::orc::SimpleCompiler(*machine))
, layout(machine->createDataLayout())
, builder(context)
@ -120,7 +174,40 @@ struct LLVMContext
builder.populateFunctionPassManager(fpm);
for (auto & function : *module)
fpm.run(function);
llvm::cantFail(compileLayer.addModule(module, std::make_shared<llvm::orc::NullResolver>()));
/// name, mangled name
std::vector<std::pair<std::string, std::string>> function_names;
function_names.reserve(module->size());
for (const auto & function : *module)
{
std::string mangled_name;
llvm::raw_string_ostream mangled_name_stream(mangled_name);
llvm::Mangler::getNameWithPrefix(mangled_name_stream, function.getName(), layout);
mangled_name_stream.flush();
function_names.emplace_back(function.getName(), mangled_name);
}
#if LLVM_VERSION_MAJOR >= 7
llvm::orc::VModuleKey module_key = execution_session.allocateVModule();
if (compileLayer.addModule(module_key, std::move(module)))
throw Exception("Cannot add module to compile layer", ErrorCodes::CANNOT_COMPILE_CODE);
#else
if (compileLayer.addModule(module, std::make_shared<llvm::orc::NullResolver>()))
throw Exception("Cannot add module to compile layer", ErrorCodes::CANNOT_COMPILE_CODE);
#endif
for (const auto & names : function_names)
{
if (auto symbol = compileLayer.findSymbol(names.second, false))
{
if (auto address_or_error = symbol.getAddress())
symbols[names.first] = reinterpret_cast<void *>(*address_or_error);
else
throw Exception("Cannot get an address of compiled symbol from a module", ErrorCodes::CANNOT_COMPILE_CODE);
}
else
throw Exception("Cannot find compiled symbol in a module", ErrorCodes::CANNOT_COMPILE_CODE);
}
}
};
@ -128,17 +215,12 @@ class LLVMPreparedFunction : public PreparedFunctionImpl
{
std::string name;
std::shared_ptr<LLVMContext> context;
const void * function;
void * function;
public:
LLVMPreparedFunction(std::string name_, std::shared_ptr<LLVMContext> context)
: name(std::move(name_)), context(context)
{
std::string mangledName;
llvm::raw_string_ostream mangledNameStream(mangledName);
llvm::Mangler::getNameWithPrefix(mangledNameStream, name, context->layout);
function = reinterpret_cast<const void *>(context->compileLayer.findSymbol(mangledNameStream.str(), false).getAddress().get());
}
: name(std::move(name_)), context(context), function(context->symbols.at(name))
{}
String getName() const override { return name; }
@ -152,11 +234,11 @@ public:
if (block_size)
{
std::vector<ColumnData> columns(arguments.size() + 1);
for (size_t i = 0; i < arguments.size(); i++)
for (size_t i = 0; i < arguments.size(); ++i)
{
auto * column = block.getByPosition(arguments[i]).column.get();
if (!column)
throw Exception("column " + block.getByPosition(arguments[i]).name + " is missing", ErrorCodes::LOGICAL_ERROR);
throw Exception("Column " + block.getByPosition(arguments[i]).name + " is missing", ErrorCodes::LOGICAL_ERROR);
columns[i] = getColumnData(column);
}
columns[arguments.size()] = getColumnData(col_res.get());
@ -168,6 +250,8 @@ public:
static void compileFunction(std::shared_ptr<LLVMContext> & context, const IFunctionBase & f)
{
ProfileEvents::increment(ProfileEvents::CompileFunction);
auto & arg_types = f.getArgumentTypes();
auto & b = context->builder;
auto * size_type = b.getIntNTy(sizeof(size_t) * 8);
@ -181,7 +265,7 @@ static void compileFunction(std::shared_ptr<LLVMContext> & context, const IFunct
auto * entry = llvm::BasicBlock::Create(b.getContext(), "entry", func);
b.SetInsertPoint(entry);
std::vector<ColumnDataPlaceholder> columns(arg_types.size() + 1);
for (size_t i = 0; i <= arg_types.size(); i++)
for (size_t i = 0; i <= arg_types.size(); ++i)
{
auto & type = i == arg_types.size() ? f.getReturnType() : arg_types[i];
auto * data = b.CreateLoad(b.CreateConstInBoundsGEP1_32(data_type, columns_arg, i));
@ -207,7 +291,7 @@ static void compileFunction(std::shared_ptr<LLVMContext> & context, const IFunct
}
}
ValuePlaceholders arguments(arg_types.size());
for (size_t i = 0; i < arguments.size(); i++)
for (size_t i = 0; i < arguments.size(); ++i)
{
arguments[i] = [&b, &col = columns[i], &type = arg_types[i]]() -> llvm::Value *
{
@ -290,7 +374,7 @@ static CompilableExpression subexpression(const IFunctionBase & f, std::vector<C
input.push_back([&]() { return arg(builder, inputs); });
auto * result = f.compile(builder, input);
if (result->getType() != toNativeType(builder, f.getReturnType()))
throw Exception("function " + f.getName() + " generated an llvm::Value of invalid type", ErrorCodes::LOGICAL_ERROR);
throw Exception("Function " + f.getName() + " generated an llvm::Value of invalid type", ErrorCodes::LOGICAL_ERROR);
return result;
};
}
@ -317,7 +401,7 @@ public:
const auto & names = action.argument_names;
const auto & types = action.function->getArgumentTypes();
std::vector<CompilableExpression> args;
for (size_t i = 0; i < names.size(); i++)
for (size_t i = 0; i < names.size(); ++i)
{
auto inserted = subexpressions.emplace(names[i], subexpression(arg_names.size()));
if (inserted.second)
@ -394,7 +478,7 @@ public:
Field right_ = right;
Monotonicity result(true, true, true);
/// monotonicity is only defined for unary functions, so the chain must describe a sequence of nested calls
for (size_t i = 0; i < originals.size(); i++)
for (size_t i = 0; i < originals.size(); ++i)
{
Monotonicity m = originals[i]->getMonotonicityForRange(*type_, left_, right_);
if (!m.is_monotonic)
@ -481,10 +565,11 @@ void compileFunctions(ExpressionActions::Actions & actions, const Names & output
}
std::vector<ExpressionActions::Actions> fused(actions.size());
for (size_t i = 0; i < actions.size(); i++)
for (size_t i = 0; i < actions.size(); ++i)
{
if (actions[i].type != ExpressionAction::APPLY_FUNCTION || !isCompilable(context->builder, *actions[i].function))
continue;
fused[i].push_back(actions[i]);
if (dependents[i].find({}) != dependents[i].end())
{
@ -496,10 +581,12 @@ void compileFunctions(ExpressionActions::Actions & actions, const Names & output
actions[i].argument_names = fn->getArgumentNames();
continue;
}
/// TODO: determine whether it's profitable to inline the function if there's more than one dependent.
for (const auto & dep : dependents[i])
fused[*dep].insert(fused[*dep].end(), fused[i].begin(), fused[i].end());
}
context->finalize();
}

3
dbms/src/Interpreters/ExpressionJIT.h
View File

@ -1,10 +1,11 @@
#pragma once
#include <Common/config.h>
#include <Interpreters/ExpressionActions.h>
#if USE_EMBEDDED_COMPILER
#include <Interpreters/ExpressionActions.h>
namespace DB
{

2
dbms/src/Interpreters/PartLog.cpp
View File

@ -49,7 +49,7 @@ Block PartLogElement::createBlock()
/// Is there an error during the execution or commit
{ColumnUInt16::create(), std::make_shared<DataTypeUInt16>(), "error"},
{ColumnString::create(), std::make_shared<DataTypeString>(), "exception"},
};
};
}
void PartLogElement::appendToBlock(Block & block) const

191
dbms/src/Interpreters/ProjectionManipulation.cpp
View File

@ -1,23 +1,20 @@
#include <Common/Exception.h>
#include <Interpreters/ExpressionActions.h>
#include <Functions/FunctionFactory.h>
#include <Interpreters/ExpressionAnalyzer.h>
#include <DataTypes/DataTypesNumber.h>
#include <Columns/ColumnsNumber.h>
#include <Interpreters/ProjectionManipulation.h>
#include <vector>
#include <string>
#include <memory>
#include <string>
#include <vector>
#include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypesNumber.h>
#include <Functions/FunctionFactory.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/ExpressionAnalyzer.h>
#include <Interpreters/ProjectionManipulation.h>
#include <Common/Exception.h>
#include <Common/typeid_cast.h>
namespace DB {
namespace DB
{
ProjectionManipulatorBase::~ProjectionManipulatorBase() {}
ProjectionManipulatorBase::~ProjectionManipulatorBase()
{}
DefaultProjectionManipulator::DefaultProjectionManipulator(ScopeStack & scopes)
: scopes(scopes)
{}
DefaultProjectionManipulator::DefaultProjectionManipulator(ScopeStack & scopes) : scopes(scopes) {}
bool DefaultProjectionManipulator::tryToGetFromUpperProjection(const std::string & column_name)
{
@ -34,22 +31,19 @@ std::string DefaultProjectionManipulator::getProjectionExpression()
return "";
}
std::string DefaultProjectionManipulator::getProjectionSourceColumn() const {
std::string DefaultProjectionManipulator::getProjectionSourceColumn() const
{
return "";
}
ConditionalTree::Node::Node()
: projection_expression_string(),
parent_node(0),
is_root(false)
{}
ConditionalTree::Node::Node() : projection_expression_string(), parent_node(0), is_root(false) {}
size_t ConditionalTree::Node::getParentNode() const
{
if (is_root)
{
throw Exception("Failed to get parent projection node of node " + projection_expression_string,
ErrorCodes::CONDITIONAL_TREE_PARENT_NOT_FOUND);
throw Exception(
"Failed to get parent projection node of node " + projection_expression_string, ErrorCodes::CONDITIONAL_TREE_PARENT_NOT_FOUND);
}
else
{
@ -75,22 +69,19 @@ std::string ConditionalTree::getColumnName(const std::string & col_name) const
return getColumnNameByIndex(col_name, current_node);
}
std::string ConditionalTree::getProjectionColumnName(const std::string & first_projection_expr,
const std::string & second_projection_expr) const
std::string ConditionalTree::getProjectionColumnName(
const std::string & first_projection_expr, const std::string & second_projection_expr) const
{
return std::string("P<") + first_projection_expr + "><" + second_projection_expr + ">";
}
std::string ConditionalTree::getProjectionColumnName(const size_t first_index, const size_t second_index) const
{
return getProjectionColumnName(
nodes[first_index].projection_expression_string,
nodes[second_index].projection_expression_string);
return getProjectionColumnName(nodes[first_index].projection_expression_string, nodes[second_index].projection_expression_string);
}
void ConditionalTree::buildProjectionCompositionRecursive(const std::vector<size_t> & path,
const size_t child_index,
const size_t parent_index)
void ConditionalTree::buildProjectionCompositionRecursive(
const std::vector<size_t> & path, const size_t child_index, const size_t parent_index)
{
std::string projection_name = getProjectionColumnName(path[parent_index], path[child_index]);
if (parent_index - child_index >= 2 && !scopes.getSampleBlock().has(projection_name))
@ -98,15 +89,12 @@ void ConditionalTree::buildProjectionCompositionRecursive(const std::vector<size
size_t middle_index = (child_index + parent_index) / 2;
buildProjectionCompositionRecursive(path, child_index, middle_index);
buildProjectionCompositionRecursive(path, middle_index, parent_index);
const FunctionBuilderPtr & function_builder = FunctionFactory::instance().get(
"__inner_build_projection_composition__", context);
scopes.addAction(ExpressionAction::applyFunction(
function_builder,
{
getProjectionColumnName(path[parent_index], path[middle_index]),
getProjectionColumnName(path[middle_index], path[child_index])
},
projection_name, getProjectionSourceColumn()));
const FunctionBuilderPtr & function_builder = FunctionFactory::instance().get("__inner_build_projection_composition__", context);
scopes.addAction(ExpressionAction::applyFunction(function_builder,
{getProjectionColumnName(path[parent_index], path[middle_index]),
getProjectionColumnName(path[middle_index], path[child_index])},
projection_name,
getProjectionSourceColumn()));
}
}
@ -126,7 +114,8 @@ void ConditionalTree::buildProjectionComposition(const size_t child_node, const
buildProjectionCompositionRecursive(path, 0, path.size() - 1);
}
std::string ConditionalTree::getProjectionSourceColumn(size_t node) const {
std::string ConditionalTree::getProjectionSourceColumn(size_t node) const
{
if (nodes[node].is_root)
{
return "";
@ -138,11 +127,7 @@ std::string ConditionalTree::getProjectionSourceColumn(size_t node) const {
}
ConditionalTree::ConditionalTree(ScopeStack & scopes, const Context & context)
: current_node(0),
nodes(1),
scopes(scopes),
context(context),
projection_expression_index()
: current_node(0), nodes(1), scopes(scopes), context(context), projection_expression_index()
{
nodes[0].is_root = true;
}
@ -155,14 +140,16 @@ void ConditionalTree::goToProjection(const std::string & field_name)
if (!scopes.getSampleBlock().has(projection_column_name))
{
const FunctionBuilderPtr & function_builder = FunctionFactory::instance().get("one_or_zero", context);
scopes.addAction(ExpressionAction::applyFunction(function_builder, {getColumnName(field_name)}, projection_column_name,
getProjectionSourceColumn()));
scopes.addAction(ExpressionAction::applyFunction(
function_builder, {getColumnName(field_name)}, projection_column_name, getProjectionSourceColumn()));
nodes.emplace_back(Node());
nodes.back().projection_expression_string = new_projection_name;
nodes.back().parent_node = current_node;
current_node = nodes.size() - 1;
projection_expression_index[projection_column_name] = current_node;
} else {
}
else
{
current_node = projection_expression_index[projection_column_name];
}
}
@ -179,27 +166,20 @@ std::string ConditionalTree::buildRestoreProjectionAndGetName(const size_t level
}
void ConditionalTree::restoreColumn(
const std::string & default_values_name,
const std::string & new_values_name,
const size_t levels_up,
const std::string & result_name
)
const std::string & default_values_name, const std::string & new_values_name, const size_t levels_up, const std::string & result_name)
{
size_t target_node = current_node;
for (size_t i = 0; i < levels_up; ++i)
{
target_node = nodes[target_node].getParentNode();
}
const FunctionBuilderPtr & function_builder = FunctionFactory::instance().get("__inner_restore_projection__",
context);
scopes.addAction(ExpressionAction::applyFunction(
function_builder,
{
getProjectionColumnName(target_node, current_node),
const FunctionBuilderPtr & function_builder = FunctionFactory::instance().get("__inner_restore_projection__", context);
scopes.addAction(ExpressionAction::applyFunction(function_builder,
{getProjectionColumnName(target_node, current_node),
getColumnNameByIndex(default_values_name, current_node),
getColumnNameByIndex(new_values_name, current_node)
},
getColumnNameByIndex(result_name, target_node), getProjectionSourceColumn()));
getColumnNameByIndex(new_values_name, current_node)},
getColumnNameByIndex(result_name, target_node),
getProjectionSourceColumn()));
}
void ConditionalTree::goUp(const size_t levels_up)
@ -220,15 +200,11 @@ bool ConditionalTree::tryToGetFromUpperProjection(const std::string & column_nam
if (node != current_node)
{
buildProjectionComposition(current_node, node);
const FunctionBuilderPtr & function_builder = FunctionFactory::instance().get("__inner_project__",
context);
scopes.addAction(ExpressionAction::applyFunction(
function_builder,
{
getColumnNameByIndex(column_name, node),
getProjectionColumnName(node, current_node)
},
getColumnName(column_name), getProjectionSourceColumn(node)));
const FunctionBuilderPtr & function_builder = FunctionFactory::instance().get("__inner_project__", context);
scopes.addAction(ExpressionAction::applyFunction(function_builder,
{getColumnNameByIndex(column_name, node), getProjectionColumnName(node, current_node)},
getColumnName(column_name),
getProjectionSourceColumn(node)));
}
return true;
}
@ -246,36 +222,32 @@ std::string ConditionalTree::getProjectionExpression()
return nodes[current_node].projection_expression_string;
}
std::string ConditionalTree::getProjectionSourceColumn() const {
std::string ConditionalTree::getProjectionSourceColumn() const
{
return getProjectionSourceColumn(current_node);
}
void DefaultProjectionAction::preArgumentAction()
{}
void DefaultProjectionAction::preArgumentAction() {}
void DefaultProjectionAction::postArgumentAction(const std::string & /*argument_name*/)
{
}
void DefaultProjectionAction::postArgumentAction(const std::string & /*argument_name*/) {}
void DefaultProjectionAction::preCalculation()
{}
void DefaultProjectionAction::preCalculation() {}
bool DefaultProjectionAction::isCalculationRequired()
{
return true;
}
AndOperatorProjectionAction::AndOperatorProjectionAction(ScopeStack & scopes,
ProjectionManipulatorPtr projection_manipulator,
const std::string & expression_name,
const Context & context)
: scopes(scopes),
projection_manipulator(projection_manipulator),
previous_argument_name(),
projection_levels_count(0),
expression_name(expression_name),
context(context)
{}
AndOperatorProjectionAction::AndOperatorProjectionAction(
ScopeStack & scopes, ProjectionManipulatorPtr projection_manipulator, const std::string & expression_name, const Context & context)
: scopes(scopes)
, projection_manipulator(projection_manipulator)
, previous_argument_name()
, projection_levels_count(0)
, expression_name(expression_name)
, context(context)
{
}
std::string AndOperatorProjectionAction::getZerosColumnName()
{
@ -292,9 +264,10 @@ void AndOperatorProjectionAction::createZerosColumn(const std::string & restore_
auto zeros_column_name = projection_manipulator->getColumnName(getZerosColumnName());
if (!scopes.getSampleBlock().has(zeros_column_name))
{
scopes.addAction(ExpressionAction::addColumn(ColumnWithTypeAndName(
ColumnUInt8::create(0, 1), std::make_shared<DataTypeUInt8>(), zeros_column_name),
restore_projection_name, true));
scopes.addAction(ExpressionAction::addColumn(
ColumnWithTypeAndName(ColumnUInt8::create(0, 1), std::make_shared<DataTypeUInt8>(), zeros_column_name),
restore_projection_name,
true));
}
}
@ -309,7 +282,8 @@ void AndOperatorProjectionAction::preArgumentAction()
}
else
{
throw Exception("Illegal projection manipulator used in AndOperatorProjectionAction", ErrorCodes::ILLEGAL_PROJECTION_MANIPULATOR);
throw Exception(
"Illegal projection manipulator used in AndOperatorProjectionAction", ErrorCodes::ILLEGAL_PROJECTION_MANIPULATOR);
}
++projection_levels_count;
}
@ -326,17 +300,13 @@ void AndOperatorProjectionAction::preCalculation()
{
auto final_column = getFinalColumnName();
const FunctionBuilderPtr & function_builder = FunctionFactory::instance().get("one_or_zero", context);
scopes.addAction(ExpressionAction::applyFunction(
function_builder,
{
projection_manipulator->getColumnName(previous_argument_name)
},
projection_manipulator->getColumnName(final_column),
projection_manipulator->getProjectionSourceColumn()));
scopes.addAction(ExpressionAction::applyFunction(function_builder,
{projection_manipulator->getColumnName(previous_argument_name)},
projection_manipulator->getColumnName(final_column),
projection_manipulator->getProjectionSourceColumn()));
std::string restore_projection_name = conditional_tree->buildRestoreProjectionAndGetName(projection_levels_count);
createZerosColumn(restore_projection_name);
conditional_tree->restoreColumn(getZerosColumnName(), final_column,
projection_levels_count, expression_name);
conditional_tree->restoreColumn(getZerosColumnName(), final_column, projection_levels_count, expression_name);
conditional_tree->goUp(projection_levels_count);
}
else
@ -350,14 +320,13 @@ bool AndOperatorProjectionAction::isCalculationRequired()
return false;
}
ProjectionActionBase::~ProjectionActionBase()
{}
ProjectionActionBase::~ProjectionActionBase() {}
ProjectionActionPtr getProjectionAction(const std::string & node_name,
ScopeStack & scopes,
ProjectionManipulatorPtr projection_manipulator,
const std::string & expression_name,
const Context & context)
ScopeStack & scopes,
ProjectionManipulatorPtr projection_manipulator,
const std::string & expression_name,
const Context & context)
{
if (typeid_cast<ConditionalTree *>(projection_manipulator.get()) && node_name == "and")
{

63
dbms/src/Interpreters/ProjectionManipulation.h
View File

@ -3,21 +3,23 @@
#include <string>
#include <Interpreters/ExpressionAnalyzer.h>
namespace DB {
namespace DB
{
class ExpressionAnalyzer;
struct ScopeStack;
namespace ErrorCodes {
extern const int CONDITIONAL_TREE_PARENT_NOT_FOUND;
extern const int ILLEGAL_PROJECTION_MANIPULATOR;
namespace ErrorCodes
{
extern const int CONDITIONAL_TREE_PARENT_NOT_FOUND;
extern const int ILLEGAL_PROJECTION_MANIPULATOR;
}
/*
* This is a base class for the ConditionalTree. Look at the description of ConditionalTree.
*/
struct ProjectionManipulatorBase {
struct ProjectionManipulatorBase
{
public:
virtual bool tryToGetFromUpperProjection(const std::string & column_name) = 0;
@ -37,9 +39,11 @@ using ProjectionManipulatorPtr = std::shared_ptr<ProjectionManipulatorBase>;
* For the better understanding of what ProjectionManipulator does,
* look at the description of ConditionalTree.
*/
struct DefaultProjectionManipulator : public ProjectionManipulatorBase {
struct DefaultProjectionManipulator : public ProjectionManipulatorBase
{
private:
ScopeStack & scopes;
public:
explicit DefaultProjectionManipulator(ScopeStack & scopes);
@ -86,9 +90,11 @@ public:
* understand whether we need to scan the expression deeply, or can it be easily computed just with the projection
* from one of the higher projection layers.
*/
struct ConditionalTree : public ProjectionManipulatorBase {
struct ConditionalTree : public ProjectionManipulatorBase
{
private:
struct Node {
struct Node
{
Node();
size_t getParentNode() const;
@ -103,21 +109,20 @@ private:
ScopeStack & scopes;
const Context & context;
std::unordered_map<std::string, size_t> projection_expression_index;
private:
std::string getColumnNameByIndex(const std::string & col_name, size_t node) const;
std::string getProjectionColumnName(const std::string & first_projection_expr,
const std::string & second_projection_expr) const;
std::string getProjectionColumnName(const std::string & first_projection_expr, const std::string & second_projection_expr) const;
std::string getProjectionColumnName(size_t first_index, size_t second_index) const;
void buildProjectionCompositionRecursive(const std::vector<size_t> & path,
size_t child_index,
size_t parent_index);
void buildProjectionCompositionRecursive(const std::vector<size_t> & path, size_t child_index, size_t parent_index);
void buildProjectionComposition(size_t child_node, size_t parent_node);
std::string getProjectionSourceColumn(size_t node) const;
public:
ConditionalTree(ScopeStack & scopes, const Context & context);
@ -128,11 +133,7 @@ public:
std::string buildRestoreProjectionAndGetName(size_t levels_up);
void restoreColumn(
const std::string & default_values_name,
const std::string & new_values_name,
size_t levels_up,
const std::string & result_name
);
const std::string & default_values_name, const std::string & new_values_name, size_t levels_up, const std::string & result_name);
void goUp(size_t levels_up);
@ -150,7 +151,8 @@ using ConditionalTreePtr = std::shared_ptr<ConditionalTree>;
* This class has two inherited classes: DefaultProjectionAction, which does nothing, and AndOperatorProjectionAction,
* which represents how function "and" uses projection manipulator.
*/
class ProjectionActionBase {
class ProjectionActionBase
{
public:
/*
* What to do before scanning the function argument (each of it)
@ -177,7 +179,8 @@ public:
using ProjectionActionPtr = std::shared_ptr<ProjectionActionBase>;
class DefaultProjectionAction : public ProjectionActionBase {
class DefaultProjectionAction : public ProjectionActionBase
{
public:
void preArgumentAction() final;
@ -191,7 +194,8 @@ public:
/*
* This is a specification of ProjectionAction specifically for the 'and' operation
*/
class AndOperatorProjectionAction : public ProjectionActionBase {
class AndOperatorProjectionAction : public ProjectionActionBase
{
private:
ScopeStack & scopes;
ProjectionManipulatorPtr projection_manipulator;
@ -205,11 +209,10 @@ private:
std::string getFinalColumnName();
void createZerosColumn(const std::string & restore_projection_name);
public:
AndOperatorProjectionAction(ScopeStack & scopes,
ProjectionManipulatorPtr projection_manipulator,
const std::string & expression_name,
const Context& context);
AndOperatorProjectionAction(
ScopeStack & scopes, ProjectionManipulatorPtr projection_manipulator, const std::string & expression_name, const Context & context);
/*
* Before scanning each argument, we should go to the next projection layer. For example, if the expression is
@ -241,9 +244,9 @@ public:
* it returns the pointer to AndOperatorProjectionAction.
*/
ProjectionActionPtr getProjectionAction(const std::string & node_name,
ScopeStack & scopes,
ProjectionManipulatorPtr projection_manipulator,
const std::string & expression_name,
const Context & context);
ScopeStack & scopes,
ProjectionManipulatorPtr projection_manipulator,
const std::string & expression_name,
const Context & context);
}

3
dbms/src/Interpreters/SettingsCommon.h
View File

@ -730,7 +730,8 @@ public:
return String(1, value);
}
void set(char x) {
void set(char x)
{
value = x;
changed = true;
}

42
dbms/src/Interpreters/tests/hash_map_string_2.cpp
View File

@ -22,27 +22,27 @@
/** Do this:
for file in MobilePhoneModel PageCharset Params URLDomain UTMSource Referer URL Title; do
for size in 30000 100000 300000 1000000 5000000; do
echo
BEST_METHOD=0
BEST_RESULT=0
for method in {1..12}; do
echo -ne $file $size $method '';
TOTAL_ELEMS=0
for i in {0..1000}; do
TOTAL_ELEMS=$(( $TOTAL_ELEMS + $size ))
if [[ $TOTAL_ELEMS -gt 25000000 ]]; then break; fi
./hash_map_string_2 $size $method < ${file}.bin 2>&1 |
grep HashMap | grep -oE '[0-9\.]+ elem';
done | awk -W interactive '{ if ($1 > x) { x = $1 }; printf(".") } END { print x }' | tee /tmp/hash_map_string_2_res;
CUR_RESULT=$(cat /tmp/hash_map_string_2_res | tr -d '.')
if [[ $CUR_RESULT -gt $BEST_RESULT ]]; then
BEST_METHOD=$method
BEST_RESULT=$CUR_RESULT
fi;
done;
echo Best: $BEST_METHOD - $BEST_RESULT
done;
for size in 30000 100000 300000 1000000 5000000; do
echo
BEST_METHOD=0
BEST_RESULT=0
for method in {1..12}; do
echo -ne $file $size $method '';
TOTAL_ELEMS=0
for i in {0..1000}; do
TOTAL_ELEMS=$(( $TOTAL_ELEMS + $size ))
if [[ $TOTAL_ELEMS -gt 25000000 ]]; then break; fi
./hash_map_string_2 $size $method < ${file}.bin 2>&1 |
grep HashMap | grep -oE '[0-9\.]+ elem';
done | awk -W interactive '{ if ($1 > x) { x = $1 }; printf(".") } END { print x }' | tee /tmp/hash_map_string_2_res;
CUR_RESULT=$(cat /tmp/hash_map_string_2_res | tr -d '.')
if [[ $CUR_RESULT -gt $BEST_RESULT ]]; then
BEST_METHOD=$method
BEST_RESULT=$CUR_RESULT
fi;
done;
echo Best: $BEST_METHOD - $BEST_RESULT
done;
done
*/

42
dbms/src/Interpreters/tests/hash_map_string_3.cpp
View File

@ -25,27 +25,27 @@
/** Do this:
for file in MobilePhoneModel PageCharset Params URLDomain UTMSource Referer URL Title; do
for size in 30000 100000 300000 1000000 5000000; do
echo
BEST_METHOD=0
BEST_RESULT=0
for method in {1..11}; do
echo -ne $file $size $method '';
TOTAL_ELEMS=0
for i in {0..1000}; do
TOTAL_ELEMS=$(( $TOTAL_ELEMS + $size ))
if [[ $TOTAL_ELEMS -gt 25000000 ]]; then break; fi
./hash_map_string_3 $size $method < ${file}.bin 2>&1 |
grep HashMap | grep -oE '[0-9\.]+ elem';
done | awk -W interactive '{ if ($1 > x) { x = $1 }; printf(".") } END { print x }' | tee /tmp/hash_map_string_3_res;
CUR_RESULT=$(cat /tmp/hash_map_string_3_res | tr -d '.')
if [[ $CUR_RESULT -gt $BEST_RESULT ]]; then
BEST_METHOD=$method
BEST_RESULT=$CUR_RESULT
fi;
done;
echo Best: $BEST_METHOD - $BEST_RESULT
done;
for size in 30000 100000 300000 1000000 5000000; do
echo
BEST_METHOD=0
BEST_RESULT=0
for method in {1..11}; do
echo -ne $file $size $method '';
TOTAL_ELEMS=0
for i in {0..1000}; do
TOTAL_ELEMS=$(( $TOTAL_ELEMS + $size ))
if [[ $TOTAL_ELEMS -gt 25000000 ]]; then break; fi
./hash_map_string_3 $size $method < ${file}.bin 2>&1 |
grep HashMap | grep -oE '[0-9\.]+ elem';
done | awk -W interactive '{ if ($1 > x) { x = $1 }; printf(".") } END { print x }' | tee /tmp/hash_map_string_3_res;
CUR_RESULT=$(cat /tmp/hash_map_string_3_res | tr -d '.')
if [[ $CUR_RESULT -gt $BEST_RESULT ]]; then
BEST_METHOD=$method
BEST_RESULT=$CUR_RESULT
fi;
done;
echo Best: $BEST_METHOD - $BEST_RESULT
done;
done
*/

7
dbms/src/Parsers/iostream_debug_helpers.cpp
View File

@ -6,13 +6,14 @@
namespace DB
{
std::ostream & operator<<(std::ostream & stream, const Token & what) {
std::ostream & operator<<(std::ostream & stream, const Token & what)
{
stream << "Token (type="<< static_cast<int>(what.type) <<"){"<< std::string{what.begin, what.end} << "}";
return stream;
}
std::ostream & operator<<(std::ostream & stream, const Expected & what) {
std::ostream & operator<<(std::ostream & stream, const Expected & what)
{
stream << "Expected {variants=";
dumpValue(stream, what.variants)
<< "; max_parsed_pos=" << what.max_parsed_pos << "}";

6
dbms/src/Server/Compiler-5.0.0/CMakeLists.txt
View File

@ -42,7 +42,9 @@ LLVMSupport
#PollyISL
#PollyPPCG
libtinfo.a
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
)
if (NOT APPLE)
target_link_libraries(clickhouse-compiler-lib PRIVATE libtinfo.a)
endif()

106
dbms/src/Server/Compiler-5.0.0/lld.cpp
View File

@ -1,22 +1,4 @@
//===- tools/lld/lld.cpp - Linker Driver Dispatcher -----------------------===//
//
// The LLVM Linker
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the lld driver. This is a thin wrapper which
// dispatches to the given platform specific driver.
//
// If there is -flavor option, it is dispatched according to the arguments.
// If the flavor parameter is not present, then it is dispatched according
// to argv[0].
//
//===----------------------------------------------------------------------===//
#include "lld/Driver/Driver.h"
#include "lld/Common/Driver.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
@ -29,93 +11,13 @@ using namespace lld;
using namespace llvm;
using namespace llvm::sys;
enum Flavor {
Invalid,
Gnu, // -flavor gnu
WinLink, // -flavor link
Darwin, // -flavor darwin
};
LLVM_ATTRIBUTE_NORETURN static void die(const Twine &S) {
errs() << S << "\n";
exit(1);
}
#if 0
static Flavor getFlavor(StringRef S) {
return StringSwitch<Flavor>(S)
.CasesLower("ld", "ld.lld", "gnu", Gnu)
.CaseLower("link", WinLink)
.CaseLower("darwin", Darwin)
.Default(Invalid);
}
static Flavor parseProgname(StringRef Progname) {
#if __APPLE__
// Use Darwin driver for "ld" on Darwin.
if (Progname == "ld")
return Darwin;
#endif
#if LLVM_ON_UNIX
// Use GNU driver for "ld" on other Unix-like system.
if (Progname == "ld")
return Gnu;
#endif
// Progname may be something like "lld-gnu". Parse it.
SmallVector<StringRef, 3> V;
Progname.split(V, "-");
for (StringRef S : V)
if (Flavor F = getFlavor(S))
return F;
return Invalid;
}
static Flavor parseFlavor(std::vector<const char *> &V) {
// Parse -flavor option.
if (V.size() > 1 && V[1] == StringRef("-flavor")) {
if (V.size() <= 2)
die("missing arg value for '-flavor'");
Flavor F = getFlavor(V[2]);
if (F == Invalid)
die("Unknown flavor: " + StringRef(V[2]));
V.erase(V.begin() + 1, V.begin() + 3);
return F;
}
// Deduct the flavor from argv[0].
StringRef Arg0 = path::filename(V[0]);
if (Arg0.endswith_lower(".exe"))
Arg0 = Arg0.drop_back(4);
return parseProgname(Arg0);
}
#endif
/// Universal linker main(). This linker emulates the gnu, darwin, or
/// windows linker based on the argv[0] or -flavor option.
int mainEntryClickHouseLLD(int Argc, char **Argv) {
int mainEntryClickHouseLLD(int Argc, char **Argv)
{
// Standard set up, so program fails gracefully.
sys::PrintStackTraceOnErrorSignal(Argv[0]);
PrettyStackTraceProgram StackPrinter(Argc, Argv);
llvm_shutdown_obj Shutdown;
std::vector<const char *> Args(Argv, Argv + Argc);
#if 0
switch (parseFlavor(Args)) {
case Gnu:
#endif
return !elf::link(Args, true);
#if 0
case WinLink:
return !coff::link(Args);
case Darwin:
return !mach_o::link(Args);
default:
die("lld is a generic driver.\n"
"Invoke ld.lld (Unix), ld (macOS) or lld-link (Windows) instead.");
}
#endif
return !elf::link(Args, true);
}

1
dbms/src/Server/Compiler-5.0.2 Symbolic link
View File

@ -0,0 +1 @@
Compiler-5.0.0

6
dbms/src/Server/Compiler-6.0.0/CMakeLists.txt
View File

@ -42,7 +42,9 @@ ${REQUIRED_LLVM_LIBRARIES}
#PollyISL
#PollyPPCG
libtinfo.a
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
)
if (NOT APPLE)
target_link_libraries(clickhouse-compiler-lib PRIVATE libtinfo.a)
endif()

120
dbms/src/Server/Compiler-6.0.0/lld.cpp
View File

@ -1,21 +1,3 @@
//===- tools/lld/lld.cpp - Linker Driver Dispatcher -----------------------===//
//
// The LLVM Linker
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the lld driver. This is a thin wrapper which
// dispatches to the given platform specific driver.
//
// If there is -flavor option, it is dispatched according to the arguments.
// If the flavor parameter is not present, then it is dispatched according
// to argv[0].
//
//===----------------------------------------------------------------------===//
#include "lld/Common/Driver.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringSwitch.h"
@ -29,109 +11,13 @@ using namespace lld;
using namespace llvm;
using namespace llvm::sys;
enum Flavor {
Invalid,
Gnu, // -flavor gnu
WinLink, // -flavor link
Darwin, // -flavor darwin
Wasm, // -flavor wasm
};
LLVM_ATTRIBUTE_NORETURN static void die(const Twine &S) {
errs() << S << "\n";
exit(1);
}
#if 0
static Flavor getFlavor(StringRef S) {
return StringSwitch<Flavor>(S)
.CasesLower("ld", "ld.lld", "gnu", Gnu)
.CasesLower("wasm", "ld-wasm", Wasm)
.CaseLower("link", WinLink)
.CasesLower("ld64", "ld64.lld", "darwin", Darwin)
.Default(Invalid);
}
static bool isPETarget(const std::vector<const char *> &V) {
for (auto It = V.begin(); It + 1 != V.end(); ++It) {
if (StringRef(*It) != "-m")
continue;
StringRef S = *(It + 1);
return S == "i386pe" || S == "i386pep" || S == "thumb2pe" || S == "arm64pe";
}
return false;
}
static Flavor parseProgname(StringRef Progname) {
#if __APPLE__
// Use Darwin driver for "ld" on Darwin.
if (Progname == "ld")
return Darwin;
#endif
#if LLVM_ON_UNIX
// Use GNU driver for "ld" on other Unix-like system.
if (Progname == "ld")
return Gnu;
#endif
// Progname may be something like "lld-gnu". Parse it.
SmallVector<StringRef, 3> V;
Progname.split(V, "-");
for (StringRef S : V)
if (Flavor F = getFlavor(S))
return F;
return Invalid;
}
static Flavor parseFlavor(std::vector<const char *> &V) {
// Parse -flavor option.
if (V.size() > 1 && V[1] == StringRef("-flavor")) {
if (V.size() <= 2)
die("missing arg value for '-flavor'");
Flavor F = getFlavor(V[2]);
if (F == Invalid)
die("Unknown flavor: " + StringRef(V[2]));
V.erase(V.begin() + 1, V.begin() + 3);
return F;
}
// Deduct the flavor from argv[0].
StringRef Arg0 = path::filename(V[0]);
if (Arg0.endswith_lower(".exe"))
Arg0 = Arg0.drop_back(4);
return parseProgname(Arg0);
}
#endif
/// Universal linker main(). This linker emulates the gnu, darwin, or
/// windows linker based on the argv[0] or -flavor option.
int mainEntryClickHouseLLD(int Argc, char **Argv) {
int mainEntryClickHouseLLD(int Argc, char **Argv)
{
// Standard set up, so program fails gracefully.
sys::PrintStackTraceOnErrorSignal(Argv[0]);
PrettyStackTraceProgram StackPrinter(Argc, Argv);
llvm_shutdown_obj Shutdown;
std::vector<const char *> Args(Argv, Argv + Argc);
#if 0
switch (parseFlavor(Args)) {
case Gnu:
if (isPETarget(Args))
return !mingw::link(Args);
#endif
return !elf::link(Args, true);
#if 0
case WinLink:
return !coff::link(Args, true);
case Darwin:
return !mach_o::link(Args);
case Wasm:
return !wasm::link(Args, true);
default:
die("lld is a generic driver.\n"
"Invoke ld.lld (Unix), ld (macOS) or lld-link (Windows) instead.");
}
#endif
return !elf::link(Args, true);
}

0
dbms/src/Server/Compiler-7.0.0 → dbms/src/Server/Compiler-6.0.1
View File

56
dbms/src/Server/Compiler-7.0.0/CMakeLists.txt Normal file
View File

@ -0,0 +1,56 @@
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error")
add_library(clickhouse-compiler-lib
driver.cpp
cc1_main.cpp
cc1as_main.cpp
lld.cpp)
target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0)
llvm_map_components_to_libnames(REQUIRED_LLVM_LIBRARIES all)
# We link statically with zlib, and LLVM (sometimes) tries to bring its own dependency.
list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-lz")
# Wrong library in freebsd:
list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-l/usr/lib/libexecinfo.so")
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS})
# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory.
target_link_libraries(clickhouse-compiler-lib PRIVATE
clangBasic clangCodeGen clangDriver
clangFrontend
clangFrontendTool
clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend
clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers
clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic
clangCrossTU clangIndex
lldCOFF
lldDriver
lldELF
lldMinGW
lldMachO
lldReaderWriter
lldYAML
lldCommon
lldCore
#lldWasm
${REQUIRED_LLVM_LIBRARIES}
Polly
PollyISL
PollyPPCG
PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads
)
if (NOT APPLE)
target_link_libraries(clickhouse-compiler-lib PRIVATE libtinfo.a)
endif()

240
dbms/src/Server/Compiler-7.0.0/cc1_main.cpp Normal file
View File

@ -0,0 +1,240 @@
//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang -cc1 functionality, which implements the
// core compiler functionality along with a number of additional tools for
// demonstration and testing purposes.
//
//===----------------------------------------------------------------------===//
#include "llvm/Option/Arg.h"
#include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
#include "clang/Config/config.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/TextDiagnosticBuffer.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "clang/FrontendTool/Utils.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/LinkAllPasses.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <cstdio>
#ifdef CLANG_HAVE_RLIMITS
#include <sys/resource.h>
#endif
using namespace clang;
using namespace llvm::opt;
//===----------------------------------------------------------------------===//
// Main driver
//===----------------------------------------------------------------------===//
static void LLVMErrorHandler(void *UserData, const std::string &Message,
bool GenCrashDiag) {
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
// Run the interrupt handlers to make sure any special cleanups get done, in
// particular that we remove files registered with RemoveFileOnSignal.
llvm::sys::RunInterruptHandlers();
// We cannot recover from llvm errors. When reporting a fatal error, exit
// with status 70 to generate crash diagnostics. For BSD systems this is
// defined as an internal software error. Otherwise, exit with status 1.
exit(GenCrashDiag ? 70 : 1);
}
#ifdef LINK_POLLY_INTO_TOOLS
namespace polly {
void initializePollyPasses(llvm::PassRegistry &Registry);
}
#endif
#ifdef CLANG_HAVE_RLIMITS
// The amount of stack we think is "sufficient". If less than this much is
// available, we may be unable to reach our template instantiation depth
// limit and other similar limits.
// FIXME: Unify this with the stack we request when spawning a thread to build
// a module.
static const int kSufficientStack = 8 << 20;
#if defined(__linux__) && defined(__PIE__)
static size_t getCurrentStackAllocation() {
// If we can't compute the current stack usage, allow for 512K of command
// line arguments and environment.
size_t Usage = 512 * 1024;
if (FILE *StatFile = fopen("/proc/self/stat", "r")) {
// We assume that the stack extends from its current address to the end of
// the environment space. In reality, there is another string literal (the
// program name) after the environment, but this is close enough (we only
// need to be within 100K or so).
unsigned long StackPtr, EnvEnd;
// Disable silly GCC -Wformat warning that complains about length
// modifiers on ignored format specifiers. We want to retain these
// for documentation purposes even though they have no effect.
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wformat"
#endif
if (fscanf(StatFile,
"%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu "
"%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu "
"%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d "
"%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d",
&StackPtr, &EnvEnd) == 2) {
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic pop
#endif
Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd;
}
fclose(StatFile);
}
return Usage;
}
#include <alloca.h>
LLVM_ATTRIBUTE_NOINLINE
static void ensureStackAddressSpace(int ExtraChunks = 0) {
// Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary
// relatively close to the stack (they are only guaranteed to be 128MiB
// apart). This results in crashes if we happen to heap-allocate more than
// 128MiB before we reach our stack high-water mark.
//
// To avoid these crashes, ensure that we have sufficient virtual memory
// pages allocated before we start running.
size_t Curr = getCurrentStackAllocation();
const int kTargetStack = kSufficientStack - 256 * 1024;
if (Curr < kTargetStack) {
volatile char *volatile Alloc =
static_cast<volatile char *>(alloca(kTargetStack - Curr));
Alloc[0] = 0;
Alloc[kTargetStack - Curr - 1] = 0;
}
}
#else
static void ensureStackAddressSpace() {}
#endif
/// Attempt to ensure that we have at least 8MiB of usable stack space.
static void ensureSufficientStack() {
struct rlimit rlim;
if (getrlimit(RLIMIT_STACK, &rlim) != 0)
return;
// Increase the soft stack limit to our desired level, if necessary and
// possible.
if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) {
// Try to allocate sufficient stack.
if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack)
rlim.rlim_cur = kSufficientStack;
else if (rlim.rlim_cur == rlim.rlim_max)
return;
else
rlim.rlim_cur = rlim.rlim_max;
if (setrlimit(RLIMIT_STACK, &rlim) != 0 ||
rlim.rlim_cur != kSufficientStack)
return;
}
// We should now have a stack of size at least kSufficientStack. Ensure
// that we can actually use that much, if necessary.
ensureStackAddressSpace();
}
#else
static void ensureSufficientStack() {}
#endif
int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
ensureSufficientStack();
std::unique_ptr<CompilerInstance> Clang(new CompilerInstance());
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
// Register the support for object-file-wrapped Clang modules.
auto PCHOps = Clang->getPCHContainerOperations();
PCHOps->registerWriter(llvm::make_unique<ObjectFilePCHContainerWriter>());
PCHOps->registerReader(llvm::make_unique<ObjectFilePCHContainerReader>());
// Initialize targets first, so that --version shows registered targets.
llvm::InitializeAllTargets();
llvm::InitializeAllTargetMCs();
llvm::InitializeAllAsmPrinters();
llvm::InitializeAllAsmParsers();
#ifdef LINK_POLLY_INTO_TOOLS
llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry();
polly::initializePollyPasses(Registry);
#endif
// Buffer diagnostics from argument parsing so that we can output them using a
// well formed diagnostic object.
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer;
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
bool Success = CompilerInvocation::CreateFromArgs(
Clang->getInvocation(), Argv.begin(), Argv.end(), Diags);
// Infer the builtin include path if unspecified.
if (Clang->getHeaderSearchOpts().UseBuiltinIncludes &&
Clang->getHeaderSearchOpts().ResourceDir.empty())
Clang->getHeaderSearchOpts().ResourceDir =
CompilerInvocation::GetResourcesPath(Argv0, MainAddr);
// Create the actual diagnostics engine.
Clang->createDiagnostics();
if (!Clang->hasDiagnostics())
return 1;
// Set an error handler, so that any LLVM backend diagnostics go through our
// error handler.
llvm::install_fatal_error_handler(LLVMErrorHandler,
static_cast<void*>(&Clang->getDiagnostics()));
DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics());
if (!Success)
return 1;
// Execute the frontend actions.
Success = ExecuteCompilerInvocation(Clang.get());
// If any timers were active but haven't been destroyed yet, print their
// results now. This happens in -disable-free mode.
llvm::TimerGroup::printAll(llvm::errs());
// Our error handler depends on the Diagnostics object, which we're
// potentially about to delete. Uninstall the handler now so that any
// later errors use the default handling behavior instead.
llvm::remove_fatal_error_handler();
// When running with -disable-free, don't do any destruction or shutdown.
if (Clang->getFrontendOpts().DisableFree) {
BuryPointer(std::move(Clang));
return !Success;
}
return !Success;
}

565
dbms/src/Server/Compiler-7.0.0/cc1as_main.cpp Normal file
View File

@ -0,0 +1,565 @@
//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang -cc1as functionality, which implements
// the direct interface to the LLVM MC based assembler.
//
//===----------------------------------------------------------------------===//
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Triple.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/MC/MCAsmBackend.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCCodeEmitter.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCObjectFileInfo.h"
#include "llvm/MC/MCParser/MCAsmParser.h"
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSubtargetInfo.h"
#include "llvm/MC/MCTargetOptions.h"
#include "llvm/Option/Arg.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/FormattedStream.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <system_error>
using namespace clang;
using namespace clang::driver;
using namespace clang::driver::options;
using namespace llvm;
using namespace llvm::opt;
/// Clang 7 with debug variant of libc++ cannot compile itself without this patch.
namespace llvm
{
inline bool operator<(const StringRef & s, const SubtargetFeatureKV & feature)
{
return s < StringRef(feature.Key);
}
}
namespace {
/// \brief Helper class for representing a single invocation of the assembler.
struct AssemblerInvocation {
/// @name Target Options
/// @{
/// The name of the target triple to assemble for.
std::string Triple;
/// If given, the name of the target CPU to determine which instructions
/// are legal.
std::string CPU;
/// The list of target specific features to enable or disable -- this should
/// be a list of strings starting with '+' or '-'.
std::vector<std::string> Features;
/// The list of symbol definitions.
std::vector<std::string> SymbolDefs;
/// @}
/// @name Language Options
/// @{
std::vector<std::string> IncludePaths;
unsigned NoInitialTextSection : 1;
unsigned SaveTemporaryLabels : 1;
unsigned GenDwarfForAssembly : 1;
unsigned RelaxELFRelocations : 1;
unsigned DwarfVersion;
std::string DwarfDebugFlags;
std::string DwarfDebugProducer;
std::string DebugCompilationDir;
llvm::DebugCompressionType CompressDebugSections =
llvm::DebugCompressionType::None;
std::string MainFileName;
/// @}
/// @name Frontend Options
/// @{
std::string InputFile;
std::vector<std::string> LLVMArgs;
std::string OutputPath;
enum FileType {
FT_Asm, ///< Assembly (.s) output, transliterate mode.
FT_Null, ///< No output, for timing purposes.
FT_Obj ///< Object file output.
};
FileType OutputType;
unsigned ShowHelp : 1;
unsigned ShowVersion : 1;
/// @}
/// @name Transliterate Options
/// @{
unsigned OutputAsmVariant;
unsigned ShowEncoding : 1;
unsigned ShowInst : 1;
/// @}
/// @name Assembler Options
/// @{
unsigned RelaxAll : 1;
unsigned NoExecStack : 1;
unsigned FatalWarnings : 1;
unsigned IncrementalLinkerCompatible : 1;
/// The name of the relocation model to use.
std::string RelocationModel;
/// @}
public:
AssemblerInvocation() {
Triple = "";
NoInitialTextSection = 0;
InputFile = "-";
OutputPath = "-";
OutputType = FT_Asm;
OutputAsmVariant = 0;
ShowInst = 0;
ShowEncoding = 0;
RelaxAll = 0;
NoExecStack = 0;
FatalWarnings = 0;
IncrementalLinkerCompatible = 0;
DwarfVersion = 0;
}
static bool CreateFromArgs(AssemblerInvocation &Res,
ArrayRef<const char *> Argv,
DiagnosticsEngine &Diags);
};
}
bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
ArrayRef<const char *> Argv,
DiagnosticsEngine &Diags) {
bool Success = true;
// Parse the arguments.
std::unique_ptr<OptTable> OptTbl(createDriverOptTable());
const unsigned IncludedFlagsBitmask = options::CC1AsOption;
unsigned MissingArgIndex, MissingArgCount;
InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount,
IncludedFlagsBitmask);
// Check for missing argument error.
if (MissingArgCount) {
Diags.Report(diag::err_drv_missing_argument)
<< Args.getArgString(MissingArgIndex) << MissingArgCount;
Success = false;
}
// Issue errors on unknown arguments.
for (const Arg *A : Args.filtered(OPT_UNKNOWN)) {
auto ArgString = A->getAsString(Args);
std::string Nearest;
if (OptTbl->findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1)
Diags.Report(diag::err_drv_unknown_argument) << ArgString;
else
Diags.Report(diag::err_drv_unknown_argument_with_suggestion)
<< ArgString << Nearest;
Success = false;
}
// Construct the invocation.
// Target Options
Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
Opts.Features = Args.getAllArgValues(OPT_target_feature);
// Use the default target triple if unspecified.
if (Opts.Triple.empty())
Opts.Triple = llvm::sys::getDefaultTargetTriple();
// Language Options
Opts.IncludePaths = Args.getAllArgValues(OPT_I);
Opts.NoInitialTextSection = Args.hasArg(OPT_n);
Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels);
// Any DebugInfoKind implies GenDwarfForAssembly.
Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ);
if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections,
OPT_compress_debug_sections_EQ)) {
if (A->getOption().getID() == OPT_compress_debug_sections) {
// TODO: be more clever about the compression type auto-detection
Opts.CompressDebugSections = llvm::DebugCompressionType::GNU;
} else {
Opts.CompressDebugSections =
llvm::StringSwitch<llvm::DebugCompressionType>(A->getValue())
.Case("none", llvm::DebugCompressionType::None)
.Case("zlib", llvm::DebugCompressionType::Z)
.Case("zlib-gnu", llvm::DebugCompressionType::GNU)
.Default(llvm::DebugCompressionType::None);
}
}
Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations);
Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags);
Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer);
Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name);
// Frontend Options
if (Args.hasArg(OPT_INPUT)) {
bool First = true;
for (const Arg *A : Args.filtered(OPT_INPUT)) {
if (First) {
Opts.InputFile = A->getValue();
First = false;
} else {
Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args);
Success = false;
}
}
}
Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
Opts.OutputPath = Args.getLastArgValue(OPT_o);
if (Arg *A = Args.getLastArg(OPT_filetype)) {
StringRef Name = A->getValue();
unsigned OutputType = StringSwitch<unsigned>(Name)
.Case("asm", FT_Asm)
.Case("null", FT_Null)
.Case("obj", FT_Obj)
.Default(~0U);
if (OutputType == ~0U) {
Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
Success = false;
} else
Opts.OutputType = FileType(OutputType);
}
Opts.ShowHelp = Args.hasArg(OPT_help);
Opts.ShowVersion = Args.hasArg(OPT_version);
// Transliterate Options
Opts.OutputAsmVariant =
getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags);
Opts.ShowEncoding = Args.hasArg(OPT_show_encoding);
Opts.ShowInst = Args.hasArg(OPT_show_inst);
// Assemble Options
Opts.RelaxAll = Args.hasArg(OPT_mrelax_all);
Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack);
Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings);
Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic");
Opts.IncrementalLinkerCompatible =
Args.hasArg(OPT_mincremental_linker_compatible);
Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym);
return Success;
}
static std::unique_ptr<raw_fd_ostream>
getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags,
bool Binary) {
if (Opts.OutputPath.empty())
Opts.OutputPath = "-";
// Make sure that the Out file gets unlinked from the disk if we get a
// SIGINT.
if (Opts.OutputPath != "-")
sys::RemoveFileOnSignal(Opts.OutputPath);
std::error_code EC;
auto Out = llvm::make_unique<raw_fd_ostream>(
Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text));
if (EC) {
Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath
<< EC.message();
return nullptr;
}
return Out;
}
static bool ExecuteAssembler(AssemblerInvocation &Opts,
DiagnosticsEngine &Diags) {
// Get the target specific parser.
std::string Error;
const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error);
if (!TheTarget)
return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
ErrorOr<std::unique_ptr<MemoryBuffer>> Buffer =
MemoryBuffer::getFileOrSTDIN(Opts.InputFile);
if (std::error_code EC = Buffer.getError()) {
Error = EC.message();
return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile;
}
SourceMgr SrcMgr;
// Tell SrcMgr about this buffer, which is what the parser will pick up.
SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc());
// Record the location of the include directories so that the lexer can find
// it later.
SrcMgr.setIncludeDirs(Opts.IncludePaths);
std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(Opts.Triple));
assert(MRI && "Unable to create target register info!");
std::unique_ptr<MCAsmInfo> MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple));
assert(MAI && "Unable to create target asm info!");
// Ensure MCAsmInfo initialization occurs before any use, otherwise sections
// may be created with a combination of default and explicit settings.
MAI->setCompressDebugSections(Opts.CompressDebugSections);
MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations);
bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj;
std::unique_ptr<raw_fd_ostream> FDOS = getOutputStream(Opts, Diags, IsBinary);
if (!FDOS)
return true;
// FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
// MCObjectFileInfo needs a MCContext reference in order to initialize itself.
std::unique_ptr<MCObjectFileInfo> MOFI(new MCObjectFileInfo());
MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr);
bool PIC = false;
if (Opts.RelocationModel == "static") {
PIC = false;
} else if (Opts.RelocationModel == "pic") {
PIC = true;
} else {
assert(Opts.RelocationModel == "dynamic-no-pic" &&
"Invalid PIC model!");
PIC = false;
}
MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx);
if (Opts.SaveTemporaryLabels)
Ctx.setAllowTemporaryLabels(false);
if (Opts.GenDwarfForAssembly)
Ctx.setGenDwarfForAssembly(true);
if (!Opts.DwarfDebugFlags.empty())
Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags));
if (!Opts.DwarfDebugProducer.empty())
Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer));
if (!Opts.DebugCompilationDir.empty())
Ctx.setCompilationDir(Opts.DebugCompilationDir);
if (!Opts.MainFileName.empty())
Ctx.setMainFileName(StringRef(Opts.MainFileName));
Ctx.setDwarfVersion(Opts.DwarfVersion);
// Build up the feature string from the target feature list.
std::string FS;
if (!Opts.Features.empty()) {
FS = Opts.Features[0];
for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i)
FS += "," + Opts.Features[i];
}
std::unique_ptr<MCStreamer> Str;
std::unique_ptr<MCInstrInfo> MCII(TheTarget->createMCInstrInfo());
std::unique_ptr<MCSubtargetInfo> STI(
TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS));
raw_pwrite_stream *Out = FDOS.get();
std::unique_ptr<buffer_ostream> BOS;
// FIXME: There is a bit of code duplication with addPassesToEmitFile.
if (Opts.OutputType == AssemblerInvocation::FT_Asm) {
MCInstPrinter *IP = TheTarget->createMCInstPrinter(
llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI);
std::unique_ptr<MCCodeEmitter> CE;
if (Opts.ShowEncoding)
CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx));
MCTargetOptions MCOptions;
std::unique_ptr<MCAsmBackend> MAB(
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
auto FOut = llvm::make_unique<formatted_raw_ostream>(*Out);
Str.reset(TheTarget->createAsmStreamer(
Ctx, std::move(FOut), /*asmverbose*/ true,
/*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB),
Opts.ShowInst));
} else if (Opts.OutputType == AssemblerInvocation::FT_Null) {
Str.reset(createNullStreamer(Ctx));
} else {
assert(Opts.OutputType == AssemblerInvocation::FT_Obj &&
"Invalid file type!");
if (!FDOS->supportsSeeking()) {
BOS = make_unique<buffer_ostream>(*FDOS);
Out = BOS.get();
}
std::unique_ptr<MCCodeEmitter> CE(
TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx));
MCTargetOptions MCOptions;
std::unique_ptr<MCAsmBackend> MAB(
TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
Triple T(Opts.Triple);
Str.reset(TheTarget->createMCObjectStreamer(
T, Ctx, std::move(MAB), *Out, std::move(CE), *STI, Opts.RelaxAll,
Opts.IncrementalLinkerCompatible,
/*DWARFMustBeAtTheEnd*/ true));
Str.get()->InitSections(Opts.NoExecStack);
}
// Assembly to object compilation should leverage assembly info.
Str->setUseAssemblerInfoForParsing(true);
bool Failed = false;
std::unique_ptr<MCAsmParser> Parser(
createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI));
// FIXME: init MCTargetOptions from sanitizer flags here.
MCTargetOptions Options;
std::unique_ptr<MCTargetAsmParser> TAP(
TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options));
if (!TAP)
Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple;
// Set values for symbols, if any.
for (auto &S : Opts.SymbolDefs) {
auto Pair = StringRef(S).split('=');
auto Sym = Pair.first;
auto Val = Pair.second;
int64_t Value;
// We have already error checked this in the driver.
Val.getAsInteger(0, Value);
Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value);
}
if (!Failed) {
Parser->setTargetParser(*TAP.get());
Failed = Parser->Run(Opts.NoInitialTextSection);
}
// Close Streamer first.
// It might have a reference to the output stream.
Str.reset();
// Close the output stream early.
BOS.reset();
FDOS.reset();
// Delete output file if there were errors.
if (Failed && Opts.OutputPath != "-")
sys::fs::remove(Opts.OutputPath);
return Failed;
}
static void LLVMErrorHandler(void *UserData, const std::string &Message,
bool /*GenCrashDiag*/) {
DiagnosticsEngine &Diags = *static_cast<DiagnosticsEngine*>(UserData);
Diags.Report(diag::err_fe_error_backend) << Message;
// We cannot recover from llvm errors.
exit(1);
}
int cc1as_main(ArrayRef<const char *> Argv, const char */*Argv0*/, void */*MainAddr*/) {
// Initialize targets and assembly printers/parsers.
InitializeAllTargetInfos();
InitializeAllTargetMCs();
InitializeAllAsmParsers();
// Construct our diagnostic client.
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
TextDiagnosticPrinter *DiagClient
= new TextDiagnosticPrinter(errs(), &*DiagOpts);
DiagClient->setPrefix("clang -cc1as");
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
// Set an error handler, so that any LLVM backend diagnostics go through our
// error handler.
ScopedFatalErrorHandler FatalErrorHandler
(LLVMErrorHandler, static_cast<void*>(&Diags));
// Parse the arguments.
AssemblerInvocation Asm;
if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags))
return 1;
if (Asm.ShowHelp) {
std::unique_ptr<OptTable> Opts(driver::createDriverOptTable());
Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler",
/*Include=*/driver::options::CC1AsOption, /*Exclude=*/0,
/*ShowAllAliases=*/false);
return 0;
}
// Honor -version.
//
// FIXME: Use a better -version message?
if (Asm.ShowVersion) {
llvm::cl::PrintVersionMessage();
return 0;
}
// Honor -mllvm.
//
// FIXME: Remove this, one day.
if (!Asm.LLVMArgs.empty()) {
unsigned NumArgs = Asm.LLVMArgs.size();
auto Args = llvm::make_unique<const char*[]>(NumArgs + 2);
Args[0] = "clang (LLVM option parsing)";
for (unsigned i = 0; i != NumArgs; ++i)
Args[i + 1] = Asm.LLVMArgs[i].c_str();
Args[NumArgs + 1] = nullptr;
llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get());
}
// Execute the invocation, unless there were parsing errors.
bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags);
// If any timers were active but haven't been destroyed yet, print their
// results now.
TimerGroup::printAll(errs());
return !!Failed;
}

512
dbms/src/Server/Compiler-7.0.0/driver.cpp Normal file
View File

@ -0,0 +1,512 @@
//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This is the entry point to the clang driver; it is a thin wrapper
// for functionality in the Driver clang library.
//
//===----------------------------------------------------------------------===//
#include "clang/Driver/Driver.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Driver/Compilation.h"
#include "clang/Driver/DriverDiagnostic.h"
#include "clang/Driver/Options.h"
#include "clang/Driver/ToolChain.h"
#include "clang/Frontend/ChainedDiagnosticConsumer.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Frontend/SerializedDiagnosticPrinter.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Option/Option.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
#include "llvm/Support/Program.h"
#include "llvm/Support/Regex.h"
#include "llvm/Support/Signals.h"
#include "llvm/Support/StringSaver.h"
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/Timer.h"
#include "llvm/Support/raw_ostream.h"
#include <memory>
#include <set>
#include <system_error>
using namespace clang;
using namespace clang::driver;
using namespace llvm::opt;
std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) {
if (!CanonicalPrefixes) {
SmallString<128> ExecutablePath(Argv0);
// Do a PATH lookup if Argv0 isn't a valid path.
if (!llvm::sys::fs::exists(ExecutablePath))
if (llvm::ErrorOr<std::string> P =
llvm::sys::findProgramByName(ExecutablePath))
ExecutablePath = *P;
return ExecutablePath.str();
}
// This just needs to be some symbol in the binary; C++ doesn't
// allow taking the address of ::main however.
void *P = (void*) (intptr_t) GetExecutablePath;
return llvm::sys::fs::getMainExecutable(Argv0, P);
}
static const char *GetStableCStr(std::set<std::string> &SavedStrings,
StringRef S) {
return SavedStrings.insert(S).first->c_str();
}
/// ApplyQAOverride - Apply a list of edits to the input argument lists.
///
/// The input string is a space separate list of edits to perform,
/// they are applied in order to the input argument lists. Edits
/// should be one of the following forms:
///
/// '#': Silence information about the changes to the command line arguments.
///
/// '^': Add FOO as a new argument at the beginning of the command line.
///
/// '+': Add FOO as a new argument at the end of the command line.
///
/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command
/// line.
///
/// 'xOPTION': Removes all instances of the literal argument OPTION.
///
/// 'XOPTION': Removes all instances of the literal argument OPTION,
/// and the following argument.
///
/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox'
/// at the end of the command line.
///
/// \param OS - The stream to write edit information to.
/// \param Args - The vector of command line arguments.
/// \param Edit - The override command to perform.
/// \param SavedStrings - Set to use for storing string representations.
static void ApplyOneQAOverride(raw_ostream &OS,
SmallVectorImpl<const char*> &Args,
StringRef Edit,
std::set<std::string> &SavedStrings) {
// This does not need to be efficient.
if (Edit[0] == '^') {
const char *Str =
GetStableCStr(SavedStrings, Edit.substr(1));
OS << "### Adding argument " << Str << " at beginning\n";
Args.insert(Args.begin() + 1, Str);
} else if (Edit[0] == '+') {
const char *Str =
GetStableCStr(SavedStrings, Edit.substr(1));
OS << "### Adding argument " << Str << " at end\n";
Args.push_back(Str);
} else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) {
StringRef MatchPattern = Edit.substr(2).split('/').first;
StringRef ReplPattern = Edit.substr(2).split('/').second;
ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1);
for (unsigned i = 1, e = Args.size(); i != e; ++i) {
// Ignore end-of-line response file markers
if (Args[i] == nullptr)
continue;
std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]);
if (Repl != Args[i]) {
OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n";
Args[i] = GetStableCStr(SavedStrings, Repl);
}
}
} else if (Edit[0] == 'x' || Edit[0] == 'X') {
auto Option = Edit.substr(1);
for (unsigned i = 1; i < Args.size();) {
if (Option == Args[i]) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
if (Edit[0] == 'X') {
if (i < Args.size()) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
} else
OS << "### Invalid X edit, end of command line!\n";
}
} else
++i;
}
} else if (Edit[0] == 'O') {
for (unsigned i = 1; i < Args.size();) {
const char *A = Args[i];
// Ignore end-of-line response file markers
if (A == nullptr)
continue;
if (A[0] == '-' && A[1] == 'O' &&
(A[2] == '\0' ||
(A[3] == '\0' && (A[2] == 's' || A[2] == 'z' ||
('0' <= A[2] && A[2] <= '9'))))) {
OS << "### Deleting argument " << Args[i] << '\n';
Args.erase(Args.begin() + i);
} else
++i;
}
OS << "### Adding argument " << Edit << " at end\n";
Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str()));
} else {
OS << "### Unrecognized edit: " << Edit << "\n";
}
}
/// ApplyQAOverride - Apply a comma separate list of edits to the
/// input argument lists. See ApplyOneQAOverride.
static void ApplyQAOverride(SmallVectorImpl<const char*> &Args,
const char *OverrideStr,
std::set<std::string> &SavedStrings) {
raw_ostream *OS = &llvm::errs();
if (OverrideStr[0] == '#') {
++OverrideStr;
OS = &llvm::nulls();
}
*OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n";
// This does not need to be efficient.
const char *S = OverrideStr;
while (*S) {
const char *End = ::strchr(S, ' ');
if (!End)
End = S + strlen(S);
if (End != S)
ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings);
S = End;
if (*S != '\0')
++S;
}
}
extern int cc1_main(ArrayRef<const char *> Argv, const char *Argv0,
void *MainAddr);
extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
void *MainAddr);
extern int cc1gen_reproducer_main(ArrayRef<const char *> Argv,
const char *Argv0, void *MainAddr);
static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
SmallVectorImpl<const char *> &ArgVector,
std::set<std::string> &SavedStrings) {
// Put target and mode arguments at the start of argument list so that
// arguments specified in command line could override them. Avoid putting
// them at index 0, as an option like '-cc1' must remain the first.
int InsertionPoint = 0;
if (ArgVector.size() > 0)
++InsertionPoint;
if (NameParts.DriverMode) {
// Add the mode flag to the arguments.
ArgVector.insert(ArgVector.begin() + InsertionPoint,
GetStableCStr(SavedStrings, NameParts.DriverMode));
}
if (NameParts.TargetIsValid) {
const char *arr[] = {"-target", GetStableCStr(SavedStrings,
NameParts.TargetPrefix)};
ArgVector.insert(ArgVector.begin() + InsertionPoint,
std::begin(arr), std::end(arr));
}
}
static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver,
SmallVectorImpl<const char *> &Opts) {
llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts);
// The first instance of '#' should be replaced with '=' in each option.
for (const char *Opt : Opts)
if (char *NumberSignPtr = const_cast<char *>(::strchr(Opt, '#')))
*NumberSignPtr = '=';
}
static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) {
// Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE.
TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS");
if (TheDriver.CCPrintOptions)
TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE");
// Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE.
TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS");
if (TheDriver.CCPrintHeaders)
TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE");
// Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE.
TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS");
if (TheDriver.CCLogDiagnostics)
TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE");
}
static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient,
const std::string &Path) {
// If the clang binary happens to be named cl.exe for compatibility reasons,
// use clang-cl.exe as the prefix to avoid confusion between clang and MSVC.
StringRef ExeBasename(llvm::sys::path::filename(Path));
if (ExeBasename.equals_lower("cl.exe"))
ExeBasename = "clang-cl.exe";
DiagClient->setPrefix(ExeBasename);
}
// This lets us create the DiagnosticsEngine with a properly-filled-out
// DiagnosticOptions instance.
static DiagnosticOptions *
CreateAndPopulateDiagOpts(ArrayRef<const char *> argv) {
auto *DiagOpts = new DiagnosticOptions;
std::unique_ptr<OptTable> Opts(createDriverOptTable());
unsigned MissingArgIndex, MissingArgCount;
InputArgList Args =
Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount);
// We ignore MissingArgCount and the return value of ParseDiagnosticArgs.
// Any errors that would be diagnosed here will also be diagnosed later,
// when the DiagnosticsEngine actually exists.
(void)ParseDiagnosticArgs(*DiagOpts, Args);
return DiagOpts;
}
static void SetInstallDir(SmallVectorImpl<const char *> &argv,
Driver &TheDriver, bool CanonicalPrefixes) {
// Attempt to find the original path used to invoke the driver, to determine
// the installed path. We do this manually, because we want to support that
// path being a symlink.
SmallString<128> InstalledPath(argv[0]);
// Do a PATH lookup, if there are no directory components.
if (llvm::sys::path::filename(InstalledPath) == InstalledPath)
if (llvm::ErrorOr<std::string> Tmp = llvm::sys::findProgramByName(
llvm::sys::path::filename(InstalledPath.str())))
InstalledPath = *Tmp;
// FIXME: We don't actually canonicalize this, we just make it absolute.
if (CanonicalPrefixes)
llvm::sys::fs::make_absolute(InstalledPath);
StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath));
if (llvm::sys::fs::exists(InstalledPathParent))
TheDriver.setInstalledDir(InstalledPathParent);
}
static int ExecuteCC1Tool(ArrayRef<const char *> argv, StringRef Tool) {
void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath;
if (Tool == "")
return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP);
if (Tool == "as")
return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP);
// Reject unknown tools.
llvm::errs() << "error: unknown integrated tool '" << Tool << "'. "
<< "Valid tools include '-cc1' and '-cc1as'.\n";
return 1;
}
int mainEntryClickHouseClang(int argc_, char **argv_) {
llvm::InitLLVM X(argc_, argv_);
SmallVector<const char *, 256> argv(argv_, argv_ + argc_);
if (llvm::sys::Process::FixupStandardFileDescriptors())
return 1;
llvm::InitializeAllTargets();
auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]);
llvm::BumpPtrAllocator A;
llvm::StringSaver Saver(A);
// Parse response files using the GNU syntax, unless we're in CL mode. There
// are two ways to put clang in CL compatibility mode: argv[0] is either
// clang-cl or cl, or --driver-mode=cl is on the command line. The normal
// command line parsing can't happen until after response file parsing, so we
// have to manually search for a --driver-mode=cl argument the hard way.
// Finally, our -cc1 tools don't care which tokenization mode we use because
// response files written by clang will tokenize the same way in either mode.
bool ClangCLMode = false;
if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") ||
std::find_if(argv.begin(), argv.end(), [](const char *F) {
return F && strcmp(F, "--driver-mode=cl") == 0;
}) != argv.end()) {
ClangCLMode = true;
}
enum { Default, POSIX, Windows } RSPQuoting = Default;
for (const char *F : argv) {
if (strcmp(F, "--rsp-quoting=posix") == 0)
RSPQuoting = POSIX;
else if (strcmp(F, "--rsp-quoting=windows") == 0)
RSPQuoting = Windows;
}
// Determines whether we want nullptr markers in argv to indicate response
// files end-of-lines. We only use this for the /LINK driver argument with
// clang-cl.exe on Windows.
bool MarkEOLs = ClangCLMode;
llvm::cl::TokenizerCallback Tokenizer;
if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode))
Tokenizer = &llvm::cl::TokenizeWindowsCommandLine;
else
Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1"))
MarkEOLs = false;
llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs);
// Handle -cc1 integrated tools, even if -cc1 was expanded from a response
// file.
auto FirstArg = std::find_if(argv.begin() + 1, argv.end(),
[](const char *A) { return A != nullptr; });
if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) {
// If -cc1 came from a response file, remove the EOL sentinels.
if (MarkEOLs) {
auto newEnd = std::remove(argv.begin(), argv.end(), nullptr);
argv.resize(newEnd - argv.begin());
}
return ExecuteCC1Tool(argv, argv[1] + 4);
}
bool CanonicalPrefixes = true;
for (int i = 1, size = argv.size(); i < size; ++i) {
// Skip end-of-line response file markers
if (argv[i] == nullptr)
continue;
if (StringRef(argv[i]) == "-no-canonical-prefixes") {
CanonicalPrefixes = false;
break;
}
}
// Handle CL and _CL_ which permits additional command line options to be
// prepended or appended.
if (ClangCLMode) {
// Arguments in "CL" are prepended.
llvm::Optional<std::string> OptCL = llvm::sys::Process::GetEnv("CL");
if (OptCL.hasValue()) {
SmallVector<const char *, 8> PrependedOpts;
getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts);
// Insert right after the program name to prepend to the argument list.
argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end());
}
// Arguments in "_CL_" are appended.
llvm::Optional<std::string> Opt_CL_ = llvm::sys::Process::GetEnv("_CL_");
if (Opt_CL_.hasValue()) {
SmallVector<const char *, 8> AppendedOpts;
getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts);
// Insert at the end of the argument list to append.
argv.append(AppendedOpts.begin(), AppendedOpts.end());
}
}
std::set<std::string> SavedStrings;
// Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the
// scenes.
if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) {
// FIXME: Driver shouldn't take extra initial argument.
ApplyQAOverride(argv, OverrideStr, SavedStrings);
}
std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes);
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts =
CreateAndPopulateDiagOpts(argv);
TextDiagnosticPrinter *DiagClient
= new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
FixupDiagPrefixExeName(DiagClient, Path);
IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient);
if (!DiagOpts->DiagnosticSerializationFile.empty()) {
auto SerializedConsumer =
clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile,
&*DiagOpts, /*MergeChildRecords=*/true);
Diags.setClient(new ChainedDiagnosticConsumer(
Diags.takeClient(), std::move(SerializedConsumer)));
}
ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false);
Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags);
SetInstallDir(argv, TheDriver, CanonicalPrefixes);
TheDriver.setTargetAndMode(TargetAndMode);
insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings);
SetBackdoorDriverOutputsFromEnvVars(TheDriver);
std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(argv));
int Res = 1;
if (C && !C->containsError()) {
SmallVector<std::pair<int, const Command *>, 4> FailingCommands;
Res = TheDriver.ExecuteCompilation(*C, FailingCommands);
// Force a crash to test the diagnostics.
if (TheDriver.GenReproducer) {
Diags.Report(diag::err_drv_force_crash)
<< !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH");
// Pretend that every command failed.
FailingCommands.clear();
for (const auto &J : C->getJobs())
if (const Command *C = dyn_cast<Command>(&J))
FailingCommands.push_back(std::make_pair(-1, C));
}
for (const auto &P : FailingCommands) {
int CommandRes = P.first;
const Command *FailingCommand = P.second;
if (!Res)
Res = CommandRes;
// If result status is < 0, then the driver command signalled an error.
// If result status is 70, then the driver command reported a fatal error.
// On Windows, abort will return an exit code of 3. In these cases,
// generate additional diagnostic information if possible.
bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70;
#ifdef _WIN32
DiagnoseCrash |= CommandRes == 3;
#endif
if (DiagnoseCrash) {
TheDriver.generateCompilationDiagnostics(*C, *FailingCommand);
break;
}
}
}
Diags.getClient()->finish();
// If any timers were active but haven't been destroyed yet, print their
// results now. This happens in -disable-free mode.
llvm::TimerGroup::printAll(llvm::errs());
#ifdef _WIN32
// Exit status should not be negative on Win32, unless abnormal termination.
// Once abnormal termiation was caught, negative status should not be
// propagated.
if (Res < 0)
Res = 1;
#endif
// If we have multiple failing commands, we return the result of the first
// failing command.
return Res;
}

10
dbms/src/Server/Compiler-7.0.0/lld.cpp Normal file
View File

@ -0,0 +1,10 @@
#include "lld/Common/Driver.h"
#include "llvm/Support/InitLLVM.h"
#include <vector>
int mainEntryClickHouseLLD(int argc, char ** argv)
{
llvm::InitLLVM X(argc, argv);
std::vector<const char *> args(argv, argv + argc);
return !lld::elf::link(args, false);
}

5
dbms/src/Server/Server.cpp
View File

@ -64,7 +64,7 @@ static std::string getCanonicalPath(std::string && path)
throw Exception("path configuration parameter is empty");
if (path.back() != '/')
path += '/';
return path;
return std::move(path);
}
void Server::uninitialize()
@ -259,7 +259,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
/* already_loaded = */ false);
/// Reload config in SYSTEM RELOAD CONFIG query.
global_context->setConfigReloadCallback([&]() {
global_context->setConfigReloadCallback([&]()
{
main_config_reloader->reload();
users_config_reloader->reload();
});

14
dbms/src/Storages/AlterCommands.cpp
View File

@ -26,11 +26,7 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const
if (type == ADD_COLUMN)
{
if (columns_description.getAll().contains(column_name))
{
throw Exception{
"Cannot add column " + column_name + ": column with this name already exists",
ErrorCodes::ILLEGAL_COLUMN};
}
throw Exception{"Cannot add column " + column_name + ": column with this name already exists", ErrorCodes::ILLEGAL_COLUMN};
const auto add_column = [this] (NamesAndTypesList & columns)
{
@ -196,17 +192,13 @@ void AlterCommands::validate(IStorage * table, const Context & context)
if (command.type == AlterCommand::ADD_COLUMN)
{
if (std::end(all_columns) != column_it)
throw Exception{
"Cannot add column " + column_name + ": column with this name already exists",
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Cannot add column " + column_name + ": column with this name already exists", ErrorCodes::ILLEGAL_COLUMN};
}
else if (command.type == AlterCommand::MODIFY_COLUMN)
{
if (std::end(all_columns) == column_it)
throw Exception{
"Wrong column name. Cannot find column " + column_name + " to modify",
ErrorCodes::ILLEGAL_COLUMN};
throw Exception{"Wrong column name. Cannot find column " + column_name + " to modify", ErrorCodes::ILLEGAL_COLUMN};
all_columns.erase(column_it);
defaults.erase(column_name);

11
dbms/src/Storages/Distributed/DirectoryMonitor.cpp
View File

@ -63,16 +63,13 @@ namespace
const char * user_pw_end = strchr(address.data(), '@');
const char * colon = strchr(address.data(), ':');
if (!user_pw_end || !colon)
throw Exception{
"Shard address '" + address + "' does not match to 'user[:password]@host:port#default_database' pattern",
throw Exception{"Shard address '" + address + "' does not match to 'user[:password]@host:port#default_database' pattern",
ErrorCodes::INCORRECT_FILE_NAME};
const bool has_pw = colon < user_pw_end;
const char * host_end = has_pw ? strchr(user_pw_end + 1, ':') : colon;
if (!host_end)
throw Exception{
"Shard address '" + address + "' does not contain port",
ErrorCodes::INCORRECT_FILE_NAME};
throw Exception{"Shard address '" + address + "' does not contain port", ErrorCodes::INCORRECT_FILE_NAME};
const char * has_db = strchr(address.data(), '#');
const char * port_end = has_db ? has_db : address_end;
@ -159,7 +156,7 @@ void StorageDistributedDirectoryMonitor::run()
ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::string & name, const StorageDistributed & storage)
{
auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.context.getSettingsRef());
const auto pool_factory = [&storage, &name, &timeouts] (const std::string & host, const UInt16 port,
const auto pool_factory = [&storage, &timeouts] (const std::string & host, const UInt16 port,
const Protocol::Secure secure,
const std::string & user, const std::string & password,
const std::string & default_database)
@ -167,7 +164,7 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri
return std::make_shared<ConnectionPool>(
1, host, port, default_database,
user, password, timeouts,
storage.getName() + '_' + name,
storage.getName() + '_' + user,
Protocol::Compression::Enable,
secure);
};

5
dbms/src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -774,10 +774,7 @@ void MergeTreeData::clearOldPartsFromFilesystem()
void MergeTreeData::setPath(const String & new_full_path)
{
if (Poco::File{new_full_path}.exists())
throw Exception{
"Target path already exists: " + new_full_path,
/// @todo existing target can also be a file, not directory
ErrorCodes::DIRECTORY_ALREADY_EXISTS};
throw Exception{"Target path already exists: " + new_full_path, ErrorCodes::DIRECTORY_ALREADY_EXISTS};
Poco::File(full_path).renameTo(new_full_path);

2
dbms/src/Storages/MergeTree/MergeTreeDataWriter.h
View File

@ -30,7 +30,7 @@ struct BlockWithPartition
using BlocksWithPartition = std::vector<BlockWithPartition>;
/** Writes new parts of data to the merge tree.
/** Writes new parts of data to the merge tree.
*/
class MergeTreeDataWriter
{

2
dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
View File

@ -260,7 +260,7 @@ std::string MergedBlockOutputStream::getPartPath() const
return part_path;
}
/// If data is pre-sorted.
/// If data is pre-sorted.
void MergedBlockOutputStream::write(const Block & block)
{
writeImpl(block, nullptr);

2
dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp
View File

@ -233,7 +233,7 @@ Date column must exist in the table and have type Date (not DateTime).
It is used for internal data partitioning and works like some kind of index.
If your source data doesn't have a column of type Date, but has a DateTime column, you may add values for Date column while loading,
or you may INSERT your source data to a table of type Log and then transform it with INSERT INTO t SELECT toDate(time) AS date, * FROM ...
or you may INSERT your source data to a table of type Log and then transform it with INSERT INTO t SELECT toDate(time) AS date, * FROM ...
If your source data doesn't have any date or time, you may just pass any constant for a date column while loading.
Next parameter is optional sampling expression. Sampling expression is used to implement SAMPLE clause in query for approximate query execution.

4
dbms/src/Storages/StorageJoin.cpp
View File

@ -30,9 +30,7 @@ StorageJoin::StorageJoin(
{
for (const auto & key : key_names)
if (!getColumns().hasPhysical(key))
throw Exception{
"Key column (" + key + ") does not exist in table declaration.",
ErrorCodes::NO_SUCH_COLUMN_IN_TABLE};
throw Exception{"Key column (" + key + ") does not exist in table declaration.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE};
/// NOTE StorageJoin doesn't use join_use_nulls setting.

6
dbms/src/Storages/tests/merge_selector2.cpp
View File

@ -10,9 +10,9 @@
/** This program tests merge-selecting algorithm.
* Usage:
clickhouse-client --query="
SELECT bytes, now() - modification_time, level, name
FROM system.parts
WHERE table = 'visits' AND active AND partition = '201610'" | ./merge_selector2
SELECT bytes, now() - modification_time, level, name
FROM system.parts
WHERE table = 'visits' AND active AND partition = '201610'" | ./merge_selector2
*/
int main(int, char **)

48
debian/.pbuilderrc vendored
View File

@ -55,13 +55,6 @@ DEBIAN_SUITES=($UNSTABLE_CODENAME $TESTING_CODENAME $STABLE_CODENAME $STABLE_BAC
# List of Ubuntu suites. Update these when needed.
UBUNTU_SUITES=("bionic" "artful" "zesty" "xenial" "trusty" "devel")
# Mirrors to use. Update these to your preferred mirror.
DEBIAN_MIRROR=${DEBIAN_MIRROR:=deb.debian.org}
UBUNTU_MIRROR=${UBUNTU_MIRROR:=archive.ubuntu.com}
#DEBIAN_MIRROR="mirror.yandex.ru"
#UBUNTU_MIRROR="mirror.yandex.ru"
# Set a default distribution if none is used. Note that you can set your own default (i.e. ${DIST:="unstable"}).
HOST_DIST=`lsb_release --short --codename`
: ${DIST:="$HOST_DIST"}
@ -104,7 +97,7 @@ ALLOWUNTRUSTED=${SET_ALLOWUNTRUSTED:=${ALLOWUNTRUSTED}}
if $(echo ${DEBIAN_SUITES[@]} | grep -q $DIST); then
# Debian configuration
OSNAME=debian
MIRRORSITE="http://$DEBIAN_MIRROR/$OSNAME/"
MIRRORSITE=${SET_MIRRORSITE="http://deb.debian.org/$OSNAME/"}
COMPONENTS="main contrib non-free"
if $(echo "$STABLE_CODENAME stable" | grep -q $DIST); then
OTHERMIRROR="$OTHERMIRROR | deb $MIRRORSITE $STABLE_BACKPORTS_SUITE $COMPONENTS"
@ -119,11 +112,16 @@ if $(echo ${DEBIAN_SUITES[@]} | grep -q $DIST); then
DEBOOTSTRAPOPTS+=( '--keyring' '/usr/share/keyrings/debian-archive-keyring.gpg' )
# DEBOOTSTRAPOPTS+=( '--keyring' '/usr/share/keyrings/debian-keyring.gpg' )
esac
elif $(echo ${UBUNTU_SUITES[@]} | grep -q $DIST); then
# Ubuntu configuration
OSNAME=ubuntu
MIRRORSITE="http://$UBUNTU_MIRROR/$OSNAME/"
if [[ "$ARCH" == "amd64" || "$ARCH" == "i386" ]]; then
MIRRORSITE=${SET_MIRRORSITE="http://archive.ubuntu.com/ubuntu/"}
else
MIRRORSITE=${SET_MIRRORSITE="http://ports.ubuntu.com/ubuntu-ports/"}
fi
COMPONENTS="main restricted universe multiverse"
OTHERMIRROR="$OTHERMIRROR | deb $MIRRORSITE $DIST-updates main restricted universe multiverse"
@ -142,7 +140,7 @@ else
exit 1
fi
echo "using $NAME $OSNAME $DIST $ARCH $LOGNAME"
echo "using $NAME $OSNAME $DIST $ARCH $LOGNAME $MIRRORSITE"
case "$DIST" in
"trusty")
@ -161,28 +159,26 @@ case "$DIST" in
;;
esac
case "$DIST" in
"bionic" )
if [ "$ARCH" != arm64 ]; then
case "$DIST" in
"bionic" | "experimental" | "unstable" | "testing")
EXTRAPACKAGES+=" liblld-6.0-dev libclang-6.0-dev liblld-6.0 "
export CMAKE_FLAGS="-DENABLE_EMBEDDED_COMPILER=1 -DLLVM_VERSION_POSTFIX=-6.0 $CMAKE_FLAGS"
export CMAKE_FLAGS="-DLLVM_VERSION_POSTFIX=-6.0 $CMAKE_FLAGS"
;;
"artful" | "experimental" | "unstable" | "testing" )
if [ "$ARCH" != arm64 ]; then
EXTRAPACKAGES+=" liblld-5.0-dev libclang-5.0-dev liblld-5.0 "
export CMAKE_FLAGS="-DENABLE_EMBEDDED_COMPILER=1 $CMAKE_FLAGS"
fi
"artful" )
EXTRAPACKAGES+=" liblld-5.0-dev libclang-5.0-dev liblld-5.0 "
;;
esac
esac
else
export CMAKE_FLAGS="-DENABLE_EMBEDDED_COMPILER=0 $CMAKE_FLAGS"
fi
# bundled zookeepeer have broken asm
case "$ARCH" in
"arm64")
EXTRAPACKAGES+=" libzookeeper-mt-dev "
;;
esac
[[ "$ARCH" == "arm64" ]] && EXTRAPACKAGES+=" libzookeeper-mt-dev "
# will test symbols
EXTRAPACKAGES+=" gdb "
#EXTRAPACKAGES+=" gdb "
[[ $CCACHE_PREFIX == 'distcc' ]] && EXTRAPACKAGES+=" $CCACHE_PREFIX "

1
debian/control vendored
View File

@ -4,7 +4,6 @@ Priority: optional
Maintainer: Alexey Milovidov <milovidov@yandex-team.ru>
Build-Depends: debhelper (>= 9),
cmake3 | cmake,
ninja-build [amd64 i386],
gcc-7, g++-7,
libc6-dev,
libmariadbclient-dev | default-libmysqlclient-dev | libmysqlclient-dev,

8
debian/rules vendored
View File

@ -43,10 +43,12 @@ endif
CMAKE_FLAGS += -DCMAKE_CXX_COMPILER=`which $(CXX)` -DCMAKE_C_COMPILER=`which $(CC)`
NINJA=$(shell which ninja)
ifndef DISABLE_NINJA
NINJA=$(shell which ninja)
ifneq ($(NINJA),)
CMAKE_FLAGS += -GNinja
export MAKE=$(NINJA)
CMAKE_FLAGS += -GNinja
export MAKE=$(NINJA)
endif
endif
ifndef DH_VERBOSE

2
docs/ru/development/build_osx.md
View File

@ -12,7 +12,7 @@ With appropriate changes, build should work on any other OS X distribution.
## Install required compilers, tools, libraries
```bash
brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext homebrew/dupes/zlib readline boost --cc=gcc-7
brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext zlib readline boost --cc=gcc-7
```
## Checkout ClickHouse sources

5
libs/libcommon/include/common/iostream_debug_helpers.h
View File

@ -145,11 +145,12 @@ Out & dump(Out & out, const char * name, T && x)
#define DUMP3(V1, V2, V3) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPVAR(V3) DUMPTAIL } while(0);
#define DUMP4(V1, V2, V3, V4) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPVAR(V3) DUMPVAR(V4) DUMPTAIL } while(0);
#define DUMP5(V1, V2, V3, V4, V5) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPVAR(V3) DUMPVAR(V4) DUMPVAR(V5) DUMPTAIL } while(0);
#define DUMP6(V1, V2, V3, V4, V5, V6) do { DUMPHEAD DUMPVAR(V1) DUMPVAR(V2) DUMPVAR(V3) DUMPVAR(V4) DUMPVAR(V5) DUMPVAR(V6) DUMPTAIL } while(0);
/// https://groups.google.com/forum/#!searchin/kona-dev/variadic$20macro%7Csort:date/kona-dev/XMA-lDOqtlI/GCzdfZsD41sJ
#define VA_NUM_ARGS_IMPL(x1, x2, x3, x4, x5, N, ...) N
#define VA_NUM_ARGS(...) VA_NUM_ARGS_IMPL(__VA_ARGS__, 5, 4, 3, 2, 1)
#define VA_NUM_ARGS_IMPL(x1, x2, x3, x4, x5, x6, N, ...) N
#define VA_NUM_ARGS(...) VA_NUM_ARGS_IMPL(__VA_ARGS__, 6, 5, 4, 3, 2, 1)
#define MAKE_VAR_MACRO_IMPL_CONCAT(PREFIX, NUM_ARGS) PREFIX ## NUM_ARGS
#define MAKE_VAR_MACRO_IMPL(PREFIX, NUM_ARGS) MAKE_VAR_MACRO_IMPL_CONCAT(PREFIX, NUM_ARGS)

12
release
View File

@ -1,5 +1,13 @@
#!/bin/bash
# Advanced usage:
# Test gcc-8:
# env DIST=bionic EXTRAPACKAGES="gcc-8 g++-8" DEB_CC=gcc-8 DEB_CXX=g++-8 CMAKE_FLAGS=" -DNO_WERROR=1 " ./release
# Clang6 build:
# env DIST=bionic EXTRAPACKAGES="clang-6.0 libstdc++-8-dev lld-6.0 liblld-6.0-dev libclang-6.0-dev liblld-6.0" DEB_CC=clang-6.0 DEB_CXX=clang++-6.0 CMAKE_FLAGS=" -DLLVM_VERSION_POSTFIX=-6.0 -DNO_WERROR=1 " ./release
# Clang6 without internal compiler (for low memory arm64):
# env DIST=bionic DISABLE_PARALLEL=1 EXTRAPACKAGES="clang-6.0 libstdc++-8-dev" DEB_CC=clang-6.0 DEB_CXX=clang++-6.0 CMAKE_FLAGS=" -DNO_WERROR=1 " ./release
set -e
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
@ -60,8 +68,8 @@ then
# todo: нужно ли отключить libtcmalloc?
LIBTCMALLOC_OPTS="-DENABLE_TCMALLOC=0"
# GLIBC_COMPATIBILITY отключен по умолчанию
export DEB_CC=clang-5.0
export DEB_CXX=clang++-5.0
export DEB_CC=${DEB_CC=clang-5.0}
export DEB_CXX=${DEB_CXX=clang++-5.0}
EXTRAPACKAGES="$EXTRAPACKAGES clang-5.0 lld-5.0"
elif [[ $BUILD_TYPE == 'valgrind' ]]; then
LIBTCMALLOC_OPTS="-DENABLE_TCMALLOC=0"

246
uncrustify.cfg Normal file
View File

@ -0,0 +1,246 @@
# Configuration file for Uncrustify code formatter.
# https://github.com/uncrustify/uncrustify
#
# Created with https://cdanu.github.io/uncrustify_config_preview/index.html
#
# You may apply it for your code with:
# uncrustify -l CPP -c uncrustify.cfg -f filename.cpp
#
# This config is in beta: it doesn't implement our style guide perfectly.
# It's not recommended to apply it for existing code base.
newlines = lf
input_tab_size = 4
output_tab_size = 4
string_replace_tab_chars = true
utf8_bom = remove
utf8_byte = true
utf8_force = true
sp_arith = force
sp_assign = force
sp_cpp_lambda_assign = remove
sp_assign_default = force
sp_enum_assign = force
sp_enum_colon = force
sp_pp_concat = force
sp_pp_stringify = remove
sp_bool = force
sp_compare = force
sp_inside_paren = remove
sp_paren_paren = remove
sp_paren_brace = force
sp_before_ptr_star = force
sp_between_ptr_star = remove
sp_after_ptr_star = force
sp_after_ptr_star_qualifier = force
sp_after_ptr_star_func = force
sp_ptr_star_paren = force
sp_before_ptr_star_func = force
sp_before_byref = force
sp_before_unnamed_byref = force
sp_after_byref = force
sp_after_byref_func = force
sp_before_byref_func = force
sp_template_angle = force
sp_before_angle = remove
sp_inside_angle = remove
sp_angle_colon = force
sp_after_angle = force
sp_angle_paren = remove
sp_angle_paren_empty = remove
sp_angle_word = force
sp_angle_shift = remove
sp_permit_cpp11_shift = true
sp_before_sparen = force
sp_inside_sparen = remove
sp_after_sparen = force
sp_sparen_brace = force
sp_special_semi = force
sp_before_semi_for = remove
sp_before_semi_for_empty = remove
sp_after_semi = force
sp_after_semi_for_empty = remove
sp_before_square = remove
sp_before_squares = remove
sp_inside_square = remove
sp_after_comma = force
sp_before_ellipsis = remove
sp_after_class_colon = force
sp_before_class_colon = force
sp_after_constr_colon = force
sp_before_constr_colon = force
sp_after_operator = remove
sp_after_operator_sym = remove
sp_after_cast = remove
sp_inside_paren_cast = remove
sp_cpp_cast_paren = remove
sp_sizeof_paren = remove
sp_inside_braces_enum = force
sp_inside_braces_struct = force
sp_inside_braces = force
sp_inside_braces_empty = remove
sp_type_func = force
sp_func_proto_paren = remove
sp_func_proto_paren_empty = remove
sp_func_def_paren = remove
sp_func_def_paren_empty = remove
sp_inside_fparens = remove
sp_inside_fparen = remove
sp_inside_tparen = remove
sp_after_tparen_close = remove
sp_square_fparen = remove
sp_fparen_brace = force
sp_func_call_paren = remove
sp_func_class_paren = remove
sp_func_class_paren_empty = remove
sp_return_paren = force
sp_attribute_paren = remove
sp_defined_paren = remove
sp_throw_paren = force
sp_after_throw = force
sp_catch_paren = force
sp_macro = add
sp_macro_func = add
sp_else_brace = force
sp_brace_else = force
sp_brace_typedef = force
sp_catch_brace = force
sp_brace_catch = force
sp_try_brace = force
sp_word_brace = remove
sp_word_brace_ns = force
sp_before_dc = remove
sp_after_dc = remove
sp_cond_colon = force
sp_cond_colon_before = force
sp_cond_colon_after = force
sp_cond_question = force
sp_cond_question_before = force
sp_cond_question_after = force
sp_cond_ternary_short = remove
sp_cmt_cpp_start = force
sp_cmt_cpp_doxygen = true
sp_cmt_cpp_qttr = true
sp_endif_cmt = force
sp_after_new = force
sp_between_new_paren = remove
sp_after_newop_paren = force
sp_inside_newop_paren = remove
sp_before_tr_emb_cmt = force
indent_columns = 4
indent_with_tabs = 0
indent_namespace = true
indent_namespace_limit = 100
indent_class = true
indent_ctor_init_leading = 1
indent_shift = true
indent_func_call_param = true
indent_func_def_param = true
indent_func_proto_param = true
indent_func_class_param = true
indent_func_ctor_var_param = true
indent_template_param = true
indent_member = 4
indent_switch_case = 4
indent_switch_pp = false
indent_label = 0
indent_access_spec = -4
indent_paren_close = 2
indent_paren_after_func_def = true
indent_paren_after_func_decl = true
indent_paren_after_func_call = true
indent_align_assign = false
indent_token_after_brace = false
indent_cpp_lambda_body = true
indent_ternary_operator = 1
nl_assign_leave_one_liners = true
nl_class_leave_one_liners = true
nl_enum_leave_one_liners = true
nl_getset_leave_one_liners = true
nl_func_leave_one_liners = true
nl_cpp_lambda_leave_one_liners = true
nl_if_leave_one_liners = true
nl_start_of_file = remove
nl_end_of_file = force
nl_enum_brace = add
nl_struct_brace = add
nl_union_brace = add
nl_if_brace = add
nl_brace_else = add
nl_else_brace = add
nl_else_if = remove
nl_before_if_closing_paren = remove
nl_try_brace = add
nl_for_brace = add
nl_catch_brace = add
nl_brace_catch = add
nl_while_brace = add
nl_do_brace = add
nl_brace_while = remove
nl_switch_brace = add
nl_multi_line_define = true
nl_before_case = true
nl_after_case = true
nl_case_colon_brace = add
nl_namespace_brace = add
nl_template_class = add
nl_class_brace = add
nl_enum_own_lines = add
nl_func_scope_name = remove
nl_func_paren = remove
nl_func_def_paren = remove
nl_func_call_paren = remove
nl_func_call_paren_empty = remove
nl_func_decl_start_multi_line = true
nl_func_def_start_multi_line = true
nl_func_decl_args_multi_line = true
nl_func_def_args_multi_line = true
nl_func_decl_end = remove
nl_func_def_end = remove
nl_func_decl_empty = remove
nl_func_def_empty = remove
nl_func_call_empty = remove
nl_func_call_start_multi_line = true
nl_func_call_args_multi_line = true
nl_fdef_brace = add
nl_after_semicolon = true
nl_constr_colon = force
nl_split_if_one_liner = true
nl_split_for_one_liner = true
nl_split_while_one_liner = true
nl_max = 3
nl_max_blank_in_func = 2
nl_after_func_proto = 1
nl_after_func_proto_group = 2
nl_after_func_class_proto = 1
nl_after_func_class_proto_group = 2
nl_before_func_body_def = 1
nl_before_func_body_proto = 1
nl_after_func_body = 3
nl_after_func_body_class = 3
nl_after_func_body_one_liner = 1
nl_after_multiline_comment = true
nl_after_struct = 3
nl_before_class = 3
nl_after_class = 3
nl_before_access_spec = 2
nl_after_access_spec = 1
nl_after_try_catch_finally = 1
eat_blanks_after_open_brace = true
eat_blanks_before_close_brace = true
nl_remove_extra_newlines = 1
nl_after_return = true
pos_constr_comma = trail_force
pos_constr_colon = lead_force
code_width = 160
ls_func_split_full = true
ls_code_width = true
align_left_shift = false
cmt_convert_tab_to_spaces = true
mod_full_brace_for = remove
mod_full_brace_if = remove
mod_full_brace_if_chain = true
mod_full_brace_while = remove
mod_paren_on_return = remove
mod_remove_extra_semicolon = true
mod_remove_empty_return = true

3
utils/build/build_debian_unbundled.sh
View File

@ -3,8 +3,7 @@
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)
# also possible: DIST=bionic DIST=testing
DIST=${DIST="unstable"}
export DIST
export DIST=${DIST=unstable}
cd $ROOT_DIR
. $ROOT_DIR/debian/.pbuilderrc

6
utils/build/build_freebsd.sh
View File

@ -36,11 +36,11 @@ cd ClickHouse/build
cmake .. -DUNBUNDLED=1 -DUSE_STATIC_LIBRARIES=0
# build with boost 1.64 from ports temporary broken
make -C dbms/src/Server -j $(nproc || sysctl -n hw.ncpu || echo 2)
make clickhouse-bundle -j $(nproc || sysctl -n hw.ncpu || echo 2)
cd ../..
# Run server:
# ClickHouse/build/dbms/src/Server/clickhouse --server --config-file=ClickHouse/dbms/src/Server/config.xml &
# ClickHouse/build/dbms/src/Server/clickhouse-server --config-file=ClickHouse/dbms/src/Server/config.xml &
# Run client:
# ClickHouse/build/dbms/src/Server/clickhouse --client
# ClickHouse/build/dbms/src/Server/clickhouse-client

45
utils/build/build_macos.sh Executable file
View File

@ -0,0 +1,45 @@
#!/bin/sh
# How to build ClickHouse on Mac OS X
# Build should work on Mac OS X 10.12. If you're using earlier version, you can try to build ClickHouse using Gentoo Prefix and clang sl in this instruction.
# With appropriate changes, build should work on any other OS X distribution.
## Install Homebrew
if [ -z `which brew` ]; then
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
fi
## Install required compilers, tools, libraries
brew install cmake gcc icu4c mysql openssl unixodbc libtool gettext zlib readline boost --cc=gcc-7
## Checkout ClickHouse sources
# To get the latest stable version:
git clone -b stable --recursive --depth=10 https://github.com/yandex/ClickHouse.git
cd ClickHouse
# For development, switch to the `master` branch.
# For the latest release candidate, switch to the `testing` branch.
## Build ClickHouse
mkdir build
cd build
cmake .. -DCMAKE_CXX_COMPILER=`which g++-8 g++-7` -DCMAKE_C_COMPILER=`which gcc-8 gcc-7`
make -j `sysctl -n hw.ncpu`
cd ../..
# Run server:
# ClickHouse/build/dbms/src/Server/clickhouse-server --config-file=ClickHouse/dbms/src/Server/config.xml &
# Run client:
# ClickHouse/build/dbms/src/Server/clickhouse-client
## Caveats
# If you intend to run clickhouse-server, make sure to increase system's maxfiles variable. See [MacOS.md](https://github.com/yandex/ClickHouse/blob/master/MacOS.md) for more details.

20
utils/check-style/check-style Executable file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env bash
# For code formatting we have clang-format.
#
# But it's not sane to apply clang-format for whole code base,
# because it sometimes makes worse for properly formatted files.
#
# It's only reasonable to blindly apply clang-format only in cases
# when the code is likely to be out of style.
#
# For this purpose we have a script that will use very primitive heuristics
# (simple regexps) to check if the code is likely to have basic style violations.
# and then to run formatter only for the specified files.
ROOT_PATH=$(git rev-parse --show-toplevel)
find $ROOT_PATH/dbms -name '*.h' -or -name '*.cpp' |
grep -vP 'Compiler|build' |
xargs grep $@ -P '((class|struct|namespace|enum|if|for|while|else|throw|switch).*|\)(\s*const)?(\s*override)?\s*)\{$|\s$|\t|^ {1,3}[^\* ]\S|\t'
# a curly brace not in a new line, but not for the case of C++11 init or agg. initialization | trailing whitespace | number of ws not a multiple of 4, but not in the case of comment continuation | a tab character

3
utils/check-style/fix-style Executable file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env bash
$(dirname ${BASH_SOURCE[0]})/check-style -l | xargs --no-run-if-empty clang-format -i

7
utils/prepare-environment/install-clang.sh
View File

@ -35,7 +35,14 @@ svn co "http://llvm.org/svn/llvm-project/libcxxabi/${BRANCH}" libcxxabi
cd ../..
mkdir build
cd build/
# NOTE You must build LLVM with the same ABI as ClickHouse.
# For example, if you compile ClickHouse with libc++, you must add
# -D LLVM_ENABLE_LIBCXX=1
# to the line below.
cmake -D CMAKE_BUILD_TYPE:STRING=Release ../llvm
make -j $THREADS
sudo make install
hash clang