thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 08:32:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Arch-based compilation

Browse Source
This commit is contained in:
Alexandr Krasheninnikov 2018-11-02 22:06:05 +03:00
parent 468ca70683
commit 6ab45d081c
10 changed files with 124 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
cmake/test_cpu.cmake
View File

@ -45,6 +45,38 @@ if (HAVE_SSE42)
set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
endif ()
set (TEST_FLAG "-mssse3")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <tmmintrin.h>
int main() {
__m64 a = _mm_abs_pi8(__m64());
(void)a;
return 0;
}
" HAVE_SSSE3)
set (TEST_FLAG "-mavx")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _mm256_insert_epi8(__m256i(), 0, 0);
(void)a;
return 0;
}
" HAVE_AVX)
set (TEST_FLAG "-mavx2")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0")
check_cxx_source_compiles("
#include <immintrin.h>
int main() {
auto a = _mm256_add_epi16(__m256i(), __m256i());
(void)a;
return 0;
}
" HAVE_AVX2)
# gcc -dM -E -mpopcnt - < /dev/null | sort > gcc-dump-popcnt
#define __POPCNT__ 1
@ -65,5 +97,3 @@ if (HAVE_POPCNT AND NOT ARCH_AARCH64)
endif ()
cmake_pop_check_state ()
# TODO: add here sse3 test if you want use it

27
contrib/base64-cmake/CMakeLists.txt
View File

@ -1,13 +1,33 @@
SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64)
set(base64_compile_instructions "")
LIST(LENGTH base64_compile_instructions 0)
macro(cast_to_bool var instruction)
if (HAVE_${var})
set(base64_${var} 1)
set(base64_${var}_opt ${instruction})
else()
set(base64_${var} 0)
endif()
endmacro()
cast_to_bool(SSSE3 "-mssse3")
cast_to_bool(SSE41 "-msse4.1")
cast_to_bool(SSE42 "-msse4.2")
cast_to_bool(AVX "-mavx")
cast_to_bool(AVX2 "-mavx2")
# write config.h file, to include it in application
file(READ config-header.tpl header)
file(WRITE config.h ${header})
file(APPEND config.h "#define HAVE_SSE41 ${HAVE_SSE41}\n")
file(APPEND config.h "#define HAVE_SSE42 ${HAVE_SSE42}\n")
file(APPEND config.h "#define HAVE_SSSE3 ${base64_SSSE3}\n")
file(APPEND config.h "#define HAVE_SSE41 ${base64_SSE41}\n")
file(APPEND config.h "#define HAVE_SSE42 ${base64_SSE42}\n")
file(APPEND config.h "#define HAVE_AVX ${base64_AVX}\n")
file(APPEND config.h "#define HAVE_AVX2 ${base64_AVX2}\n")
set(HAVE_FAST_UNALIGNED_ACCESS 0)
if (${HAVE_SSE41} OR ${HAVE_SSE42})
if (${base64_SSSE3} OR ${base64_SSE41} OR ${base64_SSE42} OR ${base64_AVX} OR ${base64_AVX2})
set(HAVE_FAST_UNALIGNED_ACCESS 1)
endif ()
@ -28,4 +48,5 @@ add_library(base64 ${LINK_MODE}
${LIBRARY_DIR}/lib/codecs.h
config.h)
target_compile_options(base64 PRIVATE ${base64_SSSE3_opt} ${base64_SSE41_opt} ${base64_SSE42_opt} ${base64_AVX_opt} ${base64_AVX2_opt})
target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include .)

3
contrib/base64-cmake/config-header.tpl
View File

@ -1,5 +1,2 @@
#define HAVE_AVX 0
#define HAVE_AVX2 0
#define HAVE_NEON32 0
#define HAVE_NEON64 0
#define HAVE_SSSE3 0

43
dbms/src/Functions/FunctionBase64Conversion.h
View File

@ -25,15 +25,31 @@ namespace ErrorCodes
struct Base64Encode
{
static constexpr auto name = "base64Encode";
static constexpr auto buffer_size_multiplier = 5.0 / 3.0;
static size_t getBufferSize(size_t string_length, size_t string_count)
{
return ( ( string_length - string_count ) / 3 + string_count ) * 4 + string_count ;
}
};
struct Base64Decode
{
static constexpr auto name = "base64Decode";
static constexpr auto buffer_size_multiplier = 3.0 / 4.0;
static size_t getBufferSize(size_t string_length, size_t string_count)
{
return ( ( string_length - string_count) / 4 + string_count) * 3 + string_count;
}
};
struct TryBase64Decode
{
static constexpr auto name = "tryBase64Decode";
static size_t getBufferSize(size_t string_length, size_t string_count)
{
return Base64Decode::getBufferSize(string_length, string_count);
}
};
template <typename Func>
class FunctionBase64Conversion : public IFunction
@ -85,7 +101,7 @@ public:
auto & dst_data = dst_column->getChars();
auto & dst_offsets = dst_column->getOffsets();
size_t reserve = ceil(input->getChars().size() * Func::buffer_size_multiplier + input->size());
size_t reserve = Func::getBufferSize(input->getChars().size(), input->size());
dst_data.resize(reserve);
dst_offsets.resize(input_rows_count);
@ -107,13 +123,26 @@ public:
{
base64_encode(source, srclen, dst_pos, &outlen, codec);
}
else
else if constexpr (std::is_same_v<Func, Base64Decode>)
{
if (!base64_decode(source, srclen, dst_pos, &outlen, codec))
{
throw Exception("Failed to " + getName() + " input '" + String(source, srclen) + "'", ErrorCodes::INCORRECT_DATA);
}
}
else
{
// during decoding character array can be partially polluted
// if fail, revert back and clean
auto savepoint = dst_pos;
if (!base64_decode(source, srclen, dst_pos, &outlen, codec))
{
outlen = 0;
dst_pos = savepoint;
// clean the symbol
dst_pos[0] = 0;
}
}
source += srclen + 1;
dst_pos += outlen + 1;
@ -130,13 +159,7 @@ public:
private:
static int getCodec()
{
#if __SSE4_2__
return BASE64_FORCE_SSE42;
#elif __SSE4_1__
return BASE64_FORCE_SSE41;
#else
return BASE64_FORCE_PLAIN;
#endif
}
};
}

2
dbms/src/Functions/registerFunctionsString.cpp
View File

@ -22,6 +22,7 @@ void registerFunctionEndsWith(FunctionFactory &);
#if USE_BASE64
void registerFunctionBase64Encode(FunctionFactory &);
void registerFunctionBase64Decode(FunctionFactory &);
void registerFunctionTryBase64Decode(FunctionFactory &);
#endif
void registerFunctionsString(FunctionFactory & factory)
@ -45,6 +46,7 @@ void registerFunctionsString(FunctionFactory & factory)
#if USE_BASE64
registerFunctionBase64Encode(factory);
registerFunctionBase64Decode(factory);
registerFunctionTryBase64Decode(factory);
#endif
}

14
dbms/src/Functions/tryBase64Decode.cpp Normal file
View File

@ -0,0 +1,14 @@
#include <Functions/FunctionBase64Conversion.h>
#if USE_BASE64
#include <DataTypes/DataTypeString.h>
#include <Functions/FunctionFactory.h>
namespace DB
{
void registerFunctionTryBase64Decode(FunctionFactory & factory)
{
factory.registerFunction<FunctionBase64Conversion<TryBase64Decode>>();
}
}
#endif

1
dbms/tests/queries/0_stateless/000732_base64_functions.reference
View File

@ -13,3 +13,4 @@ foob
fooba
foobar
1 1

1
dbms/tests/queries/0_stateless/000732_base64_functions.sql
View File

@ -2,4 +2,5 @@ SET send_logs_level = 'none';
SELECT base64Encode(val) FROM (select arrayJoin(['', 'f', 'fo', 'foo', 'foob', 'fooba', 'foobar']) val);
SELECT base64Decode(val) FROM (select arrayJoin(['', 'Zg==', 'Zm8=', 'Zm9v', 'Zm9vYg==', 'Zm9vYmE=', 'Zm9vYmFy']) val);
SELECT base64Decode(base64Encode('foo')) = 'foo', base64Encode(base64Decode('Zm9v')) == 'Zm9v';
SELECT tryBase64Decode('Zm9vYmF=Zm9v');
SELECT base64Decode('Zm9vYmF=Zm9v'); -- { serverError 117 }

8
docs/en/query_language/functions/string_functions.md
View File

@ -74,3 +74,11 @@ If the 's' string is non-empty and does not contain the 'c' character at the end
Returns the string 's' that was converted from the encoding in 'from' to the encoding in 'to'.
## base64Encode(s)
Encodes 's' string into base64
## base64Decode(s)
Decode base64-encoded string 's' into original string. In case of failure raises an exception.
## tryBase64Decode(s)
Similar to base64Decode, but in case of error an empty string would be returned.

9
docs/ru/query_language/functions/string_functions.md
View File

@ -58,3 +58,12 @@
## convertCharset(s, from, to)
Возвращает сконвертированную из кодировки from в кодировку to строку s.
## base64Encode(s)
Производит кодирование строки s в base64-представление.
## base64Decode(s)
Декодирует base64-представление s в исходную строку. При невозможности декодирования выбрасывает исключение
## tryBase64Decode(s)
Функционал аналогичен base64Decode, но при невозможности декодирования возвращает пустую строку.