mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-24 08:32:02 +00:00
Changed one base64 library to another
This commit is contained in:
Alexey Milovidov
parent
17f1754695
commit
063682ef9c
2
.gitmodules
vendored
2
.gitmodules
vendored
@ -49,7 +49,7 @@
|
||||
url = https://github.com/ClickHouse-Extras/boost.git
|
||||
[submodule "contrib/base64"]
|
||||
path = contrib/base64
|
||||
url = https://github.com/aklomp/base64.git
|
||||
url = https://github.com/powturbo/Turbo-Base64.git
|
||||
[submodule "contrib/arrow"]
|
||||
path = contrib/arrow
|
||||
url = https://github.com/apache/arrow
|
||||
|
||||
2
contrib/base64
vendored
2
contrib/base64
vendored
@ -1 +1 @@
|
||||
Subproject commit a27c565d1b6c676beaf297fe503c4518185666f7
|
||||
Subproject commit 23213812749b42529ba73e077e923bf80e96701f
|
||||
1
contrib/base64-cmake/.gitignore
vendored
1
contrib/base64-cmake/.gitignore
vendored
@ -1 +0,0 @@
|
||||
config.h
|
||||
@ -1,58 +1,19 @@
|
||||
SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64)
|
||||
|
||||
macro(cast_to_bool var instruction)
|
||||
if (HAVE_${var})
|
||||
set(base64_${var} 1)
|
||||
set(base64_${var}_opt ${instruction})
|
||||
else()
|
||||
set(base64_${var} 0)
|
||||
endif()
|
||||
endmacro()
|
||||
add_library(base64_scalar OBJECT ${LIBRARY_DIR}/turbob64c.c ${LIBRARY_DIR}/turbob64d.c)
|
||||
add_library(base64_ssse3 OBJECT ${LIBRARY_DIR}/turbob64sse.c)
|
||||
add_library(base64_avx OBJECT ${LIBRARY_DIR}/turbob64sse.c) # This is not a mistake. One file is compiled twice.
|
||||
add_library(base64_avx2 OBJECT ${LIBRARY_DIR}/turbob64avx2.c)
|
||||
|
||||
cast_to_bool(NEON32 "") # TODO flags
|
||||
cast_to_bool(NEON64 "") # TODO flags
|
||||
cast_to_bool(SSSE3 "-mssse3")
|
||||
cast_to_bool(SSE41 "-msse4.1")
|
||||
cast_to_bool(SSE42 "-msse4.2")
|
||||
cast_to_bool(AVX "-mavx")
|
||||
cast_to_bool(AVX2 "-mavx2")
|
||||
|
||||
set(HAVE_FAST_UNALIGNED_ACCESS 0)
|
||||
if(HAVE_SSSE3 OR HAVE_SSE41 OR HAVE_SSE42 OR HAVE_AVX OR HAVE_AVX2)
|
||||
set(HAVE_FAST_UNALIGNED_ACCESS 1)
|
||||
endif ()
|
||||
|
||||
configure_file(config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h)
|
||||
target_compile_options(base64_scalar PRIVATE -falign-loops)
|
||||
target_compile_options(base64_ssse3 PRIVATE -mssse3 -falign-loops)
|
||||
target_compile_options(base64_avx PRIVATE -falign-loops -mavx)
|
||||
target_compile_options(base64_avx2 PRIVATE -falign-loops -mavx2)
|
||||
|
||||
add_library(base64
|
||||
${LIBRARY_DIR}/lib/lib.c
|
||||
${LIBRARY_DIR}/lib/codec_choose.c
|
||||
${LIBRARY_DIR}/lib/arch/avx/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/avx2/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/generic/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/neon32/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/neon64/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/sse41/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/sse42/codec.c
|
||||
${LIBRARY_DIR}/lib/arch/ssse3/codec.c
|
||||
$<TARGET_OBJECTS:base64_scalar>
|
||||
$<TARGET_OBJECTS:base64_ssse3>
|
||||
$<TARGET_OBJECTS:base64_avx>
|
||||
$<TARGET_OBJECTS:base64_avx2>)
|
||||
|
||||
${LIBRARY_DIR}/lib/codecs.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/config.h)
|
||||
|
||||
if(HAVE_AVX)
|
||||
set_source_files_properties(${LIBRARY_DIR}/lib/arch/avx/codec.c PROPERTIES COMPILE_FLAGS -mavx)
|
||||
endif()
|
||||
if(HAVE_AVX2)
|
||||
set_source_files_properties(${LIBRARY_DIR}/lib/arch/avx2/codec.c PROPERTIES COMPILE_FLAGS -mavx2)
|
||||
endif()
|
||||
if(HAVE_SSE41)
|
||||
set_source_files_properties(${LIBRARY_DIR}/lib/arch/sse41/codec.c PROPERTIES COMPILE_FLAGS -msse4.1)
|
||||
endif()
|
||||
if(HAVE_SSE42)
|
||||
set_source_files_properties(${LIBRARY_DIR}/lib/arch/sse42/codec.c PROPERTIES COMPILE_FLAGS -msse4.2)
|
||||
endif()
|
||||
if(HAVE_SSSE3)
|
||||
set_source_files_properties(${LIBRARY_DIR}/lib/arch/ssse3/codec.c PROPERTIES COMPILE_FLAGS -mssse3)
|
||||
endif()
|
||||
|
||||
target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include ${CMAKE_CURRENT_BINARY_DIR})
|
||||
target_include_directories(base64 PUBLIC ${LIBRARY_DIR})
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
#define HAVE_NEON32 @base64_NEON32@
|
||||
#define HAVE_NEON64 @base64_NEON64@
|
||||
#cmakedefine HAVE_SSSE3 @base64_SSSE3@
|
||||
#cmakedefine HAVE_SSE41 @base64_SSE41@
|
||||
#cmakedefine HAVE_SSE42 @base64_SSE42@
|
||||
#cmakedefine HAVE_AVX @base64_AVX@
|
||||
#cmakedefine HAVE_AVX2 @base64_AVX2@
|
||||
#cmakedefine HAVE_FAST_UNALIGNED_ACCESS @HAVE_FAST_UNALIGNED_ACCESS@
|
||||
@ -7,7 +7,7 @@
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/GatherUtils/Algorithms.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <libbase64.h>
|
||||
#include <turbob64.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -107,40 +107,48 @@ public:
|
||||
|
||||
const ColumnString::Offsets & src_offsets = input->getOffsets();
|
||||
|
||||
auto source = reinterpret_cast<const char *>(input->getChars().data());
|
||||
auto dst = reinterpret_cast<char *>(dst_data.data());
|
||||
auto source = input->getChars().data();
|
||||
auto dst = dst_data.data();
|
||||
auto dst_pos = dst;
|
||||
|
||||
size_t src_offset_prev = 0;
|
||||
|
||||
int codec = getCodec();
|
||||
for (size_t row = 0; row < input_rows_count; ++row)
|
||||
{
|
||||
size_t srclen = src_offsets[row] - src_offset_prev - 1;
|
||||
size_t outlen = 0;
|
||||
|
||||
if (srclen >= (3U << 30))
|
||||
throw Exception("Base64 encoding/decoding doesn't support strings larger than 4 GiB", ErrorCodes::INCORRECT_DATA);
|
||||
|
||||
if constexpr (std::is_same_v<Func, Base64Encode>)
|
||||
{
|
||||
base64_encode(source, srclen, dst_pos, &outlen, codec);
|
||||
outlen = tb64enc(source, srclen, dst_pos);
|
||||
}
|
||||
else if constexpr (std::is_same_v<Func, Base64Decode>)
|
||||
{
|
||||
if (!base64_decode(source, srclen, dst_pos, &outlen, codec))
|
||||
if (srclen > 0)
|
||||
{
|
||||
throw Exception("Failed to " + getName() + " input '" + String(source, srclen) + "'", ErrorCodes::INCORRECT_DATA);
|
||||
outlen = tb64dec(source, srclen, dst_pos);
|
||||
if (!outlen)
|
||||
throw Exception("Failed to " + getName() + " input '" + String(reinterpret_cast<const char *>(source), srclen) + "'", ErrorCodes::INCORRECT_DATA);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// during decoding character array can be partially polluted
|
||||
// if fail, revert back and clean
|
||||
auto savepoint = dst_pos;
|
||||
if (!base64_decode(source, srclen, dst_pos, &outlen, codec))
|
||||
if (srclen > 0)
|
||||
{
|
||||
outlen = 0;
|
||||
dst_pos = savepoint;
|
||||
// clean the symbol
|
||||
dst_pos[0] = 0;
|
||||
// during decoding character array can be partially polluted
|
||||
// if fail, revert back and clean
|
||||
auto savepoint = dst_pos;
|
||||
outlen = tb64dec(source, srclen, dst_pos);
|
||||
if (!outlen)
|
||||
{
|
||||
outlen = 0;
|
||||
dst_pos = savepoint;
|
||||
// clean the symbol
|
||||
dst_pos[0] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -155,15 +163,6 @@ public:
|
||||
|
||||
block.getByPosition(result).column = std::move(dst_column);
|
||||
}
|
||||
|
||||
private:
|
||||
static int getCodec()
|
||||
{
|
||||
/// You can provide different value if you want to test specific codecs.
|
||||
/// Due to poor implementation of "base64" library (it will write to a global variable),
|
||||
/// it doesn't scale for multiple threads. Never use non-zero values in production.
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -8,6 +8,7 @@ namespace DB
|
||||
{
|
||||
void registerFunctionBase64Decode(FunctionFactory & factory)
|
||||
{
|
||||
tb64ini(0);
|
||||
factory.registerFunction<FunctionBase64Conversion<Base64Decode>>();
|
||||
}
|
||||
}
|
||||
|
||||
@ -9,6 +9,7 @@ namespace DB
|
||||
{
|
||||
void registerFunctionBase64Encode(FunctionFactory & factory)
|
||||
{
|
||||
tb64ini(0);
|
||||
factory.registerFunction<FunctionBase64Conversion<Base64Encode>>();
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user