thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 08:02:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into uuid-fuzz

Browse Source
This commit is contained in:
Alexey Milovidov 2021-01-05 02:51:36 +03:00
commit f5c81a5c72
15 changed files with 270 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
base/daemon/BaseDaemon.cpp
View File

@ -85,7 +85,7 @@ const char * msan_strsignal(int sig)
// Apparently strsignal is not instrumented by MemorySanitizer, so we
// have to unpoison it to avoid msan reports inside fmt library when we
// print it.
const char * signal_name = strsignal(sig);
const char * signal_name = sys_siglist[sig];
__msan_unpoison_string(signal_name);
return signal_name;
}

8
base/glibc-compatibility/musl/lgamma.c
View File

@ -258,11 +258,3 @@ double lgamma_r(double x, int *signgamp)
r = nadj - r;
return r;
}
int signgam;
double lgamma(double x)
{
return lgamma_r(x, &signgam);
}

9
base/glibc-compatibility/musl/lgammal.c
View File

@ -328,12 +328,3 @@ long double lgammal_r(long double x, int *sg)
return lgamma_r(x, sg);
}
#endif
int signgam_lgammal;
long double lgammal(long double x)
{
return lgammal_r(x, &signgam_lgammal);
}

2
base/harmful/CMakeLists.txt Normal file
View File

@ -0,0 +1,2 @@
add_library(harmful harmful.c)
install(TARGETS harmful EXPORT global ARCHIVE DESTINATION lib)

1
base/harmful/README.md Normal file
View File

@ -0,0 +1 @@
A library that traps whenever harmful functions from libc are called.

244
base/harmful/harmful.c Normal file
View File

@ -0,0 +1,244 @@
/** This library provides runtime instrumentation (hardening)
* that ensures no "harmful" functions from libc are called
* (by terminating the program immediately).
*/
/// It is only enabled in debug build (its intended use is for CI checks).
#if !defined(NDEBUG)
#if defined(__clang__)
#pragma clang diagnostic ignored "-Wincompatible-library-redeclaration"
#else
#pragma GCC diagnostic ignored "-Wbuiltin-declaration-mismatch"
#endif
/// We cannot use libc headers here.
long write(int, const void *, unsigned long);
#define TRAP(func) void func() { write(2, #func "\n", __builtin_strlen(#func) + 1); __builtin_trap(); }
/// Trap all non thread-safe functions:
/// nm -D /lib/x86_64-linux-gnu/{libc.so.6,libdl.so.2,libm.so.6,libpthread.so.0,librt.so.1,libnss_dns.so.2,libresolv.so.2} | grep -P '_r@?$' | awk '{ print $3 }' | sed -r -e 's/_r//' | grep -vP '^_'
/// See also https://reviews.llvm.org/D90944
/// You can edit this list and even comment out some functions.
/// The only purpose of the library is to force you to pay attention.
TRAP(argp_error)
TRAP(argp_help)
TRAP(argp_parse)
TRAP(argp_state_help)
TRAP(argp_usage)
TRAP(asctime)
TRAP(clearenv)
TRAP(crypt)
TRAP(ctime)
TRAP(cuserid)
TRAP(drand48)
TRAP(ecvt)
TRAP(encrypt)
TRAP(endfsent)
TRAP(endgrent)
TRAP(endhostent)
TRAP(endnetent)
TRAP(endnetgrent)
TRAP(endprotoent)
TRAP(endpwent)
TRAP(endservent)
TRAP(endutent)
TRAP(endutxent)
TRAP(erand48)
TRAP(error_at_line)
///TRAP(exit)
TRAP(fcloseall)
TRAP(fcvt)
TRAP(fgetgrent)
TRAP(fgetpwent)
TRAP(gammal)
TRAP(getchar_unlocked)
TRAP(getdate)
TRAP(getfsent)
TRAP(getfsfile)
TRAP(getfsspec)
TRAP(getgrent)
TRAP(getgrent_r)
TRAP(getgrgid)
TRAP(getgrnam)
TRAP(gethostbyaddr)
TRAP(gethostbyname)
TRAP(gethostbyname2)
TRAP(gethostent)
TRAP(getlogin)
TRAP(getmntent)
TRAP(getnetbyaddr)
TRAP(getnetbyname)
TRAP(getnetent)
TRAP(getnetgrent)
TRAP(getnetgrent_r)
TRAP(getopt)
TRAP(getopt_long)
TRAP(getopt_long_only)
TRAP(getpass)
TRAP(getprotobyname)
TRAP(getprotobynumber)
TRAP(getprotoent)
TRAP(getpwent)
TRAP(getpwent_r)
TRAP(getpwnam)
TRAP(getpwuid)
TRAP(getservbyname)
TRAP(getservbyport)
TRAP(getservent)
TRAP(getutent)
TRAP(getutent_r)
TRAP(getutid)
TRAP(getutid_r)
TRAP(getutline)
TRAP(getutline_r)
TRAP(getutxent)
TRAP(getutxid)
TRAP(getutxline)
TRAP(getwchar_unlocked)
//TRAP(glob)
//TRAP(glob64)
TRAP(gmtime)
TRAP(hcreate)
TRAP(hdestroy)
TRAP(hsearch)
TRAP(innetgr)
TRAP(jrand48)
TRAP(l64a)
TRAP(lcong48)
TRAP(lgammafNx)
TRAP(localeconv)
TRAP(localtime)
TRAP(login)
TRAP(login_tty)
TRAP(logout)
TRAP(logwtmp)
TRAP(lrand48)
TRAP(mallinfo)
TRAP(mallopt)
TRAP(mblen)
TRAP(mbrlen)
TRAP(mbrtowc)
TRAP(mbsnrtowcs)
TRAP(mbsrtowcs)
//TRAP(mbtowc) // Used by Standard C++ library
TRAP(mcheck)
TRAP(mprobe)
TRAP(mrand48)
TRAP(mtrace)
TRAP(muntrace)
TRAP(nrand48)
TRAP(__ppc_get_timebase_freq)
TRAP(ptsname)
TRAP(putchar_unlocked)
TRAP(putenv)
TRAP(pututline)
TRAP(pututxline)
TRAP(putwchar_unlocked)
TRAP(qecvt)
TRAP(qfcvt)
TRAP(register_printf_function)
TRAP(seed48)
TRAP(setenv)
TRAP(setfsent)
TRAP(setgrent)
TRAP(sethostent)
TRAP(sethostid)
TRAP(setkey)
//TRAP(setlocale) // Used by replxx at startup
TRAP(setlogmask)
TRAP(setnetent)
TRAP(setnetgrent)
TRAP(setprotoent)
TRAP(setpwent)
TRAP(setservent)
TRAP(setutent)
TRAP(setutxent)
TRAP(siginterrupt)
TRAP(sigpause)
//TRAP(sigprocmask)
TRAP(sigsuspend)
TRAP(sleep)
TRAP(srand48)
//TRAP(strerror) // Used by RocksDB and many other libraries, unfortunately.
TRAP(strsignal)
TRAP(strtok)
TRAP(tcflow)
TRAP(tcsendbreak)
TRAP(tmpnam)
TRAP(ttyname)
TRAP(unsetenv)
TRAP(updwtmp)
TRAP(utmpname)
TRAP(utmpxname)
//TRAP(valloc)
TRAP(vlimit)
//TRAP(wcrtomb) // Used by Standard C++ library
TRAP(wcsnrtombs)
TRAP(wcsrtombs)
TRAP(wctomb)
TRAP(wordexp)
TRAP(basename)
TRAP(catgets)
TRAP(dbm_clearerr)
TRAP(dbm_close)
TRAP(dbm_delete)
TRAP(dbm_error)
TRAP(dbm_fetch)
TRAP(dbm_firstkey)
TRAP(dbm_nextkey)
TRAP(dbm_open)
TRAP(dbm_store)
TRAP(dirname)
TRAP(dlerror)
TRAP(ftw)
TRAP(getc_unlocked)
//TRAP(getenv) // Ok at program startup
TRAP(inet_ntoa)
TRAP(lgamma)
TRAP(lgammaf)
TRAP(lgammal)
TRAP(nftw)
TRAP(nl_langinfo)
TRAP(putc_unlocked)
TRAP(rand)
/** In the current POSIX.1 specification (POSIX.1-2008), readdir() is not required to be thread-safe. However, in modern
* implementations (including the glibc implementation), concurrent calls to readdir() that specify different directory streams
* are thread-safe. In cases where multiple threads must read from the same directory stream, using readdir() with external
* synchronization is still preferable to the use of the deprecated readdir_r(3) function. It is expected that a future
* version of POSIX.1 will require that readdir() be thread-safe when concurrently employed on different directory streams.
* - man readdir
*/
//TRAP(readdir)
TRAP(system)
TRAP(wcstombs)
TRAP(ether_aton)
TRAP(ether_ntoa)
TRAP(fgetsgent)
TRAP(fgetspent)
TRAP(getaliasbyname)
TRAP(getaliasent)
TRAP(getrpcbyname)
TRAP(getrpcbynumber)
TRAP(getrpcent)
TRAP(getsgent)
TRAP(getsgnam)
TRAP(getspent)
TRAP(getspnam)
TRAP(initstate)
TRAP(random)
TRAP(setstate)
TRAP(sgetsgent)
TRAP(sgetspent)
TRAP(srandom)
TRAP(twalk)
TRAP(lgammaf128)
TRAP(lgammaf32)
TRAP(lgammaf32x)
TRAP(lgammaf64)
TRAP(lgammaf64x)
#endif

1
cmake/linux/default_libs.cmake
View File

@ -39,6 +39,7 @@ find_package(Threads REQUIRED)
if (NOT OS_ANDROID)
# Our compatibility layer doesn't build under Android, many errors in musl.
add_subdirectory(base/glibc-compatibility)
add_subdirectory(base/harmful)
endif ()
include (cmake/find/unwind.cmake)

8
programs/CMakeLists.txt
View File

@ -216,7 +216,13 @@ if (CLICKHOUSE_SPLIT_BINARY)
install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse)
else ()
add_executable (clickhouse main.cpp)
target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils)
# A library that prevent usage of several functions from libc.
if (ARCH_AMD64 AND OS_LINUX AND NOT OS_ANDROID)
set (HARMFUL_LIB harmful)
endif ()
target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils ${HARMFUL_LIB})
target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
if (ENABLE_CLICKHOUSE_SERVER)

5
programs/install/Install.cpp
View File

@ -21,6 +21,7 @@
#include <Common/OpenSSLHelpers.h>
#include <Common/hex.h>
#include <common/getResource.h>
#include <common/sleep.h>
#include <IO/ReadBufferFromFileDescriptor.h>
#include <IO/WriteBufferFromFileDescriptor.h>
#include <IO/ReadBufferFromFile.h>
@ -763,7 +764,7 @@ namespace
fmt::print("Server started\n");
break;
}
::sleep(1);
sleepForSeconds(1);
}
if (try_num == num_tries)
@ -875,7 +876,7 @@ namespace
fmt::print("Server stopped\n");
break;
}
::sleep(1);
sleepForSeconds(1);
}
if (try_num == num_tries)

5
src/Functions/trap.cpp
View File

@ -140,6 +140,11 @@ public:
{
executeImpl(block, result_type, input_rows_count);
}
else if (mode == "harmful function")
{
double res = drand48();
(void)res;
}
else if (mode == "mmap many")
{
std::vector<void *> maps;

2
src/IO/ReadHelpers.h
View File

@ -561,7 +561,7 @@ template <typename ReturnType = void>
inline ReturnType readDateTextImpl(LocalDate & date, ReadBuffer & buf)
{
/// Optimistic path, when whole value is in buffer.
if (buf.position() + 10 <= buf.buffer().end())
if (!buf.eof() && buf.position() + 10 <= buf.buffer().end())
{
UInt16 year = (buf.position()[0] - '0') * 1000 + (buf.position()[1] - '0') * 100 + (buf.position()[2] - '0') * 10 + (buf.position()[3] - '0');
buf.position() += 5;

4
src/IO/WriteHelpers.h
View File

@ -730,7 +730,7 @@ static const char digits100[201] =
template <char delimiter = '-'>
inline void writeDateText(const LocalDate & date, WriteBuffer & buf)
{
if (buf.position() + 10 <= buf.buffer().end())
if (reinterpret_cast<intptr_t>(buf.position()) + 10 <= reinterpret_cast<intptr_t>(buf.buffer().end()))
{
memcpy(buf.position(), &digits100[date.year() / 100 * 2], 2);
buf.position() += 2;
@ -767,7 +767,7 @@ inline void writeDateText(DayNum date, WriteBuffer & buf)
template <char date_delimeter = '-', char time_delimeter = ':', char between_date_time_delimiter = ' '>
inline void writeDateTimeText(const LocalDateTime & datetime, WriteBuffer & buf)
{
if (buf.position() + 19 <= buf.buffer().end())
if (reinterpret_cast<intptr_t>(buf.position()) + 19 <= reinterpret_cast<intptr_t>(buf.buffer().end()))
{
memcpy(buf.position(), &digits100[datetime.year() / 100 * 2], 2);
buf.position() += 2;

2
src/IO/WriteIntText.h
View File

@ -24,7 +24,7 @@ namespace detail
template <typename T>
void writeIntText(T x, WriteBuffer & buf)
{
if (likely(buf.position() + WRITE_HELPERS_MAX_INT_WIDTH < buf.buffer().end()))
if (likely(reinterpret_cast<intptr_t>(buf.position()) + WRITE_HELPERS_MAX_INT_WIDTH < reinterpret_cast<intptr_t>(buf.buffer().end())))
buf.position() = itoa(x, buf.position());
else
detail::writeUIntTextFallback(x, buf);

2
src/IO/readFloatText.h
View File

@ -152,7 +152,7 @@ ReturnType readFloatTextPreciseImpl(T & x, ReadBuffer & buf)
/// Fast path (avoid copying) if the buffer have at least MAX_LENGTH bytes.
static constexpr int MAX_LENGTH = 316;
if (likely(buf.position() + MAX_LENGTH <= buf.buffer().end()))
if (likely(!buf.eof() && buf.position() + MAX_LENGTH <= buf.buffer().end()))
{
auto initial_position = buf.position();
auto res = fast_float::from_chars(initial_position, buf.buffer().end(), x);

2
src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
View File

@ -96,7 +96,7 @@ StringRef JSONEachRowRowInputFormat::readColumnName(ReadBuffer & buf)
{
// This is just an optimization: try to avoid copying the name into current_column_name
if (nested_prefix_length == 0 && buf.position() + 1 < buf.buffer().end())
if (nested_prefix_length == 0 && !buf.eof() && buf.position() + 1 < buf.buffer().end())
{
char * next_pos = find_first_symbols<'\\', '"'>(buf.position() + 1, buf.buffer().end());