thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into system-stack-trace

Browse Source
This commit is contained in:
Alexey Milovidov 2019-12-23 19:41:23 +03:00
commit 239451a24d
10 changed files with 136 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dbms/programs/server/Server.cpp
View File

@ -18,6 +18,7 @@
#include <common/config_common.h>
#include <common/ErrorHandlers.h>
#include <common/getMemoryAmount.h>
#include <common/coverage.h>
#include <Common/ClickHouseRevision.h>
#include <Common/DNSResolver.h>
#include <Common/CurrentMetrics.h>
@ -938,6 +939,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
/// (they are effectively dangling objects, but they use global thread pool
/// and global thread pool destructor will wait for threads, preventing server shutdown).
/// Dump coverage here, because std::atexit callback would not be called.
dumpCoverageReportIfPossible();
LOG_INFO(log, "Will shutdown forcefully.");
_exit(Application::EXIT_OK);
}

132
dbms/src/Access/AllowedClientHosts.cpp
View File

@ -9,6 +9,7 @@
#include <ext/scope_guard.h>
#include <boost/range/algorithm/find.hpp>
#include <boost/range/algorithm/find_first_of.hpp>
#include <boost/algorithm/string/predicate.hpp>
#include <ifaddrs.h>
@ -23,29 +24,64 @@ namespace ErrorCodes
namespace
{
using IPAddress = Poco::Net::IPAddress;
using IPSubnet = AllowedClientHosts::IPSubnet;
const IPSubnet ALL_ADDRESSES{IPAddress{IPAddress::IPv6}, IPAddress{IPAddress::IPv6}};
const AllowedClientHosts::IPSubnet ALL_ADDRESSES = AllowedClientHosts::IPSubnet{IPAddress{IPAddress::IPv6}, IPAddress{IPAddress::IPv6}};
IPAddress toIPv6(const IPAddress & addr)
const IPAddress & getIPV6Loopback()
{
if (addr.family() == IPAddress::IPv6)
return addr;
if (addr.isLoopback())
return IPAddress("::1");
return IPAddress("::FFFF:" + addr.toString());
static const IPAddress ip("::1");
return ip;
}
IPAddress maskToIPv6(const IPAddress & mask)
bool isIPV4LoopbackMappedToIPV6(const IPAddress & ip)
{
if (mask.family() == IPAddress::IPv6)
return mask;
return IPAddress(96, IPAddress::IPv6) | toIPv6(mask);
static const IPAddress prefix("::ffff:127.0.0.0");
/// 104 == 128 - 24, we have to reset the lowest 24 bits of 128 before comparing with `prefix`
/// (IPv4 loopback means any IP from 127.0.0.0 to 127.255.255.255).
return (ip & IPAddress(104, IPAddress::IPv6)) == prefix;
}
/// Converts an address to IPv6.
/// The loopback address "127.0.0.1" (or any "127.x.y.z") is converted to "::1".
IPAddress toIPv6(const IPAddress & ip)
{
IPAddress v6;
if (ip.family() == IPAddress::IPv6)
v6 = ip;
else
v6 = IPAddress("::ffff:" + ip.toString());
// ::ffff:127.XX.XX.XX -> ::1
if (isIPV4LoopbackMappedToIPV6(v6))
v6 = getIPV6Loopback();
return v6;
}
/// Converts a subnet to IPv6.
IPSubnet toIPv6(const IPSubnet & subnet)
{
IPSubnet v6;
if (subnet.prefix.family() == IPAddress::IPv6)
v6.prefix = subnet.prefix;
else
v6.prefix = IPAddress("::ffff:" + subnet.prefix.toString());
if (subnet.mask.family() == IPAddress::IPv6)
v6.mask = subnet.mask;
else
v6.mask = IPAddress(96, IPAddress::IPv6) | IPAddress("::ffff:" + subnet.mask.toString());
v6.prefix = v6.prefix & v6.mask;
// ::ffff:127.XX.XX.XX -> ::1
if (isIPV4LoopbackMappedToIPV6(v6.prefix))
v6 = {getIPV6Loopback(), IPAddress(128, IPAddress::IPv6)};
return v6;
}
/// Helper function for isAddressOfHost().
bool isAddressOfHostImpl(const IPAddress & address, const String & host)
{
IPAddress addr_v6 = toIPv6(address);
@ -93,15 +129,15 @@ namespace
return false;
}
/// Cached version of isAddressOfHostImpl(). We need to cache DNS requests.
/// Whether a specified address is one of the addresses of a specified host.
bool isAddressOfHost(const IPAddress & address, const String & host)
{
/// We need to cache DNS requests.
static SimpleCache<decltype(isAddressOfHostImpl), isAddressOfHostImpl> cache;
return cache(address, host);
}
/// Helper function for isAddressOfLocalhost().
std::vector<IPAddress> getAddressesOfLocalhostImpl()
{
std::vector<IPAddress> addresses;
@ -114,7 +150,7 @@ namespace
int err = getifaddrs(&ifa_begin);
if (err)
return {IPAddress{"127.0.0.1"}, IPAddress{"::1"}};
return {getIPV6Loopback()};
for (const ifaddrs * ifa = ifa_begin; ifa; ifa = ifa->ifa_next)
{
@ -134,15 +170,15 @@ namespace
return addresses;
}
/// Checks if a specified address pointers to the localhost.
bool isLocalAddress(const IPAddress & address)
/// Whether a specified address is one of the addresses of the localhost.
bool isAddressOfLocalhost(const IPAddress & address)
{
/// We need to cache DNS requests.
static const std::vector<IPAddress> local_addresses = getAddressesOfLocalhostImpl();
return boost::range::find(local_addresses, address) != local_addresses.end();
return boost::range::find(local_addresses, toIPv6(address)) != local_addresses.end();
}
/// Helper function for getHostByAddress().
String getHostByAddressImpl(const IPAddress & address)
{
Poco::Net::SocketAddress sock_addr(address, 0);
@ -160,10 +196,10 @@ namespace
return host;
}
/// Cached version of getHostByAddressImpl(). We need to cache DNS requests.
/// Returns the host name by its address.
String getHostByAddress(const IPAddress & address)
{
/// We need to cache DNS requests.
static SimpleCache<decltype(getHostByAddressImpl), &getHostByAddressImpl> cache;
return cache(address);
}
@ -203,7 +239,7 @@ AllowedClientHosts::AllowedClientHosts(const AllowedClientHosts & src)
AllowedClientHosts & AllowedClientHosts::operator =(const AllowedClientHosts & src)
{
addresses = src.addresses;
loopback = src.loopback;
localhost = src.localhost;
subnets = src.subnets;
host_names = src.host_names;
host_regexps = src.host_regexps;
@ -212,28 +248,14 @@ AllowedClientHosts & AllowedClientHosts::operator =(const AllowedClientHosts & s
}
AllowedClientHosts::AllowedClientHosts(AllowedClientHosts && src)
{
*this = src;
}
AllowedClientHosts & AllowedClientHosts::operator =(AllowedClientHosts && src)
{
addresses = std::move(src.addresses);
loopback = src.loopback;
subnets = std::move(src.subnets);
host_names = std::move(src.host_names);
host_regexps = std::move(src.host_regexps);
compiled_host_regexps = std::move(src.compiled_host_regexps);
return *this;
}
AllowedClientHosts::AllowedClientHosts(AllowedClientHosts && src) = default;
AllowedClientHosts & AllowedClientHosts::operator =(AllowedClientHosts && src) = default;
void AllowedClientHosts::clear()
{
addresses.clear();
loopback = false;
localhost = false;
subnets.clear();
host_names.clear();
host_regexps.clear();
@ -250,10 +272,11 @@ bool AllowedClientHosts::empty() const
void AllowedClientHosts::addAddress(const IPAddress & address)
{
IPAddress addr_v6 = toIPv6(address);
if (boost::range::find(addresses, addr_v6) == addresses.end())
addresses.push_back(addr_v6);
if (boost::range::find(addresses, addr_v6) != addresses.end())
return;
addresses.push_back(addr_v6);
if (addr_v6.isLoopback())
loopback = true;
localhost = true;
}
@ -265,9 +288,7 @@ void AllowedClientHosts::addAddress(const String & address)
void AllowedClientHosts::addSubnet(const IPSubnet & subnet)
{
IPSubnet subnet_v6;
subnet_v6.prefix = toIPv6(subnet.prefix);
subnet_v6.mask = maskToIPv6(subnet.mask);
IPSubnet subnet_v6 = toIPv6(subnet);
if (subnet_v6.mask == IPAddress(128, IPAddress::IPv6))
{
@ -275,8 +296,6 @@ void AllowedClientHosts::addSubnet(const IPSubnet & subnet)
return;
}
subnet_v6.prefix = subnet_v6.prefix & subnet_v6.mask;
if (boost::range::find(subnets, subnet_v6) == subnets.end())
subnets.push_back(subnet_v6);
}
@ -314,8 +333,11 @@ void AllowedClientHosts::addSubnet(const String & subnet)
void AllowedClientHosts::addHostName(const String & host_name)
{
if (boost::range::find(host_names, host_name) == host_names.end())
host_names.push_back(host_name);
if (boost::range::find(host_names, host_name) != host_names.end())
return;
host_names.push_back(host_name);
if (boost::iequals(host_name, "localhost"))
localhost = true;
}
@ -360,7 +382,7 @@ bool AllowedClientHosts::contains(const IPAddress & address) const
if (boost::range::find(addresses, addr_v6) != addresses.end())
return true;
if (loopback && isLocalAddress(addr_v6))
if (localhost && isAddressOfLocalhost(addr_v6))
return true;
/// Check `ip_subnets`.

2
dbms/src/Access/AllowedClientHosts.h
View File

@ -94,7 +94,7 @@ private:
void compileRegexps() const;
std::vector<IPAddress> addresses;
bool loopback = false;
bool localhost = false;
std::vector<IPSubnet> subnets;
std::vector<String> host_names;
std::vector<String> host_regexps;

4
dbms/src/Databases/DatabaseOnDisk.cpp
View File

@ -286,7 +286,9 @@ void DatabaseOnDisk::createDictionary(
String full_name = database.getDatabaseName() + "." + dictionary_name;
auto & external_loader = const_cast<ExternalDictionariesLoader &>(context.getExternalDictionariesLoader());
if (external_loader.getCurrentStatus(full_name) != ExternalLoader::Status::NOT_EXIST)
throw Exception("Dictionary " + backQuote(full_name) + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
throw Exception(
"Dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.",
ErrorCodes::DICTIONARY_ALREADY_EXISTS);
if (database.isTableExist(context, dictionary_name))
throw Exception("Table " + backQuote(database.getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);

7
docker/test/stateless_with_coverage/run.sh
View File

@ -1,15 +1,16 @@
#!/bin/bash
kill_clickhouse () {
echo "clickhouse pids" `ps aux | grep clickhouse` | ts '%Y-%m-%d %H:%M:%S'
kill `pgrep -u clickhouse` 2>/dev/null
for i in {1..10}
do
if ! kill -0 `pgrep -u clickhouse`; then
echo "No clickhouse process"
echo "No clickhouse process" | ts '%Y-%m-%d %H:%M:%S'
break
else
echo "Process" `pgrep -u clickhouse` "still alive"
echo "Process" `pgrep -u clickhouse` "still alive" | ts '%Y-%m-%d %H:%M:%S'
sleep 10
fi
done
@ -18,7 +19,7 @@ kill_clickhouse () {
wait_llvm_profdata () {
while kill -0 `pgrep llvm-profdata-9`;
do
echo "Waiting for profdata" `pgrep llvm-profdata-9` "still alive"
echo "Waiting for profdata" `pgrep llvm-profdata-9` "still alive" | ts '%Y-%m-%d %H:%M:%S'
sleep 3
done
}

8
docs/ru/extended_roadmap.md
View File

@ -322,7 +322,7 @@ ClickHouse использует небольшое подмножество фу
Алексанр Кузьменков.
### 7.4. Включить libc++, libc++-abi при сборке с gcc.
### 7.4. + Включить libc++, libc++-abi при сборке с gcc.
Сейчас включено только при сборке с clang, но продакшен сборка использует gcc.
Требует 7.2 и, возможно, 7.1 (только в случае новой версии ICU).
@ -358,12 +358,11 @@ UBSan включен в функциональных тестах, но не в
Пока есть просто показ тестового покрытия всего кода.
### 7.13. Включение аналога -Weverything в gcc.
### 7.13. + Включение аналога -Weverything в gcc.
Мы используем -Wall -Wextra -Weverything -Werror.
При сборке с clang, -Weverything уже включено. Но в gcc есть уникальные warning-и, отсутствующие в clang.
Wolf Kreuzerkrieg. Возможно, его уже не интересует эта задача.
Низкий приоритет. Возможно, будет отменено.
Сделал Wolf Kreuzerkrieg.
### 7.14. Альтернатива для readline и libedit.
@ -425,7 +424,6 @@ https://github.com/ClickHouse/ClickHouse/issues/8027#issuecomment-566670282
Проверили на настоящем сервере Huawei, а также в специальном Docker контейнере, который содержит внутри qemu-user-static.
Также можно проверить на Cavium, на Raspberry Pi а также на твоём Android телефоне.
### 7.20. Автосборка для FreeBSD x86_64.
[Иван Лежанкин](https://github.com/abyss7).

2
libs/libcommon/CMakeLists.txt
View File

@ -24,6 +24,7 @@ add_library (common
src/sleep.cpp
src/argsToConfig.cpp
src/phdr_cache.cpp
src/coverage.cpp
include/common/SimpleCache.h
include/common/Types.h
@ -49,6 +50,7 @@ add_library (common
include/common/sleep.h
include/common/SimpleCache.h
include/common/phdr_cache.h
include/common/coverage.h
include/ext/bit_cast.h
include/ext/chrono_io.h

9
libs/libcommon/include/common/coverage.h Normal file
View File

@ -0,0 +1,9 @@
#pragma once
/// Flush coverage report to file, depending on coverage system
/// proposed by compiler (llvm for clang and gcov for gcc).
///
/// Noop if build without coverage (WITH_COVERAGE=0).
/// Thread safe (use exclusive lock).
/// Idempotent, may be called multiple times.
void dumpCoverageReportIfPossible();

31
libs/libcommon/src/coverage.cpp Normal file
View File

@ -0,0 +1,31 @@
#include <common/coverage.h>
#include <common/config_common.h>
#if WITH_COVERAGE
#include <unistd.h>
#include <mutex>
#if defined(__clang__)
extern "C" void __llvm_profile_dump();
#elif defined(__GNUC__) || defined(__GNUG__)
extern "C" void __gcov_exit();
#endif
#endif
void dumpCoverageReportIfPossible()
{
#if WITH_COVERAGE
static std::mutex mutex;
std::lock_guard lock(mutex);
#if defined(__clang__)
__llvm_profile_dump();
#elif defined(__GNUC__) || defined(__GNUG__)
__gcov_exit();
#endif
#endif
}

3
libs/libdaemon/src/BaseDaemon.cpp
View File

@ -36,6 +36,8 @@
#include <common/logger_useful.h>
#include <common/ErrorHandlers.h>
#include <common/argsToConfig.h>
#include <common/getThreadNumber.h>
#include <common/coverage.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteBufferFromFileDescriptorDiscardOnFailure.h>
@ -464,6 +466,7 @@ void BaseDaemon::terminate()
void BaseDaemon::kill()
{
dumpCoverageReportIfPossible();
pid.clear();
if (::raise(SIGKILL) != 0)
throw Poco::SystemException("cannot kill process");