thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 09:32:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into fix-materialized-pg-issue-with-generated-columns

Browse Source
This commit is contained in:
kssenii 2023-12-11 11:15:00 +01:00
commit a06179da00
143 changed files with 1246 additions and 1584 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/master.yml vendored
View File

@ -205,6 +205,12 @@ jobs:
with:
build_name: binary_amd64_compat
checkout_depth: 0
BuilderBinAmd64Musl:
needs: [DockerHubPush]
uses: ./.github/workflows/reusable_build.yml
with:
build_name: binary_amd64_musl
checkout_depth: 0
BuilderBinAarch64V80Compat:
needs: [DockerHubPush]
uses: ./.github/workflows/reusable_build.yml

5
.github/workflows/pull_request.yml vendored
View File

@ -242,6 +242,11 @@ jobs:
uses: ./.github/workflows/reusable_build.yml
with:
build_name: binary_amd64_compat
BuilderBinAmd64Musl:
needs: [FastTest, StyleCheck]
uses: ./.github/workflows/reusable_build.yml
with:
build_name: binary_amd64_musl
BuilderBinAarch64V80Compat:
needs: [FastTest, StyleCheck]
uses: ./.github/workflows/reusable_build.yml

1
base/glibc-compatibility/glibc-compatibility.c
View File

@ -30,7 +30,6 @@ int __gai_sigqueue(int sig, const union sigval val, pid_t caller_pid)
}
#include <sys/select.h>
#include <stdlib.h>
#include <features.h>

1
base/poco/Foundation/CMakeLists.txt
View File

@ -55,7 +55,6 @@ set (SRCS
src/DigestStream.cpp
src/DirectoryIterator.cpp
src/DirectoryIteratorStrategy.cpp
src/DirectoryWatcher.cpp
src/Environment.cpp
src/Error.cpp
src/ErrorHandler.cpp

228
base/poco/Foundation/include/Poco/DirectoryWatcher.h
View File

@ -1,228 +0,0 @@
//
// DirectoryWatcher.h
//
// Library: Foundation
// Package: Filesystem
// Module: DirectoryWatcher
//
// Definition of the DirectoryWatcher class.
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#ifndef Foundation_DirectoryWatcher_INCLUDED
#define Foundation_DirectoryWatcher_INCLUDED
#include "Poco/Foundation.h"
#ifndef POCO_NO_INOTIFY
# include "Poco/AtomicCounter.h"
# include "Poco/BasicEvent.h"
# include "Poco/File.h"
# include "Poco/Runnable.h"
# include "Poco/Thread.h"
namespace Poco
{
class DirectoryWatcherStrategy;
class Foundation_API DirectoryWatcher : protected Runnable
/// This class is used to get notifications about changes
/// to the filesystem, more specifically, to a specific
/// directory. Changes to a directory are reported via
/// events.
///
/// A thread will be created that watches the specified
/// directory for changes. Events are reported in the context
/// of this thread.
///
/// Note that changes to files in subdirectories of the watched
/// directory are not reported. Separate DirectoryWatcher objects
/// must be created for these directories if they should be watched.
///
/// Changes to file attributes are not reported.
///
/// On Windows, this class is implemented using FindFirstChangeNotification()/FindNextChangeNotification().
/// On Linux, this class is implemented using inotify.
/// On FreeBSD and Darwin (Mac OS X, iOS), this class uses kevent/kqueue.
/// On all other platforms, the watched directory is periodically scanned
/// for changes. This can negatively affect performance if done too often.
/// Therefore, the interval in which scans are done can be specified in
/// the constructor. Note that periodic scanning will also be done on FreeBSD
/// and Darwin if events for changes to files (DW_ITEM_MODIFIED) are enabled.
///
/// DW_ITEM_MOVED_FROM and DW_ITEM_MOVED_TO events will only be reported
/// on Linux. On other platforms, a file rename or move operation
/// will be reported via a DW_ITEM_REMOVED and a DW_ITEM_ADDED event.
/// The order of these two events is not defined.
///
/// An event mask can be specified to enable only certain events.
{
public:
enum DirectoryEventType
{
DW_ITEM_ADDED = 1,
/// A new item has been created and added to the directory.
DW_ITEM_REMOVED = 2,
/// An item has been removed from the directory.
DW_ITEM_MODIFIED = 4,
/// An item has been modified.
DW_ITEM_MOVED_FROM = 8,
/// An item has been renamed or moved. This event delivers the old name.
DW_ITEM_MOVED_TO = 16
/// An item has been renamed or moved. This event delivers the new name.
};
enum DirectoryEventMask
{
DW_FILTER_ENABLE_ALL = 31,
/// Enables all event types.
DW_FILTER_DISABLE_ALL = 0
/// Disables all event types.
};
enum
{
DW_DEFAULT_SCAN_INTERVAL = 5 /// Default scan interval for platforms that don't provide a native notification mechanism.
};
struct DirectoryEvent
{
DirectoryEvent(const File & f, DirectoryEventType ev) : item(f), event(ev) { }
const File & item; /// The directory or file that has been changed.
DirectoryEventType event; /// The kind of event.
};
BasicEvent<const DirectoryEvent> itemAdded;
/// Fired when a file or directory has been created or added to the directory.
BasicEvent<const DirectoryEvent> itemRemoved;
/// Fired when a file or directory has been removed from the directory.
BasicEvent<const DirectoryEvent> itemModified;
/// Fired when a file or directory has been modified.
BasicEvent<const DirectoryEvent> itemMovedFrom;
/// Fired when a file or directory has been renamed. This event delivers the old name.
BasicEvent<const DirectoryEvent> itemMovedTo;
/// Fired when a file or directory has been moved. This event delivers the new name.
BasicEvent<const Exception> scanError;
/// Fired when an error occurs while scanning for changes.
DirectoryWatcher(const std::string & path, int eventMask = DW_FILTER_ENABLE_ALL, int scanInterval = DW_DEFAULT_SCAN_INTERVAL);
/// Creates a DirectoryWatcher for the directory given in path.
/// To enable only specific events, an eventMask can be specified by
/// OR-ing the desired event IDs (e.g., DW_ITEM_ADDED | DW_ITEM_MODIFIED).
/// On platforms where no native filesystem notifications are available,
/// scanInterval specifies the interval in seconds between scans
/// of the directory.
DirectoryWatcher(const File & directory, int eventMask = DW_FILTER_ENABLE_ALL, int scanInterval = DW_DEFAULT_SCAN_INTERVAL);
/// Creates a DirectoryWatcher for the specified directory
/// To enable only specific events, an eventMask can be specified by
/// OR-ing the desired event IDs (e.g., DW_ITEM_ADDED | DW_ITEM_MODIFIED).
/// On platforms where no native filesystem notifications are available,
/// scanInterval specifies the interval in seconds between scans
/// of the directory.
~DirectoryWatcher();
/// Destroys the DirectoryWatcher.
void suspendEvents();
/// Suspends sending of events. Can be called multiple times, but every
/// call to suspendEvent() must be matched by a call to resumeEvents().
void resumeEvents();
/// Resumes events, after they have been suspended with a call to suspendEvents().
bool eventsSuspended() const;
/// Returns true iff events are suspended.
int eventMask() const;
/// Returns the value of the eventMask passed to the constructor.
int scanInterval() const;
/// Returns the scan interval in seconds.
const File & directory() const;
/// Returns the directory being watched.
bool supportsMoveEvents() const;
/// Returns true iff the platform supports DW_ITEM_MOVED_FROM/itemMovedFrom and
/// DW_ITEM_MOVED_TO/itemMovedTo events.
protected:
void init();
void stop();
void run();
private:
DirectoryWatcher();
DirectoryWatcher(const DirectoryWatcher &);
DirectoryWatcher & operator=(const DirectoryWatcher &);
Thread _thread;
File _directory;
int _eventMask;
AtomicCounter _eventsSuspended;
int _scanInterval;
DirectoryWatcherStrategy * _pStrategy;
};
//
// inlines
//
inline bool DirectoryWatcher::eventsSuspended() const
{
return _eventsSuspended.value() > 0;
}
inline int DirectoryWatcher::eventMask() const
{
return _eventMask;
}
inline int DirectoryWatcher::scanInterval() const
{
return _scanInterval;
}
inline const File & DirectoryWatcher::directory() const
{
return _directory;
}
} // namespace Poco
#endif // POCO_NO_INOTIFY
#endif // Foundation_DirectoryWatcher_INCLUDED

602
base/poco/Foundation/src/DirectoryWatcher.cpp
View File

@ -1,602 +0,0 @@
//
// DirectoryWatcher.cpp
//
// Library: Foundation
// Package: Filesystem
// Module: DirectoryWatcher
//
// Copyright (c) 2012, Applied Informatics Software Engineering GmbH.
// and Contributors.
//
// SPDX-License-Identifier: BSL-1.0
//
#include "Poco/DirectoryWatcher.h"
#ifndef POCO_NO_INOTIFY
#include "Poco/Path.h"
#include "Poco/Glob.h"
#include "Poco/DirectoryIterator.h"
#include "Poco/Event.h"
#include "Poco/Exception.h"
#include "Poco/Buffer.h"
#if POCO_OS == POCO_OS_LINUX || POCO_OS == POCO_OS_ANDROID
#include <sys/inotify.h>
#include <sys/select.h>
#include <unistd.h>
#elif POCO_OS == POCO_OS_MAC_OS_X || POCO_OS == POCO_OS_FREE_BSD
#include <fcntl.h>
#include <sys/types.h>
#include <sys/event.h>
#include <sys/time.h>
#include <unistd.h>
#if (POCO_OS == POCO_OS_FREE_BSD) && !defined(O_EVTONLY)
#define O_EVTONLY 0x8000
#endif
#endif
#include <algorithm>
#include <atomic>
#include <map>
namespace Poco {
class DirectoryWatcherStrategy
{
public:
DirectoryWatcherStrategy(DirectoryWatcher& owner):
_owner(owner)
{
}
virtual ~DirectoryWatcherStrategy()
{
}
DirectoryWatcher& owner()
{
return _owner;
}
virtual void run() = 0;
virtual void stop() = 0;
virtual bool supportsMoveEvents() const = 0;
protected:
struct ItemInfo
{
ItemInfo():
size(0)
{
}
ItemInfo(const ItemInfo& other):
path(other.path),
size(other.size),
lastModified(other.lastModified)
{
}
explicit ItemInfo(const File& f):
path(f.path()),
size(f.isFile() ? f.getSize() : 0),
lastModified(f.getLastModified())
{
}
std::string path;
File::FileSize size;
Timestamp lastModified;
};
typedef std::map<std::string, ItemInfo> ItemInfoMap;
void scan(ItemInfoMap& entries)
{
DirectoryIterator it(owner().directory());
DirectoryIterator end;
while (it != end)
{
entries[it.path().getFileName()] = ItemInfo(*it);
++it;
}
}
void compare(ItemInfoMap& oldEntries, ItemInfoMap& newEntries)
{
for (ItemInfoMap::iterator itn = newEntries.begin(); itn != newEntries.end(); ++itn)
{
ItemInfoMap::iterator ito = oldEntries.find(itn->first);
if (ito != oldEntries.end())
{
if ((owner().eventMask() & DirectoryWatcher::DW_ITEM_MODIFIED) && !owner().eventsSuspended())
{
if (itn->second.size != ito->second.size || itn->second.lastModified != ito->second.lastModified)
{
Poco::File f(itn->second.path);
DirectoryWatcher::DirectoryEvent ev(f, DirectoryWatcher::DW_ITEM_MODIFIED);
owner().itemModified(&owner(), ev);
}
}
oldEntries.erase(ito);
}
else if ((owner().eventMask() & DirectoryWatcher::DW_ITEM_ADDED) && !owner().eventsSuspended())
{
Poco::File f(itn->second.path);
DirectoryWatcher::DirectoryEvent ev(f, DirectoryWatcher::DW_ITEM_ADDED);
owner().itemAdded(&owner(), ev);
}
}
if ((owner().eventMask() & DirectoryWatcher::DW_ITEM_REMOVED) && !owner().eventsSuspended())
{
for (ItemInfoMap::iterator it = oldEntries.begin(); it != oldEntries.end(); ++it)
{
Poco::File f(it->second.path);
DirectoryWatcher::DirectoryEvent ev(f, DirectoryWatcher::DW_ITEM_REMOVED);
owner().itemRemoved(&owner(), ev);
}
}
}
private:
DirectoryWatcherStrategy();
DirectoryWatcherStrategy(const DirectoryWatcherStrategy&);
DirectoryWatcherStrategy& operator = (const DirectoryWatcherStrategy&);
DirectoryWatcher& _owner;
};
#if POCO_OS == POCO_OS_WINDOWS_NT
class WindowsDirectoryWatcherStrategy: public DirectoryWatcherStrategy
{
public:
WindowsDirectoryWatcherStrategy(DirectoryWatcher& owner):
DirectoryWatcherStrategy(owner)
{
_hStopped = CreateEventW(NULL, FALSE, FALSE, NULL);
if (!_hStopped)
throw SystemException("cannot create event");
}
~WindowsDirectoryWatcherStrategy()
{
CloseHandle(_hStopped);
}
void run()
{
ItemInfoMap entries;
scan(entries);
DWORD filter = FILE_NOTIFY_CHANGE_FILE_NAME | FILE_NOTIFY_CHANGE_DIR_NAME;
if (owner().eventMask() & DirectoryWatcher::DW_ITEM_MODIFIED)
filter |= FILE_NOTIFY_CHANGE_SIZE | FILE_NOTIFY_CHANGE_LAST_WRITE;
std::string path(owner().directory().path());
HANDLE hChange = FindFirstChangeNotificationA(path.c_str(), FALSE, filter);
if (hChange == INVALID_HANDLE_VALUE)
{
try
{
FileImpl::handleLastErrorImpl(path);
}
catch (Poco::Exception& exc)
{
owner().scanError(&owner(), exc);
}
return;
}
bool stopped = false;
while (!stopped)
{
try
{
HANDLE h[2];
h[0] = _hStopped;
h[1] = hChange;
switch (WaitForMultipleObjects(2, h, FALSE, INFINITE))
{
case WAIT_OBJECT_0:
stopped = true;
break;
case WAIT_OBJECT_0 + 1:
{
ItemInfoMap newEntries;
scan(newEntries);
compare(entries, newEntries);
std::swap(entries, newEntries);
if (FindNextChangeNotification(hChange) == FALSE)
{
FileImpl::handleLastErrorImpl(path);
}
}
break;
default:
throw SystemException("failed to wait for directory changes");
}
}
catch (Poco::Exception& exc)
{
owner().scanError(&owner(), exc);
}
}
FindCloseChangeNotification(hChange);
}
void stop()
{
SetEvent(_hStopped);
}
bool supportsMoveEvents() const
{
return false;
}
private:
HANDLE _hStopped;
};
#elif POCO_OS == POCO_OS_LINUX || POCO_OS == POCO_OS_ANDROID
class LinuxDirectoryWatcherStrategy: public DirectoryWatcherStrategy
{
public:
LinuxDirectoryWatcherStrategy(DirectoryWatcher& owner):
DirectoryWatcherStrategy(owner),
_fd(-1),
_stopped(false)
{
_fd = inotify_init();
if (_fd == -1) throw Poco::IOException("cannot initialize inotify", errno);
}
~LinuxDirectoryWatcherStrategy()
{
close(_fd);
}
void run()
{
int mask = 0;
if (owner().eventMask() & DirectoryWatcher::DW_ITEM_ADDED)
mask |= IN_CREATE;
if (owner().eventMask() & DirectoryWatcher::DW_ITEM_REMOVED)
mask |= IN_DELETE;
if (owner().eventMask() & DirectoryWatcher::DW_ITEM_MODIFIED)
mask |= IN_MODIFY;
if (owner().eventMask() & DirectoryWatcher::DW_ITEM_MOVED_FROM)
mask |= IN_MOVED_FROM;
if (owner().eventMask() & DirectoryWatcher::DW_ITEM_MOVED_TO)
mask |= IN_MOVED_TO;
int wd = inotify_add_watch(_fd, owner().directory().path().c_str(), mask);
if (wd == -1)
{
try
{
FileImpl::handleLastErrorImpl(owner().directory().path());
}
catch (Poco::Exception& exc)
{
owner().scanError(&owner(), exc);
}
}
Poco::Buffer<char> buffer(4096);
while (!_stopped.load(std::memory_order_relaxed))
{
fd_set fds;
FD_ZERO(&fds);
FD_SET(_fd, &fds);
struct timeval tv;
tv.tv_sec = 0;
tv.tv_usec = 200000;
if (select(_fd + 1, &fds, NULL, NULL, &tv) == 1)
{
int n = read(_fd, buffer.begin(), buffer.size());
int i = 0;
if (n > 0)
{
while (n > 0)
{
struct inotify_event* event = reinterpret_cast<struct inotify_event*>(buffer.begin() + i);
if (event->len > 0)
{
if (!owner().eventsSuspended())
{
Poco::Path p(owner().directory().path());
p.makeDirectory();
p.setFileName(event->name);
Poco::File f(p.toString());
if ((event->mask & IN_CREATE) && (owner().eventMask() & DirectoryWatcher::DW_ITEM_ADDED))
{
DirectoryWatcher::DirectoryEvent ev(f, DirectoryWatcher::DW_ITEM_ADDED);
owner().itemAdded(&owner(), ev);
}
if ((event->mask & IN_DELETE) && (owner().eventMask() & DirectoryWatcher::DW_ITEM_REMOVED))
{
DirectoryWatcher::DirectoryEvent ev(f, DirectoryWatcher::DW_ITEM_REMOVED);
owner().itemRemoved(&owner(), ev);
}
if ((event->mask & IN_MODIFY) && (owner().eventMask() & DirectoryWatcher::DW_ITEM_MODIFIED))
{
DirectoryWatcher::DirectoryEvent ev(f, DirectoryWatcher::DW_ITEM_MODIFIED);
owner().itemModified(&owner(), ev);
}
if ((event->mask & IN_MOVED_FROM) && (owner().eventMask() & DirectoryWatcher::DW_ITEM_MOVED_FROM))
{
DirectoryWatcher::DirectoryEvent ev(f, DirectoryWatcher::DW_ITEM_MOVED_FROM);
owner().itemMovedFrom(&owner(), ev);
}
if ((event->mask & IN_MOVED_TO) && (owner().eventMask() & DirectoryWatcher::DW_ITEM_MOVED_TO))
{
DirectoryWatcher::DirectoryEvent ev(f, DirectoryWatcher::DW_ITEM_MOVED_TO);
owner().itemMovedTo(&owner(), ev);
}
}
}
i += sizeof(inotify_event) + event->len;
n -= sizeof(inotify_event) + event->len;
}
}
}
}
}
void stop()
{
_stopped.store(true, std::memory_order_relaxed);
}
bool supportsMoveEvents() const
{
return true;
}
private:
int _fd;
std::atomic<bool> _stopped;
};
#elif POCO_OS == POCO_OS_MAC_OS_X || POCO_OS == POCO_OS_FREE_BSD
class BSDDirectoryWatcherStrategy: public DirectoryWatcherStrategy
{
public:
BSDDirectoryWatcherStrategy(DirectoryWatcher& owner):
DirectoryWatcherStrategy(owner),
_queueFD(-1),
_dirFD(-1),
_stopped(false)
{
_dirFD = open(owner.directory().path().c_str(), O_EVTONLY);
if (_dirFD < 0) throw Poco::FileNotFoundException(owner.directory().path());
_queueFD = kqueue();
if (_queueFD < 0)
{
close(_dirFD);
throw Poco::SystemException("Cannot create kqueue", errno);
}
}
~BSDDirectoryWatcherStrategy()
{
close(_dirFD);
close(_queueFD);
}
void run()
{
Poco::Timestamp lastScan;
ItemInfoMap entries;
scan(entries);
while (!_stopped.load(std::memory_order_relaxed))
{
struct timespec timeout;
timeout.tv_sec = 0;
timeout.tv_nsec = 200000000;
unsigned eventFilter = NOTE_WRITE;
struct kevent event;
struct kevent eventData;
EV_SET(&event, _dirFD, EVFILT_VNODE, EV_ADD | EV_CLEAR, eventFilter, 0, 0);
int nEvents = kevent(_queueFD, &event, 1, &eventData, 1, &timeout);
if (nEvents < 0 || eventData.flags == EV_ERROR)
{
try
{
FileImpl::handleLastErrorImpl(owner().directory().path());
}
catch (Poco::Exception& exc)
{
owner().scanError(&owner(), exc);
}
}
else if (nEvents > 0 || ((owner().eventMask() & DirectoryWatcher::DW_ITEM_MODIFIED) && lastScan.isElapsed(owner().scanInterval()*1000000)))
{
ItemInfoMap newEntries;
scan(newEntries);
compare(entries, newEntries);
std::swap(entries, newEntries);
lastScan.update();
}
}
}
void stop()
{
_stopped.store(true, std::memory_order_relaxed);
}
bool supportsMoveEvents() const
{
return false;
}
private:
int _queueFD;
int _dirFD;
std::atomic<bool> _stopped;
};
#else
class PollingDirectoryWatcherStrategy: public DirectoryWatcherStrategy
{
public:
PollingDirectoryWatcherStrategy(DirectoryWatcher& owner):
DirectoryWatcherStrategy(owner)
{
}
~PollingDirectoryWatcherStrategy()
{
}
void run()
{
ItemInfoMap entries;
scan(entries);
while (!_stopped.tryWait(1000*owner().scanInterval()))
{
try
{
ItemInfoMap newEntries;
scan(newEntries);
compare(entries, newEntries);
std::swap(entries, newEntries);
}
catch (Poco::Exception& exc)
{
owner().scanError(&owner(), exc);
}
}
}
void stop()
{
_stopped.set();
}
bool supportsMoveEvents() const
{
return false;
}
private:
Poco::Event _stopped;
};
#endif
DirectoryWatcher::DirectoryWatcher(const std::string& path, int eventMask, int scanInterval):
_directory(path),
_eventMask(eventMask),
_scanInterval(scanInterval)
{
init();
}
DirectoryWatcher::DirectoryWatcher(const Poco::File& directory, int eventMask, int scanInterval):
_directory(directory),
_eventMask(eventMask),
_scanInterval(scanInterval)
{
init();
}
DirectoryWatcher::~DirectoryWatcher()
{
try
{
stop();
delete _pStrategy;
}
catch (...)
{
poco_unexpected();
}
}
void DirectoryWatcher::suspendEvents()
{
poco_assert (_eventsSuspended > 0);
_eventsSuspended--;
}
void DirectoryWatcher::resumeEvents()
{
_eventsSuspended++;
}
void DirectoryWatcher::init()
{
if (!_directory.exists())
throw Poco::FileNotFoundException(_directory.path());
if (!_directory.isDirectory())
throw Poco::InvalidArgumentException("not a directory", _directory.path());
#if POCO_OS == POCO_OS_WINDOWS_NT
_pStrategy = new WindowsDirectoryWatcherStrategy(*this);
#elif POCO_OS == POCO_OS_LINUX || POCO_OS == POCO_OS_ANDROID
_pStrategy = new LinuxDirectoryWatcherStrategy(*this);
#elif POCO_OS == POCO_OS_MAC_OS_X || POCO_OS == POCO_OS_FREE_BSD
_pStrategy = new BSDDirectoryWatcherStrategy(*this);
#else
_pStrategy = new PollingDirectoryWatcherStrategy(*this);
#endif
_thread.start(*this);
}
void DirectoryWatcher::run()
{
_pStrategy->run();
}
void DirectoryWatcher::stop()
{
_pStrategy->stop();
_thread.join();
}
bool DirectoryWatcher::supportsMoveEvents() const
{
return _pStrategy->supportsMoveEvents();
}
} // namespace Poco
#endif // POCO_NO_INOTIFY

21
cmake/target.cmake
View File

@ -42,10 +42,8 @@ if (CMAKE_CROSSCOMPILING)
if (ARCH_AARCH64)
# FIXME: broken dependencies
set (ENABLE_GRPC OFF CACHE INTERNAL "")
set (ENABLE_SENTRY OFF CACHE INTERNAL "")
elseif (ARCH_PPC64LE)
set (ENABLE_GRPC OFF CACHE INTERNAL "")
set (ENABLE_SENTRY OFF CACHE INTERNAL "")
elseif (ARCH_RISCV64)
# RISC-V support is preliminary
set (GLIBC_COMPATIBILITY OFF CACHE INTERNAL "")
@ -73,19 +71,10 @@ if (CMAKE_CROSSCOMPILING)
message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!")
endif ()
if (USE_MUSL)
# use of undeclared identifier 'PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP'
set (ENABLE_SENTRY OFF CACHE INTERNAL "")
set (ENABLE_ODBC OFF CACHE INTERNAL "")
set (ENABLE_GRPC OFF CACHE INTERNAL "")
set (ENABLE_HDFS OFF CACHE INTERNAL "")
set (ENABLE_EMBEDDED_COMPILER OFF CACHE INTERNAL "")
# use of drand48_data
set (ENABLE_AZURE_BLOB_STORAGE OFF CACHE INTERNAL "")
endif ()
# Don't know why but CXX_STANDARD doesn't work for cross-compilation
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20")
message (STATUS "Cross-compiling for target: ${CMAKE_CXX_COMPILE_TARGET}")
endif ()
if (USE_MUSL)
# Does not work for unknown reason
set (ENABLE_RUST OFF CACHE INTERNAL "")
endif ()

4
contrib/CMakeLists.txt vendored
View File

@ -134,9 +134,9 @@ add_contrib (libuv-cmake libuv)
add_contrib (liburing-cmake liburing)
add_contrib (amqpcpp-cmake AMQP-CPP) # requires: libuv
add_contrib (cassandra-cmake cassandra) # requires: libuv
add_contrib (curl-cmake curl)
add_contrib (azure-cmake azure) # requires: curl
if (NOT OS_DARWIN)
add_contrib (curl-cmake curl)
add_contrib (azure-cmake azure) # requires: curl
add_contrib (sentry-native-cmake sentry-native) # requires: curl
endif()
add_contrib (fmtlib-cmake fmtlib)

2
contrib/azure vendored

@ -1 +1 @@
Subproject commit 096049bf24fffafcaccc132b9367694532716731
Subproject commit 352ff0a61cb319ac1cc38c4058443ddf70147530

5
contrib/curl-cmake/CMakeLists.txt
View File

@ -10,7 +10,7 @@ set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/curl")
set (SRCS
"${LIBRARY_DIR}/lib/altsvc.c"
"${LIBRARY_DIR}/lib/amigaos.c"
"${LIBRARY_DIR}/lib/asyn-thread.c"
"${LIBRARY_DIR}/lib/asyn-ares.c"
"${LIBRARY_DIR}/lib/base64.c"
"${LIBRARY_DIR}/lib/bufq.c"
"${LIBRARY_DIR}/lib/bufref.c"
@ -165,13 +165,14 @@ target_compile_definitions (_curl PRIVATE
libcurl_EXPORTS
OS="${CMAKE_SYSTEM_NAME}"
)
target_include_directories (_curl SYSTEM PUBLIC
"${LIBRARY_DIR}/include"
"${LIBRARY_DIR}/lib"
. # curl_config.h
)
target_link_libraries (_curl PRIVATE OpenSSL::SSL)
target_link_libraries (_curl PRIVATE OpenSSL::SSL ch_contrib::c-ares)
# The library is large - avoid bloat (XXX: is it?)
if (OMIT_HEAVY_DEBUG_SYMBOLS)

1
contrib/curl-cmake/curl_config.h
View File

@ -50,3 +50,4 @@
#define ENABLE_IPV6
#define USE_OPENSSL
#define USE_THREADS_POSIX
#define USE_ARES

2
contrib/libhdfs3 vendored

@ -1 +1 @@
Subproject commit bdcb91354b1c05b21e73043a112a6f1e3b013497
Subproject commit b9598e6016720a7c088bfe85ce1fa0410f9d2103

5
contrib/libhdfs3-cmake/CMakeLists.txt
View File

@ -26,6 +26,11 @@ ADD_DEFINITIONS(-D__STDC_FORMAT_MACROS)
ADD_DEFINITIONS(-D_GNU_SOURCE)
ADD_DEFINITIONS(-D_GLIBCXX_USE_NANOSLEEP)
ADD_DEFINITIONS(-DHAVE_NANOSLEEP)
if (USE_MUSL)
ADD_DEFINITIONS(-DSTRERROR_R_RETURN_INT)
endif ()
set(HAVE_STEADY_CLOCK 1)
set(HAVE_NESTED_EXCEPTION 1)
SET(HAVE_BOOST_CHRONO 0)

4
contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h
View File

@ -270,7 +270,7 @@ XMLPUBFUN void XMLCALL xmlCheckVersion(int version);
*
* Whether iconv support is available
*/
#if 1
#if 0
#define LIBXML_ICONV_ENABLED
#endif
@ -499,5 +499,3 @@ XMLPUBFUN void XMLCALL xmlCheckVersion(int version);
}
#endif /* __cplusplus */
#endif

2
contrib/llvm-project vendored

@ -1 +1 @@
Subproject commit e7b8befca85c8b847614432dba250c22d35fbae0
Subproject commit 1834e42289c58402c804a87be4d489892b88f3ec

2
contrib/rocksdb-cmake/CMakeLists.txt
View File

@ -117,7 +117,7 @@ endif()
add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
if (OS_LINUX OR OS_FREEBSD)
if ((OS_LINUX OR OS_FREEBSD) AND NOT USE_MUSL)
add_definitions(-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX)
endif()

2
contrib/sentry-native vendored

@ -1 +1 @@
Subproject commit ae10fb8c224c3f41571446e1ed7fd57b9e5e366b
Subproject commit bc359f86cbf0f73f6fd4b6bfb4ede0c1f8c9400f

3
contrib/sentry-native-cmake/CMakeLists.txt
View File

@ -13,6 +13,7 @@ set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/sentry-native")
set (SRCS
${SRC_DIR}/vendor/mpack.c
${SRC_DIR}/vendor/stb_sprintf.c
${SRC_DIR}/src/sentry_alloc.c
${SRC_DIR}/src/sentry_backend.c
${SRC_DIR}/src/sentry_core.c
@ -21,6 +22,7 @@ set (SRCS
${SRC_DIR}/src/sentry_json.c
${SRC_DIR}/src/sentry_logger.c
${SRC_DIR}/src/sentry_options.c
${SRC_DIR}/src/sentry_os.c
${SRC_DIR}/src/sentry_random.c
${SRC_DIR}/src/sentry_ratelimiter.c
${SRC_DIR}/src/sentry_scope.c
@ -29,6 +31,7 @@ set (SRCS
${SRC_DIR}/src/sentry_string.c
${SRC_DIR}/src/sentry_sync.c
${SRC_DIR}/src/sentry_transport.c
${SRC_DIR}/src/sentry_tracing.c
${SRC_DIR}/src/sentry_utils.c
${SRC_DIR}/src/sentry_uuid.c
${SRC_DIR}/src/sentry_value.c

4
contrib/unixodbc-cmake/CMakeLists.txt
View File

@ -1,7 +1,7 @@
option (ENABLE_ODBC "Enable ODBC library" ${ENABLE_LIBRARIES})
if (NOT OS_LINUX)
if (NOT OS_LINUX OR USE_MUSL)
if (ENABLE_ODBC)
message(STATUS "ODBC is only supported on Linux")
message(STATUS "ODBC is only supported on Linux with dynamic linking")
endif()
set (ENABLE_ODBC OFF CACHE INTERNAL "")
endif ()

9
docker/packager/packager
View File

@ -145,6 +145,7 @@ def parse_env_variables(
RISCV_SUFFIX = "-riscv64"
S390X_SUFFIX = "-s390x"
AMD64_COMPAT_SUFFIX = "-amd64-compat"
AMD64_MUSL_SUFFIX = "-amd64-musl"
result = []
result.append("OUTPUT_DIR=/output")
@ -163,6 +164,7 @@ def parse_env_variables(
is_cross_s390x = compiler.endswith(S390X_SUFFIX)
is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
is_amd64_compat = compiler.endswith(AMD64_COMPAT_SUFFIX)
is_amd64_musl = compiler.endswith(AMD64_MUSL_SUFFIX)
if is_cross_darwin:
cc = compiler[: -len(DARWIN_SUFFIX)]
@ -232,6 +234,12 @@ def parse_env_variables(
cc = compiler[: -len(AMD64_COMPAT_SUFFIX)]
result.append("DEB_ARCH=amd64")
cmake_flags.append("-DNO_SSE3_OR_HIGHER=1")
elif is_amd64_musl:
cc = compiler[: -len(AMD64_MUSL_SUFFIX)]
result.append("DEB_ARCH=amd64")
cmake_flags.append(
"-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake"
)
else:
cc = compiler
result.append("DEB_ARCH=amd64")
@ -396,6 +404,7 @@ def parse_args() -> argparse.Namespace:
"clang-17-riscv64",
"clang-17-s390x",
"clang-17-amd64-compat",
"clang-17-amd64-musl",
"clang-17-freebsd",
),
default="clang-17",

4
docker/test/fasttest/run.sh
View File

@ -16,7 +16,7 @@ export LLVM_VERSION=${LLVM_VERSION:-17}
# it being undefined. Also read it as array so that we can pass an empty list
# of additional variable to cmake properly, and it doesn't generate an extra
# empty parameter.
# Read it as CMAKE_FLAGS to not lose exported FASTTEST_CMAKE_FLAGS on subsequential launch
# Read it as CMAKE_FLAGS to not lose exported FASTTEST_CMAKE_FLAGS on subsequent launch
read -ra CMAKE_FLAGS <<< "${FASTTEST_CMAKE_FLAGS:-}"
# Run only matching tests.
@ -197,7 +197,7 @@ function run_cmake
(
cd "$FASTTEST_BUILD"
cmake "$FASTTEST_SOURCE" -DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" "${CMAKE_LIBS_CONFIG[@]}" "${CMAKE_FLAGS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/cmake_log.txt"
cmake "$FASTTEST_SOURCE" -DCMAKE_CXX_COMPILER="clang++-${LLVM_VERSION}" -DCMAKE_C_COMPILER="clang-${LLVM_VERSION}" -DCMAKE_TOOLCHAIN_FILE="${FASTTEST_SOURCE}/cmake/linux/toolchain-x86_64-musl.cmake" "${CMAKE_LIBS_CONFIG[@]}" "${CMAKE_FLAGS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/cmake_log.txt"
)
}

8
docs/en/development/build-cross-osx.md
View File

@ -28,18 +28,20 @@ sudo apt-get install clang-17
Let’s remember the path where we install `cctools` as ${CCTOOLS}
``` bash
mkdir ~/cctools
export CCTOOLS=$(cd ~/cctools && pwd)
mkdir ${CCTOOLS}
cd ${CCTOOLS}
git clone --depth=1 https://github.com/tpoechtrager/apple-libtapi.git
git clone https://github.com/tpoechtrager/apple-libtapi.git
cd apple-libtapi
git checkout 15dfc2a8c9a2a89d06ff227560a69f5265b692f9
INSTALLPREFIX=${CCTOOLS} ./build.sh
./install.sh
cd ..
git clone --depth=1 https://github.com/tpoechtrager/cctools-port.git
git clone https://github.com/tpoechtrager/cctools-port.git
cd cctools-port/cctools
git checkout 2a3e1c2a6ff54a30f898b70cfb9ba1692a55fad7
./configure --prefix=$(readlink -f ${CCTOOLS}) --with-libtapi=$(readlink -f ${CCTOOLS}) --target=x86_64-apple-darwin
make install
```

1
docs/en/operations/utilities/clickhouse-local.md
View File

@ -216,7 +216,6 @@ Arguments:
- `--logger.level` — Log level.
- `--ignore-error` — do not stop processing if a query failed.
- `-c`, `--config-file` — path to configuration file in same format as for ClickHouse server, by default the configuration empty.
- `--no-system-tables` — do not attach system tables.
- `--help` — arguments references for `clickhouse-local`.
- `-V`, `--version` — print version information and exit.

4
docs/en/sql-reference/functions/hash-functions.md
View File

@ -319,9 +319,9 @@ This is a relatively fast non-cryptographic hash function of average quality for
Calculates a 64-bit hash code from any type of integer.
It works faster than intHash32. Average quality.
## SHA1, SHA224, SHA256, SHA512
## SHA1, SHA224, SHA256, SHA512, SHA512_256
Calculates SHA-1, SHA-224, SHA-256, SHA-512 hash from a string and returns the resulting set of bytes as [FixedString](/docs/en/sql-reference/data-types/fixedstring.md).
Calculates SHA-1, SHA-224, SHA-256, SHA-512, SHA-512-256 hash from a string and returns the resulting set of bytes as [FixedString](/docs/en/sql-reference/data-types/fixedstring.md).
**Syntax**

147
docs/en/sql-reference/statements/create/view.md
View File

@ -90,152 +90,11 @@ Views look the same as normal tables. For example, they are listed in the result
To delete a view, use [DROP VIEW](../../../sql-reference/statements/drop.md#drop-view). Although `DROP TABLE` works for VIEWs as well.
## Live View [Experimental]
## Live View [Deprecated]
:::note
This is an experimental feature that may change in backwards-incompatible ways in the future releases. Enable usage of live views and `WATCH` query using [allow_experimental_live_view](../../../operations/settings/settings.md#allow-experimental-live-view) setting. Input the command `set allow_experimental_live_view = 1`.
:::
This feature is deprecated and will be removed in the future.
```sql
CREATE LIVE VIEW [IF NOT EXISTS] [db.]table_name [WITH REFRESH [value_in_sec]] AS SELECT ...
```
Live views store result of the corresponding [SELECT](../../../sql-reference/statements/select/index.md) query and are updated any time the result of the query changes. Query result as well as partial result needed to combine with new data are stored in memory providing increased performance for repeated queries. Live views can provide push notifications when query result changes using the [WATCH](../../../sql-reference/statements/watch.md) query.
Live views are triggered by insert into the innermost table specified in the query.
Live views work similarly to how a query in a distributed table works. But instead of combining partial results from different servers they combine partial result from current data with partial result from the new data. When a live view query includes a subquery then the cached partial result is only stored for the innermost subquery.
:::info
- [Table function](../../../sql-reference/table-functions/index.md) is not supported as the innermost table.
- Tables that do not have inserts such as a [dictionary](../../../sql-reference/dictionaries/index.md), [system table](../../../operations/system-tables/index.md), a [normal view](#normal), or a [materialized view](#materialized) will not trigger a live view.
- Only queries where one can combine partial result from the old data plus partial result from the new data will work. Live view will not work for queries that require the complete data set to compute the final result or aggregations where the state of the aggregation must be preserved.
- Does not work with replicated or distributed tables where inserts are performed on different nodes.
- Can't be triggered by multiple tables.
See [WITH REFRESH](#live-view-with-refresh) to force periodic updates of a live view that in some cases can be used as a workaround.
:::
### Monitoring Live View Changes
You can monitor changes in the `LIVE VIEW` query result using [WATCH](../../../sql-reference/statements/watch.md) query.
```sql
WATCH [db.]live_view
```
**Example:**
```sql
CREATE TABLE mt (x Int8) Engine = MergeTree ORDER BY x;
CREATE LIVE VIEW lv AS SELECT sum(x) FROM mt;
```
Watch a live view while doing a parallel insert into the source table.
```sql
WATCH lv;
```
```bash
┌─sum(x)─┬─_version─┐
│ 1 │ 1 │
└────────┴──────────┘
┌─sum(x)─┬─_version─┐
│ 3 │ 2 │
└────────┴──────────┘
┌─sum(x)─┬─_version─┐
│ 6 │ 3 │
└────────┴──────────┘
```
```sql
INSERT INTO mt VALUES (1);
INSERT INTO mt VALUES (2);
INSERT INTO mt VALUES (3);
```
Or add [EVENTS](../../../sql-reference/statements/watch.md#events-clause) clause to just get change events.
```sql
WATCH [db.]live_view EVENTS;
```
**Example:**
```sql
WATCH lv EVENTS;
```
```bash
┌─version─┐
│ 1 │
└─────────┘
┌─version─┐
│ 2 │
└─────────┘
┌─version─┐
│ 3 │
└─────────┘
```
You can execute [SELECT](../../../sql-reference/statements/select/index.md) query on a live view in the same way as for any regular view or a table. If the query result is cached it will return the result immediately without running the stored query on the underlying tables.
```sql
SELECT * FROM [db.]live_view WHERE ...
```
### Force Live View Refresh
You can force live view refresh using the `ALTER LIVE VIEW [db.]table_name REFRESH` statement.
### WITH REFRESH Clause
When a live view is created with a `WITH REFRESH` clause then it will be automatically refreshed after the specified number of seconds elapse since the last refresh or trigger.
```sql
CREATE LIVE VIEW [db.]table_name WITH REFRESH [value_in_sec] AS SELECT ...
```
If the refresh value is not specified then the value specified by the [periodic_live_view_refresh](../../../operations/settings/settings.md#periodic-live-view-refresh) setting is used.
**Example:**
```sql
CREATE LIVE VIEW lv WITH REFRESH 5 AS SELECT now();
WATCH lv
```
```bash
┌───────────────now()─┬─_version─┐
│ 2021-02-21 08:47:05 │ 1 │
└─────────────────────┴──────────┘
┌───────────────now()─┬─_version─┐
│ 2021-02-21 08:47:10 │ 2 │
└─────────────────────┴──────────┘
┌───────────────now()─┬─_version─┐
│ 2021-02-21 08:47:15 │ 3 │
└─────────────────────┴──────────┘
```
```sql
WATCH lv
```
```
Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.lv does not exist..
```
### Live View Usage
Most common uses of live view tables include:
- Providing push notifications for query result changes to avoid polling.
- Caching results of most frequent queries to provide immediate query results.
- Watching for table changes and triggering a follow-up select queries.
- Watching metrics from system tables using periodic refresh.
**See Also**
- [ALTER LIVE VIEW](../alter/view.md#alter-live-view)
For your convenience, the old documentation is located [here](https://pastila.nl/?00f32652/fdf07272a7b54bda7e13b919264e449f.md)
## Window View [Experimental]

1
docs/ru/operations/utilities/clickhouse-local.md
View File

@ -45,7 +45,6 @@ $ clickhouse-local --structure "table_structure" --input-format "format_of_incom
- `--logger.level` — уровень логирования.
- `--ignore-error` — не прекращать обработку если запрос выдал ошибку.
- `-c`, `--config-file` — путь к файлу конфигурации. По умолчанию `clickhouse-local` запускается с пустой конфигурацией. Конфигурационный файл имеет тот же формат, что и для сервера ClickHouse, и в нём можно использовать все конфигурационные параметры сервера. Обычно подключение конфигурации не требуется; если требуется установить отдельный параметр, то это можно сделать ключом с именем параметра.
- `--no-system-tables` — запуск без использования системных таблиц.
- `--help` — вывод справочной информации о `clickhouse-local`.
- `-V`, `--version` — вывод текущей версии и выход.

1
docs/zh/operations/utilities/clickhouse-local.md
View File

@ -45,7 +45,6 @@ clickhouse-local --structure "table_structure" --input-format "format_of_incomin
- `--logger.level` — 日志级别。
- `--ignore-error` — 当查询失败时，不停止处理。
- `-c`, `--config-file` — 与ClickHouse服务器格式相同配置文件的路径，默认情况下配置为空。
- `--no-system-tables` — 不附加系统表。
- `--help` — `clickhouse-local`使用帮助信息。
- `-V`, `--version` — 打印版本信息并退出。

7
programs/keeper/Keeper.cpp
View File

@ -328,6 +328,13 @@ try
config().getUInt("max_thread_pool_free_size", 1000),
config().getUInt("thread_pool_queue_size", 10000)
);
/// Wait for all threads to avoid possible use-after-free (for example logging objects can be already destroyed).
SCOPE_EXIT({
Stopwatch watch;
LOG_INFO(log, "Waiting for background threads");
GlobalThreadPool::instance().shutdown();
LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds());
});
static ServerErrorHandler error_handler;
Poco::ErrorHandler::set(&error_handler);

9
programs/local/LocalServer.cpp
View File

@ -744,7 +744,7 @@ void LocalServer::processConfig()
LOG_DEBUG(log, "Loading metadata from {}", path);
auto startup_system_tasks = loadMetadataSystem(global_context);
attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
attachSystemTablesLocal</* lazy= */ true>(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
waitLoad(TablesLoaderForegroundPoolId, startup_system_tasks);
@ -761,9 +761,9 @@ void LocalServer::processConfig()
LOG_DEBUG(log, "Loaded metadata.");
}
else if (!config().has("no-system-tables"))
else
{
attachSystemTablesLocal(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
attachSystemTablesLocal</* lazy= */ true>(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA));
attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE));
}
@ -842,7 +842,6 @@ void LocalServer::addOptions(OptionsDescription & options_description)
("logger.log", po::value<std::string>(), "Log file name")
("logger.level", po::value<std::string>(), "Log level")
("no-system-tables", "do not attach system tables (better startup time)")
("path", po::value<std::string>(), "Storage path")
("only-system-tables", "attach only system tables from specified path")
("top_level_domains_path", po::value<std::string>(), "Path to lists with custom TLDs")
@ -871,8 +870,6 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp
config().setString("table-file", options["file"].as<std::string>());
if (options.count("structure"))
config().setString("table-structure", options["structure"].as<std::string>());
if (options.count("no-system-tables"))
config().setBool("no-system-tables", true);
if (options.count("only-system-tables"))
config().setBool("only-system-tables", true);
if (options.count("database"))

125
programs/server/Server.cpp
View File

@ -657,6 +657,11 @@ try
CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision());
CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger());
Poco::ThreadPool server_pool(3, server_settings.max_connections);
std::mutex servers_lock;
std::vector<ProtocolServerAdapter> servers;
std::vector<ProtocolServerAdapter> servers_to_start_before_tables;
/** Context contains all that query execution is dependent:
* settings, available functions, data types, aggregate functions, databases, ...
*/
@ -697,6 +702,68 @@ try
server_settings.max_thread_pool_size,
server_settings.max_thread_pool_free_size,
server_settings.thread_pool_queue_size);
/// Wait for all threads to avoid possible use-after-free (for example logging objects can be already destroyed).
SCOPE_EXIT({
Stopwatch watch;
LOG_INFO(log, "Waiting for background threads");
GlobalThreadPool::instance().shutdown();
LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds());
});
/// NOTE: global context should be destroyed *before* GlobalThreadPool::shutdown()
/// Otherwise GlobalThreadPool::shutdown() will hang, since Context holds some threads.
SCOPE_EXIT({
/** Ask to cancel background jobs all table engines,
* and also query_log.
* It is important to do early, not in destructor of Context, because
* table engines could use Context on destroy.
*/
LOG_INFO(log, "Shutting down storages.");
global_context->shutdown();
LOG_DEBUG(log, "Shut down storages.");
if (!servers_to_start_before_tables.empty())
{
LOG_DEBUG(log, "Waiting for current connections to servers for tables to finish.");
size_t current_connections = 0;
{
std::lock_guard lock(servers_lock);
for (auto & server : servers_to_start_before_tables)
{
server.stop();
current_connections += server.currentConnections();
}
}
if (current_connections)
LOG_INFO(log, "Closed all listening sockets. Waiting for {} outstanding connections.", current_connections);
else
LOG_INFO(log, "Closed all listening sockets.");
if (current_connections > 0)
current_connections = waitServersToFinish(servers_to_start_before_tables, servers_lock, server_settings.shutdown_wait_unfinished);
if (current_connections)
LOG_INFO(log, "Closed connections to servers for tables. But {} remain. Probably some tables of other users cannot finish their connections after context shutdown.", current_connections);
else
LOG_INFO(log, "Closed connections to servers for tables.");
}
global_context->shutdownKeeperDispatcher();
/// Wait server pool to avoid use-after-free of destroyed context in the handlers
server_pool.joinAll();
/** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available.
* At this moment, no one could own shared part of Context.
*/
global_context.reset();
shared_context.reset();
LOG_DEBUG(log, "Destroyed global context.");
});
#if USE_AZURE_BLOB_STORAGE
/// It makes sense to deinitialize libxml after joining of all threads
@ -755,10 +822,6 @@ try
}
}
Poco::ThreadPool server_pool(3, server_settings.max_connections);
std::mutex servers_lock;
std::vector<ProtocolServerAdapter> servers;
std::vector<ProtocolServerAdapter> servers_to_start_before_tables;
/// This object will periodically calculate some metrics.
ServerAsynchronousMetrics async_metrics(
global_context,
@ -1598,60 +1661,6 @@ try
/// try set up encryption. There are some errors in config, error will be printed and server wouldn't start.
CompressionCodecEncrypted::Configuration::instance().load(config(), "encryption_codecs");
SCOPE_EXIT({
async_metrics.stop();
/** Ask to cancel background jobs all table engines,
* and also query_log.
* It is important to do early, not in destructor of Context, because
* table engines could use Context on destroy.
*/
LOG_INFO(log, "Shutting down storages.");
global_context->shutdown();
LOG_DEBUG(log, "Shut down storages.");
if (!servers_to_start_before_tables.empty())
{
LOG_DEBUG(log, "Waiting for current connections to servers for tables to finish.");
size_t current_connections = 0;
{
std::lock_guard lock(servers_lock);
for (auto & server : servers_to_start_before_tables)
{
server.stop();
current_connections += server.currentConnections();
}
}
if (current_connections)
LOG_INFO(log, "Closed all listening sockets. Waiting for {} outstanding connections.", current_connections);
else
LOG_INFO(log, "Closed all listening sockets.");
if (current_connections > 0)
current_connections = waitServersToFinish(servers_to_start_before_tables, servers_lock, server_settings.shutdown_wait_unfinished);
if (current_connections)
LOG_INFO(log, "Closed connections to servers for tables. But {} remain. Probably some tables of other users cannot finish their connections after context shutdown.", current_connections);
else
LOG_INFO(log, "Closed connections to servers for tables.");
global_context->shutdownKeeperDispatcher();
}
/// Wait server pool to avoid use-after-free of destroyed context in the handlers
server_pool.joinAll();
/** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available.
* At this moment, no one could own shared part of Context.
*/
global_context.reset();
shared_context.reset();
LOG_DEBUG(log, "Destroyed global context.");
});
/// DNSCacheUpdater uses BackgroundSchedulePool which lives in shared context
/// and thus this object must be created after the SCOPE_EXIT object where shared
/// context is destroyed.

2
src/AggregateFunctions/AggregateFunctionAny.cpp
View File

@ -110,7 +110,7 @@ public:
}
}
}
else
else if (row_begin < row_end)
{
size_t pos = First ? row_begin : row_end - 1;
add(place, columns, pos, arena);

6
src/AggregateFunctions/IAggregateFunction.h
View File

@ -549,8 +549,10 @@ public:
auto to = std::lower_bound(offsets.begin(), offsets.end(), row_end) - offsets.begin() + 1;
size_t num_defaults = (row_end - row_begin) - (to - from);
static_cast<const Derived *>(this)->addBatchSinglePlace(from, to, place, &values, arena, -1);
static_cast<const Derived *>(this)->addManyDefaults(place, &values, num_defaults, arena);
if (from < to)
static_cast<const Derived *>(this)->addBatchSinglePlace(from, to, place, &values, arena, -1);
if (num_defaults > 0)
static_cast<const Derived *>(this)->addManyDefaults(place, &values, num_defaults, arena);
}
void addBatchSinglePlaceNotNull( /// NOLINT

9
src/Client/LineReader.cpp
View File

@ -7,7 +7,7 @@
#include <cassert>
#include <cstring>
#include <unistd.h>
#include <sys/select.h>
#include <poll.h>
#include <sys/time.h>
#include <sys/types.h>
@ -27,11 +27,8 @@ void trim(String & s)
/// Allows delaying the start of query execution until the entirety of query is inserted.
bool hasInputData()
{
timeval timeout = {0, 0};
fd_set fds{};
FD_ZERO(&fds);
FD_SET(STDIN_FILENO, &fds);
return select(1, &fds, nullptr, nullptr, &timeout) == 1;
pollfd fd{STDIN_FILENO, POLLIN, 0};
return poll(&fd, 1, 0) == 1;
}
struct NoCaseCompare

1
src/Common/ProfileEvents.cpp
View File

@ -238,6 +238,7 @@
M(DictCacheLockReadNs, "Number of nanoseconds spend in waiting for read lock to lookup the data for the dictionaries of 'cache' types.") \
\
M(DistributedSyncInsertionTimeoutExceeded, "A timeout has exceeded while waiting for shards during synchronous insertion into a Distributed table (with 'distributed_foreground_insert' = 1)") \
M(DistributedAsyncInsertionFailures, "Number of failures for asynchronous insertion into a Distributed table (with 'distributed_foreground_insert' = 0)") \
M(DataAfterMergeDiffersFromReplica, R"(
Number of times data after merge is not byte-identical to the data on another replicas. There could be several reasons:
1. Using newer version of compression library after server update.

7
src/Common/ThreadPool.cpp
View File

@ -500,3 +500,10 @@ GlobalThreadPool & GlobalThreadPool::instance()
return *the_instance;
}
void GlobalThreadPool::shutdown()
{
if (the_instance)
{
the_instance->finalize();
}
}

3
src/Common/ThreadPool.h
View File

@ -109,6 +109,8 @@ public:
void addOnDestroyCallback(OnDestroyCallback && callback);
private:
friend class GlobalThreadPool;
mutable std::mutex mutex;
std::condition_variable job_finished;
std::condition_variable new_job_or_shutdown;
@ -205,6 +207,7 @@ class GlobalThreadPool : public FreeThreadPool, private boost::noncopyable
public:
static void initialize(size_t max_threads = 10000, size_t max_free_threads = 1000, size_t queue_size = 10000);
static GlobalThreadPool & instance();
static void shutdown();
};

73
src/Common/mysqlxx/Pool.cpp
View File

@ -52,8 +52,7 @@ void Pool::Entry::decrementRefCount()
Pool::Pool(const Poco::Util::AbstractConfiguration & cfg, const std::string & config_name,
unsigned default_connections_, unsigned max_connections_,
const char * parent_config_name_)
: logger(Poco::Logger::get("mysqlxx::Pool"))
, default_connections(default_connections_)
: default_connections(default_connections_)
, max_connections(max_connections_)
{
server = cfg.getString(config_name + ".host");
@ -127,6 +126,38 @@ Pool::Pool(const Poco::Util::AbstractConfiguration & cfg, const std::string & co
}
Pool::Pool(
const std::string & db_,
const std::string & server_,
const std::string & user_,
const std::string & password_,
unsigned port_,
const std::string & socket_,
unsigned connect_timeout_,
unsigned rw_timeout_,
unsigned default_connections_,
unsigned max_connections_,
unsigned enable_local_infile_,
bool opt_reconnect_)
: default_connections(default_connections_)
, max_connections(max_connections_)
, db(db_)
, server(server_)
, user(user_)
, password(password_)
, port(port_)
, socket(socket_)
, connect_timeout(connect_timeout_)
, rw_timeout(rw_timeout_)
, enable_local_infile(enable_local_infile_)
, opt_reconnect(opt_reconnect_)
{
LOG_DEBUG(log,
"Created MySQL Pool with settings: connect_timeout={}, read_write_timeout={}, default_connections_number={}, max_connections_number={}",
connect_timeout, rw_timeout, default_connections, max_connections);
}
Pool::~Pool()
{
std::lock_guard lock(mutex);
@ -148,29 +179,29 @@ Pool::Entry Pool::get(uint64_t wait_timeout)
initialize();
for (;;)
{
logger.trace("(%s): Iterating through existing MySQL connections", getDescription());
LOG_TRACE(log, "{}: Iterating through existing MySQL connections", getDescription());
for (auto & connection : connections)
{
if (connection->ref_count == 0)
{
logger.test("Found free connection in pool, returning it to the caller");
LOG_TEST(log, "Found free connection in pool, returning it to the caller");
return Entry(connection, this);
}
}
logger.trace("(%s): Trying to allocate a new connection.", getDescription());
LOG_TRACE(log, "{}: Trying to allocate a new connection.", getDescription());
if (connections.size() < static_cast<size_t>(max_connections))
{
Connection * conn = allocConnection();
if (conn)
return Entry(conn, this);
logger.trace("(%s): Unable to create a new connection: Allocation failed.", getDescription());
LOG_TRACE(log, "{}: Unable to create a new connection: Allocation failed.", getDescription());
}
else
{
logger.trace("(%s): Unable to create a new connection: Max number of connections has been reached.", getDescription());
LOG_TRACE(log, "{}: Unable to create a new connection: Max number of connections has been reached.", getDescription());
}
if (!wait_timeout)
@ -180,7 +211,7 @@ Pool::Entry Pool::get(uint64_t wait_timeout)
throw Poco::Exception("mysqlxx::Pool is full (connection_wait_timeout is exceeded)");
lock.unlock();
logger.trace("(%s): Sleeping for %d seconds.", getDescription(), MYSQLXX_POOL_SLEEP_ON_CONNECT_FAIL);
LOG_TRACE(log, "{}: Sleeping for {} seconds.", getDescription(), MYSQLXX_POOL_SLEEP_ON_CONNECT_FAIL);
sleepForSeconds(MYSQLXX_POOL_SLEEP_ON_CONNECT_FAIL);
lock.lock();
}
@ -206,7 +237,7 @@ Pool::Entry Pool::tryGet()
return res;
}
logger.debug("(%s): Idle connection to MySQL server cannot be recovered, dropping it.", getDescription());
LOG_DEBUG(log, "{}: Idle connection to MySQL server cannot be recovered, dropping it.", getDescription());
/// This one is disconnected, cannot be reestablished and so needs to be disposed of.
connection_it = connections.erase(connection_it);
@ -229,7 +260,7 @@ Pool::Entry Pool::tryGet()
void Pool::removeConnection(Connection* connection)
{
logger.trace("(%s): Removing connection.", getDescription());
LOG_TRACE(log, "{}: Removing connection.", getDescription());
std::lock_guard lock(mutex);
if (connection)
@ -260,8 +291,8 @@ void Pool::Entry::forceConnected() const
else
sleepForSeconds(MYSQLXX_POOL_SLEEP_ON_CONNECT_FAIL);
pool->logger.debug(
"Creating a new MySQL connection to %s with settings: connect_timeout=%u, read_write_timeout=%u",
LOG_DEBUG(pool->log,
"Creating a new MySQL connection to {} with settings: connect_timeout={}, read_write_timeout={}",
pool->description, pool->connect_timeout, pool->rw_timeout);
data->conn.connect(
@ -287,21 +318,21 @@ bool Pool::Entry::tryForceConnected() const
auto * const mysql_driver = data->conn.getDriver();
const auto prev_connection_id = mysql_thread_id(mysql_driver);
pool->logger.trace("Entry(connection %lu): sending PING to check if it is alive.", prev_connection_id);
LOG_TRACE(pool->log, "Entry(connection {}): sending PING to check if it is alive.", prev_connection_id);
if (data->conn.ping()) /// Attempts to reestablish lost connection
{
const auto current_connection_id = mysql_thread_id(mysql_driver);
if (prev_connection_id != current_connection_id)
{
pool->logger.debug("Entry(connection %lu): Reconnected to MySQL server. Connection id changed: %lu -> %lu",
current_connection_id, prev_connection_id, current_connection_id);
LOG_DEBUG(pool->log, "Entry(connection {}): Reconnected to MySQL server. Connection id changed: {} -> {}",
current_connection_id, prev_connection_id, current_connection_id);
}
pool->logger.trace("Entry(connection %lu): PING ok.", current_connection_id);
LOG_TRACE(pool->log, "Entry(connection {}): PING ok.", current_connection_id);
return true;
}
pool->logger.trace("Entry(connection %lu): PING failed.", prev_connection_id);
LOG_TRACE(pool->log, "Entry(connection {}): PING failed.", prev_connection_id);
return false;
}
@ -326,10 +357,10 @@ Pool::Connection * Pool::allocConnection(bool dont_throw_if_failed_first_time)
try
{
logger.debug("Connecting to %s", description);
LOG_DEBUG(log, "Connecting to {}", description);
logger.debug(
"Creating a new MySQL connection to %s with settings: connect_timeout=%u, read_write_timeout=%u",
LOG_DEBUG(log,
"Creating a new MySQL connection to {} with settings: connect_timeout={}, read_write_timeout={}",
description, connect_timeout, rw_timeout);
conn_ptr->conn.connect(
@ -349,7 +380,7 @@ Pool::Connection * Pool::allocConnection(bool dont_throw_if_failed_first_time)
}
catch (mysqlxx::ConnectionFailed & e)
{
logger.error(e.what());
LOG_ERROR(log, "Failed to connect to MySQL ({}): {}", description, e.what());
if ((!was_successful && !dont_throw_if_failed_first_time)
|| e.errnum() == ER_ACCESS_DENIED_ERROR

24
src/Common/mysqlxx/mysqlxx/Pool.h
View File

@ -169,28 +169,10 @@ public:
unsigned default_connections_ = MYSQLXX_POOL_DEFAULT_START_CONNECTIONS,
unsigned max_connections_ = MYSQLXX_POOL_DEFAULT_MAX_CONNECTIONS,
unsigned enable_local_infile_ = MYSQLXX_DEFAULT_ENABLE_LOCAL_INFILE,
bool opt_reconnect_ = MYSQLXX_DEFAULT_MYSQL_OPT_RECONNECT)
: logger(Poco::Logger::get("mysqlxx::Pool"))
, default_connections(default_connections_)
, max_connections(max_connections_)
, db(db_)
, server(server_)
, user(user_)
, password(password_)
, port(port_)
, socket(socket_)
, connect_timeout(connect_timeout_)
, rw_timeout(rw_timeout_)
, enable_local_infile(enable_local_infile_)
, opt_reconnect(opt_reconnect_)
{
logger.debug(
"Created MySQL Pool with settings: connect_timeout=%u, read_write_timeout=%u, default_connections_number=%u, max_connections_number=%u",
connect_timeout, rw_timeout, default_connections, max_connections);
}
bool opt_reconnect_ = MYSQLXX_DEFAULT_MYSQL_OPT_RECONNECT);
Pool(const Pool & other)
: logger(other.logger), default_connections{other.default_connections},
: default_connections{other.default_connections},
max_connections{other.max_connections},
db{other.db}, server{other.server},
user{other.user}, password{other.password},
@ -220,7 +202,7 @@ public:
void removeConnection(Connection * connection);
protected:
Poco::Logger & logger;
Poco::Logger * log = &Poco::Logger::get("mysqlxx::Pool");
/// Number of MySQL connections which are created at launch.
unsigned default_connections;

1
src/Coordination/CoordinationSettings.h
View File

@ -43,6 +43,7 @@ struct Settings;
M(UInt64, max_requests_batch_bytes_size, 100*1024, "Max size in bytes of batch of requests that can be sent to RAFT", 0) \
M(UInt64, max_flush_batch_size, 1000, "Max size of batch of requests that can be flushed together", 0) \
M(UInt64, max_requests_quick_batch_size, 100, "Max size of batch of requests to try to get before proceeding with RAFT. Keeper will not wait for requests but take only requests that are already in queue" , 0) \
M(UInt64, max_memory_usage_soft_limit, 0, "Soft limit in bytes of keeper memory usage", 0) \
M(Bool, quorum_reads, false, "Execute read requests as writes through whole RAFT consesus with similar speed", 0) \
M(Bool, force_sync, true, "Call fsync on each change in RAFT changelog", 0) \
M(Bool, compress_logs, false, "Write compressed coordination logs in ZSTD format", 0) \

6
src/Coordination/KeeperAsynchronousMetrics.cpp
View File

@ -113,6 +113,12 @@ KeeperAsynchronousMetrics::KeeperAsynchronousMetrics(
{
}
KeeperAsynchronousMetrics::~KeeperAsynchronousMetrics()
{
/// NOTE: stop() from base class is not enough, since this leads to leak on vptr
stop();
}
void KeeperAsynchronousMetrics::updateImpl(AsynchronousMetricValues & new_values, TimePoint /*update_time*/, TimePoint /*current_time*/)
{
#if USE_NURAFT

1
src/Coordination/KeeperAsynchronousMetrics.h
View File

@ -14,6 +14,7 @@ class KeeperAsynchronousMetrics : public AsynchronousMetrics
public:
KeeperAsynchronousMetrics(
ContextPtr context_, int update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_);
~KeeperAsynchronousMetrics() override;
private:
ContextPtr context;

58
src/Coordination/KeeperDispatcher.cpp
View File

@ -51,6 +51,56 @@ namespace ErrorCodes
extern const int SYSTEM_ERROR;
}
namespace
{
bool checkIfRequestIncreaseMem(const Coordination::ZooKeeperRequestPtr & request)
{
if (request->getOpNum() == Coordination::OpNum::Create
|| request->getOpNum() == Coordination::OpNum::CreateIfNotExists
|| request->getOpNum() == Coordination::OpNum::Set)
{
return true;
}
else if (request->getOpNum() == Coordination::OpNum::Multi)
{
Coordination::ZooKeeperMultiRequest & multi_req = dynamic_cast<Coordination::ZooKeeperMultiRequest &>(*request);
Int64 memory_delta = 0;
for (const auto & sub_req : multi_req.requests)
{
auto sub_zk_request = std::dynamic_pointer_cast<Coordination::ZooKeeperRequest>(sub_req);
switch (sub_zk_request->getOpNum())
{
case Coordination::OpNum::Create:
case Coordination::OpNum::CreateIfNotExists:
{
Coordination::ZooKeeperCreateRequest & create_req = dynamic_cast<Coordination::ZooKeeperCreateRequest &>(*sub_zk_request);
memory_delta += create_req.bytesSize();
break;
}
case Coordination::OpNum::Set:
{
Coordination::ZooKeeperSetRequest & set_req = dynamic_cast<Coordination::ZooKeeperSetRequest &>(*sub_zk_request);
memory_delta += set_req.bytesSize();
break;
}
case Coordination::OpNum::Remove:
{
Coordination::ZooKeeperRemoveRequest & remove_req = dynamic_cast<Coordination::ZooKeeperRemoveRequest &>(*sub_zk_request);
memory_delta -= remove_req.bytesSize();
break;
}
default:
break;
}
}
return memory_delta > 0;
}
return false;
}
}
KeeperDispatcher::KeeperDispatcher()
: responses_queue(std::numeric_limits<size_t>::max())
@ -93,6 +143,14 @@ void KeeperDispatcher::requestThread()
if (shutdown_called)
break;
Int64 mem_soft_limit = configuration_and_settings->coordination_settings->max_memory_usage_soft_limit;
if (configuration_and_settings->standalone_keeper && mem_soft_limit > 0 && total_memory_tracker.get() >= mem_soft_limit && checkIfRequestIncreaseMem(request.request))
{
LOG_TRACE(log, "Processing requests refused because of max_memory_usage_soft_limit {}, the total used memory is {}, request type is {}", mem_soft_limit, total_memory_tracker.get(), request.request->getOpNum());
addErrorResponses({request}, Coordination::Error::ZCONNECTIONLOSS);
continue;
}
KeeperStorage::RequestsForSessions current_batch;
size_t current_batch_bytes_size = 0;

17
src/Coordination/Standalone/Context.cpp
View File

@ -4,6 +4,7 @@
#include <Common/Macros.h>
#include <Common/ThreadPool.h>
#include <Common/callOnce.h>
#include <Disks/IO/IOUringReader.h>
#include <Core/ServerSettings.h>
@ -62,6 +63,11 @@ struct ContextSharedPart : boost::noncopyable
mutable std::unique_ptr<IAsynchronousReader> asynchronous_local_fs_reader;
mutable std::unique_ptr<IAsynchronousReader> synchronous_local_fs_reader;
#if USE_LIBURING
mutable OnceFlag io_uring_reader_initialized;
mutable std::unique_ptr<IOUringReader> io_uring_reader;
#endif
mutable OnceFlag threadpool_writer_initialized;
mutable std::unique_ptr<ThreadPool> threadpool_writer;
@ -225,6 +231,17 @@ IAsynchronousReader & Context::getThreadPoolReader(FilesystemReaderType type) co
}
}
#if USE_LIBURING
IOUringReader & Context::getIOURingReader() const
{
callOnce(shared->io_uring_reader_initialized, [&] {
shared->io_uring_reader = std::make_unique<IOUringReader>(512);
});
return *shared->io_uring_reader;
}
#endif
std::shared_ptr<FilesystemCacheLog> Context::getFilesystemCacheLog() const
{
return nullptr;

6
src/Coordination/Standalone/Context.h
View File

@ -20,6 +20,8 @@
#include <memory>
#include "config.h"
namespace DB
{
@ -28,6 +30,7 @@ class Macros;
class FilesystemCacheLog;
class FilesystemReadPrefetchesLog;
class BlobStorageLog;
class IOUringReader;
/// A small class which owns ContextShared.
/// We don't use something like unique_ptr directly to allow ContextShared type to be incomplete.
@ -127,6 +130,9 @@ public:
ApplicationType getApplicationType() const { return ApplicationType::KEEPER; }
IAsynchronousReader & getThreadPoolReader(FilesystemReaderType type) const;
#if USE_LIBURING
IOUringReader & getIOURingReader() const;
#endif
std::shared_ptr<AsyncReadCounters> getAsyncReadCounters() const;
ThreadPool & getThreadPoolWriter() const;

1
src/Core/Settings.h
View File

@ -156,6 +156,7 @@ class IColumn;
M(Bool, allow_suspicious_low_cardinality_types, false, "In CREATE TABLE statement allows specifying LowCardinality modifier for types of small fixed size (8 or less). Enabling this may increase merge times and memory consumption.", 0) \
M(Bool, allow_suspicious_fixed_string_types, false, "In CREATE TABLE statement allows creating columns of type FixedString(n) with n > 256. FixedString with length >= 256 is suspicious and most likely indicates misusage", 0) \
M(Bool, allow_suspicious_indices, false, "Reject primary/secondary indexes and sorting keys with identical expressions", 0) \
M(Bool, allow_suspicious_ttl_expressions, false, "Reject TTL expressions that don't depend on any of table's columns. It indicates a user error most of the time.", 0) \
M(Bool, compile_expressions, false, "Compile some scalar functions and operators to native code.", 0) \
M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \
M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \

2
src/Core/SettingsChangesHistory.h
View File

@ -7,6 +7,7 @@
#include <boost/algorithm/string.hpp>
#include <map>
namespace DB
{
@ -80,6 +81,7 @@ namespace SettingsChangesHistory
/// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972)
static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> settings_changes_history =
{
{"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."}}},
{"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
{"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
{"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},

7
src/Databases/DatabaseAtomic.cpp
View File

@ -89,15 +89,14 @@ void DatabaseAtomic::drop(ContextPtr)
fs::remove_all(getMetadataPath());
}
void DatabaseAtomic::attachTable(ContextPtr /* context_ */, const String & name, const StoragePtr & table, const String & relative_table_path)
void DatabaseAtomic::attachTableUnlocked(ContextPtr local_context, const String & name, const StoragePtr & table, const String & relative_table_path)
{
assert(relative_table_path != data_path && !relative_table_path.empty());
DetachedTables not_in_use;
std::lock_guard lock(mutex);
not_in_use = cleanupDetachedTables();
auto table_id = table->getStorageID();
assertDetachedTableNotInUse(table_id.uuid);
DatabaseOrdinary::attachTableUnlocked(name, table);
DatabaseOrdinary::attachTableUnlocked(local_context, name, table, relative_table_path);
table_name_to_path.emplace(std::make_pair(name, relative_table_path));
}
@ -325,7 +324,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora
/// It throws if `table_metadata_path` already exists (it's possible if table was detached)
renameNoReplace(table_metadata_tmp_path, table_metadata_path); /// Commit point (a sort of)
attachTableUnlocked(query.getTable(), table); /// Should never throw
DatabaseWithOwnTablesBase::attachTableUnlocked(query_context, query.getTable(), table, /*relative_table_path=*/ {}); /// Should never throw
table_name_to_path.emplace(query.getTable(), table_data_path);
}
catch (...)

3
src/Databases/DatabaseAtomic.h
View File

@ -38,7 +38,6 @@ public:
void dropTable(ContextPtr context, const String & table_name, bool sync) override;
void dropTableImpl(ContextPtr context, const String & table_name, bool sync);
void attachTable(ContextPtr context, const String & name, const StoragePtr & table, const String & relative_table_path) override;
StoragePtr detachTable(ContextPtr context, const String & name) override;
String getTableDataPath(const String & table_name) const override;
@ -66,6 +65,8 @@ public:
void setDetachedTableNotInUseForce(const UUID & uuid) override;
protected:
void attachTableUnlocked(ContextPtr local_context, const String & name, const StoragePtr & table, const String & relative_table_path) TSA_REQUIRES(mutex) override;
void commitAlterTable(const StorageID & table_id, const String & table_metadata_tmp_path, const String & table_metadata_path, const String & statement, ContextPtr query_context) override;
void commitCreateTable(const ASTCreateQuery & query, const StoragePtr & table,
const String & table_metadata_tmp_path, const String & table_metadata_path, ContextPtr query_context) override;

3
src/Databases/DatabaseLazy.cpp
View File

@ -161,10 +161,9 @@ bool DatabaseLazy::empty() const
return tables_cache.empty();
}
void DatabaseLazy::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & table, const String &)
void DatabaseLazy::attachTableUnlocked(ContextPtr /* context_ */, const String & table_name, const StoragePtr & table, const String &)
{
LOG_DEBUG(log, "Attach table {}.", backQuote(table_name));
std::lock_guard lock(mutex);
time_t current_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
auto [it, inserted] = tables_cache.emplace(std::piecewise_construct,

5
src/Databases/DatabaseLazy.h
View File

@ -64,14 +64,15 @@ public:
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override;
void attachTable(ContextPtr context, const String & table_name, const StoragePtr & table, const String & relative_table_path) override;
StoragePtr detachTable(ContextPtr context, const String & table_name) override;
void shutdown() override;
~DatabaseLazy() override;
protected:
void attachTableUnlocked(ContextPtr context, const String & table_name, const StoragePtr & table, const String & relative_table_path) TSA_REQUIRES(mutex) override;
private:
struct CacheExpirationQueueElement
{

8
src/Databases/DatabaseMemory.cpp
View File

@ -33,13 +33,13 @@ DatabaseMemory::DatabaseMemory(const String & name_, ContextPtr context_)
}
void DatabaseMemory::createTable(
ContextPtr /*context*/,
ContextPtr local_context,
const String & table_name,
const StoragePtr & table,
const ASTPtr & query)
{
std::lock_guard lock{mutex};
attachTableUnlocked(table_name, table);
attachTableUnlocked(local_context, table_name, table, /*relative_table_path=*/ {});
/// Clean the query from temporary flags.
ASTPtr query_to_store = query;
@ -56,7 +56,7 @@ void DatabaseMemory::createTable(
}
void DatabaseMemory::dropTable(
ContextPtr /*context*/,
ContextPtr local_context,
const String & table_name,
bool /*sync*/)
{
@ -83,7 +83,7 @@ void DatabaseMemory::dropTable(
catch (...)
{
std::lock_guard lock{mutex};
attachTableUnlocked(table_name, table);
attachTableUnlocked(local_context, table_name, table, /*relative_table_path=*/ {});
throw;
}

57
src/Databases/DatabasesCommon.cpp
View File

@ -7,6 +7,7 @@
#include <Parsers/formatAST.h>
#include <Storages/StorageDictionary.h>
#include <Storages/StorageFactory.h>
#include <Common/quoteString.h>
#include <Common/typeid_cast.h>
#include <Common/escapeForFileName.h>
#include <TableFunctions/TableFunctionFactory.h>
@ -193,7 +194,7 @@ DatabaseWithOwnTablesBase::DatabaseWithOwnTablesBase(const String & name_, const
bool DatabaseWithOwnTablesBase::isTableExist(const String & table_name, ContextPtr) const
{
std::lock_guard lock(mutex);
return tables.find(table_name) != tables.end();
return tables.find(table_name) != tables.end() || lazy_tables.find(table_name) != lazy_tables.end();
}
StoragePtr DatabaseWithOwnTablesBase::tryGetTable(const String & table_name, ContextPtr) const
@ -205,6 +206,9 @@ StoragePtr DatabaseWithOwnTablesBase::tryGetTable(const String & table_name, Con
DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(ContextPtr, const FilterByNameFunction & filter_by_table_name) const
{
std::lock_guard lock(mutex);
loadLazyTables();
if (!filter_by_table_name)
return std::make_unique<DatabaseTablesSnapshotIterator>(tables, database_name);
@ -250,13 +254,7 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
return res;
}
void DatabaseWithOwnTablesBase::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & table, const String &)
{
std::lock_guard lock(mutex);
attachTableUnlocked(table_name, table);
}
void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, const StoragePtr & table)
void DatabaseWithOwnTablesBase::attachTableUnlocked(ContextPtr, const String & name, const StoragePtr & table, const String &)
{
auto table_id = table->getStorageID();
if (table_id.database_name != database_name)
@ -269,7 +267,7 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
DatabaseCatalog::instance().addUUIDMapping(table_id.uuid, shared_from_this(), table);
}
if (!tables.emplace(table_name, table).second)
if (!tables.emplace(name, table).second)
{
if (table_id.hasUUID())
DatabaseCatalog::instance().removeUUIDMapping(table_id.uuid);
@ -281,6 +279,12 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
table->is_detached = false;
}
void DatabaseWithOwnTablesBase::registerLazyTableUnlocked(const String & table_name, LazyTableCreator table_creator, const String & relative_table_path)
{
if (!lazy_tables.emplace(table_name, std::make_pair(relative_table_path, std::move(table_creator))).second)
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {} already registered.", table_name);
}
void DatabaseWithOwnTablesBase::shutdown()
{
/// You can not hold a lock during shutdown.
@ -381,10 +385,45 @@ void DatabaseWithOwnTablesBase::createTableRestoredFromBackup(const ASTPtr & cre
StoragePtr DatabaseWithOwnTablesBase::tryGetTableNoWait(const String & table_name) const
{
std::lock_guard lock(mutex);
auto it = tables.find(table_name);
if (it != tables.end())
return it->second;
const auto lazy_it = lazy_tables.find(table_name);
if (lazy_it != lazy_tables.end())
{
LOG_DEBUG(log, "Attaching lazy table {}", backQuoteIfNeed(table_name));
auto relative_table_path = lazy_it->second.first;
auto storage = lazy_it->second.second();
lazy_tables.erase(lazy_it);
(const_cast<DatabaseWithOwnTablesBase *>(this))->attachTableUnlocked(Context::getGlobalContextInstance(), table_name, storage, relative_table_path);
it = tables.find(table_name);
if (it != tables.end())
return it->second;
}
return {};
}
void DatabaseWithOwnTablesBase::loadLazyTables() const
{
if (lazy_tables.empty())
return;
ContextPtr global_context = Context::getGlobalContextInstance();
while (!lazy_tables.empty())
{
auto lazy_it = lazy_tables.begin();
const auto table_name = lazy_it->first;
LOG_DEBUG(log, "Attaching lazy table {}", backQuoteIfNeed(table_name));
auto relative_table_path = lazy_it->second.first;
auto storage = lazy_it->second.second();
lazy_tables.erase(lazy_it);
(const_cast<DatabaseWithOwnTablesBase *>(this))->attachTableUnlocked(global_context, table_name, storage, relative_table_path);
}
}
}

9
src/Databases/DatabasesCommon.h
View File

@ -30,8 +30,6 @@ public:
bool empty() const override;
void attachTable(ContextPtr context, const String & table_name, const StoragePtr & table, const String & relative_table_path) override;
StoragePtr detachTable(ContextPtr context, const String & table_name) override;
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) const override;
@ -45,14 +43,19 @@ public:
protected:
Tables tables TSA_GUARDED_BY(mutex);
/// Tables that are attached lazily
mutable LazyTables lazy_tables TSA_GUARDED_BY(mutex);
Poco::Logger * log;
DatabaseWithOwnTablesBase(const String & name_, const String & logger, ContextPtr context);
void attachTableUnlocked(const String & table_name, const StoragePtr & table) TSA_REQUIRES(mutex);
void attachTableUnlocked(ContextPtr context, const String & name, const StoragePtr & table, const String & relative_table_path) TSA_REQUIRES(mutex) override;
void registerLazyTableUnlocked(const String & table_name, LazyTableCreator table_creator, const String & relative_table_path) TSA_REQUIRES(mutex) override;
StoragePtr detachTableUnlocked(const String & table_name) TSA_REQUIRES(mutex);
StoragePtr getTableUnlocked(const String & table_name) const TSA_REQUIRES(mutex);
StoragePtr tryGetTableNoWait(const String & table_name) const;
void loadLazyTables() const TSA_REQUIRES(mutex);
};
}

16
src/Databases/IDatabase.cpp
View File

@ -46,4 +46,20 @@ void IDatabase::createTableRestoredFromBackup(const ASTPtr & create_table_query,
backQuoteIfNeed(create_table_query->as<const ASTCreateQuery &>().getTable()));
}
/// Add a table to the database, but do not add it to the metadata. The database may not support this method.
///
/// Note: ATTACH TABLE statement actually uses createTable method.
void IDatabase::attachTable(ContextPtr context, const String & name, const StoragePtr & table, const String & relative_table_path) /// NOLINT
{
std::lock_guard lock(mutex);
attachTableUnlocked(context, name, table, relative_table_path);
}
void IDatabase::registerLazyTable(ContextPtr, const String & table_name, LazyTableCreator table_creator, const String & relative_table_path) /// NOLINT
{
std::lock_guard lock(mutex);
registerLazyTableUnlocked(table_name, std::move(table_creator), relative_table_path);
}
}

31
src/Databases/IDatabase.h
View File

@ -125,7 +125,6 @@ public:
using DatabaseTablesIteratorPtr = std::unique_ptr<IDatabaseTablesIterator>;
/** Database engine.
* It is responsible for:
* - initialization of set of known tables and dictionaries;
@ -138,6 +137,10 @@ using DatabaseTablesIteratorPtr = std::unique_ptr<IDatabaseTablesIterator>;
class IDatabase : public std::enable_shared_from_this<IDatabase>
{
public:
using LazyTableCreator = std::function<StoragePtr()>;
/// Map{table_name, Pair{relative_table_path, LazyTableCreator}}
using LazyTables = std::map<String, std::pair<String, LazyTableCreator>>;
IDatabase() = delete;
explicit IDatabase(String database_name_) : database_name(std::move(database_name_)) {}
@ -269,11 +272,17 @@ public:
/// Add a table to the database, but do not add it to the metadata. The database may not support this method.
///
/// Note: ATTACH TABLE statement actually uses createTable method.
virtual void attachTable(ContextPtr /* context */, const String & /*name*/, const StoragePtr & /*table*/, [[maybe_unused]] const String & relative_table_path = {}) /// NOLINT
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no ATTACH TABLE query for Database{}", getEngineName());
}
/// @param relative_table_path - only for Atomic engine
///
/// Note:
/// - ATTACH TABLE statement actually uses createTable method.
/// - Instead of overriding this method you should override attachTableUnlocked()
/// (This method is only for DatabasesOverlay to override)
virtual void attachTable(ContextPtr context, const String & name, const StoragePtr & table, const String & relative_table_path = {}); /// NOLINT
/// Register tables lazily (attach will be done only when the table will be used) instead of attaching it.
/// This is needed to improve startup time of clickhouse-local.
virtual void registerLazyTable(ContextPtr context, const String & table_name, LazyTableCreator table_creator, const String & relative_table_path = {});
/// Forget about the table without deleting it, and return it. The database may not support this method.
virtual StoragePtr detachTable(ContextPtr /* context */, const String & /*name*/)
@ -430,6 +439,16 @@ protected:
return nullptr;
}
virtual void attachTableUnlocked(ContextPtr /*context*/, const String & /*name*/, const StoragePtr & /*table*/, const String & /*relative_table_path*/ = {}) TSA_REQUIRES(mutex) /// NOLINT
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no ATTACH TABLE query for Database{}", getEngineName());
}
virtual void registerLazyTableUnlocked(const String & /* table_name */, LazyTableCreator /* table_creator */, const String & /* relative_table_path */) TSA_REQUIRES(mutex) /// NOLINT
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There lazy table initialization support for Database{}", getEngineName());
}
mutable std::mutex mutex;
String database_name TSA_GUARDED_BY(mutex);
String comment TSA_GUARDED_BY(mutex);

4
src/Databases/MySQL/DatabaseMaterializedMySQL.cpp
View File

@ -101,10 +101,10 @@ void DatabaseMaterializedMySQL::dropTable(ContextPtr context_, const String & na
DatabaseAtomic::dropTable(context_, name, sync);
}
void DatabaseMaterializedMySQL::attachTable(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path)
void DatabaseMaterializedMySQL::attachTableUnlocked(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path)
{
checkIsInternalQuery(context_, "ATTACH TABLE");
DatabaseAtomic::attachTable(context_, name, table, relative_table_path);
DatabaseAtomic::attachTableUnlocked(context_, name, table, relative_table_path);
}
StoragePtr DatabaseMaterializedMySQL::detachTable(ContextPtr context_, const String & name)

4
src/Databases/MySQL/DatabaseMaterializedMySQL.h
View File

@ -48,6 +48,8 @@ protected:
LoadTaskPtr startup_mysql_database_task;
void attachTableUnlocked(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path) TSA_REQUIRES(mutex) override;
public:
String getEngineName() const override { return "MaterializedMySQL"; }
@ -58,8 +60,6 @@ public:
void dropTable(ContextPtr context_, const String & name, bool sync) override;
void attachTable(ContextPtr context_, const String & name, const StoragePtr & table, const String & relative_table_path) override;
StoragePtr detachTable(ContextPtr context_, const String & name) override;
void renameTable(ContextPtr context_, const String & name, IDatabase & to_database, const String & to_name, bool exchange, bool dictionary) override;

4
src/Databases/MySQL/DatabaseMySQL.cpp
View File

@ -361,10 +361,8 @@ void DatabaseMySQL::cleanOutdatedTables()
}
}
void DatabaseMySQL::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & storage, const String &)
void DatabaseMySQL::attachTableUnlocked(ContextPtr /* context_ */, const String & table_name, const StoragePtr & storage, const String &)
{
std::lock_guard lock{mutex};
if (!local_tables_cache.contains(table_name))
throw Exception(ErrorCodes::UNKNOWN_TABLE, "Cannot attach table {}.{} because it does not exist.",
backQuoteIfNeed(database_name), backQuoteIfNeed(table_name));

4
src/Databases/MySQL/DatabaseMySQL.h
View File

@ -84,9 +84,9 @@ public:
void dropTable(ContextPtr context, const String & table_name, bool sync) override;
void attachTable(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) override;
protected:
void attachTableUnlocked(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) TSA_REQUIRES(mutex) override;
ASTPtr getCreateTableQueryImpl(const String & name, ContextPtr context, bool throw_on_error) const override;
private:

4
src/Databases/PostgreSQL/DatabasePostgreSQL.cpp
View File

@ -216,10 +216,8 @@ StoragePtr DatabasePostgreSQL::fetchTable(const String & table_name, ContextPtr
}
void DatabasePostgreSQL::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & storage, const String &)
void DatabasePostgreSQL::attachTableUnlocked(ContextPtr /* context_ */, const String & table_name, const StoragePtr & storage, const String &)
{
std::lock_guard lock{mutex};
if (!checkPostgresTable(table_name))
throw Exception(ErrorCodes::UNKNOWN_TABLE,
"Cannot attach PostgreSQL table {} because it does not exist in PostgreSQL (database: {})",

3
src/Databases/PostgreSQL/DatabasePostgreSQL.h
View File

@ -54,13 +54,14 @@ public:
void createTable(ContextPtr, const String & table_name, const StoragePtr & storage, const ASTPtr & create_query) override;
void dropTable(ContextPtr, const String & table_name, bool sync) override;
void attachTable(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) override;
StoragePtr detachTable(ContextPtr context, const String & table_name) override;
void drop(ContextPtr /*context*/) override;
void shutdown() override;
protected:
void attachTableUnlocked(ContextPtr context, const String & table_name, const StoragePtr & storage, const String & relative_table_path) TSA_REQUIRES(mutex) override;
ASTPtr getCreateTableQueryImpl(const String & table_name, ContextPtr context, bool throw_on_error) const override;
private:

10
src/Disks/IO/createReadBufferFromFileBase.cpp
View File

@ -101,12 +101,16 @@ std::unique_ptr<ReadBufferFromFileBase> createReadBufferFromFileBase(
else if (settings.local_fs_method == LocalFSReadMethod::io_uring)
{
#if USE_LIBURING
static std::shared_ptr<IOUringReader> reader = std::make_shared<IOUringReader>(512);
if (!reader->isSupported())
auto global_context = Context::getGlobalContextInstance();
if (!global_context)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot obtain io_uring reader (global context not initialized)");
auto & reader = global_context->getIOURingReader();
if (!reader.isSupported())
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "io_uring is not supported by this system");
res = std::make_unique<AsynchronousReadBufferFromFileWithDescriptorsCache>(
*reader,
reader,
settings.priority,
filename,
buffer_size,

7
src/Functions/FunctionsAES.cpp
View File

@ -15,14 +15,13 @@ namespace ErrorCodes
{
extern const int OPENSSL_ERROR;
}
}
namespace OpenSSLDetails
{
void onError(std::string error_message)
{
error_message += ". OpenSSL error code: " + std::to_string(ERR_get_error());
throw DB::Exception::createDeprecated(error_message, DB::ErrorCodes::OPENSSL_ERROR);
throw Exception(ErrorCodes::OPENSSL_ERROR, "{}. OpenSSL error code: {}", error_message, ERR_get_error());
}
StringRef foldEncryptionKeyInMySQLCompatitableMode(size_t cipher_key_size, StringRef key, std::array<char, EVP_MAX_KEY_LENGTH> & folded_key)
@ -48,4 +47,6 @@ const EVP_CIPHER * getCipherByName(StringRef cipher_name)
}
}
#endif

21
src/Functions/FunctionsAES.h
View File

@ -25,13 +25,14 @@
#include <string.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
}
namespace OpenSSLDetails
{
@ -60,7 +61,7 @@ struct KeyHolder
inline StringRef setKey(size_t cipher_key_size, StringRef key) const
{
if (key.size != cipher_key_size)
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid key size: {} expected {}", key.size, cipher_key_size);
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid key size: {} expected {}", key.size, cipher_key_size);
return key;
}
@ -72,7 +73,7 @@ struct KeyHolder<CipherMode::MySQLCompatibility>
inline StringRef setKey(size_t cipher_key_size, StringRef key)
{
if (key.size < cipher_key_size)
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid key size: {} expected {}", key.size, cipher_key_size);
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid key size: {} expected {}", key.size, cipher_key_size);
// MySQL does something fancy with the keys that are too long,
// ruining compatibility with OpenSSL and not improving security.
@ -118,7 +119,7 @@ inline void validateCipherMode(const EVP_CIPHER * evp_cipher)
}
}
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Unsupported cipher mode");
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported cipher mode");
}
template <CipherMode mode>
@ -127,13 +128,11 @@ inline void validateIV(StringRef iv_value, const size_t cipher_iv_size)
// In MySQL mode we don't care if IV is longer than expected, only if shorter.
if ((mode == CipherMode::MySQLCompatibility && iv_value.size != 0 && iv_value.size < cipher_iv_size)
|| (mode == CipherMode::OpenSSLCompatibility && iv_value.size != 0 && iv_value.size != cipher_iv_size))
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid IV size: {} expected {}", iv_value.size, cipher_iv_size);
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid IV size: {} expected {}", iv_value.size, cipher_iv_size);
}
}
namespace DB
{
template <typename Impl>
class FunctionEncrypt : public IFunction
{
@ -313,12 +312,12 @@ private:
// in GCM mode IV can be of arbitrary size (>0), IV is optional for other modes.
if (mode == CipherMode::RFC5116_AEAD_AES_GCM && iv_value.size == 0)
{
throw Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid IV size {} != expected size {}", iv_value.size, iv_size);
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid IV size {} != expected size {}", iv_value.size, iv_size);
}
if (mode != CipherMode::RFC5116_AEAD_AES_GCM && key_value.size != key_size)
{
throw Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid key size {} != expected size {}", key_value.size, key_size);
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid key size {} != expected size {}", key_value.size, key_size);
}
}
@ -608,12 +607,12 @@ private:
// in GCM mode IV can be of arbitrary size (>0), for other modes IV is optional.
if (mode == CipherMode::RFC5116_AEAD_AES_GCM && iv_value.size == 0)
{
throw Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid IV size {} != expected size {}", iv_value.size, iv_size);
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid IV size {} != expected size {}", iv_value.size, iv_size);
}
if (key_value.size != key_size)
{
throw Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid key size {} != expected size {}", key_value.size, key_size);
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid key size {} != expected size {}", key_value.size, key_size);
}
}

25
src/Functions/FunctionsHashing.h
View File

@ -28,6 +28,11 @@
# include <openssl/md4.h>
# include <openssl/md5.h>
# include <openssl/sha.h>
#if USE_BORINGSSL
# include <openssl/digest.h>
#else
# include <openssl/evp.h>
#endif
#endif
#include <bit>
@ -318,6 +323,25 @@ struct SHA512Impl
SHA512_Final(out_char_data, &ctx);
}
};
struct SHA512Impl256
{
static constexpr auto name = "SHA512_256";
enum { length = 32 };
static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
{
/// Here, we use the EVP interface that is common to both BoringSSL and OpenSSL. Though BoringSSL is the default
/// SSL library that we use, for S390X architecture only OpenSSL is supported. But the SHA512-256, SHA512_256_Init,
/// SHA512_256_Update, SHA512_256_Final methods to calculate hash (similar to the other SHA functions) aren't available
/// in the current version of OpenSSL that we use which necessitates the use of the EVP interface.
auto md_ctx = EVP_MD_CTX_create();
EVP_DigestInit_ex(md_ctx, EVP_sha512_256(), nullptr /*engine*/);
EVP_DigestUpdate(md_ctx, begin, size);
EVP_DigestFinal_ex(md_ctx, out_char_data, nullptr /*size*/);
EVP_MD_CTX_destroy(md_ctx);
}
};
#endif
struct SipHash64Impl
@ -1801,6 +1825,7 @@ using FunctionSHA224 = FunctionStringHashFixedString<SHA224Impl>;
using FunctionSHA256 = FunctionStringHashFixedString<SHA256Impl>;
using FunctionSHA384 = FunctionStringHashFixedString<SHA384Impl>;
using FunctionSHA512 = FunctionStringHashFixedString<SHA512Impl>;
using FunctionSHA512_256 = FunctionStringHashFixedString<SHA512Impl256>;
#endif
using FunctionSipHash128 = FunctionAnyHash<SipHash128Impl>;
using FunctionSipHash128Keyed = FunctionAnyHash<SipHash128KeyedImpl, true, SipHash128KeyedImpl::Key, SipHash128KeyedImpl::KeyColumns>;

16
src/Functions/FunctionsHashingSSL.cpp
View File

@ -22,6 +22,22 @@ REGISTER_FUNCTION(HashingSSL)
factory.registerFunction<FunctionSHA256>();
factory.registerFunction<FunctionSHA384>();
factory.registerFunction<FunctionSHA512>();
factory.registerFunction<FunctionSHA512_256>(FunctionDocumentation{
.description = R"(Calculates the SHA512_256 hash of the given string.)",
.syntax = "SELECT SHA512_256(s);",
.arguments = {{"s", "The input [String](../../sql-reference/data-types/string.md)."}},
.returned_value
= "The SHA512_256 hash of the given input string returned as a [FixedString](../../sql-reference/data-types/fixedstring.md).",
.examples
= {{"",
"SELECT HEX(SHA512_256('abc'));",
R"(
┌─hex(SHA512_256('abc'))───────────────────────────────────────────┐
│ 53048E2681941EF99B2E29B76B4C7DABE4C2D0C634FC6D46E0E2F13107E7AF23 │
└──────────────────────────────────────────────────────────────────┘
)"
}}
});
}
}

7
src/Functions/aes_decrypt_mysql.cpp
View File

@ -5,6 +5,10 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsAES.h>
namespace DB
{
namespace
{
@ -17,9 +21,6 @@ struct DecryptMySQLModeImpl
}
namespace DB
{
REGISTER_FUNCTION(AESDecryptMysql)
{
factory.registerFunction<FunctionDecrypt<DecryptMySQLModeImpl>>();

6
src/Functions/aes_encrypt_mysql.cpp
View File

@ -5,6 +5,9 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsAES.h>
namespace DB
{
namespace
{
@ -16,9 +19,6 @@ struct EncryptMySQLModeImpl
}
namespace DB
{
REGISTER_FUNCTION(AESEncryptMysql)
{
factory.registerFunction<FunctionEncrypt<EncryptMySQLModeImpl>>();

2
src/Functions/date_trunc.cpp
View File

@ -47,7 +47,7 @@ public:
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be constant string: "
"name of datepart", getName());
datepart_param = datepart_column->getValue<String>();
datepart_param = Poco::toLower(datepart_column->getValue<String>());
if (datepart_param.empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "First argument (name of datepart) for function {} cannot be empty",
getName());

6
src/Functions/decrypt.cpp
View File

@ -5,6 +5,9 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsAES.h>
namespace DB
{
namespace
{
@ -17,9 +20,6 @@ struct DecryptImpl
}
namespace DB
{
REGISTER_FUNCTION(Decrypt)
{
factory.registerFunction<FunctionDecrypt<DecryptImpl>>();

6
src/Functions/encrypt.cpp
View File

@ -5,6 +5,9 @@
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsAES.h>
namespace DB
{
namespace
{
@ -16,9 +19,6 @@ struct EncryptImpl
}
namespace DB
{
REGISTER_FUNCTION(Encrypt)
{
factory.registerFunction<FunctionEncrypt<EncryptImpl>>();

11
src/Functions/tryDecrypt.cpp
View File

@ -3,8 +3,12 @@
#if USE_SSL
# include <Functions/FunctionFactory.h>
# include <Functions/FunctionsAES.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionsAES.h>
namespace DB
{
namespace
{
@ -18,9 +22,6 @@ struct TryDecryptImpl
}
namespace DB
{
REGISTER_FUNCTION(TryDecrypt)
{
factory.registerFunction<FunctionDecrypt<TryDecryptImpl>>(FunctionDocumentation{

24
src/IO/FileEncryptionCommon.cpp
View File

@ -97,7 +97,7 @@ namespace
uint8_t * ciphertext = reinterpret_cast<uint8_t *>(out.position());
int ciphertext_size = 0;
if (!EVP_EncryptUpdate(evp_ctx, ciphertext, &ciphertext_size, &in[in_size], static_cast<int>(part_size)))
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to encrypt");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to encrypt: {}", ERR_get_error());
in_size += part_size;
if (ciphertext_size)
@ -120,7 +120,7 @@ namespace
uint8_t ciphertext[kBlockSize];
int ciphertext_size = 0;
if (!EVP_EncryptUpdate(evp_ctx, ciphertext, &ciphertext_size, padded_data, safe_cast<int>(padded_data_size)))
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to encrypt");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to encrypt: {}", ERR_get_error());
if (!ciphertext_size)
return 0;
@ -140,7 +140,7 @@ namespace
int ciphertext_size = 0;
if (!EVP_EncryptFinal_ex(evp_ctx,
ciphertext, &ciphertext_size))
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to finalize encrypting");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to finalize encrypting: {}", ERR_get_error());
if (ciphertext_size)
out.write(reinterpret_cast<const char *>(ciphertext), ciphertext_size);
return ciphertext_size;
@ -152,7 +152,7 @@ namespace
uint8_t * plaintext = reinterpret_cast<uint8_t *>(out);
int plaintext_size = 0;
if (!EVP_DecryptUpdate(evp_ctx, plaintext, &plaintext_size, in, safe_cast<int>(size)))
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to decrypt");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to decrypt: {}", ERR_get_error());
return plaintext_size;
}
@ -165,7 +165,7 @@ namespace
uint8_t plaintext[kBlockSize];
int plaintext_size = 0;
if (!EVP_DecryptUpdate(evp_ctx, plaintext, &plaintext_size, padded_data, safe_cast<int>(padded_data_size)))
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to decrypt");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to decrypt: {}", ERR_get_error());
if (!plaintext_size)
return 0;
@ -184,7 +184,7 @@ namespace
uint8_t plaintext[kBlockSize];
int plaintext_size = 0;
if (!EVP_DecryptFinal_ex(evp_ctx, plaintext, &plaintext_size))
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to finalize decrypting");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to finalize decrypting: {}", ERR_get_error());
if (plaintext_size)
memcpy(out, plaintext, plaintext_size);
return plaintext_size;
@ -291,11 +291,11 @@ void Encryptor::encrypt(const char * data, size_t size, WriteBuffer & out)
auto * evp_ctx = evp_ctx_ptr.get();
if (!EVP_EncryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr))
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to initialize encryption context with cipher");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to initialize encryption context with cipher: {}", ERR_get_error());
if (!EVP_EncryptInit_ex(evp_ctx, nullptr, nullptr,
reinterpret_cast<const uint8_t*>(key.c_str()), reinterpret_cast<const uint8_t*>(current_iv.c_str())))
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to set key and IV for encryption");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to set key and IV for encryption: {}", ERR_get_error());
size_t in_size = 0;
size_t out_size = 0;
@ -320,7 +320,7 @@ void Encryptor::encrypt(const char * data, size_t size, WriteBuffer & out)
out_size += encryptFinal(evp_ctx, out);
if (out_size != in_size)
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Only part of the data was encrypted");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Only part of the data was encrypted: {} out of {} bytes", out_size, in_size);
offset += in_size;
}
@ -335,11 +335,11 @@ void Encryptor::decrypt(const char * data, size_t size, char * out)
auto * evp_ctx = evp_ctx_ptr.get();
if (!EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr))
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to initialize decryption context with cipher");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to initialize decryption context with cipher: {}", ERR_get_error());
if (!EVP_DecryptInit_ex(evp_ctx, nullptr, nullptr,
reinterpret_cast<const uint8_t*>(key.c_str()), reinterpret_cast<const uint8_t*>(current_iv.c_str())))
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to set key and IV for decryption");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Failed to set key and IV for decryption: {}", ERR_get_error());
size_t in_size = 0;
size_t out_size = 0;
@ -364,7 +364,7 @@ void Encryptor::decrypt(const char * data, size_t size, char * out)
out_size += decryptFinal(evp_ctx, &out[out_size]);
if (out_size != in_size)
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Only part of the data was decrypted");
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Only part of the data was decrypted: {} out of {} bytes", out_size, in_size);
offset += in_size;
}

19
src/IO/ReadBufferFromFileDescriptor.cpp
View File

@ -39,7 +39,6 @@ namespace ErrorCodes
extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;
extern const int ARGUMENT_OUT_OF_BOUND;
extern const int CANNOT_SEEK_THROUGH_FILE;
extern const int CANNOT_SELECT;
extern const int CANNOT_ADVISE;
}
@ -249,24 +248,6 @@ void ReadBufferFromFileDescriptor::rewind()
file_offset_of_buffer_end = 0;
}
/// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout.
bool ReadBufferFromFileDescriptor::poll(size_t timeout_microseconds) const
{
fd_set fds;
FD_ZERO(&fds);
FD_SET(fd, &fds);
timeval timeout = { time_t(timeout_microseconds / 1000000), suseconds_t(timeout_microseconds % 1000000) };
int res = select(1, &fds, nullptr, nullptr, &timeout);
if (-1 == res)
throwFromErrno("Cannot select", ErrorCodes::CANNOT_SELECT);
return res > 0;
}
size_t ReadBufferFromFileDescriptor::getFileSize()
{
return getSizeFromFileDescriptor(fd, getFileName());

4
src/IO/ReadBufferFromFileDescriptor.h
View File

@ -75,10 +75,6 @@ public:
size_t readBigAt(char * to, size_t n, size_t offset, const std::function<bool(size_t)> &) override;
bool supportsReadAt() override { return use_pread; }
private:
/// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout.
bool poll(size_t timeout_microseconds) const;
};

19
src/Interpreters/Context.cpp
View File

@ -35,6 +35,7 @@
#include <Disks/ObjectStorages/DiskObjectStorage.h>
#include <Disks/ObjectStorages/IObjectStorage.h>
#include <Disks/StoragePolicy.h>
#include <Disks/IO/IOUringReader.h>
#include <IO/SynchronousReader.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Interpreters/ActionLocksManager.h>
@ -309,6 +310,11 @@ struct ContextSharedPart : boost::noncopyable
mutable OnceFlag threadpool_writer_initialized;
mutable std::unique_ptr<ThreadPool> threadpool_writer;
#if USE_LIBURING
mutable OnceFlag io_uring_reader_initialized;
mutable std::unique_ptr<IOUringReader> io_uring_reader;
#endif
mutable ThrottlerPtr replicated_fetches_throttler; /// A server-wide throttler for replicated fetches
mutable ThrottlerPtr replicated_sends_throttler; /// A server-wide throttler for replicated sends
@ -4811,7 +4817,7 @@ void Context::initializeBackgroundExecutorsIfNeeded()
shared->are_background_executors_initialized = true;
}
bool Context::areBackgroundExecutorsInitialized()
bool Context::areBackgroundExecutorsInitialized() const
{
SharedLockGuard lock(shared->background_executors_mutex);
return shared->are_background_executors_initialized;
@ -4861,6 +4867,17 @@ IAsynchronousReader & Context::getThreadPoolReader(FilesystemReaderType type) co
}
}
#if USE_LIBURING
IOUringReader & Context::getIOURingReader() const
{
callOnce(shared->io_uring_reader_initialized, [&] {
shared->io_uring_reader = std::make_unique<IOUringReader>(512);
});
return *shared->io_uring_reader;
}
#endif
ThreadPool & Context::getThreadPoolWriter() const
{
callOnce(shared->threadpool_writer_initialized, [&] {

6
src/Interpreters/Context.h
View File

@ -109,6 +109,7 @@ class AsynchronousInsertLog;
class BackupLog;
class BlobStorageLog;
class IAsynchronousReader;
class IOUringReader;
struct MergeTreeSettings;
struct InitialAllRangesAnnouncement;
struct ParallelReadRequest;
@ -1202,7 +1203,7 @@ public:
/// Background executors related methods
void initializeBackgroundExecutorsIfNeeded();
bool areBackgroundExecutorsInitialized();
bool areBackgroundExecutorsInitialized() const;
MergeMutateBackgroundExecutorPtr getMergeMutateExecutor() const;
OrdinaryBackgroundExecutorPtr getMovesExecutor() const;
@ -1210,6 +1211,9 @@ public:
OrdinaryBackgroundExecutorPtr getCommonExecutor() const;
IAsynchronousReader & getThreadPoolReader(FilesystemReaderType type) const;
#if USE_LIBURING
IOUringReader & getIOURingReader() const;
#endif
std::shared_ptr<AsyncReadCounters> getAsyncReadCounters() const;

10
src/Interpreters/ServerAsynchronousMetrics.cpp
View File

@ -54,8 +54,8 @@ ServerAsynchronousMetrics::ServerAsynchronousMetrics(
int update_period_seconds,
int heavy_metrics_update_period_seconds,
const ProtocolServerMetricsFunc & protocol_server_metrics_func_)
: AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_)
, WithContext(global_context_)
: WithContext(global_context_)
, AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_)
, heavy_metric_update_period(heavy_metrics_update_period_seconds)
{
/// sanity check
@ -63,6 +63,12 @@ ServerAsynchronousMetrics::ServerAsynchronousMetrics(
throw Exception(ErrorCodes::INVALID_SETTING_VALUE, "Setting asynchronous_metrics_update_period_s and asynchronous_heavy_metrics_update_period_s must not be zero");
}
ServerAsynchronousMetrics::~ServerAsynchronousMetrics()
{
/// NOTE: stop() from base class is not enough, since this leads to leak on vptr
stop();
}
void ServerAsynchronousMetrics::updateImpl(AsynchronousMetricValues & new_values, TimePoint update_time, TimePoint current_time)
{
if (auto mark_cache = getContext()->getMarkCache())

4
src/Interpreters/ServerAsynchronousMetrics.h
View File

@ -7,7 +7,7 @@
namespace DB
{
class ServerAsynchronousMetrics : public AsynchronousMetrics, WithContext
class ServerAsynchronousMetrics : WithContext, public AsynchronousMetrics
{
public:
ServerAsynchronousMetrics(
@ -15,6 +15,8 @@ public:
int update_period_seconds,
int heavy_metrics_update_period_seconds,
const ProtocolServerMetricsFunc & protocol_server_metrics_func_);
~ServerAsynchronousMetrics() override;
private:
void updateImpl(AsynchronousMetricValues & new_values, TimePoint update_time, TimePoint current_time) override;
void logImpl(AsynchronousMetricValues & new_values) override;

28
src/Planner/PlannerJoinTree.cpp
View File

@ -988,18 +988,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
if (join_strictness == JoinStrictness::All || join_strictness == JoinStrictness::Semi || join_strictness == JoinStrictness::Anti)
join_constant = tryExtractConstantFromJoinNode(join_table_expression);
if (join_constant)
{
/** If there is JOIN with always true constant, we transform it to cross.
* If there is JOIN with always false constant, we do not process JOIN keys.
* It is expected by join algorithm to handle such case.
*
* Example: SELECT * FROM test_table AS t1 INNER JOIN test_table AS t2 ON 1;
*/
if (*join_constant)
join_kind = JoinKind::Cross;
}
else if (join_node.isOnJoinExpression())
if (!join_constant && join_node.isOnJoinExpression())
{
join_clauses_and_actions = buildJoinClausesAndActions(left_plan_output_columns,
right_plan_output_columns,
@ -1079,7 +1068,6 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
const auto & query_context = planner_context->getQueryContext();
const auto & settings = query_context->getSettingsRef();
bool join_use_nulls = settings.join_use_nulls;
auto to_nullable_function = FunctionFactory::instance().get("toNullable", query_context);
auto join_cast_plan_columns_to_nullable = [&](QueryPlan & plan_to_add_cast)
@ -1105,7 +1093,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
plan_to_add_cast.addStep(std::move(cast_join_columns_step));
};
if (join_use_nulls)
if (settings.join_use_nulls)
{
if (isFull(join_kind))
{
@ -1124,6 +1112,18 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
auto table_join = std::make_shared<TableJoin>(settings, query_context->getGlobalTemporaryVolume());
table_join->getTableJoin() = join_node.toASTTableJoin()->as<ASTTableJoin &>();
if (join_constant)
{
/** If there is JOIN with always true constant, we transform it to cross.
* If there is JOIN with always false constant, we do not process JOIN keys.
* It is expected by join algorithm to handle such case.
*
* Example: SELECT * FROM test_table AS t1 INNER JOIN test_table AS t2 ON 1;
*/
if (*join_constant)
join_kind = JoinKind::Cross;
}
table_join->getTableJoin().kind = join_kind;
if (join_kind == JoinKind::Comma)

6
src/Storages/AlterCommands.cpp
View File

@ -707,7 +707,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context)
}
else if (type == MODIFY_TTL)
{
metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl, metadata.columns, context, metadata.primary_key);
metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(ttl, metadata.columns, context, metadata.primary_key, context->getSettingsRef().allow_suspicious_ttl_expressions);
}
else if (type == REMOVE_TTL)
{
@ -1136,13 +1136,13 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, ContextPtr context
metadata_copy.column_ttls_by_name.clear();
for (const auto & [name, ast] : column_ttl_asts)
{
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, metadata_copy.columns, context, metadata_copy.primary_key);
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, metadata_copy.columns, context, metadata_copy.primary_key, context->getSettingsRef().allow_suspicious_ttl_expressions);
metadata_copy.column_ttls_by_name[name] = new_ttl_entry;
}
if (metadata_copy.table_ttl.definition_ast != nullptr)
metadata_copy.table_ttl = TTLTableDescription::getTTLForTableFromAST(
metadata_copy.table_ttl.definition_ast, metadata_copy.columns, context, metadata_copy.primary_key);
metadata_copy.table_ttl.definition_ast, metadata_copy.columns, context, metadata_copy.primary_key, context->getSettingsRef().allow_suspicious_ttl_expressions);
metadata = std::move(metadata_copy);
}

25
src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
View File

@ -5,6 +5,7 @@
#include <Storages/StorageDistributed.h>
#include <QueryPipeline/RemoteInserter.h>
#include <Common/CurrentMetrics.h>
#include <base/defines.h>
#include <IO/Operators.h>
#include <IO/WriteBufferFromFile.h>
@ -163,6 +164,22 @@ void DistributedAsyncInsertBatch::deserialize()
readText(in);
}
bool DistributedAsyncInsertBatch::valid()
{
chassert(!files.empty());
bool res = true;
for (const auto & file : files)
{
if (!fs::exists(file))
{
LOG_WARNING(parent.log, "File {} does not exists, likely due abnormal shutdown", file);
res = false;
}
}
return res;
}
void DistributedAsyncInsertBatch::writeText(WriteBuffer & out)
{
for (const auto & file : files)
@ -201,14 +218,6 @@ void DistributedAsyncInsertBatch::sendBatch()
{
for (const auto & file : files)
{
/// In case of recovery it is possible that some of files will be
/// missing, if server had been restarted abnormally
if (recovered && !fs::exists(file))
{
LOG_WARNING(parent.log, "File {} does not exists, likely due abnormal shutdown", file);
continue;
}
ReadBufferFromFile in(file);
const auto & distributed_header = DistributedAsyncInsertHeader::read(in, parent.log);

7
src/Storages/Distributed/DistributedAsyncInsertBatch.h
View File

@ -18,9 +18,16 @@ public:
bool isEnoughSize() const;
void send();
/// Write batch to current_batch.txt
void serialize();
/// Read batch from current_batch.txt
void deserialize();
/// Does all required files exists?
/// (The only way variant when it is valid is during restoring batch from disk).
bool valid();
size_t total_rows = 0;
size_t total_bytes = 0;
std::vector<std::string> files;

39
src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
View File

@ -16,13 +16,14 @@
#include <Common/StringUtils/StringUtils.h>
#include <Common/SipHash.h>
#include <Common/quoteString.h>
#include <base/hex.h>
#include <Common/ProfileEvents.h>
#include <Common/ActionBlocker.h>
#include <Common/formatReadable.h>
#include <Common/Stopwatch.h>
#include <Common/logger_useful.h>
#include <Compression/CheckingCompressedReadBuffer.h>
#include <IO/Operators.h>
#include <base/hex.h>
#include <boost/algorithm/string/find_iterator.hpp>
#include <boost/algorithm/string/finder.hpp>
#include <boost/range/adaptor/indexed.hpp>
@ -38,6 +39,11 @@ namespace CurrentMetrics
extern const Metric BrokenDistributedBytesToInsert;
}
namespace ProfileEvents
{
extern const Event DistributedAsyncInsertionFailures;
}
namespace fs = std::filesystem;
namespace DB
@ -195,6 +201,15 @@ void DistributedAsyncInsertDirectoryQueue::run()
/// No errors while processing existing files.
/// Let's see maybe there are more files to process.
do_sleep = false;
const auto now = std::chrono::system_clock::now();
if (now - last_decrease_time > decrease_error_count_period)
{
std::lock_guard status_lock(status_mutex);
status.error_count /= 2;
last_decrease_time = now;
}
}
catch (...)
{
@ -213,15 +228,6 @@ void DistributedAsyncInsertDirectoryQueue::run()
else
LOG_TEST(LogFrequencyLimiter(log, 30), "Skipping send data over distributed table.");
const auto now = std::chrono::system_clock::now();
if (now - last_decrease_time > decrease_error_count_period)
{
std::lock_guard status_lock(status_mutex);
status.error_count /= 2;
last_decrease_time = now;
}
if (do_sleep)
break;
}
@ -376,6 +382,8 @@ try
}
catch (...)
{
ProfileEvents::increment(ProfileEvents::DistributedAsyncInsertionFailures);
std::lock_guard status_lock(status_mutex);
++status.error_count;
@ -516,7 +524,16 @@ void DistributedAsyncInsertDirectoryQueue::processFilesWithBatching()
DistributedAsyncInsertBatch batch(*this);
batch.deserialize();
batch.send();
/// In case of recovery it is possible that some of files will be
/// missing, if server had been restarted abnormally
/// (between unlink(*.bin) and unlink(current_batch.txt)).
///
/// But current_batch_file_path should be removed anyway, since if some
/// file was missing, then the batch is not complete and there is no
/// point in trying to pretend that it will not break deduplication.
if (batch.valid())
batch.send();
auto dir_sync_guard = getDirectorySyncGuard(relative_path);
fs::remove(current_batch_file_path);

8
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -2957,7 +2957,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
if (!settings.allow_non_metadata_alters)
{
auto mutation_commands = commands.getMutationCommands(new_metadata, settings.materialize_ttl_after_modify, getContext());
auto mutation_commands = commands.getMutationCommands(new_metadata, settings.materialize_ttl_after_modify, local_context);
if (!mutation_commands.empty())
throw Exception(ErrorCodes::ALTER_OF_COLUMN_IS_FORBIDDEN,
@ -2966,7 +2966,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
queryToString(mutation_commands.ast()));
}
commands.apply(new_metadata, getContext());
commands.apply(new_metadata, local_context);
if (commands.hasInvertedIndex(new_metadata) && !settings.allow_experimental_inverted_index)
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
@ -3253,7 +3253,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
if (!columns_to_check_conversion.empty())
{
auto old_header = old_metadata.getSampleBlock();
performRequiredConversions(old_header, columns_to_check_conversion, getContext());
performRequiredConversions(old_header, columns_to_check_conversion, local_context);
}
if (old_metadata.hasSettingsChanges())
@ -3285,7 +3285,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
}
if (setting_name == "storage_policy")
checkStoragePolicy(getContext()->getStoragePolicy(new_value.safeGet<String>()));
checkStoragePolicy(local_context->getStoragePolicy(new_value.safeGet<String>()));
}
/// Check if it is safe to reset the settings

6
src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp
View File

@ -409,7 +409,7 @@ StorageInMemoryMetadata ReplicatedMergeTreeTableMetadata::Diff::getNewMetadata(c
ParserTTLExpressionList parser;
auto ttl_for_table_ast = parseQuery(parser, new_ttl_table, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(
ttl_for_table_ast, new_metadata.columns, context, new_metadata.primary_key);
ttl_for_table_ast, new_metadata.columns, context, new_metadata.primary_key, true /* allow_suspicious; because it is replication */);
}
else /// TTL was removed
{
@ -422,7 +422,7 @@ StorageInMemoryMetadata ReplicatedMergeTreeTableMetadata::Diff::getNewMetadata(c
new_metadata.column_ttls_by_name.clear();
for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs())
{
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, context, new_metadata.primary_key);
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, context, new_metadata.primary_key, true /* allow_suspicious; because it is replication */);
new_metadata.column_ttls_by_name[name] = new_ttl_entry;
}
@ -454,7 +454,7 @@ StorageInMemoryMetadata ReplicatedMergeTreeTableMetadata::Diff::getNewMetadata(c
if (!ttl_table_changed && new_metadata.table_ttl.definition_ast != nullptr)
new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(
new_metadata.table_ttl.definition_ast, new_metadata.columns, context, new_metadata.primary_key);
new_metadata.table_ttl.definition_ast, new_metadata.columns, context, new_metadata.primary_key, true /* allow_suspicious; because it is replication */);
if (!projections_changed)
{

6
src/Storages/MergeTree/registerStorageMergeTree.cpp
View File

@ -581,10 +581,12 @@ static StoragePtr create(const StorageFactory::Arguments & args)
if (args.storage_def->sample_by)
metadata.sampling_key = KeyDescription::getKeyFromAST(args.storage_def->sample_by->ptr(), metadata.columns, context);
bool allow_suspicious_ttl = args.attach || args.getLocalContext()->getSettingsRef().allow_suspicious_ttl_expressions;
if (args.storage_def->ttl_table)
{
metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST(
args.storage_def->ttl_table->ptr(), metadata.columns, context, metadata.primary_key);
args.storage_def->ttl_table->ptr(), metadata.columns, context, metadata.primary_key, allow_suspicious_ttl);
}
if (args.query.columns_list && args.query.columns_list->indices)
@ -602,7 +604,7 @@ static StoragePtr create(const StorageFactory::Arguments & args)
auto column_ttl_asts = columns.getColumnTTLs();
for (const auto & [name, ast] : column_ttl_asts)
{
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, columns, context, metadata.primary_key);
auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, columns, context, metadata.primary_key, allow_suspicious_ttl);
metadata.column_ttls_by_name[name] = new_ttl_entry;
}

1
src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -1379,7 +1379,6 @@ void StorageReplicatedMergeTree::setTableStructure(const StorageID & table_id, c
checkTTLExpressions(new_metadata, old_metadata);
setProperties(new_metadata, old_metadata);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(local_context, table_id, new_metadata);
}

13
src/Storages/System/StorageSystemServerSettings.cpp
View File

@ -46,17 +46,20 @@ void StorageSystemServerSettings::fillData(MutableColumns & res_columns, Context
{"max_concurrent_insert_queries", std::to_string(context->getProcessList().getMaxInsertQueriesAmount())},
{"max_concurrent_select_queries", std::to_string(context->getProcessList().getMaxSelectQueriesAmount())},
{"background_pool_size", std::to_string(context->getMergeMutateExecutor()->getMaxThreads())},
{"background_move_pool_size", std::to_string(context->getMovesExecutor()->getMaxThreads())},
{"background_fetches_pool_size", std::to_string(context->getFetchesExecutor()->getMaxThreads())},
{"background_common_pool_size", std::to_string(context->getCommonExecutor()->getMaxThreads())},
{"background_buffer_flush_schedule_pool_size", std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundBufferFlushSchedulePoolSize))},
{"background_schedule_pool_size", std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundSchedulePoolSize))},
{"background_message_broker_schedule_pool_size", std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundMessageBrokerSchedulePoolSize))},
{"background_distributed_schedule_pool_size", std::to_string(CurrentMetrics::get(CurrentMetrics::BackgroundDistributedSchedulePoolSize))}
};
if (context->areBackgroundExecutorsInitialized())
{
updated.insert({"background_pool_size", std::to_string(context->getMergeMutateExecutor()->getMaxThreads())});
updated.insert({"background_move_pool_size", std::to_string(context->getMovesExecutor()->getMaxThreads())});
updated.insert({"background_fetches_pool_size", std::to_string(context->getFetchesExecutor()->getMaxThreads())});
updated.insert({"background_common_pool_size", std::to_string(context->getCommonExecutor()->getMaxThreads())});
}
const auto & config = context->getConfigRef();
ServerSettings settings;
settings.loadSettingsFromConfig(config);

18
src/Storages/System/StorageSystemStackTrace.cpp
View File

@ -57,12 +57,13 @@ std::atomic<bool> signal_latch = false; /// Only need for thread sanitizer.
/** Notes:
* Only one query from the table can be processed at the moment of time.
* This is ensured by the mutex in fillData function.
* This is ensured by the mutex in StorageSystemStackTraceSource.
* We obtain information about threads by sending signal and receiving info from the signal handler.
* Information is passed via global variables and pipe is used for signaling.
* Actually we can send all information via pipe, but we read from it with timeout just in case,
* so it's convenient to use is only for signaling.
* so it's convenient to use it only for signaling.
*/
std::mutex mutex;
StackTrace stack_trace{NoCapture{}};
@ -189,7 +190,7 @@ ThreadIdToName getFilteredThreadNames(ASTPtr query, ContextPtr context, const Pa
tid_to_name[tid] = thread_name;
all_thread_names->insert(thread_name);
}
LOG_TEST(log, "Read {} thread names for {} threads, took {} ms", tid_to_name.size(), thread_ids.size(), watch.elapsedMilliseconds());
LOG_TRACE(log, "Read {} thread names for {} threads, took {} ms", tid_to_name.size(), thread_ids.size(), watch.elapsedMilliseconds());
Block block { ColumnWithTypeAndName(std::move(all_thread_names), std::make_shared<DataTypeString>(), "thread_name") };
VirtualColumnUtils::filterBlockWithQuery(query, block, context);
@ -229,6 +230,8 @@ public:
, pipe_read_timeout_ms(static_cast<int>(context->getSettingsRef().storage_system_stack_trace_pipe_read_timeout_ms.totalMilliseconds()))
, log(log_)
, proc_it("/proc/self/task")
/// It shouldn't be possible to do concurrent reads from this table.
, lock(mutex)
{
/// Create a mask of what columns are needed in the result.
NameSet names_set(column_names.begin(), column_names.end());
@ -241,16 +244,13 @@ public:
protected:
Chunk generate() override
{
/// It shouldn't be possible to do concurrent reads from this table.
std::lock_guard lock(mutex);
MutableColumns res_columns = header.cloneEmptyColumns();
ColumnPtr thread_ids;
{
Stopwatch watch;
thread_ids = getFilteredThreadIds();
LOG_TEST(log, "Read {} threads, took {} ms", thread_ids->size(), watch.elapsedMilliseconds());
LOG_TRACE(log, "Read {} threads, took {} ms", thread_ids->size(), watch.elapsedMilliseconds());
}
if (thread_ids->empty())
return Chunk();
@ -332,7 +332,7 @@ protected:
++sequence_num;
}
}
LOG_TEST(log, "Send signal to {} threads (total), took {} ms", signals_sent, signals_sent_ms);
LOG_TRACE(log, "Send signal to {} threads (total), took {} ms", signals_sent, signals_sent_ms);
UInt64 num_rows = res_columns.at(0)->size();
Chunk chunk(std::move(res_columns), num_rows);
@ -357,7 +357,7 @@ private:
size_t signals_sent = 0;
size_t signals_sent_ms = 0;
std::mutex mutex;
std::unique_lock<std::mutex> lock;
ColumnPtr getFilteredThreadIds()
{

107
src/Storages/System/attachSystemTables.cpp
View File

@ -108,72 +108,75 @@
namespace DB
{
template <bool lazy>
void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database)
{
attach<StorageSystemOne>(context, system_database, "one");
attach<StorageSystemNumbers>(context, system_database, "numbers", false);
attach<StorageSystemNumbers>(context, system_database, "numbers_mt", true);
attach<StorageSystemZeros>(context, system_database, "zeros", false);
attach<StorageSystemZeros>(context, system_database, "zeros_mt", true);
attach<StorageSystemDatabases>(context, system_database, "databases");
attach<StorageSystemTables>(context, system_database, "tables");
attach<StorageSystemColumns>(context, system_database, "columns");
attach<StorageSystemFunctions>(context, system_database, "functions");
attach<StorageSystemEvents>(context, system_database, "events");
attach<StorageSystemSettings>(context, system_database, "settings");
attach<StorageSystemServerSettings>(context, system_database, "server_settings");
attach<StorageSystemSettingsChanges>(context, system_database, "settings_changes");
attach<SystemMergeTreeSettings<false>>(context, system_database, "merge_tree_settings");
attach<SystemMergeTreeSettings<true>>(context, system_database, "replicated_merge_tree_settings");
attach<StorageSystemBuildOptions>(context, system_database, "build_options");
attach<StorageSystemFormats>(context, system_database, "formats");
attach<StorageSystemTableFunctions>(context, system_database, "table_functions");
attach<StorageSystemAggregateFunctionCombinators>(context, system_database, "aggregate_function_combinators");
attach<StorageSystemDataTypeFamilies>(context, system_database, "data_type_families");
attach<StorageSystemCollations>(context, system_database, "collations");
attach<StorageSystemTableEngines>(context, system_database, "table_engines");
attach<StorageSystemContributors>(context, system_database, "contributors");
attach<StorageSystemUsers>(context, system_database, "users");
attach<StorageSystemRoles>(context, system_database, "roles");
attach<StorageSystemGrants>(context, system_database, "grants");
attach<StorageSystemRoleGrants>(context, system_database, "role_grants");
attach<StorageSystemCurrentRoles>(context, system_database, "current_roles");
attach<StorageSystemEnabledRoles>(context, system_database, "enabled_roles");
attach<StorageSystemSettingsProfiles>(context, system_database, "settings_profiles");
attach<StorageSystemSettingsProfileElements>(context, system_database, "settings_profile_elements");
attach<StorageSystemRowPolicies>(context, system_database, "row_policies");
attach<StorageSystemQuotas>(context, system_database, "quotas");
attach<StorageSystemQuotaLimits>(context, system_database, "quota_limits");
attach<StorageSystemQuotaUsage>(context, system_database, "quota_usage");
attach<StorageSystemQuotasUsage>(context, system_database, "quotas_usage");
attach<StorageSystemUserDirectories>(context, system_database, "user_directories");
attach<StorageSystemPrivileges>(context, system_database, "privileges");
attach<StorageSystemErrors>(context, system_database, "errors");
attach<StorageSystemWarnings>(context, system_database, "warnings");
attach<StorageSystemDataSkippingIndices>(context, system_database, "data_skipping_indices");
attach<StorageSystemLicenses>(context, system_database, "licenses");
attach<StorageSystemTimeZones>(context, system_database, "time_zones");
attach<StorageSystemBackups>(context, system_database, "backups");
attach<StorageSystemSchemaInferenceCache>(context, system_database, "schema_inference_cache");
attach<StorageSystemDroppedTables>(context, system_database, "dropped_tables");
attach<StorageSystemScheduler>(context, system_database, "scheduler");
attachLazyOrNot<lazy, StorageSystemOne>(context, system_database, "one");
attachLazyOrNot<lazy, StorageSystemNumbers>(context, system_database, "numbers", false);
attachLazyOrNot<lazy, StorageSystemNumbers>(context, system_database, "numbers_mt", true);
attachLazyOrNot<lazy, StorageSystemZeros>(context, system_database, "zeros", false);
attachLazyOrNot<lazy, StorageSystemZeros>(context, system_database, "zeros_mt", true);
attachLazyOrNot<lazy, StorageSystemDatabases>(context, system_database, "databases");
attachLazyOrNot<lazy, StorageSystemTables>(context, system_database, "tables");
attachLazyOrNot<lazy, StorageSystemColumns>(context, system_database, "columns");
attachLazyOrNot<lazy, StorageSystemFunctions>(context, system_database, "functions");
attachLazyOrNot<lazy, StorageSystemEvents>(context, system_database, "events");
attachLazyOrNot<lazy, StorageSystemSettings>(context, system_database, "settings");
attachLazyOrNot<lazy, StorageSystemServerSettings>(context, system_database, "server_settings");
attachLazyOrNot<lazy, StorageSystemSettingsChanges>(context, system_database, "settings_changes");
attachLazyOrNot<lazy, SystemMergeTreeSettings<false>>(context, system_database, "merge_tree_settings");
attachLazyOrNot<lazy, SystemMergeTreeSettings<true>>(context, system_database, "replicated_merge_tree_settings");
attachLazyOrNot<lazy, StorageSystemBuildOptions>(context, system_database, "build_options");
attachLazyOrNot<lazy, StorageSystemFormats>(context, system_database, "formats");
attachLazyOrNot<lazy, StorageSystemTableFunctions>(context, system_database, "table_functions");
attachLazyOrNot<lazy, StorageSystemAggregateFunctionCombinators>(context, system_database, "aggregate_function_combinators");
attachLazyOrNot<lazy, StorageSystemDataTypeFamilies>(context, system_database, "data_type_families");
attachLazyOrNot<lazy, StorageSystemCollations>(context, system_database, "collations");
attachLazyOrNot<lazy, StorageSystemTableEngines>(context, system_database, "table_engines");
attachLazyOrNot<lazy, StorageSystemContributors>(context, system_database, "contributors");
attachLazyOrNot<lazy, StorageSystemUsers>(context, system_database, "users");
attachLazyOrNot<lazy, StorageSystemRoles>(context, system_database, "roles");
attachLazyOrNot<lazy, StorageSystemGrants>(context, system_database, "grants");
attachLazyOrNot<lazy, StorageSystemRoleGrants>(context, system_database, "role_grants");
attachLazyOrNot<lazy, StorageSystemCurrentRoles>(context, system_database, "current_roles");
attachLazyOrNot<lazy, StorageSystemEnabledRoles>(context, system_database, "enabled_roles");
attachLazyOrNot<lazy, StorageSystemSettingsProfiles>(context, system_database, "settings_profiles");
attachLazyOrNot<lazy, StorageSystemSettingsProfileElements>(context, system_database, "settings_profile_elements");
attachLazyOrNot<lazy, StorageSystemRowPolicies>(context, system_database, "row_policies");
attachLazyOrNot<lazy, StorageSystemQuotas>(context, system_database, "quotas");
attachLazyOrNot<lazy, StorageSystemQuotaLimits>(context, system_database, "quota_limits");
attachLazyOrNot<lazy, StorageSystemQuotaUsage>(context, system_database, "quota_usage");
attachLazyOrNot<lazy, StorageSystemQuotasUsage>(context, system_database, "quotas_usage");
attachLazyOrNot<lazy, StorageSystemUserDirectories>(context, system_database, "user_directories");
attachLazyOrNot<lazy, StorageSystemPrivileges>(context, system_database, "privileges");
attachLazyOrNot<lazy, StorageSystemErrors>(context, system_database, "errors");
attachLazyOrNot<lazy, StorageSystemWarnings>(context, system_database, "warnings");
attachLazyOrNot<lazy, StorageSystemDataSkippingIndices>(context, system_database, "data_skipping_indices");
attachLazyOrNot<lazy, StorageSystemLicenses>(context, system_database, "licenses");
attachLazyOrNot<lazy, StorageSystemTimeZones>(context, system_database, "time_zones");
attachLazyOrNot<lazy, StorageSystemBackups>(context, system_database, "backups");
attachLazyOrNot<lazy, StorageSystemSchemaInferenceCache>(context, system_database, "schema_inference_cache");
attachLazyOrNot<lazy, StorageSystemDroppedTables>(context, system_database, "dropped_tables");
attachLazyOrNot<lazy, StorageSystemScheduler>(context, system_database, "scheduler");
#if defined(__ELF__) && !defined(OS_FREEBSD)
attach<StorageSystemSymbols>(context, system_database, "symbols");
attachLazyOrNot<lazy, StorageSystemSymbols>(context, system_database, "symbols");
#endif
#if USE_RDKAFKA
attach<StorageSystemKafkaConsumers>(context, system_database, "kafka_consumers");
attachLazyOrNot<lazy, StorageSystemKafkaConsumers>(context, system_database, "kafka_consumers");
#endif
#ifdef OS_LINUX
attach<StorageSystemStackTrace>(context, system_database, "stack_trace");
attachLazyOrNot<lazy, StorageSystemStackTrace>(context, system_database, "stack_trace");
#endif
#if USE_ROCKSDB
attach<StorageSystemRocksDB>(context, system_database, "rocksdb");
attachLazyOrNot<lazy, StorageSystemRocksDB>(context, system_database, "rocksdb");
#endif
}
template void attachSystemTablesLocal<false>(ContextPtr context, IDatabase & system_database);
template void attachSystemTablesLocal<true>(ContextPtr context, IDatabase & system_database);
void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper)
{
attachSystemTablesLocal(context, system_database);
attachSystemTablesLocal</* lazy= */ false>(context, system_database);
attach<StorageSystemParts>(context, system_database, "parts");
attach<StorageSystemProjectionParts>(context, system_database, "projection_parts");

6
src/Storages/System/attachSystemTables.h
View File

@ -9,8 +9,12 @@ namespace DB
class AsynchronousMetrics;
class IDatabase;
void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper);
template <bool lazy>
void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database);
void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, bool has_zookeeper);
void attachSystemTablesAsync(ContextPtr context, IDatabase & system_database, AsynchronousMetrics & async_metrics);
extern template void attachSystemTablesLocal<false>(ContextPtr context, IDatabase & system_database);
extern template void attachSystemTablesLocal<true>(ContextPtr context, IDatabase & system_database);
}

40
src/Storages/System/attachSystemTablesImpl.h
View File

@ -1,15 +1,42 @@
#pragma once
#include <Databases/IDatabase.h>
#include <Databases/DatabasesCommon.h>
#include <Interpreters/DatabaseCatalog.h>
namespace DB
{
template<typename StorageT, typename... StorageArgs>
void attachLazy(ContextPtr context, IDatabase & system_database, const String & table_name, StorageArgs && ... args)
{
if (system_database.getUUID() == UUIDHelpers::Nil)
{
/// Attach to Ordinary database.
auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name);
system_database.registerLazyTable(context, table_name, [table_id, ... captured_args = std::forward<StorageArgs>(args)] mutable
{
return std::make_shared<StorageT>(table_id, std::forward<StorageArgs>(captured_args)...);
});
}
else
{
/// Attach to Atomic database.
/// NOTE: UUIDs are not persistent, but it's ok since no data are stored on disk for these storages
/// and path is actually not used
auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name, UUIDHelpers::generateV4());
DatabaseCatalog::instance().addUUIDMapping(table_id.uuid);
String path = "store/" + DatabaseCatalog::getPathForUUID(table_id.uuid);
system_database.registerLazyTable(context, table_name, [table_id, ... captured_args = std::forward<StorageArgs>(args)] mutable
{
return std::make_shared<StorageT>(table_id, std::forward<StorageArgs>(captured_args)...);
}, path);
}
}
template<typename StorageT, typename... StorageArgs>
void attach(ContextPtr context, IDatabase & system_database, const String & table_name, StorageArgs && ... args)
{
assert(system_database.getDatabaseName() == DatabaseCatalog::SYSTEM_DATABASE);
if (system_database.getUUID() == UUIDHelpers::Nil)
{
/// Attach to Ordinary database.
@ -28,4 +55,15 @@ void attach(ContextPtr context, IDatabase & system_database, const String & tabl
}
}
template<bool lazy, typename StorageT, typename... StorageArgs>
void attachLazyOrNot(ContextPtr context, IDatabase & system_database, const String & table_name, StorageArgs && ... args)
{
assert(system_database.getDatabaseName() == DatabaseCatalog::SYSTEM_DATABASE);
if constexpr (lazy)
attachLazy<StorageT>(context, system_database, table_name, std::forward<StorageArgs>(args)...);
else
attach<StorageT>(context, system_database, table_name, std::forward<StorageArgs>(args)...);
}
}

37
src/Storages/TTLDescription.cpp
View File

@ -55,22 +55,29 @@ TTLAggregateDescription & TTLAggregateDescription::operator=(const TTLAggregateD
namespace
{
void checkTTLExpression(const ExpressionActionsPtr & ttl_expression, const String & result_column_name)
void checkTTLExpression(const ExpressionActionsPtr & ttl_expression, const String & result_column_name, bool allow_suspicious)
{
for (const auto & action : ttl_expression->getActions())
/// Do not apply this check in ATTACH queries for compatibility reasons and if explicitly allowed.
if (!allow_suspicious)
{
if (action.node->type == ActionsDAG::ActionType::FUNCTION)
if (ttl_expression->getRequiredColumns().empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"TTL expression {} does not depend on any of the columns of the table", result_column_name);
for (const auto & action : ttl_expression->getActions())
{
const IFunctionBase & func = *action.node->function_base;
if (!func.isDeterministic())
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"TTL expression cannot contain non-deterministic functions, but contains function {}",
func.getName());
if (action.node->type == ActionsDAG::ActionType::FUNCTION)
{
const IFunctionBase & func = *action.node->function_base;
if (!func.isDeterministic())
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"TTL expression cannot contain non-deterministic functions, but contains function {}",
func.getName());
}
}
}
const auto & result_column = ttl_expression->getSampleBlock().getByName(result_column_name);
if (!typeid_cast<const DataTypeDateTime *>(result_column.type.get())
&& !typeid_cast<const DataTypeDate *>(result_column.type.get()))
{
@ -162,7 +169,8 @@ TTLDescription TTLDescription::getTTLFromAST(
const ASTPtr & definition_ast,
const ColumnsDescription & columns,
ContextPtr context,
const KeyDescription & primary_key)
const KeyDescription & primary_key,
bool is_attach)
{
TTLDescription result;
const auto * ttl_element = definition_ast->as<ASTTTLElement>();
@ -289,7 +297,7 @@ TTLDescription TTLDescription::getTTLFromAST(
}
}
checkTTLExpression(result.expression, result.result_column);
checkTTLExpression(result.expression, result.result_column, is_attach || context->getSettingsRef().allow_suspicious_ttl_expressions);
return result;
}
@ -327,7 +335,8 @@ TTLTableDescription TTLTableDescription::getTTLForTableFromAST(
const ASTPtr & definition_ast,
const ColumnsDescription & columns,
ContextPtr context,
const KeyDescription & primary_key)
const KeyDescription & primary_key,
bool is_attach)
{
TTLTableDescription result;
if (!definition_ast)
@ -338,7 +347,7 @@ TTLTableDescription TTLTableDescription::getTTLForTableFromAST(
bool have_unconditional_delete_ttl = false;
for (const auto & ttl_element_ptr : definition_ast->children)
{
auto ttl = TTLDescription::getTTLFromAST(ttl_element_ptr, columns, context, primary_key);
auto ttl = TTLDescription::getTTLFromAST(ttl_element_ptr, columns, context, primary_key, is_attach);
if (ttl.mode == TTLMode::DELETE)
{
if (!ttl.where_expression)
@ -380,7 +389,7 @@ TTLTableDescription TTLTableDescription::parse(const String & str, const Columns
ASTPtr ast = parseQuery(parser, str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
FunctionNameNormalizer().visit(ast.get());
return getTTLForTableFromAST(ast, columns, context, primary_key);
return getTTLForTableFromAST(ast, columns, context, primary_key, context->getSettingsRef().allow_suspicious_ttl_expressions);
}
}

9
src/Storages/TTLDescription.h
View File

@ -1,4 +1,5 @@
#pragma once
#include <Parsers/IAST_fwd.h>
#include <Storages/DataDestinationType.h>
#include <Storages/ColumnsDescription.h>
@ -7,6 +8,7 @@
#include <Interpreters/AggregateDescription.h>
#include <Storages/TTLMode.h>
namespace DB
{
@ -82,9 +84,8 @@ struct TTLDescription
/// Codec name which will be used to recompress data
ASTPtr recompression_codec;
/// Parse TTL structure from definition. Able to parse both column and table
/// TTLs.
static TTLDescription getTTLFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, ContextPtr context, const KeyDescription & primary_key);
/// Parse TTL structure from definition. Able to parse both column and table TTLs.
static TTLDescription getTTLFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, ContextPtr context, const KeyDescription & primary_key, bool is_attach);
TTLDescription() = default;
TTLDescription(const TTLDescription & other);
@ -121,7 +122,7 @@ struct TTLTableDescription
TTLTableDescription & operator=(const TTLTableDescription & other);
static TTLTableDescription getTTLForTableFromAST(
const ASTPtr & definition_ast, const ColumnsDescription & columns, ContextPtr context, const KeyDescription & primary_key);
const ASTPtr & definition_ast, const ColumnsDescription & columns, ContextPtr context, const KeyDescription & primary_key, bool is_attach);
/// Parse description from string
static TTLTableDescription parse(const String & str, const ColumnsDescription & columns, ContextPtr context, const KeyDescription & primary_key);

8
tests/ci/ci_config.py
View File

@ -208,6 +208,13 @@ CI_CONFIG = CiConfig(
static_binary_name="amd64compat",
comment="SSE2-only build",
),
"binary_amd64_musl": BuildConfig(
name="binary_amd64_musl",
compiler="clang-17-amd64-musl",
package_type="binary",
static_binary_name="amd64musl",
comment="Build with Musl",
),
"binary_riscv64": BuildConfig(
name="binary_riscv64",
compiler="clang-17-riscv64",
@ -249,6 +256,7 @@ CI_CONFIG = CiConfig(
"binary_riscv64",
"binary_s390x",
"binary_amd64_compat",
"binary_amd64_musl",
],
},
test_configs={

14
tests/clickhouse-test
View File

@ -2205,7 +2205,7 @@ def reportLogStats(args):
GROUP BY message_format_string
ORDER BY count DESC
LIMIT 100
FORMAT TSVWithNamesAndTypes
FORMAT PrettySpaceNoEscapes
"""
value = clickhouse_execute(args, query).decode(errors="replace")
print("\nTop patterns of log messages:\n")
@ -2225,7 +2225,7 @@ def reportLogStats(args):
GROUP BY pattern
ORDER BY count DESC
LIMIT 30
FORMAT TSVWithNamesAndTypes
FORMAT PrettySpaceNoEscapes
"""
value = clickhouse_execute(args, query).decode(errors="replace")
print("\nTop messages without format string (fmt::runtime):\n")
@ -2238,10 +2238,10 @@ def reportLogStats(args):
WHERE (now() - toIntervalMinute(240)) < event_time
AND (message NOT LIKE (replaceRegexpAll(message_format_string, '{[:.0-9dfx]*}', '%') AS s))
AND (message NOT LIKE concat('%Exception: ', s, '%'))
GROUP BY message_format_string ORDER BY count() DESC LIMIT 20 FORMAT TSVWithNamesAndTypes
GROUP BY message_format_string ORDER BY count() DESC LIMIT 20 FORMAT PrettySpaceNoEscapes
"""
value = clickhouse_execute(args, query).decode(errors="replace")
print("\nTop messages that does not match its format string:\n")
print("\nTop messages not matching their format strings:\n")
print(value)
print("\n")
@ -2269,13 +2269,13 @@ def reportLogStats(args):
'Attempt to read after eof', 'String size is too big ({}), maximum: {}'
) AS known_short_messages
SELECT count() AS c, message_format_string, substr(any(message), 1, 120),
min(if(length(regexpExtract(message, '(.*)\\([A-Z0-9_]+\\)')) as pref > 0, pref, length(message)) - 26 AS length_without_exception_boilerplate) AS min_length_without_exception_boilerplate
min(if(notEmpty(regexpExtract(message, '(.*)\\([A-Z0-9_]+\\)') as prefix), prefix, length(message)) - 26 AS length_without_exception_boilerplate) AS min_length_without_exception_boilerplate
FROM system.text_log
WHERE (now() - toIntervalMinute(240)) < event_time
AND (length(message_format_string) < 16
OR (message ilike '%DB::Exception%' AND length_without_exception_boilerplate < 30))
OR (message ILIKE '%DB::Exception%' AND length_without_exception_boilerplate < 30))
AND message_format_string NOT IN known_short_messages
GROUP BY message_format_string ORDER BY c DESC LIMIT 50 FORMAT TSVWithNamesAndTypes
GROUP BY message_format_string ORDER BY c DESC LIMIT 50 FORMAT PrettySpaceNoEscapes
"""
value = clickhouse_execute(args, query).decode(errors="replace")
print("\nTop short messages:\n")

11
tests/integration/helpers/cluster.py
View File

@ -390,6 +390,7 @@ class ClickHouseCluster:
odbc_bridge_bin_path=None,
library_bridge_bin_path=None,
zookeeper_config_path=None,
keeper_config_dir=None,
custom_dockerd_host=None,
zookeeper_keyfile=None,
zookeeper_certfile=None,
@ -426,6 +427,12 @@ class ClickHouseCluster:
else p.join(HELPERS_DIR, "zookeeper_config.xml")
)
self.keeper_config_dir = (
p.join(self.base_dir, keeper_config_dir)
if keeper_config_dir
else HELPERS_DIR
)
project_name = (
pwd.getpwuid(os.getuid()).pw_name + p.basename(self.base_dir) + self.name
)
@ -2725,7 +2732,9 @@ class ClickHouseCluster:
if self.use_keeper: # TODO: remove hardcoded paths from here
for i in range(1, 4):
shutil.copy(
os.path.join(HELPERS_DIR, f"keeper_config{i}.xml"),
os.path.join(
self.keeper_config_dir, f"keeper_config{i}.xml"
),
os.path.join(
self.keeper_instance_dir_prefix + f"{i}", "config"
),

14
tests/integration/helpers/postgres_utility.py
View File

@ -193,11 +193,11 @@ class PostgresManager:
database_name = self.database_or_default(database_name)
self.drop_postgres_db(database_name)
self.created_postgres_db_list.add(database_name)
self.cursor.execute(f"CREATE DATABASE {database_name}")
self.cursor.execute(f'CREATE DATABASE "{database_name}"')
def drop_postgres_db(self, database_name=""):
database_name = self.database_or_default(database_name)
self.cursor.execute(f"DROP DATABASE IF EXISTS {database_name} WITH (FORCE)")
self.cursor.execute(f'DROP DATABASE IF EXISTS "{database_name}" WITH (FORCE)')
if database_name in self.created_postgres_db_list:
self.created_postgres_db_list.remove(database_name)
@ -216,19 +216,19 @@ class PostgresManager:
if len(schema_name) == 0:
self.instance.query(
f"""
CREATE DATABASE {database_name}
CREATE DATABASE \"{database_name}\"
ENGINE = PostgreSQL('{self.ip}:{self.port}', '{postgres_database}', 'postgres', 'mysecretpassword')"""
)
else:
self.instance.query(
f"""
CREATE DATABASE {database_name}
CREATE DATABASE \"{database_name}\"
ENGINE = PostgreSQL('{self.ip}:{self.port}', '{postgres_database}', 'postgres', 'mysecretpassword', '{schema_name}')"""
)
def drop_clickhouse_postgres_db(self, database_name=""):
database_name = self.database_or_default(database_name)
self.instance.query(f"DROP DATABASE IF EXISTS {database_name}")
self.instance.query(f'DROP DATABASE IF EXISTS "{database_name}"')
if database_name in self.created_ch_postgres_db_list:
self.created_ch_postgres_db_list.remove(database_name)
@ -368,7 +368,7 @@ def check_tables_are_synchronized(
result_query = f"select * from {table_path} order by {order_by};"
expected = instance.query(
f"select * from {postgres_database}.{table_name} order by {order_by};"
f"select * from `{postgres_database}`.`{table_name}` order by {order_by};"
)
result = instance.query(result_query)
@ -382,7 +382,7 @@ def check_tables_are_synchronized(
if result != expected:
count = int(instance.query(f"select count() from {table_path}"))
expected_count = int(
instance.query(f"select count() from {postgres_database}.{table_name}")
instance.query(f"select count() from `{postgres_database}`.`{table_name}`")
)
print(f"Having {count}, expected {expected_count}")
assert result == expected

12
tests/integration/test_dictionaries_update_and_reload/test.py
View File

@ -92,16 +92,16 @@ def test_reload_while_loading(started_cluster):
assert get_status("slow") == "NOT_LOADED"
assert get_loading_duration("slow") == 0
# It's not possible to get a value from the dictionary within 0.5 second, so the following query fails by timeout.
# It's not possible to get a value from the dictionary within 1 second, so the following query fails by timeout.
with pytest.raises(QueryTimeoutExceedException):
query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout=0.5)
query("SELECT dictGetInt32('slow', 'a', toUInt64(5))", timeout=1)
# The dictionary is now loading.
assert get_status("slow") == "LOADING"
start_time, duration = get_loading_start_time("slow"), get_loading_duration("slow")
assert duration > 0
time.sleep(0.5) # Still loading.
time.sleep(1) # Still loading.
assert get_status("slow") == "LOADING"
prev_start_time, prev_duration = start_time, duration
start_time, duration = get_loading_start_time("slow"), get_loading_duration("slow")
@ -110,14 +110,14 @@ def test_reload_while_loading(started_cluster):
# SYSTEM RELOAD DICTIONARY should restart loading.
with pytest.raises(QueryTimeoutExceedException):
query("SYSTEM RELOAD DICTIONARY 'slow'", timeout=0.5)
query("SYSTEM RELOAD DICTIONARY 'slow'", timeout=1)
assert get_status("slow") == "LOADING"
prev_start_time, prev_duration = start_time, duration
start_time, duration = get_loading_start_time("slow"), get_loading_duration("slow")
assert start_time > prev_start_time
assert duration < prev_duration
time.sleep(0.5) # Still loading.
time.sleep(1) # Still loading.
assert get_status("slow") == "LOADING"
prev_start_time, prev_duration = start_time, duration
start_time, duration = get_loading_start_time("slow"), get_loading_duration("slow")
@ -128,7 +128,7 @@ def test_reload_while_loading(started_cluster):
replace_in_file_in_container(
"/etc/clickhouse-server/dictionaries/slow.xml", "sleep 100", "sleep 0"
)
time.sleep(5) # Configuration files are reloaded once in 5 seconds.
query("SYSTEM RELOAD CONFIG")
# This time loading should finish quickly.
assert get_status("slow") == "LOADED"

Some files were not shown because too many files have changed in this diff Show More