mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-17 21:24:28 +00:00
98e3a99a88
Since this hides real problems, since destructor does final flush and if
it fails, then data will be lost.
One of such examples if MEMORY_LIMIT_EXCEEDED exception, so lock
exceptions from destructors, by using
MemoryTracker::LockExceptionInThread to block these exception, and allow
others (so std::terminate will be called, since this is c++11 with
noexcept for destructors by default).
Here is an example, that leads to empty block in the distributed batch:
2021.01.21 12:43:18.619739 [ 46468 ] {7bd60d75-ebcb-45d2-874d-260df9a4ddac} <Error> virtual DB::CompressedWriteBuffer::~CompressedWriteBuffer(): Code: 241, e.displayText() = DB::Exception: Memory limit (for user) exceeded: would use 332.07 GiB (attempt to allocate chunk of 4355342 bytes), maximum: 256.00 GiB, Stack trace (when copying this message, always include the lines below):
0. DB::Exception::Exception<>() @ 0x86f7b88 in /usr/bin/clickhouse
...
4. void DB::PODArrayBase<>::resize<>(unsigned long) @ 0xe9e878d in /usr/bin/clickhouse
5. DB::CompressedWriteBuffer::nextImpl() @ 0xe9f0296 in /usr/bin/clickhouse
6. DB::CompressedWriteBuffer::~CompressedWriteBuffer() @ 0xe9f0415 in /usr/bin/clickhouse
7. DB::DistributedBlockOutputStream::writeToShard() @ 0xf6bed4a in /usr/bin/clickhouse
106 lines
2.3 KiB
C++
106 lines
2.3 KiB
C++
#include <sys/stat.h>
|
|
#include <fcntl.h>
|
|
#include <errno.h>
|
|
|
|
#include <Common/ProfileEvents.h>
|
|
#include <Common/MemoryTracker.h>
|
|
|
|
#include <IO/WriteBufferFromFile.h>
|
|
#include <IO/WriteHelpers.h>
|
|
|
|
|
|
namespace ProfileEvents
|
|
{
|
|
extern const Event FileOpen;
|
|
}
|
|
|
|
namespace DB
|
|
{
|
|
|
|
namespace ErrorCodes
|
|
{
|
|
extern const int FILE_DOESNT_EXIST;
|
|
extern const int CANNOT_OPEN_FILE;
|
|
extern const int CANNOT_CLOSE_FILE;
|
|
}
|
|
|
|
|
|
WriteBufferFromFile::WriteBufferFromFile(
|
|
const std::string & file_name_,
|
|
size_t buf_size,
|
|
int flags,
|
|
mode_t mode,
|
|
char * existing_memory,
|
|
size_t alignment)
|
|
: WriteBufferFromFileDescriptor(-1, buf_size, existing_memory, alignment), file_name(file_name_)
|
|
{
|
|
ProfileEvents::increment(ProfileEvents::FileOpen);
|
|
|
|
#ifdef __APPLE__
|
|
bool o_direct = (flags != -1) && (flags & O_DIRECT);
|
|
if (o_direct)
|
|
flags = flags & ~O_DIRECT;
|
|
#endif
|
|
|
|
fd = ::open(file_name.c_str(), flags == -1 ? O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC : flags | O_CLOEXEC, mode);
|
|
|
|
if (-1 == fd)
|
|
throwFromErrnoWithPath("Cannot open file " + file_name, file_name,
|
|
errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE);
|
|
|
|
#ifdef __APPLE__
|
|
if (o_direct)
|
|
{
|
|
if (fcntl(fd, F_NOCACHE, 1) == -1)
|
|
throwFromErrnoWithPath("Cannot set F_NOCACHE on file " + file_name, file_name, ErrorCodes::CANNOT_OPEN_FILE);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
|
|
/// Use pre-opened file descriptor.
|
|
WriteBufferFromFile::WriteBufferFromFile(
|
|
int & fd_,
|
|
const std::string & original_file_name,
|
|
size_t buf_size,
|
|
char * existing_memory,
|
|
size_t alignment)
|
|
:
|
|
WriteBufferFromFileDescriptor(fd_, buf_size, existing_memory, alignment),
|
|
file_name(original_file_name.empty() ? "(fd = " + toString(fd_) + ")" : original_file_name)
|
|
{
|
|
fd_ = -1;
|
|
}
|
|
|
|
|
|
WriteBufferFromFile::~WriteBufferFromFile()
|
|
{
|
|
if (fd < 0)
|
|
return;
|
|
|
|
/// FIXME move final flush into the caller
|
|
MemoryTracker::LockExceptionInThread lock;
|
|
|
|
next();
|
|
|
|
::close(fd);
|
|
}
|
|
|
|
|
|
/// Close file before destruction of object.
|
|
void WriteBufferFromFile::close()
|
|
{
|
|
if (fd < 0)
|
|
return;
|
|
|
|
next();
|
|
|
|
if (0 != ::close(fd))
|
|
throw Exception("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE);
|
|
|
|
fd = -1;
|
|
metric_increment.destroy();
|
|
}
|
|
|
|
}
|