2011-12-30 17:57:02 +00:00
|
|
|
#pragma once
|
2011-06-27 18:22:14 +00:00
|
|
|
|
2012-06-22 16:54:51 +00:00
|
|
|
#include <boost/noncopyable.hpp>
|
2011-06-27 18:22:14 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Common/ProfileEvents.h>
|
|
|
|
#include <Common/Allocator.h>
|
2014-01-04 04:53:07 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Common/Exception.h>
|
|
|
|
#include <Core/Defines.h>
|
2011-06-27 18:22:14 +00:00
|
|
|
|
2022-08-17 22:35:50 +00:00
|
|
|
#include <base/arithmeticOverflow.h>
|
|
|
|
|
2011-06-27 18:22:14 +00:00
|
|
|
|
2016-10-24 02:02:37 +00:00
|
|
|
namespace ProfileEvents
|
|
|
|
{
|
|
|
|
extern const Event IOBufferAllocs;
|
|
|
|
extern const Event IOBufferAllocBytes;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2011-06-27 18:22:14 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2022-08-17 22:35:50 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int ARGUMENT_OUT_OF_BOUND;
|
|
|
|
}
|
|
|
|
|
2011-06-27 18:22:14 +00:00
|
|
|
|
2016-10-24 02:02:37 +00:00
|
|
|
/** Replacement for std::vector<char> to use in buffers.
|
|
|
|
* Differs in that is doesn't do unneeded memset. (And also tries to do as little as possible.)
|
|
|
|
* Also allows to allocate aligned piece of memory (to use with O_DIRECT, for example).
|
2013-09-08 05:53:10 +00:00
|
|
|
*/
|
2019-04-06 15:27:39 +00:00
|
|
|
template <typename Allocator = Allocator<false>>
|
|
|
|
struct Memory : boost::noncopyable, Allocator
|
2012-06-21 18:43:29 +00:00
|
|
|
{
|
2022-10-25 11:56:28 +00:00
|
|
|
static constexpr size_t pad_right = PADDING_FOR_SIMD - 1;
|
2018-08-30 20:12:15 +00:00
|
|
|
|
|
|
|
size_t m_capacity = 0; /// With padding.
|
2014-03-13 20:12:40 +00:00
|
|
|
size_t m_size = 0;
|
|
|
|
char * m_data = nullptr;
|
|
|
|
size_t alignment = 0;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2021-02-15 13:10:14 +00:00
|
|
|
Memory() = default;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2016-10-24 02:02:37 +00:00
|
|
|
/// If alignment != 0, then allocate memory aligned to specified value.
|
2022-08-17 22:35:50 +00:00
|
|
|
explicit Memory(size_t size_, size_t alignment_ = 0) : alignment(alignment_)
|
2013-10-13 06:53:11 +00:00
|
|
|
{
|
2022-08-17 22:35:50 +00:00
|
|
|
alloc(size_);
|
2013-10-13 06:53:11 +00:00
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2013-09-08 05:53:10 +00:00
|
|
|
~Memory()
|
|
|
|
{
|
2013-10-13 06:53:11 +00:00
|
|
|
dealloc();
|
2013-09-08 05:53:10 +00:00
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2021-07-26 00:34:36 +00:00
|
|
|
void swap(Memory & rhs) noexcept
|
2014-03-13 20:12:40 +00:00
|
|
|
{
|
|
|
|
std::swap(m_capacity, rhs.m_capacity);
|
|
|
|
std::swap(m_size, rhs.m_size);
|
|
|
|
std::swap(m_data, rhs.m_data);
|
|
|
|
std::swap(alignment, rhs.alignment);
|
2021-07-26 00:34:36 +00:00
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2021-07-26 00:34:36 +00:00
|
|
|
Memory(Memory && rhs) noexcept
|
|
|
|
{
|
|
|
|
swap(rhs);
|
|
|
|
}
|
|
|
|
|
|
|
|
Memory & operator=(Memory && rhs) noexcept
|
|
|
|
{
|
|
|
|
swap(rhs);
|
2014-03-13 20:12:40 +00:00
|
|
|
return *this;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2013-09-08 05:53:10 +00:00
|
|
|
size_t size() const { return m_size; }
|
|
|
|
const char & operator[](size_t i) const { return m_data[i]; }
|
|
|
|
char & operator[](size_t i) { return m_data[i]; }
|
2016-07-31 05:56:36 +00:00
|
|
|
const char * data() const { return m_data; }
|
|
|
|
char * data() { return m_data; }
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2013-09-08 05:53:10 +00:00
|
|
|
void resize(size_t new_size)
|
|
|
|
{
|
2022-08-17 22:35:50 +00:00
|
|
|
if (!m_data)
|
2015-09-01 15:08:37 +00:00
|
|
|
{
|
2022-08-17 22:35:50 +00:00
|
|
|
alloc(new_size);
|
|
|
|
return;
|
2015-09-01 15:08:37 +00:00
|
|
|
}
|
2022-08-17 22:35:50 +00:00
|
|
|
|
|
|
|
if (new_size <= m_capacity - pad_right)
|
2012-06-21 18:43:29 +00:00
|
|
|
{
|
2013-09-08 05:53:10 +00:00
|
|
|
m_size = new_size;
|
|
|
|
return;
|
|
|
|
}
|
2021-10-14 18:07:49 +00:00
|
|
|
|
2022-08-18 12:25:52 +00:00
|
|
|
size_t new_capacity = withPadding(new_size);
|
2021-10-14 18:07:49 +00:00
|
|
|
|
2022-08-17 22:35:50 +00:00
|
|
|
size_t diff = new_capacity - m_capacity;
|
|
|
|
ProfileEvents::increment(ProfileEvents::IOBufferAllocBytes, diff);
|
|
|
|
|
|
|
|
m_data = static_cast<char *>(Allocator::realloc(m_data, m_capacity, new_capacity, alignment));
|
|
|
|
m_capacity = new_capacity;
|
|
|
|
m_size = new_size;
|
2013-10-13 06:53:11 +00:00
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-09-07 21:04:48 +00:00
|
|
|
private:
|
2022-08-18 12:25:52 +00:00
|
|
|
static size_t withPadding(size_t value)
|
2015-08-16 13:00:22 +00:00
|
|
|
{
|
2022-08-17 22:35:50 +00:00
|
|
|
size_t res = 0;
|
|
|
|
|
2022-08-18 12:25:52 +00:00
|
|
|
if (common::addOverflow<size_t>(value, pad_right, res))
|
2023-01-23 21:13:58 +00:00
|
|
|
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "value is too big to apply padding");
|
2022-08-17 22:35:50 +00:00
|
|
|
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
|
|
|
void alloc(size_t new_size)
|
|
|
|
{
|
|
|
|
if (!new_size)
|
2013-10-13 06:53:11 +00:00
|
|
|
{
|
2014-04-08 07:31:51 +00:00
|
|
|
m_data = nullptr;
|
2013-10-13 06:53:11 +00:00
|
|
|
return;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2022-08-18 12:25:52 +00:00
|
|
|
size_t new_capacity = withPadding(new_size);
|
2022-08-17 22:35:50 +00:00
|
|
|
|
2014-01-04 04:53:07 +00:00
|
|
|
ProfileEvents::increment(ProfileEvents::IOBufferAllocs);
|
2022-08-17 22:35:50 +00:00
|
|
|
ProfileEvents::increment(ProfileEvents::IOBufferAllocBytes, new_capacity);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2016-10-24 02:02:37 +00:00
|
|
|
m_data = static_cast<char *>(Allocator::alloc(new_capacity, alignment));
|
2015-08-16 13:00:22 +00:00
|
|
|
m_capacity = new_capacity;
|
2022-08-17 22:35:50 +00:00
|
|
|
m_size = new_size;
|
2013-10-13 06:53:11 +00:00
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2013-10-13 06:53:11 +00:00
|
|
|
void dealloc()
|
|
|
|
{
|
2014-05-03 22:57:43 +00:00
|
|
|
if (!m_data)
|
|
|
|
return;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2016-10-24 02:02:37 +00:00
|
|
|
Allocator::free(m_data, m_capacity);
|
|
|
|
m_data = nullptr; /// To avoid double free if next alloc will throw an exception.
|
2013-09-08 05:53:10 +00:00
|
|
|
}
|
|
|
|
};
|
2012-06-21 18:43:29 +00:00
|
|
|
|
|
|
|
|
2016-10-24 02:02:37 +00:00
|
|
|
/** Buffer that could own its working memory.
|
|
|
|
* Template parameter: ReadBuffer or WriteBuffer
|
2011-06-27 18:22:14 +00:00
|
|
|
*/
|
|
|
|
template <typename Base>
|
|
|
|
class BufferWithOwnMemory : public Base
|
|
|
|
{
|
|
|
|
protected:
|
2019-04-06 15:27:39 +00:00
|
|
|
Memory<> memory;
|
2011-06-27 18:22:14 +00:00
|
|
|
public:
|
2016-10-24 02:02:37 +00:00
|
|
|
/// If non-nullptr 'existing_memory' is passed, then buffer will not create its own memory and will use existing_memory without ownership.
|
2021-02-15 13:10:14 +00:00
|
|
|
explicit BufferWithOwnMemory(size_t size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0)
|
2016-07-31 05:56:36 +00:00
|
|
|
: Base(nullptr, 0), memory(existing_memory ? 0 : size, alignment)
|
2011-06-27 18:22:14 +00:00
|
|
|
{
|
2016-07-31 05:56:36 +00:00
|
|
|
Base::set(existing_memory ? existing_memory : memory.data(), size);
|
Padding for IO buffers.
Testing data
```
select 'aaaaaaaa','bbbbbbbb','cccccccc','dddddddd','eeeeeeee','ffffffff','gggg','hhh' from numbers(3000000) into outfile '/tmp/test.tsv'
```
Testing command
```
echo "select count() from file('/tmp/test.tsv', CSV, 'a String, b String, c String, d String, e String, f String, g String, h String') where not ignore(e)" | clickhouse-benchmark
```
TSV parser has less overhead than CSV, using it would better unveil the benefits of memcpySmall.
Before
```
QPS: 1.662, RPS: 4985463.906, MiB/s: 603.823, result RPS: 1.662, result MiB/s: 0.000.
0.000% 0.559 sec.
10.000% 0.564 sec.
20.000% 0.568 sec.
30.000% 0.572 sec.
40.000% 0.575 sec.
50.000% 0.581 sec.
60.000% 0.592 sec.
70.000% 0.624 sec.
80.000% 0.639 sec.
90.000% 0.664 sec.
95.000% 0.686 sec.
99.000% 0.711 sec.
99.900% 0.715 sec.
99.990% 0.716 sec.
```
After
```
QPS: 1.861, RPS: 5582303.107, MiB/s: 676.110, result RPS: 1.861, result MiB/s: 0.000.
0.000% 0.510 sec.
10.000% 0.514 sec.
20.000% 0.517 sec.
30.000% 0.521 sec.
40.000% 0.523 sec.
50.000% 0.527 sec.
60.000% 0.530 sec.
70.000% 0.539 sec.
80.000% 0.558 sec.
90.000% 0.584 sec.
95.000% 0.589 sec.
99.000% 0.608 sec.
99.900% 0.655 sec.
99.990% 0.663 sec.
```
2018-08-27 19:14:15 +00:00
|
|
|
Base::padded = !existing_memory;
|
2011-06-27 18:22:14 +00:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
|
2020-06-11 23:00:49 +00:00
|
|
|
/** Buffer that could write data to external memory which came from outside
|
|
|
|
* Template parameter: ReadBuffer or WriteBuffer
|
|
|
|
*/
|
|
|
|
template <typename Base>
|
|
|
|
class BufferWithOutsideMemory : public Base
|
|
|
|
{
|
|
|
|
protected:
|
|
|
|
Memory<> & memory;
|
|
|
|
public:
|
|
|
|
|
2020-08-14 00:34:35 +00:00
|
|
|
explicit BufferWithOutsideMemory(Memory<> & memory_)
|
2020-10-05 15:59:29 +00:00
|
|
|
: Base(memory_.data(), memory_.size()), memory(memory_)
|
2020-06-11 23:00:49 +00:00
|
|
|
{
|
2020-10-05 15:59:29 +00:00
|
|
|
Base::set(memory.data(), memory.size(), 0);
|
2020-06-11 23:00:49 +00:00
|
|
|
Base::padded = false;
|
|
|
|
}
|
2020-08-14 00:34:35 +00:00
|
|
|
|
|
|
|
size_t getActualSize()
|
|
|
|
{
|
|
|
|
return Base::count();
|
|
|
|
}
|
|
|
|
|
2020-06-11 23:00:49 +00:00
|
|
|
private:
|
2022-06-27 21:00:37 +00:00
|
|
|
void nextImpl() final
|
2020-06-11 23:00:49 +00:00
|
|
|
{
|
2020-12-29 22:34:26 +00:00
|
|
|
const size_t prev_size = Base::position() - memory.data();
|
2020-10-05 15:59:29 +00:00
|
|
|
memory.resize(2 * prev_size + 1);
|
2020-10-06 14:02:01 +00:00
|
|
|
Base::set(memory.data() + prev_size, memory.size() - prev_size, 0);
|
2020-06-11 23:00:49 +00:00
|
|
|
}
|
2023-11-18 16:30:49 +00:00
|
|
|
|
|
|
|
void finalizeImpl() final
|
|
|
|
{
|
|
|
|
/// there is no need to allocate twice more memory at finalize()
|
|
|
|
/// So make that call no op, do not call here nextImpl()
|
|
|
|
}
|
2020-06-11 23:00:49 +00:00
|
|
|
};
|
|
|
|
|
2011-06-27 18:22:14 +00:00
|
|
|
}
|