2020-01-28 16:08:30 +00:00
|
|
|
#include <gtest/gtest.h>
|
|
|
|
|
|
|
|
#include <Columns/ColumnsNumber.h>
|
|
|
|
#include <DataStreams/IBlockOutputStream.h>
|
|
|
|
#include <DataStreams/copyData.h>
|
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
2020-01-28 16:09:24 +00:00
|
|
|
#include <Disks/tests/gtest_disk.h>
|
2020-01-28 16:08:30 +00:00
|
|
|
#include <Formats/FormatFactory.h>
|
|
|
|
#include <IO/ReadHelpers.h>
|
|
|
|
#include <IO/WriteBufferFromOStream.h>
|
|
|
|
#include <Interpreters/Context.h>
|
|
|
|
#include <Storages/StorageLog.h>
|
2020-09-10 19:55:36 +00:00
|
|
|
#include <Storages/SelectQueryInfo.h>
|
2020-01-28 16:08:30 +00:00
|
|
|
#include <Common/typeid_cast.h>
|
2020-02-07 09:51:48 +00:00
|
|
|
#include <Common/tests/gtest_global_context.h>
|
2020-11-04 10:14:23 +00:00
|
|
|
#include <Common/tests/gtest_global_register.h>
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-01-30 19:29:30 +00:00
|
|
|
#include <memory>
|
2020-07-31 13:23:19 +00:00
|
|
|
#include <Processors/Executors/PipelineExecutingBlockInputStream.h>
|
|
|
|
#include <Processors/QueryPipeline.h>
|
2020-01-30 19:29:30 +00:00
|
|
|
|
2021-04-18 09:17:02 +00:00
|
|
|
#if !defined(__clang__)
|
2020-01-28 16:08:30 +00:00
|
|
|
# pragma GCC diagnostic push
|
|
|
|
# pragma GCC diagnostic ignored "-Wsuggest-override"
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
DB::StoragePtr createStorage(DB::DiskPtr & disk)
|
|
|
|
{
|
|
|
|
using namespace DB;
|
|
|
|
|
|
|
|
NamesAndTypesList names_and_types;
|
|
|
|
names_and_types.emplace_back("a", std::make_shared<DataTypeUInt64>());
|
|
|
|
|
|
|
|
StoragePtr table = StorageLog::create(
|
2021-04-25 06:51:21 +00:00
|
|
|
disk, "table/", StorageID("test", "test"), ColumnsDescription{names_and_types}, ConstraintsDescription{}, String{}, false, 1048576);
|
2020-01-28 16:08:30 +00:00
|
|
|
|
|
|
|
table->startup();
|
|
|
|
|
|
|
|
return table;
|
|
|
|
}
|
|
|
|
|
|
|
|
template <typename T>
|
|
|
|
class StorageLogTest : public testing::Test
|
|
|
|
{
|
|
|
|
public:
|
2020-01-30 19:29:30 +00:00
|
|
|
|
2020-01-28 16:08:30 +00:00
|
|
|
void SetUp() override
|
|
|
|
{
|
2020-03-23 02:12:31 +00:00
|
|
|
disk = createDisk<T>();
|
|
|
|
table = createStorage(disk);
|
2020-01-28 16:08:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void TearDown() override
|
|
|
|
{
|
2020-03-23 02:12:31 +00:00
|
|
|
table->shutdown();
|
|
|
|
destroyDisk<T>(disk);
|
2020-01-28 16:08:30 +00:00
|
|
|
}
|
|
|
|
|
2020-03-23 02:12:31 +00:00
|
|
|
const DB::DiskPtr & getDisk() { return disk; }
|
|
|
|
DB::StoragePtr & getTable() { return table; }
|
2020-01-28 16:08:30 +00:00
|
|
|
|
|
|
|
private:
|
2020-03-23 02:12:31 +00:00
|
|
|
DB::DiskPtr disk;
|
|
|
|
DB::StoragePtr table;
|
2020-01-28 16:08:30 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
2020-03-09 02:15:19 +00:00
|
|
|
using DiskImplementations = testing::Types<DB::DiskMemory, DB::DiskLocal>;
|
2020-01-28 16:08:30 +00:00
|
|
|
TYPED_TEST_SUITE(StorageLogTest, DiskImplementations);
|
|
|
|
|
2020-01-30 19:29:30 +00:00
|
|
|
// Returns data written to table in Values format.
|
2021-04-10 23:33:54 +00:00
|
|
|
std::string writeData(int rows, DB::StoragePtr & table, const DB::ContextPtr context)
|
2020-01-28 16:08:30 +00:00
|
|
|
{
|
|
|
|
using namespace DB;
|
2020-06-15 19:08:58 +00:00
|
|
|
auto metadata_snapshot = table->getInMemoryMetadataPtr();
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-01-30 19:29:30 +00:00
|
|
|
std::string data;
|
2020-01-28 16:08:30 +00:00
|
|
|
|
|
|
|
Block block;
|
|
|
|
|
|
|
|
{
|
2020-06-17 16:39:58 +00:00
|
|
|
const auto & storage_columns = metadata_snapshot->getColumns();
|
2020-01-28 16:08:30 +00:00
|
|
|
ColumnWithTypeAndName column;
|
|
|
|
column.name = "a";
|
2020-04-27 15:38:35 +00:00
|
|
|
column.type = storage_columns.getPhysical("a").type;
|
2020-01-28 16:08:30 +00:00
|
|
|
auto col = column.type->createColumn();
|
|
|
|
ColumnUInt64::Container & vec = typeid_cast<ColumnUInt64 &>(*col).getData();
|
|
|
|
|
|
|
|
vec.resize(rows);
|
|
|
|
for (size_t i = 0; i < rows; ++i)
|
2020-01-30 19:29:30 +00:00
|
|
|
{
|
2020-01-28 16:08:30 +00:00
|
|
|
vec[i] = i;
|
2020-01-30 19:29:30 +00:00
|
|
|
if (i > 0)
|
|
|
|
data += ",";
|
|
|
|
data += "(" + std::to_string(i) + ")";
|
|
|
|
}
|
2020-01-28 16:08:30 +00:00
|
|
|
|
|
|
|
column.column = std::move(col);
|
|
|
|
block.insert(column);
|
|
|
|
}
|
|
|
|
|
2020-06-15 19:08:58 +00:00
|
|
|
BlockOutputStreamPtr out = table->write({}, metadata_snapshot, context);
|
2020-01-30 19:29:30 +00:00
|
|
|
out->write(block);
|
2020-07-16 02:37:12 +00:00
|
|
|
out->writeSuffix();
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-01-30 19:29:30 +00:00
|
|
|
return data;
|
|
|
|
}
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-01-30 19:29:30 +00:00
|
|
|
// Returns all table data in Values format.
|
2021-04-10 23:33:54 +00:00
|
|
|
std::string readData(DB::StoragePtr & table, const DB::ContextPtr context)
|
2020-01-30 19:29:30 +00:00
|
|
|
{
|
|
|
|
using namespace DB;
|
2020-06-15 19:08:58 +00:00
|
|
|
auto metadata_snapshot = table->getInMemoryMetadataPtr();
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-01-30 19:29:30 +00:00
|
|
|
Names column_names;
|
|
|
|
column_names.push_back("a");
|
|
|
|
|
2020-09-10 19:55:36 +00:00
|
|
|
SelectQueryInfo query_info;
|
|
|
|
QueryProcessingStage::Enum stage = table->getQueryProcessingStage(
|
2021-04-22 13:32:17 +00:00
|
|
|
context, QueryProcessingStage::Complete, metadata_snapshot, query_info);
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-07-31 13:23:19 +00:00
|
|
|
QueryPipeline pipeline;
|
2020-09-20 17:52:17 +00:00
|
|
|
pipeline.init(table->read(column_names, metadata_snapshot, query_info, context, stage, 8192, 1));
|
2020-07-31 13:23:19 +00:00
|
|
|
BlockInputStreamPtr in = std::make_shared<PipelineExecutingBlockInputStream>(std::move(pipeline));
|
2020-01-30 19:29:30 +00:00
|
|
|
|
|
|
|
Block sample;
|
2020-01-28 16:08:30 +00:00
|
|
|
{
|
2020-01-30 19:29:30 +00:00
|
|
|
ColumnWithTypeAndName col;
|
|
|
|
col.type = std::make_shared<DataTypeUInt64>();
|
|
|
|
sample.insert(std::move(col));
|
|
|
|
}
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-11-04 10:14:23 +00:00
|
|
|
tryRegisterFormats();
|
|
|
|
|
2020-11-09 19:07:38 +00:00
|
|
|
WriteBufferFromOwnString out_buf;
|
2020-12-30 03:07:30 +00:00
|
|
|
BlockOutputStreamPtr output = FormatFactory::instance().getOutputStream("Values", out_buf, sample, context);
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-01-30 19:29:30 +00:00
|
|
|
copyData(*in, *output);
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-01-30 19:29:30 +00:00
|
|
|
output->flush();
|
|
|
|
|
2020-11-09 19:07:38 +00:00
|
|
|
return out_buf.str();
|
2020-01-30 19:29:30 +00:00
|
|
|
}
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-01-30 19:29:30 +00:00
|
|
|
TYPED_TEST(StorageLogTest, testReadWrite)
|
|
|
|
{
|
|
|
|
using namespace DB;
|
2020-05-28 18:52:08 +00:00
|
|
|
const auto & context_holder = getContext();
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-01-30 19:29:30 +00:00
|
|
|
std::string data;
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-01-30 19:29:30 +00:00
|
|
|
// Write several chunks of data.
|
2020-04-20 09:42:56 +00:00
|
|
|
data += writeData(10, this->getTable(), context_holder.context);
|
2020-01-30 19:29:30 +00:00
|
|
|
data += ",";
|
2020-04-20 09:42:56 +00:00
|
|
|
data += writeData(20, this->getTable(), context_holder.context);
|
2020-01-30 19:29:30 +00:00
|
|
|
data += ",";
|
2020-04-20 09:42:56 +00:00
|
|
|
data += writeData(10, this->getTable(), context_holder.context);
|
2020-01-28 16:08:30 +00:00
|
|
|
|
2020-04-20 09:42:56 +00:00
|
|
|
ASSERT_EQ(data, readData(this->getTable(), context_holder.context));
|
2020-01-28 16:08:30 +00:00
|
|
|
}
|