From 84592c521f3f9f33732e0dccaa66bfc2c2233fc4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 4 Aug 2009 16:15:25 +0000 Subject: [PATCH] dbms: development. --- dbms/src/StorageNoKey.cpp | 17 ++- dbms/src/tests/column_storage_no_key.cpp | 171 +++++++++++++++++++++++ dbms/src/tests/istream_test.cpp | 12 +- 3 files changed, 194 insertions(+), 6 deletions(-) create mode 100644 dbms/src/tests/column_storage_no_key.cpp diff --git a/dbms/src/StorageNoKey.cpp b/dbms/src/StorageNoKey.cpp index 4797088cbf9..5cb44bf6dba 100644 --- a/dbms/src/StorageNoKey.cpp +++ b/dbms/src/StorageNoKey.cpp @@ -31,15 +31,24 @@ void StorageNoKey::merge(const AggregatedRowSet & data, const ColumnMask & mask) for (AggregatedRowSet::const_iterator it = data.begin(); it != data.end(); ++it) { + size_t column_num = 0; for (size_t j = 0; j != it->first.size(); ++j) + { if (mask[j]) - table->columns->at(column_group->column_numbers[j]).type->serializeBinary(it->first[j], ostr); + { + table->columns->at(column_group->column_numbers[column_num]).type->serializeBinary(it->first[j], ostr); + ++column_num; + } + } for (size_t j = 0; j != it->second.size(); ++j) + { if (mask[j + it->first.size()]) - table->columns->at( - column_group->column_numbers[j + it->first.size()] - ).type->serializeBinary(it->second[j], ostr); + { + table->columns->at(column_group->column_numbers[column_num]).type->serializeBinary(it->second[j], ostr); + ++column_num; + } + } } } diff --git a/dbms/src/tests/column_storage_no_key.cpp b/dbms/src/tests/column_storage_no_key.cpp new file mode 100644 index 00000000000..eed368fdc62 --- /dev/null +++ b/dbms/src/tests/column_storage_no_key.cpp @@ -0,0 +1,171 @@ +#include +#include + +#include +#include +#include +#include +#include + + +int main(int argc, char ** argv) +{ + Poco::Stopwatch stopwatch; + + /// создаём таблицу + + Poco::SharedPtr columns = new DB::Table::Columns; + + columns->push_back(DB::Column("WatchID", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("ChunkID", new DB::ColumnTypeUInt64)); + columns->push_back(DB::Column("Random", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("JavaEnable", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("FrameEnable", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("Title", new DB::ColumnTypeText)); + columns->push_back(DB::Column("GoodEvent", new DB::ColumnTypeVarInt)); + columns->push_back(DB::Column("EventTime", new DB::ColumnTypeUInt32)); + columns->push_back(DB::Column("CounterID", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("ClientIP", new DB::ColumnTypeUInt32)); + columns->push_back(DB::Column("RegionID", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("UniqID", new DB::ColumnTypeUInt64)); + columns->push_back(DB::Column("SessID", new DB::ColumnTypeUInt32)); + columns->push_back(DB::Column("CounterClass", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("OS", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("UserAgent", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("URL", new DB::ColumnTypeText)); + columns->push_back(DB::Column("Referer", new DB::ColumnTypeText)); + columns->push_back(DB::Column("Refresh", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("ResolutionWidth", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("ResolutionHeight", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("ResolutionDepth", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("FlashMajor", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("FlashMinor", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("FlashMinor2", new DB::ColumnTypeText)); + columns->push_back(DB::Column("NetMajor", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("NetMinor", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("UserAgentMajor", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("UserAgentMinor", new DB::ColumnTypeFixedText(2))); + columns->push_back(DB::Column("CookieEnable", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("JavascriptEnable", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("IsMobile", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("MobilePhone", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("MobilePhoneModel", new DB::ColumnTypeText)); + columns->push_back(DB::Column("Params", new DB::ColumnTypeText)); + columns->push_back(DB::Column("IPNetworkID", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("TraficSourceID", new DB::ColumnTypeVarInt)); + columns->push_back(DB::Column("SearchEngineID", new DB::ColumnTypeVarUInt)); + columns->push_back(DB::Column("SearchPhrase", new DB::ColumnTypeText)); + columns->push_back(DB::Column("AdvEngineID", new DB::ColumnTypeVarUInt)); + + Poco::SharedPtr primary_key_column_numbers = new DB::Table::ColumnNumbers; + primary_key_column_numbers->push_back(0); + + Poco::SharedPtr column_groups = new DB::Table::ColumnGroups; + for (size_t i = 0; i < columns->size(); ++i) + { + DB::ColumnGroup column_group; + column_group.column_numbers.push_back(i); + column_group.storage = new DB::StorageNoKey("./", "TestStorageNoKeyColumn" + Poco::NumberFormatter::format(i)); + column_groups->push_back(column_group); + } + + DB::Table table("TestTable", columns, primary_key_column_numbers, column_groups); + + /// создаём набор данных + DB::AggregatedRowSet data; + DB::Row key; + DB::Row value; + + key.push_back(DB::Field(DB::UInt(65765691660ULL))); + value.push_back(DB::Field(DB::UInt(20090724165002400ULL))); + value.push_back(DB::Field(DB::UInt(9154640))); + value.push_back(DB::Field(DB::UInt(1))); + value.push_back(DB::Field(DB::UInt(0))); + value.push_back(DB::Field(DB::String("Китайские ученые перепрограммировали клетки и создали из них мышей. Иностранная пресса о событиях в "))); + value.push_back(DB::Field(DB::Int(1))); + value.push_back(DB::Field(DB::UInt(1248456711))); + value.push_back(DB::Field(DB::UInt(71551))); + value.push_back(DB::Field(DB::UInt(1220865079))); + value.push_back(DB::Field(DB::UInt(84))); + value.push_back(DB::Field(DB::UInt(5243575589842965681ULL))); + value.push_back(DB::Field(DB::UInt(0))); + value.push_back(DB::Field(DB::UInt(1))); + value.push_back(DB::Field(DB::UInt(3))); + value.push_back(DB::Field(DB::UInt(5))); + value.push_back(DB::Field(DB::String("http://www.example.ru/wsj/2009/07/24/15:10:00/mouse"))); + value.push_back(DB::Field(DB::String("http://www.example.com/"))); + value.push_back(DB::Field(DB::UInt(0))); + value.push_back(DB::Field(DB::UInt(1024))); + value.push_back(DB::Field(DB::UInt(768))); + value.push_back(DB::Field(DB::UInt(16))); + value.push_back(DB::Field(DB::UInt(10))); + value.push_back(DB::Field(DB::UInt(0))); + value.push_back(DB::Field(DB::String(""))); + value.push_back(DB::Field(DB::UInt(3))); + value.push_back(DB::Field(DB::UInt(5))); + value.push_back(DB::Field(DB::UInt(8))); + value.push_back(DB::Field(DB::String("0 "))); + value.push_back(DB::Field(DB::UInt(1))); + value.push_back(DB::Field(DB::UInt(1))); + value.push_back(DB::Field(DB::UInt(0))); + value.push_back(DB::Field(DB::UInt(0))); + value.push_back(DB::Field(DB::String(""))); + value.push_back(DB::Field(DB::String(""))); + value.push_back(DB::Field(DB::UInt(0))); + value.push_back(DB::Field(DB::Int(1))); + value.push_back(DB::Field(DB::UInt(0))); + value.push_back(DB::Field(DB::String(""))); + value.push_back(DB::Field(DB::UInt(0))); + + { + stopwatch.restart(); + + for (DB::UInt i = 0; i < 1000000; ++i) + { + data[key] = value; + ++boost::get(key[0]); + } + + stopwatch.stop(); + std::cout << "Filling data: " << static_cast(stopwatch.elapsed()) / 1000000 << std::endl; + } + + /// заполняем таблицу + { + DB::ColumnMask mask(columns->size(), false); + + stopwatch.restart(); + + for (size_t i = 0; i < column_groups->size(); ++i) + { + mask[i] = true; + (*column_groups)[i].storage->merge(data, mask); + mask[i] = false; + } + + stopwatch.stop(); + std::cout << "Saving data: " << static_cast(stopwatch.elapsed()) / 1000000 << std::endl; + } + + /// читаем таблицу + { + DB::Row key; + Poco::SharedPtr reader((*column_groups)[30].storage->read(key)); /// UniqID + + stopwatch.restart(); + + DB::UInt i = 0; + DB::Row row; + while (reader->fetch(row)) + { + ++i; + } + if (i != 1000000) + throw Poco::Exception("Number of rows doesn't match"); + + stopwatch.stop(); + std::cout << "Reading data: " << static_cast(stopwatch.elapsed()) / 1000000 << std::endl; + } + + return 0; +} diff --git a/dbms/src/tests/istream_test.cpp b/dbms/src/tests/istream_test.cpp index 23b6866a78a..2388d606656 100644 --- a/dbms/src/tests/istream_test.cpp +++ b/dbms/src/tests/istream_test.cpp @@ -1,10 +1,12 @@ #include #include +#include #include +#include int main(int argc, char ** argv) { - std::stringstream s; +/* std::stringstream s; s << "192.168.1.1fls"; unsigned x; @@ -18,7 +20,13 @@ int main(int argc, char ** argv) s.get(); std::cout << x << std::endl; s >> x; - std::cout << x << std::endl; + std::cout << x << std::endl;*/ + + std::ofstream f("test"); + Poco::BinaryWriter w(f); + + for (int i = 0; i < 1048576; ++i) + w << rand() % 128; return 0; }