thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

dbms: development.

Browse Source
This commit is contained in:
Alexey Milovidov 2010-05-24 17:55:57 +00:00
parent 78c982e205
commit abf4c94551
3 changed files with 142 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dbms/include/DB/DataStreams/copyData.h
View File

@ -1,6 +1,7 @@
#ifndef DBMS_DATA_STREAMS_COPY_DATA_H
#define DBMS_DATA_STREAMS_COPY_DATA_H
#include <DB/Core/Block.h>
#include <DB/DataStreams/IBlockInputStream.h>
#include <DB/DataStreams/IBlockOutputStream.h>
#include <DB/DataStreams/IRowInputStream.h>
@ -16,6 +17,7 @@ namespace DB
void copyData(IBlockInputStream & from, IBlockOutputStream & to);
void copyData(IRowInputStream & from, IRowOutputStream & to);
void copyData(IBlockInputStream & from, IRowOutputStream & to);
void copyData(IRowInputStream & from, IBlockOutputStream & to, const Block & sample);
}

8
dbms/src/DataStreams/copyData.cpp
View File

@ -1,4 +1,5 @@
#include <DB/DataStreams/RowInputStreamFromBlockInputStream.h>
#include <DB/DataStreams/BlockInputStreamFromRowInputStream.h>
#include <DB/DataStreams/copyData.h>
@ -31,4 +32,11 @@ void copyData(IBlockInputStream & from, IRowOutputStream & to)
copyData(row_input, to);
}
void copyData(IRowInputStream & from, IBlockOutputStream & to, const Block & sample)
{
BlockInputStreamFromRowInputStream block_input(from, sample);
copyData(block_input, to);
}
}

132
dbms/src/Storages/tests/hit_log.cpp Normal file
View File

@ -0,0 +1,132 @@
#include <map>
#include <list>
#include <iostream>
#include <boost/assign/list_inserter.hpp>
#include <Poco/SharedPtr.h>
#include <DB/DataTypes/DataTypesNumberFixed.h>
#include <DB/DataTypes/DataTypeString.h>
#include <DB/DataStreams/TabSeparatedRowInputStream.h>
#include <DB/DataStreams/TabSeparatedRowOutputStream.h>
#include <DB/DataStreams/copyData.h>
#include <DB/Storages/StorageLog.h>
using Poco::SharedPtr;
int main(int argc, char ** argv)
{
try
{
typedef std::pair<std::string, SharedPtr<DB::IDataType> > NameAndTypePair;
typedef std::list<NameAndTypePair> NamesAndTypesList;
NamesAndTypesList names_and_types_list;
boost::assign::push_back(names_and_types_list)
("WatchID", new DB::DataTypeUInt64)
("ChunkID", new DB::DataTypeUInt64)
("Random", new DB::DataTypeUInt32)
("JavaEnable", new DB::DataTypeUInt8)
("FrameEnable", new DB::DataTypeUInt8)
("Title", new DB::DataTypeString)
("GoodEvent", new DB::DataTypeUInt32)
// ("EventTime", new DB::DataTypeDateTime)
("CounterID", new DB::DataTypeUInt32)
("ClientIP", new DB::DataTypeUInt32)
("RegionID", new DB::DataTypeUInt32)
("UniqID", new DB::DataTypeUInt64)
("SessID", new DB::DataTypeUInt32)
("CounterClass", new DB::DataTypeUInt8)
("OS", new DB::DataTypeUInt8)
("UserAgent", new DB::DataTypeUInt8)
("URL", new DB::DataTypeString)
("Referer", new DB::DataTypeString)
("Refresh", new DB::DataTypeUInt8)
("ResolutionWidth", new DB::DataTypeUInt16)
("ResolutionHeight", new DB::DataTypeUInt16)
("ResolutionDepth", new DB::DataTypeUInt8)
("FlashMajor", new DB::DataTypeUInt8)
("FlashMinor", new DB::DataTypeUInt8)
("FlashMinor2", new DB::DataTypeString)
("NetMajor", new DB::DataTypeUInt8)
("NetMinor", new DB::DataTypeUInt8)
("UserAgentMajor", new DB::DataTypeUInt16)
// ("UserAgentMinor", new DB::DataTypeFixedString(2))
("CookieEnable", new DB::DataTypeUInt8)
("JavascriptEnable", new DB::DataTypeUInt8)
("IsMobile", new DB::DataTypeUInt8)
("MobilePhone", new DB::DataTypeUInt8)
("MobilePhoneModel", new DB::DataTypeString)
("Params", new DB::DataTypeString)
("IPNetworkID", new DB::DataTypeUInt32)
("TraficSourceID", new DB::DataTypeInt8)
("SearchEngineID", new DB::DataTypeUInt16)
("SearchPhrase", new DB::DataTypeString)
("AdvEngineID", new DB::DataTypeUInt8)
("IsArtifical", new DB::DataTypeUInt8)
("WindowClientWidth", new DB::DataTypeUInt16)
("WindowClientHeight", new DB::DataTypeUInt16)
("ClientTimeZone", new DB::DataTypeInt16)
// ("ClientEventTime", new DB::DataTypeDateTime)
("SilverlightVersion1", new DB::DataTypeUInt8)
("SilverlightVersion2", new DB::DataTypeUInt8)
("SilverlightVersion3", new DB::DataTypeUInt32)
("SilverlightVersion4", new DB::DataTypeUInt16)
("PageCharset", new DB::DataTypeString)
;
SharedPtr<DB::NamesAndTypes> names_and_types_map = new DB::NamesAndTypes;
SharedPtr<DB::DataTypes> data_types;
DB::ColumnNames column_names;
for (NamesAndTypesList::const_iterator it = names_and_types_list.begin(); it != names_and_types_list.end(); ++it)
{
names_and_types_map->insert(*it);
data_types->push_back(it->second);
column_names.push_back(it->first);
}
/// создаём таблицу хит лога
DB::StorageLog table("./", "HitLog", names_and_types_map, ".bin");
/// создаём описание, как читать данные из tab separated дампа
DB::Block sample;
for (NamesAndTypesList::const_iterator it = names_and_types_list.begin(); it != names_and_types_list.end(); ++it)
{
DB::ColumnWithNameAndType elem;
elem.name = it->first;
elem.type = it->second;
elem.column = elem.type->createColumn();
sample.insert(elem);
}
/// читаем данные из tsv файла и одновременно пишем в таблицу
{
DB::TabSeparatedRowInputStream in(std::cin, data_types);
SharedPtr<DB::IBlockOutputStream> out = table.write(0);
DB::copyData(in, *out, sample);
}
/// читаем из неё
{
SharedPtr<DB::IBlockInputStream> in = table.read(column_names, 0);
DB::TabSeparatedRowOutputStream out(std::cout, data_types);
DB::copyData(*in, out);
}
}
catch (const DB::Exception & e)
{
std::cerr << e.what() << ", " << e.message() << std::endl;
return 1;
}
return 0;
}