thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

DB/IO: improved writing strings [#CONV-2807].

Browse Source
This commit is contained in:
Alexey Milovidov 2013-05-15 22:58:57 +00:00
parent b89bbf8463
commit af6a7b2697
11 changed files with 122 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
dbms/include/DB/Core/Field.h
View File

@ -1,7 +1,6 @@
#pragma once
#include <vector>
#include <sstream>
#include <tr1/type_traits>
#include <boost/static_assert.hpp>
@ -17,6 +16,7 @@
#include <DB/Core/ErrorCodes.h>
#include <DB/IO/ReadHelpers.h>
#include <DB/IO/WriteHelpers.h>
#include <DB/IO/WriteBufferFromString.h>
namespace DB
@ -539,69 +539,86 @@ template <> struct TypeName<Array> { static std::string get() { return "Array";
/** Возвращает строковый дамп типа */
class FieldVisitorDump : public StaticVisitor<std::string>
class FieldVisitorDump : public StaticVisitor<String>
{
private:
template <typename T>
static inline String formatQuotedWithPrefix(T x, const char * prefix)
{
String res;
WriteBufferFromString wb(res);
wb.write(prefix, strlen(prefix));
writeQuoted(x, wb);
return res;
}
public:
String operator() (const Null & x) const { return "NULL"; }
String operator() (const UInt64 & x) const { return "UInt64_" + Poco::NumberFormatter::format(x); }
String operator() (const Int64 & x) const { return "Int64_" + Poco::NumberFormatter::format(x); }
String operator() (const Float64 & x) const { return "Float64_" + Poco::NumberFormatter::format(x); }
String operator() (const UInt64 & x) const { return formatQuotedWithPrefix(x, "UInt64_"); }
String operator() (const Int64 & x) const { return formatQuotedWithPrefix(x, "Int64_"); }
String operator() (const Float64 & x) const { return formatQuotedWithPrefix(x, "Float64_"); }
String operator() (const String & x) const
{
std::stringstream s;
s << mysqlxx::quote << x;
return s.str();
String res;
WriteBufferFromString wb(res);
writeQuoted(x, wb);
return res;
}
String operator() (const Array & x) const
{
std::stringstream s;
String res;
WriteBufferFromString wb(res);
FieldVisitorDump visitor;
s << "Array_[";
wb.write("Array_[", 7);
for (Array::const_iterator it = x.begin(); it != x.end(); ++it)
{
if (it != x.begin())
s << ", ";
s << apply_visitor(FieldVisitorDump(), *it);
wb.write(", ", 2);
writeString(apply_visitor(visitor, *it), wb);
}
s << "]";
writeChar(']', wb);
return s.str();
return res;
}
};
/** Выводит текстовое представление типа, как литерала в SQL запросе */
class FieldVisitorToString : public StaticVisitor<String>
{
private:
template <typename T>
static inline String formatQuoted(T x)
{
String res;
WriteBufferFromString wb(res);
writeQuoted(x, wb);
return res;
}
public:
String operator() (const Null & x) const { return "NULL"; }
String operator() (const UInt64 & x) const { return Poco::NumberFormatter::format(x); }
String operator() (const Int64 & x) const { return Poco::NumberFormatter::format(x); }
String operator() (const Float64 & x) const { return Poco::NumberFormatter::format(x); }
String operator() (const String & x) const
{
std::stringstream s;
s << mysqlxx::quote << x;
return s.str();
}
String operator() (const UInt64 & x) const { return formatQuoted(x); }
String operator() (const Int64 & x) const { return formatQuoted(x); }
String operator() (const Float64 & x) const { return formatQuoted(x); }
String operator() (const String & x) const { return formatQuoted(x); }
String operator() (const Array & x) const
{
std::stringstream s;
String res;
WriteBufferFromString wb(res);
FieldVisitorToString visitor;
s << "[";
writeChar('[', wb);
for (Array::const_iterator it = x.begin(); it != x.end(); ++it)
{
if (it != x.begin())
s << ", ";
s << apply_visitor(visitor, *it);
wb.write(", ", 2);
writeString(apply_visitor(visitor, *it), wb);
}
s << "]";
writeChar(']', wb);
return s.str();
return res;
}
};

12
dbms/include/DB/IO/WriteHelpers.h
View File

@ -85,6 +85,18 @@ inline void writeString(const String & s, WriteBuffer & buf)
buf.write(s.data(), s.size());
}
inline void writeString(const char * data, size_t size, WriteBuffer & buf)
{
buf.write(data, size);
}
/** Пишет С-строку без создания временного объекта. Если строка - литерал, то strlen выполняется на этапе компиляции.
* Используйте, когда строка - литерал.
*/
#define writeCString(s, buf) \
(buf).write((s), strlen(s))
template <char c>
void writeAnyEscapedString(const char * begin, const char * end, WriteBuffer & buf)

4
dbms/include/DB/Parsers/ASTFunction.h
View File

@ -47,7 +47,7 @@ public:
for (ASTs::const_iterator it = parameters->children.begin(); it != parameters->children.end(); ++it)
{
if (it != parameters->children.begin())
writeString(", ", wb);
writeCString(", ", wb);
writeString((*it)->getColumnName(), wb);
}
writeChar(')', wb);
@ -57,7 +57,7 @@ public:
for (ASTs::const_iterator it = arguments->children.begin(); it != arguments->children.end(); ++it)
{
if (it != arguments->children.begin())
writeString(", ", wb);
writeCString(", ", wb);
writeString((*it)->getColumnName(), wb);
}
writeChar(')', wb);

6
dbms/src/DataStreams/JSONCompactRowOutputStream.cpp
View File

@ -24,15 +24,15 @@ void JSONCompactRowOutputStream::writeField(const Field & field)
void JSONCompactRowOutputStream::writeFieldDelimiter()
{
writeString(", ", ostr);
writeCString(", ", ostr);
}
void JSONCompactRowOutputStream::writeRowStartDelimiter()
{
if (row_count > 0)
writeString(",\n", ostr);
writeString("\t\t\t[", ostr);
writeCString(",\n", ostr);
writeCString("\t\t\t[", ostr);
}

40
dbms/src/DataStreams/JSONRowOutputStream.cpp
View File

@ -19,39 +19,39 @@ JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & samp
void JSONRowOutputStream::writePrefix()
{
writeString("{\n", ostr);
writeString("\t\"meta\":\n", ostr);
writeString("\t[\n", ostr);
writeCString("{\n", ostr);
writeCString("\t\"meta\":\n", ostr);
writeCString("\t[\n", ostr);
for (size_t i = 0; i < fields.size(); ++i)
{
writeString("\t\t{\n", ostr);
writeCString("\t\t{\n", ostr);
writeString("\t\t\t\"name\": ", ostr);
writeCString("\t\t\t\"name\": ", ostr);
writeDoubleQuotedString(fields[i].first, ostr);
writeString(",\n", ostr);
writeString("\t\t\t\"type\": ", ostr);
writeCString(",\n", ostr);
writeCString("\t\t\t\"type\": ", ostr);
writeDoubleQuotedString(fields[i].second->getName(), ostr);
writeChar('\n', ostr);
writeString("\t\t}", ostr);
writeCString("\t\t}", ostr);
if (i + 1 < fields.size())
writeChar(',', ostr);
writeChar('\n', ostr);
}
writeString("\t],\n", ostr);
writeCString("\t],\n", ostr);
writeChar('\n', ostr);
writeString("\t\"data\":\n", ostr);
writeString("\t[\n", ostr);
writeCString("\t\"data\":\n", ostr);
writeCString("\t[\n", ostr);
}
void JSONRowOutputStream::writeField(const Field & field)
{
writeString("\t\t\t", ostr);
writeCString("\t\t\t", ostr);
writeDoubleQuotedString(fields[field_number].first, ostr);
writeString(": ", ostr);
writeCString(": ", ostr);
fields[field_number].second->serializeTextQuoted(field, ostr);
++field_number;
}
@ -59,22 +59,22 @@ void JSONRowOutputStream::writeField(const Field & field)
void JSONRowOutputStream::writeFieldDelimiter()
{
writeString(",\n", ostr);
writeCString(",\n", ostr);
}
void JSONRowOutputStream::writeRowStartDelimiter()
{
if (row_count > 0)
writeString(",\n", ostr);
writeString("\t\t{\n", ostr);
writeCString(",\n", ostr);
writeCString("\t\t{\n", ostr);
}
void JSONRowOutputStream::writeRowEndDelimiter()
{
writeChar('\n', ostr);
writeString("\t\t}", ostr);
writeCString("\t\t}", ostr);
field_number = 0;
++row_count;
}
@ -83,12 +83,12 @@ void JSONRowOutputStream::writeRowEndDelimiter()
void JSONRowOutputStream::writeSuffix()
{
writeChar('\n', ostr);
writeString("\t],\n", ostr);
writeCString("\t],\n", ostr);
writeChar('\n', ostr);
writeString("\t\"rows\": ", ostr);
writeCString("\t\"rows\": ", ostr);
writeIntText(row_count, ostr);
writeChar('\n', ostr);
writeString("}\n", ostr);
writeCString("}\n", ostr);
}
}

20
dbms/src/DataStreams/PrettyBlockOutputStream.cpp
View File

@ -131,16 +131,16 @@ void PrettyBlockOutputStream::write(const Block & block_)
writeString(top_separator_s, ostr);
/// Имена
writeString("┃ ", ostr);
writeCString("┃ ", ostr);
for (size_t i = 0; i < columns; ++i)
{
if (i != 0)
writeString(" ┃ ", ostr);
writeCString(" ┃ ", ostr);
const ColumnWithNameAndType & col = block.getByPosition(i);
if (!no_escapes)
writeString("\033[1;37m", ostr);
writeCString("\033[1;37m", ostr);
if (col.type->isNumeric())
{
@ -158,9 +158,9 @@ void PrettyBlockOutputStream::write(const Block & block_)
}
if (!no_escapes)
writeString("\033[0m", ostr);
writeCString("\033[0m", ostr);
}
writeString(" ┃\n", ostr);
writeCString(" ┃\n", ostr);
writeString(middle_names_separator_s, ostr);
@ -169,12 +169,12 @@ void PrettyBlockOutputStream::write(const Block & block_)
if (i != 0)
writeString(middle_values_separator_s, ostr);
writeString("│ ", ostr);
writeCString("│ ", ostr);
for (size_t j = 0; j < columns; ++j)
{
if (j != 0)
writeString(" │ ", ostr);
writeCString(" │ ", ostr);
const ColumnWithNameAndType & col = block.getByPosition(j);
@ -196,7 +196,7 @@ void PrettyBlockOutputStream::write(const Block & block_)
}
}
writeString(" │\n", ostr);
writeCString(" │\n", ostr);
}
writeString(bottom_separator_s, ostr);
@ -209,9 +209,9 @@ void PrettyBlockOutputStream::writeSuffix()
{
if (total_rows >= max_rows)
{
writeString(" Showed first ", ostr);
writeCString(" Showed first ", ostr);
writeIntText(max_rows, ostr);
writeString(".\n", ostr);
writeCString(".\n", ostr);
}
}

24
dbms/src/DataStreams/PrettyCompactBlockOutputStream.cpp
View File

@ -45,47 +45,47 @@ void PrettyCompactBlockOutputStream::write(const Block & block_)
std::string bottom_separator_s = bottom_separator.str();
/// Имена
writeString("┌─", ostr);
writeCString("┌─", ostr);
for (size_t i = 0; i < columns; ++i)
{
if (i != 0)
writeString("─┬─", ostr);
writeCString("─┬─", ostr);
const ColumnWithNameAndType & col = block.getByPosition(i);
if (col.type->isNumeric())
{
for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k)
writeString("─", ostr);
writeCString("─", ostr);
if (!no_escapes)
writeString("\033[1;37m", ostr);
writeCString("\033[1;37m", ostr);
writeEscapedString(col.name, ostr);
if (!no_escapes)
writeString("\033[0m", ostr);
writeCString("\033[0m", ostr);
}
else
{
if (!no_escapes)
writeString("\033[1;37m", ostr);
writeCString("\033[1;37m", ostr);
writeEscapedString(col.name, ostr);
if (!no_escapes)
writeString("\033[0m", ostr);
writeCString("\033[0m", ostr);
for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k)
writeString("─", ostr);
writeCString("─", ostr);
}
}
writeString("─┐\n", ostr);
writeCString("─┐\n", ostr);
for (size_t i = 0; i < rows && total_rows + i < max_rows; ++i)
{
writeString("│ ", ostr);
writeCString("│ ", ostr);
for (size_t j = 0; j < columns; ++j)
{
if (j != 0)
writeString(" │ ", ostr);
writeCString(" │ ", ostr);
const ColumnWithNameAndType & col = block.getByPosition(j);
@ -107,7 +107,7 @@ void PrettyCompactBlockOutputStream::write(const Block & block_)
}
}
writeString(" │\n", ostr);
writeCString(" │\n", ostr);
}
writeString(bottom_separator_s, ostr);

18
dbms/src/DataStreams/PrettySpaceBlockOutputStream.cpp
View File

@ -38,7 +38,7 @@ void PrettySpaceBlockOutputStream::write(const Block & block_)
for (size_t i = 0; i < columns; ++i)
{
if (i != 0)
writeString(" ", ostr);
writeCString(" ", ostr);
const ColumnWithNameAndType & col = block.getByPosition(i);
@ -48,31 +48,31 @@ void PrettySpaceBlockOutputStream::write(const Block & block_)
writeChar(' ', ostr);
if (!no_escapes)
writeString("\033[1;37m", ostr);
writeCString("\033[1;37m", ostr);
writeEscapedString(col.name, ostr);
if (!no_escapes)
writeString("\033[0m", ostr);
writeCString("\033[0m", ostr);
}
else
{
if (!no_escapes)
writeString("\033[1;37m", ostr);
writeCString("\033[1;37m", ostr);
writeEscapedString(col.name, ostr);
if (!no_escapes)
writeString("\033[0m", ostr);
writeCString("\033[0m", ostr);
for (ssize_t k = 0; k < std::max(0L, static_cast<ssize_t>(max_widths[i] - name_widths[i])); ++k)
writeChar(' ', ostr);
}
}
writeString("\n\n", ostr);
writeCString("\n\n", ostr);
for (size_t i = 0; i < rows && total_rows + i < max_rows; ++i)
{
for (size_t j = 0; j < columns; ++j)
{
if (j != 0)
writeString(" ", ostr);
writeCString(" ", ostr);
const ColumnWithNameAndType & col = block.getByPosition(j);
@ -105,9 +105,9 @@ void PrettySpaceBlockOutputStream::writeSuffix()
{
if (total_rows >= max_rows)
{
writeString("\nShowed first ", ostr);
writeCString("\nShowed first ", ostr);
writeIntText(max_rows, ostr);
writeString(".", ostr);
writeCString(".", ostr);
}
}

2
dbms/src/DataStreams/ValuesRowOutputStream.cpp
View File

@ -46,7 +46,7 @@ void ValuesRowOutputStream::writeRowEndDelimiter()
void ValuesRowOutputStream::writeRowBetweenDelimiter()
{
writeString(",", ostr);
writeCString(",", ostr);
field_number = 0;
}

10
dbms/src/DataStreams/VerticalRowOutputStream.cpp
View File

@ -39,7 +39,7 @@ VerticalRowOutputStream::VerticalRowOutputStream(WriteBuffer & ostr_, const Bloc
void VerticalRowOutputStream::writeField(const Field & field)
{
writeEscapedString(names[field_number], ostr);
writeString(": ", ostr);
writeCString(": ", ostr);
writeString(pads[field_number], ostr);
data_types[field_number]->serializeTextEscaped(field, ostr);
@ -52,20 +52,20 @@ void VerticalRowOutputStream::writeField(const Field & field)
void VerticalRowOutputStream::writeRowStartDelimiter()
{
++row_number;
writeString("Row ", ostr);
writeCString("Row ", ostr);
writeIntText(row_number, ostr);
writeString(":\n", ostr);
writeCString(":\n", ostr);
size_t width = log10(row_number + 1) + 1 + strlen("Row :");
for (size_t i = 0; i < width; ++i)
writeString("─", ostr);
writeCString("─", ostr);
writeChar('\n', ostr);
}
void VerticalRowOutputStream::writeRowBetweenDelimiter()
{
writeString("\n", ostr);
writeCString("\n", ostr);
field_number = 0;
}

19
dbms/src/DataStreams/tests/glue_streams.cpp
View File

@ -65,25 +65,8 @@ int main(int argc, char ** argv)
WriteBufferFromFileDescriptor wb(STDOUT_FILENO);
BlockOutputStreamPtr out1 = context.getFormatFactory().getOutput("TabSeparated", wb, io1.in_sample);
/* out1->writePrefix();
while (Block block = io1.in->read())
{
out1->write(block);
wb.next();
}
out1->writeSuffix();*/
BlockOutputStreamPtr out2 = context.getFormatFactory().getOutput("TabSeparated", wb, io2.in_sample);
/* out2->writePrefix();
while (Block block = io2.in->read())
{
out2->write(block);
wb.next();
}
out2->writeSuffix();*/
BlockInputStreams inputs;
inputs.push_back(io1.in);
inputs.push_back(io2.in);
@ -93,7 +76,7 @@ int main(int argc, char ** argv)
Forks forks;
// glueBlockInputStreams(inputs, forks);
glueBlockInputStreams(inputs, forks);
std::cerr << forks.size() << std::endl;