thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #67354 from Avogar/json-as-object-change

Browse Source 
Don't use PeekableReadBuffer in JSONAsObject format
This commit is contained in:
Kruglov Pavel 2024-07-30 10:43:56 +00:00 committed by GitHub
commit e3a820b0fb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 41 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp
View File

@ -15,11 +15,8 @@ namespace ErrorCodes
extern const int ILLEGAL_COLUMN;
}
JSONAsRowInputFormat::JSONAsRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_)
: JSONAsRowInputFormat(header_, std::make_unique<PeekableReadBuffer>(in_), params_, format_settings_) {}
JSONAsRowInputFormat::JSONAsRowInputFormat(const Block & header_, std::unique_ptr<PeekableReadBuffer> buf_, Params params_, const FormatSettings & format_settings_) :
JSONEachRowRowInputFormat(*buf_, header_, std::move(params_), format_settings_, false), buf(std::move(buf_))
JSONAsRowInputFormat::JSONAsRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_) :
JSONEachRowRowInputFormat(in_, header_, std::move(params_), format_settings_, false)
{
if (header_.columns() > 1)
throw Exception(ErrorCodes::BAD_ARGUMENTS,
@ -27,19 +24,6 @@ JSONAsRowInputFormat::JSONAsRowInputFormat(const Block & header_, std::unique_pt
header_.columns());
}
void JSONAsRowInputFormat::setReadBuffer(ReadBuffer & in_)
{
buf = std::make_unique<PeekableReadBuffer>(in_);
JSONEachRowRowInputFormat::setReadBuffer(*buf);
}
void JSONAsRowInputFormat::resetReadBuffer()
{
buf.reset();
JSONEachRowRowInputFormat::resetReadBuffer();
}
bool JSONAsRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &)
{
assert(columns.size() == 1);
@ -48,35 +32,41 @@ bool JSONAsRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &)
if (!allow_new_rows)
return false;
skipWhitespaceIfAny(*buf);
if (!buf->eof())
skipWhitespaceIfAny(*in);
if (!in->eof())
{
if (!data_in_square_brackets && *buf->position() == ';')
if (!data_in_square_brackets && *in->position() == ';')
{
/// ';' means the end of query, but it cannot be before ']'.
return allow_new_rows = false;
}
else if (data_in_square_brackets && *buf->position() == ']')
else if (data_in_square_brackets && *in->position() == ']')
{
/// ']' means the end of query.
return allow_new_rows = false;
}
}
if (!buf->eof())
if (!in->eof())
readJSONObject(*columns[0]);
skipWhitespaceIfAny(*buf);
if (!buf->eof() && *buf->position() == ',')
++buf->position();
skipWhitespaceIfAny(*buf);
skipWhitespaceIfAny(*in);
if (!in->eof() && *in->position() == ',')
++in->position();
skipWhitespaceIfAny(*in);
return !buf->eof();
return !in->eof();
}
JSONAsStringRowInputFormat::JSONAsStringRowInputFormat(
const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_)
: JSONAsRowInputFormat(header_, in_, params_, format_settings_)
const Block & header_, ReadBuffer & in_, IRowInputFormat::Params params_, const FormatSettings & format_settings_)
: JSONAsStringRowInputFormat(header_, std::make_unique<PeekableReadBuffer>(in_), params_, format_settings_)
{
}
JSONAsStringRowInputFormat::JSONAsStringRowInputFormat(
const Block & header_, std::unique_ptr<PeekableReadBuffer> buf_, Params params_, const FormatSettings & format_settings_)
: JSONAsRowInputFormat(header_, *buf_, params_, format_settings_), buf(std::move(buf_))
{
if (!isString(removeNullable(removeLowCardinality(header_.getByPosition(0).type))))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
@ -84,6 +74,18 @@ JSONAsStringRowInputFormat::JSONAsStringRowInputFormat(
header_.getByPosition(0).type->getName());
}
void JSONAsStringRowInputFormat::setReadBuffer(ReadBuffer & in_)
{
buf = std::make_unique<PeekableReadBuffer>(in_);
JSONAsRowInputFormat::setReadBuffer(*buf);
}
void JSONAsStringRowInputFormat::resetReadBuffer()
{
buf.reset();
JSONAsRowInputFormat::resetReadBuffer();
}
void JSONAsStringRowInputFormat::readJSONObject(IColumn & column)
{
PeekableReadBufferCheckpoint checkpoint{*buf};
@ -174,7 +176,7 @@ JSONAsObjectRowInputFormat::JSONAsObjectRowInputFormat(
void JSONAsObjectRowInputFormat::readJSONObject(IColumn & column)
{
serializations[0]->deserializeTextJSON(column, *buf, format_settings);
serializations[0]->deserializeTextJSON(column, *in, format_settings);
}
Chunk JSONAsObjectRowInputFormat::getChunkForCount(size_t rows)

15
src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h
View File

@ -18,17 +18,11 @@ class JSONAsRowInputFormat : public JSONEachRowRowInputFormat
public:
JSONAsRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings);
void setReadBuffer(ReadBuffer & in_) override;
void resetReadBuffer() override;
private:
JSONAsRowInputFormat(const Block & header_, std::unique_ptr<PeekableReadBuffer> buf_, Params params_, const FormatSettings & format_settings);
bool readRow(MutableColumns & columns, RowReadExtension & ext) override;
protected:
virtual void readJSONObject(IColumn & column) = 0;
std::unique_ptr<PeekableReadBuffer> buf;
};
/// Each JSON object is parsed as a whole to string.
@ -36,11 +30,18 @@ protected:
class JSONAsStringRowInputFormat final : public JSONAsRowInputFormat
{
public:
JSONAsStringRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings);
JSONAsStringRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_);
String getName() const override { return "JSONAsStringRowInputFormat"; }
void setReadBuffer(ReadBuffer & in_) override;
void resetReadBuffer() override;
private:
JSONAsStringRowInputFormat(const Block & header_, std::unique_ptr<PeekableReadBuffer> buf_, Params params_, const FormatSettings & format_settings_);
void readJSONObject(IColumn & column) override;
std::unique_ptr<PeekableReadBuffer> buf;
};