mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-28 12:40:49 +00:00
Merge remote-tracking branch 'upstream/master' into fix27
This commit is contained in:
commit
40b5011367
@ -186,8 +186,8 @@ endif()
|
||||
list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
|
||||
list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})
|
||||
|
||||
list (APPEND dbms_sources src/Functions/IFunction.cpp src/Functions/FunctionFactory.cpp src/Functions/FunctionHelpers.cpp)
|
||||
list (APPEND dbms_headers src/Functions/IFunctionImpl.h src/Functions/FunctionFactory.h src/Functions/FunctionHelpers.h)
|
||||
list (APPEND dbms_sources src/Functions/IFunction.cpp src/Functions/FunctionFactory.cpp src/Functions/FunctionHelpers.cpp src/Functions/extractTimeZoneFromFunctionArguments.cpp)
|
||||
list (APPEND dbms_headers src/Functions/IFunctionImpl.h src/Functions/FunctionFactory.h src/Functions/FunctionHelpers.h src/Functions/extractTimeZoneFromFunctionArguments.h)
|
||||
|
||||
list (APPEND dbms_sources
|
||||
src/AggregateFunctions/AggregateFunctionFactory.cpp
|
||||
|
@ -127,8 +127,8 @@ public:
|
||||
uint64_t line = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* Find the file and line number information corresponding to address.
|
||||
/** Find the file and line number information corresponding to address.
|
||||
* The address must be physical - offset in object file without offset in virtual memory where the object is loaded.
|
||||
*/
|
||||
bool findAddress(uintptr_t address, LocationInfo & info, LocationInfoMode mode) const;
|
||||
|
||||
|
@ -258,10 +258,14 @@ static void toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offs
|
||||
|
||||
for (size_t i = offset; i < size; ++i)
|
||||
{
|
||||
const void * addr = frames[i];
|
||||
const void * virtual_addr = frames[i];
|
||||
auto object = symbol_index.findObject(virtual_addr);
|
||||
uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0;
|
||||
const void * physical_addr = reinterpret_cast<const void *>(uintptr_t(virtual_addr) - virtual_offset);
|
||||
|
||||
out << i << ". " << addr << " ";
|
||||
auto symbol = symbol_index.findSymbol(addr);
|
||||
out << i << ". " << physical_addr << " ";
|
||||
|
||||
auto symbol = symbol_index.findSymbol(virtual_addr);
|
||||
if (symbol)
|
||||
{
|
||||
int status = 0;
|
||||
@ -272,18 +276,17 @@ static void toStringEveryLineImpl(const StackTrace::Frames & frames, size_t offs
|
||||
|
||||
out << " ";
|
||||
|
||||
if (auto object = symbol_index.findObject(addr))
|
||||
if (object)
|
||||
{
|
||||
if (std::filesystem::exists(object->name))
|
||||
{
|
||||
auto dwarf_it = dwarfs.try_emplace(object->name, *object->elf).first;
|
||||
|
||||
DB::Dwarf::LocationInfo location;
|
||||
if (dwarf_it->second.findAddress(uintptr_t(addr) - uintptr_t(object->address_begin), location, DB::Dwarf::LocationInfoMode::FAST))
|
||||
if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, DB::Dwarf::LocationInfoMode::FAST))
|
||||
out << location.file.toString() << ":" << location.line;
|
||||
else
|
||||
out << object->name;
|
||||
}
|
||||
out << " in " << object->name;
|
||||
}
|
||||
else
|
||||
out << "?";
|
||||
|
@ -38,6 +38,7 @@ public:
|
||||
std::unique_ptr<Elf> elf;
|
||||
};
|
||||
|
||||
/// Address in virtual memory should be passed. These addresses include offset where the object is loaded in memory.
|
||||
const Symbol * findSymbol(const void * address) const;
|
||||
const Object * findObject(const void * address) const;
|
||||
|
||||
|
@ -49,7 +49,7 @@ int main(int argc, char ** argv)
|
||||
Dwarf dwarf(*object->elf);
|
||||
|
||||
Dwarf::LocationInfo location;
|
||||
if (dwarf.findAddress(uintptr_t(address), location, Dwarf::LocationInfoMode::FAST))
|
||||
if (dwarf.findAddress(uintptr_t(address) - uintptr_t(info.dli_fbase), location, Dwarf::LocationInfoMode::FAST))
|
||||
std::cerr << location.file.toString() << ":" << location.line << "\n";
|
||||
else
|
||||
std::cerr << "Dwarf: Not found\n";
|
||||
|
@ -65,7 +65,7 @@ void ParallelParsingBlockInputStream::parserThreadFunction(size_t current_unit_n
|
||||
*/
|
||||
ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0);
|
||||
auto parser = std::make_unique<InputStreamFromInputFormat>(
|
||||
input_processor_creator(read_buffer, header, context,
|
||||
input_processor_creator(read_buffer, header,
|
||||
row_input_format_params, format_settings));
|
||||
|
||||
unit.block_ext.block.clear();
|
||||
|
@ -55,23 +55,21 @@ private:
|
||||
using InputProcessorCreator = std::function<InputFormatPtr(
|
||||
ReadBuffer & buf,
|
||||
const Block & header,
|
||||
const Context & context,
|
||||
const RowInputFormatParams & params,
|
||||
const FormatSettings & settings)>;
|
||||
public:
|
||||
struct InputCreatorParams
|
||||
{
|
||||
const Block &sample;
|
||||
const Context &context;
|
||||
const RowInputFormatParams& row_input_format_params;
|
||||
const Block & sample;
|
||||
const RowInputFormatParams & row_input_format_params;
|
||||
const FormatSettings &settings;
|
||||
};
|
||||
|
||||
struct Params
|
||||
{
|
||||
ReadBuffer & read_buffer;
|
||||
const InputProcessorCreator &input_processor_creator;
|
||||
const InputCreatorParams &input_creator_params;
|
||||
const InputProcessorCreator & input_processor_creator;
|
||||
const InputCreatorParams & input_creator_params;
|
||||
FormatFactory::FileSegmentationEngine file_segmentation_engine;
|
||||
int max_threads;
|
||||
size_t min_chunk_bytes;
|
||||
@ -79,7 +77,6 @@ public:
|
||||
|
||||
explicit ParallelParsingBlockInputStream(const Params & params)
|
||||
: header(params.input_creator_params.sample),
|
||||
context(params.input_creator_params.context),
|
||||
row_input_format_params(params.input_creator_params.row_input_format_params),
|
||||
format_settings(params.input_creator_params.settings),
|
||||
input_processor_creator(params.input_processor_creator),
|
||||
@ -149,7 +146,6 @@ protected:
|
||||
|
||||
private:
|
||||
const Block header;
|
||||
const Context context;
|
||||
const RowInputFormatParams row_input_format_params;
|
||||
const FormatSettings format_settings;
|
||||
const InputProcessorCreator input_processor_creator;
|
||||
|
@ -12,6 +12,8 @@
|
||||
#include <Processors/Formats/OutputStreamToOutputFormat.h>
|
||||
#include <DataStreams/SquashingBlockOutputStream.h>
|
||||
#include <DataStreams/NativeBlockInputStream.h>
|
||||
#include <Processors/Formats/Impl/ValuesBlockInputFormat.h>
|
||||
#include <Processors/Formats/Impl/MySQLOutputFormat.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -34,7 +36,7 @@ const FormatFactory::Creators & FormatFactory::getCreators(const String & name)
|
||||
}
|
||||
|
||||
|
||||
static FormatSettings getInputFormatSetting(const Settings & settings)
|
||||
static FormatSettings getInputFormatSetting(const Settings & settings, const Context & context)
|
||||
{
|
||||
FormatSettings format_settings;
|
||||
format_settings.csv.delimiter = settings.format_csv_delimiter;
|
||||
@ -56,11 +58,21 @@ static FormatSettings getInputFormatSetting(const Settings & settings)
|
||||
format_settings.template_settings.row_format = settings.format_template_row;
|
||||
format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
|
||||
format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default;
|
||||
format_settings.schema.format_schema = settings.format_schema;
|
||||
format_settings.schema.format_schema_path = context.getFormatSchemaPath();
|
||||
format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);
|
||||
format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter;
|
||||
format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter;
|
||||
format_settings.custom.escaping_rule = settings.format_custom_escaping_rule;
|
||||
format_settings.custom.field_delimiter = settings.format_custom_field_delimiter;
|
||||
format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter;
|
||||
format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter;
|
||||
format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter;
|
||||
|
||||
return format_settings;
|
||||
}
|
||||
|
||||
static FormatSettings getOutputFormatSetting(const Settings & settings)
|
||||
static FormatSettings getOutputFormatSetting(const Settings & settings, const Context & context)
|
||||
{
|
||||
FormatSettings format_settings;
|
||||
format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers;
|
||||
@ -77,6 +89,16 @@ static FormatSettings getOutputFormatSetting(const Settings & settings)
|
||||
format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
|
||||
format_settings.write_statistics = settings.output_format_write_statistics;
|
||||
format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size;
|
||||
format_settings.schema.format_schema = settings.format_schema;
|
||||
format_settings.schema.format_schema_path = context.getFormatSchemaPath();
|
||||
format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);
|
||||
format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter;
|
||||
format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter;
|
||||
format_settings.custom.escaping_rule = settings.format_custom_escaping_rule;
|
||||
format_settings.custom.field_delimiter = settings.format_custom_field_delimiter;
|
||||
format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter;
|
||||
format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter;
|
||||
format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter;
|
||||
|
||||
return format_settings;
|
||||
}
|
||||
@ -100,9 +122,9 @@ BlockInputStreamPtr FormatFactory::getInput(
|
||||
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
FormatSettings format_settings = getInputFormatSetting(settings);
|
||||
FormatSettings format_settings = getInputFormatSetting(settings, context);
|
||||
|
||||
return input_getter(buf, sample, context, max_block_size, callback ? callback : ReadCallback(), format_settings);
|
||||
return input_getter(buf, sample, max_block_size, callback ? callback : ReadCallback(), format_settings);
|
||||
}
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
@ -118,7 +140,7 @@ BlockInputStreamPtr FormatFactory::getInput(
|
||||
if (!input_getter)
|
||||
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
|
||||
|
||||
FormatSettings format_settings = getInputFormatSetting(settings);
|
||||
FormatSettings format_settings = getInputFormatSetting(settings, context);
|
||||
|
||||
RowInputFormatParams row_input_format_params;
|
||||
row_input_format_params.max_block_size = max_block_size;
|
||||
@ -128,7 +150,7 @@ BlockInputStreamPtr FormatFactory::getInput(
|
||||
row_input_format_params.max_execution_time = settings.max_execution_time;
|
||||
row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode;
|
||||
|
||||
auto input_creator_params = ParallelParsingBlockInputStream::InputCreatorParams{sample, context, row_input_format_params, format_settings};
|
||||
auto input_creator_params = ParallelParsingBlockInputStream::InputCreatorParams{sample, row_input_format_params, format_settings};
|
||||
ParallelParsingBlockInputStream::Params params{buf, input_getter,
|
||||
input_creator_params, file_segmentation_engine,
|
||||
static_cast<int>(settings.max_threads),
|
||||
@ -164,16 +186,16 @@ BlockOutputStreamPtr FormatFactory::getOutput(
|
||||
throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
FormatSettings format_settings = getOutputFormatSetting(settings);
|
||||
FormatSettings format_settings = getOutputFormatSetting(settings, context);
|
||||
|
||||
/** Materialization is needed, because formats can use the functions `IDataType`,
|
||||
* which only work with full columns.
|
||||
*/
|
||||
return std::make_shared<MaterializingBlockOutputStream>(
|
||||
output_getter(buf, sample, context, callback, format_settings), sample);
|
||||
output_getter(buf, sample, std::move(callback), format_settings), sample);
|
||||
}
|
||||
|
||||
auto format = getOutputFormat(name, buf, sample, context, callback);
|
||||
auto format = getOutputFormat(name, buf, sample, context, std::move(callback));
|
||||
return std::make_shared<MaterializingBlockOutputStream>(std::make_shared<OutputStreamToOutputFormat>(format), sample);
|
||||
}
|
||||
|
||||
@ -191,7 +213,7 @@ InputFormatPtr FormatFactory::getInputFormat(
|
||||
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
FormatSettings format_settings = getInputFormatSetting(settings);
|
||||
FormatSettings format_settings = getInputFormatSetting(settings, context);
|
||||
|
||||
RowInputFormatParams params;
|
||||
params.max_block_size = max_block_size;
|
||||
@ -201,7 +223,13 @@ InputFormatPtr FormatFactory::getInputFormat(
|
||||
params.max_execution_time = settings.max_execution_time;
|
||||
params.timeout_overflow_mode = settings.timeout_overflow_mode;
|
||||
|
||||
return input_getter(buf, sample, context, params, format_settings);
|
||||
auto format = input_getter(buf, sample, params, format_settings);
|
||||
|
||||
/// It's a kludge. Because I cannot remove context from values format.
|
||||
if (auto * values = typeid_cast<ValuesBlockInputFormat *>(format.get()))
|
||||
values->setContext(context);
|
||||
|
||||
return format;
|
||||
}
|
||||
|
||||
|
||||
@ -213,12 +241,18 @@ OutputFormatPtr FormatFactory::getOutputFormat(
|
||||
throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
FormatSettings format_settings = getOutputFormatSetting(settings);
|
||||
FormatSettings format_settings = getOutputFormatSetting(settings, context);
|
||||
|
||||
/** TODO: Materialization is needed, because formats can use the functions `IDataType`,
|
||||
* which only work with full columns.
|
||||
*/
|
||||
return output_getter(buf, sample, context, callback, format_settings);
|
||||
auto format = output_getter(buf, sample, std::move(callback), format_settings);
|
||||
|
||||
/// It's a kludge. Because I cannot remove context from MySQL format.
|
||||
if (auto * mysql = typeid_cast<MySQLOutputFormat *>(format.get()))
|
||||
mysql->setContext(context);
|
||||
|
||||
return format;
|
||||
}
|
||||
|
||||
|
||||
@ -259,7 +293,7 @@ void FormatFactory::registerFileSegmentationEngine(const String & name, FileSegm
|
||||
auto & target = dict[name].file_segmentation_engine;
|
||||
if (target)
|
||||
throw Exception("FormatFactory: File segmentation engine " + name + " is already registered", ErrorCodes::LOGICAL_ERROR);
|
||||
target = file_segmentation_engine;
|
||||
target = std::move(file_segmentation_engine);
|
||||
}
|
||||
|
||||
FormatFactory::FormatFactory()
|
||||
|
@ -59,7 +59,6 @@ private:
|
||||
using InputCreator = std::function<BlockInputStreamPtr(
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
UInt64 max_block_size,
|
||||
ReadCallback callback,
|
||||
const FormatSettings & settings)>;
|
||||
@ -67,21 +66,18 @@ private:
|
||||
using OutputCreator = std::function<BlockOutputStreamPtr(
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
WriteCallback callback,
|
||||
const FormatSettings & settings)>;
|
||||
|
||||
using InputProcessorCreator = std::function<InputFormatPtr(
|
||||
ReadBuffer & buf,
|
||||
const Block & header,
|
||||
const Context & context,
|
||||
const RowInputFormatParams & params,
|
||||
const FormatSettings & settings)>;
|
||||
|
||||
using OutputProcessorCreator = std::function<OutputFormatPtr(
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
WriteCallback callback,
|
||||
const FormatSettings & settings)>;
|
||||
|
||||
|
@ -26,7 +26,7 @@ namespace
|
||||
}
|
||||
|
||||
|
||||
FormatSchemaInfo::FormatSchemaInfo(const Context & context, const String & format_schema, const String & format, bool require_message)
|
||||
FormatSchemaInfo::FormatSchemaInfo(const String & format_schema, const String & format, bool require_message, bool is_server, const std::string & format_schema_path)
|
||||
{
|
||||
if (format_schema.empty())
|
||||
throw Exception(
|
||||
@ -54,29 +54,25 @@ FormatSchemaInfo::FormatSchemaInfo(const Context & context, const String & forma
|
||||
else
|
||||
path.assign(format_schema).makeFile().getFileName();
|
||||
|
||||
auto default_schema_directory = [&context]()
|
||||
auto default_schema_directory = [&format_schema_path]()
|
||||
{
|
||||
static const String str = Poco::Path(context.getFormatSchemaPath()).makeAbsolute().makeDirectory().toString();
|
||||
static const String str = Poco::Path(format_schema_path).makeAbsolute().makeDirectory().toString();
|
||||
return str;
|
||||
};
|
||||
auto is_server = [&context]()
|
||||
{
|
||||
return context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);
|
||||
};
|
||||
|
||||
if (path.getExtension().empty() && !default_file_extension.empty())
|
||||
path.setExtension(default_file_extension);
|
||||
|
||||
if (path.isAbsolute())
|
||||
{
|
||||
if (is_server())
|
||||
if (is_server)
|
||||
throw Exception("Absolute path in the 'format_schema' setting is prohibited: " + path.toString(), ErrorCodes::BAD_ARGUMENTS);
|
||||
schema_path = path.getFileName();
|
||||
schema_directory = path.makeParent().toString();
|
||||
}
|
||||
else if (path.depth() >= 1 && path.directory(0) == "..")
|
||||
{
|
||||
if (is_server())
|
||||
if (is_server)
|
||||
throw Exception(
|
||||
"Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: " + path.toString(),
|
||||
ErrorCodes::BAD_ARGUMENTS);
|
||||
|
@ -10,7 +10,7 @@ class Context;
|
||||
class FormatSchemaInfo
|
||||
{
|
||||
public:
|
||||
FormatSchemaInfo(const Context & context, const String & format_schema, const String & format, bool require_message);
|
||||
FormatSchemaInfo(const String & format_schema, const String & format, bool require_message, bool is_server, const std::string & format_schema_path);
|
||||
|
||||
/// Returns path to the schema file.
|
||||
const String & schemaPath() const { return schema_path; }
|
||||
|
@ -89,6 +89,27 @@ struct FormatSettings
|
||||
UInt64 row_group_size = 1000000;
|
||||
} parquet;
|
||||
|
||||
struct Schema
|
||||
{
|
||||
std::string format_schema;
|
||||
std::string format_schema_path;
|
||||
bool is_server = false;
|
||||
};
|
||||
|
||||
Schema schema;
|
||||
|
||||
struct Custom
|
||||
{
|
||||
std::string result_before_delimiter;
|
||||
std::string result_after_delimiter;
|
||||
std::string row_before_delimiter;
|
||||
std::string row_after_delimiter;
|
||||
std::string row_between_delimiter;
|
||||
std::string field_delimiter;
|
||||
std::string escaping_rule;
|
||||
};
|
||||
|
||||
Custom custom;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -11,7 +11,6 @@ void registerInputFormatNative(FormatFactory & factory)
|
||||
factory.registerInputFormat("Native", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
UInt64 /* max_block_size */,
|
||||
FormatFactory::ReadCallback /* callback */,
|
||||
const FormatSettings &)
|
||||
@ -25,7 +24,6 @@ void registerOutputFormatNative(FormatFactory & factory)
|
||||
factory.registerOutputFormat("Native", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings &)
|
||||
{
|
||||
|
@ -10,7 +10,6 @@ void registerOutputFormatNull(FormatFactory & factory)
|
||||
factory.registerOutputFormat("Null", [](
|
||||
WriteBuffer &,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings &)
|
||||
{
|
||||
|
@ -234,36 +234,32 @@ void ParsedTemplateFormatString::throwInvalidFormat(const String & message, size
|
||||
ErrorCodes::INVALID_TEMPLATE_FORMAT);
|
||||
}
|
||||
|
||||
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(const Context & context)
|
||||
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(const FormatSettings::Custom & settings)
|
||||
{
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
|
||||
/// Set resultset format to "result_before_delimiter ${data} result_after_delimiter"
|
||||
ParsedTemplateFormatString resultset_format;
|
||||
resultset_format.delimiters.emplace_back(settings.format_custom_result_before_delimiter);
|
||||
resultset_format.delimiters.emplace_back(settings.format_custom_result_after_delimiter);
|
||||
resultset_format.delimiters.emplace_back(settings.result_before_delimiter);
|
||||
resultset_format.delimiters.emplace_back(settings.result_after_delimiter);
|
||||
resultset_format.formats.emplace_back(ParsedTemplateFormatString::ColumnFormat::None);
|
||||
resultset_format.format_idx_to_column_idx.emplace_back(0);
|
||||
resultset_format.column_names.emplace_back("data");
|
||||
return resultset_format;
|
||||
}
|
||||
|
||||
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedRowFormat(const Context & context, const Block & sample)
|
||||
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedRowFormat(const FormatSettings::Custom & settings, const Block & sample)
|
||||
{
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
|
||||
/// Set row format to
|
||||
/// "row_before_delimiter ${Col0:escaping} field_delimiter ${Col1:escaping} field_delimiter ... ${ColN:escaping} row_after_delimiter"
|
||||
ParsedTemplateFormatString::ColumnFormat escaping = ParsedTemplateFormatString::stringToFormat(settings.format_custom_escaping_rule);
|
||||
ParsedTemplateFormatString::ColumnFormat escaping = ParsedTemplateFormatString::stringToFormat(settings.escaping_rule);
|
||||
ParsedTemplateFormatString row_format;
|
||||
row_format.delimiters.emplace_back(settings.format_custom_row_before_delimiter);
|
||||
row_format.delimiters.emplace_back(settings.row_before_delimiter);
|
||||
for (size_t i = 0; i < sample.columns(); ++i)
|
||||
{
|
||||
row_format.formats.emplace_back(escaping);
|
||||
row_format.format_idx_to_column_idx.emplace_back(i);
|
||||
row_format.column_names.emplace_back(sample.getByPosition(i).name);
|
||||
bool last_column = i == sample.columns() - 1;
|
||||
row_format.delimiters.emplace_back(last_column ? settings.format_custom_row_after_delimiter : settings.format_custom_field_delimiter);
|
||||
row_format.delimiters.emplace_back(last_column ? settings.row_after_delimiter : settings.field_delimiter);
|
||||
}
|
||||
return row_format;
|
||||
}
|
||||
|
@ -4,6 +4,7 @@
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <Formats/FormatSchemaInfo.h>
|
||||
#include <Formats/FormatSettings.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -49,8 +50,8 @@ struct ParsedTemplateFormatString
|
||||
String dump() const;
|
||||
[[noreturn]] void throwInvalidFormat(const String & message, size_t column) const;
|
||||
|
||||
static ParsedTemplateFormatString setupCustomSeparatedResultsetFormat(const Context & context);
|
||||
static ParsedTemplateFormatString setupCustomSeparatedRowFormat(const Context & context, const Block & sample);
|
||||
static ParsedTemplateFormatString setupCustomSeparatedResultsetFormat(const FormatSettings::Custom & settings);
|
||||
static ParsedTemplateFormatString setupCustomSeparatedRowFormat(const FormatSettings::Custom & settings, const Block & sample);
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -5,29 +5,6 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
void throwExceptionForIncompletelyParsedValue(
|
||||
ReadBuffer & read_buffer, Block & block, size_t result)
|
||||
{
|
||||
const IDataType & to_type = *block.getByPosition(result).type;
|
||||
|
||||
WriteBufferFromOwnString message_buf;
|
||||
message_buf << "Cannot parse string " << quote << String(read_buffer.buffer().begin(), read_buffer.buffer().size())
|
||||
<< " as " << to_type.getName()
|
||||
<< ": syntax error";
|
||||
|
||||
if (read_buffer.offset())
|
||||
message_buf << " at position " << read_buffer.offset()
|
||||
<< " (parsed just " << quote << String(read_buffer.buffer().begin(), read_buffer.offset()) << ")";
|
||||
else
|
||||
message_buf << " at begin of string";
|
||||
|
||||
if (isNativeNumber(to_type))
|
||||
message_buf << ". Note: there are to" << to_type.getName() << "OrZero and to" << to_type.getName() << "OrNull functions, which returns zero/NULL instead of throwing exception.";
|
||||
|
||||
throw Exception(message_buf.str(), ErrorCodes::CANNOT_PARSE_TEXT);
|
||||
}
|
||||
|
||||
|
||||
void registerFunctionsConversion(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionToUInt8>();
|
||||
|
@ -501,7 +501,26 @@ inline bool tryParseImpl<DataTypeDateTime>(DataTypeDateTime::FieldType & x, Read
|
||||
|
||||
/** Throw exception with verbose message when string value is not parsed completely.
|
||||
*/
|
||||
[[noreturn]] void throwExceptionForIncompletelyParsedValue(ReadBuffer & read_buffer, Block & block, size_t result);
|
||||
[[noreturn]] inline void throwExceptionForIncompletelyParsedValue(ReadBuffer & read_buffer, Block & block, size_t result)
|
||||
{
|
||||
const IDataType & to_type = *block.getByPosition(result).type;
|
||||
|
||||
WriteBufferFromOwnString message_buf;
|
||||
message_buf << "Cannot parse string " << quote << String(read_buffer.buffer().begin(), read_buffer.buffer().size())
|
||||
<< " as " << to_type.getName()
|
||||
<< ": syntax error";
|
||||
|
||||
if (read_buffer.offset())
|
||||
message_buf << " at position " << read_buffer.offset()
|
||||
<< " (parsed just " << quote << String(read_buffer.buffer().begin(), read_buffer.offset()) << ")";
|
||||
else
|
||||
message_buf << " at begin of string";
|
||||
|
||||
if (isNativeNumber(to_type))
|
||||
message_buf << ". Note: there are to" << to_type.getName() << "OrZero and to" << to_type.getName() << "OrNull functions, which returns zero/NULL instead of throwing exception.";
|
||||
|
||||
throw Exception(message_buf.str(), ErrorCodes::CANNOT_PARSE_TEXT);
|
||||
}
|
||||
|
||||
|
||||
enum class ConvertFromStringExceptionMode
|
||||
@ -886,6 +905,7 @@ public:
|
||||
static constexpr bool to_datetime64 = std::is_same_v<ToDataType, DataTypeDateTime64>;
|
||||
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionConvert>(); }
|
||||
static FunctionPtr create() { return std::make_shared<FunctionConvert>(); }
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
@ -1083,6 +1103,7 @@ public:
|
||||
std::is_same_v<ToDataType, DataTypeDecimal<Decimal128>>;
|
||||
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionConvertFromString>(); }
|
||||
static FunctionPtr create() { return std::make_shared<FunctionConvertFromString>(); }
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
@ -1231,6 +1252,7 @@ class FunctionToFixedString : public IFunction
|
||||
public:
|
||||
static constexpr auto name = "toFixedString";
|
||||
static FunctionPtr create(const Context &) { return std::make_shared<FunctionToFixedString>(); }
|
||||
static FunctionPtr create() { return std::make_shared<FunctionToFixedString>(); }
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
@ -1686,9 +1708,9 @@ public:
|
||||
using WrapperType = std::function<void(Block &, const ColumnNumbers &, size_t, size_t)>;
|
||||
using MonotonicityForRange = std::function<Monotonicity(const IDataType &, const Field &, const Field &)>;
|
||||
|
||||
FunctionCast(const Context & context_, const char * name_, MonotonicityForRange && monotonicity_for_range_
|
||||
FunctionCast(const char * name_, MonotonicityForRange && monotonicity_for_range_
|
||||
, const DataTypes & argument_types_, const DataTypePtr & return_type_)
|
||||
: context(context_), name(name_), monotonicity_for_range(monotonicity_for_range_)
|
||||
: name(name_), monotonicity_for_range(monotonicity_for_range_)
|
||||
, argument_types(argument_types_), return_type(return_type_)
|
||||
{
|
||||
}
|
||||
@ -1719,7 +1741,6 @@ public:
|
||||
|
||||
private:
|
||||
|
||||
const Context & context;
|
||||
const char * name;
|
||||
MonotonicityForRange monotonicity_for_range;
|
||||
|
||||
@ -1735,10 +1756,10 @@ private:
|
||||
{
|
||||
/// In case when converting to Nullable type, we apply different parsing rule,
|
||||
/// that will not throw an exception but return NULL in case of malformed input.
|
||||
function = FunctionConvertFromString<DataType, NameCast, ConvertFromStringExceptionMode::Null>::create(context);
|
||||
function = FunctionConvertFromString<DataType, NameCast, ConvertFromStringExceptionMode::Null>::create();
|
||||
}
|
||||
else
|
||||
function = FunctionTo<DataType>::Type::create(context);
|
||||
function = FunctionTo<DataType>::Type::create();
|
||||
|
||||
auto function_adaptor =
|
||||
FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
|
||||
@ -1752,7 +1773,7 @@ private:
|
||||
|
||||
WrapperType createStringWrapper(const DataTypePtr & from_type) const
|
||||
{
|
||||
FunctionPtr function = FunctionToString::create(context);
|
||||
FunctionPtr function = FunctionToString::create();
|
||||
|
||||
auto function_adaptor =
|
||||
FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
|
||||
@ -1780,7 +1801,7 @@ private:
|
||||
if (requested_result_is_nullable)
|
||||
throw Exception{"CAST AS Nullable(UUID) is not implemented", ErrorCodes::NOT_IMPLEMENTED};
|
||||
|
||||
FunctionPtr function = FunctionTo<DataTypeUUID>::Type::create(context);
|
||||
FunctionPtr function = FunctionTo<DataTypeUUID>::Type::create();
|
||||
|
||||
auto function_adaptor =
|
||||
FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
|
||||
@ -1985,7 +2006,7 @@ private:
|
||||
return createStringToEnumWrapper<ColumnFixedString, EnumType>();
|
||||
else if (isNativeNumber(from_type) || isEnum(from_type))
|
||||
{
|
||||
auto function = Function::create(context);
|
||||
auto function = Function::create();
|
||||
auto func_or_adaptor = FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
|
||||
.build(ColumnsWithTypeAndName{{nullptr, from_type, "" }});
|
||||
|
||||
@ -2337,9 +2358,10 @@ public:
|
||||
using MonotonicityForRange = FunctionCast::MonotonicityForRange;
|
||||
|
||||
static constexpr auto name = "CAST";
|
||||
static FunctionOverloadResolverImplPtr create(const Context & context) { return std::make_unique<CastOverloadResolver>(context); }
|
||||
static FunctionOverloadResolverImplPtr create(const Context &) { return createImpl(); }
|
||||
static FunctionOverloadResolverImplPtr createImpl() { return std::make_unique<CastOverloadResolver>(); }
|
||||
|
||||
CastOverloadResolver(const Context & context_) : context(context_) {}
|
||||
CastOverloadResolver() {}
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
@ -2357,7 +2379,7 @@ protected:
|
||||
data_types[i] = arguments[i].type;
|
||||
|
||||
auto monotonicity = getMonotonicityInformation(arguments.front().type, return_type.get());
|
||||
return std::make_unique<FunctionCast>(context, name, std::move(monotonicity), data_types, return_type);
|
||||
return std::make_unique<FunctionCast>(name, std::move(monotonicity), data_types, return_type);
|
||||
}
|
||||
|
||||
DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments) const override
|
||||
@ -2418,8 +2440,6 @@ private:
|
||||
/// other types like Null, FixedString, Array and Tuple have no monotonicity defined
|
||||
return {};
|
||||
}
|
||||
|
||||
const Context & context;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -339,6 +339,8 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run)
|
||||
affected_materialized.emplace(mat_column);
|
||||
}
|
||||
|
||||
/// Just to be sure, that we don't change type
|
||||
/// after update expression execution.
|
||||
const auto & update_expr = kv.second;
|
||||
auto updated_column = makeASTFunction("CAST",
|
||||
makeASTFunction("if",
|
||||
|
@ -65,8 +65,9 @@ private:
|
||||
/// Each stage has output_columns that contain columns that are changed at the end of that stage
|
||||
/// plus columns needed for the next mutations.
|
||||
///
|
||||
/// First stage is special: it can contain only DELETEs and is executed using InterpreterSelectQuery
|
||||
/// to take advantage of table indexes (if there are any).
|
||||
/// First stage is special: it can contain only filters and is executed using InterpreterSelectQuery
|
||||
/// to take advantage of table indexes (if there are any). It's necessary because all mutations have
|
||||
/// `WHERE clause` part.
|
||||
|
||||
struct Stage
|
||||
{
|
||||
@ -83,7 +84,7 @@ private:
|
||||
|
||||
/// A chain of actions needed to execute this stage.
|
||||
/// First steps calculate filter columns for DELETEs (in the same order as in `filter_column_names`),
|
||||
/// then there is (possibly) an UPDATE stage, and finally a projection stage.
|
||||
/// then there is (possibly) an UPDATE step, and finally a projection step.
|
||||
ExpressionActionsChain expressions_chain;
|
||||
Names filter_column_names;
|
||||
};
|
||||
|
@ -2,13 +2,14 @@
|
||||
#include <Interpreters/castColumn.h>
|
||||
#include <Interpreters/ExpressionActions.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/IFunctionAdaptors.h>
|
||||
#include <Functions/FunctionsConversion.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type, const Context & context)
|
||||
ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type)
|
||||
{
|
||||
if (arg.type->equals(*type))
|
||||
return arg.column;
|
||||
@ -28,7 +29,7 @@ ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type
|
||||
}
|
||||
};
|
||||
|
||||
FunctionOverloadResolverPtr func_builder_cast = FunctionFactory::instance().get("CAST", context);
|
||||
FunctionOverloadResolverPtr func_builder_cast = std::make_shared<FunctionOverloadResolverAdaptor>(CastOverloadResolver::createImpl());
|
||||
|
||||
ColumnsWithTypeAndName arguments{ temporary_block.getByPosition(0), temporary_block.getByPosition(1) };
|
||||
auto func_cast = func_builder_cast->build(arguments);
|
||||
@ -37,4 +38,9 @@ ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type
|
||||
return temporary_block.getByPosition(2).column;
|
||||
}
|
||||
|
||||
ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type, const Context &)
|
||||
{
|
||||
return castColumn(arg, type);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -6,7 +6,7 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type);
|
||||
ColumnPtr castColumn(const ColumnWithTypeAndName & arg, const DataTypePtr & type, const Context & context);
|
||||
|
||||
}
|
||||
|
@ -248,7 +248,7 @@ namespace DB
|
||||
|
||||
void ArrowColumnToCHColumn::arrowTableToCHChunk(Chunk &res, std::shared_ptr<arrow::Table> &table,
|
||||
arrow::Status &read_status, const Block &header,
|
||||
int &row_group_current, const Context &context, std::string format_name)
|
||||
int &row_group_current, std::string format_name)
|
||||
{
|
||||
Columns columns_list;
|
||||
UInt64 num_rows = 0;
|
||||
@ -389,7 +389,7 @@ namespace DB
|
||||
else
|
||||
column.column = std::move(read_column);
|
||||
|
||||
column.column = castColumn(column, column_type, context);
|
||||
column.column = castColumn(column, column_type);
|
||||
column.type = column_type;
|
||||
num_rows = column.column->size();
|
||||
columns_list.push_back(std::move(column.column));
|
||||
|
@ -39,7 +39,7 @@ namespace DB
|
||||
|
||||
static void arrowTableToCHChunk(Chunk &res, std::shared_ptr<arrow::Table> &table,
|
||||
arrow::Status &read_status, const Block &header,
|
||||
int &row_group_current, const Context &context, std::string format_name);
|
||||
int &row_group_current, std::string format_name);
|
||||
};
|
||||
}
|
||||
#endif
|
||||
|
@ -61,7 +61,6 @@ void registerInputFormatProcessorRowBinary(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor("RowBinary", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
const IRowInputFormat::Params & params,
|
||||
const FormatSettings &)
|
||||
{
|
||||
@ -71,7 +70,6 @@ void registerInputFormatProcessorRowBinary(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor("RowBinaryWithNamesAndTypes", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
const IRowInputFormat::Params & params,
|
||||
const FormatSettings &)
|
||||
{
|
||||
|
@ -52,7 +52,6 @@ void registerOutputFormatProcessorRowBinary(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("RowBinary", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings &)
|
||||
{
|
||||
@ -62,7 +61,6 @@ void registerOutputFormatProcessorRowBinary(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("RowBinaryWithNamesAndTypes", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings &)
|
||||
{
|
||||
|
@ -421,7 +421,6 @@ void registerInputFormatProcessorCSV(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor(with_names ? "CSVWithNames" : "CSV", [=](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
|
@ -76,7 +76,6 @@ void registerOutputFormatProcessorCSV(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor(with_names ? "CSVWithNames" : "CSV", [=](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
|
@ -301,10 +301,11 @@ void registerInputFormatProcessorCapnProto(FormatFactory & factory)
|
||||
{
|
||||
factory.registerInputFormatProcessor(
|
||||
"CapnProto",
|
||||
[](ReadBuffer & buf, const Block & sample, const Context & context, IRowInputFormat::Params params, const FormatSettings &)
|
||||
[](ReadBuffer & buf, const Block & sample, IRowInputFormat::Params params, const FormatSettings & settings)
|
||||
{
|
||||
return std::make_shared<CapnProtoRowInputFormat>(buf, sample, std::move(params),
|
||||
FormatSchemaInfo(context, context.getSettingsRef().format_schema, "CapnProto", true));
|
||||
FormatSchemaInfo(settings.schema.format_schema, "CapnProto", true,
|
||||
settings.schema.is_server, settings.schema.format_schema_path));
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -217,7 +217,6 @@ void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor("JSONCompactEachRow", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
@ -227,7 +226,6 @@ void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
|
@ -94,7 +94,6 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("JSONCompactEachRow", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
@ -104,7 +103,6 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", [](
|
||||
WriteBuffer &buf,
|
||||
const Block &sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings &format_settings)
|
||||
{
|
||||
|
@ -80,7 +80,6 @@ void registerOutputFormatProcessorJSONCompact(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("JSONCompact", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
|
@ -271,7 +271,6 @@ void registerInputFormatProcessorJSONEachRow(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor("JSONEachRow", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
|
@ -56,7 +56,6 @@ void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("JSONEachRow", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
|
@ -33,7 +33,6 @@ void registerOutputFormatProcessorJSONEachRowWithProgress(FormatFactory & factor
|
||||
factory.registerOutputFormatProcessor("JSONEachRowWithProgress", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
|
@ -246,7 +246,6 @@ void registerOutputFormatProcessorJSON(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("JSON", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
|
@ -12,13 +12,10 @@ namespace DB
|
||||
using namespace MySQLProtocol;
|
||||
|
||||
|
||||
MySQLOutputFormat::MySQLOutputFormat(WriteBuffer & out_, const Block & header_, const Context & context_, const FormatSettings & settings_)
|
||||
MySQLOutputFormat::MySQLOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & settings_)
|
||||
: IOutputFormat(header_, out_)
|
||||
, context(context_)
|
||||
, packet_sender(out, const_cast<uint8_t &>(context_.mysql.sequence_id)) /// TODO: fix it
|
||||
, format_settings(settings_)
|
||||
{
|
||||
packet_sender.max_packet_size = context_.mysql.max_packet_size;
|
||||
}
|
||||
|
||||
void MySQLOutputFormat::initialize()
|
||||
@ -32,17 +29,17 @@ void MySQLOutputFormat::initialize()
|
||||
|
||||
if (header.columns())
|
||||
{
|
||||
packet_sender.sendPacket(LengthEncodedNumber(header.columns()));
|
||||
packet_sender->sendPacket(LengthEncodedNumber(header.columns()));
|
||||
|
||||
for (size_t i = 0; i < header.columns(); i++)
|
||||
{
|
||||
const auto & column_name = header.getColumnsWithTypeAndName()[i].name;
|
||||
packet_sender.sendPacket(getColumnDefinition(column_name, data_types[i]->getTypeId()));
|
||||
packet_sender->sendPacket(getColumnDefinition(column_name, data_types[i]->getTypeId()));
|
||||
}
|
||||
|
||||
if (!(context.mysql.client_capabilities & Capability::CLIENT_DEPRECATE_EOF))
|
||||
if (!(context->mysql.client_capabilities & Capability::CLIENT_DEPRECATE_EOF))
|
||||
{
|
||||
packet_sender.sendPacket(EOF_Packet(0, 0));
|
||||
packet_sender->sendPacket(EOF_Packet(0, 0));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -53,7 +50,7 @@ void MySQLOutputFormat::consume(Chunk chunk)
|
||||
for (size_t i = 0; i < chunk.getNumRows(); i++)
|
||||
{
|
||||
ProtocolText::ResultsetRow row_packet(data_types, chunk.getColumns(), i);
|
||||
packet_sender.sendPacket(row_packet);
|
||||
packet_sender->sendPacket(row_packet);
|
||||
}
|
||||
}
|
||||
|
||||
@ -61,7 +58,7 @@ void MySQLOutputFormat::finalize()
|
||||
{
|
||||
size_t affected_rows = 0;
|
||||
std::stringstream human_readable_info;
|
||||
if (QueryStatus * process_list_elem = context.getProcessListElement())
|
||||
if (QueryStatus * process_list_elem = context->getProcessListElement())
|
||||
{
|
||||
CurrentThread::finalizePerformanceCounters();
|
||||
QueryStatusInfo info = process_list_elem->getInfo();
|
||||
@ -74,17 +71,17 @@ void MySQLOutputFormat::finalize()
|
||||
|
||||
const auto & header = getPort(PortKind::Main).getHeader();
|
||||
if (header.columns() == 0)
|
||||
packet_sender.sendPacket(OK_Packet(0x0, context.mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info.str()), true);
|
||||
packet_sender->sendPacket(OK_Packet(0x0, context->mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info.str()), true);
|
||||
else
|
||||
if (context.mysql.client_capabilities & CLIENT_DEPRECATE_EOF)
|
||||
packet_sender.sendPacket(OK_Packet(0xfe, context.mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info.str()), true);
|
||||
if (context->mysql.client_capabilities & CLIENT_DEPRECATE_EOF)
|
||||
packet_sender->sendPacket(OK_Packet(0xfe, context->mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info.str()), true);
|
||||
else
|
||||
packet_sender.sendPacket(EOF_Packet(0, 0), true);
|
||||
packet_sender->sendPacket(EOF_Packet(0, 0), true);
|
||||
}
|
||||
|
||||
void MySQLOutputFormat::flush()
|
||||
{
|
||||
packet_sender.out->next();
|
||||
packet_sender->out->next();
|
||||
}
|
||||
|
||||
void registerOutputFormatProcessorMySQLWrite(FormatFactory & factory)
|
||||
@ -93,9 +90,8 @@ void registerOutputFormatProcessorMySQLWrite(FormatFactory & factory)
|
||||
"MySQLWire",
|
||||
[](WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings & settings) { return std::make_shared<MySQLOutputFormat>(buf, sample, context, settings); });
|
||||
const FormatSettings & settings) { return std::make_shared<MySQLOutputFormat>(buf, sample, settings); });
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -16,13 +16,20 @@ class Context;
|
||||
|
||||
/** A stream for outputting data in a binary line-by-line format.
|
||||
*/
|
||||
class MySQLOutputFormat: public IOutputFormat
|
||||
class MySQLOutputFormat final : public IOutputFormat
|
||||
{
|
||||
public:
|
||||
MySQLOutputFormat(WriteBuffer & out_, const Block & header_, const Context & context_, const FormatSettings & settings_);
|
||||
MySQLOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & settings_);
|
||||
|
||||
String getName() const override { return "MySQLOutputFormat"; }
|
||||
|
||||
void setContext(const Context & context_)
|
||||
{
|
||||
context = &context_;
|
||||
packet_sender = std::make_unique<MySQLProtocol::PacketSender>(out, const_cast<uint8_t &>(context_.mysql.sequence_id)); /// TODO: fix it
|
||||
packet_sender->max_packet_size = context_.mysql.max_packet_size;
|
||||
}
|
||||
|
||||
void consume(Chunk) override;
|
||||
void finalize() override;
|
||||
void flush() override;
|
||||
@ -34,8 +41,8 @@ private:
|
||||
|
||||
bool initialized = false;
|
||||
|
||||
const Context & context;
|
||||
MySQLProtocol::PacketSender packet_sender;
|
||||
const Context * context = nullptr;
|
||||
std::unique_ptr<MySQLProtocol::PacketSender> packet_sender;
|
||||
FormatSettings format_settings;
|
||||
DataTypes data_types;
|
||||
};
|
||||
|
@ -156,7 +156,6 @@ void registerInputFormatProcessorNative(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor("Native", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
const RowInputFormatParams &,
|
||||
const FormatSettings &)
|
||||
{
|
||||
@ -169,7 +168,6 @@ void registerOutputFormatProcessorNative(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("Native", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings &)
|
||||
{
|
||||
|
@ -21,7 +21,6 @@ void registerOutputFormatProcessorNull(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("Null", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings &)
|
||||
{
|
||||
|
@ -107,7 +107,7 @@ void ODBCDriver2BlockOutputFormat::writePrefix()
|
||||
void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory)
|
||||
{
|
||||
factory.registerOutputFormatProcessor(
|
||||
"ODBCDriver2", [](WriteBuffer & buf, const Block & sample, const Context &, FormatFactory::WriteCallback, const FormatSettings & format_settings)
|
||||
"ODBCDriver2", [](WriteBuffer & buf, const Block & sample, FormatFactory::WriteCallback, const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<ODBCDriver2BlockOutputFormat>(buf, sample, format_settings);
|
||||
});
|
||||
|
@ -69,7 +69,6 @@ void registerOutputFormatProcessorODBCDriver(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("ODBCDriver", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
|
@ -12,8 +12,8 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
ORCBlockInputFormat::ORCBlockInputFormat(ReadBuffer &in_, Block header_, const Context &context_)
|
||||
: IInputFormat(std::move(header_), in_), context{context_} {
|
||||
ORCBlockInputFormat::ORCBlockInputFormat(ReadBuffer &in_, Block header_) : IInputFormat(std::move(header_), in_)
|
||||
{
|
||||
}
|
||||
|
||||
Chunk ORCBlockInputFormat::generate()
|
||||
@ -57,7 +57,7 @@ namespace DB
|
||||
|
||||
arrow::Status read_status = file_reader->Read(&table);
|
||||
|
||||
ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, context, "ORC");
|
||||
ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, "ORC");
|
||||
|
||||
return res;
|
||||
}
|
||||
@ -78,11 +78,10 @@ namespace DB
|
||||
"ORC",
|
||||
[](ReadBuffer &buf,
|
||||
const Block &sample,
|
||||
const Context &context,
|
||||
const RowInputFormatParams &,
|
||||
const FormatSettings & /* settings */)
|
||||
{
|
||||
return std::make_shared<ORCBlockInputFormat>(buf, sample, context);
|
||||
return std::make_shared<ORCBlockInputFormat>(buf, sample);
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -17,7 +17,7 @@ class Context;
|
||||
class ORCBlockInputFormat: public IInputFormat
|
||||
{
|
||||
public:
|
||||
ORCBlockInputFormat(ReadBuffer & in_, Block header_, const Context & context_);
|
||||
ORCBlockInputFormat(ReadBuffer & in_, Block header_);
|
||||
|
||||
String getName() const override { return "ORCBlockInputFormat"; }
|
||||
|
||||
@ -30,8 +30,6 @@ private:
|
||||
|
||||
// TODO: check that this class implements every part of its parent
|
||||
|
||||
const Context & context;
|
||||
|
||||
std::unique_ptr<arrow::adapters::orc::ORCFileReader> file_reader;
|
||||
std::string file_data;
|
||||
int row_group_total = 0;
|
||||
|
@ -15,8 +15,9 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
ParquetBlockInputFormat::ParquetBlockInputFormat(ReadBuffer &in_, Block header_, const Context &context_)
|
||||
: IInputFormat(std::move(header_), in_), context{context_} {
|
||||
ParquetBlockInputFormat::ParquetBlockInputFormat(ReadBuffer & in_, Block header_)
|
||||
: IInputFormat(std::move(header_), in_)
|
||||
{
|
||||
}
|
||||
|
||||
Chunk ParquetBlockInputFormat::generate()
|
||||
@ -59,7 +60,7 @@ namespace DB
|
||||
std::shared_ptr<arrow::Table> table;
|
||||
arrow::Status read_status = file_reader->ReadRowGroup(row_group_current, &table);
|
||||
|
||||
ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, context, "Parquet");
|
||||
ArrowColumnToCHColumn::arrowTableToCHChunk(res, table, read_status, header, row_group_current, "Parquet");
|
||||
return res;
|
||||
}
|
||||
|
||||
@ -80,11 +81,10 @@ namespace DB
|
||||
"Parquet",
|
||||
[](ReadBuffer &buf,
|
||||
const Block &sample,
|
||||
const Context &context,
|
||||
const RowInputFormatParams &,
|
||||
const FormatSettings & /* settings */)
|
||||
{
|
||||
return std::make_shared<ParquetBlockInputFormat>(buf, sample, context);
|
||||
return std::make_shared<ParquetBlockInputFormat>(buf, sample);
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -16,7 +16,7 @@ class Context;
|
||||
class ParquetBlockInputFormat: public IInputFormat
|
||||
{
|
||||
public:
|
||||
ParquetBlockInputFormat(ReadBuffer & in_, Block header_, const Context & context_);
|
||||
ParquetBlockInputFormat(ReadBuffer & in_, Block header_);
|
||||
|
||||
void resetParser() override;
|
||||
|
||||
@ -30,8 +30,6 @@ private:
|
||||
|
||||
// TODO: check that this class implements every part of its parent
|
||||
|
||||
const Context & context;
|
||||
|
||||
std::unique_ptr<parquet::arrow::FileReader> file_reader;
|
||||
std::string file_data;
|
||||
std::unique_ptr<arrow::Buffer> buffer;
|
||||
|
@ -426,7 +426,6 @@ void registerOutputFormatProcessorParquet(FormatFactory & factory)
|
||||
"Parquet",
|
||||
[](WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & /*context*/,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
|
@ -260,7 +260,6 @@ void registerOutputFormatProcessorPretty(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("Pretty", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
@ -270,7 +269,6 @@ void registerOutputFormatProcessorPretty(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("PrettyNoEscapes", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
|
@ -134,7 +134,6 @@ void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("PrettyCompact", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
@ -144,7 +143,6 @@ void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("PrettyCompactNoEscapes", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
@ -157,7 +155,6 @@ void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory)
|
||||
// factory.registerOutputFormat("PrettyCompactMonoBlock", [](
|
||||
// WriteBuffer & buf,
|
||||
// const Block & sample,
|
||||
// const Context &,
|
||||
// const FormatSettings & format_settings)
|
||||
// {
|
||||
// BlockOutputStreamPtr impl = std::make_shared<PrettyCompactBlockOutputFormat>(buf, sample, format_settings);
|
||||
|
@ -97,7 +97,6 @@ void registerOutputFormatProcessorPrettySpace(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("PrettySpace", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
@ -107,7 +106,6 @@ void registerOutputFormatProcessorPrettySpace(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("PrettySpaceNoEscapes", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
|
@ -70,12 +70,12 @@ void registerInputFormatProcessorProtobuf(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor("Protobuf", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings &)
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
return std::make_shared<ProtobufRowInputFormat>(buf, sample, std::move(params),
|
||||
FormatSchemaInfo(context, context.getSettingsRef().format_schema, "Protobuf", true));
|
||||
FormatSchemaInfo(settings.schema.format_schema, "Protobuf", true,
|
||||
settings.schema.is_server, settings.schema.format_schema_path));
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -50,12 +50,12 @@ void registerOutputFormatProcessorProtobuf(FormatFactory & factory)
|
||||
"Protobuf",
|
||||
[](WriteBuffer & buf,
|
||||
const Block & header,
|
||||
const Context & context,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings &)
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
return std::make_shared<ProtobufRowOutputFormat>(buf, header, callback,
|
||||
FormatSchemaInfo(context, context.getSettingsRef().format_schema, "Protobuf", true));
|
||||
return std::make_shared<ProtobufRowOutputFormat>(buf, header, std::move(callback),
|
||||
FormatSchemaInfo(settings.schema.format_schema, "Protobuf", true,
|
||||
settings.schema.is_server, settings.schema.format_schema_path));
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -210,7 +210,6 @@ void registerInputFormatProcessorTSKV(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor("TSKV", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
|
@ -45,7 +45,6 @@ void registerOutputFormatProcessorTSKV(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("TSKV", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
|
@ -356,7 +356,6 @@ void registerInputFormatProcessorTabSeparated(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor(name, [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
@ -369,7 +368,6 @@ void registerInputFormatProcessorTabSeparated(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor(name, [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
@ -382,7 +380,6 @@ void registerInputFormatProcessorTabSeparated(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor(name, [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
|
@ -78,7 +78,6 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor(name, [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
@ -91,7 +90,6 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor(name, [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
@ -104,7 +102,6 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor(name, [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
@ -117,7 +114,6 @@ void registerOutputFormatProcessorTabSeparated(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor(name, [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
|
@ -14,8 +14,10 @@ namespace ErrorCodes
|
||||
}
|
||||
|
||||
TemplateBlockOutputFormat::TemplateBlockOutputFormat(const Block & header_, WriteBuffer & out_, const FormatSettings & settings_,
|
||||
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_)
|
||||
: IOutputFormat(header_, out_), settings(settings_), format(std::move(format_)), row_format(std::move(row_format_))
|
||||
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_,
|
||||
std::string row_between_delimiter_)
|
||||
: IOutputFormat(header_, out_), settings(settings_), format(std::move(format_))
|
||||
, row_format(std::move(row_format_)), row_between_delimiter(std::move(row_between_delimiter_))
|
||||
{
|
||||
auto & sample = getPort(PortKind::Main).getHeader();
|
||||
size_t columns = sample.columns();
|
||||
@ -152,7 +154,7 @@ void TemplateBlockOutputFormat::consume(Chunk chunk)
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
if (row_count)
|
||||
writeString(settings.template_settings.row_between_delimiter, out);
|
||||
writeString(row_between_delimiter, out);
|
||||
|
||||
writeRow(chunk, i);
|
||||
++row_count;
|
||||
@ -230,7 +232,6 @@ void registerOutputFormatProcessorTemplate(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("Template", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
@ -247,7 +248,8 @@ void registerOutputFormatProcessorTemplate(FormatFactory & factory)
|
||||
{
|
||||
/// Read format string from file
|
||||
resultset_format = ParsedTemplateFormatString(
|
||||
FormatSchemaInfo(context, settings.template_settings.resultset_format, "Template", false),
|
||||
FormatSchemaInfo(settings.template_settings.resultset_format, "Template", false,
|
||||
settings.schema.is_server, settings.schema.format_schema_path),
|
||||
[&](const String & partName)
|
||||
{
|
||||
return static_cast<size_t>(TemplateBlockOutputFormat::stringToResultsetPart(partName));
|
||||
@ -255,28 +257,26 @@ void registerOutputFormatProcessorTemplate(FormatFactory & factory)
|
||||
}
|
||||
|
||||
ParsedTemplateFormatString row_format = ParsedTemplateFormatString(
|
||||
FormatSchemaInfo(context, settings.template_settings.row_format, "Template", false),
|
||||
FormatSchemaInfo(settings.template_settings.row_format, "Template", false,
|
||||
settings.schema.is_server, settings.schema.format_schema_path),
|
||||
[&](const String & colName)
|
||||
{
|
||||
return sample.getPositionByName(colName);
|
||||
});
|
||||
|
||||
return std::make_shared<TemplateBlockOutputFormat>(sample, buf, settings, resultset_format, row_format);
|
||||
return std::make_shared<TemplateBlockOutputFormat>(sample, buf, settings, resultset_format, row_format, settings.template_settings.row_between_delimiter);
|
||||
});
|
||||
|
||||
factory.registerOutputFormatProcessor("CustomSeparated", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
FormatFactory::WriteCallback,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
ParsedTemplateFormatString resultset_format = ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(context);
|
||||
ParsedTemplateFormatString row_format = ParsedTemplateFormatString::setupCustomSeparatedRowFormat(context, sample);
|
||||
FormatSettings format_settings = settings;
|
||||
format_settings.template_settings.row_between_delimiter = context.getSettingsRef().format_custom_row_between_delimiter;
|
||||
ParsedTemplateFormatString resultset_format = ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(settings.custom);
|
||||
ParsedTemplateFormatString row_format = ParsedTemplateFormatString::setupCustomSeparatedRowFormat(settings.custom, sample);
|
||||
|
||||
return std::make_shared<TemplateBlockOutputFormat>(sample, buf, format_settings, resultset_format, row_format);
|
||||
return std::make_shared<TemplateBlockOutputFormat>(sample, buf, settings, resultset_format, row_format, settings.custom.row_between_delimiter);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
@ -15,7 +15,8 @@ class TemplateBlockOutputFormat : public IOutputFormat
|
||||
using ColumnFormat = ParsedTemplateFormatString::ColumnFormat;
|
||||
public:
|
||||
TemplateBlockOutputFormat(const Block & header_, WriteBuffer & out_, const FormatSettings & settings_,
|
||||
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_);
|
||||
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_,
|
||||
std::string row_between_delimiter_);
|
||||
|
||||
String getName() const override { return "TemplateBlockOutputFormat"; }
|
||||
|
||||
@ -65,6 +66,8 @@ protected:
|
||||
|
||||
size_t row_count = 0;
|
||||
bool need_write_prefix = true;
|
||||
|
||||
std::string row_between_delimiter;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -20,12 +20,13 @@ extern const int SYNTAX_ERROR;
|
||||
|
||||
|
||||
TemplateRowInputFormat::TemplateRowInputFormat(const Block & header_, ReadBuffer & in_, const Params & params_,
|
||||
const FormatSettings & settings_, bool ignore_spaces_,
|
||||
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_)
|
||||
FormatSettings settings_, bool ignore_spaces_,
|
||||
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_,
|
||||
std::string row_between_delimiter_)
|
||||
: RowInputFormatWithDiagnosticInfo(header_, buf, params_), buf(in_), data_types(header_.getDataTypes()),
|
||||
settings(settings_), ignore_spaces(ignore_spaces_),
|
||||
settings(std::move(settings_)), ignore_spaces(ignore_spaces_),
|
||||
format(std::move(format_)), row_format(std::move(row_format_)),
|
||||
default_csv_delimiter(settings.csv.delimiter)
|
||||
default_csv_delimiter(settings.csv.delimiter), row_between_delimiter(std::move(row_between_delimiter_))
|
||||
{
|
||||
/// Validate format string for result set
|
||||
bool has_data = false;
|
||||
@ -160,7 +161,7 @@ bool TemplateRowInputFormat::readRow(MutableColumns & columns, RowReadExtension
|
||||
updateDiagnosticInfo();
|
||||
|
||||
if (likely(row_num != 1))
|
||||
assertString(settings.template_settings.row_between_delimiter, buf);
|
||||
assertString(row_between_delimiter, buf);
|
||||
|
||||
extra.read_columns.assign(columns.size(), false);
|
||||
|
||||
@ -339,11 +340,11 @@ bool TemplateRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & col
|
||||
try
|
||||
{
|
||||
if (likely(row_num != 1))
|
||||
assertString(settings.template_settings.row_between_delimiter, buf);
|
||||
assertString(row_between_delimiter, buf);
|
||||
}
|
||||
catch (const DB::Exception &)
|
||||
{
|
||||
writeErrorStringForWrongDelimiter(out, "delimiter between rows", settings.template_settings.row_between_delimiter);
|
||||
writeErrorStringForWrongDelimiter(out, "delimiter between rows", row_between_delimiter);
|
||||
|
||||
return false;
|
||||
}
|
||||
@ -428,7 +429,7 @@ bool TemplateRowInputFormat::isGarbageAfterField(size_t, ReadBuffer::Position)
|
||||
|
||||
bool TemplateRowInputFormat::allowSyncAfterError() const
|
||||
{
|
||||
return !row_format.delimiters.back().empty() || !settings.template_settings.row_between_delimiter.empty();
|
||||
return !row_format.delimiters.back().empty() || !row_between_delimiter.empty();
|
||||
}
|
||||
|
||||
void TemplateRowInputFormat::syncAfterError()
|
||||
@ -450,10 +451,10 @@ void TemplateRowInputFormat::syncAfterError()
|
||||
|
||||
bool last_delimiter_in_row_found = !row_format.delimiters.back().empty();
|
||||
|
||||
if (last_delimiter_in_row_found && checkString(settings.template_settings.row_between_delimiter, buf))
|
||||
if (last_delimiter_in_row_found && checkString(row_between_delimiter, buf))
|
||||
at_beginning_of_row_or_eof = true;
|
||||
else
|
||||
skipToNextDelimiterOrEof(settings.template_settings.row_between_delimiter);
|
||||
skipToNextDelimiterOrEof(row_between_delimiter);
|
||||
|
||||
if (buf.eof())
|
||||
at_beginning_of_row_or_eof = end_of_stream = true;
|
||||
@ -509,7 +510,6 @@ void registerInputFormatProcessorTemplate(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor(ignore_spaces ? "TemplateIgnoreSpaces" : "Template", [=](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
@ -526,7 +526,8 @@ void registerInputFormatProcessorTemplate(FormatFactory & factory)
|
||||
{
|
||||
/// Read format string from file
|
||||
resultset_format = ParsedTemplateFormatString(
|
||||
FormatSchemaInfo(context, settings.template_settings.resultset_format, "Template", false),
|
||||
FormatSchemaInfo(settings.template_settings.resultset_format, "Template", false,
|
||||
settings.schema.is_server, settings.schema.format_schema_path),
|
||||
[&](const String & partName) -> std::optional<size_t>
|
||||
{
|
||||
if (partName == "data")
|
||||
@ -537,13 +538,14 @@ void registerInputFormatProcessorTemplate(FormatFactory & factory)
|
||||
}
|
||||
|
||||
ParsedTemplateFormatString row_format = ParsedTemplateFormatString(
|
||||
FormatSchemaInfo(context, settings.template_settings.row_format, "Template", false),
|
||||
FormatSchemaInfo(settings.template_settings.row_format, "Template", false,
|
||||
settings.schema.is_server, settings.schema.format_schema_path),
|
||||
[&](const String & colName) -> std::optional<size_t>
|
||||
{
|
||||
return sample.getPositionByName(colName);
|
||||
});
|
||||
|
||||
return std::make_shared<TemplateRowInputFormat>(sample, buf, params, settings, ignore_spaces, resultset_format, row_format);
|
||||
return std::make_shared<TemplateRowInputFormat>(sample, buf, params, settings, ignore_spaces, resultset_format, row_format, settings.template_settings.row_between_delimiter);
|
||||
});
|
||||
}
|
||||
|
||||
@ -552,16 +554,13 @@ void registerInputFormatProcessorTemplate(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor(ignore_spaces ? "CustomSeparatedIgnoreSpaces" : "CustomSeparated", [=](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context & context,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
ParsedTemplateFormatString resultset_format = ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(context);
|
||||
ParsedTemplateFormatString row_format = ParsedTemplateFormatString::setupCustomSeparatedRowFormat(context, sample);
|
||||
FormatSettings format_settings = settings;
|
||||
format_settings.template_settings.row_between_delimiter = context.getSettingsRef().format_custom_row_between_delimiter;
|
||||
ParsedTemplateFormatString resultset_format = ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(settings.custom);
|
||||
ParsedTemplateFormatString row_format = ParsedTemplateFormatString::setupCustomSeparatedRowFormat(settings.custom, sample);
|
||||
|
||||
return std::make_shared<TemplateRowInputFormat>(sample, buf, params, format_settings, ignore_spaces, resultset_format, row_format);
|
||||
return std::make_shared<TemplateRowInputFormat>(sample, buf, params, settings, ignore_spaces, resultset_format, row_format, settings.custom.row_between_delimiter);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
@ -16,8 +16,9 @@ class TemplateRowInputFormat : public RowInputFormatWithDiagnosticInfo
|
||||
using ColumnFormat = ParsedTemplateFormatString::ColumnFormat;
|
||||
public:
|
||||
TemplateRowInputFormat(const Block & header_, ReadBuffer & in_, const Params & params_,
|
||||
const FormatSettings & settings_, bool ignore_spaces_,
|
||||
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_);
|
||||
FormatSettings settings_, bool ignore_spaces_,
|
||||
ParsedTemplateFormatString format_, ParsedTemplateFormatString row_format_,
|
||||
std::string row_between_delimiter);
|
||||
|
||||
String getName() const override { return "TemplateRowInputFormat"; }
|
||||
|
||||
@ -61,6 +62,8 @@ private:
|
||||
bool end_of_stream = false;
|
||||
std::vector<size_t> always_default_columns;
|
||||
char default_csv_delimiter;
|
||||
|
||||
std::string row_between_delimiter;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -33,8 +33,8 @@ namespace ErrorCodes
|
||||
|
||||
|
||||
ValuesBlockInputFormat::ValuesBlockInputFormat(ReadBuffer & in_, const Block & header_, const RowInputFormatParams & params_,
|
||||
const Context & context_, const FormatSettings & format_settings_)
|
||||
: IInputFormat(header_, buf), buf(in_), params(params_), context(std::make_unique<Context>(context_)),
|
||||
const FormatSettings & format_settings_)
|
||||
: IInputFormat(header_, buf), buf(in_), params(params_),
|
||||
format_settings(format_settings_), num_columns(header_.columns()),
|
||||
parser_type_for_column(num_columns, ParserType::Streaming),
|
||||
attempts_to_deduce_template(num_columns), attempts_to_deduce_template_cached(num_columns),
|
||||
@ -424,11 +424,10 @@ void registerInputFormatProcessorValues(FormatFactory & factory)
|
||||
factory.registerInputFormatProcessor("Values", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & header,
|
||||
const Context & context,
|
||||
const RowInputFormatParams & params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
return std::make_shared<ValuesBlockInputFormat>(buf, header, params, context, settings);
|
||||
return std::make_shared<ValuesBlockInputFormat>(buf, header, params, settings);
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -18,7 +18,7 @@ class ReadBuffer;
|
||||
|
||||
/** Stream to read data in VALUES format (as in INSERT query).
|
||||
*/
|
||||
class ValuesBlockInputFormat : public IInputFormat
|
||||
class ValuesBlockInputFormat final : public IInputFormat
|
||||
{
|
||||
public:
|
||||
/** Data is parsed using fast, streaming parser.
|
||||
@ -29,12 +29,15 @@ public:
|
||||
* than interpreting expressions in each row separately, but it's still slower than streaming parsing)
|
||||
*/
|
||||
ValuesBlockInputFormat(ReadBuffer & in_, const Block & header_, const RowInputFormatParams & params_,
|
||||
const Context & context_, const FormatSettings & format_settings_);
|
||||
const FormatSettings & format_settings_);
|
||||
|
||||
String getName() const override { return "ValuesBlockInputFormat"; }
|
||||
|
||||
void resetParser() override;
|
||||
|
||||
/// TODO: remove context somehow.
|
||||
void setContext(const Context & context_) { context = std::make_unique<Context>(context_); }
|
||||
|
||||
const BlockMissingValues & getMissingValues() const override { return block_missing_values; }
|
||||
|
||||
private:
|
||||
|
@ -46,7 +46,6 @@ void registerOutputFormatProcessorValues(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("Values", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
|
@ -168,7 +168,6 @@ void registerOutputFormatProcessorVertical(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("Vertical", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
|
@ -245,7 +245,6 @@ void registerOutputFormatProcessorXML(FormatFactory & factory)
|
||||
factory.registerOutputFormatProcessor("XML", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
const Context &,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
|
@ -109,7 +109,7 @@ std::optional<AlterCommand> AlterCommand::parse(const ASTAlterCommand * command_
|
||||
if (ast_col_decl.comment)
|
||||
{
|
||||
const auto & ast_comment = ast_col_decl.comment->as<ASTLiteral &>();
|
||||
command.comment = ast_comment.value.get<String>();
|
||||
command.comment.emplace(ast_comment.value.get<String>());
|
||||
}
|
||||
|
||||
if (ast_col_decl.ttl)
|
||||
@ -225,7 +225,9 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri
|
||||
column.default_desc.kind = default_kind;
|
||||
column.default_desc.expression = default_expression;
|
||||
}
|
||||
column.comment = comment;
|
||||
if (comment)
|
||||
column.comment = *comment;
|
||||
|
||||
column.codec = codec;
|
||||
column.ttl = ttl;
|
||||
|
||||
@ -251,19 +253,22 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri
|
||||
column.codec = codec;
|
||||
}
|
||||
|
||||
if (!isMutable())
|
||||
{
|
||||
column.comment = comment;
|
||||
return;
|
||||
}
|
||||
if (comment)
|
||||
column.comment = *comment;
|
||||
|
||||
if (ttl)
|
||||
column.ttl = ttl;
|
||||
|
||||
column.type = data_type;
|
||||
if (data_type)
|
||||
column.type = data_type;
|
||||
|
||||
column.default_desc.kind = default_kind;
|
||||
column.default_desc.expression = default_expression;
|
||||
/// User specified default expression or changed
|
||||
/// datatype. We have to replace default.
|
||||
if (default_expression || data_type)
|
||||
{
|
||||
column.default_desc.kind = default_kind;
|
||||
column.default_desc.expression = default_expression;
|
||||
}
|
||||
});
|
||||
}
|
||||
else if (type == MODIFY_ORDER_BY)
|
||||
@ -279,7 +284,7 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri
|
||||
}
|
||||
else if (type == COMMENT_COLUMN)
|
||||
{
|
||||
columns_description.modify(column_name, [&](ColumnDescription & column) { column.comment = comment; });
|
||||
columns_description.modify(column_name, [&](ColumnDescription & column) { column.comment = *comment; });
|
||||
}
|
||||
else if (type == ADD_INDEX)
|
||||
{
|
||||
@ -390,13 +395,15 @@ void AlterCommand::apply(ColumnsDescription & columns_description, IndicesDescri
|
||||
throw Exception("Wrong parameter type in ALTER query", ErrorCodes::LOGICAL_ERROR);
|
||||
}
|
||||
|
||||
bool AlterCommand::isMutable() const
|
||||
bool AlterCommand::isModifyingData() const
|
||||
{
|
||||
if (type == COMMENT_COLUMN || type == MODIFY_SETTING)
|
||||
return false;
|
||||
/// Possible change data representation on disk
|
||||
if (type == MODIFY_COLUMN)
|
||||
return data_type.get() || default_expression;
|
||||
return true;
|
||||
return data_type != nullptr;
|
||||
|
||||
return type == ADD_COLUMN /// We need to change columns.txt in each part for MergeTree
|
||||
|| type == DROP_COLUMN /// We need to change columns.txt in each part for MergeTree
|
||||
|| type == DROP_INDEX; /// We need to remove file from filesystem for MergeTree
|
||||
}
|
||||
|
||||
bool AlterCommand::isSettingsAlter() const
|
||||
@ -666,11 +673,11 @@ void AlterCommands::applyForSettingsOnly(SettingsChanges & changes) const
|
||||
changes = std::move(out_changes);
|
||||
}
|
||||
|
||||
bool AlterCommands::isMutable() const
|
||||
bool AlterCommands::isModifyingData() const
|
||||
{
|
||||
for (const auto & param : *this)
|
||||
{
|
||||
if (param.isMutable())
|
||||
if (param.isModifyingData())
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -47,7 +47,7 @@ struct AlterCommand
|
||||
|
||||
ColumnDefaultKind default_kind{};
|
||||
ASTPtr default_expression{};
|
||||
String comment;
|
||||
std::optional<String> comment;
|
||||
|
||||
/// For ADD - after which column to add a new one. If an empty string, add to the end. To add to the beginning now it is impossible.
|
||||
String after_column;
|
||||
@ -102,8 +102,11 @@ struct AlterCommand
|
||||
ConstraintsDescription & constraints_description, ASTPtr & order_by_ast,
|
||||
ASTPtr & primary_key_ast, ASTPtr & ttl_table_ast, SettingsChanges & changes) const;
|
||||
|
||||
/// Checks that not only metadata touched by that command
|
||||
bool isMutable() const;
|
||||
/// Checks that alter query changes data. For MergeTree:
|
||||
/// * column files (data and marks)
|
||||
/// * each part meta (columns.txt)
|
||||
/// in each part on disk (it's not lightweight alter).
|
||||
bool isModifyingData() const;
|
||||
|
||||
/// checks that only settings changed by alter
|
||||
bool isSettingsAlter() const;
|
||||
@ -124,7 +127,7 @@ public:
|
||||
void applyForSettingsOnly(SettingsChanges & changes) const;
|
||||
|
||||
void validate(const IStorage & table, const Context & context);
|
||||
bool isMutable() const;
|
||||
bool isModifyingData() const;
|
||||
bool isSettingsAlter() const;
|
||||
};
|
||||
|
||||
|
@ -402,7 +402,7 @@ void IStorage::alter(
|
||||
const Context & context,
|
||||
TableStructureWriteLockHolder & table_lock_holder)
|
||||
{
|
||||
if (params.isMutable())
|
||||
if (params.isModifyingData())
|
||||
throw Exception("Method alter supports only change comment of column for storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
|
||||
|
||||
const String database_name = getDatabaseName();
|
||||
|
@ -1342,11 +1342,11 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
|
||||
"before using data skipping indices.", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
/// Set of columns that shouldn't be altered.
|
||||
NameSet columns_alter_forbidden;
|
||||
NameSet columns_alter_type_forbidden;
|
||||
|
||||
/// Primary key columns can be ALTERed only if they are used in the key as-is
|
||||
/// (and not as a part of some expression) and if the ALTER only affects column metadata.
|
||||
NameSet columns_alter_metadata_only;
|
||||
NameSet columns_alter_type_metadata_only;
|
||||
|
||||
if (partition_key_expr)
|
||||
{
|
||||
@ -1354,13 +1354,13 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
|
||||
/// TODO: in some cases (e.g. adding an Enum value) a partition key column can still be ALTERed.
|
||||
/// We should allow it.
|
||||
for (const String & col : partition_key_expr->getRequiredColumns())
|
||||
columns_alter_forbidden.insert(col);
|
||||
columns_alter_type_forbidden.insert(col);
|
||||
}
|
||||
|
||||
for (const auto & index : skip_indices)
|
||||
{
|
||||
for (const String & col : index->expr->getRequiredColumns())
|
||||
columns_alter_forbidden.insert(col);
|
||||
columns_alter_type_forbidden.insert(col);
|
||||
}
|
||||
|
||||
if (sorting_key_expr)
|
||||
@ -1368,17 +1368,16 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
|
||||
for (const ExpressionAction & action : sorting_key_expr->getActions())
|
||||
{
|
||||
auto action_columns = action.getNeededColumns();
|
||||
columns_alter_forbidden.insert(action_columns.begin(), action_columns.end());
|
||||
columns_alter_type_forbidden.insert(action_columns.begin(), action_columns.end());
|
||||
}
|
||||
for (const String & col : sorting_key_expr->getRequiredColumns())
|
||||
columns_alter_metadata_only.insert(col);
|
||||
columns_alter_type_metadata_only.insert(col);
|
||||
|
||||
/// We don't process sample_by_ast separately because it must be among the primary key columns
|
||||
/// and we don't process primary_key_expr separately because it is a prefix of sorting_key_expr.
|
||||
}
|
||||
|
||||
if (!merging_params.sign_column.empty())
|
||||
columns_alter_forbidden.insert(merging_params.sign_column);
|
||||
columns_alter_type_forbidden.insert(merging_params.sign_column);
|
||||
|
||||
std::map<String, const IDataType *> old_types;
|
||||
for (const auto & column : getColumns().getAllPhysical())
|
||||
@ -1386,34 +1385,26 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
|
||||
|
||||
for (const AlterCommand & command : commands)
|
||||
{
|
||||
if (!command.isMutable())
|
||||
if (command.type == AlterCommand::MODIFY_ORDER_BY && !is_custom_partitioned)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (columns_alter_forbidden.count(command.column_name))
|
||||
throw Exception("Trying to ALTER key column " + command.column_name, ErrorCodes::ILLEGAL_COLUMN);
|
||||
|
||||
if (columns_alter_metadata_only.count(command.column_name))
|
||||
{
|
||||
if (command.type == AlterCommand::MODIFY_COLUMN)
|
||||
{
|
||||
auto it = old_types.find(command.column_name);
|
||||
if (it != old_types.end() && isMetadataOnlyConversion(it->second, command.data_type.get()))
|
||||
continue;
|
||||
}
|
||||
|
||||
throw Exception(
|
||||
"ALTER of key column " + command.column_name + " must be metadata-only",
|
||||
ErrorCodes::ILLEGAL_COLUMN);
|
||||
"ALTER MODIFY ORDER BY is not supported for default-partitioned tables created with the old syntax",
|
||||
ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
||||
if (command.type == AlterCommand::MODIFY_ORDER_BY)
|
||||
else if (command.isModifyingData())
|
||||
{
|
||||
if (!is_custom_partitioned)
|
||||
throw Exception(
|
||||
"ALTER MODIFY ORDER BY is not supported for default-partitioned tables created with the old syntax",
|
||||
ErrorCodes::BAD_ARGUMENTS);
|
||||
if (columns_alter_type_forbidden.count(command.column_name))
|
||||
throw Exception("Trying to ALTER key column " + command.column_name, ErrorCodes::ILLEGAL_COLUMN);
|
||||
|
||||
if (columns_alter_type_metadata_only.count(command.column_name))
|
||||
{
|
||||
if (command.type == AlterCommand::MODIFY_COLUMN)
|
||||
{
|
||||
auto it = old_types.find(command.column_name);
|
||||
if (it == old_types.end() || !isMetadataOnlyConversion(it->second, command.data_type.get()))
|
||||
throw Exception("ALTER of key column " + command.column_name + " must be metadata-only", ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1425,17 +1416,20 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c
|
||||
for (const auto & setting : new_changes)
|
||||
checkSettingCanBeChanged(setting.name);
|
||||
|
||||
/// Check that type conversions are possible.
|
||||
ExpressionActionsPtr unused_expression;
|
||||
NameToNameMap unused_map;
|
||||
bool unused_bool;
|
||||
createConvertExpression(nullptr, getColumns().getAllPhysical(), new_columns.getAllPhysical(),
|
||||
getIndices().indices, new_indices.indices, unused_expression, unused_map, unused_bool);
|
||||
if (commands.isModifyingData())
|
||||
{
|
||||
/// Check that type conversions are possible.
|
||||
ExpressionActionsPtr unused_expression;
|
||||
NameToNameMap unused_map;
|
||||
bool unused_bool;
|
||||
createConvertExpression(nullptr, getColumns().getAllPhysical(), new_columns.getAllPhysical(),
|
||||
getIndices().indices, new_indices.indices, unused_expression, unused_map, unused_bool);
|
||||
}
|
||||
}
|
||||
|
||||
void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns,
|
||||
const IndicesASTs & old_indices, const IndicesASTs & new_indices, ExpressionActionsPtr & out_expression,
|
||||
NameToNameMap & out_rename_map, bool & out_force_update_metadata) const
|
||||
void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns,
|
||||
const NamesAndTypesList & new_columns, const IndicesASTs & old_indices, const IndicesASTs & new_indices,
|
||||
ExpressionActionsPtr & out_expression, NameToNameMap & out_rename_map, bool & out_force_update_metadata) const
|
||||
{
|
||||
const auto settings = getSettings();
|
||||
out_expression = nullptr;
|
||||
@ -1457,7 +1451,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name
|
||||
|
||||
|
||||
/// Remove old indices
|
||||
std::set<String> new_indices_set;
|
||||
std::unordered_set<String> new_indices_set;
|
||||
for (const auto & index_decl : new_indices)
|
||||
new_indices_set.emplace(index_decl->as<ASTIndexDeclaration &>().name);
|
||||
for (const auto & index_decl : old_indices)
|
||||
@ -1465,8 +1459,8 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name
|
||||
const auto & index = index_decl->as<ASTIndexDeclaration &>();
|
||||
if (!new_indices_set.count(index.name))
|
||||
{
|
||||
out_rename_map["skp_idx_" + index.name + ".idx"] = "";
|
||||
out_rename_map["skp_idx_" + index.name + part_mrk_file_extension] = "";
|
||||
out_rename_map["skp_idx_" + index.name + ".idx"] = ""; /// drop this file
|
||||
out_rename_map["skp_idx_" + index.name + part_mrk_file_extension] = ""; /// and this one
|
||||
}
|
||||
}
|
||||
|
||||
@ -1494,8 +1488,8 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name
|
||||
/// Delete files if they are no longer shared with another column.
|
||||
if (--stream_counts[file_name] == 0)
|
||||
{
|
||||
out_rename_map[file_name + ".bin"] = "";
|
||||
out_rename_map[file_name + part_mrk_file_extension] = "";
|
||||
out_rename_map[file_name + ".bin"] = ""; /// drop this file
|
||||
out_rename_map[file_name + part_mrk_file_extension] = ""; /// and this one
|
||||
}
|
||||
}, {});
|
||||
}
|
||||
@ -1847,7 +1841,7 @@ void MergeTreeData::AlterDataPartTransaction::commit()
|
||||
mutable_part.checksums = new_checksums;
|
||||
mutable_part.columns = new_columns;
|
||||
|
||||
/// 3) Delete the old files.
|
||||
/// 3) Delete the old files and drop required columns (DROP COLUMN)
|
||||
for (const auto & from_to : rename_map)
|
||||
{
|
||||
String name = from_to.second.empty() ? from_to.first : from_to.second;
|
||||
|
@ -234,7 +234,7 @@ public:
|
||||
const NamesAndTypesList & getNewColumns() const { return new_columns; }
|
||||
const DataPart::Checksums & getNewChecksums() const { return new_checksums; }
|
||||
|
||||
AlterDataPartTransaction(DataPartPtr data_part_) : data_part(data_part_), alter_lock(data_part->alter_mutex) {}
|
||||
AlterDataPartTransaction(DataPartPtr data_part_) : data_part(data_part_) {}
|
||||
const DataPartPtr & getDataPart() const { return data_part; }
|
||||
bool isValid() const;
|
||||
|
||||
@ -244,9 +244,7 @@ public:
|
||||
|
||||
bool valid = true;
|
||||
|
||||
//don't interchange order of data_part & alter_lock
|
||||
DataPartPtr data_part;
|
||||
DataPartsLock alter_lock;
|
||||
|
||||
DataPart::Checksums new_checksums;
|
||||
NamesAndTypesList new_columns;
|
||||
|
@ -227,16 +227,6 @@ struct MergeTreeDataPart
|
||||
*/
|
||||
mutable std::shared_mutex columns_lock;
|
||||
|
||||
/** It is taken for the whole time ALTER a part: from the beginning of the recording of the temporary files to their renaming to permanent.
|
||||
* It is taken with unlocked `columns_lock`.
|
||||
*
|
||||
* NOTE: "You can" do without this mutex if you could turn ReadRWLock into WriteRWLock without removing the lock.
|
||||
* This transformation is impossible, because it would create a deadlock, if you do it from two threads at once.
|
||||
* Taking this mutex means that we want to lock columns_lock on read with intention then, not
|
||||
* unblocking, block it for writing.
|
||||
*/
|
||||
mutable std::mutex alter_mutex;
|
||||
|
||||
MergeTreeIndexGranularityInfo index_granularity_info;
|
||||
|
||||
~MergeTreeDataPart();
|
||||
|
@ -249,34 +249,10 @@ void StorageMergeTree::alter(
|
||||
const String current_database_name = getDatabaseName();
|
||||
const String current_table_name = getTableName();
|
||||
|
||||
if (!params.isMutable())
|
||||
{
|
||||
lockStructureExclusively(table_lock_holder, context.getCurrentQueryId());
|
||||
auto new_columns = getColumns();
|
||||
auto new_indices = getIndices();
|
||||
auto new_constraints = getConstraints();
|
||||
ASTPtr new_order_by_ast = order_by_ast;
|
||||
ASTPtr new_primary_key_ast = primary_key_ast;
|
||||
ASTPtr new_ttl_table_ast = ttl_table_ast;
|
||||
SettingsChanges new_changes;
|
||||
|
||||
params.apply(new_columns, new_indices, new_constraints, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes);
|
||||
|
||||
changeSettings(new_changes, table_lock_holder);
|
||||
|
||||
IDatabase::ASTModifier settings_modifier = getSettingsModifier(new_changes);
|
||||
context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, settings_modifier);
|
||||
setColumns(std::move(new_columns));
|
||||
return;
|
||||
}
|
||||
|
||||
/// NOTE: Here, as in ReplicatedMergeTree, you can do ALTER which does not block the writing of data for a long time.
|
||||
/// Also block moves, because they can replace part with old state
|
||||
auto merge_blocker = merger_mutator.merges_blocker.cancel();
|
||||
auto moves_blocked = parts_mover.moves_blocker.cancel();
|
||||
|
||||
lockNewDataStructureExclusively(table_lock_holder, context.getCurrentQueryId());
|
||||
|
||||
checkAlter(params, context);
|
||||
|
||||
auto new_columns = getColumns();
|
||||
auto new_indices = getIndices();
|
||||
auto new_constraints = getConstraints();
|
||||
@ -284,13 +260,11 @@ void StorageMergeTree::alter(
|
||||
ASTPtr new_primary_key_ast = primary_key_ast;
|
||||
ASTPtr new_ttl_table_ast = ttl_table_ast;
|
||||
SettingsChanges new_changes;
|
||||
|
||||
params.apply(new_columns, new_indices, new_constraints, new_order_by_ast, new_primary_key_ast, new_ttl_table_ast, new_changes);
|
||||
|
||||
auto transactions = prepareAlterTransactions(new_columns, new_indices, context);
|
||||
|
||||
lockStructureExclusively(table_lock_holder, context.getCurrentQueryId());
|
||||
|
||||
IDatabase::ASTModifier storage_modifier = [&] (IAST & ast)
|
||||
/// Modifier for storage AST in /metadata/storage_db/storage.sql
|
||||
IDatabase::ASTModifier storage_modifier = [&](IAST & ast)
|
||||
{
|
||||
auto & storage_ast = ast.as<ASTStorage &>();
|
||||
|
||||
@ -310,24 +284,51 @@ void StorageMergeTree::alter(
|
||||
}
|
||||
};
|
||||
|
||||
changeSettings(new_changes, table_lock_holder);
|
||||
|
||||
context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, storage_modifier);
|
||||
|
||||
|
||||
/// Reinitialize primary key because primary key column types might have changed.
|
||||
setProperties(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints);
|
||||
|
||||
setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast);
|
||||
|
||||
for (auto & transaction : transactions)
|
||||
/// Update metdata in memory
|
||||
auto update_metadata = [&]()
|
||||
{
|
||||
transaction->commit();
|
||||
transaction.reset();
|
||||
}
|
||||
changeSettings(new_changes, table_lock_holder);
|
||||
/// Reinitialize primary key because primary key column types might have changed.
|
||||
setProperties(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints);
|
||||
|
||||
/// Columns sizes could be changed
|
||||
recalculateColumnSizes();
|
||||
setTTLExpressions(new_columns.getColumnTTLs(), new_ttl_table_ast);
|
||||
};
|
||||
|
||||
/// This alter can be performed at metadata level only
|
||||
if (!params.isModifyingData())
|
||||
{
|
||||
lockStructureExclusively(table_lock_holder, context.getCurrentQueryId());
|
||||
|
||||
context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, storage_modifier);
|
||||
|
||||
update_metadata();
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
/// NOTE: Here, as in ReplicatedMergeTree, you can do ALTER which does not block the writing of data for a long time.
|
||||
/// Also block moves, because they can replace part with old state
|
||||
auto merge_blocker = merger_mutator.merges_blocker.cancel();
|
||||
auto moves_blocked = parts_mover.moves_blocker.cancel();
|
||||
|
||||
|
||||
auto transactions = prepareAlterTransactions(new_columns, new_indices, context);
|
||||
|
||||
lockStructureExclusively(table_lock_holder, context.getCurrentQueryId());
|
||||
|
||||
context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, storage_modifier);
|
||||
|
||||
update_metadata();
|
||||
|
||||
for (auto & transaction : transactions)
|
||||
{
|
||||
transaction->commit();
|
||||
transaction.reset();
|
||||
}
|
||||
|
||||
/// Columns sizes could be changed
|
||||
recalculateColumnSizes();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -453,7 +454,6 @@ void StorageMergeTree::mutate(const MutationCommands & commands, const Context &
|
||||
auto check = [version, this]() { return isMutationDone(version); };
|
||||
std::unique_lock lock(mutation_wait_mutex);
|
||||
mutation_wait_event.wait(lock, check);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3231,6 +3231,10 @@ void StorageReplicatedMergeTree::alter(
|
||||
const String current_database_name = getDatabaseName();
|
||||
const String current_table_name = getTableName();
|
||||
|
||||
/// We cannot check this alter commands with method isModifyingData()
|
||||
/// because ReplicatedMergeTree stores both columns and metadata for
|
||||
/// each replica. So we have to wait AlterThread even with lightweight
|
||||
/// metadata alter.
|
||||
if (params.isSettingsAlter())
|
||||
{
|
||||
/// We don't replicate storage_settings_ptr ALTER. It's local operation.
|
||||
|
@ -21,6 +21,14 @@ mv ./dbms/unit_tests_dbms /output
|
||||
find . -name '*.so' -print -exec mv '{}' /output \;
|
||||
find . -name '*.so.*' -print -exec mv '{}' /output \;
|
||||
|
||||
# Different files for performance test.
|
||||
if [ "performance" == "$COMBINED_OUTPUT" ]
|
||||
then
|
||||
cp -r ../dbms/tests/performance /output
|
||||
rm /output/unit_tests_dbms ||:
|
||||
rm /output/clickhouse-odbc-bridge ||:
|
||||
fi
|
||||
|
||||
# May be set for split build or for performance test.
|
||||
if [ "" != "$COMBINED_OUTPUT" ]
|
||||
then
|
||||
|
146
docs/en/operations/utils/clickhouse-benchmark.md
Normal file
146
docs/en/operations/utils/clickhouse-benchmark.md
Normal file
@ -0,0 +1,146 @@
|
||||
# clickhouse-benchmark
|
||||
|
||||
Connects to a ClickHouse server and repeatedly sends specified queries.
|
||||
|
||||
Syntax:
|
||||
|
||||
```bash
|
||||
$ echo "single query" | clickhouse-benchmark [keys]
|
||||
```
|
||||
or
|
||||
```bash
|
||||
$ clickhouse-benchmark [keys] <<< "single query"
|
||||
```
|
||||
|
||||
If you want to send a set of queries, create a text file and place each query on the individual string in this file. For example:
|
||||
|
||||
```sql
|
||||
SELECT * FROM system.numbers LIMIT 10000000
|
||||
SELECT 1
|
||||
```
|
||||
|
||||
Then pass this file to a standard input of `clickhouse-benchmark`.
|
||||
|
||||
```bash
|
||||
clickhouse-benchmark [keys] < queries_file
|
||||
```
|
||||
|
||||
## Keys {#clickhouse-benchmark-keys}
|
||||
|
||||
- `-c N`, `--concurrency=N` — Number of queries that `clickhouse-benchmark` sends simultaneously. Default value: 1.
|
||||
- `-d N`, `--delay=N` — Interval in seconds between intermediate reports (set 0 to disable reports). Default value: 1.
|
||||
- `-h WORD`, `--host=WORD` — Server host. Default value: `localhost`. For the [comparison mode](#clickhouse-benchmark-comparison-mode) you can use multiple `-h` keys.
|
||||
- `-p N`, `--port=N` — Server port. Default value: 9000. For the [comparison mode](#clickhouse-benchmark-comparison-mode) you can use multiple `-p` keys.
|
||||
- `-i N`, `--iterations=N` — Total number of queries. Default value: 0.
|
||||
- `-r`, `--randomize` — Random order of queries execution if there is more then one input query.
|
||||
- `-s`, `--secure` — Using TLS connection.
|
||||
- `-t N`, `--timelimit=N` — Time limit in seconds. `clickhouse-benchmark` stops sending queries when the specified time limit is reached. Default value: 0 (time limit disabled).
|
||||
- `--confidence=N` — Level of confidence for T-test. Possible values: 0 (80%), 1 (90%), 2 (95%), 3 (98%), 4 (99%), 5 (99.5%). Default value: 5. In the [comparison mode](#clickhouse-benchmark-comparison-mode) `clickhouse-benchmark` performs the [Independent two-sample Student's t-test](https://en.wikipedia.org/wiki/Student%27s_t-test#Independent_two-sample_t-test) test to determine whether the two distributions aren't different with the selected level of confidence.
|
||||
- `--cumulative` — Printing cumulative data instead of data per interval.
|
||||
- `--database=DATABASE_NAME` — ClickHouse database name. Default value: `default`.
|
||||
- `--json=FILEPATH` — JSON output. When the key is set, `clickhouse-benchmark` outputs a report to the specified JSON-file.
|
||||
- `--user=USERNAME` — ClickHouse user name. Default value: `default`.
|
||||
- `--password=PSWD` — ClickHouse user password. Default value: empty string.
|
||||
- `--stacktrace` — Stack traces output. When the key is set, `clickhouse-bencmark` outputs stack traces of exceptions.
|
||||
- `--stage=WORD` — Query processing stage at server. ClickHouse stops query processing and returns answer to `clickhouse-benchmark` at the specified stage. Possible values: `complete`, `fetch_columns`, `with_mergeable_state`. Default value: `complete`.
|
||||
- `--help` — Shows the help message.
|
||||
|
||||
If you want to apply some [settings](../../operations/settings/index.md) for queries, pass them as a key `--<session setting name>= SETTING_VALUE`. For example, `--max_memory_usage=1048576`.
|
||||
|
||||
## Output {#clickhouse-benchmark-output}
|
||||
|
||||
By default, `clickhouse-benchmark` reports for each `--delay` interval.
|
||||
|
||||
Example of the report:
|
||||
|
||||
```text
|
||||
Queries executed: 10.
|
||||
|
||||
localhost:9000, queries 10, QPS: 6.772, RPS: 67904487.440, MiB/s: 518.070, result RPS: 67721584.984, result MiB/s: 516.675.
|
||||
|
||||
0.000% 0.145 sec.
|
||||
10.000% 0.146 sec.
|
||||
20.000% 0.146 sec.
|
||||
30.000% 0.146 sec.
|
||||
40.000% 0.147 sec.
|
||||
50.000% 0.148 sec.
|
||||
60.000% 0.148 sec.
|
||||
70.000% 0.148 sec.
|
||||
80.000% 0.149 sec.
|
||||
90.000% 0.150 sec.
|
||||
95.000% 0.150 sec.
|
||||
99.000% 0.150 sec.
|
||||
99.900% 0.150 sec.
|
||||
99.990% 0.150 sec.
|
||||
```
|
||||
|
||||
In the report you can find:
|
||||
|
||||
- Number of queries in the `Queries executed: ` field.
|
||||
- Status string containing (in order):
|
||||
|
||||
- Endpoint of ClickHouse server.
|
||||
- Number of processed queries.
|
||||
- QPS: QPS: How many queries server performed per second during a period specified in the `--delay` argument.
|
||||
- RPS: How many rows server read per second during a period specified in the `--delay` argument.
|
||||
- MiB/s: How many mebibytes server read per second during a period specified in the `--delay` argument.
|
||||
- result RPS: How many rows placed by server to the result of a query per second during a period specified in the `--delay` argument.
|
||||
- result MiB/s. How many mebibytes placed by server to the result of a query per second during a period specified in the `--delay` argument.
|
||||
|
||||
- Percentiles of queries execution time.
|
||||
|
||||
|
||||
## Comparison mode {#clickhouse-benchmark-comparison-mode}
|
||||
|
||||
`clickhouse-benchmark` can compare performances for two running ClickHouse servers.
|
||||
|
||||
To use the comparison mode, specify endpoints of both servers by two pairs of `--host`, `--port` keys. Keys matched together by position in arguments list, the first `--host` is matched with the first `--port` and so on. `clickhouse-benchmark` establishes connections to both servers, then sends queries. Each query addressed to a randomly selected server. The results are shown for each server separately.
|
||||
|
||||
## Example {#clickhouse-benchmark-example}
|
||||
|
||||
```bash
|
||||
$ echo "SELECT * FROM system.numbers LIMIT 10000000 OFFSET 10000000" | clickhouse-benchmark -i 10
|
||||
```
|
||||
```text
|
||||
Loaded 1 queries.
|
||||
|
||||
Queries executed: 6.
|
||||
|
||||
localhost:9000, queries 6, QPS: 6.153, RPS: 123398340.957, MiB/s: 941.455, result RPS: 61532982.200, result MiB/s: 469.459.
|
||||
|
||||
0.000% 0.159 sec.
|
||||
10.000% 0.159 sec.
|
||||
20.000% 0.159 sec.
|
||||
30.000% 0.160 sec.
|
||||
40.000% 0.160 sec.
|
||||
50.000% 0.162 sec.
|
||||
60.000% 0.164 sec.
|
||||
70.000% 0.165 sec.
|
||||
80.000% 0.166 sec.
|
||||
90.000% 0.166 sec.
|
||||
95.000% 0.167 sec.
|
||||
99.000% 0.167 sec.
|
||||
99.900% 0.167 sec.
|
||||
99.990% 0.167 sec.
|
||||
|
||||
|
||||
|
||||
Queries executed: 10.
|
||||
|
||||
localhost:9000, queries 10, QPS: 6.082, RPS: 121959604.568, MiB/s: 930.478, result RPS: 60815551.642, result MiB/s: 463.986.
|
||||
|
||||
0.000% 0.159 sec.
|
||||
10.000% 0.159 sec.
|
||||
20.000% 0.160 sec.
|
||||
30.000% 0.163 sec.
|
||||
40.000% 0.164 sec.
|
||||
50.000% 0.165 sec.
|
||||
60.000% 0.166 sec.
|
||||
70.000% 0.166 sec.
|
||||
80.000% 0.167 sec.
|
||||
90.000% 0.167 sec.
|
||||
95.000% 0.170 sec.
|
||||
99.000% 0.172 sec.
|
||||
99.900% 0.172 sec.
|
||||
99.990% 0.172 sec.
|
||||
```
|
@ -1,7 +1,7 @@
|
||||
# ClickHouse Utility
|
||||
|
||||
* [clickhouse-local](clickhouse-local.md) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` does this.
|
||||
* [clickhouse-copier](clickhouse-copier.md) — Copies (and reshards) data from one cluster to another cluster.
|
||||
|
||||
- [clickhouse-local](clickhouse-local.md) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` does this.
|
||||
- [clickhouse-copier](clickhouse-copier.md) — Copies (and reshards) data from one cluster to another cluster.
|
||||
- [clickhouse-benchmark](clickhouse-benchmark.md) — Loads server with the custom queries and settings.
|
||||
|
||||
[Original article](https://clickhouse.yandex/docs/en/operations/utils/) <!--hide-->
|
||||
|
@ -114,18 +114,18 @@ If a query does not list any columns (for example, `SELECT count() FROM t`), som
|
||||
|
||||
#### FINAL Modifier {#select-from-final}
|
||||
|
||||
Appliable when selecting data from tables of the [MergeTree](../operations/table_engines/mergetree.md)-engine family, except `GraphiteMergeTree`. When `FINAL` is specified, ClickHouse fully merges data before returning the result and thus performs all data transformations that are supposed to happen during merges for given table engine.
|
||||
Applicable when selecting data from tables from the [MergeTree](../operations/table_engines/mergetree.md)-engine family other than `GraphiteMergeTree`. When `FINAL` is specified, ClickHouse fully merges the data before returning the result and thus performs all data transformations that happen during merges for the given table engine.
|
||||
|
||||
Also supported for:
|
||||
- [Replicated](../operations/table_engines/replication.md) versions of `MergeTree` engines.
|
||||
- [View](../operations/table_engines/view.md), [Buffer](../operations/table_engines/buffer.md), [Distributed](../operations/table_engines/distributed.md), [MaterializedView](../operations/table_engines/materializedview.md) engines that operate over other engines, if they created over `MergeTree`-engine tables.
|
||||
- [View](../operations/table_engines/view.md), [Buffer](../operations/table_engines/buffer.md), [Distributed](../operations/table_engines/distributed.md), and [MaterializedView](../operations/table_engines/materializedview.md) engines that operate over other engines, provided they were created over `MergeTree`-engine tables.
|
||||
|
||||
The queries that use `FINAL` are executed slower than similar queries that don't, because:
|
||||
Queries that use `FINAL` are executed not as fast as similar queries that don't, because:
|
||||
|
||||
- Query is executed in a single thread, and data is merged during query execution.
|
||||
- Queries with `FINAL` read primary key columns additionally to the columns specified in the query.
|
||||
- Query is executed in a single thread and data is merged during query execution.
|
||||
- Queries with `FINAL` read primary key columns in addition to the columns specified in the query.
|
||||
|
||||
In the most cases, avoid using `FINAL`.
|
||||
In most cases, avoid using `FINAL`.
|
||||
|
||||
### SAMPLE Clause {#select-sample-clause}
|
||||
|
||||
|
1
docs/fa/operations/utils/clickhouse-benchmark.md
Symbolic link
1
docs/fa/operations/utils/clickhouse-benchmark.md
Symbolic link
@ -0,0 +1 @@
|
||||
../../../en/operations/utils/clickhouse-benchmark.md
|
1
docs/ja/operations/utils/clickhouse-benchmark.md
Symbolic link
1
docs/ja/operations/utils/clickhouse-benchmark.md
Symbolic link
@ -0,0 +1 @@
|
||||
../../../en/operations/utils/clickhouse-benchmark.md
|
@ -114,6 +114,23 @@ Cекция `FROM` определяет источник данных:
|
||||
Модификатор `FINAL` может быть использован в запросе `SELECT` из таблиц семейства [MergeTree](../operations/table_engines/mergetree.md). При указании `FINAL`, данные будут выбираться полностью "домерженными". Стоит учитывать, что использование `FINAL` приводит к чтению также столбцов, относящихся к первичному ключу. Также, запрос будет выполняться в один поток, и при выполнении запроса будет выполняться слияние данных. Это приводит к тому, что при использовании `FINAL`, запрос выполняется медленнее. В большинстве случаев, следует избегать использования `FINAL`.
|
||||
Модификатор `FINAL` может быть использован для всех таблиц семейства `MergeTree`, которые производят преобразования данных в процессе фоновых слияний (кроме GraphiteMergeTree).
|
||||
|
||||
#### FINAL Modifier {#select-from-final}
|
||||
|
||||
Применим при выборке данных из таблиц с движками таблиц семейства [MergeTree](../operations/table_engines/mergetree.md), кроме `GraphiteMergeTree`. Если в запросе используется `FINAL`, то ClickHouse полностью мёржит данные перед выдачей результата, таким образом выполняя все преобразования данных, которые производятся движком таблиц при мёржах.
|
||||
|
||||
Также поддержан для движков:
|
||||
|
||||
- [Replicated](../operations/table_engines/replication.md)-версий `MergeTree`.
|
||||
- [View](../operations/table_engines/view.md), [Buffer](../operations/table_engines/buffer.md), [Distributed](../operations/table_engines/distributed.md), и [MaterializedView](../operations/table_engines/materializedview.md), которые работают поверх других движков, если они созданы для таблиц с движками семейства `MergeTree`.
|
||||
|
||||
Запросы, использующие `FINAL` исполняются медленнее аналогичных запросов без `FINAL`, поскольку:
|
||||
|
||||
- Запрос исполняется в один поток и данные мёржатся в процессе выполнения.
|
||||
- Запросы с модификатором `FINAL` дополнительно к столбцам, указанным в запросе, читают столбцы первичного ключа.
|
||||
|
||||
По возможности не используйте модификатор `FINAL`.
|
||||
|
||||
|
||||
### Секция SAMPLE {#select-sample-clause}
|
||||
|
||||
Секция `SAMPLE` позволяет выполнять запросы приближённо. Например, чтобы посчитать статистику по всем визитам, можно обработать 1/10 всех визитов и результат домножить на 10.
|
||||
|
@ -212,6 +212,7 @@ nav:
|
||||
- 'Overview': 'operations/utils/index.md'
|
||||
- 'clickhouse-copier': 'operations/utils/clickhouse-copier.md'
|
||||
- 'clickhouse-local': 'operations/utils/clickhouse-local.md'
|
||||
- 'clickhouse-benchmark': 'operations/utils/clickhouse-benchmark.md'
|
||||
|
||||
- 'Development':
|
||||
- 'hidden': 'development/index.md'
|
||||
|
@ -208,7 +208,8 @@ nav:
|
||||
- 'Overview': 'operations/utils/index.md'
|
||||
- 'clickhouse-copier': 'operations/utils/clickhouse-copier.md'
|
||||
- 'clickhouse-local': 'operations/utils/clickhouse-local.md'
|
||||
|
||||
- 'clickhouse-benchmark': 'operations/utils/clickhouse-benchmark.md'
|
||||
|
||||
- 'F.A.Q.':
|
||||
- 'General Questions': 'faq/general.md'
|
||||
|
||||
|
@ -212,7 +212,8 @@ nav:
|
||||
- 'Overview': 'operations/utils/index.md'
|
||||
- 'clickhouse-copier': 'operations/utils/clickhouse-copier.md'
|
||||
- 'clickhouse-local': 'operations/utils/clickhouse-local.md'
|
||||
|
||||
- 'clickhouse-benchmark': 'operations/utils/clickhouse-benchmark.md'
|
||||
|
||||
- 'Development':
|
||||
- 'hidden': 'development/index.md'
|
||||
- 'Overview of ClickHouse Architecture': 'development/architecture.md'
|
||||
|
@ -207,7 +207,8 @@ nav:
|
||||
- '介绍': 'operations/utils/index.md'
|
||||
- 'clickhouse-copier': 'operations/utils/clickhouse-copier.md'
|
||||
- 'clickhouse-local': 'operations/utils/clickhouse-local.md'
|
||||
|
||||
- 'clickhouse-benchmark': 'operations/utils/clickhouse-benchmark.md'
|
||||
|
||||
- '常见问题':
|
||||
- '一般的问题': 'faq/general.md'
|
||||
|
||||
|
1
docs/zh/operations/utils/clickhouse-benchmark.md
Symbolic link
1
docs/zh/operations/utils/clickhouse-benchmark.md
Symbolic link
@ -0,0 +1 @@
|
||||
../../../en/operations/utils/clickhouse-benchmark.md
|
@ -1,7 +1,7 @@
|
||||
# ClickHouse Utility
|
||||
|
||||
* [clickhouse-local](clickhouse-local.md) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` does this.
|
||||
* [clickhouse-copier](clickhouse-copier.md) — Copies (and reshards) data from one cluster to another cluster.
|
||||
|
||||
- [clickhouse-local](clickhouse-local.md) — Allows running SQL queries on data without stopping the ClickHouse server, similar to how `awk` does this.
|
||||
- [clickhouse-copier](clickhouse-copier.md) — Copies (and reshards) data from one cluster to another cluster.
|
||||
- [clickhouse-benchmark](clickhouse-benchmark.md) — Loads server with the custom queries and settings.
|
||||
|
||||
[Original article](https://clickhouse.yandex/docs/en/operations/utils/) <!--hide-->
|
||||
|
Loading…
Reference in New Issue
Block a user