thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 09:32:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #50950 from alekar/improve-outfile

Browse Source 
Address some usability issues with INTO OUTFILE usage.
This commit is contained in:
Nikolay Degterinsky 2023-06-25 18:40:55 +02:00 committed by GitHub
commit 4b02d83999
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 77 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
src/Client/ClientBase.cpp
View File

@ -102,6 +102,7 @@ namespace ErrorCodes
extern const int UNRECOGNIZED_ARGUMENTS;
extern const int LOGICAL_ERROR;
extern const int CANNOT_OPEN_FILE;
extern const int FILE_ALREADY_EXISTS;
}
}
@ -567,30 +568,17 @@ try
CompressionMethod compression_method = chooseCompressionMethod(out_file, compression_method_string);
UInt64 compression_level = 3;
if (query_with_output->is_outfile_append && compression_method != CompressionMethod::None)
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Cannot append to compressed file. Please use uncompressed file or remove APPEND keyword.");
}
if (query_with_output->compression_level)
{
const auto & compression_level_node = query_with_output->compression_level->as<ASTLiteral &>();
bool res = compression_level_node.value.tryGet<UInt64>(compression_level);
auto range = getCompressionLevelRange(compression_method);
if (!res || compression_level < range.first || compression_level > range.second)
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Invalid compression level, must be positive integer in range {}-{}",
range.first,
range.second);
compression_level_node.value.tryGet<UInt64>(compression_level);
}
auto flags = O_WRONLY | O_EXCL;
if (query_with_output->is_outfile_append)
flags |= O_APPEND;
else if (query_with_output->is_outfile_truncate)
flags |= O_TRUNC;
else
flags |= O_CREAT;
@ -871,6 +859,67 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa
}
}
// Run some local checks to make sure queries into output file will work before sending to server.
if (const auto * query_with_output = dynamic_cast<const ASTQueryWithOutput *>(parsed_query.get()))
{
String out_file;
if (query_with_output->out_file)
{
const auto & out_file_node = query_with_output->out_file->as<ASTLiteral &>();
out_file = out_file_node.value.safeGet<std::string>();
std::string compression_method_string;
if (query_with_output->compression)
{
const auto & compression_method_node = query_with_output->compression->as<ASTLiteral &>();
compression_method_string = compression_method_node.value.safeGet<std::string>();
}
CompressionMethod compression_method = chooseCompressionMethod(out_file, compression_method_string);
UInt64 compression_level = 3;
if (query_with_output->is_outfile_append && query_with_output->is_outfile_truncate)
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Cannot use INTO OUTFILE with APPEND and TRUNCATE simultaneously.");
}
if (query_with_output->is_outfile_append && compression_method != CompressionMethod::None)
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Cannot append to compressed file. Please use uncompressed file or remove APPEND keyword.");
}
if (query_with_output->compression_level)
{
const auto & compression_level_node = query_with_output->compression_level->as<ASTLiteral &>();
bool res = compression_level_node.value.tryGet<UInt64>(compression_level);
auto range = getCompressionLevelRange(compression_method);
if (!res || compression_level < range.first || compression_level > range.second)
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Invalid compression level, must be positive integer in range {}-{}",
range.first,
range.second);
}
if (fs::exists(out_file))
{
if (!query_with_output->is_outfile_append && !query_with_output->is_outfile_truncate)
{
throw Exception(
ErrorCodes::FILE_ALREADY_EXISTS,
"File {} exists, consider using APPEND or TRUNCATE.",
out_file);
}
}
}
}
const auto & settings = global_context->getSettingsRef();
const Int32 signals_before_stop = settings.partial_result_on_first_cancel ? 2 : 1;

2
src/Client/Suggest.cpp
View File

@ -43,7 +43,7 @@ Suggest::Suggest()
"IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", "ROLE",
"PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE",
"IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "RANDOMIZED",
"INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE", "CLEANUP"
"INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE", "CLEANUP", "APPEND"
});
}

2
src/Parsers/ASTQueryWithOutput.cpp
View File

@ -39,6 +39,8 @@ void ASTQueryWithOutput::formatImpl(const FormatSettings & s, FormatState & stat
s.ostr << (s.hilite ? hilite_keyword : "");
if (is_outfile_append)
s.ostr << " APPEND";
if (is_outfile_truncate)
s.ostr << " TRUNCATE";
if (is_into_outfile_with_stdout)
s.ostr << " AND STDOUT";
s.ostr << (s.hilite ? hilite_none : "");

1
src/Parsers/ASTQueryWithOutput.h
View File

@ -17,6 +17,7 @@ public:
ASTPtr out_file;
bool is_into_outfile_with_stdout = false;
bool is_outfile_append = false;
bool is_outfile_truncate = false;
ASTPtr format;
ASTPtr settings_ast;
ASTPtr compression;

6
src/Parsers/ParserQueryWithOutput.cpp
View File

@ -109,6 +109,12 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
query_with_output.is_outfile_append = true;
}
ParserKeyword s_truncate("TRUNCATE");
if (s_truncate.ignore(pos, expected))
{
query_with_output.is_outfile_truncate = true;
}
ParserKeyword s_stdout("AND STDOUT");
if (s_stdout.ignore(pos, expected))
{

2
tests/queries/0_stateless/02050_clickhouse_client_local_exception.sh
View File

@ -10,4 +10,4 @@ function cleanup()
rm "${CLICKHOUSE_TMP}/test_exception"
}
trap cleanup EXIT
$CLICKHOUSE_LOCAL --query="SELECT 1 INTO OUTFILE '${CLICKHOUSE_TMP}/test_exception' FORMAT Native" 2>&1 | grep -q "Code: 76. DB::ErrnoException:" && echo 'OK' || echo 'FAIL' ||:
$CLICKHOUSE_LOCAL --query="SELECT 1 INTO OUTFILE '${CLICKHOUSE_TMP}/test_exception' FORMAT Native" 2>&1 | grep -q "Code: 504. DB::Exception:" && echo 'OK' || echo 'FAIL' ||:

2
tests/queries/0_stateless/02346_into_outfile_and_stdout.sh
View File

@ -66,7 +66,7 @@ performBadQuery "bad_query_incorrect_usage" "SELECT 1, 2, 3 INTO OUTFILE AND STD
performBadQuery "bad_query_no_into_outfile" "SELECT 1, 2, 3 AND STDOUT'" "SYNTAX_ERROR"
performFileExists "bad_query_file_exists" "SELECT 1, 2, 3 INTO OUTFILE '${CLICKHOUSE_TMP}/test_into_outfile_and_stdout_bad_query_file_exists.out' AND STDOUT" "File exists. (CANNOT_OPEN_FILE)"
performFileExists "bad_query_file_exists" "SELECT 1, 2, 3 INTO OUTFILE '${CLICKHOUSE_TMP}/test_into_outfile_and_stdout_bad_query_file_exists.out' AND STDOUT" "File ${CLICKHOUSE_TMP}/test_into_outfile_and_stdout_bad_query_file_exists.out exists, consider using APPEND or TRUNCATE."
performCompression "compression" "SELECT * FROM (SELECT 'Hello, World! From clickhouse.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_into_outfile_and_stdout_compression.gz' AND STDOUT COMPRESSION 'GZ' FORMAT TabSeparated"