mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 01:25:21 +00:00
change settings name
This commit is contained in:
parent
7939787302
commit
f13eedd644
@ -96,7 +96,7 @@ class IColumn;
|
||||
M(Bool, s3_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in s3 engine tables", 0) \
|
||||
M(Bool, s3_check_objects_after_upload, false, "Check each uploaded object to s3 with head request to be sure that upload was successful", 0) \
|
||||
M(Bool, s3_allow_parallel_part_upload, true, "Use multiple threads for s3 multipart upload. It may lead to slightly higher memory usage", 0) \
|
||||
M(Bool, s3_allow_throw_if_mismatch_files, false, "Allow throw error, when LIST request can not match any files", 0) \
|
||||
M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \
|
||||
M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \
|
||||
M(UInt64, hdfs_replication, 0, "The actual number of replications can be specified when the hdfs file is created.", 0) \
|
||||
M(Bool, hdfs_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables", 0) \
|
||||
|
@ -264,7 +264,7 @@ private:
|
||||
outcome_future = listObjectsAsync();
|
||||
}
|
||||
|
||||
if (request_settings.allow_throw_if_mismatch_files && result_batch.empty())
|
||||
if (request_settings.throw_on_zero_files_match && result_batch.empty())
|
||||
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Can not match any files using prefix {}", request.GetPrefix());
|
||||
|
||||
KeysWithInfo temp_buffer;
|
||||
|
@ -167,7 +167,7 @@ S3Settings::RequestSettings::RequestSettings(const NamedCollection & collection)
|
||||
max_connections = collection.getOrDefault<UInt64>("max_connections", max_connections);
|
||||
list_object_keys_size = collection.getOrDefault<UInt64>("list_object_keys_size", list_object_keys_size);
|
||||
allow_head_object_request = collection.getOrDefault<bool>("allow_head_object_request", allow_head_object_request);
|
||||
allow_throw_if_mismatch_files = collection.getOrDefault<bool>("allow_throw_if_mismatch_files", allow_throw_if_mismatch_files);
|
||||
throw_on_zero_files_match = collection.getOrDefault<bool>("throw_on_zero_files_match", throw_on_zero_files_match);
|
||||
}
|
||||
|
||||
S3Settings::RequestSettings::RequestSettings(
|
||||
@ -183,7 +183,7 @@ S3Settings::RequestSettings::RequestSettings(
|
||||
check_objects_after_upload = config.getBool(key + "check_objects_after_upload", settings.s3_check_objects_after_upload);
|
||||
list_object_keys_size = config.getUInt64(key + "list_object_keys_size", settings.s3_list_object_keys_size);
|
||||
allow_head_object_request = config.getBool(key + "allow_head_object_request", allow_head_object_request);
|
||||
allow_throw_if_mismatch_files = config.getBool(key + "allow_throw_if_mismatch_files", settings.s3_allow_throw_if_mismatch_files);
|
||||
throw_on_zero_files_match = config.getBool(key + "throw_on_zero_files_match", settings.s3_throw_on_zero_files_match);
|
||||
|
||||
/// NOTE: it would be better to reuse old throttlers to avoid losing token bucket state on every config reload,
|
||||
/// which could lead to exceeding limit for short time. But it is good enough unless very high `burst` values are used.
|
||||
@ -234,8 +234,8 @@ void S3Settings::RequestSettings::updateFromSettingsImpl(const Settings & settin
|
||||
put_request_throttler = std::make_shared<Throttler>(
|
||||
settings.s3_max_put_rps, settings.s3_max_put_burst ? settings.s3_max_put_burst : Throttler::default_burst_seconds * settings.s3_max_put_rps);
|
||||
|
||||
if (!if_changed || settings.s3_allow_throw_if_mismatch_files)
|
||||
allow_throw_if_mismatch_files = settings.s3_allow_throw_if_mismatch_files;
|
||||
if (!if_changed || settings.s3_throw_on_zero_files_match)
|
||||
throw_on_zero_files_match = settings.s3_throw_on_zero_files_match;
|
||||
}
|
||||
|
||||
void S3Settings::RequestSettings::updateFromSettings(const Settings & settings)
|
||||
|
@ -77,7 +77,7 @@ struct S3Settings
|
||||
/// See https://github.com/aws/aws-sdk-cpp/issues/1558 and also the function S3ErrorMarshaller::ExtractRegion() for more information.
|
||||
bool allow_head_object_request = true;
|
||||
|
||||
bool allow_throw_if_mismatch_files = false;
|
||||
bool throw_on_zero_files_match = false;
|
||||
|
||||
const PartUploadSettings & getUploadSettings() const { return upload_settings; }
|
||||
|
||||
|
@ -1,43 +1,43 @@
|
||||
-- { echo }
|
||||
drop table if exists test_02480_write;
|
||||
drop table if exists test_02480_write2;
|
||||
create table test_02480_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_{_partition_id}', format=Parquet) partition by a;
|
||||
drop table if exists test_02480_support_wildcard_write;
|
||||
drop table if exists test_02480_support_wildcard_write2;
|
||||
create table test_02480_support_wildcard_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a;
|
||||
set s3_truncate_on_insert=1;
|
||||
insert into test_02480_write values (1, 'a'), (22, 'b'), (333, 'c');
|
||||
select a, b from s3(s3_conn, filename='test_02480_*', format=Parquet) order by a;
|
||||
insert into test_02480_support_wildcard_write values (1, 'a'), (22, 'b'), (333, 'c');
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_*', format=Parquet) order by a;
|
||||
1 a
|
||||
22 b
|
||||
333 c
|
||||
select a, b from s3(s3_conn, filename='test_02480_?', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_?', format=Parquet) order by a;
|
||||
1 a
|
||||
select a, b from s3(s3_conn, filename='test_02480_??', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_??', format=Parquet) order by a;
|
||||
22 b
|
||||
select a, b from s3(s3_conn, filename='test_02480_?*?', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_?*?', format=Parquet) order by a;
|
||||
22 b
|
||||
333 c
|
||||
select a, b from s3(s3_conn, filename='test_02480_{1,333}', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_{1,333}', format=Parquet) order by a;
|
||||
1 a
|
||||
333 c
|
||||
select a, b from s3(s3_conn, filename='test_02480_{1..333}', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_{1..333}', format=Parquet) order by a;
|
||||
1 a
|
||||
22 b
|
||||
333 c
|
||||
create table test_02480_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_{_partition_id}', format=Parquet) partition by a;
|
||||
create table test_02480_support_wildcard_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a;
|
||||
set s3_truncate_on_insert=1;
|
||||
insert into test_02480_write2 values (4, 'd'), (55, 'f'), (666, 'g');
|
||||
select a, b from s3(s3_conn, filename='*/test_02480_*', format=Parquet) order by a;
|
||||
insert into test_02480_support_wildcard_write2 values (4, 'd'), (55, 'f'), (666, 'g');
|
||||
select a, b from s3(s3_conn, filename='*/test_02480_support_wildcard_*', format=Parquet) order by a;
|
||||
4 d
|
||||
55 f
|
||||
666 g
|
||||
select a, b from s3(s3_conn, filename='*/test_02480_?', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='*/test_02480_support_wildcard_?', format=Parquet) order by a;
|
||||
4 d
|
||||
select a, b from s3(s3_conn, filename='prefix/test_02480_??', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='prefix/test_02480_support_wildcard_??', format=Parquet) order by a;
|
||||
55 f
|
||||
select a, b from s3(s3_conn, filename='prefi?/test_02480_*', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='prefi?/test_02480_support_wildcard_*', format=Parquet) order by a;
|
||||
4 d
|
||||
55 f
|
||||
666 g
|
||||
select a, b from s3(s3_conn, filename='p?*/test_02480_{56..666}', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='p?*/test_02480_support_wildcard_{56..666}', format=Parquet) order by a;
|
||||
666 g
|
||||
drop table test_02480_write;
|
||||
drop table test_02480_write2;
|
||||
drop table test_02480_support_wildcard_write;
|
||||
drop table test_02480_support_wildcard_write2;
|
||||
|
@ -2,28 +2,28 @@
|
||||
-- Tag no-fasttest: Depends on AWS
|
||||
|
||||
-- { echo }
|
||||
drop table if exists test_02480_write;
|
||||
drop table if exists test_02480_write2;
|
||||
create table test_02480_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_{_partition_id}', format=Parquet) partition by a;
|
||||
drop table if exists test_02480_support_wildcard_write;
|
||||
drop table if exists test_02480_support_wildcard_write2;
|
||||
create table test_02480_support_wildcard_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a;
|
||||
set s3_truncate_on_insert=1;
|
||||
insert into test_02480_write values (1, 'a'), (22, 'b'), (333, 'c');
|
||||
insert into test_02480_support_wildcard_write values (1, 'a'), (22, 'b'), (333, 'c');
|
||||
|
||||
select a, b from s3(s3_conn, filename='test_02480_*', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_?', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_??', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_?*?', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_{1,333}', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_{1..333}', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_*', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_?', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_??', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_?*?', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_{1,333}', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_{1..333}', format=Parquet) order by a;
|
||||
|
||||
create table test_02480_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_{_partition_id}', format=Parquet) partition by a;
|
||||
create table test_02480_support_wildcard_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a;
|
||||
set s3_truncate_on_insert=1;
|
||||
insert into test_02480_write2 values (4, 'd'), (55, 'f'), (666, 'g');
|
||||
insert into test_02480_support_wildcard_write2 values (4, 'd'), (55, 'f'), (666, 'g');
|
||||
|
||||
select a, b from s3(s3_conn, filename='*/test_02480_*', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='*/test_02480_?', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='prefix/test_02480_??', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='prefi?/test_02480_*', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='p?*/test_02480_{56..666}', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='*/test_02480_support_wildcard_*', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='*/test_02480_support_wildcard_?', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='prefix/test_02480_support_wildcard_??', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='prefi?/test_02480_support_wildcard_*', format=Parquet) order by a;
|
||||
select a, b from s3(s3_conn, filename='p?*/test_02480_support_wildcard_{56..666}', format=Parquet) order by a;
|
||||
|
||||
drop table test_02480_write;
|
||||
drop table test_02480_write2;
|
||||
drop table test_02480_support_wildcard_write;
|
||||
drop table test_02480_support_wildcard_write2;
|
||||
|
@ -1,7 +1,7 @@
|
||||
-- { echo }
|
||||
drop table if exists test_02480_mismatch_files;
|
||||
create table test_02480_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_mismatch_files_{_partition_id}', format=Parquet) partition by a;
|
||||
drop table if exists test_02481_mismatch_files;
|
||||
create table test_02481_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02481_mismatch_files_{_partition_id}', format=Parquet) partition by a;
|
||||
set s3_truncate_on_insert=1;
|
||||
insert into test_02480_mismatch_files values (1, 'a'), (22, 'b'), (333, 'c');
|
||||
select a, b from s3(s3_conn, filename='test_02480_mismatch_filesxxx*', format=Parquet); -- { serverError 636 }
|
||||
select a, b from s3(s3_conn, filename='test_02480_mismatch_filesxxx*', format=Parquet) settings s3_allow_throw_if_mismatch_files=1; -- { serverError 107 }
|
||||
insert into test_02481_mismatch_files values (1, 'a'), (22, 'b'), (333, 'c');
|
||||
select a, b from s3(s3_conn, filename='test_02481_mismatch_filesxxx*', format=Parquet); -- { serverError 636 }
|
||||
select a, b from s3(s3_conn, filename='test_02481_mismatch_filesxxx*', format=Parquet) settings s3_throw_on_zero_files_match=1; -- { serverError 107 }
|
||||
|
@ -2,11 +2,11 @@
|
||||
-- Tag no-fasttest: Depends on AWS
|
||||
|
||||
-- { echo }
|
||||
drop table if exists test_02480_mismatch_files;
|
||||
create table test_02480_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_mismatch_files_{_partition_id}', format=Parquet) partition by a;
|
||||
drop table if exists test_02481_mismatch_files;
|
||||
create table test_02481_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02481_mismatch_files_{_partition_id}', format=Parquet) partition by a;
|
||||
set s3_truncate_on_insert=1;
|
||||
insert into test_02480_mismatch_files values (1, 'a'), (22, 'b'), (333, 'c');
|
||||
insert into test_02481_mismatch_files values (1, 'a'), (22, 'b'), (333, 'c');
|
||||
|
||||
select a, b from s3(s3_conn, filename='test_02480_mismatch_filesxxx*', format=Parquet); -- { serverError 636 }
|
||||
select a, b from s3(s3_conn, filename='test_02481_mismatch_filesxxx*', format=Parquet); -- { serverError 636 }
|
||||
|
||||
select a, b from s3(s3_conn, filename='test_02480_mismatch_filesxxx*', format=Parquet) settings s3_allow_throw_if_mismatch_files=1; -- { serverError 107 }
|
||||
select a, b from s3(s3_conn, filename='test_02481_mismatch_filesxxx*', format=Parquet) settings s3_throw_on_zero_files_match=1; -- { serverError 107 }
|
||||
|
Loading…
Reference in New Issue
Block a user