change settings name

This commit is contained in:
xiedeyantu 2023-02-04 22:11:14 +08:00
parent 7939787302
commit f13eedd644
8 changed files with 55 additions and 55 deletions

View File

@ -96,7 +96,7 @@ class IColumn;
M(Bool, s3_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in s3 engine tables", 0) \
M(Bool, s3_check_objects_after_upload, false, "Check each uploaded object to s3 with head request to be sure that upload was successful", 0) \
M(Bool, s3_allow_parallel_part_upload, true, "Use multiple threads for s3 multipart upload. It may lead to slightly higher memory usage", 0) \
M(Bool, s3_allow_throw_if_mismatch_files, false, "Allow throw error, when LIST request can not match any files", 0) \
M(Bool, s3_throw_on_zero_files_match, false, "Throw an error, when ListObjects request cannot match any files", 0) \
M(Bool, enable_s3_requests_logging, false, "Enable very explicit logging of S3 requests. Makes sense for debug only.", 0) \
M(UInt64, hdfs_replication, 0, "The actual number of replications can be specified when the hdfs file is created.", 0) \
M(Bool, hdfs_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables", 0) \

View File

@ -264,7 +264,7 @@ private:
outcome_future = listObjectsAsync();
}
if (request_settings.allow_throw_if_mismatch_files && result_batch.empty())
if (request_settings.throw_on_zero_files_match && result_batch.empty())
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Can not match any files using prefix {}", request.GetPrefix());
KeysWithInfo temp_buffer;

View File

@ -167,7 +167,7 @@ S3Settings::RequestSettings::RequestSettings(const NamedCollection & collection)
max_connections = collection.getOrDefault<UInt64>("max_connections", max_connections);
list_object_keys_size = collection.getOrDefault<UInt64>("list_object_keys_size", list_object_keys_size);
allow_head_object_request = collection.getOrDefault<bool>("allow_head_object_request", allow_head_object_request);
allow_throw_if_mismatch_files = collection.getOrDefault<bool>("allow_throw_if_mismatch_files", allow_throw_if_mismatch_files);
throw_on_zero_files_match = collection.getOrDefault<bool>("throw_on_zero_files_match", throw_on_zero_files_match);
}
S3Settings::RequestSettings::RequestSettings(
@ -183,7 +183,7 @@ S3Settings::RequestSettings::RequestSettings(
check_objects_after_upload = config.getBool(key + "check_objects_after_upload", settings.s3_check_objects_after_upload);
list_object_keys_size = config.getUInt64(key + "list_object_keys_size", settings.s3_list_object_keys_size);
allow_head_object_request = config.getBool(key + "allow_head_object_request", allow_head_object_request);
allow_throw_if_mismatch_files = config.getBool(key + "allow_throw_if_mismatch_files", settings.s3_allow_throw_if_mismatch_files);
throw_on_zero_files_match = config.getBool(key + "throw_on_zero_files_match", settings.s3_throw_on_zero_files_match);
/// NOTE: it would be better to reuse old throttlers to avoid losing token bucket state on every config reload,
/// which could lead to exceeding limit for short time. But it is good enough unless very high `burst` values are used.
@ -234,8 +234,8 @@ void S3Settings::RequestSettings::updateFromSettingsImpl(const Settings & settin
put_request_throttler = std::make_shared<Throttler>(
settings.s3_max_put_rps, settings.s3_max_put_burst ? settings.s3_max_put_burst : Throttler::default_burst_seconds * settings.s3_max_put_rps);
if (!if_changed || settings.s3_allow_throw_if_mismatch_files)
allow_throw_if_mismatch_files = settings.s3_allow_throw_if_mismatch_files;
if (!if_changed || settings.s3_throw_on_zero_files_match)
throw_on_zero_files_match = settings.s3_throw_on_zero_files_match;
}
void S3Settings::RequestSettings::updateFromSettings(const Settings & settings)

View File

@ -77,7 +77,7 @@ struct S3Settings
/// See https://github.com/aws/aws-sdk-cpp/issues/1558 and also the function S3ErrorMarshaller::ExtractRegion() for more information.
bool allow_head_object_request = true;
bool allow_throw_if_mismatch_files = false;
bool throw_on_zero_files_match = false;
const PartUploadSettings & getUploadSettings() const { return upload_settings; }

View File

@ -1,43 +1,43 @@
-- { echo }
drop table if exists test_02480_write;
drop table if exists test_02480_write2;
create table test_02480_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_{_partition_id}', format=Parquet) partition by a;
drop table if exists test_02480_support_wildcard_write;
drop table if exists test_02480_support_wildcard_write2;
create table test_02480_support_wildcard_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a;
set s3_truncate_on_insert=1;
insert into test_02480_write values (1, 'a'), (22, 'b'), (333, 'c');
select a, b from s3(s3_conn, filename='test_02480_*', format=Parquet) order by a;
insert into test_02480_support_wildcard_write values (1, 'a'), (22, 'b'), (333, 'c');
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_*', format=Parquet) order by a;
1 a
22 b
333 c
select a, b from s3(s3_conn, filename='test_02480_?', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_?', format=Parquet) order by a;
1 a
select a, b from s3(s3_conn, filename='test_02480_??', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_??', format=Parquet) order by a;
22 b
select a, b from s3(s3_conn, filename='test_02480_?*?', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_?*?', format=Parquet) order by a;
22 b
333 c
select a, b from s3(s3_conn, filename='test_02480_{1,333}', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_{1,333}', format=Parquet) order by a;
1 a
333 c
select a, b from s3(s3_conn, filename='test_02480_{1..333}', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_{1..333}', format=Parquet) order by a;
1 a
22 b
333 c
create table test_02480_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_{_partition_id}', format=Parquet) partition by a;
create table test_02480_support_wildcard_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a;
set s3_truncate_on_insert=1;
insert into test_02480_write2 values (4, 'd'), (55, 'f'), (666, 'g');
select a, b from s3(s3_conn, filename='*/test_02480_*', format=Parquet) order by a;
insert into test_02480_support_wildcard_write2 values (4, 'd'), (55, 'f'), (666, 'g');
select a, b from s3(s3_conn, filename='*/test_02480_support_wildcard_*', format=Parquet) order by a;
4 d
55 f
666 g
select a, b from s3(s3_conn, filename='*/test_02480_?', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='*/test_02480_support_wildcard_?', format=Parquet) order by a;
4 d
select a, b from s3(s3_conn, filename='prefix/test_02480_??', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='prefix/test_02480_support_wildcard_??', format=Parquet) order by a;
55 f
select a, b from s3(s3_conn, filename='prefi?/test_02480_*', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='prefi?/test_02480_support_wildcard_*', format=Parquet) order by a;
4 d
55 f
666 g
select a, b from s3(s3_conn, filename='p?*/test_02480_{56..666}', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='p?*/test_02480_support_wildcard_{56..666}', format=Parquet) order by a;
666 g
drop table test_02480_write;
drop table test_02480_write2;
drop table test_02480_support_wildcard_write;
drop table test_02480_support_wildcard_write2;

View File

@ -2,28 +2,28 @@
-- Tag no-fasttest: Depends on AWS
-- { echo }
drop table if exists test_02480_write;
drop table if exists test_02480_write2;
create table test_02480_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_{_partition_id}', format=Parquet) partition by a;
drop table if exists test_02480_support_wildcard_write;
drop table if exists test_02480_support_wildcard_write2;
create table test_02480_support_wildcard_write (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a;
set s3_truncate_on_insert=1;
insert into test_02480_write values (1, 'a'), (22, 'b'), (333, 'c');
insert into test_02480_support_wildcard_write values (1, 'a'), (22, 'b'), (333, 'c');
select a, b from s3(s3_conn, filename='test_02480_*', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_?', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_??', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_?*?', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_{1,333}', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_{1..333}', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_*', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_?', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_??', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_?*?', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_{1,333}', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='test_02480_support_wildcard_{1..333}', format=Parquet) order by a;
create table test_02480_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_{_partition_id}', format=Parquet) partition by a;
create table test_02480_support_wildcard_write2 (a UInt64, b String) engine = S3(s3_conn, filename='prefix/test_02480_support_wildcard_{_partition_id}', format=Parquet) partition by a;
set s3_truncate_on_insert=1;
insert into test_02480_write2 values (4, 'd'), (55, 'f'), (666, 'g');
insert into test_02480_support_wildcard_write2 values (4, 'd'), (55, 'f'), (666, 'g');
select a, b from s3(s3_conn, filename='*/test_02480_*', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='*/test_02480_?', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='prefix/test_02480_??', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='prefi?/test_02480_*', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='p?*/test_02480_{56..666}', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='*/test_02480_support_wildcard_*', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='*/test_02480_support_wildcard_?', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='prefix/test_02480_support_wildcard_??', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='prefi?/test_02480_support_wildcard_*', format=Parquet) order by a;
select a, b from s3(s3_conn, filename='p?*/test_02480_support_wildcard_{56..666}', format=Parquet) order by a;
drop table test_02480_write;
drop table test_02480_write2;
drop table test_02480_support_wildcard_write;
drop table test_02480_support_wildcard_write2;

View File

@ -1,7 +1,7 @@
-- { echo }
drop table if exists test_02480_mismatch_files;
create table test_02480_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_mismatch_files_{_partition_id}', format=Parquet) partition by a;
drop table if exists test_02481_mismatch_files;
create table test_02481_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02481_mismatch_files_{_partition_id}', format=Parquet) partition by a;
set s3_truncate_on_insert=1;
insert into test_02480_mismatch_files values (1, 'a'), (22, 'b'), (333, 'c');
select a, b from s3(s3_conn, filename='test_02480_mismatch_filesxxx*', format=Parquet); -- { serverError 636 }
select a, b from s3(s3_conn, filename='test_02480_mismatch_filesxxx*', format=Parquet) settings s3_allow_throw_if_mismatch_files=1; -- { serverError 107 }
insert into test_02481_mismatch_files values (1, 'a'), (22, 'b'), (333, 'c');
select a, b from s3(s3_conn, filename='test_02481_mismatch_filesxxx*', format=Parquet); -- { serverError 636 }
select a, b from s3(s3_conn, filename='test_02481_mismatch_filesxxx*', format=Parquet) settings s3_throw_on_zero_files_match=1; -- { serverError 107 }

View File

@ -2,11 +2,11 @@
-- Tag no-fasttest: Depends on AWS
-- { echo }
drop table if exists test_02480_mismatch_files;
create table test_02480_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02480_mismatch_files_{_partition_id}', format=Parquet) partition by a;
drop table if exists test_02481_mismatch_files;
create table test_02481_mismatch_files (a UInt64, b String) engine = S3(s3_conn, filename='test_02481_mismatch_files_{_partition_id}', format=Parquet) partition by a;
set s3_truncate_on_insert=1;
insert into test_02480_mismatch_files values (1, 'a'), (22, 'b'), (333, 'c');
insert into test_02481_mismatch_files values (1, 'a'), (22, 'b'), (333, 'c');
select a, b from s3(s3_conn, filename='test_02480_mismatch_filesxxx*', format=Parquet); -- { serverError 636 }
select a, b from s3(s3_conn, filename='test_02481_mismatch_filesxxx*', format=Parquet); -- { serverError 636 }
select a, b from s3(s3_conn, filename='test_02480_mismatch_filesxxx*', format=Parquet) settings s3_allow_throw_if_mismatch_files=1; -- { serverError 107 }
select a, b from s3(s3_conn, filename='test_02481_mismatch_filesxxx*', format=Parquet) settings s3_throw_on_zero_files_match=1; -- { serverError 107 }