thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 07:31:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix special build, add randomization to stateless tests

Browse Source
This commit is contained in:
Sema Checherinda 2024-01-03 16:15:56 +01:00
parent 2861cc70ba
commit c59a72b987
8 changed files with 22 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docker/test/stateful/run.sh
View File

@ -44,6 +44,9 @@ if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TR
# It is not needed, we will explicitly create tables on s3.
# We do not have statefull tests with s3 storage run in public repository, but this is needed for another repository.
rm /etc/clickhouse-server/config.d/s3_storage_policy_for_merge_tree_by_default.xml
rm /etc/clickhouse-server/config.d/storage_metadata_with_full_object_key.xml
rm /etc/clickhouse-server/config.d/s3_storage_policy_with_template_object_key.xml
fi
function start()

3
docker/test/stateless/stress_tests.lib
View File

@ -9,9 +9,6 @@ FAIL="\tFAIL\t\\N\t"
FAILURE_CONTEXT_LINES=100
FAILURE_CONTEXT_MAX_LINE_WIDTH=300
export S3_OBJECT_KEY_TYPES=("generate-suffix" "generate-full-key" "generate-template-key")
export USE_S3_STORAGE_WITH_OBJECT_KEY_TYPE="${S3_OBJECT_KEY_TYPES[0]}"
function escaped()
{
# That's the simplest way I found to escape a string in bash. Yep, bash is the most convenient programming language.

2
docker/test/stress/run.sh
View File

@ -193,8 +193,8 @@ stop
# Let's enable S3 storage by default
export USE_S3_STORAGE_FOR_MERGE_TREE=1
export $RANDOMIZE_OBJECT_KEY_TYPE=1
export ZOOKEEPER_FAULT_INJECTION=1
export USE_S3_STORAGE_WITH_OBJECT_KEY_TYPE="${S3_OBJECT_KEY_TYPES[$((RANDOM % ${#S3_OBJECT_KEY_TYPES[@]}))]}"
configure
# But we still need default disk because some tables loaded only into it

15
src/Common/MatchGenerator.cpp
View File

@ -188,7 +188,7 @@ private:
using CharRanges = std::vector<std::pair<re2::Rune, re2::Rune>>;
public:
RegexpCharClassFunction(Regexp * re_)
explicit RegexpCharClassFunction(Regexp * re_)
{
CharClass * cc = re_->cc();
chassert(cc);
@ -198,7 +198,7 @@ private:
char_count = cc->size();
char_ranges.reserve(std::distance(cc->begin(), cc->end()));
for (auto it = cc->begin(); it != cc->end(); ++it)
for (const auto * it = cc->begin(); it != cc->end(); ++it)
{
char_ranges.emplace_back(it->lo, it->hi);
}
@ -246,7 +246,7 @@ private:
class RegexpLiteralStringFunction : public NodeFunction
{
public:
RegexpLiteralStringFunction(Regexp * re_)
explicit RegexpLiteralStringFunction(Regexp * re_)
{
if (re_->nrunes() == 0)
return;
@ -279,7 +279,7 @@ private:
class RegexpLiteralFunction : public NodeFunction
{
public:
RegexpLiteralFunction(Regexp * re_)
explicit RegexpLiteralFunction(Regexp * re_)
{
char buffer[UTFmax];
@ -308,7 +308,7 @@ private:
class ThrowExceptionFunction : public NodeFunction
{
public:
ThrowExceptionFunction(Regexp * re_)
explicit ThrowExceptionFunction(Regexp * re_)
: operation(magic_enum::enum_name(re_->op()))
{
}
@ -332,7 +332,7 @@ private:
public:
RandomStringPrepareWalker(bool logging)
explicit RandomStringPrepareWalker(bool logging)
: logger(logging ? &Poco::Logger::get("GeneratorCombiner") : nullptr)
{
if (logger)
@ -344,7 +344,7 @@ public:
if (root == nullptr)
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "no root has been set");
if (generators.size() == 0)
if (generators.empty())
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "no generators");
auto root_func = generators.at(root);
@ -458,7 +458,6 @@ private:
return pre_arg;
}
private:
Poco::Logger * logger = nullptr;
Regexp * root = nullptr;

4
src/Common/ObjectStorageKeyGenerator.cpp
View File

@ -15,7 +15,6 @@ public:
, re_gen(key_template, /*logging*/ false)
{
}
DB::ObjectStorageKey generate(const String &) const override
{
return DB::ObjectStorageKey::createAsAbsolute(re_gen.generate());
@ -30,7 +29,7 @@ private:
class GeneratorWithPrefix : public DB::IObjectStorageKeysGenerator
{
public:
GeneratorWithPrefix(String key_prefix_)
explicit GeneratorWithPrefix(String key_prefix_)
: key_prefix(std::move(key_prefix_))
{}
@ -41,7 +40,6 @@ public:
/// Total length is 32 a-z characters for enough randomness.
/// First 3 characters are used as a prefix for
/// https://aws.amazon.com/premiumsupport/knowledge-center/s3-object-key-naming-pattern/
constexpr size_t key_name_total_size = 32;
constexpr size_t key_name_prefix_size = 3;

1
tests/ci/functional_test_check.py
View File

@ -55,6 +55,7 @@ def get_additional_envs(
result.append("USE_PARALLEL_REPLICAS=1")
if "s3 storage" in check_name:
result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1")
result.append("RANDOMIZE_OBJECT_KEY_TYPE=1")
if "analyzer" in check_name:
result.append("USE_NEW_ANALYZER=1")

1
tests/ci/libfuzzer_test_check.py
View File

@ -47,6 +47,7 @@ def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total):
result.append("USE_PARALLEL_REPLICAS=1")
if "s3 storage" in check_name:
result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1")
result.append("RANDOMIZE_OBJECT_KEY_TYPE=1")
if "analyzer" in check_name:
result.append("USE_NEW_ANALYZER=1")

9
tests/config/install.sh
View File

@ -141,7 +141,14 @@ if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; th
fi
if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
case "$USE_S3_STORAGE_WITH_OBJECT_KEY_TYPE" in
object_key_types_options=("generate-suffix" "generate-full-key" "generate-template-key")
object_key_type="${object_key_types_options[$((RANDOM % ${#object_key_types_options[0]}))]}"
if [[ -n "$RANDOMIZE_OBJECT_KEY_TYPE" ]] && [[ "$RANDOMIZE_OBJECT_KEY_TYPE" -eq 1 ]]; then
object_key_type="${randomize_options[$((RANDOM % ${#randomize_options[@]}))]}"
fi
case object_key_type in
"generate-full-key")
ln -sf $SRC_PATH/config.d/storage_metadata_with_full_object_key.xml $DEST_SERVER_PATH/config.d/
;;