thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #62565 from ClickHouse/ci_add_azure_tests

Browse Source 
CI: Add tests with Azure storage
This commit is contained in:
alesapin 2024-04-27 15:13:42 +00:00 committed by GitHub
commit 9c3d40749b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 143 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/master.yml vendored
View File

@ -448,6 +448,14 @@ jobs:
test_name: Stateless tests (debug)
runner_type: func-tester
data: ${{ needs.RunConfig.outputs.data }}
FunctionalStatelessTestAsanAzure:
needs: [RunConfig, BuilderDebAsan]
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable_test.yml
with:
test_name: Stateless tests (azure, asan)
runner_type: func-tester
data: ${{ needs.RunConfig.outputs.data }}
##############################################################################################
############################ FUNCTIONAl STATEFUL TESTS #######################################
##############################################################################################
@ -598,6 +606,14 @@ jobs:
test_name: Stress test (tsan)
runner_type: stress-tester
data: ${{ needs.RunConfig.outputs.data }}
StressTestTsanAzure:
needs: [RunConfig, BuilderDebTsan]
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable_test.yml
with:
test_name: Stress test (azure, tsan)
runner_type: stress-tester
data: ${{ needs.RunConfig.outputs.data }}
StressTestMsan:
needs: [RunConfig, BuilderDebMsan]
if: ${{ !failure() && !cancelled() }}

2
docker/test/stateful/run.sh
View File

@ -19,7 +19,7 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
# install test configs
/usr/share/clickhouse-test/config/install.sh
azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence &
./setup_minio.sh stateful
config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml

23
docker/test/stateless/run.sh
View File

@ -42,12 +42,6 @@ source /utils.lib
# install test configs
/usr/share/clickhouse-test/config/install.sh
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
echo "Azure is disabled"
else
azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
fi
./setup_minio.sh stateless
./setup_hdfs_minicluster.sh
@ -97,12 +91,11 @@ if [ "$NUM_TRIES" -gt "1" ]; then
export THREAD_FUZZER_pthread_mutex_unlock_AFTER_SLEEP_TIME_US_MAX=10000
mkdir -p /var/run/clickhouse-server
# simplest way to forward env variables to server
sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml --daemon --pid-file /var/run/clickhouse-server/clickhouse-server.pid
else
sudo clickhouse start
fi
# simplest way to forward env variables to server
sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml --daemon --pid-file /var/run/clickhouse-server/clickhouse-server.pid
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
sudo sed -i "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
@ -212,6 +205,14 @@ function run_tests()
ADDITIONAL_OPTIONS+=('--s3-storage')
fi
if [[ -n "$USE_AZURE_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
# to disable the same tests
ADDITIONAL_OPTIONS+=('--s3-storage')
# azurite is slow, but with these two settings it can be super slow
ADDITIONAL_OPTIONS+=('--no-random-settings')
ADDITIONAL_OPTIONS+=('--no-random-merge-tree-settings')
fi
if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
ADDITIONAL_OPTIONS+=('--shared-catalog')
fi
@ -286,7 +287,7 @@ stop_logs_replication
failed_to_save_logs=0
for table in query_log zookeeper_log trace_log transactions_info_log metric_log
do
err=$( { clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst; } 2>&1 )
err=$(clickhouse-client -q "select * from system.$table into outfile '/test_output/$table.tsv.gz' format TSVWithNamesAndTypes")
echo "$err"
[[ "0" != "${#err}" ]] && failed_to_save_logs=1
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then

2
docker/test/stateless/stress_tests.lib
View File

@ -279,7 +279,7 @@ function check_logs_for_critical_errors()
function collect_query_and_trace_logs()
{
for table in query_log trace_log
for table in query_log trace_log metric_log
do
clickhouse-local --config-file=/etc/clickhouse-server/config.xml --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||:
done

1
docker/test/stress/run.sh
View File

@ -52,7 +52,6 @@ export ZOOKEEPER_FAULT_INJECTION=1
# available for dump via clickhouse-local
configure
azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
./setup_minio.sh stateless # to have a proper environment
config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml

9
tests/ci/ci_config.py
View File

@ -107,6 +107,7 @@ class JobNames(metaclass=WithIter):
# STATELESS_TEST_S3_RELEASE = "Stateless tests (release, s3 storage)"
STATELESS_TEST_S3_DEBUG = "Stateless tests (debug, s3 storage)"
STATELESS_TEST_S3_TSAN = "Stateless tests (tsan, s3 storage)"
STATELESS_TEST_AZURE_ASAN = "Stateless tests (azure, asan)"
STATELESS_TEST_FLAKY_ASAN = "Stateless tests flaky check (asan)"
STATEFUL_TEST_DEBUG = "Stateful tests (debug)"
@ -129,6 +130,7 @@ class JobNames(metaclass=WithIter):
STRESS_TEST_UBSAN = "Stress test (ubsan)"
STRESS_TEST_MSAN = "Stress test (msan)"
STRESS_TEST_DEBUG = "Stress test (debug)"
STRESS_TEST_AZURE_TSAN = "Stress test (azure, tsan)"
INTEGRATION_TEST = "Integration tests (release)"
INTEGRATION_TEST_ASAN = "Integration tests (asan)"
@ -1201,6 +1203,10 @@ CI_CONFIG = CIConfig(
Build.PACKAGE_DEBUG,
job_config=JobConfig(num_batches=6, **statless_test_common_params), # type: ignore
),
JobNames.STATELESS_TEST_AZURE_ASAN: TestConfig(
Build.PACKAGE_ASAN,
job_config=JobConfig(num_batches=4, **statless_test_common_params, release_only=True), # type: ignore
),
JobNames.STATELESS_TEST_S3_TSAN: TestConfig(
Build.PACKAGE_TSAN,
job_config=JobConfig(num_batches=5, **statless_test_common_params), # type: ignore
@ -1223,6 +1229,9 @@ CI_CONFIG = CIConfig(
JobNames.UPGRADE_TEST_ASAN: TestConfig(
Build.PACKAGE_ASAN, job_config=JobConfig(pr_only=True, random_bucket="upgrade_with_sanitizer", **upgrade_test_common_params) # type: ignore
),
JobNames.STRESS_TEST_AZURE_TSAN: TestConfig(
Build.PACKAGE_TSAN, job_config=JobConfig(**stress_test_common_params, release_only=True) # type: ignore
),
JobNames.UPGRADE_TEST_TSAN: TestConfig(
Build.PACKAGE_TSAN, job_config=JobConfig(pr_only=True, random_bucket="upgrade_with_sanitizer", **upgrade_test_common_params) # type: ignore
),

6
tests/ci/functional_test_check.py
View File

@ -15,6 +15,7 @@ from clickhouse_helper import CiLogsCredentials
from docker_images_helper import DockerImage, get_docker_image, pull_image
from download_release_packages import download_last_release
from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH
from get_robot_token import get_parameter_from_ssm
from pr_info import PRInfo
from report import ERROR, SUCCESS, JobReport, StatusType, TestResults, read_test_results
from stopwatch import Stopwatch
@ -27,6 +28,8 @@ def get_additional_envs(
check_name: str, run_by_hash_num: int, run_by_hash_total: int
) -> List[str]:
result = []
azure_connection_string = get_parameter_from_ssm("azure_connection_string")
result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'")
if "DatabaseReplicated" in check_name:
result.append("USE_DATABASE_REPLICATED=1")
if "DatabaseOrdinary" in check_name:
@ -40,6 +43,9 @@ def get_additional_envs(
result.append("RANDOMIZE_OBJECT_KEY_TYPE=1")
if "analyzer" in check_name:
result.append("USE_OLD_ANALYZER=1")
if "azure" in check_name:
assert "USE_S3_STORAGE_FOR_MERGE_TREE=1" not in result
result.append("USE_AZURE_STORAGE_FOR_MERGE_TREE=1")
if run_by_hash_total != 0:
result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}")

9
tests/ci/stress_check.py
View File

@ -12,18 +12,23 @@ from build_download_helper import download_all_deb_packages
from clickhouse_helper import CiLogsCredentials
from docker_images_helper import DockerImage, get_docker_image, pull_image
from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH
from get_robot_token import get_parameter_from_ssm
from pr_info import PRInfo
from report import ERROR, JobReport, TestResult, TestResults, read_test_results
from stopwatch import Stopwatch
from tee_popen import TeePopen
def get_additional_envs() -> List[str]:
def get_additional_envs(check_name: str) -> List[str]:
result = []
azure_connection_string = get_parameter_from_ssm("azure_connection_string")
result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'")
# some cloud-specific features require feature flags enabled
# so we need this ENV to be able to disable the randomization
# of feature flags
result.append("RANDOMIZE_KEEPER_FEATURE_FLAGS=1")
if "azure" in check_name:
result.append("USE_AZURE_STORAGE_FOR_MERGE_TREE=1")
return result
@ -143,7 +148,7 @@ def run_stress_test(docker_image_name: str) -> None:
pr_info, stopwatch.start_time_str, check_name
)
additional_envs = get_additional_envs()
additional_envs = get_additional_envs(check_name)
run_command = get_run_command(
packages_path,

87
tests/ci/test_ci_options.py
View File

@ -4,7 +4,6 @@
import unittest
from ci import CiOptions
from ci_config import JobNames
_TEST_BODY_1 = """
#### Run only:
@ -44,6 +43,85 @@ _TEST_BODY_3 = """
- [x] <!---ci_include_analyzer--> Must include all tests for analyzer
"""
_TEST_JOB_LIST = [
"Style check",
"Fast test",
"package_release",
"package_asan",
"Docker server image",
"Docker keeper image",
"Install packages (amd64)",
"Install packages (arm64)",
"Stateless tests (debug)",
"Stateless tests (release)",
"Stateless tests (coverage)",
"Stateless tests (aarch64)",
"Stateless tests (asan)",
"Stateless tests (tsan)",
"Stateless tests (msan)",
"Stateless tests (ubsan)",
"Stateless tests (release, old analyzer, s3, DatabaseReplicated)",
"Stateless tests (debug, s3 storage)",
"Stateless tests (tsan, s3 storage)",
"Stateless tests flaky check (asan)",
"Stateful tests (debug)",
"Stateful tests (release)",
"Stateful tests (coverage)",
"Stateful tests (aarch64)",
"Stateful tests (asan)",
"Stateful tests (tsan)",
"Stateful tests (msan)",
"Stateful tests (ubsan)",
"Stateful tests (release, ParallelReplicas)",
"Stateful tests (debug, ParallelReplicas)",
"Stateful tests (asan, ParallelReplicas)",
"Stateful tests (msan, ParallelReplicas)",
"Stateful tests (ubsan, ParallelReplicas)",
"Stateful tests (tsan, ParallelReplicas)",
"Stress test (asan)",
"Stress test (tsan)",
"Stress test (ubsan)",
"Stress test (msan)",
"Stress test (debug)",
"Integration tests (release)",
"Integration tests (asan)",
"Integration tests (asan, old analyzer)",
"Integration tests (tsan)",
"Integration tests (aarch64)",
"Integration tests flaky check (asan)",
"Upgrade check (debug)",
"Upgrade check (asan)",
"Upgrade check (tsan)",
"Upgrade check (msan)",
"Unit tests (release)",
"Unit tests (asan)",
"Unit tests (msan)",
"Unit tests (tsan)",
"Unit tests (ubsan)",
"AST fuzzer (debug)",
"AST fuzzer (asan)",
"AST fuzzer (msan)",
"AST fuzzer (tsan)",
"AST fuzzer (ubsan)",
"ClickHouse Keeper Jepsen",
"ClickHouse Server Jepsen",
"Performance Comparison",
"Performance Comparison Aarch64",
"Sqllogic test (release)",
"SQLancer (release)",
"SQLancer (debug)",
"SQLTest",
"Compatibility check (amd64)",
"Compatibility check (aarch64)",
"ClickBench (amd64)",
"ClickBench (aarch64)",
"libFuzzer tests",
"ClickHouse build check",
"ClickHouse special build check",
"Docs check",
"Bugfix validation",
]
class TestCIOptions(unittest.TestCase):
def test_pr_body_parsing(self):
@ -69,7 +147,7 @@ class TestCIOptions(unittest.TestCase):
ci_options.exclude_keywords,
["tsan", "aarch64", "analyzer", "s3_storage", "coverage"],
)
jobs_to_do = list(JobNames)
jobs_to_do = list(_TEST_JOB_LIST)
jobs_to_skip = []
job_params = {}
jobs_to_do, jobs_to_skip, job_params = ci_options.apply(
@ -81,9 +159,6 @@ class TestCIOptions(unittest.TestCase):
"Style check",
"package_release",
"package_asan",
"package_ubsan",
"package_debug",
"package_msan",
"Stateless tests (asan)",
"Stateless tests flaky check (asan)",
"Stateless tests (msan)",
@ -103,7 +178,7 @@ class TestCIOptions(unittest.TestCase):
)
self.assertCountEqual(ci_options.include_keywords, ["analyzer"])
self.assertIsNone(ci_options.exclude_keywords)
jobs_to_do = list(JobNames)
jobs_to_do = list(_TEST_JOB_LIST)
jobs_to_skip = []
job_params = {}
jobs_to_do, jobs_to_skip, job_params = ci_options.apply(

7
tests/config/config.d/azure_storage_conf.xml
View File

@ -4,13 +4,10 @@
<azure>
<type>object_storage</type>
<object_storage_type>azure</object_storage_type>
<storage_account_url>http://localhost:10000/devstoreaccount1</storage_account_url>
<container_name>cont</container_name>
<skip_access_check>false</skip_access_check>
<!-- default credentials for Azurite storage account -->
<account_name>devstoreaccount1</account_name>
<account_key>Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==</account_key>
<max_single_part_upload_size>33554432</max_single_part_upload_size>
<container_name>openbucketforpublicci</container_name>
<connection_string from_env="AZURE_CONNECTION_STRING"/>
</azure>
<cached_azure>
<type>cache</type>

5
tests/config/config.d/azure_storage_policy_by_default.xml Normal file
View File

@ -0,0 +1,5 @@
<clickhouse>
<merge_tree>
<storage_policy>azure_cache</storage_policy>
</merge_tree>
</clickhouse>

4
tests/config/install.sh
View File

@ -156,7 +156,7 @@ if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; th
ln -sf $SRC_PATH/users.d/database_ordinary.xml $DEST_SERVER_PATH/users.d/
fi
if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
if [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" == "1" ]]; then
object_key_types_options=("generate-suffix" "generate-full-key" "generate-template-key")
object_key_type="${object_key_types_options[0]}"
@ -177,6 +177,8 @@ if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TR
ln -sf $SRC_PATH/config.d/s3_storage_policy_by_default.xml $DEST_SERVER_PATH/config.d/
;;
esac
elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then
ln -sf $SRC_PATH/config.d/azure_storage_policy_by_default.xml $DEST_SERVER_PATH/config.d/
fi
ARM="aarch64"