thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #58130 from ClickHouse/integration-tests-images

Browse Source 
Use the single images list for integration tests everywhere
This commit is contained in:
Alexey Milovidov 2023-12-22 06:04:43 +01:00 committed by GitHub
commit 7f7e80ea56
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 60 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docker/test/integration/runner/compose/docker_compose_minio.yml
View File

@ -34,7 +34,7 @@ services:
# Empty container to run proxy resolver.
resolver:
image: clickhouse/python-bottle
image: clickhouse/python-bottle:${DOCKER_PYTHON_BOTTLE_TAG:-latest}
expose:
- "8080"
tty: true

33
tests/ci/ci_config.py
View File

@ -2,12 +2,13 @@
from enum import Enum
import logging
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable, Dict, Iterable, List, Literal, Optional, Union
from integration_test_images import IMAGES
class Labels(Enum):
DO_NOT_TEST_LABEL = "do not test"
@ -145,20 +146,7 @@ upgrade_check_digest = DigestConfig(
integration_check_digest = DigestConfig(
include_paths=["./tests/ci/integration_test_check.py", "./tests/integration"],
exclude_files=[".md"],
docker=[
"clickhouse/dotnet-client",
"clickhouse/integration-helper",
"clickhouse/integration-test",
"clickhouse/integration-tests-runner",
"clickhouse/kerberized-hadoop",
"clickhouse/kerberos-kdc",
"clickhouse/mysql-golang-client",
"clickhouse/mysql-java-client",
"clickhouse/mysql-js-client",
"clickhouse/mysql-php-client",
"clickhouse/nginx-dav",
"clickhouse/postgresql-java-client",
],
docker=IMAGES.copy(),
)
ast_fuzzer_check_digest = DigestConfig(
@ -202,20 +190,9 @@ bugfix_validate_check = DigestConfig(
"./tests/ci/bugfix_validate_check.py",
],
exclude_files=[".md"],
docker=[
docker=IMAGES.copy()
+ [
"clickhouse/stateless-test",
"clickhouse/dotnet-client",
"clickhouse/integration-helper",
"clickhouse/integration-test",
"clickhouse/integration-tests-runner",
"clickhouse/kerberized-hadoop",
"clickhouse/kerberos-kdc",
"clickhouse/mysql-golang-client",
"clickhouse/mysql-java-client",
"clickhouse/mysql-js-client",
"clickhouse/mysql-php-client",
"clickhouse/nginx-dav",
"clickhouse/postgresql-java-client",
],
)
# common test params

33
tests/ci/integration_test_check.py
View File

@ -10,13 +10,8 @@ import sys
from pathlib import Path
from typing import Dict, List, Tuple
from github import Github
from build_download_helper import download_all_deb_packages
from clickhouse_helper import (
ClickHouseHelper,
prepare_tests_results_for_clickhouse,
)
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
from commit_status_helper import (
RerunHelper,
get_commit,
@ -24,10 +19,12 @@ from commit_status_helper import (
post_commit_status,
post_commit_status_to_file,
)
from docker_images_helper import DockerImage, pull_image, get_docker_image
from docker_images_helper import DockerImage, get_docker_image, pull_image
from download_release_packages import download_last_release
from env_helper import REPORT_PATH, TEMP_PATH, REPO_COPY
from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH
from get_robot_token import get_best_robot_token
from github_helper import GitHub
from integration_test_images import IMAGES
from pr_info import PRInfo
from report import ERROR, TestResult, TestResults, read_test_results
from s3_helper import S3Helper
@ -36,24 +33,6 @@ from tee_popen import TeePopen
from upload_result_helper import upload_results
# When update, update
# tests/integration/ci-runner.py:ClickhouseIntegrationTestsRunner.get_images_names too
IMAGES = [
"clickhouse/dotnet-client",
"clickhouse/integration-helper",
"clickhouse/integration-test",
"clickhouse/integration-tests-runner",
"clickhouse/kerberized-hadoop",
"clickhouse/kerberos-kdc",
"clickhouse/mysql-golang-client",
"clickhouse/mysql-java-client",
"clickhouse/mysql-js-client",
"clickhouse/mysql-php-client",
"clickhouse/nginx-dav",
"clickhouse/postgresql-java-client",
]
def get_json_params_dict(
check_name: str,
pr_info: PRInfo,
@ -210,7 +189,7 @@ def main():
logging.info("Skipping '%s' (no pr-bugfix in '%s')", check_name, pr_info.labels)
sys.exit(0)
gh = Github(get_best_robot_token(), per_page=100)
gh = GitHub(get_best_robot_token())
commit = get_commit(gh, pr_info.sha)
rerun_helper = RerunHelper(commit, check_name_with_group)

31
tests/ci/integration_test_images.py Normal file
View File

@ -0,0 +1,31 @@
#!/usr/bin/env python3
IMAGES_ENV = {
"clickhouse/dotnet-client": "DOCKER_DOTNET_CLIENT_TAG",
"clickhouse/integration-helper": "DOCKER_HELPER_TAG",
"clickhouse/integration-test": "DOCKER_BASE_TAG",
"clickhouse/integration-tests-runner": "",
"clickhouse/kerberized-hadoop": "DOCKER_KERBERIZED_HADOOP_TAG",
"clickhouse/kerberos-kdc": "DOCKER_KERBEROS_KDC_TAG",
"clickhouse/mysql-golang-client": "DOCKER_MYSQL_GOLANG_CLIENT_TAG",
"clickhouse/mysql-java-client": "DOCKER_MYSQL_JAVA_CLIENT_TAG",
"clickhouse/mysql-js-client": "DOCKER_MYSQL_JS_CLIENT_TAG",
"clickhouse/mysql-php-client": "DOCKER_MYSQL_PHP_CLIENT_TAG",
"clickhouse/nginx-dav": "DOCKER_NGINX_DAV_TAG",
"clickhouse/postgresql-java-client": "DOCKER_POSTGRESQL_JAVA_CLIENT_TAG",
"clickhouse/python-bottle": "DOCKER_PYTHON_BOTTLE_TAG",
}
IMAGES = list(IMAGES_ENV.keys())
def get_image_env(image: str) -> str:
return IMAGES_ENV.get(image, "")
def get_docker_env(image: str, tag: str) -> str:
"if image belongs to IMAGES_ENV, return `-e` argument for docker command"
env = get_image_env(image)
if not env:
return env
return f"-e {env}={tag} "

24
tests/integration/ci-runner.py
View File

@ -1,6 +1,5 @@
#!/usr/bin/env python3
from collections import defaultdict
import csv
import glob
import json
@ -8,13 +7,15 @@ import logging
import os
import random
import re
import shlex
import shutil
import string
import subprocess
import time
import shlex
import zlib # for crc32
from collections import defaultdict
from integration_test_images import IMAGES
MAX_RETRY = 1
NUM_WORKERS = 5
@ -301,23 +302,6 @@ class ClickhouseIntegrationTestsRunner:
def shuffle_test_groups(self):
return self.shuffle_groups != 0
@staticmethod
def get_images_names():
return [
"clickhouse/dotnet-client",
"clickhouse/integration-helper",
"clickhouse/integration-test",
"clickhouse/integration-tests-runner",
"clickhouse/kerberized-hadoop",
"clickhouse/kerberos-kdc",
"clickhouse/mysql-golang-client",
"clickhouse/mysql-java-client",
"clickhouse/mysql-js-client",
"clickhouse/mysql-php-client",
"clickhouse/nginx-dav",
"clickhouse/postgresql-java-client",
]
def _pre_pull_images(self, repo_path):
image_cmd = self._get_runner_image_cmd(repo_path)
@ -523,7 +507,7 @@ class ClickhouseIntegrationTestsRunner:
os.path.join(repo_path, "tests/integration", "runner"),
"--docker-image-version",
):
for img in self.get_images_names():
for img in IMAGES:
if img == "clickhouse/integration-tests-runner":
runner_version = self.get_image_version(img)
logging.info(

1
tests/integration/integration_test_images.py Symbolic link
View File

@ -0,0 +1 @@
../ci/integration_test_images.py

43
tests/integration/runner
View File

@ -1,17 +1,17 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import subprocess
import os
import getpass
import glob
import argparse
import glob
import logging
import signal
import subprocess
import sys
import string
import os
import random
import shlex
import signal
import string
import subprocess
import sys
from integration_test_images import get_docker_env
def random_str(length=6):
@ -335,30 +335,11 @@ if __name__ == "__main__":
if args.docker_compose_images_tags is not None:
for img_tag in args.docker_compose_images_tags:
[image, tag] = img_tag.split(":")
if image == "clickhouse/dotnet-client":
env_tags += "-e {}={} ".format("DOCKER_DOTNET_CLIENT_TAG", tag)
elif image == "clickhouse/integration-helper":
env_tags += "-e {}={} ".format("DOCKER_HELPER_TAG", tag)
elif image == "clickhouse/integration-test":
env_tags += "-e {}={} ".format("DOCKER_BASE_TAG", tag)
elif image == "clickhouse/kerberized-hadoop":
env_tags += "-e {}={} ".format("DOCKER_KERBERIZED_HADOOP_TAG", tag)
elif image == "clickhouse/kerberos-kdc":
env_tags += "-e {}={} ".format("DOCKER_KERBEROS_KDC_TAG", tag)
elif image == "clickhouse/mysql-golang-client":
env_tags += "-e {}={} ".format("DOCKER_MYSQL_GOLANG_CLIENT_TAG", tag)
elif image == "clickhouse/mysql-java-client":
env_tags += "-e {}={} ".format("DOCKER_MYSQL_JAVA_CLIENT_TAG", tag)
elif image == "clickhouse/mysql-js-client":
env_tags += "-e {}={} ".format("DOCKER_MYSQL_JS_CLIENT_TAG", tag)
elif image == "clickhouse/mysql-php-client":
env_tags += "-e {}={} ".format("DOCKER_MYSQL_PHP_CLIENT_TAG", tag)
elif image == "clickhouse/nginx-dav":
env_tags += "-e {}={} ".format("DOCKER_NGINX_DAV_TAG", tag)
elif image == "clickhouse/postgresql-java-client":
env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag)
env_tag = get_docker_env(image, tag)
if env_tag:
env_tags += env_tag
else:
logging.info("Unknown image %s" % (image))
logging.info("Unknown image %s", image)
# create named volume which will be used inside to store images and other docker related files,
# to avoid redownloading it every time