thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add host regexp multiple PTR records integration tests

Browse Source
This commit is contained in:
Arthur Passos 2022-07-20 14:09:38 -03:00
parent abb550d2ff
commit 828de2a674
8 changed files with 177 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
docker/test/integration/runner/compose/docker_compose_coredns.yml Normal file
View File

@ -0,0 +1,9 @@
version: "2.3"
services:
coredns:
image: coredns/coredns:latest
restart: always
volumes:
- ${COREDNS_CONFIG_DIR}/example.com:/example.com
- ${COREDNS_CONFIG_DIR}/Corefile:/Corefile

55
tests/integration/helpers/cluster.py
View File

@ -364,6 +364,7 @@ class ClickHouseCluster:
self.with_jdbc_bridge = False
self.with_nginx = False
self.with_hive = False
self.with_coredns = False
self.with_minio = False
self.minio_dir = os.path.join(self.instances_dir, "minio")
@ -407,6 +408,10 @@ class ClickHouseCluster:
self.schema_registry_port = get_free_port()
self.kafka_docker_id = self.get_instance_docker_id(self.kafka_host)
self.coredns_host = "coredns"
self.coredns_port = 53
self.coredns_docker_id = self.get_instance_docker_id(self.coredns_host)
# available when with_kerberozed_kafka == True
self.kerberized_kafka_host = "kerberized_kafka1"
self.kerberized_kafka_port = get_free_port()
@ -1056,6 +1061,25 @@ class ClickHouseCluster:
]
return self.base_mongo_cmd
def setup_coredns_cmd(self, instance, env_variables, docker_compose_yml_dir):
self.with_coredns = True
env_variables["COREDNS_CONFIG_DIR"] = instance.path + "/" + "coredns_config"
self.base_cmd.extend(
["--file", p.join(docker_compose_yml_dir, "docker_compose_coredns.yml")]
)
self.base_coredns_cmd = [
"docker-compose",
"--env-file",
instance.env_file,
"--project-name",
self.project_name,
"--file",
p.join(docker_compose_yml_dir, "docker_compose_coredns.yml"),
]
return self.base_coredns_cmd
def setup_meili_cmd(self, instance, env_variables, docker_compose_yml_dir):
self.with_meili = True
env_variables["MEILI_HOST"] = self.meili_host
@ -1218,6 +1242,7 @@ class ClickHouseCluster:
with_cassandra=False,
with_jdbc_bridge=False,
with_hive=False,
with_coredns=False,
hostname=None,
env_variables=None,
image="clickhouse/integration-test",
@ -1301,6 +1326,7 @@ class ClickHouseCluster:
with_cassandra=with_cassandra,
with_jdbc_bridge=with_jdbc_bridge,
with_hive=with_hive,
with_coredns=with_coredns,
server_bin_path=self.server_bin_path,
odbc_bridge_bin_path=self.odbc_bridge_bin_path,
library_bridge_bin_path=self.library_bridge_bin_path,
@ -1460,6 +1486,11 @@ class ClickHouseCluster:
)
)
if with_coredns and not self.with_coredns:
cmds.append(
self.setup_coredns_cmd(instance, env_variables, docker_compose_yml_dir)
)
if with_meili and not self.with_meili:
cmds.append(
self.setup_meili_cmd(instance, env_variables, docker_compose_yml_dir)
@ -1576,6 +1607,16 @@ class ClickHouseCluster:
"IPAddress"
]
def get_instance_global_ipv6(self, instance_name):
logging.debug("get_instance_ip instance_name={}".format(instance_name))
docker_id = self.get_instance_docker_id(instance_name)
# for cont in self.docker_client.containers.list():
# logging.debug("CONTAINERS LIST: ID={} NAME={} STATUS={}".format(cont.id, cont.name, cont.status))
handle = self.docker_client.containers.get(docker_id)
return list(handle.attrs["NetworkSettings"]["Networks"].values())[0][
"GlobalIPv6Address"
]
def get_container_id(self, instance_name):
return self.get_instance_docker_id(instance_name)
# docker_id = self.get_instance_docker_id(instance_name)
@ -2380,6 +2421,12 @@ class ClickHouseCluster:
self.up_called = True
self.wait_mongo_to_start(30, secure=self.with_mongo_secure)
if self.with_coredns and self.base_coredns_cmd:
logging.debug("Setup coredns")
run_and_check(self.base_coredns_cmd + common_opts)
self.up_called = True
time.sleep(10)
if self.with_meili and self.base_meili_cmd:
logging.debug("Setup MeiliSearch")
run_and_check(self.base_meili_cmd + common_opts)
@ -2717,6 +2764,7 @@ class ClickHouseInstance:
with_azurite,
with_jdbc_bridge,
with_hive,
with_coredns,
with_cassandra,
server_bin_path,
odbc_bridge_bin_path,
@ -2799,6 +2847,8 @@ class ClickHouseInstance:
self.with_cassandra = with_cassandra
self.with_jdbc_bridge = with_jdbc_bridge
self.with_hive = with_hive
self.with_coredns = with_coredns
self.coredns_config_dir = p.abspath(p.join(base_path, "coredns_config"))
self.main_config_name = main_config_name
self.users_config_name = users_config_name
@ -3703,6 +3753,11 @@ class ClickHouseInstance:
self.kerberos_secrets_dir, p.abspath(p.join(self.path, "secrets"))
)
if self.with_coredns:
shutil.copytree(
self.coredns_config_dir, p.abspath(p.join(self.path, "coredns_config"))
)
# Copy config.d configs
logging.debug(
f"Copy custom test config files {self.custom_main_config_paths} to {self.config_d_dir}"

0
tests/integration/test_host_regexp_multiple_ptr_records/__init__.py Normal file
View File

11
tests/integration/test_host_regexp_multiple_ptr_records/configs/host_regexp.xml Normal file
View File

@ -0,0 +1,11 @@
<yandex>
<users>
<test_dns>
<password/>
<networks>
<host_regexp>test1\.example\.com$</host_regexp>
</networks>
<profile>default</profile>
</test_dns>
</users>
</yandex>

5
tests/integration/test_host_regexp_multiple_ptr_records/configs/listen_host.xml Normal file
View File

@ -0,0 +1,5 @@
<yandex>
<listen_host>::</listen_host>
<listen_host>0.0.0.0</listen_host>
<listen_try>1</listen_try>
</yandex>

8
tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/Corefile Normal file
View File

@ -0,0 +1,8 @@
. {
hosts /example.com {
reload "200ms"
fallthrough
}
forward . 127.0.0.11
log
}

1
tests/integration/test_host_regexp_multiple_ptr_records/coredns_config/example.com Normal file
View File

@ -0,0 +1 @@
filled in runtime, but needs to exist in order to be volume mapped in docker

88
tests/integration/test_host_regexp_multiple_ptr_records/test.py Normal file
View File

@ -0,0 +1,88 @@
import pytest
from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check
import os
DOCKER_COMPOSE_PATH = get_docker_compose_path()
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
cluster = ClickHouseCluster(__file__)
ch_server = cluster.add_instance(
"clickhouse-server",
with_coredns=True,
main_configs=["configs/listen_host.xml"],
user_configs=["configs/host_regexp.xml"],
ipv6_address="2001:3984:3989::1:1111",
)
client = cluster.add_instance(
"clickhouse-client",
ipv6_address="2001:3984:3989::1:1112",
)
@pytest.fixture(scope="module")
def started_cluster():
global cluster
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def setup_dns_server(ip):
domains_string = "test3.example.com test2.example.com test1.example.com"
example_file_path = f'{ch_server.env_variables["COREDNS_CONFIG_DIR"]}/example.com'
run_and_check(f"echo '{example_file_path}' > /custom_log", shell=True)
run_and_check(f"echo '{ip} {domains_string}' > {example_file_path}", shell=True)
def setup_ch_server(dns_server_ip):
ch_server.exec_in_container((["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"]))
ch_server.exec_in_container((["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"]))
ch_server.query("SYSTEM DROP DNS CACHE")
def build_endpoint_v4(ip):
return f"'http://{ip}:8123/?query=SELECT+1&user=test_dns'"
def build_endpoint_v6(ip):
return build_endpoint_v4(f"[{ip}]")
def test_host_regexp_multiple_ptr_v4_fails_with_wrong_resolution(started_cluster):
server_ip = cluster.get_instance_ip("clickhouse-server")
random_ip = "9.9.9.9"
dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
setup_dns_server(random_ip)
setup_ch_server(dns_server_ip)
endpoint = build_endpoint_v4(server_ip)
assert "1\n" != client.exec_in_container((["bash", "-c", f"curl {endpoint}"]))
def test_host_regexp_multiple_ptr_v4(started_cluster):
server_ip = cluster.get_instance_ip("clickhouse-server")
client_ip = cluster.get_instance_ip("clickhouse-client")
dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
setup_dns_server(client_ip)
setup_ch_server(dns_server_ip)
endpoint = build_endpoint_v4(server_ip)
assert "1\n" == client.exec_in_container((["bash", "-c", f"curl {endpoint}"]))
def test_host_regexp_multiple_ptr_v6(started_cluster):
setup_dns_server(client.ipv6_address)
setup_ch_server(cluster.get_instance_global_ipv6(cluster.coredns_host))
endpoint = build_endpoint_v6(ch_server.ipv6_address)
assert "1\n" == client.exec_in_container((["bash", "-c", f"curl -6 {endpoint}"]))