Merge pull request #53286 from arthurpassos/pin_coredns_version_and_fix_tests

Bring back **garbage** dns tests
This commit is contained in:
Alexey Milovidov 2023-08-12 17:24:38 +03:00 committed by GitHub
commit 0792d36415
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 408 additions and 1 deletions

View File

@ -2,7 +2,7 @@ version: "2.3"
services:
coredns:
image: coredns/coredns:latest
image: coredns/coredns:1.9.3 # :latest broke this test
restart: always
volumes:
- ${COREDNS_CONFIG_DIR}/example.com:/example.com

View File

@ -0,0 +1,11 @@
<clickhouse>
<users>
<test_dns>
<password/>
<networks>
<host_regexp>test1\.example\.com$</host_regexp>
</networks>
<profile>default</profile>
</test_dns>
</users>
</clickhouse>

View File

@ -0,0 +1,5 @@
<clickhouse>
<listen_host>::</listen_host>
<listen_host>0.0.0.0</listen_host>
<listen_try>1</listen_try>
</clickhouse>

View File

@ -0,0 +1,8 @@
. {
hosts /example.com {
reload "20ms"
fallthrough
}
forward . 127.0.0.11
log
}

View File

@ -0,0 +1 @@
filled in runtime, but needs to exist in order to be volume mapped in docker

View File

@ -0,0 +1,109 @@
import pytest
import socket
from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check
from time import sleep
import os
DOCKER_COMPOSE_PATH = get_docker_compose_path()
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
cluster = ClickHouseCluster(__file__)
ch_server = cluster.add_instance(
"clickhouse-server",
with_coredns=True,
main_configs=["configs/listen_host.xml"],
user_configs=["configs/host_regexp.xml"],
ipv6_address="2001:3984:3989::1:1111",
)
client = cluster.add_instance(
"clickhouse-client",
ipv6_address="2001:3984:3989::1:1112",
)
@pytest.fixture(scope="module")
def started_cluster():
global cluster
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def check_ptr_record(ip, hostname):
try:
host, aliaslist, ipaddrlist = socket.gethostbyaddr(ip)
if hostname.lower() == host.lower():
return True
except socket.herror:
pass
return False
def setup_dns_server(ip):
domains_string = "test3.example.com test2.example.com test1.example.com"
example_file_path = f'{ch_server.env_variables["COREDNS_CONFIG_DIR"]}/example.com'
run_and_check(f"echo '{ip} {domains_string}' > {example_file_path}", shell=True)
# DNS server takes time to reload the configuration.
for try_num in range(10):
if all(check_ptr_record(ip, host) for host in domains_string.split()):
break
sleep(1)
def setup_ch_server(dns_server_ip):
ch_server.exec_in_container(
(["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"])
)
ch_server.exec_in_container(
(["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"])
)
ch_server.query("SYSTEM DROP DNS CACHE")
def build_endpoint_v4(ip):
return f"'http://{ip}:8123/?query=SELECT+1&user=test_dns'"
def build_endpoint_v6(ip):
return build_endpoint_v4(f"[{ip}]")
def test_host_regexp_multiple_ptr_v4_fails_with_wrong_resolution(started_cluster):
server_ip = cluster.get_instance_ip("clickhouse-server")
random_ip = "9.9.9.9"
dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
setup_dns_server(random_ip)
setup_ch_server(dns_server_ip)
endpoint = build_endpoint_v4(server_ip)
assert "1\n" != client.exec_in_container(["bash", "-c", f"curl {endpoint}"])
def test_host_regexp_multiple_ptr_v4(started_cluster):
server_ip = cluster.get_instance_ip("clickhouse-server")
client_ip = cluster.get_instance_ip("clickhouse-client")
dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
setup_dns_server(client_ip)
setup_ch_server(dns_server_ip)
endpoint = build_endpoint_v4(server_ip)
assert "1\n" == client.exec_in_container(["bash", "-c", f"curl {endpoint}"])
def test_host_regexp_multiple_ptr_v6(started_cluster):
setup_dns_server(client.ipv6_address)
setup_ch_server(cluster.get_instance_global_ipv6(cluster.coredns_host))
endpoint = build_endpoint_v6(ch_server.ipv6_address)
assert "1\n" == client.exec_in_container(["bash", "-c", f"curl -6 {endpoint}"])

View File

@ -0,0 +1,4 @@
<clickhouse>
<disable_internal_dns_cache>1</disable_internal_dns_cache>
<max_concurrent_queries>250</max_concurrent_queries>
</clickhouse>

View File

@ -0,0 +1,11 @@
<clickhouse>
<users>
<test_dns>
<password/>
<networks>
<host_regexp>test1\.example\.com$</host_regexp>
</networks>
<profile>default</profile>
</test_dns>
</users>
</clickhouse>

View File

@ -0,0 +1,5 @@
<clickhouse>
<listen_host>::</listen_host>
<listen_host>0.0.0.0</listen_host>
<listen_try>1</listen_try>
</clickhouse>

View File

@ -0,0 +1,8 @@
. {
hosts /example.com {
reload "20ms"
fallthrough
}
forward . 127.0.0.11
log
}

View File

@ -0,0 +1 @@
filled in runtime, but needs to exist in order to be volume mapped in docker

View File

@ -0,0 +1,62 @@
import pycurl
import threading
from io import BytesIO
import sys
client_ip = sys.argv[1]
server_ip = sys.argv[2]
mutex = threading.Lock()
success_counter = 0
number_of_threads = 100
number_of_iterations = 100
def perform_request():
buffer = BytesIO()
crl = pycurl.Curl()
crl.setopt(pycurl.INTERFACE, client_ip)
crl.setopt(crl.WRITEDATA, buffer)
crl.setopt(crl.URL, f"http://{server_ip}:8123/?query=select+1&user=test_dns")
crl.perform()
# End curl session
crl.close()
str_response = buffer.getvalue().decode("iso-8859-1")
expected_response = "1\n"
mutex.acquire()
global success_counter
if str_response == expected_response:
success_counter += 1
mutex.release()
def perform_multiple_requests(n):
for request_number in range(n):
perform_request()
threads = []
for i in range(number_of_threads):
thread = threading.Thread(
target=perform_multiple_requests, args=(number_of_iterations,)
)
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
if success_counter == number_of_threads * number_of_iterations:
exit(0)
exit(1)

View File

@ -0,0 +1,88 @@
import pytest
import socket
from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check
from time import sleep
import os
DOCKER_COMPOSE_PATH = get_docker_compose_path()
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
cluster = ClickHouseCluster(__file__)
ch_server = cluster.add_instance(
"clickhouse-server",
with_coredns=True,
main_configs=["configs/config.xml", "configs/listen_host.xml"],
user_configs=["configs/host_regexp.xml"],
)
client = cluster.add_instance(
"clickhouse-client",
)
@pytest.fixture(scope="module")
def started_cluster():
global cluster
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def check_ptr_record(ip, hostname):
try:
host, aliaslist, ipaddrlist = socket.gethostbyaddr(ip)
if hostname.lower() == host.lower():
return True
except socket.herror:
pass
return False
def setup_dns_server(ip):
domains_string = "test3.example.com test2.example.com test1.example.com"
example_file_path = f'{ch_server.env_variables["COREDNS_CONFIG_DIR"]}/example.com'
run_and_check(f"echo '{ip} {domains_string}' > {example_file_path}", shell=True)
# DNS server takes time to reload the configuration.
for try_num in range(10):
if all(check_ptr_record(ip, host) for host in domains_string.split()):
break
sleep(1)
def setup_ch_server(dns_server_ip):
ch_server.exec_in_container(
(["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"])
)
ch_server.exec_in_container(
(["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"])
)
ch_server.query("SYSTEM DROP DNS CACHE")
def build_endpoint_v4(ip):
return f"'http://{ip}:8123/?query=SELECT+1&user=test_dns'"
def build_endpoint_v6(ip):
return build_endpoint_v4(f"[{ip}]")
def test_host_regexp_multiple_ptr_v4(started_cluster):
server_ip = cluster.get_instance_ip("clickhouse-server")
client_ip = cluster.get_instance_ip("clickhouse-client")
dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
setup_dns_server(client_ip)
setup_ch_server(dns_server_ip)
current_dir = os.path.dirname(__file__)
client.copy_file_to_container(
os.path.join(current_dir, "scripts", "stress_test.py"), "stress_test.py"
)
client.exec_in_container(["python3", f"stress_test.py", client_ip, server_ip])

View File

@ -0,0 +1,3 @@
<clickhouse>
<disable_internal_dns_cache>1</disable_internal_dns_cache>
</clickhouse>

View File

@ -0,0 +1,5 @@
<clickhouse>
<listen_host>::</listen_host>
<listen_host>0.0.0.0</listen_host>
<listen_try>1</listen_try>
</clickhouse>

View File

@ -0,0 +1,3 @@
<clickhouse>
<allow_reverse_dns_query_function>1</allow_reverse_dns_query_function>
</clickhouse>

View File

@ -0,0 +1,8 @@
. {
hosts /example.com {
reload "20ms"
fallthrough
}
forward . 127.0.0.11
log
}

View File

@ -0,0 +1 @@
filled in runtime, but needs to exist in order to be volume mapped in docker

View File

@ -0,0 +1,74 @@
import pytest
import socket
from helpers.cluster import ClickHouseCluster, get_docker_compose_path, run_and_check
from time import sleep
import os
DOCKER_COMPOSE_PATH = get_docker_compose_path()
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
cluster = ClickHouseCluster(__file__)
ch_server = cluster.add_instance(
"clickhouse-server",
with_coredns=True,
main_configs=[
"configs/config.xml",
"configs/reverse_dns_function.xml",
"configs/listen_host.xml",
],
)
@pytest.fixture(scope="module")
def started_cluster():
global cluster
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def check_ptr_record(ip, hostname):
try:
host, aliaslist, ipaddrlist = socket.gethostbyaddr(ip)
if hostname.lower() == host.lower():
return True
except socket.herror:
pass
return False
def setup_dns_server(ip):
domains_string = "test.example.com"
example_file_path = f'{ch_server.env_variables["COREDNS_CONFIG_DIR"]}/example.com'
run_and_check(f"echo '{ip} {domains_string}' > {example_file_path}", shell=True)
# DNS server takes time to reload the configuration.
for try_num in range(10):
if all(check_ptr_record(ip, host) for host in domains_string.split()):
break
sleep(1)
def setup_ch_server(dns_server_ip):
ch_server.exec_in_container(
(["bash", "-c", f"echo 'nameserver {dns_server_ip}' > /etc/resolv.conf"])
)
ch_server.exec_in_container(
(["bash", "-c", "echo 'options ndots:0' >> /etc/resolv.conf"])
)
ch_server.query("SYSTEM DROP DNS CACHE")
def test_reverse_dns_query(started_cluster):
dns_server_ip = cluster.get_instance_ip(cluster.coredns_host)
random_ipv6 = "4ae8:fa0f:ee1d:68c5:0b76:1b79:7ae6:1549" # https://commentpicker.com/ip-address-generator.php
setup_dns_server(random_ipv6)
setup_ch_server(dns_server_ip)
for _ in range(0, 200):
response = ch_server.query(f"select reverseDNSQuery('{random_ipv6}')")
assert response == "['test.example.com']\n"