thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Removed leftovers

Browse Source
This commit is contained in:
Alexey Milovidov 2020-03-21 19:43:31 +03:00
parent 59976318e4
commit 1bfe312834
6 changed files with 0 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dbms/tests/clickhouse-test-server
View File

@ -130,19 +130,16 @@ if [ -n "$*" ]; then
$* $*
else else
TEST_RUN=${TEST_RUN=1} TEST_RUN=${TEST_RUN=1}
TEST_PERF=${TEST_PERF=1}
TEST_DICT=${TEST_DICT=1} TEST_DICT=${TEST_DICT=1}
CLICKHOUSE_CLIENT_QUERY="${CLICKHOUSE_CLIENT} --config ${CLICKHOUSE_CONFIG_CLIENT} --port $CLICKHOUSE_PORT_TCP -m -n -q" CLICKHOUSE_CLIENT_QUERY="${CLICKHOUSE_CLIENT} --config ${CLICKHOUSE_CONFIG_CLIENT} --port $CLICKHOUSE_PORT_TCP -m -n -q"
$CLICKHOUSE_CLIENT_QUERY 'SELECT * from system.build_options; SELECT * FROM system.clusters;' $CLICKHOUSE_CLIENT_QUERY 'SELECT * from system.build_options; SELECT * FROM system.clusters;'
CLICKHOUSE_TEST="env ${TEST_DIR}clickhouse-test --force-color --binary ${BIN_DIR}${CLICKHOUSE_BINARY_NAME} --configclient $CLICKHOUSE_CONFIG_CLIENT --configserver $CLICKHOUSE_CONFIG --tmp $DATA_DIR/tmp --queries $QUERIES_DIR $TEST_OPT0 $TEST_OPT" CLICKHOUSE_TEST="env ${TEST_DIR}clickhouse-test --force-color --binary ${BIN_DIR}${CLICKHOUSE_BINARY_NAME} --configclient $CLICKHOUSE_CONFIG_CLIENT --configserver $CLICKHOUSE_CONFIG --tmp $DATA_DIR/tmp --queries $QUERIES_DIR $TEST_OPT0 $TEST_OPT"
CLICKHOUSE_PERFORMANCE_TEST="${BIN_DIR}clickhouse-performance-test --port $CLICKHOUSE_PORT_TCP --recursive $CUR_DIR/performance --skip-tags=long"
if [ "${TEST_RUN_STRESS}" ]; then if [ "${TEST_RUN_STRESS}" ]; then
# Running test in parallel will fail some results (tests can create/fill/drop same tables) # Running test in parallel will fail some results (tests can create/fill/drop same tables)
TEST_NPROC=${TEST_NPROC:=$(( `nproc || sysctl -n hw.ncpu || echo 2` * 2))} TEST_NPROC=${TEST_NPROC:=$(( `nproc || sysctl -n hw.ncpu || echo 2` * 2))}
for i in `seq 1 ${TEST_NPROC}`; do for i in `seq 1 ${TEST_NPROC}`; do
$CLICKHOUSE_TEST --order=random --testname --tmp=$DATA_DIR/tmp/tmp${i} & $CLICKHOUSE_TEST --order=random --testname --tmp=$DATA_DIR/tmp/tmp${i} &
done done
$CLICKHOUSE_PERFORMANCE_TEST &
fi fi
if [ "${TEST_RUN_PARALLEL}" ]; then if [ "${TEST_RUN_PARALLEL}" ]; then
@ -164,8 +161,6 @@ else
( [ "$TEST_RUN" ] && $CLICKHOUSE_TEST ) || ${TEST_TRUE:=false} ( [ "$TEST_RUN" ] && $CLICKHOUSE_TEST ) || ${TEST_TRUE:=false}
fi fi
( [ "$TEST_PERF" ] && $CLICKHOUSE_PERFORMANCE_TEST $* ) || true
#( [ "$TEST_DICT" ] && mkdir -p $DATA_DIR/etc/dictionaries/ && cd $CUR_DIR/external_dictionaries && python generate_and_test.py --port=$CLICKHOUSE_PORT_TCP --client=$CLICKHOUSE_CLIENT --source=$CUR_DIR/external_dictionaries/source.tsv --reference=$CUR_DIR/external_dictionaries/reference --generated=$DATA_DIR/etc/dictionaries/ --no_mysql --no_mongo ) || true
$CLICKHOUSE_CLIENT_QUERY "SELECT event, value FROM system.events; SELECT metric, value FROM system.metrics; SELECT metric, value FROM system.asynchronous_metrics;" $CLICKHOUSE_CLIENT_QUERY "SELECT event, value FROM system.events; SELECT metric, value FROM system.metrics; SELECT metric, value FROM system.asynchronous_metrics;"
$CLICKHOUSE_CLIENT_QUERY "SELECT 'Still alive'" $CLICKHOUSE_CLIENT_QUERY "SELECT 'Still alive'"
fi fi

1
debian/clickhouse-test.install vendored
View File

@ -1,6 +1,5 @@
usr/bin/clickhouse-test usr/bin/clickhouse-test
usr/bin/clickhouse-test-server usr/bin/clickhouse-test-server
usr/bin/clickhouse-performance-test
usr/share/clickhouse-test/* usr/share/clickhouse-test/*
etc/clickhouse-client/client-test.xml etc/clickhouse-client/client-test.xml
etc/clickhouse-server/server-test.xml etc/clickhouse-server/server-test.xml

1
docker/packager/README.md
View File

@ -36,6 +36,5 @@ lrwxrwxrwx 1 root root 10 clickhouse-lld -> clickhouse
lrwxrwxrwx 1 root root 10 clickhouse-local -> clickhouse lrwxrwxrwx 1 root root 10 clickhouse-local -> clickhouse
lrwxrwxrwx 1 root root 10 clickhouse-obfuscator -> clickhouse lrwxrwxrwx 1 root root 10 clickhouse-obfuscator -> clickhouse
lrwxrwxrwx 1 root root 10 clickhouse-odbc-bridge -> clickhouse lrwxrwxrwx 1 root root 10 clickhouse-odbc-bridge -> clickhouse
lrwxrwxrwx 1 root root 10 clickhouse-performance-test -> clickhouse
lrwxrwxrwx 1 root root 10 clickhouse-server -> clickhouse lrwxrwxrwx 1 root root 10 clickhouse-server -> clickhouse
``` ```

16
docker/test/performance/Dockerfile
View File

@ -1,16 +0,0 @@
# docker build -t yandex/clickhouse-performance-test .
FROM yandex/clickhouse-stateful-test
RUN apt-get update -y \
&& env DEBIAN_FRONTEND=noninteractive \
apt-get install --yes --no-install-recommends \
python-requests
COPY s3downloader /s3downloader
COPY run.sh /run.sh
ENV OPEN_DATASETS="hits values_with_expressions"
ENV PRIVATE_DATASETS="hits_100m_single hits_10m_single"
ENV DOWNLOAD_DATASETS=1
CMD /run.sh

39
docker/test/performance/run.sh
View File

@ -1,39 +0,0 @@
#!/bin/bash
set -ex
install_packages() {
dpkg -i package_folder/clickhouse-common-static_*.deb
dpkg -i package_folder/clickhouse-server_*.deb
dpkg -i package_folder/clickhouse-client_*.deb
dpkg -i package_folder/clickhouse-test_*.deb
ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/
service clickhouse-server start && sleep 5
}
download_data() {
clickhouse-client --query "ATTACH DATABASE IF NOT EXISTS datasets ENGINE = Ordinary"
clickhouse-client --query "CREATE DATABASE IF NOT EXISTS test"
/s3downloader --dataset-names $OPEN_DATASETS
/s3downloader --dataset-names $PRIVATE_DATASETS --url 'https://s3.mds.yandex.net/clickhouse-private-datasets'
chmod 777 -R /var/lib/clickhouse
service clickhouse-server restart && sleep 5
clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
}
run() {
clickhouse-performance-test $TESTS_TO_RUN | tee test_output/test_result.json
}
install_packages
if [ $DOWNLOAD_DATASETS -eq 1 ]; then
download_data
fi
clickhouse-client --query "select * from system.settings where name = 'log_queries'"
tree /etc/clickhouse-server
cat /var/lib/clickhouse/preprocessed_configs/config.xml
cat /var/lib/clickhouse/preprocessed_configs/users.xml
run

98
docker/test/performance/s3downloader
View File

@ -1,98 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import tarfile
import logging
import argparse
import requests
import tempfile
DEFAULT_URL = 'https://clickhouse-datasets.s3.yandex.net'
AVAILABLE_DATASETS = {
'hits': 'hits_v1.tar',
'visits': 'visits_v1.tar',
'values_with_expressions': 'test_values.tar',
'hits_100m_single': 'hits_100m_single.tar',
'hits_1000m_single': 'hits_1000m_single.tar',
'hits_10m_single': 'hits_10m_single.tar',
'trips_mergetree': 'trips_mergetree.tar',
}
def _get_temp_file_name():
return os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names()))
def build_url(base_url, dataset):
return os.path.join(base_url, dataset, 'partitions', AVAILABLE_DATASETS[dataset])
def dowload_with_progress(url, path):
logging.info("Downloading from %s to temp path %s", url, path)
with open(path, 'w') as f:
response = requests.get(url, stream=True)
response.raise_for_status()
total_length = response.headers.get('content-length')
if total_length is None or int(total_length) == 0:
logging.info("No content-length, will download file without progress")
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
logging.info("Content length is %ld bytes", total_length)
counter = 0
for data in response.iter_content(chunk_size=4096):
dl += len(data)
counter += 1
f.write(data)
done = int(50 * dl / total_length)
percent = int(100 * float(dl) / total_length)
if sys.stdout.isatty():
sys.stdout.write("\r[{}{}] {}%".format('=' * done, ' ' * (50-done), percent))
sys.stdout.flush()
elif counter % 1000 == 0:
sys.stdout.write("{}%".format(percent))
sys.stdout.flush()
sys.stdout.write("\n")
logging.info("Downloading finished")
def unpack_to_clickhouse_directory(tar_path, clickhouse_path):
logging.info("Will unpack data from temp path %s to clickhouse db %s", tar_path, clickhouse_path)
with tarfile.open(tar_path, 'r') as comp_file:
comp_file.extractall(path=clickhouse_path)
logging.info("Unpack finished")
if __name__ == "__main__":
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s %(levelname)s: %(message)s')
parser = argparse.ArgumentParser(
description="Simple tool for dowloading datasets for clickhouse from S3")
parser.add_argument('--dataset-names', required=True, nargs='+', choices=AVAILABLE_DATASETS.keys())
parser.add_argument('--url-prefix', default=DEFAULT_URL)
parser.add_argument('--clickhouse-data-path', default='/var/lib/clickhouse/')
args = parser.parse_args()
datasets = args.dataset_names
logging.info("Will fetch following datasets: %s", ', '.join(datasets))
for dataset in datasets:
logging.info("Processing %s", dataset)
temp_archive_path = _get_temp_file_name()
try:
download_url_for_dataset = build_url(args.url_prefix, dataset)
dowload_with_progress(download_url_for_dataset, temp_archive_path)
unpack_to_clickhouse_directory(temp_archive_path, args.clickhouse_data_path)
except Exception as ex:
logging.info("Some exception occured %s", str(ex))
raise
finally:
logging.info("Will remove dowloaded file %s from filesystem if it exists", temp_archive_path)
if os.path.exists(temp_archive_path):
os.remove(temp_archive_path)
logging.info("Processing of %s finished, table placed at", dataset)
logging.info("Fetch finished, enjoy your tables!")