mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-17 13:13:36 +00:00
Merge branch 'nikvas0/index' into nikvas0/parse_index
This commit is contained in:
Nikita Vasilev
commit
2a76e86338
11
.gitignore
vendored
11
.gitignore
vendored
@ -180,18 +180,10 @@ utils/zookeeper-create-entry-to-download-part/zookeeper-create-entry-to-download
|
||||
utils/zookeeper-dump-tree/zookeeper-dump-tree
|
||||
utils/zookeeper-remove-by-list/zookeeper-remove-by-list
|
||||
dbms/src/Storages/tests/remove_symlink_directory
|
||||
debian/control
|
||||
debian/copyright
|
||||
debian/tmp/
|
||||
libs/libcommon/src/tests/json_test
|
||||
utils/compressor/zstd_test
|
||||
utils/wikistat-loader/wikistat-loader
|
||||
dbms/src/Common/tests/pod_array
|
||||
debian/clickhouse-benchmark/
|
||||
debian/clickhouse-client/
|
||||
debian/clickhouse-server-base/
|
||||
debian/clickhouse-server-common/
|
||||
debian/files
|
||||
|
||||
dbms/src/Server/data/*
|
||||
dbms/src/Server/metadata/*
|
||||
@ -210,9 +202,6 @@ vgcore*
|
||||
*.changes
|
||||
build-stamp
|
||||
configure-stamp
|
||||
debian/*.debhelper.log
|
||||
debian/*.debhelper
|
||||
debian/*.substvars
|
||||
|
||||
*.bin
|
||||
*.mrk
|
||||
|
||||
@ -1,50 +0,0 @@
|
||||
language: generic
|
||||
|
||||
matrix:
|
||||
fast_finish: true
|
||||
include:
|
||||
# We need to have gcc7 headers to compile c++17 code on clang
|
||||
# - os: linux
|
||||
#
|
||||
# cache:
|
||||
# ccache: true
|
||||
# timeout: 1000
|
||||
# directories:
|
||||
# - /home/travis/.ccache
|
||||
#
|
||||
# addons:
|
||||
# apt:
|
||||
# update: true
|
||||
# sources:
|
||||
# - ubuntu-toolchain-r-test
|
||||
# - llvm-toolchain-trusty-5.0
|
||||
# packages: [ ninja-build, g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl]
|
||||
#
|
||||
# env:
|
||||
# - MATRIX_EVAL="export CC=clang-5.0 CXX=clang++-5.0"
|
||||
#
|
||||
# script:
|
||||
# - utils/travis/normal.sh
|
||||
|
||||
- os: linux
|
||||
|
||||
sudo: required
|
||||
|
||||
cache:
|
||||
timeout: 1000
|
||||
directories:
|
||||
- /var/cache/pbuilder/ccache
|
||||
|
||||
addons:
|
||||
apt:
|
||||
update: true
|
||||
packages: [ pbuilder, fakeroot, debhelper ]
|
||||
|
||||
script:
|
||||
- utils/travis/pbuilder.sh
|
||||
|
||||
allow_failures:
|
||||
- os: osx
|
||||
|
||||
before_script:
|
||||
- eval "${MATRIX_EVAL}"
|
||||
@ -2,10 +2,10 @@
|
||||
set(VERSION_REVISION 54413)
|
||||
set(VERSION_MAJOR 19)
|
||||
set(VERSION_MINOR 1)
|
||||
set(VERSION_PATCH 1)
|
||||
set(VERSION_GITHASH 4e7747117123f5a1b027a64865844b4faa10447d)
|
||||
set(VERSION_DESCRIBE v19.1.1-testing)
|
||||
set(VERSION_STRING 19.1.1)
|
||||
set(VERSION_PATCH 3)
|
||||
set(VERSION_GITHASH ac0060079ab221278338db343ca9eaf006fc4ee1)
|
||||
set(VERSION_DESCRIBE v19.1.3-testing)
|
||||
set(VERSION_STRING 19.1.3)
|
||||
# end of autochange
|
||||
|
||||
set(VERSION_EXTRA "" CACHE STRING "")
|
||||
|
||||
@ -186,10 +186,10 @@ SharedLibraryPtr Compiler::getOrCount(
|
||||
static void addCodeToAssertHeadersMatch(WriteBuffer & out)
|
||||
{
|
||||
out <<
|
||||
"#define STRING2(x) #x\n"
|
||||
"#define STRING(x) STRING2(x)\n"
|
||||
"#include <Common/config_version.h>\n"
|
||||
"#if VERSION_REVISION != " << ClickHouseRevision::get() << "\n"
|
||||
"#define STRING2(x) #x\n"
|
||||
"#define STRING(x) STRING2(x)\n"
|
||||
"#pragma message \"ClickHouse headers revision = \" STRING(VERSION_REVISION) \n"
|
||||
"#error \"ClickHouse headers revision doesn't match runtime revision of the server (" << ClickHouseRevision::get() << ").\"\n"
|
||||
"#endif\n\n";
|
||||
|
||||
@ -136,7 +136,7 @@ void NO_INLINE Aggregator::executeSpecializedCase(
|
||||
AggregateDataPtr overflow_row) const
|
||||
{
|
||||
/// For all rows.
|
||||
typename Method::Key prev_key;
|
||||
typename Method::Key prev_key{};
|
||||
AggregateDataPtr value = nullptr;
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
|
||||
12
debian/.gitignore
vendored
Normal file
12
debian/.gitignore
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
control
|
||||
copyright
|
||||
tmp/
|
||||
clickhouse-benchmark/
|
||||
clickhouse-client/
|
||||
clickhouse-server-base/
|
||||
clickhouse-server-common/
|
||||
files
|
||||
*.debhelper.log
|
||||
*.debhelper
|
||||
*.substvars
|
||||
|
||||
4
debian/changelog
vendored
4
debian/changelog
vendored
@ -1,5 +1,5 @@
|
||||
clickhouse (19.1.1) unstable; urgency=low
|
||||
clickhouse (19.1.3) unstable; urgency=low
|
||||
|
||||
* Modified source code
|
||||
|
||||
-- <root@yandex-team.ru> Wed, 16 Jan 2019 14:04:37 +0300
|
||||
-- <root@yandex-team.ru> Mon, 21 Jan 2019 16:26:13 +0300
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
|
||||
ARG version=19.1.1
|
||||
ARG version=19.1.3
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends \
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
|
||||
ARG version=19.1.1
|
||||
ARG version=19.1.3
|
||||
ARG gosu_ver=1.10
|
||||
|
||||
RUN apt-get update \
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
|
||||
ARG version=19.1.1
|
||||
ARG version=19.1.3
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y apt-transport-https dirmngr && \
|
||||
|
||||
29
docker/test/stateful/Dockerfile
Normal file
29
docker/test/stateful/Dockerfile
Normal file
@ -0,0 +1,29 @@
|
||||
FROM yandex/clickhouse-stateless-test
|
||||
|
||||
RUN apt-get update -y \
|
||||
&& env DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get install --yes --no-install-recommends \
|
||||
python-requests
|
||||
|
||||
COPY s3downloader /s3downloader
|
||||
|
||||
ENV DATASETS="hits visits"
|
||||
|
||||
CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
|
||||
dpkg -i package_folder/clickhouse-server_*.deb; \
|
||||
dpkg -i package_folder/clickhouse-client_*.deb; \
|
||||
dpkg -i package_folder/clickhouse-test_*.deb; \
|
||||
service zookeeper start; sleep 5; \
|
||||
service clickhouse-server start && sleep 5 \
|
||||
&& /s3downloader --dataset-names $DATASETS \
|
||||
&& chmod 777 -R /var/lib/clickhouse \
|
||||
&& clickhouse-client --query "SHOW DATABASES" \
|
||||
&& clickhouse-client --query "CREATE DATABASE datasets" \
|
||||
&& clickhouse-client --query "CREATE DATABASE test" \
|
||||
&& service clickhouse-server restart && sleep 5 \
|
||||
&& clickhouse-client --query "SHOW TABLES FROM datasets" \
|
||||
&& clickhouse-client --query "SHOW TABLES FROM test" \
|
||||
&& clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" \
|
||||
&& clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" \
|
||||
&& clickhouse-client --query "SHOW TABLES FROM test" \
|
||||
&& clickhouse-test --shard --zookeeper --no-stateless $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
1
docker/test/stateful/listen.xml
Normal file
1
docker/test/stateful/listen.xml
Normal file
@ -0,0 +1 @@
|
||||
<yandex><listen_host>::</listen_host></yandex>
|
||||
7
docker/test/stateful/log_queries.xml
Normal file
7
docker/test/stateful/log_queries.xml
Normal file
@ -0,0 +1,7 @@
|
||||
<yandex>
|
||||
<profiles>
|
||||
<default>
|
||||
<log_queries>1</log_queries>
|
||||
</default>
|
||||
</profiles>
|
||||
</yandex>
|
||||
8
docker/test/stateful/part_log.xml
Normal file
8
docker/test/stateful/part_log.xml
Normal file
@ -0,0 +1,8 @@
|
||||
<yandex>
|
||||
<part_log>
|
||||
<database>system</database>
|
||||
<table>part_log</table>
|
||||
|
||||
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
|
||||
</part_log>
|
||||
</yandex>
|
||||
86
docker/test/stateful/s3downloader
Executable file
86
docker/test/stateful/s3downloader
Executable file
@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import sys
|
||||
import tarfile
|
||||
import logging
|
||||
import argparse
|
||||
import requests
|
||||
import tempfile
|
||||
|
||||
|
||||
DEFAULT_URL = 'https://clickhouse-datasets.s3.yandex.net'
|
||||
|
||||
AVAILABLE_DATASETS = {
|
||||
'hits': 'hits_v1.tar',
|
||||
'visits': 'visits_v1.tar',
|
||||
}
|
||||
|
||||
def _get_temp_file_name():
|
||||
return os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names()))
|
||||
|
||||
def build_url(base_url, dataset):
|
||||
return os.path.join(base_url, dataset, 'partitions', AVAILABLE_DATASETS[dataset])
|
||||
|
||||
def dowload_with_progress(url, path):
|
||||
logging.info("Downloading from %s to temp path %s", url, path)
|
||||
with open(path, 'w') as f:
|
||||
response = requests.get(url, stream=True)
|
||||
response.raise_for_status()
|
||||
total_length = response.headers.get('content-length')
|
||||
if total_length is None or int(total_length) == 0:
|
||||
logging.info("No content-length, will download file without progress")
|
||||
f.write(response.content)
|
||||
else:
|
||||
dl = 0
|
||||
total_length = int(total_length)
|
||||
logging.info("Content length is %ld bytes", total_length)
|
||||
for data in response.iter_content(chunk_size=4096):
|
||||
dl += len(data)
|
||||
f.write(data)
|
||||
if sys.stdout.isatty():
|
||||
done = int(50 * dl / total_length)
|
||||
percent = int(100 * float(dl) / total_length)
|
||||
sys.stdout.write("\r[{}{}] {}%".format('=' * done, ' ' * (50-done), percent))
|
||||
sys.stdout.flush()
|
||||
sys.stdout.write("\n")
|
||||
logging.info("Downloading finished")
|
||||
|
||||
def unpack_to_clickhouse_directory(tar_path, clickhouse_path):
|
||||
logging.info("Will unpack data from temp path %s to clickhouse db %s", tar_path, clickhouse_path)
|
||||
with tarfile.open(tar_path, 'r') as comp_file:
|
||||
comp_file.extractall(path=clickhouse_path)
|
||||
logging.info("Unpack finished")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Simple tool for dowloading datasets for clickhouse from S3")
|
||||
|
||||
parser.add_argument('--dataset-names', required=True, nargs='+', choices=AVAILABLE_DATASETS.keys())
|
||||
parser.add_argument('--url-prefix', default=DEFAULT_URL)
|
||||
parser.add_argument('--clickhouse-data-path', default='/var/lib/clickhouse/')
|
||||
|
||||
args = parser.parse_args()
|
||||
datasets = args.dataset_names
|
||||
logging.info("Will fetch following datasets: %s", ', '.join(datasets))
|
||||
for dataset in datasets:
|
||||
logging.info("Processing %s", dataset)
|
||||
temp_archive_path = _get_temp_file_name()
|
||||
try:
|
||||
download_url_for_dataset = build_url(args.url_prefix, dataset)
|
||||
dowload_with_progress(download_url_for_dataset, temp_archive_path)
|
||||
unpack_to_clickhouse_directory(temp_archive_path, args.clickhouse_data_path)
|
||||
except Exception as ex:
|
||||
logging.info("Some exception occured %s", str(ex))
|
||||
raise
|
||||
finally:
|
||||
logging.info("Will remove dowloaded file %s from filesystem if it exists", temp_archive_path)
|
||||
if os.path.exists(temp_archive_path):
|
||||
os.remove(temp_archive_path)
|
||||
logging.info("Processing of %s finished", dataset)
|
||||
logging.info("Fetch finished, enjoy your tables!")
|
||||
|
||||
|
||||
16
docker/test/stateful/zookeeper.xml
Normal file
16
docker/test/stateful/zookeeper.xml
Normal file
@ -0,0 +1,16 @@
|
||||
<yandex>
|
||||
<zookeeper>
|
||||
<node>
|
||||
<host>localhost</host>
|
||||
<port>2181</port>
|
||||
</node>
|
||||
<node>
|
||||
<host>yandex.ru</host>
|
||||
<port>2181</port>
|
||||
</node>
|
||||
<node>
|
||||
<host>111.0.1.2</host>
|
||||
<port>2181</port>
|
||||
</node>
|
||||
</zookeeper>
|
||||
</yandex>
|
||||
Loading…
Reference in New Issue
Block a user