mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-29 21:20:49 +00:00
Merge branch 'nikvas0/index' into nikvas0/parse_index
This commit is contained in:
commit
2a76e86338
11
.gitignore
vendored
11
.gitignore
vendored
@ -180,18 +180,10 @@ utils/zookeeper-create-entry-to-download-part/zookeeper-create-entry-to-download
|
|||||||
utils/zookeeper-dump-tree/zookeeper-dump-tree
|
utils/zookeeper-dump-tree/zookeeper-dump-tree
|
||||||
utils/zookeeper-remove-by-list/zookeeper-remove-by-list
|
utils/zookeeper-remove-by-list/zookeeper-remove-by-list
|
||||||
dbms/src/Storages/tests/remove_symlink_directory
|
dbms/src/Storages/tests/remove_symlink_directory
|
||||||
debian/control
|
|
||||||
debian/copyright
|
|
||||||
debian/tmp/
|
|
||||||
libs/libcommon/src/tests/json_test
|
libs/libcommon/src/tests/json_test
|
||||||
utils/compressor/zstd_test
|
utils/compressor/zstd_test
|
||||||
utils/wikistat-loader/wikistat-loader
|
utils/wikistat-loader/wikistat-loader
|
||||||
dbms/src/Common/tests/pod_array
|
dbms/src/Common/tests/pod_array
|
||||||
debian/clickhouse-benchmark/
|
|
||||||
debian/clickhouse-client/
|
|
||||||
debian/clickhouse-server-base/
|
|
||||||
debian/clickhouse-server-common/
|
|
||||||
debian/files
|
|
||||||
|
|
||||||
dbms/src/Server/data/*
|
dbms/src/Server/data/*
|
||||||
dbms/src/Server/metadata/*
|
dbms/src/Server/metadata/*
|
||||||
@ -210,9 +202,6 @@ vgcore*
|
|||||||
*.changes
|
*.changes
|
||||||
build-stamp
|
build-stamp
|
||||||
configure-stamp
|
configure-stamp
|
||||||
debian/*.debhelper.log
|
|
||||||
debian/*.debhelper
|
|
||||||
debian/*.substvars
|
|
||||||
|
|
||||||
*.bin
|
*.bin
|
||||||
*.mrk
|
*.mrk
|
||||||
|
@ -1,50 +0,0 @@
|
|||||||
language: generic
|
|
||||||
|
|
||||||
matrix:
|
|
||||||
fast_finish: true
|
|
||||||
include:
|
|
||||||
# We need to have gcc7 headers to compile c++17 code on clang
|
|
||||||
# - os: linux
|
|
||||||
#
|
|
||||||
# cache:
|
|
||||||
# ccache: true
|
|
||||||
# timeout: 1000
|
|
||||||
# directories:
|
|
||||||
# - /home/travis/.ccache
|
|
||||||
#
|
|
||||||
# addons:
|
|
||||||
# apt:
|
|
||||||
# update: true
|
|
||||||
# sources:
|
|
||||||
# - ubuntu-toolchain-r-test
|
|
||||||
# - llvm-toolchain-trusty-5.0
|
|
||||||
# packages: [ ninja-build, g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl]
|
|
||||||
#
|
|
||||||
# env:
|
|
||||||
# - MATRIX_EVAL="export CC=clang-5.0 CXX=clang++-5.0"
|
|
||||||
#
|
|
||||||
# script:
|
|
||||||
# - utils/travis/normal.sh
|
|
||||||
|
|
||||||
- os: linux
|
|
||||||
|
|
||||||
sudo: required
|
|
||||||
|
|
||||||
cache:
|
|
||||||
timeout: 1000
|
|
||||||
directories:
|
|
||||||
- /var/cache/pbuilder/ccache
|
|
||||||
|
|
||||||
addons:
|
|
||||||
apt:
|
|
||||||
update: true
|
|
||||||
packages: [ pbuilder, fakeroot, debhelper ]
|
|
||||||
|
|
||||||
script:
|
|
||||||
- utils/travis/pbuilder.sh
|
|
||||||
|
|
||||||
allow_failures:
|
|
||||||
- os: osx
|
|
||||||
|
|
||||||
before_script:
|
|
||||||
- eval "${MATRIX_EVAL}"
|
|
@ -2,10 +2,10 @@
|
|||||||
set(VERSION_REVISION 54413)
|
set(VERSION_REVISION 54413)
|
||||||
set(VERSION_MAJOR 19)
|
set(VERSION_MAJOR 19)
|
||||||
set(VERSION_MINOR 1)
|
set(VERSION_MINOR 1)
|
||||||
set(VERSION_PATCH 1)
|
set(VERSION_PATCH 3)
|
||||||
set(VERSION_GITHASH 4e7747117123f5a1b027a64865844b4faa10447d)
|
set(VERSION_GITHASH ac0060079ab221278338db343ca9eaf006fc4ee1)
|
||||||
set(VERSION_DESCRIBE v19.1.1-testing)
|
set(VERSION_DESCRIBE v19.1.3-testing)
|
||||||
set(VERSION_STRING 19.1.1)
|
set(VERSION_STRING 19.1.3)
|
||||||
# end of autochange
|
# end of autochange
|
||||||
|
|
||||||
set(VERSION_EXTRA "" CACHE STRING "")
|
set(VERSION_EXTRA "" CACHE STRING "")
|
||||||
|
@ -186,10 +186,10 @@ SharedLibraryPtr Compiler::getOrCount(
|
|||||||
static void addCodeToAssertHeadersMatch(WriteBuffer & out)
|
static void addCodeToAssertHeadersMatch(WriteBuffer & out)
|
||||||
{
|
{
|
||||||
out <<
|
out <<
|
||||||
"#define STRING2(x) #x\n"
|
|
||||||
"#define STRING(x) STRING2(x)\n"
|
|
||||||
"#include <Common/config_version.h>\n"
|
"#include <Common/config_version.h>\n"
|
||||||
"#if VERSION_REVISION != " << ClickHouseRevision::get() << "\n"
|
"#if VERSION_REVISION != " << ClickHouseRevision::get() << "\n"
|
||||||
|
"#define STRING2(x) #x\n"
|
||||||
|
"#define STRING(x) STRING2(x)\n"
|
||||||
"#pragma message \"ClickHouse headers revision = \" STRING(VERSION_REVISION) \n"
|
"#pragma message \"ClickHouse headers revision = \" STRING(VERSION_REVISION) \n"
|
||||||
"#error \"ClickHouse headers revision doesn't match runtime revision of the server (" << ClickHouseRevision::get() << ").\"\n"
|
"#error \"ClickHouse headers revision doesn't match runtime revision of the server (" << ClickHouseRevision::get() << ").\"\n"
|
||||||
"#endif\n\n";
|
"#endif\n\n";
|
||||||
|
@ -136,7 +136,7 @@ void NO_INLINE Aggregator::executeSpecializedCase(
|
|||||||
AggregateDataPtr overflow_row) const
|
AggregateDataPtr overflow_row) const
|
||||||
{
|
{
|
||||||
/// For all rows.
|
/// For all rows.
|
||||||
typename Method::Key prev_key;
|
typename Method::Key prev_key{};
|
||||||
AggregateDataPtr value = nullptr;
|
AggregateDataPtr value = nullptr;
|
||||||
for (size_t i = 0; i < rows; ++i)
|
for (size_t i = 0; i < rows; ++i)
|
||||||
{
|
{
|
||||||
|
12
debian/.gitignore
vendored
Normal file
12
debian/.gitignore
vendored
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
control
|
||||||
|
copyright
|
||||||
|
tmp/
|
||||||
|
clickhouse-benchmark/
|
||||||
|
clickhouse-client/
|
||||||
|
clickhouse-server-base/
|
||||||
|
clickhouse-server-common/
|
||||||
|
files
|
||||||
|
*.debhelper.log
|
||||||
|
*.debhelper
|
||||||
|
*.substvars
|
||||||
|
|
4
debian/changelog
vendored
4
debian/changelog
vendored
@ -1,5 +1,5 @@
|
|||||||
clickhouse (19.1.1) unstable; urgency=low
|
clickhouse (19.1.3) unstable; urgency=low
|
||||||
|
|
||||||
* Modified source code
|
* Modified source code
|
||||||
|
|
||||||
-- <root@yandex-team.ru> Wed, 16 Jan 2019 14:04:37 +0300
|
-- <root@yandex-team.ru> Mon, 21 Jan 2019 16:26:13 +0300
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
FROM ubuntu:18.04
|
FROM ubuntu:18.04
|
||||||
|
|
||||||
ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
|
ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
|
||||||
ARG version=19.1.1
|
ARG version=19.1.3
|
||||||
|
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install --yes --no-install-recommends \
|
&& apt-get install --yes --no-install-recommends \
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
FROM ubuntu:18.04
|
FROM ubuntu:18.04
|
||||||
|
|
||||||
ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
|
ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
|
||||||
ARG version=19.1.1
|
ARG version=19.1.3
|
||||||
ARG gosu_ver=1.10
|
ARG gosu_ver=1.10
|
||||||
|
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
FROM ubuntu:18.04
|
FROM ubuntu:18.04
|
||||||
|
|
||||||
ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
|
ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/"
|
||||||
ARG version=19.1.1
|
ARG version=19.1.3
|
||||||
|
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
apt-get install -y apt-transport-https dirmngr && \
|
apt-get install -y apt-transport-https dirmngr && \
|
||||||
|
29
docker/test/stateful/Dockerfile
Normal file
29
docker/test/stateful/Dockerfile
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
FROM yandex/clickhouse-stateless-test
|
||||||
|
|
||||||
|
RUN apt-get update -y \
|
||||||
|
&& env DEBIAN_FRONTEND=noninteractive \
|
||||||
|
apt-get install --yes --no-install-recommends \
|
||||||
|
python-requests
|
||||||
|
|
||||||
|
COPY s3downloader /s3downloader
|
||||||
|
|
||||||
|
ENV DATASETS="hits visits"
|
||||||
|
|
||||||
|
CMD dpkg -i package_folder/clickhouse-common-static_*.deb; \
|
||||||
|
dpkg -i package_folder/clickhouse-server_*.deb; \
|
||||||
|
dpkg -i package_folder/clickhouse-client_*.deb; \
|
||||||
|
dpkg -i package_folder/clickhouse-test_*.deb; \
|
||||||
|
service zookeeper start; sleep 5; \
|
||||||
|
service clickhouse-server start && sleep 5 \
|
||||||
|
&& /s3downloader --dataset-names $DATASETS \
|
||||||
|
&& chmod 777 -R /var/lib/clickhouse \
|
||||||
|
&& clickhouse-client --query "SHOW DATABASES" \
|
||||||
|
&& clickhouse-client --query "CREATE DATABASE datasets" \
|
||||||
|
&& clickhouse-client --query "CREATE DATABASE test" \
|
||||||
|
&& service clickhouse-server restart && sleep 5 \
|
||||||
|
&& clickhouse-client --query "SHOW TABLES FROM datasets" \
|
||||||
|
&& clickhouse-client --query "SHOW TABLES FROM test" \
|
||||||
|
&& clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" \
|
||||||
|
&& clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" \
|
||||||
|
&& clickhouse-client --query "SHOW TABLES FROM test" \
|
||||||
|
&& clickhouse-test --shard --zookeeper --no-stateless $SKIP_TESTS_OPTION 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
1
docker/test/stateful/listen.xml
Normal file
1
docker/test/stateful/listen.xml
Normal file
@ -0,0 +1 @@
|
|||||||
|
<yandex><listen_host>::</listen_host></yandex>
|
7
docker/test/stateful/log_queries.xml
Normal file
7
docker/test/stateful/log_queries.xml
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
<yandex>
|
||||||
|
<profiles>
|
||||||
|
<default>
|
||||||
|
<log_queries>1</log_queries>
|
||||||
|
</default>
|
||||||
|
</profiles>
|
||||||
|
</yandex>
|
8
docker/test/stateful/part_log.xml
Normal file
8
docker/test/stateful/part_log.xml
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
<yandex>
|
||||||
|
<part_log>
|
||||||
|
<database>system</database>
|
||||||
|
<table>part_log</table>
|
||||||
|
|
||||||
|
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
|
||||||
|
</part_log>
|
||||||
|
</yandex>
|
86
docker/test/stateful/s3downloader
Executable file
86
docker/test/stateful/s3downloader
Executable file
@ -0,0 +1,86 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import logging
|
||||||
|
import argparse
|
||||||
|
import requests
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_URL = 'https://clickhouse-datasets.s3.yandex.net'
|
||||||
|
|
||||||
|
AVAILABLE_DATASETS = {
|
||||||
|
'hits': 'hits_v1.tar',
|
||||||
|
'visits': 'visits_v1.tar',
|
||||||
|
}
|
||||||
|
|
||||||
|
def _get_temp_file_name():
|
||||||
|
return os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names()))
|
||||||
|
|
||||||
|
def build_url(base_url, dataset):
|
||||||
|
return os.path.join(base_url, dataset, 'partitions', AVAILABLE_DATASETS[dataset])
|
||||||
|
|
||||||
|
def dowload_with_progress(url, path):
|
||||||
|
logging.info("Downloading from %s to temp path %s", url, path)
|
||||||
|
with open(path, 'w') as f:
|
||||||
|
response = requests.get(url, stream=True)
|
||||||
|
response.raise_for_status()
|
||||||
|
total_length = response.headers.get('content-length')
|
||||||
|
if total_length is None or int(total_length) == 0:
|
||||||
|
logging.info("No content-length, will download file without progress")
|
||||||
|
f.write(response.content)
|
||||||
|
else:
|
||||||
|
dl = 0
|
||||||
|
total_length = int(total_length)
|
||||||
|
logging.info("Content length is %ld bytes", total_length)
|
||||||
|
for data in response.iter_content(chunk_size=4096):
|
||||||
|
dl += len(data)
|
||||||
|
f.write(data)
|
||||||
|
if sys.stdout.isatty():
|
||||||
|
done = int(50 * dl / total_length)
|
||||||
|
percent = int(100 * float(dl) / total_length)
|
||||||
|
sys.stdout.write("\r[{}{}] {}%".format('=' * done, ' ' * (50-done), percent))
|
||||||
|
sys.stdout.flush()
|
||||||
|
sys.stdout.write("\n")
|
||||||
|
logging.info("Downloading finished")
|
||||||
|
|
||||||
|
def unpack_to_clickhouse_directory(tar_path, clickhouse_path):
|
||||||
|
logging.info("Will unpack data from temp path %s to clickhouse db %s", tar_path, clickhouse_path)
|
||||||
|
with tarfile.open(tar_path, 'r') as comp_file:
|
||||||
|
comp_file.extractall(path=clickhouse_path)
|
||||||
|
logging.info("Unpack finished")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Simple tool for dowloading datasets for clickhouse from S3")
|
||||||
|
|
||||||
|
parser.add_argument('--dataset-names', required=True, nargs='+', choices=AVAILABLE_DATASETS.keys())
|
||||||
|
parser.add_argument('--url-prefix', default=DEFAULT_URL)
|
||||||
|
parser.add_argument('--clickhouse-data-path', default='/var/lib/clickhouse/')
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
datasets = args.dataset_names
|
||||||
|
logging.info("Will fetch following datasets: %s", ', '.join(datasets))
|
||||||
|
for dataset in datasets:
|
||||||
|
logging.info("Processing %s", dataset)
|
||||||
|
temp_archive_path = _get_temp_file_name()
|
||||||
|
try:
|
||||||
|
download_url_for_dataset = build_url(args.url_prefix, dataset)
|
||||||
|
dowload_with_progress(download_url_for_dataset, temp_archive_path)
|
||||||
|
unpack_to_clickhouse_directory(temp_archive_path, args.clickhouse_data_path)
|
||||||
|
except Exception as ex:
|
||||||
|
logging.info("Some exception occured %s", str(ex))
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
logging.info("Will remove dowloaded file %s from filesystem if it exists", temp_archive_path)
|
||||||
|
if os.path.exists(temp_archive_path):
|
||||||
|
os.remove(temp_archive_path)
|
||||||
|
logging.info("Processing of %s finished", dataset)
|
||||||
|
logging.info("Fetch finished, enjoy your tables!")
|
||||||
|
|
||||||
|
|
16
docker/test/stateful/zookeeper.xml
Normal file
16
docker/test/stateful/zookeeper.xml
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
<yandex>
|
||||||
|
<zookeeper>
|
||||||
|
<node>
|
||||||
|
<host>localhost</host>
|
||||||
|
<port>2181</port>
|
||||||
|
</node>
|
||||||
|
<node>
|
||||||
|
<host>yandex.ru</host>
|
||||||
|
<port>2181</port>
|
||||||
|
</node>
|
||||||
|
<node>
|
||||||
|
<host>111.0.1.2</host>
|
||||||
|
<port>2181</port>
|
||||||
|
</node>
|
||||||
|
</zookeeper>
|
||||||
|
</yandex>
|
Loading…
Reference in New Issue
Block a user