mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-27 01:51:59 +00:00
Merge branch 'master' into async-inserts-log
This commit is contained in:
Anton Popov
GitHub
commit
9a2cabd3a1
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -284,3 +284,6 @@
|
||||
[submodule "contrib/llvm-project"]
|
||||
path = contrib/llvm-project
|
||||
url = https://github.com/ClickHouse/llvm-project.git
|
||||
[submodule "contrib/corrosion"]
|
||||
path = contrib/corrosion
|
||||
url = https://github.com/corrosion-rs/corrosion.git
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
cmake_minimum_required(VERSION 3.15)
|
||||
cmake_minimum_required(VERSION 3.20)
|
||||
|
||||
project(ClickHouse LANGUAGES C CXX ASM)
|
||||
|
||||
@ -557,9 +557,9 @@ macro (clickhouse_add_executable target)
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
# With cross-compiling, all targets are built for the target platform which usually different from the host
|
||||
# platform. This is problematic if a build artifact X (e.g. a file or an executable) is generated by running
|
||||
# another executable Y previously produced in the build. This is solved by compiling and running Y for/on
|
||||
# With cross-compiling, all targets are built for the target platform which usually different from the host
|
||||
# platform. This is problematic if a build artifact X (e.g. a file or an executable) is generated by running
|
||||
# another executable Y previously produced in the build. This is solved by compiling and running Y for/on
|
||||
# the host platform. Add target to the list:
|
||||
# add_native_target(<target> ...)
|
||||
set_property (GLOBAL PROPERTY NATIVE_BUILD_TARGETS)
|
||||
@ -574,6 +574,10 @@ include_directories(${ConfigIncludePath})
|
||||
include (cmake/warnings.cmake)
|
||||
include (cmake/print_flags.cmake)
|
||||
|
||||
if (ENABLE_RUST)
|
||||
add_subdirectory (rust)
|
||||
endif()
|
||||
|
||||
add_subdirectory (base)
|
||||
add_subdirectory (src)
|
||||
add_subdirectory (programs)
|
||||
@ -584,7 +588,7 @@ include (cmake/sanitize_target_link_libraries.cmake)
|
||||
|
||||
# Build native targets if necessary
|
||||
get_property(NATIVE_BUILD_TARGETS GLOBAL PROPERTY NATIVE_BUILD_TARGETS)
|
||||
if (NATIVE_BUILD_TARGETS
|
||||
if (NATIVE_BUILD_TARGETS
|
||||
AND NOT(
|
||||
CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME
|
||||
AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR
|
||||
|
||||
@ -176,6 +176,249 @@ void __explicit_bzero_chk(void * buf, size_t len, size_t unused)
|
||||
}
|
||||
|
||||
|
||||
#include <unistd.h>
|
||||
#include "syscall.h"
|
||||
|
||||
ssize_t copy_file_range(int fd_in, off_t *off_in, int fd_out, off_t *off_out, size_t len, unsigned flags)
|
||||
{
|
||||
return syscall(SYS_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags);
|
||||
}
|
||||
|
||||
|
||||
long splice(int fd_in, off_t *off_in, int fd_out, off_t *off_out, size_t len, unsigned flags)
|
||||
{
|
||||
return syscall(SYS_splice, fd_in, off_in, fd_out, off_out, len, flags);
|
||||
}
|
||||
|
||||
|
||||
#define _BSD_SOURCE
|
||||
#include <sys/stat.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#if !defined(__aarch64__)
|
||||
struct statx {
|
||||
uint32_t stx_mask;
|
||||
uint32_t stx_blksize;
|
||||
uint64_t stx_attributes;
|
||||
uint32_t stx_nlink;
|
||||
uint32_t stx_uid;
|
||||
uint32_t stx_gid;
|
||||
uint16_t stx_mode;
|
||||
uint16_t pad1;
|
||||
uint64_t stx_ino;
|
||||
uint64_t stx_size;
|
||||
uint64_t stx_blocks;
|
||||
uint64_t stx_attributes_mask;
|
||||
struct {
|
||||
int64_t tv_sec;
|
||||
uint32_t tv_nsec;
|
||||
int32_t pad;
|
||||
} stx_atime, stx_btime, stx_ctime, stx_mtime;
|
||||
uint32_t stx_rdev_major;
|
||||
uint32_t stx_rdev_minor;
|
||||
uint32_t stx_dev_major;
|
||||
uint32_t stx_dev_minor;
|
||||
uint64_t spare[14];
|
||||
};
|
||||
#endif
|
||||
|
||||
int statx(int fd, const char *restrict path, int flag,
|
||||
unsigned int mask, struct statx *restrict statxbuf)
|
||||
{
|
||||
return syscall(SYS_statx, fd, path, flag, mask, statxbuf);
|
||||
}
|
||||
|
||||
|
||||
#include <syscall.h>
|
||||
|
||||
ssize_t getrandom(void *buf, size_t buflen, unsigned flags)
|
||||
{
|
||||
/// There was cancellable syscall (syscall_cp), but I don't care too.
|
||||
return syscall(SYS_getrandom, buf, buflen, flags);
|
||||
}
|
||||
|
||||
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
|
||||
#define ALIGN (sizeof(size_t))
|
||||
#define ONES ((size_t)-1/UCHAR_MAX)
|
||||
#define HIGHS (ONES * (UCHAR_MAX/2+1))
|
||||
#define HASZERO(x) ((x)-ONES & ~(x) & HIGHS)
|
||||
|
||||
char *__strchrnul(const char *s, int c)
|
||||
{
|
||||
c = (unsigned char)c;
|
||||
if (!c) return (char *)s + strlen(s);
|
||||
|
||||
#ifdef __GNUC__
|
||||
typedef size_t __attribute__((__may_alias__)) word;
|
||||
const word *w;
|
||||
for (; (uintptr_t)s % ALIGN; s++)
|
||||
if (!*s || *(unsigned char *)s == c) return (char *)s;
|
||||
size_t k = ONES * c;
|
||||
for (w = (void *)s; !HASZERO(*w) && !HASZERO(*w^k); w++);
|
||||
s = (void *)w;
|
||||
#endif
|
||||
for (; *s && *(unsigned char *)s != c; s++);
|
||||
return (char *)s;
|
||||
}
|
||||
|
||||
int __execvpe(const char *file, char *const argv[], char *const envp[])
|
||||
{
|
||||
const char *p, *z, *path = getenv("PATH");
|
||||
size_t l, k;
|
||||
int seen_eacces = 0;
|
||||
|
||||
errno = ENOENT;
|
||||
if (!*file) return -1;
|
||||
|
||||
if (strchr(file, '/'))
|
||||
return execve(file, argv, envp);
|
||||
|
||||
if (!path) path = "/usr/local/bin:/bin:/usr/bin";
|
||||
k = strnlen(file, NAME_MAX+1);
|
||||
if (k > NAME_MAX) {
|
||||
errno = ENAMETOOLONG;
|
||||
return -1;
|
||||
}
|
||||
l = strnlen(path, PATH_MAX-1)+1;
|
||||
|
||||
for(p=path; ; p=z) {
|
||||
char b[l+k+1];
|
||||
z = __strchrnul(p, ':');
|
||||
if (z-p >= l) {
|
||||
if (!*z++) break;
|
||||
continue;
|
||||
}
|
||||
memcpy(b, p, z-p);
|
||||
b[z-p] = '/';
|
||||
memcpy(b+(z-p)+(z>p), file, k+1);
|
||||
execve(b, argv, envp);
|
||||
switch (errno) {
|
||||
case EACCES:
|
||||
seen_eacces = 1;
|
||||
case ENOENT:
|
||||
case ENOTDIR:
|
||||
break;
|
||||
default:
|
||||
return -1;
|
||||
}
|
||||
if (!*z++) break;
|
||||
}
|
||||
if (seen_eacces) errno = EACCES;
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
#include "spawn.h"
|
||||
|
||||
int posix_spawnp(pid_t *restrict res, const char *restrict file,
|
||||
const posix_spawn_file_actions_t *fa,
|
||||
const posix_spawnattr_t *restrict attr,
|
||||
char *const argv[restrict], char *const envp[restrict])
|
||||
{
|
||||
posix_spawnattr_t spawnp_attr = { 0 };
|
||||
if (attr) spawnp_attr = *attr;
|
||||
spawnp_attr.__fn = (void *)__execvpe;
|
||||
return posix_spawn(res, file, fa, &spawnp_attr, argv, envp);
|
||||
}
|
||||
|
||||
#define FDOP_CLOSE 1
|
||||
#define FDOP_DUP2 2
|
||||
#define FDOP_OPEN 3
|
||||
#define FDOP_CHDIR 4
|
||||
#define FDOP_FCHDIR 5
|
||||
|
||||
#define ENOMEM 12
|
||||
#define EBADF 9
|
||||
|
||||
struct fdop {
|
||||
struct fdop *next, *prev;
|
||||
int cmd, fd, srcfd, oflag;
|
||||
mode_t mode;
|
||||
char path[];
|
||||
};
|
||||
|
||||
int posix_spawn_file_actions_init(posix_spawn_file_actions_t *fa) {
|
||||
fa->__actions = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int posix_spawn_file_actions_addchdir_np(posix_spawn_file_actions_t *restrict fa, const char *restrict path) {
|
||||
struct fdop *op = malloc(sizeof *op + strlen(path) + 1);
|
||||
if (!op) return ENOMEM;
|
||||
op->cmd = FDOP_CHDIR;
|
||||
op->fd = -1;
|
||||
strcpy(op->path, path);
|
||||
if ((op->next = fa->__actions)) op->next->prev = op;
|
||||
op->prev = 0;
|
||||
fa->__actions = op;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t *fa, int fd) {
|
||||
if (fd < 0) return EBADF;
|
||||
struct fdop *op = malloc(sizeof *op);
|
||||
if (!op) return ENOMEM;
|
||||
op->cmd = FDOP_CLOSE;
|
||||
op->fd = fd;
|
||||
if ((op->next = fa->__actions)) op->next->prev = op;
|
||||
op->prev = 0;
|
||||
fa->__actions = op;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int posix_spawn_file_actions_adddup2(posix_spawn_file_actions_t *fa, int srcfd, int fd) {
|
||||
if (srcfd < 0 || fd < 0) return EBADF;
|
||||
struct fdop *op = malloc(sizeof *op);
|
||||
if (!op) return ENOMEM;
|
||||
op->cmd = FDOP_DUP2;
|
||||
op->srcfd = srcfd;
|
||||
op->fd = fd;
|
||||
if ((op->next = fa->__actions)) op->next->prev = op;
|
||||
op->prev = 0;
|
||||
fa->__actions = op;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int posix_spawn_file_actions_addfchdir_np(posix_spawn_file_actions_t *fa, int fd) {
|
||||
if (fd < 0) return EBADF;
|
||||
struct fdop *op = malloc(sizeof *op);
|
||||
if (!op) return ENOMEM;
|
||||
op->cmd = FDOP_FCHDIR;
|
||||
op->fd = fd;
|
||||
if ((op->next = fa->__actions)) op->next->prev = op;
|
||||
op->prev = 0;
|
||||
fa->__actions = op;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int posix_spawn_file_actions_addopen(posix_spawn_file_actions_t *restrict fa, int fd, const char *restrict path, int flags, mode_t mode) {
|
||||
if (fd < 0) return EBADF;
|
||||
struct fdop *op = malloc(sizeof *op + strlen(path) + 1);
|
||||
if (!op) return ENOMEM;
|
||||
op->cmd = FDOP_OPEN;
|
||||
op->fd = fd;
|
||||
op->oflag = flags;
|
||||
op->mode = mode;
|
||||
strcpy(op->path, path);
|
||||
if ((op->next = fa->__actions)) op->next->prev = op;
|
||||
op->prev = 0;
|
||||
fa->__actions = op;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int posix_spawn_file_actions_destroy(posix_spawn_file_actions_t *fa) {
|
||||
struct fdop *op = fa->__actions, *next;
|
||||
while (op) {
|
||||
next = op->next;
|
||||
free(op);
|
||||
op = next;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
32
base/glibc-compatibility/spawn.h
Normal file
32
base/glibc-compatibility/spawn.h
Normal file
@ -0,0 +1,32 @@
|
||||
#ifndef _SPAWN_H
|
||||
#define _SPAWN_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <features.h>
|
||||
|
||||
typedef struct {
|
||||
int __flags;
|
||||
pid_t __pgrp;
|
||||
sigset_t __def, __mask;
|
||||
int __prio, __pol;
|
||||
void *__fn;
|
||||
char __pad[64-sizeof(void *)];
|
||||
} posix_spawnattr_t;
|
||||
|
||||
typedef struct {
|
||||
int __pad0[2];
|
||||
void *__actions;
|
||||
int __pad[16];
|
||||
} posix_spawn_file_actions_t;
|
||||
|
||||
int posix_spawn(pid_t *__restrict, const char *__restrict, const posix_spawn_file_actions_t *,
|
||||
const posix_spawnattr_t *__restrict, char *const *__restrict, char *const *__restrict);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
2
contrib/CMakeLists.txt
vendored
2
contrib/CMakeLists.txt
vendored
@ -92,6 +92,8 @@ add_contrib (openldap-cmake openldap)
|
||||
add_contrib (grpc-cmake grpc)
|
||||
add_contrib (msgpack-c-cmake msgpack-c)
|
||||
|
||||
add_contrib (corrosion-cmake corrosion)
|
||||
|
||||
if (ENABLE_FUZZING)
|
||||
add_contrib (libprotobuf-mutator-cmake libprotobuf-mutator)
|
||||
endif()
|
||||
|
||||
1
contrib/corrosion
vendored
Submodule
1
contrib/corrosion
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit d9dfdefaa3d9ec4ba1245c7070727359c65c7869
|
||||
46
contrib/corrosion-cmake/CMakeLists.txt
Normal file
46
contrib/corrosion-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,46 @@
|
||||
if (NOT ENABLE_LIBRARIES)
|
||||
set(DEFAULT_ENABLE_RUST FALSE)
|
||||
elseif((CMAKE_TOOLCHAIN_FILE MATCHES "darwin") AND (CMAKE_TOOLCHAIN_FILE MATCHES "aarch64"))
|
||||
message(STATUS "Rust is not available on aarch64-apple-darwin")
|
||||
set(DEFAULT_ENABLE_RUST FALSE)
|
||||
else()
|
||||
list (APPEND CMAKE_MODULE_PATH "${ClickHouse_SOURCE_DIR}/contrib/corrosion/cmake")
|
||||
find_package(Rust)
|
||||
set(DEFAULT_ENABLE_RUST ${Rust_FOUND})
|
||||
endif()
|
||||
|
||||
option(ENABLE_RUST "Enable rust" ${DEFAULT_ENABLE_RUST})
|
||||
|
||||
message(STATUS ${ENABLE_RUST})
|
||||
|
||||
if(NOT ENABLE_RUST)
|
||||
message(STATUS "Not using rust")
|
||||
return()
|
||||
endif()
|
||||
|
||||
message(STATUS "Checking Rust toolchain for current target")
|
||||
|
||||
if(CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64")
|
||||
set(Rust_CARGO_TARGET "x86_64-unknown-linux-gnu")
|
||||
endif()
|
||||
|
||||
if(CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-aarch64")
|
||||
set(Rust_CARGO_TARGET "aarch64-unknown-linux-gnu")
|
||||
endif()
|
||||
|
||||
if((CMAKE_TOOLCHAIN_FILE MATCHES "darwin") AND (CMAKE_TOOLCHAIN_FILE MATCHES "x86_64"))
|
||||
set(Rust_CARGO_TARGET "x86_64-apple-darwin")
|
||||
endif()
|
||||
|
||||
if((CMAKE_TOOLCHAIN_FILE MATCHES "freebsd") AND (CMAKE_TOOLCHAIN_FILE MATCHES "x86_64"))
|
||||
set(Rust_CARGO_TARGET "x86_64-unknown-freebsd")
|
||||
endif()
|
||||
|
||||
if(CMAKE_TOOLCHAIN_FILE MATCHES "ppc64le")
|
||||
set(Rust_CARGO_TARGET "powerpc64le-unknown-linux-gnu")
|
||||
endif()
|
||||
|
||||
message(STATUS "Switched Rust target to ${Rust_CARGO_TARGET}")
|
||||
|
||||
# Define function corrosion_import_crate()
|
||||
include ("${ClickHouse_SOURCE_DIR}/contrib/corrosion/cmake/Corrosion.cmake")
|
||||
@ -3,6 +3,33 @@
|
||||
ARG FROM_TAG=latest
|
||||
FROM clickhouse/test-util:$FROM_TAG
|
||||
|
||||
# Rust toolchain and libraries
|
||||
ENV RUSTUP_HOME=/rust/rustup
|
||||
ENV CARGO_HOME=/rust/cargo
|
||||
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
|
||||
RUN chmod 777 -R /rust
|
||||
ENV PATH="/rust/cargo/env:${PATH}"
|
||||
ENV PATH="/rust/cargo/bin:${PATH}"
|
||||
RUN rustup target add aarch64-unknown-linux-gnu && \
|
||||
rustup target add x86_64-apple-darwin && \
|
||||
rustup target add x86_64-unknown-freebsd && \
|
||||
rustup target add aarch64-apple-darwin && \
|
||||
rustup target add powerpc64le-unknown-linux-gnu
|
||||
RUN apt-get install \
|
||||
gcc-aarch64-linux-gnu \
|
||||
build-essential \
|
||||
libc6 \
|
||||
libc6-dev \
|
||||
libc6-dev-arm64-cross \
|
||||
--yes
|
||||
|
||||
# Install CMake 3.20+ for Rust compilation
|
||||
# Used https://askubuntu.com/a/1157132 as reference
|
||||
RUN apt purge cmake --yes
|
||||
RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
|
||||
RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main'
|
||||
RUN apt update && apt install cmake --yes
|
||||
|
||||
ENV CC=clang-${LLVM_VERSION}
|
||||
ENV CXX=clang++-${LLVM_VERSION}
|
||||
|
||||
|
||||
@ -19,6 +19,12 @@ RUN apt-get update \
|
||||
pv \
|
||||
--yes --no-install-recommends
|
||||
|
||||
# Install CMake 3.20+ for Rust compilation
|
||||
RUN apt purge cmake --yes
|
||||
RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
|
||||
RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main'
|
||||
RUN apt update && apt install cmake --yes
|
||||
|
||||
RUN pip3 install numpy scipy pandas Jinja2
|
||||
|
||||
ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
|
||||
|
||||
@ -35,6 +35,8 @@ RUN apt-get update \
|
||||
tzdata \
|
||||
vim \
|
||||
wget \
|
||||
rustc \
|
||||
cargo \
|
||||
&& pip3 --no-cache-dir install 'clickhouse-driver==0.2.1' scipy \
|
||||
&& apt-get purge --yes python3-dev g++ \
|
||||
&& apt-get autoremove --yes \
|
||||
|
||||
@ -35,12 +35,13 @@ RUN apt-get update -y \
|
||||
tree \
|
||||
unixodbc \
|
||||
wget \
|
||||
rustc \
|
||||
cargo \
|
||||
zstd \
|
||||
file \
|
||||
pv \
|
||||
&& apt-get clean
|
||||
|
||||
|
||||
RUN pip3 install numpy scipy pandas Jinja2
|
||||
|
||||
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
|
||||
|
||||
@ -1,8 +1,7 @@
|
||||
position: 10
|
||||
position: 1
|
||||
label: 'Example Datasets'
|
||||
collapsible: true
|
||||
collapsed: true
|
||||
link:
|
||||
type: generated-index
|
||||
title: Example Datasets
|
||||
slug: /en/getting-started/example-datasets
|
||||
type: doc
|
||||
id: en/getting-started/example-datasets/
|
||||
|
||||
@ -1,9 +1,16 @@
|
||||
---
|
||||
slug: /en/getting-started/example-datasets/cell-towers
|
||||
sidebar_label: Cell Towers
|
||||
sidebar_position: 3
|
||||
title: "Cell Towers"
|
||||
---
|
||||
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import CodeBlock from '@theme/CodeBlock';
|
||||
import ActionsMenu from '@site/docs/en/_snippets/_service_actions_menu.md';
|
||||
import SQLConsoleDetail from '@site/docs/en/_snippets/_launch_sql_console.md';
|
||||
|
||||
This dataset is from [OpenCellid](https://www.opencellid.org/) - The world's largest Open Database of Cell Towers.
|
||||
|
||||
As of 2021, it contains more than 40 million records about cell towers (GSM, LTE, UMTS, etc.) around the world with their geographical coordinates and metadata (country code, network, etc).
|
||||
@ -13,6 +20,26 @@ OpenCelliD Project is licensed under a Creative Commons Attribution-ShareAlike 4
|
||||
|
||||
## Get the Dataset {#get-the-dataset}
|
||||
|
||||
<Tabs groupId="deployMethod">
|
||||
<TabItem value="serverless" label="ClickHouse Cloud" default>
|
||||
|
||||
ClickHouse Cloud provides an easy-button for uploading this dataset from S3. Log in to your ClickHouse Cloud organization, or create a free trial at [ClickHouse.cloud](https://clickhouse.cloud).
|
||||
<ActionsMenu menu="Load Data" />
|
||||
|
||||
Choose the **Cell Towers** dataset from the **Sample data** tab, and **Load data**:
|
||||
|
||||
|
||||
|
||||
Examine the schema of the cell_towers table:
|
||||
```sql
|
||||
DESCRIBE TABLE cell_towers
|
||||
```
|
||||
|
||||
<SQLConsoleDetail />
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="selfmanaged" label="Self-managed">
|
||||
|
||||
1. Download the snapshot of the dataset from February 2021: [cell_towers.csv.xz](https://datasets.clickhouse.com/cell_towers.csv.xz) (729 MB).
|
||||
|
||||
2. Validate the integrity (optional step):
|
||||
@ -56,7 +83,10 @@ ENGINE = MergeTree ORDER BY (radio, mcc, net, created);
|
||||
clickhouse-client --query "INSERT INTO cell_towers FORMAT CSVWithNames" < cell_towers.csv
|
||||
```
|
||||
|
||||
## Examples {#examples}
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Example queries {#examples}
|
||||
|
||||
1. A number of cell towers by type:
|
||||
|
||||
@ -101,18 +131,31 @@ So, the top countries are: the USA, Germany, and Russia.
|
||||
|
||||
You may want to create an [External Dictionary](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) in ClickHouse to decode these values.
|
||||
|
||||
|
||||
## Use case {#use-case}
|
||||
## Use case: Incorporate geo data {#use-case}
|
||||
|
||||
Using `pointInPolygon` function.
|
||||
|
||||
1. Create a table where we will store polygons:
|
||||
|
||||
<Tabs groupId="deployMethod">
|
||||
<TabItem value="serverless" label="ClickHouse Cloud" default>
|
||||
|
||||
```sql
|
||||
CREATE TABLE moscow (polygon Array(Tuple(Float64, Float64)))
|
||||
ORDER BY polygon;
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="selfmanaged" label="Self-managed">
|
||||
|
||||
```sql
|
||||
CREATE TEMPORARY TABLE
|
||||
moscow (polygon Array(Tuple(Float64, Float64)));
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
2. This is a rough shape of Moscow (without "new Moscow"):
|
||||
|
||||
```sql
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -13,16 +13,6 @@ Description of the fields: https://www.gov.uk/guidance/about-the-price-paid-data
|
||||
|
||||
Contains HM Land Registry data © Crown copyright and database right 2021. This data is licensed under the Open Government Licence v3.0.
|
||||
|
||||
## Download the Dataset {#download-dataset}
|
||||
|
||||
Run the command:
|
||||
|
||||
```bash
|
||||
wget http://prod.publicdata.landregistry.gov.uk.s3-website-eu-west-1.amazonaws.com/pp-complete.csv
|
||||
```
|
||||
|
||||
Download will take about 2 minutes with good internet connection.
|
||||
|
||||
## Create the Table {#create-table}
|
||||
|
||||
```sql
|
||||
@ -41,31 +31,49 @@ CREATE TABLE uk_price_paid
|
||||
locality LowCardinality(String),
|
||||
town LowCardinality(String),
|
||||
district LowCardinality(String),
|
||||
county LowCardinality(String),
|
||||
category UInt8
|
||||
) ENGINE = MergeTree ORDER BY (postcode1, postcode2, addr1, addr2);
|
||||
county LowCardinality(String)
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY (postcode1, postcode2, addr1, addr2);
|
||||
```
|
||||
|
||||
## Preprocess and Import Data {#preprocess-import-data}
|
||||
## Preprocess and Insert the Data {#preprocess-import-data}
|
||||
|
||||
We will use `clickhouse-local` tool for data preprocessing and `clickhouse-client` to upload it.
|
||||
We will use the `url` function to stream the data into ClickHouse. We need to preprocess some of the incoming data first, which includes:
|
||||
- splitting the `postcode` to two different columns - `postcode1` and `postcode2`, which is better for storage and queries
|
||||
- converting the `time` field to date as it only contains 00:00 time
|
||||
- ignoring the [UUid](../../sql-reference/data-types/uuid.md) field because we don't need it for analysis
|
||||
- transforming `type` and `duration` to more readable `Enum` fields using the [transform](../../sql-reference/functions/other-functions.md#transform) function
|
||||
- transforming the `is_new` field from a single-character string (`Y`/`N`) to a [UInt8](../../sql-reference/data-types/int-uint.md#uint8-uint16-uint32-uint64-uint256-int8-int16-int32-int64-int128-int256) field with 0 or 1
|
||||
- drop the last two columns since they all have the same value (which is 0)
|
||||
|
||||
In this example, we define the structure of source data from the CSV file and specify a query to preprocess the data with `clickhouse-local`.
|
||||
The `url` function streams the data from the web server into your ClickHouse table. The following command inserts 5 million rows into the `uk_price_paid` table:
|
||||
|
||||
The preprocessing is:
|
||||
- splitting the postcode to two different columns `postcode1` and `postcode2` that is better for storage and queries;
|
||||
- coverting the `time` field to date as it only contains 00:00 time;
|
||||
- ignoring the [UUid](../../sql-reference/data-types/uuid.md) field because we don't need it for analysis;
|
||||
- transforming `type` and `duration` to more readable Enum fields with function [transform](../../sql-reference/functions/other-functions.md#transform);
|
||||
- transforming `is_new` and `category` fields from single-character string (`Y`/`N` and `A`/`B`) to [UInt8](../../sql-reference/data-types/int-uint.md#uint8-uint16-uint32-uint64-uint256-int8-int16-int32-int64-int128-int256) field with 0 and 1.
|
||||
|
||||
Preprocessed data is piped directly to `clickhouse-client` to be inserted into ClickHouse table in streaming fashion.
|
||||
|
||||
```bash
|
||||
clickhouse-local --input-format CSV --structure '
|
||||
uuid String,
|
||||
price UInt32,
|
||||
time DateTime,
|
||||
```sql
|
||||
INSERT INTO uk_price_paid
|
||||
WITH
|
||||
splitByChar(' ', postcode) AS p
|
||||
SELECT
|
||||
toUInt32(price_string) AS price,
|
||||
parseDateTimeBestEffortUS(time) AS date,
|
||||
p[1] AS postcode1,
|
||||
p[2] AS postcode2,
|
||||
transform(a, ['T', 'S', 'D', 'F', 'O'], ['terraced', 'semi-detached', 'detached', 'flat', 'other']) AS type,
|
||||
b = 'Y' AS is_new,
|
||||
transform(c, ['F', 'L', 'U'], ['freehold', 'leasehold', 'unknown']) AS duration,
|
||||
addr1,
|
||||
addr2,
|
||||
street,
|
||||
locality,
|
||||
town,
|
||||
district,
|
||||
county
|
||||
FROM url(
|
||||
'http://prod.publicdata.landregistry.gov.uk.s3-website-eu-west-1.amazonaws.com/pp-complete.csv',
|
||||
'CSV',
|
||||
'uuid_string String,
|
||||
price_string String,
|
||||
time String,
|
||||
postcode String,
|
||||
a String,
|
||||
b String,
|
||||
@ -78,154 +86,136 @@ clickhouse-local --input-format CSV --structure '
|
||||
district String,
|
||||
county String,
|
||||
d String,
|
||||
e String
|
||||
' --query "
|
||||
WITH splitByChar(' ', postcode) AS p
|
||||
SELECT
|
||||
price,
|
||||
toDate(time) AS date,
|
||||
p[1] AS postcode1,
|
||||
p[2] AS postcode2,
|
||||
transform(a, ['T', 'S', 'D', 'F', 'O'], ['terraced', 'semi-detached', 'detached', 'flat', 'other']) AS type,
|
||||
b = 'Y' AS is_new,
|
||||
transform(c, ['F', 'L', 'U'], ['freehold', 'leasehold', 'unknown']) AS duration,
|
||||
addr1,
|
||||
addr2,
|
||||
street,
|
||||
locality,
|
||||
town,
|
||||
district,
|
||||
county,
|
||||
d = 'B' AS category
|
||||
FROM table" --date_time_input_format best_effort < pp-complete.csv | clickhouse-client --query "INSERT INTO uk_price_paid FORMAT TSV"
|
||||
e String'
|
||||
) SETTINGS max_http_get_redirects=10;
|
||||
```
|
||||
|
||||
It will take about 40 seconds.
|
||||
Wait for the data to insert - it will take a minute or two depending on the network speed.
|
||||
|
||||
## Validate the Data {#validate-data}
|
||||
|
||||
Query:
|
||||
Let's verify it worked by seeing how many rows were inserted:
|
||||
|
||||
```sql
|
||||
SELECT count() FROM uk_price_paid;
|
||||
SELECT count()
|
||||
FROM uk_price_paid
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌──count()─┐
|
||||
│ 26321785 │
|
||||
└──────────┘
|
||||
```
|
||||
|
||||
The size of dataset in ClickHouse is just 278 MiB, check it.
|
||||
|
||||
Query:
|
||||
At the time this query was executed, the dataset had 27,450,499 rows. Let's see what the storage size is of the table in ClickHouse:
|
||||
|
||||
```sql
|
||||
SELECT formatReadableSize(total_bytes) FROM system.tables WHERE name = 'uk_price_paid';
|
||||
SELECT formatReadableSize(total_bytes)
|
||||
FROM system.tables
|
||||
WHERE name = 'uk_price_paid'
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─formatReadableSize(total_bytes)─┐
|
||||
│ 278.80 MiB │
|
||||
└─────────────────────────────────┘
|
||||
```
|
||||
Notice the size of the table is just 221.43 MiB!
|
||||
|
||||
## Run Some Queries {#run-queries}
|
||||
|
||||
Let's run some queries to analyze the data:
|
||||
|
||||
### Query 1. Average Price Per Year {#average-price}
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT toYear(date) AS year, round(avg(price)) AS price, bar(price, 0, 1000000, 80) FROM uk_price_paid GROUP BY year ORDER BY year;
|
||||
SELECT
|
||||
toYear(date) AS year,
|
||||
round(avg(price)) AS price,
|
||||
bar(price, 0, 1000000, 80
|
||||
)
|
||||
FROM uk_price_paid
|
||||
GROUP BY year
|
||||
ORDER BY year
|
||||
```
|
||||
|
||||
Result:
|
||||
The result looks like:
|
||||
|
||||
```text
|
||||
```response
|
||||
┌─year─┬──price─┬─bar(round(avg(price)), 0, 1000000, 80)─┐
|
||||
│ 1995 │ 67932 │ █████▍ │
|
||||
│ 1996 │ 71505 │ █████▋ │
|
||||
│ 1997 │ 78532 │ ██████▎ │
|
||||
│ 1998 │ 85436 │ ██████▋ │
|
||||
│ 1999 │ 96037 │ ███████▋ │
|
||||
│ 2000 │ 107479 │ ████████▌ │
|
||||
│ 2001 │ 118885 │ █████████▌ │
|
||||
│ 2002 │ 137941 │ ███████████ │
|
||||
│ 2003 │ 155889 │ ████████████▍ │
|
||||
│ 2004 │ 178885 │ ██████████████▎ │
|
||||
│ 2005 │ 189351 │ ███████████████▏ │
|
||||
│ 2006 │ 203528 │ ████████████████▎ │
|
||||
│ 2007 │ 219378 │ █████████████████▌ │
|
||||
│ 1995 │ 67934 │ █████▍ │
|
||||
│ 1996 │ 71508 │ █████▋ │
|
||||
│ 1997 │ 78536 │ ██████▎ │
|
||||
│ 1998 │ 85441 │ ██████▋ │
|
||||
│ 1999 │ 96038 │ ███████▋ │
|
||||
│ 2000 │ 107487 │ ████████▌ │
|
||||
│ 2001 │ 118888 │ █████████▌ │
|
||||
│ 2002 │ 137948 │ ███████████ │
|
||||
│ 2003 │ 155893 │ ████████████▍ │
|
||||
│ 2004 │ 178888 │ ██████████████▎ │
|
||||
│ 2005 │ 189359 │ ███████████████▏ │
|
||||
│ 2006 │ 203532 │ ████████████████▎ │
|
||||
│ 2007 │ 219375 │ █████████████████▌ │
|
||||
│ 2008 │ 217056 │ █████████████████▎ │
|
||||
│ 2009 │ 213419 │ █████████████████ │
|
||||
│ 2010 │ 236109 │ ██████████████████▊ │
|
||||
│ 2010 │ 236110 │ ██████████████████▊ │
|
||||
│ 2011 │ 232805 │ ██████████████████▌ │
|
||||
│ 2012 │ 238367 │ ███████████████████ │
|
||||
│ 2013 │ 256931 │ ████████████████████▌ │
|
||||
│ 2014 │ 279915 │ ██████████████████████▍ │
|
||||
│ 2015 │ 297266 │ ███████████████████████▋ │
|
||||
│ 2016 │ 313201 │ █████████████████████████ │
|
||||
│ 2017 │ 346097 │ ███████████████████████████▋ │
|
||||
│ 2018 │ 350116 │ ████████████████████████████ │
|
||||
│ 2019 │ 351013 │ ████████████████████████████ │
|
||||
│ 2020 │ 369420 │ █████████████████████████████▌ │
|
||||
│ 2021 │ 386903 │ ██████████████████████████████▊ │
|
||||
│ 2012 │ 238381 │ ███████████████████ │
|
||||
│ 2013 │ 256927 │ ████████████████████▌ │
|
||||
│ 2014 │ 280008 │ ██████████████████████▍ │
|
||||
│ 2015 │ 297263 │ ███████████████████████▋ │
|
||||
│ 2016 │ 313518 │ █████████████████████████ │
|
||||
│ 2017 │ 346371 │ ███████████████████████████▋ │
|
||||
│ 2018 │ 350556 │ ████████████████████████████ │
|
||||
│ 2019 │ 352184 │ ████████████████████████████▏ │
|
||||
│ 2020 │ 375808 │ ██████████████████████████████ │
|
||||
│ 2021 │ 381105 │ ██████████████████████████████▍ │
|
||||
│ 2022 │ 362572 │ █████████████████████████████ │
|
||||
└──────┴────────┴────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Query 2. Average Price per Year in London {#average-price-london}
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT toYear(date) AS year, round(avg(price)) AS price, bar(price, 0, 2000000, 100) FROM uk_price_paid WHERE town = 'LONDON' GROUP BY year ORDER BY year;
|
||||
SELECT
|
||||
toYear(date) AS year,
|
||||
round(avg(price)) AS price,
|
||||
bar(price, 0, 2000000, 100
|
||||
)
|
||||
FROM uk_price_paid
|
||||
WHERE town = 'LONDON'
|
||||
GROUP BY year
|
||||
ORDER BY year
|
||||
```
|
||||
|
||||
Result:
|
||||
The result looks like:
|
||||
|
||||
```text
|
||||
```response
|
||||
┌─year─┬───price─┬─bar(round(avg(price)), 0, 2000000, 100)───────────────┐
|
||||
│ 1995 │ 109116 │ █████▍ │
|
||||
│ 1996 │ 118667 │ █████▊ │
|
||||
│ 1997 │ 136518 │ ██████▋ │
|
||||
│ 1998 │ 152983 │ ███████▋ │
|
||||
│ 1999 │ 180637 │ █████████ │
|
||||
│ 2000 │ 215838 │ ██████████▋ │
|
||||
│ 2001 │ 232994 │ ███████████▋ │
|
||||
│ 2002 │ 263670 │ █████████████▏ │
|
||||
│ 2003 │ 278394 │ █████████████▊ │
|
||||
│ 2004 │ 304666 │ ███████████████▏ │
|
||||
│ 2005 │ 322875 │ ████████████████▏ │
|
||||
│ 2006 │ 356191 │ █████████████████▋ │
|
||||
│ 2007 │ 404054 │ ████████████████████▏ │
|
||||
│ 1995 │ 109110 │ █████▍ │
|
||||
│ 1996 │ 118659 │ █████▊ │
|
||||
│ 1997 │ 136526 │ ██████▋ │
|
||||
│ 1998 │ 153002 │ ███████▋ │
|
||||
│ 1999 │ 180633 │ █████████ │
|
||||
│ 2000 │ 215849 │ ██████████▋ │
|
||||
│ 2001 │ 232987 │ ███████████▋ │
|
||||
│ 2002 │ 263668 │ █████████████▏ │
|
||||
│ 2003 │ 278424 │ █████████████▊ │
|
||||
│ 2004 │ 304664 │ ███████████████▏ │
|
||||
│ 2005 │ 322887 │ ████████████████▏ │
|
||||
│ 2006 │ 356195 │ █████████████████▋ │
|
||||
│ 2007 │ 404062 │ ████████████████████▏ │
|
||||
│ 2008 │ 420741 │ █████████████████████ │
|
||||
│ 2009 │ 427753 │ █████████████████████▍ │
|
||||
│ 2010 │ 480306 │ ████████████████████████ │
|
||||
│ 2011 │ 496274 │ ████████████████████████▋ │
|
||||
│ 2012 │ 519442 │ █████████████████████████▊ │
|
||||
│ 2013 │ 616212 │ ██████████████████████████████▋ │
|
||||
│ 2014 │ 724154 │ ████████████████████████████████████▏ │
|
||||
│ 2015 │ 792129 │ ███████████████████████████████████████▌ │
|
||||
│ 2016 │ 843655 │ ██████████████████████████████████████████▏ │
|
||||
│ 2017 │ 982642 │ █████████████████████████████████████████████████▏ │
|
||||
│ 2018 │ 1016835 │ ██████████████████████████████████████████████████▋ │
|
||||
│ 2019 │ 1042849 │ ████████████████████████████████████████████████████▏ │
|
||||
│ 2020 │ 1011889 │ ██████████████████████████████████████████████████▌ │
|
||||
│ 2021 │ 960343 │ ████████████████████████████████████████████████ │
|
||||
│ 2009 │ 427754 │ █████████████████████▍ │
|
||||
│ 2010 │ 480322 │ ████████████████████████ │
|
||||
│ 2011 │ 496278 │ ████████████████████████▋ │
|
||||
│ 2012 │ 519482 │ █████████████████████████▊ │
|
||||
│ 2013 │ 616195 │ ██████████████████████████████▋ │
|
||||
│ 2014 │ 724121 │ ████████████████████████████████████▏ │
|
||||
│ 2015 │ 792101 │ ███████████████████████████████████████▌ │
|
||||
│ 2016 │ 843589 │ ██████████████████████████████████████████▏ │
|
||||
│ 2017 │ 983523 │ █████████████████████████████████████████████████▏ │
|
||||
│ 2018 │ 1016753 │ ██████████████████████████████████████████████████▋ │
|
||||
│ 2019 │ 1041673 │ ████████████████████████████████████████████████████ │
|
||||
│ 2020 │ 1060027 │ █████████████████████████████████████████████████████ │
|
||||
│ 2021 │ 958249 │ ███████████████████████████████████████████████▊ │
|
||||
│ 2022 │ 902596 │ █████████████████████████████████████████████▏ │
|
||||
└──────┴─────────┴───────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Something happened in 2013. I don't have a clue. Maybe you have a clue what happened in 2020?
|
||||
Something happened to home prices in 2020! But that is probably not a surprise...
|
||||
|
||||
### Query 3. The Most Expensive Neighborhoods {#most-expensive-neighborhoods}
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
town,
|
||||
@ -240,124 +230,123 @@ GROUP BY
|
||||
district
|
||||
HAVING c >= 100
|
||||
ORDER BY price DESC
|
||||
LIMIT 100;
|
||||
LIMIT 100
|
||||
```
|
||||
|
||||
Result:
|
||||
The result looks like:
|
||||
|
||||
```text
|
||||
|
||||
┌─town─────────────────┬─district───────────────┬────c─┬───price─┬─bar(round(avg(price)), 0, 5000000, 100)────────────────────────────┐
|
||||
│ LONDON │ CITY OF WESTMINSTER │ 3606 │ 3280239 │ █████████████████████████████████████████████████████████████████▌ │
|
||||
│ LONDON │ CITY OF LONDON │ 274 │ 3160502 │ ███████████████████████████████████████████████████████████████▏ │
|
||||
│ LONDON │ KENSINGTON AND CHELSEA │ 2550 │ 2308478 │ ██████████████████████████████████████████████▏ │
|
||||
│ LEATHERHEAD │ ELMBRIDGE │ 114 │ 1897407 │ █████████████████████████████████████▊ │
|
||||
│ LONDON │ CAMDEN │ 3033 │ 1805404 │ ████████████████████████████████████ │
|
||||
│ VIRGINIA WATER │ RUNNYMEDE │ 156 │ 1753247 │ ███████████████████████████████████ │
|
||||
│ WINDLESHAM │ SURREY HEATH │ 108 │ 1677613 │ █████████████████████████████████▌ │
|
||||
│ THORNTON HEATH │ CROYDON │ 546 │ 1671721 │ █████████████████████████████████▍ │
|
||||
│ BARNET │ ENFIELD │ 124 │ 1505840 │ ██████████████████████████████ │
|
||||
│ COBHAM │ ELMBRIDGE │ 387 │ 1237250 │ ████████████████████████▋ │
|
||||
│ LONDON │ ISLINGTON │ 2668 │ 1236980 │ ████████████████████████▋ │
|
||||
│ OXFORD │ SOUTH OXFORDSHIRE │ 321 │ 1220907 │ ████████████████████████▍ │
|
||||
│ LONDON │ RICHMOND UPON THAMES │ 704 │ 1215551 │ ████████████████████████▎ │
|
||||
│ LONDON │ HOUNSLOW │ 671 │ 1207493 │ ████████████████████████▏ │
|
||||
│ ASCOT │ WINDSOR AND MAIDENHEAD │ 407 │ 1183299 │ ███████████████████████▋ │
|
||||
│ BEACONSFIELD │ BUCKINGHAMSHIRE │ 330 │ 1175615 │ ███████████████████████▌ │
|
||||
│ RICHMOND │ RICHMOND UPON THAMES │ 874 │ 1110444 │ ██████████████████████▏ │
|
||||
│ LONDON │ HAMMERSMITH AND FULHAM │ 3086 │ 1053983 │ █████████████████████ │
|
||||
│ SURBITON │ ELMBRIDGE │ 100 │ 1011800 │ ████████████████████▏ │
|
||||
│ RADLETT │ HERTSMERE │ 283 │ 1011712 │ ████████████████████▏ │
|
||||
│ SALCOMBE │ SOUTH HAMS │ 127 │ 1011624 │ ████████████████████▏ │
|
||||
│ WEYBRIDGE │ ELMBRIDGE │ 655 │ 1007265 │ ████████████████████▏ │
|
||||
│ ESHER │ ELMBRIDGE │ 485 │ 986581 │ ███████████████████▋ │
|
||||
│ LEATHERHEAD │ GUILDFORD │ 202 │ 977320 │ ███████████████████▌ │
|
||||
│ BURFORD │ WEST OXFORDSHIRE │ 111 │ 966893 │ ███████████████████▎ │
|
||||
│ BROCKENHURST │ NEW FOREST │ 129 │ 956675 │ ███████████████████▏ │
|
||||
│ HINDHEAD │ WAVERLEY │ 137 │ 953753 │ ███████████████████ │
|
||||
│ GERRARDS CROSS │ BUCKINGHAMSHIRE │ 419 │ 951121 │ ███████████████████ │
|
||||
│ EAST MOLESEY │ ELMBRIDGE │ 192 │ 936769 │ ██████████████████▋ │
|
||||
│ CHALFONT ST GILES │ BUCKINGHAMSHIRE │ 146 │ 925515 │ ██████████████████▌ │
|
||||
│ LONDON │ TOWER HAMLETS │ 4388 │ 918304 │ ██████████████████▎ │
|
||||
│ OLNEY │ MILTON KEYNES │ 235 │ 910646 │ ██████████████████▏ │
|
||||
│ HENLEY-ON-THAMES │ SOUTH OXFORDSHIRE │ 540 │ 902418 │ ██████████████████ │
|
||||
│ LONDON │ SOUTHWARK │ 3885 │ 892997 │ █████████████████▋ │
|
||||
│ KINGSTON UPON THAMES │ KINGSTON UPON THAMES │ 960 │ 885969 │ █████████████████▋ │
|
||||
│ LONDON │ EALING │ 2658 │ 871755 │ █████████████████▍ │
|
||||
│ CRANBROOK │ TUNBRIDGE WELLS │ 431 │ 862348 │ █████████████████▏ │
|
||||
│ LONDON │ MERTON │ 2099 │ 859118 │ █████████████████▏ │
|
||||
│ BELVEDERE │ BEXLEY │ 346 │ 842423 │ ████████████████▋ │
|
||||
│ GUILDFORD │ WAVERLEY │ 143 │ 841277 │ ████████████████▋ │
|
||||
│ HARPENDEN │ ST ALBANS │ 657 │ 841216 │ ████████████████▋ │
|
||||
│ LONDON │ HACKNEY │ 3307 │ 837090 │ ████████████████▋ │
|
||||
│ LONDON │ WANDSWORTH │ 6566 │ 832663 │ ████████████████▋ │
|
||||
│ MAIDENHEAD │ BUCKINGHAMSHIRE │ 123 │ 824299 │ ████████████████▍ │
|
||||
│ KINGS LANGLEY │ DACORUM │ 145 │ 821331 │ ████████████████▍ │
|
||||
│ BERKHAMSTED │ DACORUM │ 543 │ 818415 │ ████████████████▎ │
|
||||
│ GREAT MISSENDEN │ BUCKINGHAMSHIRE │ 226 │ 802807 │ ████████████████ │
|
||||
│ BILLINGSHURST │ CHICHESTER │ 144 │ 797829 │ ███████████████▊ │
|
||||
│ WOKING │ GUILDFORD │ 176 │ 793494 │ ███████████████▋ │
|
||||
│ STOCKBRIDGE │ TEST VALLEY │ 178 │ 793269 │ ███████████████▋ │
|
||||
│ EPSOM │ REIGATE AND BANSTEAD │ 172 │ 791862 │ ███████████████▋ │
|
||||
│ TONBRIDGE │ TUNBRIDGE WELLS │ 360 │ 787876 │ ███████████████▋ │
|
||||
│ TEDDINGTON │ RICHMOND UPON THAMES │ 595 │ 786492 │ ███████████████▋ │
|
||||
│ TWICKENHAM │ RICHMOND UPON THAMES │ 1155 │ 786193 │ ███████████████▋ │
|
||||
│ LYNDHURST │ NEW FOREST │ 102 │ 785593 │ ███████████████▋ │
|
||||
│ LONDON │ LAMBETH │ 5228 │ 774574 │ ███████████████▍ │
|
||||
│ LONDON │ BARNET │ 3955 │ 773259 │ ███████████████▍ │
|
||||
│ OXFORD │ VALE OF WHITE HORSE │ 353 │ 772088 │ ███████████████▍ │
|
||||
│ TONBRIDGE │ MAIDSTONE │ 305 │ 770740 │ ███████████████▍ │
|
||||
│ LUTTERWORTH │ HARBOROUGH │ 538 │ 768634 │ ███████████████▎ │
|
||||
│ WOODSTOCK │ WEST OXFORDSHIRE │ 140 │ 766037 │ ███████████████▎ │
|
||||
│ MIDHURST │ CHICHESTER │ 257 │ 764815 │ ███████████████▎ │
|
||||
│ MARLOW │ BUCKINGHAMSHIRE │ 327 │ 761876 │ ███████████████▏ │
|
||||
│ LONDON │ NEWHAM │ 3237 │ 761784 │ ███████████████▏ │
|
||||
│ ALDERLEY EDGE │ CHESHIRE EAST │ 178 │ 757318 │ ███████████████▏ │
|
||||
│ LUTON │ CENTRAL BEDFORDSHIRE │ 212 │ 754283 │ ███████████████ │
|
||||
│ PETWORTH │ CHICHESTER │ 154 │ 754220 │ ███████████████ │
|
||||
│ ALRESFORD │ WINCHESTER │ 219 │ 752718 │ ███████████████ │
|
||||
│ POTTERS BAR │ WELWYN HATFIELD │ 174 │ 748465 │ ██████████████▊ │
|
||||
│ HASLEMERE │ CHICHESTER │ 128 │ 746907 │ ██████████████▊ │
|
||||
│ TADWORTH │ REIGATE AND BANSTEAD │ 502 │ 743252 │ ██████████████▋ │
|
||||
│ THAMES DITTON │ ELMBRIDGE │ 244 │ 741913 │ ██████████████▋ │
|
||||
│ REIGATE │ REIGATE AND BANSTEAD │ 581 │ 738198 │ ██████████████▋ │
|
||||
│ BOURNE END │ BUCKINGHAMSHIRE │ 138 │ 735190 │ ██████████████▋ │
|
||||
│ SEVENOAKS │ SEVENOAKS │ 1156 │ 730018 │ ██████████████▌ │
|
||||
│ OXTED │ TANDRIDGE │ 336 │ 729123 │ ██████████████▌ │
|
||||
│ INGATESTONE │ BRENTWOOD │ 166 │ 728103 │ ██████████████▌ │
|
||||
│ LONDON │ BRENT │ 2079 │ 720605 │ ██████████████▍ │
|
||||
│ LONDON │ HARINGEY │ 3216 │ 717780 │ ██████████████▎ │
|
||||
│ PURLEY │ CROYDON │ 575 │ 716108 │ ██████████████▎ │
|
||||
│ WELWYN │ WELWYN HATFIELD │ 222 │ 710603 │ ██████████████▏ │
|
||||
│ RICKMANSWORTH │ THREE RIVERS │ 798 │ 704571 │ ██████████████ │
|
||||
│ BANSTEAD │ REIGATE AND BANSTEAD │ 401 │ 701293 │ ██████████████ │
|
||||
│ CHIGWELL │ EPPING FOREST │ 261 │ 701203 │ ██████████████ │
|
||||
│ PINNER │ HARROW │ 528 │ 698885 │ █████████████▊ │
|
||||
│ HASLEMERE │ WAVERLEY │ 280 │ 696659 │ █████████████▊ │
|
||||
│ SLOUGH │ BUCKINGHAMSHIRE │ 396 │ 694917 │ █████████████▊ │
|
||||
│ WALTON-ON-THAMES │ ELMBRIDGE │ 946 │ 692395 │ █████████████▋ │
|
||||
│ READING │ SOUTH OXFORDSHIRE │ 318 │ 691988 │ █████████████▋ │
|
||||
│ NORTHWOOD │ HILLINGDON │ 271 │ 690643 │ █████████████▋ │
|
||||
│ FELTHAM │ HOUNSLOW │ 763 │ 688595 │ █████████████▋ │
|
||||
│ ASHTEAD │ MOLE VALLEY │ 303 │ 687923 │ █████████████▋ │
|
||||
│ BARNET │ BARNET │ 975 │ 686980 │ █████████████▋ │
|
||||
│ WOKING │ SURREY HEATH │ 283 │ 686669 │ █████████████▋ │
|
||||
│ MALMESBURY │ WILTSHIRE │ 323 │ 683324 │ █████████████▋ │
|
||||
│ AMERSHAM │ BUCKINGHAMSHIRE │ 496 │ 680962 │ █████████████▌ │
|
||||
│ CHISLEHURST │ BROMLEY │ 430 │ 680209 │ █████████████▌ │
|
||||
│ HYTHE │ FOLKESTONE AND HYTHE │ 490 │ 676908 │ █████████████▌ │
|
||||
│ MAYFIELD │ WEALDEN │ 101 │ 676210 │ █████████████▌ │
|
||||
│ ASCOT │ BRACKNELL FOREST │ 168 │ 676004 │ █████████████▌ │
|
||||
└──────────────────────┴────────────────────────┴──────┴─────────┴────────────────────────────────────────────────────────────────────┘
|
||||
```response
|
||||
┌─town─────────────────┬─district───────────────┬─────c─┬───price─┬─bar(round(avg(price)), 0, 5000000, 100)─────────────────────────┐
|
||||
│ LONDON │ CITY OF LONDON │ 578 │ 3149590 │ ██████████████████████████████████████████████████████████████▊ │
|
||||
│ LONDON │ CITY OF WESTMINSTER │ 7083 │ 2903794 │ ██████████████████████████████████████████████████████████ │
|
||||
│ LONDON │ KENSINGTON AND CHELSEA │ 4986 │ 2333782 │ ██████████████████████████████████████████████▋ │
|
||||
│ LEATHERHEAD │ ELMBRIDGE │ 203 │ 2071595 │ █████████████████████████████████████████▍ │
|
||||
│ VIRGINIA WATER │ RUNNYMEDE │ 308 │ 1939465 │ ██████████████████████████████████████▋ │
|
||||
│ LONDON │ CAMDEN │ 5750 │ 1673687 │ █████████████████████████████████▍ │
|
||||
│ WINDLESHAM │ SURREY HEATH │ 182 │ 1428358 │ ████████████████████████████▌ │
|
||||
│ NORTHWOOD │ THREE RIVERS │ 112 │ 1404170 │ ████████████████████████████ │
|
||||
│ BARNET │ ENFIELD │ 259 │ 1338299 │ ██████████████████████████▋ │
|
||||
│ LONDON │ ISLINGTON │ 5504 │ 1275520 │ █████████████████████████▌ │
|
||||
│ LONDON │ RICHMOND UPON THAMES │ 1345 │ 1261935 │ █████████████████████████▏ │
|
||||
│ COBHAM │ ELMBRIDGE │ 727 │ 1251403 │ █████████████████████████ │
|
||||
│ BEACONSFIELD │ BUCKINGHAMSHIRE │ 680 │ 1199970 │ ███████████████████████▊ │
|
||||
│ LONDON │ TOWER HAMLETS │ 10012 │ 1157827 │ ███████████████████████▏ │
|
||||
│ LONDON │ HOUNSLOW │ 1278 │ 1144389 │ ██████████████████████▊ │
|
||||
│ BURFORD │ WEST OXFORDSHIRE │ 182 │ 1139393 │ ██████████████████████▋ │
|
||||
│ RICHMOND │ RICHMOND UPON THAMES │ 1649 │ 1130076 │ ██████████████████████▌ │
|
||||
│ KINGSTON UPON THAMES │ RICHMOND UPON THAMES │ 147 │ 1126111 │ ██████████████████████▌ │
|
||||
│ ASCOT │ WINDSOR AND MAIDENHEAD │ 773 │ 1106109 │ ██████████████████████ │
|
||||
│ LONDON │ HAMMERSMITH AND FULHAM │ 6162 │ 1056198 │ █████████████████████ │
|
||||
│ RADLETT │ HERTSMERE │ 513 │ 1045758 │ ████████████████████▊ │
|
||||
│ LEATHERHEAD │ GUILDFORD │ 354 │ 1045175 │ ████████████████████▊ │
|
||||
│ WEYBRIDGE │ ELMBRIDGE │ 1275 │ 1036702 │ ████████████████████▋ │
|
||||
│ FARNHAM │ EAST HAMPSHIRE │ 107 │ 1033682 │ ████████████████████▋ │
|
||||
│ ESHER │ ELMBRIDGE │ 915 │ 1032753 │ ████████████████████▋ │
|
||||
│ FARNHAM │ HART │ 102 │ 1002692 │ ████████████████████ │
|
||||
│ GERRARDS CROSS │ BUCKINGHAMSHIRE │ 845 │ 983639 │ ███████████████████▋ │
|
||||
│ CHALFONT ST GILES │ BUCKINGHAMSHIRE │ 286 │ 973993 │ ███████████████████▍ │
|
||||
│ SALCOMBE │ SOUTH HAMS │ 215 │ 965724 │ ███████████████████▎ │
|
||||
│ SURBITON │ ELMBRIDGE │ 181 │ 960346 │ ███████████████████▏ │
|
||||
│ BROCKENHURST │ NEW FOREST │ 226 │ 951278 │ ███████████████████ │
|
||||
│ SUTTON COLDFIELD │ LICHFIELD │ 110 │ 930757 │ ██████████████████▌ │
|
||||
│ EAST MOLESEY │ ELMBRIDGE │ 372 │ 927026 │ ██████████████████▌ │
|
||||
│ LLANGOLLEN │ WREXHAM │ 127 │ 925681 │ ██████████████████▌ │
|
||||
│ OXFORD │ SOUTH OXFORDSHIRE │ 638 │ 923830 │ ██████████████████▍ │
|
||||
│ LONDON │ MERTON │ 4383 │ 923194 │ ██████████████████▍ │
|
||||
│ GUILDFORD │ WAVERLEY │ 261 │ 905733 │ ██████████████████ │
|
||||
│ TEDDINGTON │ RICHMOND UPON THAMES │ 1147 │ 894856 │ █████████████████▊ │
|
||||
│ HARPENDEN │ ST ALBANS │ 1271 │ 893079 │ █████████████████▋ │
|
||||
│ HENLEY-ON-THAMES │ SOUTH OXFORDSHIRE │ 1042 │ 887557 │ █████████████████▋ │
|
||||
│ POTTERS BAR │ WELWYN HATFIELD │ 314 │ 863037 │ █████████████████▎ │
|
||||
│ LONDON │ WANDSWORTH │ 13210 │ 857318 │ █████████████████▏ │
|
||||
│ BILLINGSHURST │ CHICHESTER │ 255 │ 856508 │ █████████████████▏ │
|
||||
│ LONDON │ SOUTHWARK │ 7742 │ 843145 │ ████████████████▋ │
|
||||
│ LONDON │ HACKNEY │ 6656 │ 839716 │ ████████████████▋ │
|
||||
│ LUTTERWORTH │ HARBOROUGH │ 1096 │ 836546 │ ████████████████▋ │
|
||||
│ KINGSTON UPON THAMES │ KINGSTON UPON THAMES │ 1846 │ 828990 │ ████████████████▌ │
|
||||
│ LONDON │ EALING │ 5583 │ 820135 │ ████████████████▍ │
|
||||
│ INGATESTONE │ CHELMSFORD │ 120 │ 815379 │ ████████████████▎ │
|
||||
│ MARLOW │ BUCKINGHAMSHIRE │ 718 │ 809943 │ ████████████████▏ │
|
||||
│ EAST GRINSTEAD │ TANDRIDGE │ 105 │ 809461 │ ████████████████▏ │
|
||||
│ CHIGWELL │ EPPING FOREST │ 484 │ 809338 │ ████████████████▏ │
|
||||
│ EGHAM │ RUNNYMEDE │ 989 │ 807858 │ ████████████████▏ │
|
||||
│ HASLEMERE │ CHICHESTER │ 223 │ 804173 │ ████████████████ │
|
||||
│ PETWORTH │ CHICHESTER │ 288 │ 803206 │ ████████████████ │
|
||||
│ TWICKENHAM │ RICHMOND UPON THAMES │ 2194 │ 802616 │ ████████████████ │
|
||||
│ WEMBLEY │ BRENT │ 1698 │ 801733 │ ████████████████ │
|
||||
│ HINDHEAD │ WAVERLEY │ 233 │ 801482 │ ████████████████ │
|
||||
│ LONDON │ BARNET │ 8083 │ 792066 │ ███████████████▋ │
|
||||
│ WOKING │ GUILDFORD │ 343 │ 789360 │ ███████████████▋ │
|
||||
│ STOCKBRIDGE │ TEST VALLEY │ 318 │ 777909 │ ███████████████▌ │
|
||||
│ BERKHAMSTED │ DACORUM │ 1049 │ 776138 │ ███████████████▌ │
|
||||
│ MAIDENHEAD │ BUCKINGHAMSHIRE │ 236 │ 775572 │ ███████████████▌ │
|
||||
│ SOLIHULL │ STRATFORD-ON-AVON │ 142 │ 770727 │ ███████████████▍ │
|
||||
│ GREAT MISSENDEN │ BUCKINGHAMSHIRE │ 431 │ 764493 │ ███████████████▎ │
|
||||
│ TADWORTH │ REIGATE AND BANSTEAD │ 920 │ 757511 │ ███████████████▏ │
|
||||
│ LONDON │ BRENT │ 4124 │ 757194 │ ███████████████▏ │
|
||||
│ THAMES DITTON │ ELMBRIDGE │ 470 │ 750828 │ ███████████████ │
|
||||
│ LONDON │ LAMBETH │ 10431 │ 750532 │ ███████████████ │
|
||||
│ RICKMANSWORTH │ THREE RIVERS │ 1500 │ 747029 │ ██████████████▊ │
|
||||
│ KINGS LANGLEY │ DACORUM │ 281 │ 746536 │ ██████████████▊ │
|
||||
│ HARLOW │ EPPING FOREST │ 172 │ 739423 │ ██████████████▋ │
|
||||
│ TONBRIDGE │ SEVENOAKS │ 103 │ 738740 │ ██████████████▋ │
|
||||
│ BELVEDERE │ BEXLEY │ 686 │ 736385 │ ██████████████▋ │
|
||||
│ CRANBROOK │ TUNBRIDGE WELLS │ 769 │ 734328 │ ██████████████▋ │
|
||||
│ SOLIHULL │ WARWICK │ 116 │ 733286 │ ██████████████▋ │
|
||||
│ ALDERLEY EDGE │ CHESHIRE EAST │ 357 │ 732882 │ ██████████████▋ │
|
||||
│ WELWYN │ WELWYN HATFIELD │ 404 │ 730281 │ ██████████████▌ │
|
||||
│ CHISLEHURST │ BROMLEY │ 870 │ 730279 │ ██████████████▌ │
|
||||
│ LONDON │ HARINGEY │ 6488 │ 726715 │ ██████████████▌ │
|
||||
│ AMERSHAM │ BUCKINGHAMSHIRE │ 965 │ 725426 │ ██████████████▌ │
|
||||
│ SEVENOAKS │ SEVENOAKS │ 2183 │ 725102 │ ██████████████▌ │
|
||||
│ BOURNE END │ BUCKINGHAMSHIRE │ 269 │ 724595 │ ██████████████▍ │
|
||||
│ NORTHWOOD │ HILLINGDON │ 568 │ 722436 │ ██████████████▍ │
|
||||
│ PURFLEET │ THURROCK │ 143 │ 722205 │ ██████████████▍ │
|
||||
│ SLOUGH │ BUCKINGHAMSHIRE │ 832 │ 721529 │ ██████████████▍ │
|
||||
│ INGATESTONE │ BRENTWOOD │ 301 │ 718292 │ ██████████████▎ │
|
||||
│ EPSOM │ REIGATE AND BANSTEAD │ 315 │ 709264 │ ██████████████▏ │
|
||||
│ ASHTEAD │ MOLE VALLEY │ 524 │ 708646 │ ██████████████▏ │
|
||||
│ BETCHWORTH │ MOLE VALLEY │ 155 │ 708525 │ ██████████████▏ │
|
||||
│ OXTED │ TANDRIDGE │ 645 │ 706946 │ ██████████████▏ │
|
||||
│ READING │ SOUTH OXFORDSHIRE │ 593 │ 705466 │ ██████████████ │
|
||||
│ FELTHAM │ HOUNSLOW │ 1536 │ 703815 │ ██████████████ │
|
||||
│ TUNBRIDGE WELLS │ WEALDEN │ 207 │ 703296 │ ██████████████ │
|
||||
│ LEWES │ WEALDEN │ 116 │ 701349 │ ██████████████ │
|
||||
│ OXFORD │ OXFORD │ 3656 │ 700813 │ ██████████████ │
|
||||
│ MAYFIELD │ WEALDEN │ 177 │ 698158 │ █████████████▊ │
|
||||
│ PINNER │ HARROW │ 997 │ 697876 │ █████████████▊ │
|
||||
│ LECHLADE │ COTSWOLD │ 155 │ 696262 │ █████████████▊ │
|
||||
│ WALTON-ON-THAMES │ ELMBRIDGE │ 1850 │ 690102 │ █████████████▋ │
|
||||
└──────────────────────┴────────────────────────┴───────┴─────────┴─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Let's Speed Up Queries Using Projections {#speedup-with-projections}
|
||||
|
||||
[Projections](../../sql-reference/statements/alter/projection.md) allow to improve queries speed by storing pre-aggregated data.
|
||||
[Projections](../../sql-reference/statements/alter/projection.md) allow you to improve query speeds by storing pre-aggregated data in whatever format you want. In this example, we create a projection that keeps track of the average price, total price, and count of properties grouped by the year, district and town. At execution time, ClickHouse will use your projection if it thinks the projection can improve the performance fo the query (you don't have to do anything special to use the projection - ClickHouse decides for you when the projection will be useful).
|
||||
|
||||
### Build a Projection {#build-projection}
|
||||
|
||||
Create an aggregate projection by dimensions `toYear(date)`, `district`, `town`:
|
||||
Let's create an aggregate projection by the dimensions `toYear(date)`, `district`, and `town`:
|
||||
|
||||
```sql
|
||||
ALTER TABLE uk_price_paid
|
||||
@ -374,25 +363,23 @@ ALTER TABLE uk_price_paid
|
||||
toYear(date),
|
||||
district,
|
||||
town
|
||||
);
|
||||
)
|
||||
```
|
||||
|
||||
Populate the projection for existing data (without it projection will be created for only newly inserted data):
|
||||
Populate the projection for existing data. (Without materializing it, the projection will be created for only newly inserted data):
|
||||
|
||||
```sql
|
||||
ALTER TABLE uk_price_paid
|
||||
MATERIALIZE PROJECTION projection_by_year_district_town
|
||||
SETTINGS mutations_sync = 1;
|
||||
SETTINGS mutations_sync = 1
|
||||
```
|
||||
|
||||
## Test Performance {#test-performance}
|
||||
|
||||
Let's run the same 3 queries.
|
||||
Let's run the same 3 queries again:
|
||||
|
||||
### Query 1. Average Price Per Year {#average-price-projections}
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toYear(date) AS year,
|
||||
@ -400,47 +387,18 @@ SELECT
|
||||
bar(price, 0, 1000000, 80)
|
||||
FROM uk_price_paid
|
||||
GROUP BY year
|
||||
ORDER BY year ASC;
|
||||
ORDER BY year ASC
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─year─┬──price─┬─bar(round(avg(price)), 0, 1000000, 80)─┐
|
||||
│ 1995 │ 67932 │ █████▍ │
|
||||
│ 1996 │ 71505 │ █████▋ │
|
||||
│ 1997 │ 78532 │ ██████▎ │
|
||||
│ 1998 │ 85436 │ ██████▋ │
|
||||
│ 1999 │ 96037 │ ███████▋ │
|
||||
│ 2000 │ 107479 │ ████████▌ │
|
||||
│ 2001 │ 118885 │ █████████▌ │
|
||||
│ 2002 │ 137941 │ ███████████ │
|
||||
│ 2003 │ 155889 │ ████████████▍ │
|
||||
│ 2004 │ 178885 │ ██████████████▎ │
|
||||
│ 2005 │ 189351 │ ███████████████▏ │
|
||||
│ 2006 │ 203528 │ ████████████████▎ │
|
||||
│ 2007 │ 219378 │ █████████████████▌ │
|
||||
│ 2008 │ 217056 │ █████████████████▎ │
|
||||
│ 2009 │ 213419 │ █████████████████ │
|
||||
│ 2010 │ 236109 │ ██████████████████▊ │
|
||||
│ 2011 │ 232805 │ ██████████████████▌ │
|
||||
│ 2012 │ 238367 │ ███████████████████ │
|
||||
│ 2013 │ 256931 │ ████████████████████▌ │
|
||||
│ 2014 │ 279915 │ ██████████████████████▍ │
|
||||
│ 2015 │ 297266 │ ███████████████████████▋ │
|
||||
│ 2016 │ 313201 │ █████████████████████████ │
|
||||
│ 2017 │ 346097 │ ███████████████████████████▋ │
|
||||
│ 2018 │ 350116 │ ████████████████████████████ │
|
||||
│ 2019 │ 351013 │ ████████████████████████████ │
|
||||
│ 2020 │ 369420 │ █████████████████████████████▌ │
|
||||
│ 2021 │ 386903 │ ██████████████████████████████▊ │
|
||||
└──────┴────────┴────────────────────────────────────────┘
|
||||
The result is the same, but the performance is better!
|
||||
```response
|
||||
No projection: 28 rows in set. Elapsed: 1.775 sec. Processed 27.45 million rows, 164.70 MB (15.47 million rows/s., 92.79 MB/s.)
|
||||
With projection: 28 rows in set. Elapsed: 0.665 sec. Processed 87.51 thousand rows, 3.21 MB (131.51 thousand rows/s., 4.82 MB/s.)
|
||||
```
|
||||
|
||||
|
||||
### Query 2. Average Price Per Year in London {#average-price-london-projections}
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
toYear(date) AS year,
|
||||
@ -449,48 +407,19 @@ SELECT
|
||||
FROM uk_price_paid
|
||||
WHERE town = 'LONDON'
|
||||
GROUP BY year
|
||||
ORDER BY year ASC;
|
||||
ORDER BY year ASC
|
||||
```
|
||||
|
||||
Result:
|
||||
Same result, but notice the improvement in query performance:
|
||||
|
||||
```text
|
||||
┌─year─┬───price─┬─bar(round(avg(price)), 0, 2000000, 100)───────────────┐
|
||||
│ 1995 │ 109116 │ █████▍ │
|
||||
│ 1996 │ 118667 │ █████▊ │
|
||||
│ 1997 │ 136518 │ ██████▋ │
|
||||
│ 1998 │ 152983 │ ███████▋ │
|
||||
│ 1999 │ 180637 │ █████████ │
|
||||
│ 2000 │ 215838 │ ██████████▋ │
|
||||
│ 2001 │ 232994 │ ███████████▋ │
|
||||
│ 2002 │ 263670 │ █████████████▏ │
|
||||
│ 2003 │ 278394 │ █████████████▊ │
|
||||
│ 2004 │ 304666 │ ███████████████▏ │
|
||||
│ 2005 │ 322875 │ ████████████████▏ │
|
||||
│ 2006 │ 356191 │ █████████████████▋ │
|
||||
│ 2007 │ 404054 │ ████████████████████▏ │
|
||||
│ 2008 │ 420741 │ █████████████████████ │
|
||||
│ 2009 │ 427753 │ █████████████████████▍ │
|
||||
│ 2010 │ 480306 │ ████████████████████████ │
|
||||
│ 2011 │ 496274 │ ████████████████████████▋ │
|
||||
│ 2012 │ 519442 │ █████████████████████████▊ │
|
||||
│ 2013 │ 616212 │ ██████████████████████████████▋ │
|
||||
│ 2014 │ 724154 │ ████████████████████████████████████▏ │
|
||||
│ 2015 │ 792129 │ ███████████████████████████████████████▌ │
|
||||
│ 2016 │ 843655 │ ██████████████████████████████████████████▏ │
|
||||
│ 2017 │ 982642 │ █████████████████████████████████████████████████▏ │
|
||||
│ 2018 │ 1016835 │ ██████████████████████████████████████████████████▋ │
|
||||
│ 2019 │ 1042849 │ ████████████████████████████████████████████████████▏ │
|
||||
│ 2020 │ 1011889 │ ██████████████████████████████████████████████████▌ │
|
||||
│ 2021 │ 960343 │ ████████████████████████████████████████████████ │
|
||||
└──────┴─────────┴───────────────────────────────────────────────────────┘
|
||||
```response
|
||||
No projection: 28 rows in set. Elapsed: 0.720 sec. Processed 27.45 million rows, 46.61 MB (38.13 million rows/s., 64.74 MB/s.)
|
||||
With projection: 28 rows in set. Elapsed: 0.015 sec. Processed 87.51 thousand rows, 3.51 MB (5.74 million rows/s., 230.24 MB/s.)
|
||||
```
|
||||
|
||||
### Query 3. The Most Expensive Neighborhoods {#most-expensive-neighborhoods-projections}
|
||||
|
||||
The condition (date >= '2020-01-01') needs to be modified to match projection dimension (toYear(date) >= 2020).
|
||||
|
||||
Query:
|
||||
The condition (date >= '2020-01-01') needs to be modified so that it matches the projection dimension (`toYear(date) >= 2020)`:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
@ -506,138 +435,16 @@ GROUP BY
|
||||
district
|
||||
HAVING c >= 100
|
||||
ORDER BY price DESC
|
||||
LIMIT 100;
|
||||
LIMIT 100
|
||||
```
|
||||
|
||||
Result:
|
||||
Again, the result is the same but notice the improvement in query performance:
|
||||
|
||||
```text
|
||||
┌─town─────────────────┬─district───────────────┬────c─┬───price─┬─bar(round(avg(price)), 0, 5000000, 100)────────────────────────────┐
|
||||
│ LONDON │ CITY OF WESTMINSTER │ 3606 │ 3280239 │ █████████████████████████████████████████████████████████████████▌ │
|
||||
│ LONDON │ CITY OF LONDON │ 274 │ 3160502 │ ███████████████████████████████████████████████████████████████▏ │
|
||||
│ LONDON │ KENSINGTON AND CHELSEA │ 2550 │ 2308478 │ ██████████████████████████████████████████████▏ │
|
||||
│ LEATHERHEAD │ ELMBRIDGE │ 114 │ 1897407 │ █████████████████████████████████████▊ │
|
||||
│ LONDON │ CAMDEN │ 3033 │ 1805404 │ ████████████████████████████████████ │
|
||||
│ VIRGINIA WATER │ RUNNYMEDE │ 156 │ 1753247 │ ███████████████████████████████████ │
|
||||
│ WINDLESHAM │ SURREY HEATH │ 108 │ 1677613 │ █████████████████████████████████▌ │
|
||||
│ THORNTON HEATH │ CROYDON │ 546 │ 1671721 │ █████████████████████████████████▍ │
|
||||
│ BARNET │ ENFIELD │ 124 │ 1505840 │ ██████████████████████████████ │
|
||||
│ COBHAM │ ELMBRIDGE │ 387 │ 1237250 │ ████████████████████████▋ │
|
||||
│ LONDON │ ISLINGTON │ 2668 │ 1236980 │ ████████████████████████▋ │
|
||||
│ OXFORD │ SOUTH OXFORDSHIRE │ 321 │ 1220907 │ ████████████████████████▍ │
|
||||
│ LONDON │ RICHMOND UPON THAMES │ 704 │ 1215551 │ ████████████████████████▎ │
|
||||
│ LONDON │ HOUNSLOW │ 671 │ 1207493 │ ████████████████████████▏ │
|
||||
│ ASCOT │ WINDSOR AND MAIDENHEAD │ 407 │ 1183299 │ ███████████████████████▋ │
|
||||
│ BEACONSFIELD │ BUCKINGHAMSHIRE │ 330 │ 1175615 │ ███████████████████████▌ │
|
||||
│ RICHMOND │ RICHMOND UPON THAMES │ 874 │ 1110444 │ ██████████████████████▏ │
|
||||
│ LONDON │ HAMMERSMITH AND FULHAM │ 3086 │ 1053983 │ █████████████████████ │
|
||||
│ SURBITON │ ELMBRIDGE │ 100 │ 1011800 │ ████████████████████▏ │
|
||||
│ RADLETT │ HERTSMERE │ 283 │ 1011712 │ ████████████████████▏ │
|
||||
│ SALCOMBE │ SOUTH HAMS │ 127 │ 1011624 │ ████████████████████▏ │
|
||||
│ WEYBRIDGE │ ELMBRIDGE │ 655 │ 1007265 │ ████████████████████▏ │
|
||||
│ ESHER │ ELMBRIDGE │ 485 │ 986581 │ ███████████████████▋ │
|
||||
│ LEATHERHEAD │ GUILDFORD │ 202 │ 977320 │ ███████████████████▌ │
|
||||
│ BURFORD │ WEST OXFORDSHIRE │ 111 │ 966893 │ ███████████████████▎ │
|
||||
│ BROCKENHURST │ NEW FOREST │ 129 │ 956675 │ ███████████████████▏ │
|
||||
│ HINDHEAD │ WAVERLEY │ 137 │ 953753 │ ███████████████████ │
|
||||
│ GERRARDS CROSS │ BUCKINGHAMSHIRE │ 419 │ 951121 │ ███████████████████ │
|
||||
│ EAST MOLESEY │ ELMBRIDGE │ 192 │ 936769 │ ██████████████████▋ │
|
||||
│ CHALFONT ST GILES │ BUCKINGHAMSHIRE │ 146 │ 925515 │ ██████████████████▌ │
|
||||
│ LONDON │ TOWER HAMLETS │ 4388 │ 918304 │ ██████████████████▎ │
|
||||
│ OLNEY │ MILTON KEYNES │ 235 │ 910646 │ ██████████████████▏ │
|
||||
│ HENLEY-ON-THAMES │ SOUTH OXFORDSHIRE │ 540 │ 902418 │ ██████████████████ │
|
||||
│ LONDON │ SOUTHWARK │ 3885 │ 892997 │ █████████████████▋ │
|
||||
│ KINGSTON UPON THAMES │ KINGSTON UPON THAMES │ 960 │ 885969 │ █████████████████▋ │
|
||||
│ LONDON │ EALING │ 2658 │ 871755 │ █████████████████▍ │
|
||||
│ CRANBROOK │ TUNBRIDGE WELLS │ 431 │ 862348 │ █████████████████▏ │
|
||||
│ LONDON │ MERTON │ 2099 │ 859118 │ █████████████████▏ │
|
||||
│ BELVEDERE │ BEXLEY │ 346 │ 842423 │ ████████████████▋ │
|
||||
│ GUILDFORD │ WAVERLEY │ 143 │ 841277 │ ████████████████▋ │
|
||||
│ HARPENDEN │ ST ALBANS │ 657 │ 841216 │ ████████████████▋ │
|
||||
│ LONDON │ HACKNEY │ 3307 │ 837090 │ ████████████████▋ │
|
||||
│ LONDON │ WANDSWORTH │ 6566 │ 832663 │ ████████████████▋ │
|
||||
│ MAIDENHEAD │ BUCKINGHAMSHIRE │ 123 │ 824299 │ ████████████████▍ │
|
||||
│ KINGS LANGLEY │ DACORUM │ 145 │ 821331 │ ████████████████▍ │
|
||||
│ BERKHAMSTED │ DACORUM │ 543 │ 818415 │ ████████████████▎ │
|
||||
│ GREAT MISSENDEN │ BUCKINGHAMSHIRE │ 226 │ 802807 │ ████████████████ │
|
||||
│ BILLINGSHURST │ CHICHESTER │ 144 │ 797829 │ ███████████████▊ │
|
||||
│ WOKING │ GUILDFORD │ 176 │ 793494 │ ███████████████▋ │
|
||||
│ STOCKBRIDGE │ TEST VALLEY │ 178 │ 793269 │ ███████████████▋ │
|
||||
│ EPSOM │ REIGATE AND BANSTEAD │ 172 │ 791862 │ ███████████████▋ │
|
||||
│ TONBRIDGE │ TUNBRIDGE WELLS │ 360 │ 787876 │ ███████████████▋ │
|
||||
│ TEDDINGTON │ RICHMOND UPON THAMES │ 595 │ 786492 │ ███████████████▋ │
|
||||
│ TWICKENHAM │ RICHMOND UPON THAMES │ 1155 │ 786193 │ ███████████████▋ │
|
||||
│ LYNDHURST │ NEW FOREST │ 102 │ 785593 │ ███████████████▋ │
|
||||
│ LONDON │ LAMBETH │ 5228 │ 774574 │ ███████████████▍ │
|
||||
│ LONDON │ BARNET │ 3955 │ 773259 │ ███████████████▍ │
|
||||
│ OXFORD │ VALE OF WHITE HORSE │ 353 │ 772088 │ ███████████████▍ │
|
||||
│ TONBRIDGE │ MAIDSTONE │ 305 │ 770740 │ ███████████████▍ │
|
||||
│ LUTTERWORTH │ HARBOROUGH │ 538 │ 768634 │ ███████████████▎ │
|
||||
│ WOODSTOCK │ WEST OXFORDSHIRE │ 140 │ 766037 │ ███████████████▎ │
|
||||
│ MIDHURST │ CHICHESTER │ 257 │ 764815 │ ███████████████▎ │
|
||||
│ MARLOW │ BUCKINGHAMSHIRE │ 327 │ 761876 │ ███████████████▏ │
|
||||
│ LONDON │ NEWHAM │ 3237 │ 761784 │ ███████████████▏ │
|
||||
│ ALDERLEY EDGE │ CHESHIRE EAST │ 178 │ 757318 │ ███████████████▏ │
|
||||
│ LUTON │ CENTRAL BEDFORDSHIRE │ 212 │ 754283 │ ███████████████ │
|
||||
│ PETWORTH │ CHICHESTER │ 154 │ 754220 │ ███████████████ │
|
||||
│ ALRESFORD │ WINCHESTER │ 219 │ 752718 │ ███████████████ │
|
||||
│ POTTERS BAR │ WELWYN HATFIELD │ 174 │ 748465 │ ██████████████▊ │
|
||||
│ HASLEMERE │ CHICHESTER │ 128 │ 746907 │ ██████████████▊ │
|
||||
│ TADWORTH │ REIGATE AND BANSTEAD │ 502 │ 743252 │ ██████████████▋ │
|
||||
│ THAMES DITTON │ ELMBRIDGE │ 244 │ 741913 │ ██████████████▋ │
|
||||
│ REIGATE │ REIGATE AND BANSTEAD │ 581 │ 738198 │ ██████████████▋ │
|
||||
│ BOURNE END │ BUCKINGHAMSHIRE │ 138 │ 735190 │ ██████████████▋ │
|
||||
│ SEVENOAKS │ SEVENOAKS │ 1156 │ 730018 │ ██████████████▌ │
|
||||
│ OXTED │ TANDRIDGE │ 336 │ 729123 │ ██████████████▌ │
|
||||
│ INGATESTONE │ BRENTWOOD │ 166 │ 728103 │ ██████████████▌ │
|
||||
│ LONDON │ BRENT │ 2079 │ 720605 │ ██████████████▍ │
|
||||
│ LONDON │ HARINGEY │ 3216 │ 717780 │ ██████████████▎ │
|
||||
│ PURLEY │ CROYDON │ 575 │ 716108 │ ██████████████▎ │
|
||||
│ WELWYN │ WELWYN HATFIELD │ 222 │ 710603 │ ██████████████▏ │
|
||||
│ RICKMANSWORTH │ THREE RIVERS │ 798 │ 704571 │ ██████████████ │
|
||||
│ BANSTEAD │ REIGATE AND BANSTEAD │ 401 │ 701293 │ ██████████████ │
|
||||
│ CHIGWELL │ EPPING FOREST │ 261 │ 701203 │ ██████████████ │
|
||||
│ PINNER │ HARROW │ 528 │ 698885 │ █████████████▊ │
|
||||
│ HASLEMERE │ WAVERLEY │ 280 │ 696659 │ █████████████▊ │
|
||||
│ SLOUGH │ BUCKINGHAMSHIRE │ 396 │ 694917 │ █████████████▊ │
|
||||
│ WALTON-ON-THAMES │ ELMBRIDGE │ 946 │ 692395 │ █████████████▋ │
|
||||
│ READING │ SOUTH OXFORDSHIRE │ 318 │ 691988 │ █████████████▋ │
|
||||
│ NORTHWOOD │ HILLINGDON │ 271 │ 690643 │ █████████████▋ │
|
||||
│ FELTHAM │ HOUNSLOW │ 763 │ 688595 │ █████████████▋ │
|
||||
│ ASHTEAD │ MOLE VALLEY │ 303 │ 687923 │ █████████████▋ │
|
||||
│ BARNET │ BARNET │ 975 │ 686980 │ █████████████▋ │
|
||||
│ WOKING │ SURREY HEATH │ 283 │ 686669 │ █████████████▋ │
|
||||
│ MALMESBURY │ WILTSHIRE │ 323 │ 683324 │ █████████████▋ │
|
||||
│ AMERSHAM │ BUCKINGHAMSHIRE │ 496 │ 680962 │ █████████████▌ │
|
||||
│ CHISLEHURST │ BROMLEY │ 430 │ 680209 │ █████████████▌ │
|
||||
│ HYTHE │ FOLKESTONE AND HYTHE │ 490 │ 676908 │ █████████████▌ │
|
||||
│ MAYFIELD │ WEALDEN │ 101 │ 676210 │ █████████████▌ │
|
||||
│ ASCOT │ BRACKNELL FOREST │ 168 │ 676004 │ █████████████▌ │
|
||||
└──────────────────────┴────────────────────────┴──────┴─────────┴────────────────────────────────────────────────────────────────────┘
|
||||
```response
|
||||
No projection: 100 rows in set. Elapsed: 0.928 sec. Processed 27.45 million rows, 103.80 MB (29.56 million rows/s., 111.80 MB/s.)
|
||||
With projection: 100 rows in set. Elapsed: 0.336 sec. Processed 17.32 thousand rows, 1.23 MB (51.61 thousand rows/s., 3.65 MB/s.)
|
||||
```
|
||||
|
||||
### Summary {#summary}
|
||||
|
||||
All 3 queries work much faster and read fewer rows.
|
||||
|
||||
```text
|
||||
Query 1
|
||||
|
||||
no projection: 27 rows in set. Elapsed: 0.158 sec. Processed 26.32 million rows, 157.93 MB (166.57 million rows/s., 999.39 MB/s.)
|
||||
projection: 27 rows in set. Elapsed: 0.007 sec. Processed 105.96 thousand rows, 3.33 MB (14.58 million rows/s., 458.13 MB/s.)
|
||||
|
||||
|
||||
Query 2
|
||||
|
||||
no projection: 27 rows in set. Elapsed: 0.163 sec. Processed 26.32 million rows, 80.01 MB (161.75 million rows/s., 491.64 MB/s.)
|
||||
projection: 27 rows in set. Elapsed: 0.008 sec. Processed 105.96 thousand rows, 3.67 MB (13.29 million rows/s., 459.89 MB/s.)
|
||||
|
||||
Query 3
|
||||
|
||||
no projection: 100 rows in set. Elapsed: 0.069 sec. Processed 26.32 million rows, 62.47 MB (382.13 million rows/s., 906.93 MB/s.)
|
||||
projection: 100 rows in set. Elapsed: 0.029 sec. Processed 8.08 thousand rows, 511.08 KB (276.06 thousand rows/s., 17.47 MB/s.)
|
||||
```
|
||||
|
||||
### Test It in Playground {#playground}
|
||||
### Test it in the Playground {#playground}
|
||||
|
||||
The dataset is also available in the [Online Playground](https://play.clickhouse.com/play?user=play#U0VMRUNUIHRvd24sIGRpc3RyaWN0LCBjb3VudCgpIEFTIGMsIHJvdW5kKGF2ZyhwcmljZSkpIEFTIHByaWNlLCBiYXIocHJpY2UsIDAsIDUwMDAwMDAsIDEwMCkgRlJPTSB1a19wcmljZV9wYWlkIFdIRVJFIGRhdGUgPj0gJzIwMjAtMDEtMDEnIEdST1VQIEJZIHRvd24sIGRpc3RyaWN0IEhBVklORyBjID49IDEwMCBPUkRFUiBCWSBwcmljZSBERVNDIExJTUlUIDEwMA==).
|
||||
|
||||
26
docs/en/getting-started/index.md
Normal file
26
docs/en/getting-started/index.md
Normal file
@ -0,0 +1,26 @@
|
||||
---
|
||||
slug: /en/getting-started/example-datasets/
|
||||
sidebar_position: 0
|
||||
sidebar_label: Overview
|
||||
keywords: [clickhouse, install, tutorial, sample, datasets]
|
||||
pagination_next: 'en/tutorial'
|
||||
---
|
||||
|
||||
# Tutorials and Example Datasets
|
||||
|
||||
We have a lot of resources for helping you get started and learn how ClickHouse works:
|
||||
|
||||
- If you need to get ClickHouse up and running, check out our [Quick Start](../quick-start.mdx)
|
||||
- The [ClickHouse Tutorial](../tutorial.md) analyzes a dataset of New York City taxi rides
|
||||
|
||||
In addition, the sample datasets provide a great experience on working with ClickHouse,
|
||||
learning important techniques and tricks, and seeing how to take advantage of the many powerful
|
||||
functions in ClickHouse. The sample datasets include:
|
||||
|
||||
- The [UK Property Price Paid dataset](../getting-started/example-datasets/uk-price-paid.md) is a good starting point with some interesting SQL queries
|
||||
- The [New York Taxi Data](../getting-started/example-datasets/nyc-taxi.md) has an example of how to insert data from S3 into ClickHouse
|
||||
- The [Cell Towers dataset](../getting-started/example-datasets/cell-towers.md) imports a CSV into ClickHouse
|
||||
- The [NYPD Complaint Data](../getting-started/example-datasets/nypd_complaint_data.md) demonstrates how to use data inference to simplify creating tables
|
||||
- The ["What's on the Menu?" dataset](../getting-started/example-datasets/menus.md) has an example of denormalizing data
|
||||
|
||||
View the **Tutorials and Datasets** menu for a complete list of sample datasets.
|
||||
@ -3,6 +3,7 @@ slug: /en/interfaces/cli
|
||||
sidebar_position: 17
|
||||
sidebar_label: Command-Line Client
|
||||
---
|
||||
import ConnectionDetails from '@site/docs/en/_snippets/_gather_your_details_native.md';
|
||||
|
||||
# Command-line Client
|
||||
|
||||
@ -24,26 +25,76 @@ Connected to ClickHouse server version 20.13.1 revision 54442.
|
||||
Different client and server versions are compatible with one another, but some features may not be available in older clients. We recommend using the same version of the client as the server app. When you try to use a client of the older version, then the server, `clickhouse-client` displays the message:
|
||||
|
||||
```response
|
||||
ClickHouse client version is older than ClickHouse server. It may lack support for new features.
|
||||
ClickHouse client version is older than ClickHouse server.
|
||||
It may lack support for new features.
|
||||
```
|
||||
|
||||
## Usage {#cli_usage}
|
||||
|
||||
The client can be used in interactive and non-interactive (batch) mode. To use batch mode, specify the ‘query’ parameter, or send data to ‘stdin’ (it verifies that ‘stdin’ is not a terminal), or both. Similar to the HTTP interface, when using the ‘query’ parameter and sending data to ‘stdin’, the request is a concatenation of the ‘query’ parameter, a line feed, and the data in ‘stdin’. This is convenient for large INSERT queries.
|
||||
The client can be used in interactive and non-interactive (batch) mode.
|
||||
|
||||
Example of using the client to insert data:
|
||||
### Gather your connection details
|
||||
<ConnectionDetails />
|
||||
|
||||
### Interactive
|
||||
|
||||
To connect to your ClickHouse Cloud service, or any ClickHouse server using TLS and passwords, interactively use `--secure`, port 9440, and provide your username and password:
|
||||
|
||||
```bash
|
||||
clickhouse-client --host <HOSTNAME> \
|
||||
--secure \
|
||||
--port 9440 \
|
||||
--user <USERNAME> \
|
||||
--password <PASSWORD>
|
||||
```
|
||||
|
||||
To connect to a self-managed ClickHouse server you will need the details for that server. Whether or not TLS is used, port numbers, and passwords are all configurable. Use the above example for ClickHouse Cloud as a starting point.
|
||||
|
||||
|
||||
### Batch
|
||||
|
||||
To use batch mode, specify the ‘query’ parameter, or send data to ‘stdin’ (it verifies that ‘stdin’ is not a terminal), or both. Similar to the HTTP interface, when using the ‘query’ parameter and sending data to ‘stdin’, the request is a concatenation of the ‘query’ parameter, a line feed, and the data in ‘stdin’. This is convenient for large INSERT queries.
|
||||
|
||||
Examples of using the client to insert data:
|
||||
|
||||
#### Inserting a CSV file into a remote ClickHouse service
|
||||
|
||||
This example is appropriate for ClickHouse Cloud, or any ClickHouse server using TLS and a password. In this example a sample dataset CSV file, `cell_towers.csv` is inserted into an existing table `cell_towers` in the `default` database:
|
||||
|
||||
```bash
|
||||
clickhouse-client --host HOSTNAME.clickhouse.cloud \
|
||||
--secure \
|
||||
--port 9440 \
|
||||
--user default \
|
||||
--password PASSWORD \
|
||||
--query "INSERT INTO cell_towers FORMAT CSVWithNames" \
|
||||
< cell_towers.csv
|
||||
```
|
||||
|
||||
:::note
|
||||
To concentrate on the query syntax, the rest of the examples leave off the connection details (`--host`, `--port`, etc.). Add them in when you try the commands.
|
||||
:::
|
||||
|
||||
#### Three different ways of inserting data
|
||||
|
||||
``` bash
|
||||
$ echo -ne "1, 'some text', '2016-08-14 00:00:00'\n2, 'some more text', '2016-08-14 00:00:01'" | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV";
|
||||
echo -ne "1, 'some text', '2016-08-14 00:00:00'\n2, 'some more text', '2016-08-14 00:00:01'" | \
|
||||
clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV";
|
||||
```
|
||||
|
||||
$ cat <<_EOF | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV";
|
||||
```bash
|
||||
cat <<_EOF | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV";
|
||||
3, 'some text', '2016-08-14 00:00:00'
|
||||
4, 'some more text', '2016-08-14 00:00:01'
|
||||
_EOF
|
||||
|
||||
$ cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV";
|
||||
```
|
||||
|
||||
```bash
|
||||
cat file.csv | clickhouse-client --database=test --query="INSERT INTO test FORMAT CSV";
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
In batch mode, the default data format is TabSeparated. You can set the format in the FORMAT clause of the query.
|
||||
|
||||
By default, you can only process a single query in batch mode. To make multiple queries from a “script,” use the `--multiquery` parameter. This works for all queries except INSERT. Query results are output consecutively without additional separators. Similarly, to process a large number of queries, you can run ‘clickhouse-client’ for each query. Note that it may take tens of milliseconds to launch the ‘clickhouse-client’ program.
|
||||
|
||||
@ -6,16 +6,32 @@ sidebar_label: MySQL Interface
|
||||
|
||||
# MySQL Interface
|
||||
|
||||
ClickHouse supports MySQL wire protocol. It can be enabled by [mysql_port](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-mysql_port) setting in configuration file:
|
||||
ClickHouse supports MySQL wire protocol. To enable the MySQL wire protocol, add the [mysql_port](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-mysql_port) setting to your server's configuration file. For example, you could define the port in a new XML file in your `config.d` folder:
|
||||
|
||||
``` xml
|
||||
<mysql_port>9004</mysql_port>
|
||||
<clickhouse>
|
||||
<mysql_port>9004</mysql_port>
|
||||
</clickhouse>
|
||||
```
|
||||
|
||||
Example of connecting using command-line tool `mysql`:
|
||||
Startup your ClickHouse server and look for a log message similar to the following that mentions Listening for MySQL compatibility protocol:
|
||||
|
||||
```
|
||||
{} <Information> Application: Listening for MySQL compatibility protocol: 127.0.0.1:9004
|
||||
```
|
||||
|
||||
## Connect mysql to ClickHouse
|
||||
|
||||
The following command demonstrates how to connect the MySQL client `mysql` to ClickHouse:
|
||||
|
||||
```bash
|
||||
mysql --protocol tcp -h [hostname] -u [username] -P [port_number] [database_name]
|
||||
```
|
||||
|
||||
For example:
|
||||
|
||||
``` bash
|
||||
$ mysql --protocol tcp -u default -P 9004
|
||||
$ mysql --protocol tcp -h 127.0.0.1 -u default -P 9004 default
|
||||
```
|
||||
|
||||
Output if a connection succeeded:
|
||||
|
||||
@ -5,6 +5,9 @@ sidebar_label: ClickHouse Keeper
|
||||
---
|
||||
|
||||
# ClickHouse Keeper
|
||||
import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_automated.md';
|
||||
|
||||
<SelfManaged />
|
||||
|
||||
ClickHouse Keeper provides the coordination system for data [replication](../engines/table-engines/mergetree-family/replication.md) and [distributed DDL](../sql-reference/distributed-ddl.md) queries execution. ClickHouse Keeper is compatible with ZooKeeper.
|
||||
|
||||
|
||||
@ -3,7 +3,11 @@ slug: /en/operations/external-authenticators/
|
||||
sidebar_position: 48
|
||||
sidebar_label: External User Authenticators and Directories
|
||||
title: "External User Authenticators and Directories"
|
||||
pagination_next: 'en/operations/external-authenticators/kerberos'
|
||||
---
|
||||
import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md';
|
||||
|
||||
<SelfManaged />
|
||||
|
||||
ClickHouse supports authenticating and managing users using external services.
|
||||
|
||||
|
||||
@ -2,6 +2,9 @@
|
||||
slug: /en/operations/external-authenticators/kerberos
|
||||
---
|
||||
# Kerberos
|
||||
import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md';
|
||||
|
||||
<SelfManaged />
|
||||
|
||||
Existing and properly configured ClickHouse users can be authenticated via Kerberos authentication protocol.
|
||||
|
||||
|
||||
@ -2,6 +2,9 @@
|
||||
slug: /en/operations/external-authenticators/ldap
|
||||
title: "LDAP"
|
||||
---
|
||||
import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md';
|
||||
|
||||
<SelfManaged />
|
||||
|
||||
LDAP server can be used to authenticate ClickHouse users. There are two different approaches for doing this:
|
||||
|
||||
|
||||
@ -2,6 +2,9 @@
|
||||
slug: /en/operations/external-authenticators/ssl-x509
|
||||
title: "SSL X.509 certificate authentication"
|
||||
---
|
||||
import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md';
|
||||
|
||||
<SelfManaged />
|
||||
|
||||
[SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` field of the certificate is used to identify connected user. This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration.
|
||||
|
||||
|
||||
@ -5,6 +5,9 @@ sidebar_label: Monitoring
|
||||
---
|
||||
|
||||
# Monitoring
|
||||
import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_automated.md';
|
||||
|
||||
<SelfManaged />
|
||||
|
||||
You can monitor:
|
||||
|
||||
|
||||
@ -3,9 +3,12 @@ slug: /en/operations/optimizing-performance/sampling-query-profiler
|
||||
sidebar_position: 54
|
||||
sidebar_label: Query Profiling
|
||||
---
|
||||
import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md';
|
||||
|
||||
# Sampling Query Profiler
|
||||
|
||||
<SelfManaged />
|
||||
|
||||
ClickHouse runs sampling profiler that allows analyzing query execution. Using profiler you can find source code routines that used the most frequently during query execution. You can trace CPU time and wall-clock time spent including idle time.
|
||||
|
||||
To use profiler:
|
||||
|
||||
@ -5,6 +5,10 @@ sidebar_label: Testing Hardware
|
||||
title: "How to Test Your Hardware with ClickHouse"
|
||||
---
|
||||
|
||||
import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.md';
|
||||
|
||||
<SelfManaged />
|
||||
|
||||
You can run a basic ClickHouse performance test on any server without installation of ClickHouse packages.
|
||||
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
slug: /en/operations/server-configuration-parameters/
|
||||
sidebar_position: 54
|
||||
sidebar_label: Server Configuration Parameters
|
||||
pagination_next: en/operations/server-configuration-parameters/settings
|
||||
---
|
||||
|
||||
# Server Configuration Parameters
|
||||
|
||||
@ -666,6 +666,7 @@ Keys:
|
||||
- `http_proxy` - Configure HTTP proxy for sending crash reports.
|
||||
- `debug` - Sets the Sentry client into debug mode.
|
||||
- `tmp_path` - Filesystem path for temporary crash report state.
|
||||
- `environment` - An arbitrary name of an environment in which the ClickHouse server is running. It will be mentioned in each crash report. The default value is `test` or `prod` depending on the version of ClickHouse.
|
||||
|
||||
**Recommended way to use**
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
sidebar_label: Settings
|
||||
sidebar_position: 51
|
||||
slug: /en/operations/settings/
|
||||
pagination_next: en/operations/settings/settings
|
||||
---
|
||||
|
||||
# Settings Overview
|
||||
|
||||
@ -5,6 +5,9 @@ sidebar_label: Secured Communication with Zookeeper
|
||||
---
|
||||
|
||||
# Optional secured communication between ClickHouse and Zookeeper
|
||||
import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_automated.md';
|
||||
|
||||
<SelfManaged />
|
||||
|
||||
You should specify `ssl.keyStore.location`, `ssl.keyStore.password` and `ssl.trustStore.location`, `ssl.trustStore.password` for communication with ClickHouse client over SSL. These options are available from Zookeeper version 3.5.2.
|
||||
|
||||
|
||||
@ -4,6 +4,9 @@ sidebar_position: 58
|
||||
sidebar_label: Usage Recommendations
|
||||
title: "Usage Recommendations"
|
||||
---
|
||||
import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_automated.md';
|
||||
|
||||
<SelfManaged />
|
||||
|
||||
## CPU Scaling Governor
|
||||
|
||||
|
||||
@ -10,7 +10,7 @@ Creates new [roles](../../../operations/access-rights.md#role-management). Role
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [, name2 ...]
|
||||
CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
|
||||
@ -13,7 +13,7 @@ Creates a new view. Views can be [normal](#normal-view), [materialized](#materia
|
||||
Syntax:
|
||||
|
||||
``` sql
|
||||
CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] AS SELECT ...
|
||||
CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster_name] AS SELECT ...
|
||||
```
|
||||
|
||||
Normal views do not store any data. They just perform a read from another table on each access. In other words, a normal view is nothing more than a saved query. When reading from a view, this saved query is used as a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause.
|
||||
|
||||
@ -430,9 +430,9 @@ FROM
|
||||
### Cumulative sum.
|
||||
|
||||
```sql
|
||||
CREATE TABLE events
|
||||
CREATE TABLE warehouse
|
||||
(
|
||||
`metric` String,
|
||||
`item` String,
|
||||
`ts` DateTime,
|
||||
`value` Float
|
||||
)
|
||||
|
||||
@ -624,6 +624,7 @@ ClickHouse поддерживает динамическое изменение
|
||||
- `http_proxy` - Настройка HTTP proxy для отсылки отчетов о сбоях.
|
||||
- `debug` - Настроить клиентскую библиотеку Sentry в debug режим.
|
||||
- `tmp_path` - Путь в файловой системе для временного хранения состояния отчетов о сбоях перед отправкой на сервер Sentry.
|
||||
- `environment` - Произвольное название среды, в которой запущен сервер ClickHouse, которое будет упомянуто в каждом отчете от сбое. По умолчанию имеет значение `test` или `prod` в зависимости от версии ClickHouse.
|
||||
|
||||
**Рекомендованные настройки**
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ sidebar_label: "Роль"
|
||||
Синтаксис:
|
||||
|
||||
```sql
|
||||
CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [, name2 ...]
|
||||
CREATE ROLE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...]
|
||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | PROFILE 'profile_name'] [,...]
|
||||
```
|
||||
|
||||
@ -47,4 +47,4 @@ SET ROLE accountant;
|
||||
SELECT * FROM db.*;
|
||||
```
|
||||
|
||||
<!--hide-->
|
||||
<!--hide-->
|
||||
|
||||
@ -11,7 +11,7 @@ sidebar_label: "Представление"
|
||||
## Обычные представления {#normal}
|
||||
|
||||
``` sql
|
||||
CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] AS SELECT ...
|
||||
CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster_name] AS SELECT ...
|
||||
```
|
||||
|
||||
Обычные представления не хранят никаких данных, они выполняют чтение данных из другой таблицы при каждом доступе. Другими словами, обычное представление — это не что иное, как сохраненный запрос. При чтении данных из представления этот сохраненный запрос используется как подзапрос в секции [FROM](../../../sql-reference/statements/select/from.md).
|
||||
|
||||
@ -13,7 +13,7 @@ sidebar_label: VIEW
|
||||
语法:
|
||||
|
||||
``` sql
|
||||
CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] AS SELECT ...
|
||||
CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster_name] AS SELECT ...
|
||||
```
|
||||
|
||||
普通视图不存储任何数据。 他们只是在每次访问时从另一个表执行读取。换句话说，普通视图只不过是一个保存的查询。 从视图中读取时，此保存的查询用作[FROM](../../../sql-reference/statements/select/from.md)子句中的子查询.
|
||||
|
||||
4
rust/BLAKE3/CMakeLists.txt
Executable file
4
rust/BLAKE3/CMakeLists.txt
Executable file
@ -0,0 +1,4 @@
|
||||
corrosion_import_crate(MANIFEST_PATH Cargo.toml NO_STD)
|
||||
|
||||
target_include_directories(_ch_rust_blake3 INTERFACE include)
|
||||
add_library(ch_rust::blake3 ALIAS _ch_rust_blake3)
|
||||
92
rust/BLAKE3/Cargo.lock
generated
Normal file
92
rust/BLAKE3/Cargo.lock
generated
Normal file
@ -0,0 +1,92 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "_ch_rust_blake3"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrayref"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
|
||||
|
||||
[[package]]
|
||||
name = "blake3"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "526c210b4520e416420759af363083471656e819a75e831b8d2c9d5a584f2413"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"arrayvec",
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"constant_time_eq",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.73"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.132"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
|
||||
13
rust/BLAKE3/Cargo.toml
Normal file
13
rust/BLAKE3/Cargo.toml
Normal file
@ -0,0 +1,13 @@
|
||||
[package]
|
||||
name = "_ch_rust_blake3"
|
||||
version = "0.1.0"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
blake3 = "1.2.0"
|
||||
libc = "0.2.132"
|
||||
|
||||
[lib]
|
||||
crate-type = ["staticlib"]
|
||||
|
||||
17
rust/BLAKE3/include/blake3.h
Normal file
17
rust/BLAKE3/include/blake3.h
Normal file
@ -0,0 +1,17 @@
|
||||
#ifndef BLAKE3_H
|
||||
#define BLAKE3_H
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
|
||||
extern "C" {
|
||||
|
||||
char *blake3_apply_shim(const char *begin, uint32_t _size, uint8_t *out_char_data);
|
||||
|
||||
char *blake3_apply_shim_msan_compat(const char *begin, uint32_t size, uint8_t *out_char_data);
|
||||
|
||||
void blake3_free_char_pointer(char *ptr_to_free);
|
||||
|
||||
} // extern "C"
|
||||
|
||||
#endif /* BLAKE3_H */
|
||||
55
rust/BLAKE3/src/lib.rs
Normal file
55
rust/BLAKE3/src/lib.rs
Normal file
@ -0,0 +1,55 @@
|
||||
extern crate blake3;
|
||||
extern crate libc;
|
||||
|
||||
use std::ffi::{CStr, CString};
|
||||
use std::os::raw::c_char;
|
||||
use std::mem;
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn blake3_apply_shim(
|
||||
begin: *const c_char,
|
||||
_size: u32,
|
||||
out_char_data: *mut u8,
|
||||
) -> *mut c_char {
|
||||
if begin.is_null() {
|
||||
let err_str = CString::new("input was a null pointer").unwrap();
|
||||
return err_str.into_raw();
|
||||
}
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
let input_bytes = CStr::from_ptr(begin);
|
||||
let input_res = input_bytes.to_bytes();
|
||||
hasher.update(input_res);
|
||||
let mut reader = hasher.finalize_xof();
|
||||
reader.fill(std::slice::from_raw_parts_mut(out_char_data, blake3::OUT_LEN));
|
||||
std::ptr::null_mut()
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn blake3_apply_shim_msan_compat(
|
||||
mut begin: *const c_char,
|
||||
size: u32,
|
||||
out_char_data: *mut u8,
|
||||
) -> *mut c_char {
|
||||
if begin.is_null() {
|
||||
let err_str = CString::new("input was a null pointer").unwrap();
|
||||
return err_str.into_raw();
|
||||
}
|
||||
libc::memset(out_char_data as *mut libc::c_void, 0, mem::size_of::<u8>());
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
let mut vec = Vec::<u8>::new();
|
||||
for _ in 0..size {
|
||||
vec.push(*begin as u8);
|
||||
begin = begin.add(1);
|
||||
}
|
||||
let input_res = vec.as_mut_slice();
|
||||
hasher.update(input_res);
|
||||
let mut reader = hasher.finalize_xof();
|
||||
reader.fill(std::slice::from_raw_parts_mut(out_char_data, blake3::OUT_LEN));
|
||||
std::ptr::null_mut()
|
||||
}
|
||||
|
||||
// Freeing memory according to docs: https://doc.rust-lang.org/std/ffi/struct.CString.html#method.into_raw
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn blake3_free_char_pointer(ptr_to_free: *mut c_char) {
|
||||
std::mem::drop(CString::from_raw(ptr_to_free));
|
||||
}
|
||||
1
rust/CMakeLists.txt
Normal file
1
rust/CMakeLists.txt
Normal file
@ -0,0 +1 @@
|
||||
add_subdirectory (BLAKE3)
|
||||
@ -368,8 +368,6 @@ target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2_st)
|
||||
target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2)
|
||||
|
||||
target_link_libraries(clickhouse_common_io
|
||||
PRIVATE
|
||||
${EXECINFO_LIBRARIES}
|
||||
PUBLIC
|
||||
boost::program_options
|
||||
boost::system
|
||||
|
||||
@ -261,8 +261,8 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
|
||||
M(Bool, fallback_to_stale_replicas_for_distributed_queries, true, "Suppose max_replica_delay_for_distributed_queries is set and all replicas for the queried table are stale. If this setting is enabled, the query will be performed anyway, otherwise the error will be reported.", 0) \
|
||||
M(UInt64, preferred_max_column_in_block_size_bytes, 0, "Limit on max column size in block while reading. Helps to decrease cache misses count. Should be close to L2 cache size.", 0) \
|
||||
\
|
||||
M(UInt64, parts_to_delay_insert, 150, "If the destination table contains at least that many active parts in a single partition, artificially slow down insert into table.", 0) \
|
||||
M(UInt64, parts_to_throw_insert, 300, "If more than this number active parts in a single partition of the destination table, throw 'Too many parts ...' exception.", 0) \
|
||||
M(UInt64, parts_to_delay_insert, 0, "If the destination table contains at least that many active parts in a single partition, artificially slow down insert into table.", 0) \
|
||||
M(UInt64, parts_to_throw_insert, 0, "If more than this number active parts in a single partition of the destination table, throw 'Too many parts ...' exception.", 0) \
|
||||
M(Bool, insert_distributed_sync, false, "If setting is enabled, insert query into distributed waits until data will be sent to all nodes in cluster.", 0) \
|
||||
M(UInt64, insert_distributed_timeout, 0, "Timeout for insert query into distributed. Setting is used only with insert_distributed_sync enabled. Zero value means no timeout.", 0) \
|
||||
M(Int64, distributed_ddl_task_timeout, 180, "Timeout for DDL query responses from all hosts in cluster. If a ddl request has not been performed on all hosts, a response will contain a timeout error and a request will be executed in an async mode. Negative value means infinite. Zero means async mode.", 0) \
|
||||
|
||||
@ -11,8 +11,8 @@ if (OS_DARWIN AND NOT USE_STATIC_LIBRARIES)
|
||||
target_link_libraries (daemon PUBLIC -Wl,-undefined,dynamic_lookup)
|
||||
endif()
|
||||
|
||||
target_link_libraries (daemon PUBLIC loggers common PRIVATE clickhouse_common_io clickhouse_common_config ${EXECINFO_LIBRARIES})
|
||||
target_link_libraries (daemon PUBLIC loggers common PRIVATE clickhouse_common_io clickhouse_common_config)
|
||||
|
||||
if (TARGET ch_contrib::sentry)
|
||||
target_link_libraries (daemon PRIVATE ch_contrib::sentry)
|
||||
target_link_libraries (daemon PRIVATE ch_contrib::sentry dbms)
|
||||
endif ()
|
||||
|
||||
@ -96,14 +96,14 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config)
|
||||
}
|
||||
sentry_options_set_dsn(options, endpoint.c_str());
|
||||
sentry_options_set_database_path(options, temp_folder_path.c_str());
|
||||
|
||||
/// This value will be attached to each report
|
||||
String environment_default_value = "test";
|
||||
if (strstr(VERSION_DESCRIBE, "-stable") || strstr(VERSION_DESCRIBE, "-lts"))
|
||||
{
|
||||
sentry_options_set_environment(options, "prod");
|
||||
}
|
||||
else
|
||||
{
|
||||
sentry_options_set_environment(options, "test");
|
||||
}
|
||||
environment_default_value = "prod";
|
||||
/// If the value is set in config - use it
|
||||
auto value = config.getString("send_crash_reports.environment", environment_default_value);
|
||||
sentry_options_set_environment(options, value.c_str());
|
||||
|
||||
const std::string & http_proxy = config.getString("send_crash_reports.http_proxy", "");
|
||||
if (!http_proxy.empty())
|
||||
|
||||
@ -205,10 +205,9 @@ inline ReturnType convertToDecimalImpl(const typename FromDataType::FieldType &
|
||||
if (!std::isfinite(value))
|
||||
{
|
||||
if constexpr (throw_exception)
|
||||
throw Exception(std::string(ToDataType::family_name) + " convert overflow. Cannot convert infinity or NaN to decimal",
|
||||
ErrorCodes::DECIMAL_OVERFLOW);
|
||||
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Cannot convert infinity or NaN to decimal", ToDataType::family_name);
|
||||
else
|
||||
return false;
|
||||
return ReturnType(false);
|
||||
}
|
||||
|
||||
auto out = value * static_cast<FromFieldType>(DecimalUtils::scaleMultiplier<ToNativeType>(scale));
|
||||
@ -217,8 +216,7 @@ inline ReturnType convertToDecimalImpl(const typename FromDataType::FieldType &
|
||||
out >= static_cast<FromFieldType>(std::numeric_limits<ToNativeType>::max()))
|
||||
{
|
||||
if constexpr (throw_exception)
|
||||
throw Exception(std::string(ToDataType::family_name) + " convert overflow. Float is out of Decimal range",
|
||||
ErrorCodes::DECIMAL_OVERFLOW);
|
||||
throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "{} convert overflow. Float is out of Decimal range", ToDataType::family_name);
|
||||
else
|
||||
return ReturnType(false);
|
||||
}
|
||||
|
||||
@ -33,6 +33,13 @@ list (APPEND PRIVATE_LIBS
|
||||
divide_impl
|
||||
)
|
||||
|
||||
|
||||
if (TARGET ch_rust::blake3)
|
||||
list (APPEND PUBLIC_LIBS
|
||||
ch_rust::blake3
|
||||
)
|
||||
endif()
|
||||
|
||||
if (TARGET OpenSSL::Crypto)
|
||||
list (APPEND PUBLIC_LIBS OpenSSL::Crypto)
|
||||
endif()
|
||||
|
||||
@ -323,13 +323,13 @@ struct ToDateTimeImpl
|
||||
{
|
||||
static constexpr auto name = "toDateTime";
|
||||
|
||||
static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
static UInt32 execute(UInt16 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
auto date_time = time_zone.fromDayNum(ExtendedDayNum(d));
|
||||
return date_time <= 0xffffffff ? UInt32(date_time) : UInt32(0xffffffff);
|
||||
}
|
||||
|
||||
static inline UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
static UInt32 execute(Int32 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
if (d < 0)
|
||||
return 0;
|
||||
@ -338,12 +338,12 @@ struct ToDateTimeImpl
|
||||
return date_time <= 0xffffffff ? date_time : 0xffffffff;
|
||||
}
|
||||
|
||||
static inline UInt32 execute(UInt32 dt, const DateLUTImpl & /*time_zone*/)
|
||||
static UInt32 execute(UInt32 dt, const DateLUTImpl & /*time_zone*/)
|
||||
{
|
||||
return dt;
|
||||
}
|
||||
|
||||
static inline UInt32 execute(Int64 d, const DateLUTImpl & time_zone)
|
||||
static UInt32 execute(Int64 d, const DateLUTImpl & time_zone)
|
||||
{
|
||||
if (d < 0)
|
||||
return 0;
|
||||
@ -352,7 +352,7 @@ struct ToDateTimeImpl
|
||||
return date_time <= 0xffffffff ? date_time : 0xffffffff;
|
||||
}
|
||||
|
||||
static inline UInt32 execute(const DecimalUtils::DecimalComponents<DateTime64> & t, const DateLUTImpl & /*time_zone*/)
|
||||
static UInt32 execute(const DecimalUtils::DecimalComponents<DateTime64> & t, const DateLUTImpl & /*time_zone*/)
|
||||
{
|
||||
if (t.whole < 0 || (t.whole >= 0 && t.fractional < 0))
|
||||
return 0;
|
||||
@ -374,7 +374,7 @@ struct ToDateTransform32Or64
|
||||
{
|
||||
static constexpr auto name = "toDate";
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone)
|
||||
static NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone)
|
||||
{
|
||||
// since converting to Date, no need in values outside of default LUT range.
|
||||
if (from < 0)
|
||||
@ -391,7 +391,7 @@ struct ToDateTransform32Or64Signed
|
||||
{
|
||||
static constexpr auto name = "toDate";
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone)
|
||||
static NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone)
|
||||
{
|
||||
// TODO: decide narrow or extended range based on FromType
|
||||
/// The function should be monotonic (better for query optimizations), so we saturate instead of overflow.
|
||||
@ -413,7 +413,7 @@ struct ToDateTransform8Or16Signed
|
||||
{
|
||||
static constexpr auto name = "toDate";
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl &)
|
||||
static NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl &)
|
||||
{
|
||||
if (from < 0)
|
||||
return 0;
|
||||
@ -431,7 +431,7 @@ struct ToDate32Transform32Or64
|
||||
{
|
||||
static constexpr auto name = "toDate32";
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone)
|
||||
static NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone)
|
||||
{
|
||||
return (from < DATE_LUT_MAX_EXTEND_DAY_NUM)
|
||||
? from
|
||||
@ -444,7 +444,7 @@ struct ToDate32Transform32Or64Signed
|
||||
{
|
||||
static constexpr auto name = "toDate32";
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone)
|
||||
static NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone)
|
||||
{
|
||||
static const Int32 daynum_min_offset = -static_cast<Int32>(DateLUT::instance().getDayNumOffsetEpoch());
|
||||
if (from < daynum_min_offset)
|
||||
@ -460,7 +460,7 @@ struct ToDate32Transform8Or16Signed
|
||||
{
|
||||
static constexpr auto name = "toDate32";
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl &)
|
||||
static NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl &)
|
||||
{
|
||||
return from;
|
||||
}
|
||||
@ -529,7 +529,7 @@ struct ToDateTimeTransform64
|
||||
{
|
||||
static constexpr auto name = "toDateTime";
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl &)
|
||||
static NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl &)
|
||||
{
|
||||
return std::min<Int64>(Int64(from), Int64(0xFFFFFFFF));
|
||||
}
|
||||
@ -540,7 +540,7 @@ struct ToDateTimeTransformSigned
|
||||
{
|
||||
static constexpr auto name = "toDateTime";
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl &)
|
||||
static NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl &)
|
||||
{
|
||||
if (from < 0)
|
||||
return 0;
|
||||
@ -553,7 +553,7 @@ struct ToDateTimeTransform64Signed
|
||||
{
|
||||
static constexpr auto name = "toDateTime";
|
||||
|
||||
static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & /* time_zone */)
|
||||
static NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & /* time_zone */)
|
||||
{
|
||||
if (from < 0)
|
||||
return 0;
|
||||
@ -581,9 +581,9 @@ template <typename Name> struct ConvertImpl<DataTypeFloat32, DataTypeDateTime, N
|
||||
template <typename Name> struct ConvertImpl<DataTypeFloat64, DataTypeDateTime, Name>
|
||||
: DateTimeTransformImpl<DataTypeFloat64, DataTypeDateTime, ToDateTimeTransform64Signed<Float64, UInt32>> {};
|
||||
|
||||
const time_t LUT_MIN_TIME = -2208988800l; // 1900-01-01 UTC
|
||||
constexpr time_t LUT_MIN_TIME = -2208988800l; // 1900-01-01 UTC
|
||||
|
||||
const time_t LUT_MAX_TIME = 10413791999l; // 2299-12-31 UTC
|
||||
constexpr time_t LUT_MAX_TIME = 10413791999l; // 2299-12-31 UTC
|
||||
|
||||
/** Conversion of numeric to DateTime64
|
||||
*/
|
||||
@ -599,7 +599,7 @@ struct ToDateTime64TransformUnsigned
|
||||
: scale_multiplier(DecimalUtils::scaleMultiplier<DateTime64::NativeType>(scale))
|
||||
{}
|
||||
|
||||
inline NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const
|
||||
NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const
|
||||
{
|
||||
from = std::min<time_t>(from, LUT_MAX_TIME);
|
||||
return DecimalUtils::decimalFromComponentsWithMultiplier<DateTime64>(from, 0, scale_multiplier);
|
||||
@ -616,7 +616,7 @@ struct ToDateTime64TransformSigned
|
||||
: scale_multiplier(DecimalUtils::scaleMultiplier<DateTime64::NativeType>(scale))
|
||||
{}
|
||||
|
||||
inline NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const
|
||||
NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const
|
||||
{
|
||||
from = std::max<time_t>(from, LUT_MIN_TIME);
|
||||
from = std::min<time_t>(from, LUT_MAX_TIME);
|
||||
@ -634,11 +634,10 @@ struct ToDateTime64TransformFloat
|
||||
: scale(scale_)
|
||||
{}
|
||||
|
||||
inline NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const
|
||||
NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const
|
||||
{
|
||||
if (from < 0)
|
||||
return 0;
|
||||
from = std::min<FromType>(from, FromType(0xFFFFFFFF));
|
||||
from = std::max(from, static_cast<FromType>(LUT_MIN_TIME));
|
||||
from = std::min(from, static_cast<FromType>(LUT_MAX_TIME));
|
||||
return convertToDecimal<FromDataType, DataTypeDateTime64>(from, scale);
|
||||
}
|
||||
};
|
||||
@ -672,7 +671,7 @@ struct FromDateTime64Transform
|
||||
: scale_multiplier(DecimalUtils::scaleMultiplier<DateTime64::NativeType>(scale))
|
||||
{}
|
||||
|
||||
inline auto execute(DateTime64::NativeType dt, const DateLUTImpl & time_zone) const
|
||||
auto execute(DateTime64::NativeType dt, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
const auto c = DecimalUtils::splitWithScaleMultiplier(DateTime64(dt), scale_multiplier);
|
||||
return Transform::execute(static_cast<UInt32>(c.whole), time_zone);
|
||||
@ -694,19 +693,19 @@ struct ToDateTime64Transform
|
||||
: scale_multiplier(DecimalUtils::scaleMultiplier<DateTime64::NativeType>(scale))
|
||||
{}
|
||||
|
||||
inline DateTime64::NativeType execute(UInt16 d, const DateLUTImpl & time_zone) const
|
||||
DateTime64::NativeType execute(UInt16 d, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
const auto dt = ToDateTimeImpl::execute(d, time_zone);
|
||||
return execute(dt, time_zone);
|
||||
}
|
||||
|
||||
inline DateTime64::NativeType execute(Int32 d, const DateLUTImpl & time_zone) const
|
||||
DateTime64::NativeType execute(Int32 d, const DateLUTImpl & time_zone) const
|
||||
{
|
||||
const auto dt = time_zone.fromDayNum(ExtendedDayNum(d));
|
||||
return DecimalUtils::decimalFromComponentsWithMultiplier<DateTime64>(dt, 0, scale_multiplier);
|
||||
}
|
||||
|
||||
inline DateTime64::NativeType execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) const
|
||||
DateTime64::NativeType execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) const
|
||||
{
|
||||
return DecimalUtils::decimalFromComponentsWithMultiplier<DateTime64>(dt, 0, scale_multiplier);
|
||||
}
|
||||
|
||||
@ -41,5 +41,22 @@ REGISTER_FUNCTION(Hashing)
|
||||
factory.registerFunction<FunctionXxHash64>();
|
||||
|
||||
factory.registerFunction<FunctionWyHash64>();
|
||||
|
||||
|
||||
#if USE_BLAKE3
|
||||
factory.registerFunction<FunctionBLAKE3>(
|
||||
{
|
||||
R"(
|
||||
Calculates BLAKE3 hash string and returns the resulting set of bytes as FixedString.
|
||||
This cryptographic hash-function is integrated into ClickHouse with BLAKE3 Rust library.
|
||||
The function is rather fast and shows approximately two times faster performance compared to SHA-2, while generating hashes of the same length as SHA-256.
|
||||
It returns a BLAKE3 hash as a byte array with type FixedString(32).
|
||||
)",
|
||||
Documentation::Examples{
|
||||
{"hash", "SELECT hex(blake3('ABC'))"}},
|
||||
Documentation::Categories{"Hash"}
|
||||
},
|
||||
FunctionFactory::CaseSensitive);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@ -10,6 +10,10 @@
|
||||
#include "config_functions.h"
|
||||
#include "config_core.h"
|
||||
|
||||
#if USE_BLAKE3
|
||||
# include <blake3.h>
|
||||
#endif
|
||||
|
||||
#include <Common/SipHash.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/HashTable/Hash.h>
|
||||
@ -615,6 +619,32 @@ struct ImplXxHash64
|
||||
static constexpr bool use_int_hash_for_pods = false;
|
||||
};
|
||||
|
||||
|
||||
#if USE_BLAKE3
|
||||
struct ImplBLAKE3
|
||||
{
|
||||
static constexpr auto name = "blake3";
|
||||
enum { length = 32 };
|
||||
|
||||
static void apply(const char * begin, const size_t size, unsigned char* out_char_data)
|
||||
{
|
||||
#if defined(MEMORY_SANITIZER)
|
||||
auto err_msg = blake3_apply_shim_msan_compat(begin, size, out_char_data);
|
||||
__msan_unpoison(out_char_data, length);
|
||||
#else
|
||||
auto err_msg = blake3_apply_shim(begin, size, out_char_data);
|
||||
#endif
|
||||
if (err_msg != nullptr)
|
||||
{
|
||||
auto err_st = std::string(err_msg);
|
||||
blake3_free_char_pointer(err_msg);
|
||||
throw Exception("Function returned error message: " + std::string(err_msg), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
|
||||
template <typename Impl>
|
||||
class FunctionStringHashFixedString : public IFunction
|
||||
{
|
||||
@ -1474,4 +1504,8 @@ using FunctionXxHash64 = FunctionAnyHash<ImplXxHash64>;
|
||||
|
||||
using FunctionWyHash64 = FunctionAnyHash<ImplWyHash64>;
|
||||
|
||||
#if USE_BLAKE3
|
||||
using FunctionBLAKE3 = FunctionStringHashFixedString<ImplBLAKE3>;
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
@ -8,5 +8,6 @@
|
||||
#cmakedefine01 USE_H3
|
||||
#cmakedefine01 USE_S2_GEOMETRY
|
||||
#cmakedefine01 USE_FASTOPS
|
||||
#cmakedefine01 USE_BLAKE3
|
||||
#cmakedefine01 USE_NLP
|
||||
#cmakedefine01 USE_VECTORSCAN
|
||||
|
||||
@ -110,7 +110,7 @@ namespace ErrorCodes
|
||||
|
||||
namespace
|
||||
{
|
||||
bool tryAddHeadersFromConfig(HTTPServerResponse & response, const Poco::Util::LayeredConfiguration & config)
|
||||
bool tryAddHttpOptionHeadersFromConfig(HTTPServerResponse & response, const Poco::Util::LayeredConfiguration & config)
|
||||
{
|
||||
if (config.has("http_options_response"))
|
||||
{
|
||||
@ -138,7 +138,7 @@ bool tryAddHeadersFromConfig(HTTPServerResponse & response, const Poco::Util::La
|
||||
void processOptionsRequest(HTTPServerResponse & response, const Poco::Util::LayeredConfiguration & config)
|
||||
{
|
||||
/// If can add some headers from config
|
||||
if (tryAddHeadersFromConfig(response, config))
|
||||
if (tryAddHttpOptionHeadersFromConfig(response, config))
|
||||
{
|
||||
response.setKeepAlive(false);
|
||||
response.setStatusAndReason(HTTPResponse::HTTP_NO_CONTENT);
|
||||
@ -774,16 +774,11 @@ void HTTPHandler::processQuery(
|
||||
if (in_post_compressed && settings.http_native_compression_disable_checksumming_on_decompress)
|
||||
static_cast<CompressedReadBuffer &>(*in_post_maybe_compressed).disableChecksumming();
|
||||
|
||||
/// Add CORS header if 'add_http_cors_header' setting is turned on send * in Access-Control-Allow-Origin,
|
||||
/// or if config has http_options_response, which means that there
|
||||
/// are some headers to be sent, and the client passed Origin header.
|
||||
if (!request.get("Origin", "").empty())
|
||||
{
|
||||
if (config.has("http_options_response"))
|
||||
tryAddHeadersFromConfig(response, config);
|
||||
else if (settings.add_http_cors_header)
|
||||
used_output.out->addHeaderCORS(true);
|
||||
}
|
||||
/// Add CORS header if 'add_http_cors_header' setting is turned on send * in Access-Control-Allow-Origin
|
||||
/// Note that whether the header is added is determined by the settings, and we can only get the user settings after authentication.
|
||||
/// Once the authentication fails, the header can't be added.
|
||||
if (settings.add_http_cors_header && !request.get("Origin", "").empty() && !config.has("http_options_response"))
|
||||
used_output.out->addHeaderCORS(true);
|
||||
|
||||
auto append_callback = [context = context] (ProgressCallback callback)
|
||||
{
|
||||
@ -971,6 +966,10 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
|
||||
|
||||
response.setContentType("text/plain; charset=UTF-8");
|
||||
response.set("X-ClickHouse-Server-Display-Name", server_display_name);
|
||||
|
||||
if (!request.get("Origin", "").empty())
|
||||
tryAddHttpOptionHeadersFromConfig(response, server.config());
|
||||
|
||||
/// For keep-alive to work.
|
||||
if (request.getVersion() == HTTPServerRequest::HTTP_1_1)
|
||||
response.setChunkedTransferEncoding(true);
|
||||
|
||||
@ -1231,19 +1231,18 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
|
||||
for (const auto & disk_ptr : disks)
|
||||
defined_disk_names.insert(disk_ptr->getName());
|
||||
|
||||
for (const auto & [_, disk_ptr] : getContext()->getDisksMap())
|
||||
for (const auto & [disk_name, disk_ptr] : getContext()->getDisksMap())
|
||||
{
|
||||
/// In composable cache with the underlying source disk there might the following structure:
|
||||
/// DiskObjectStorage(CachedObjectStorage(...(CachedObjectStored(ObjectStorage)...)))
|
||||
/// In configuration file each of these layers has a different name, but data path
|
||||
/// (getPath() result) is the same. We need to take it into account here.
|
||||
if (disk_ptr->supportsCache())
|
||||
if (disk_ptr->supportsCache() && defined_disk_names.contains(disk_ptr->getName()))
|
||||
{
|
||||
if (defined_disk_names.contains(disk_ptr->getName()))
|
||||
{
|
||||
auto caches = disk_ptr->getCacheLayersNames();
|
||||
disk_names_wrapped_in_cache.insert(caches.begin(), caches.end());
|
||||
}
|
||||
auto caches = disk_ptr->getCacheLayersNames();
|
||||
disk_names_wrapped_in_cache.insert(caches.begin(), caches.end());
|
||||
LOG_TEST(log, "Cache layers for cache disk `{}`, inner disk `{}`: {}",
|
||||
disk_name, disk_ptr->getName(), fmt::join(caches, ", "));
|
||||
}
|
||||
}
|
||||
|
||||
@ -1262,8 +1261,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
|
||||
{
|
||||
throw Exception(
|
||||
ErrorCodes::UNKNOWN_DISK,
|
||||
"Part {} ({}) was found on disk {} which is not defined in the storage policy",
|
||||
backQuote(it->name()), backQuote(it->path()), backQuote(disk_name));
|
||||
"Part {} ({}) was found on disk {} which is not defined in the storage policy (defined disks: {}, wrapped disks: {})",
|
||||
backQuote(it->name()), backQuote(it->path()), backQuote(disk_name),
|
||||
fmt::join(defined_disk_names, ", "), fmt::join(disk_names_wrapped_in_cache, ", "));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -3511,8 +3511,8 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until, ContextPtr q
|
||||
k_inactive = static_cast<ssize_t>(inactive_parts_count_in_partition) - static_cast<ssize_t>(settings->inactive_parts_to_delay_insert);
|
||||
}
|
||||
|
||||
auto parts_to_delay_insert = query_settings.parts_to_delay_insert.changed ? query_settings.parts_to_delay_insert : settings->parts_to_delay_insert;
|
||||
auto parts_to_throw_insert = query_settings.parts_to_throw_insert.changed ? query_settings.parts_to_throw_insert : settings->parts_to_throw_insert;
|
||||
auto parts_to_delay_insert = query_settings.parts_to_delay_insert ? query_settings.parts_to_delay_insert : settings->parts_to_delay_insert;
|
||||
auto parts_to_throw_insert = query_settings.parts_to_throw_insert ? query_settings.parts_to_throw_insert : settings->parts_to_throw_insert;
|
||||
|
||||
if (parts_count_in_partition >= parts_to_throw_insert)
|
||||
{
|
||||
|
||||
@ -19,6 +19,9 @@ endif()
|
||||
if (TARGET ch_contrib::rdkafka)
|
||||
set(USE_RDKAFKA 1)
|
||||
endif()
|
||||
if (TARGET ch_rust::blake3)
|
||||
set(USE_BLAKE3 1)
|
||||
endif()
|
||||
if (TARGET OpenSSL::SSL)
|
||||
set(USE_SSL 1)
|
||||
endif()
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
<value>SHA224</value>
|
||||
<value>SHA256</value>
|
||||
<value>halfMD5</value>
|
||||
<value>blake3</value>
|
||||
</values>
|
||||
</substitution>
|
||||
<substitution>
|
||||
|
||||
@ -10,11 +10,11 @@ SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64(0, 'Asia/Istanbul'));
|
||||
2020-01-01 00:00:00
|
||||
SELECT toDateTime64(bitShiftLeft(toUInt64(1), 33), 2, 'Asia/Istanbul') FORMAT Null;
|
||||
SELECT toTimeZone(toDateTime(-2., 2), 'Asia/Istanbul');
|
||||
1970-01-01 02:00:00.00
|
||||
1970-01-01 01:59:58.00
|
||||
SELECT toDateTime64(-2., 2, 'Asia/Istanbul');
|
||||
1970-01-01 02:00:00.00
|
||||
1970-01-01 01:59:58.00
|
||||
SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Asia/Istanbul');
|
||||
2106-02-07 09:28:16.00
|
||||
2242-03-16 15:56:32.00
|
||||
SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Asia/Istanbul') FORMAT Null;
|
||||
-- These are outsize of extended range and hence clamped
|
||||
SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1), 35), 2, 'Asia/Istanbul');
|
||||
|
||||
@ -1,3 +1,13 @@
|
||||
2005-03-18 03:58:31.222
|
||||
2005-03-18 03:58:31.222
|
||||
2005-03-18 03:58:31.222
|
||||
1970-01-01 00:00:00.000000000
|
||||
1970-01-01 00:00:00.000000000
|
||||
1900-04-15 00:53:20.000000000
|
||||
1900-04-15 00:53:20.000000000
|
||||
1900-01-01 00:00:00.000000000
|
||||
1900-01-01 00:00:00.000000000
|
||||
1900-01-01 00:00:00.000000000
|
||||
1900-01-01 00:00:00.000000000
|
||||
2261-07-15 11:33:20.000000000
|
||||
2261-07-15 11:33:20.000000000
|
||||
|
||||
@ -1,3 +1,22 @@
|
||||
SELECT CAST(1111111111.222 AS DateTime64(3, 'Asia/Istanbul'));
|
||||
SELECT toDateTime(1111111111.222, 3, 'Asia/Istanbul');
|
||||
SELECT toDateTime64(1111111111.222, 3, 'Asia/Istanbul');
|
||||
|
||||
SELECT toDateTime64(0.0, 9, 'UTC') ;
|
||||
SELECT toDateTime64(0, 9, 'UTC');
|
||||
|
||||
SELECT toDateTime64(-2200000000.0, 9, 'UTC'); -- 1900-01-01 < value
|
||||
SELECT toDateTime64(-2200000000, 9, 'UTC');
|
||||
|
||||
SELECT toDateTime64(-2300000000.0, 9, 'UTC'); -- value < 1900-01-01
|
||||
SELECT toDateTime64(-2300000000, 9, 'UTC');
|
||||
|
||||
SELECT toDateTime64(-999999999999.0, 9, 'UTC'); -- value << 1900-01-01
|
||||
SELECT toDateTime64(-999999999999, 9, 'UTC');
|
||||
|
||||
SELECT toDateTime64(9200000000.0, 9, 'UTC'); -- value < 2262-04-11
|
||||
SELECT toDateTime64(9200000000, 9, 'UTC');
|
||||
|
||||
SELECT toDateTime64(9300000000.0, 9, 'UTC'); -- { serverError 407 } # 2262-04-11 < value
|
||||
SELECT toDateTime64(9300000000, 9, 'UTC'); -- { serverError 407 }
|
||||
|
||||
|
||||
@ -3,3 +3,8 @@
|
||||
< Access-Control-Allow-Headers: origin, x-requested-with
|
||||
< Access-Control-Allow-Methods: POST, GET, OPTIONS
|
||||
< Access-Control-Max-Age: 86400
|
||||
< HTTP/1.1 403 Forbidden
|
||||
< Access-Control-Allow-Origin: *
|
||||
< Access-Control-Allow-Headers: origin, x-requested-with
|
||||
< Access-Control-Allow-Methods: POST, GET, OPTIONS
|
||||
< Access-Control-Max-Age: 86400
|
||||
|
||||
@ -6,3 +6,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
# grep all fields, that should be set for CORS support (see CORS.xml)
|
||||
$CLICKHOUSE_CURL "${CLICKHOUSE_URL}" -X OPTIONS -vs 2>&1 | grep -E "HTTP/1.1 204 No Content|Access-Control-Allow-Origin|Access-Control-Allow-Headers|Access-Control-Allow-Methods|Access-Control-Max-Age"
|
||||
|
||||
# grep all fields, that should be set for CORS support (see CORS.xml)
|
||||
echo 'SELECT 1' | $CLICKHOUSE_CURL -X POST -H 'Origin: clickhouse-test' "${CLICKHOUSE_URL}&password=wrong_password" --data @- -vs 2>&1 | grep -E "HTTP/1.1 403 Forbidden|Access-Control-Allow-Origin|Access-Control-Allow-Headers|Access-Control-Allow-Methods|Access-Control-Max-Age"
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: race, zookeeper, no-parallel
|
||||
# Tags: race, zookeeper, no-parallel, disabled
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: race, zookeeper, no-parallel, no-backward-compatibility-check
|
||||
# Tags: race, zookeeper, no-parallel, no-backward-compatibility-check, disabled
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
|
||||
@ -0,0 +1,3 @@
|
||||
0C673DA1EF75D2DAA895483138340F041881EA975D57C1435D487F454A111B74
|
||||
007ED777B7A1CBA08D37BDA339EFABB42FA460D953070779903125B0F4D5FB5F
|
||||
E25232688E2A4D3A55174DECB33815A27B2A92DC8839E3CDA456105C259BB071
|
||||
@ -0,0 +1,5 @@
|
||||
-- Tags: no-fasttest
|
||||
|
||||
SELECT hex(blake3('test_1'));
|
||||
SELECT hex(blake3('test_2'));
|
||||
SELECT hex(blake3('test_3'));
|
||||
@ -13,15 +13,15 @@ export CLICKHOUSE_TEST_NAME
|
||||
export CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_NAME}_${CLICKHOUSE_DATABASE}"
|
||||
export CLICKHOUSE_TEST_UNIQUE_NAME="${CLICKHOUSE_TEST_NAME}_${CLICKHOUSE_DATABASE}"
|
||||
|
||||
[ -v CLICKHOUSE_CONFIG_CLIENT ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} "
|
||||
[ -v CLICKHOUSE_HOST ] && CLICKHOUSE_CLIENT_OPT0+=" --host=${CLICKHOUSE_HOST} "
|
||||
[ -v CLICKHOUSE_PORT_TCP ] && CLICKHOUSE_CLIENT_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} "
|
||||
[ -v CLICKHOUSE_PORT_TCP ] && CLICKHOUSE_BENCHMARK_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} "
|
||||
[ -v CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL ] && CLICKHOUSE_CLIENT_OPT0+=" --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL} "
|
||||
[ -v CLICKHOUSE_DATABASE ] && CLICKHOUSE_CLIENT_OPT0+=" --database=${CLICKHOUSE_DATABASE} "
|
||||
[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_CLIENT_OPT0+=" --log_comment $(printf '%q' ${CLICKHOUSE_LOG_COMMENT}) "
|
||||
[ -v CLICKHOUSE_DATABASE ] && CLICKHOUSE_BENCHMARK_OPT0+=" --database=${CLICKHOUSE_DATABASE} "
|
||||
[ -v CLICKHOUSE_LOG_COMMENT ] && CLICKHOUSE_BENCHMARK_OPT0+=" --log_comment $(printf '%q' ${CLICKHOUSE_LOG_COMMENT}) "
|
||||
[ -n "${CLICKHOUSE_CONFIG_CLIENT:-}" ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} "
|
||||
[ -n "${CLICKHOUSE_HOST:-}" ] && CLICKHOUSE_CLIENT_OPT0+=" --host=${CLICKHOUSE_HOST} "
|
||||
[ -n "${CLICKHOUSE_PORT_TCP:-}" ] && CLICKHOUSE_CLIENT_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} "
|
||||
[ -n "${CLICKHOUSE_PORT_TCP:-}" ] && CLICKHOUSE_BENCHMARK_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} "
|
||||
[ -n "${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL:-}" ] && CLICKHOUSE_CLIENT_OPT0+=" --send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL} "
|
||||
[ -n "${CLICKHOUSE_DATABASE:-}" ] && CLICKHOUSE_CLIENT_OPT0+=" --database=${CLICKHOUSE_DATABASE} "
|
||||
[ -n "${CLICKHOUSE_LOG_COMMENT:-}" ] && CLICKHOUSE_CLIENT_OPT0+=" --log_comment $(printf '%q' ${CLICKHOUSE_LOG_COMMENT}) "
|
||||
[ -n "${CLICKHOUSE_DATABASE:-}" ] && CLICKHOUSE_BENCHMARK_OPT0+=" --database=${CLICKHOUSE_DATABASE} "
|
||||
[ -n "${CLICKHOUSE_LOG_COMMENT:-}" ] && CLICKHOUSE_BENCHMARK_OPT0+=" --log_comment $(printf '%q' ${CLICKHOUSE_LOG_COMMENT}) "
|
||||
|
||||
export CLICKHOUSE_BINARY=${CLICKHOUSE_BINARY:="clickhouse"}
|
||||
# client
|
||||
@ -81,20 +81,20 @@ export CLICKHOUSE_PORT_KEEPER=${CLICKHOUSE_PORT_KEEPER:="9181"}
|
||||
export CLICKHOUSE_CLIENT_SECURE=${CLICKHOUSE_CLIENT_SECURE:=$(echo "${CLICKHOUSE_CLIENT}" | sed 's/--secure //' | sed 's/'"--port=${CLICKHOUSE_PORT_TCP}"'//g; s/$/'"--secure --accept-invalid-certificate --port=${CLICKHOUSE_PORT_TCP_SECURE}"'/g')}
|
||||
|
||||
# Add database and log comment to url params
|
||||
if [ -v CLICKHOUSE_URL_PARAMS ]
|
||||
if [ -n "${CLICKHOUSE_URL_PARAMS:-}" ]
|
||||
then
|
||||
export CLICKHOUSE_URL_PARAMS="${CLICKHOUSE_URL_PARAMS}&database=${CLICKHOUSE_DATABASE}"
|
||||
else
|
||||
export CLICKHOUSE_URL_PARAMS="database=${CLICKHOUSE_DATABASE}"
|
||||
fi
|
||||
# Note: missing url encoding of the log comment.
|
||||
[ -v CLICKHOUSE_LOG_COMMENT ] && export CLICKHOUSE_URL_PARAMS="${CLICKHOUSE_URL_PARAMS}&log_comment=${CLICKHOUSE_LOG_COMMENT}"
|
||||
[ -n "${CLICKHOUSE_LOG_COMMENT:-}" ] && export CLICKHOUSE_URL_PARAMS="${CLICKHOUSE_URL_PARAMS}&log_comment=${CLICKHOUSE_LOG_COMMENT}"
|
||||
|
||||
export CLICKHOUSE_URL=${CLICKHOUSE_URL:="${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/"}
|
||||
export CLICKHOUSE_URL_HTTPS=${CLICKHOUSE_URL_HTTPS:="https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTPS}/"}
|
||||
|
||||
# Add url params to url
|
||||
if [ -v CLICKHOUSE_URL_PARAMS ]
|
||||
if [ -n "${CLICKHOUSE_URL_PARAMS:-}" ]
|
||||
then
|
||||
export CLICKHOUSE_URL="${CLICKHOUSE_URL}?${CLICKHOUSE_URL_PARAMS}"
|
||||
export CLICKHOUSE_URL_HTTPS="${CLICKHOUSE_URL_HTTPS}?${CLICKHOUSE_URL_PARAMS}"
|
||||
@ -117,10 +117,10 @@ mkdir -p ${CLICKHOUSE_TMP}
|
||||
export MYSQL_CLIENT_BINARY=${MYSQL_CLIENT_BINARY:="mysql"}
|
||||
export MYSQL_CLIENT_CLICKHOUSE_USER=${MYSQL_CLIENT_CLICKHOUSE_USER:="default"}
|
||||
# Avoids "Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock'" when connecting to localhost
|
||||
[ -v CLICKHOUSE_HOST ] && MYSQL_CLIENT_OPT0+=" --protocol tcp "
|
||||
[ -v CLICKHOUSE_HOST ] && MYSQL_CLIENT_OPT0+=" --host ${CLICKHOUSE_HOST} "
|
||||
[ -v CLICKHOUSE_PORT_MYSQL ] && MYSQL_CLIENT_OPT0+=" --port ${CLICKHOUSE_PORT_MYSQL} "
|
||||
[ -v CLICKHOUSE_DATABASE ] && MYSQL_CLIENT_OPT0+=" --database ${CLICKHOUSE_DATABASE} "
|
||||
[ -n "${CLICKHOUSE_HOST:-}" ] && MYSQL_CLIENT_OPT0+=" --protocol tcp "
|
||||
[ -n "${CLICKHOUSE_HOST:-}" ] && MYSQL_CLIENT_OPT0+=" --host ${CLICKHOUSE_HOST} "
|
||||
[ -n "${CLICKHOUSE_PORT_MYSQL:-}" ] && MYSQL_CLIENT_OPT0+=" --port ${CLICKHOUSE_PORT_MYSQL} "
|
||||
[ -n "${CLICKHOUSE_DATABASE:-}" ] && MYSQL_CLIENT_OPT0+=" --database ${CLICKHOUSE_DATABASE} "
|
||||
MYSQL_CLIENT_OPT0+=" --user ${MYSQL_CLIENT_CLICKHOUSE_USER} "
|
||||
export MYSQL_CLIENT_OPT="${MYSQL_CLIENT_OPT0:-} ${MYSQL_CLIENT_OPT:-}"
|
||||
export MYSQL_CLIENT=${MYSQL_CLIENT:="$MYSQL_CLIENT_BINARY ${MYSQL_CLIENT_OPT:-}"}
|
||||
|
||||
@ -502,3 +502,7 @@ zkcopy
|
||||
zlib
|
||||
znodes
|
||||
zstd
|
||||
NYPD
|
||||
denormalizing
|
||||
ConnectionDetails
|
||||
SelfManaged
|
||||
|
||||
Loading…
Reference in New Issue
Block a user