diff --git a/contrib/NuRaft b/contrib/NuRaft index 95d6bbba579..2a1bf7d87b4 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit 95d6bbba579b3a4e4c2dede954f541ff6f3dba51 +Subproject commit 2a1bf7d87b4a03561fc66fbb49cee8a288983c5d diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index cff49a805fa..0665ab7560f 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -1,5 +1,5 @@ # docker build -t yandex/clickhouse-integration-tests-runner . -FROM ubuntu:18.04 +FROM ubuntu:20.04 RUN apt-get update \ && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ @@ -14,7 +14,6 @@ RUN apt-get update \ wget \ git \ iproute2 \ - module-init-tools \ cgroupfs-mount \ python3-pip \ tzdata \ @@ -42,7 +41,6 @@ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ENV DOCKER_CHANNEL stable -ENV DOCKER_VERSION 5:19.03.13~3-0~ubuntu-bionic RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" @@ -66,17 +64,18 @@ RUN python3 -m pip install \ dict2xml \ dicttoxml \ docker \ - docker-compose==1.22.0 \ + docker-compose==1.28.2 \ grpcio \ grpcio-tools \ kafka-python \ kazoo \ minio \ protobuf \ - psycopg2-binary==2.7.5 \ + psycopg2-binary==2.8.6 \ pymongo \ pytest \ pytest-timeout \ + pytest-xdist \ redis \ tzlocal \ urllib3 \ @@ -86,6 +85,7 @@ RUN python3 -m pip install \ COPY modprobe.sh /usr/local/bin/modprobe COPY dockerd-entrypoint.sh /usr/local/bin/ COPY compose/ /compose/ +COPY misc/ /misc/ RUN set -x \ && addgroup --system dockremap \ @@ -94,7 +94,6 @@ RUN set -x \ && echo 'dockremap:165536:65536' >> /etc/subuid \ && echo 'dockremap:165536:65536' >> /etc/subgid -VOLUME /var/lib/docker EXPOSE 2375 ENTRYPOINT ["dockerd-entrypoint.sh"] CMD ["sh", "-c", "pytest $PYTEST_OPTS"] diff --git a/docker/test/integration/runner/compose/docker_compose_cassandra.yml b/docker/test/integration/runner/compose/docker_compose_cassandra.yml index c5cdfac5ce7..b6190a11d73 100644 --- a/docker/test/integration/runner/compose/docker_compose_cassandra.yml +++ b/docker/test/integration/runner/compose/docker_compose_cassandra.yml @@ -1,7 +1,5 @@ version: '2.3' services: cassandra1: - image: cassandra + image: cassandra:4.0 restart: always - ports: - - 9043:9042 diff --git a/docker/test/integration/runner/compose/docker_compose_hdfs.yml b/docker/test/integration/runner/compose/docker_compose_hdfs.yml index 43dd1aa43d3..12bdc292b2f 100644 --- a/docker/test/integration/runner/compose/docker_compose_hdfs.yml +++ b/docker/test/integration/runner/compose/docker_compose_hdfs.yml @@ -5,6 +5,10 @@ services: hostname: hdfs1 restart: always ports: - - 50075:50075 - - 50070:50070 + - ${HDFS_NAME_EXTERNAL_PORT}:${HDFS_NAME_INTERNAL_PORT} #50070 + - ${HDFS_DATA_EXTERNAL_PORT}:${HDFS_DATA_INTERNAL_PORT} #50075 entrypoint: /etc/bootstrap.sh -d + volumes: + - type: ${HDFS_FS:-tmpfs} + source: ${HDFS_LOGS:-} + target: /usr/local/hadoop/logs \ No newline at end of file diff --git a/docker/test/integration/runner/compose/docker_compose_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kafka.yml index b77542f7e11..4552e6f0696 100644 --- a/docker/test/integration/runner/compose/docker_compose_kafka.yml +++ b/docker/test/integration/runner/compose/docker_compose_kafka.yml @@ -15,10 +15,11 @@ services: image: confluentinc/cp-kafka:5.2.0 hostname: kafka1 ports: - - "9092:9092" + - ${KAFKA_EXTERNAL_PORT}:${KAFKA_EXTERNAL_PORT} environment: - KAFKA_ADVERTISED_LISTENERS: INSIDE://localhost:9092,OUTSIDE://kafka1:19092 - KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:19092 + KAFKA_ADVERTISED_LISTENERS: INSIDE://localhost:${KAFKA_EXTERNAL_PORT},OUTSIDE://kafka1:19092 + KAFKA_ADVERTISED_HOST_NAME: kafka1 + KAFKA_LISTENERS: INSIDE://0.0.0.0:${KAFKA_EXTERNAL_PORT},OUTSIDE://0.0.0.0:19092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE KAFKA_BROKER_ID: 1 @@ -34,7 +35,7 @@ services: image: confluentinc/cp-schema-registry:5.2.0 hostname: schema-registry ports: - - "8081:8081" + - ${SCHEMA_REGISTRY_EXTERNAL_PORT}:${SCHEMA_REGISTRY_INTERNAL_PORT} environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml index e2e15975e22..9ffd0037b50 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml @@ -11,10 +11,12 @@ services: - ${KERBERIZED_HDFS_DIR}/../../hdfs_configs/bootstrap.sh:/etc/bootstrap.sh:ro - ${KERBERIZED_HDFS_DIR}/secrets:/usr/local/hadoop/etc/hadoop/conf - ${KERBERIZED_HDFS_DIR}/secrets/krb_long.conf:/etc/krb5.conf:ro + - type: ${KERBERIZED_HDFS_FS:-tmpfs} + source: ${KERBERIZED_HDFS_LOGS:-} + target: /var/log/hadoop-hdfs ports: - - 1006:1006 - - 50070:50070 - - 9010:9010 + - ${KERBERIZED_HDFS_NAME_EXTERNAL_PORT}:${KERBERIZED_HDFS_NAME_INTERNAL_PORT} #50070 + - ${KERBERIZED_HDFS_DATA_EXTERNAL_PORT}:${KERBERIZED_HDFS_DATA_INTERNAL_PORT} #1006 depends_on: - hdfskerberos entrypoint: /etc/bootstrap.sh -d diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml index 64a3ef3e956..081b90c4f27 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml @@ -23,13 +23,13 @@ services: # restart: always hostname: kerberized_kafka1 ports: - - "9092:9092" - - "9093:9093" + - ${KERBERIZED_KAFKA_EXTERNAL_PORT}:${KERBERIZED_KAFKA_EXTERNAL_PORT} environment: - KAFKA_LISTENERS: OUTSIDE://:19092,UNSECURED_OUTSIDE://:19093,UNSECURED_INSIDE://:9093 - KAFKA_ADVERTISED_LISTENERS: OUTSIDE://kerberized_kafka1:19092,UNSECURED_OUTSIDE://kerberized_kafka1:19093,UNSECURED_INSIDE://localhost:9093 + KAFKA_LISTENERS: OUTSIDE://:19092,UNSECURED_OUTSIDE://:19093,UNSECURED_INSIDE://0.0.0.0:${KERBERIZED_KAFKA_EXTERNAL_PORT} + KAFKA_ADVERTISED_LISTENERS: OUTSIDE://kerberized_kafka1:19092,UNSECURED_OUTSIDE://kerberized_kafka1:19093,UNSECURED_INSIDE://localhost:${KERBERIZED_KAFKA_EXTERNAL_PORT} # KAFKA_LISTENERS: INSIDE://kerberized_kafka1:9092,OUTSIDE://kerberized_kafka1:19092 # KAFKA_ADVERTISED_LISTENERS: INSIDE://localhost:9092,OUTSIDE://kerberized_kafka1:19092 + KAFKA_ADVERTISED_HOST_NAME: kerberized_kafka1 KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: GSSAPI KAFKA_SASL_ENABLED_MECHANISMS: GSSAPI KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka diff --git a/docker/test/integration/runner/compose/docker_compose_minio.yml b/docker/test/integration/runner/compose/docker_compose_minio.yml index dbb29f9711a..96a5f8bdc31 100644 --- a/docker/test/integration/runner/compose/docker_compose_minio.yml +++ b/docker/test/integration/runner/compose/docker_compose_minio.yml @@ -6,8 +6,8 @@ services: volumes: - data1-1:/data1 - ${MINIO_CERTS_DIR:-}:/certs - ports: - - "9001:9001" + expose: + - ${MINIO_PORT} environment: MINIO_ACCESS_KEY: minio MINIO_SECRET_KEY: minio123 @@ -20,14 +20,14 @@ services: # HTTP proxies for Minio. proxy1: image: yandex/clickhouse-s3-proxy - ports: + expose: - "8080" # Redirect proxy port - "80" # Reverse proxy port - "443" # Reverse proxy port (secure) proxy2: image: yandex/clickhouse-s3-proxy - ports: + expose: - "8080" - "80" - "443" @@ -35,7 +35,7 @@ services: # Empty container to run proxy resolver. resolver: image: yandex/clickhouse-python-bottle - ports: + expose: - "8080" tty: true depends_on: diff --git a/docker/test/integration/runner/compose/docker_compose_mongo.yml b/docker/test/integration/runner/compose/docker_compose_mongo.yml index 6c98fde2303..e794966bd08 100644 --- a/docker/test/integration/runner/compose/docker_compose_mongo.yml +++ b/docker/test/integration/runner/compose/docker_compose_mongo.yml @@ -7,5 +7,5 @@ services: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: clickhouse ports: - - 27018:27017 + - ${MONGO_EXTERNAL_PORT}:${MONGO_INTERNAL_PORT} command: --profile=2 --verbose diff --git a/docker/test/integration/runner/compose/docker_compose_mysql.yml b/docker/test/integration/runner/compose/docker_compose_mysql.yml index 5b15d517f37..c9b45af6563 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql.yml @@ -1,10 +1,24 @@ version: '2.3' services: - mysql1: + mysql57: image: mysql:5.7 restart: always environment: MYSQL_ROOT_PASSWORD: clickhouse - ports: - - 3308:3306 - command: --server_id=100 --log-bin='mysql-bin-1.log' --default-time-zone='+3:00' --gtid-mode="ON" --enforce-gtid-consistency + MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST} + DATADIR: /mysql/ + expose: + - ${MYSQL_PORT} + command: --server_id=100 + --log-bin='mysql-bin-1.log' + --default-time-zone='+3:00' + --gtid-mode="ON" + --enforce-gtid-consistency + --log-error-verbosity=3 + --log-error=/mysql/error.log + --general-log=ON + --general-log-file=/mysql/general.log + volumes: + - type: ${MYSQL_LOGS_FS:-tmpfs} + source: ${MYSQL_LOGS:-} + target: /mysql/ \ No newline at end of file diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialize_mysql.yml b/docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialize_mysql.yml index 5aa13ba91c7..ba693fd9fb4 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialize_mysql.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialize_mysql.yml @@ -12,3 +12,10 @@ services: --gtid-mode="ON" --enforce-gtid-consistency --log-error-verbosity=3 + --log-error=/var/log/mysqld/error.log + --general-log=ON + --general-log-file=/var/log/mysqld/general.log + volumes: + - type: ${MYSQL_LOGS_FS:-tmpfs} + source: ${MYSQL_LOGS:-} + target: /var/log/mysqld/ \ No newline at end of file diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml b/docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml new file mode 100644 index 00000000000..e13076c4e2e --- /dev/null +++ b/docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml @@ -0,0 +1,23 @@ +version: '2.3' +services: + mysql80: + image: mysql:8.0 + restart: always + environment: + MYSQL_ROOT_PASSWORD: clickhouse + MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST} + DATADIR: /mysql/ + expose: + - ${MYSQL8_PORT} + command: --server_id=100 --log-bin='mysql-bin-1.log' + --default_authentication_plugin='mysql_native_password' + --default-time-zone='+3:00' --gtid-mode="ON" + --enforce-gtid-consistency + --log-error-verbosity=3 + --log-error=/mysql/error.log + --general-log=ON + --general-log-file=/mysql/general.log + volumes: + - type: ${MYSQL8_LOGS_FS:-tmpfs} + source: ${MYSQL8_LOGS:-} + target: /mysql/ \ No newline at end of file diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_8_0_for_materialize_mysql.yml b/docker/test/integration/runner/compose/docker_compose_mysql_8_0_for_materialize_mysql.yml deleted file mode 100644 index 93bfee35caf..00000000000 --- a/docker/test/integration/runner/compose/docker_compose_mysql_8_0_for_materialize_mysql.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '2.3' -services: - mysql8_0: - image: mysql:8.0 - restart: 'no' - environment: - MYSQL_ROOT_PASSWORD: clickhouse - ports: - - 3309:3306 - command: --server_id=100 --log-bin='mysql-bin-1.log' - --default_authentication_plugin='mysql_native_password' - --default-time-zone='+3:00' - --gtid-mode="ON" - --enforce-gtid-consistency - --log-error-verbosity=3 diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_client.yml b/docker/test/integration/runner/compose/docker_compose_mysql_client.yml index 5e4565d64c3..5b37b6e6c09 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_client.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: - mysql1: + mysql_client: image: mysql:5.7 restart: always environment: diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml b/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml index d0674362709..6ced7536812 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml @@ -5,19 +5,64 @@ services: restart: always environment: MYSQL_ROOT_PASSWORD: clickhouse - ports: - - 3348:3306 + MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST} + DATADIR: /mysql/ + expose: + - ${MYSQL_CLUSTER_PORT} + command: --server_id=100 + --log-bin='mysql-bin-2.log' + --default-time-zone='+3:00' + --gtid-mode="ON" + --enforce-gtid-consistency + --log-error-verbosity=3 + --log-error=/mysql/2_error.log + --general-log=ON + --general-log-file=/mysql/2_general.log + volumes: + - type: ${MYSQL_CLUSTER_LOGS_FS:-tmpfs} + source: ${MYSQL_CLUSTER_LOGS:-} + target: /mysql/ mysql3: image: mysql:5.7 restart: always environment: MYSQL_ROOT_PASSWORD: clickhouse - ports: - - 3388:3306 + MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST} + DATADIR: /mysql/ + expose: + - ${MYSQL_CLUSTER_PORT} + command: --server_id=100 + --log-bin='mysql-bin-3.log' + --default-time-zone='+3:00' + --gtid-mode="ON" + --enforce-gtid-consistency + --log-error-verbosity=3 + --log-error=/mysql/3_error.log + --general-log=ON + --general-log-file=/mysql/3_general.log + volumes: + - type: ${MYSQL_CLUSTER_LOGS_FS:-tmpfs} + source: ${MYSQL_CLUSTER_LOGS:-} + target: /mysql/ mysql4: image: mysql:5.7 restart: always environment: MYSQL_ROOT_PASSWORD: clickhouse - ports: - - 3368:3306 + MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST} + DATADIR: /mysql/ + expose: + - ${MYSQL_CLUSTER_PORT} + command: --server_id=100 + --log-bin='mysql-bin-4.log' + --default-time-zone='+3:00' + --gtid-mode="ON" + --enforce-gtid-consistency + --log-error-verbosity=3 + --log-error=/mysql/4_error.log + --general-log=ON + --general-log-file=/mysql/4_general.log + volumes: + - type: ${MYSQL_CLUSTER_LOGS_FS:-tmpfs} + source: ${MYSQL_CLUSTER_LOGS:-} + target: /mysql/ \ No newline at end of file diff --git a/docker/test/integration/runner/compose/docker_compose_postgres.yml b/docker/test/integration/runner/compose/docker_compose_postgres.yml index 5657352e1b3..c4a506ad356 100644 --- a/docker/test/integration/runner/compose/docker_compose_postgres.yml +++ b/docker/test/integration/runner/compose/docker_compose_postgres.yml @@ -2,12 +2,24 @@ version: '2.3' services: postgres1: image: postgres + command: ["postgres", "-c", "logging_collector=on", "-c", "log_directory=/postgres/logs", "-c", "log_filename=postgresql.log", "-c", "log_statement=all"] restart: always - environment: - POSTGRES_PASSWORD: mysecretpassword - ports: - - 5432:5432 + expose: + - ${POSTGRES_PORT} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 networks: - default: - aliases: - - postgre-sql.local + default: + aliases: + - postgre-sql.local + environment: + POSTGRES_HOST_AUTH_METHOD: "trust" + POSTGRES_PASSWORD: mysecretpassword + PGDATA: /postgres/data + volumes: + - type: ${POSTGRES_LOGS_FS:-tmpfs} + source: ${POSTGRES_DIR:-} + target: /postgres/ \ No newline at end of file diff --git a/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml b/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml index d04c8a2f3a6..94b941b74da 100644 --- a/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml +++ b/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml @@ -2,22 +2,43 @@ version: '2.3' services: postgres2: image: postgres + command: ["postgres", "-c", "logging_collector=on", "-c", "log_directory=/postgres/logs", "-c", "log_filename=postgresql.log", "-c", "log_statement=all"] restart: always environment: + POSTGRES_HOST_AUTH_METHOD: "trust" POSTGRES_PASSWORD: mysecretpassword - ports: - - 5421:5432 + PGDATA: /postgres/data + expose: + - ${POSTGRES_PORT} + volumes: + - type: ${POSTGRES_LOGS_FS:-tmpfs} + source: ${POSTGRES2_DIR:-} + target: /postgres/ postgres3: image: postgres + command: ["postgres", "-c", "logging_collector=on", "-c", "log_directory=/postgres/logs", "-c", "log_filename=postgresql.log", "-c", "log_statement=all"] restart: always environment: + POSTGRES_HOST_AUTH_METHOD: "trust" POSTGRES_PASSWORD: mysecretpassword - ports: - - 5441:5432 + PGDATA: /postgres/data + expose: + - ${POSTGRES_PORT} + volumes: + - type: ${POSTGRES_LOGS_FS:-tmpfs} + source: ${POSTGRES3_DIR:-} + target: /postgres/ postgres4: image: postgres + command: ["postgres", "-c", "logging_collector=on", "-c", "log_directory=/postgres/logs", "-c", "log_filename=postgresql.log", "-c", "log_statement=all"] restart: always environment: + POSTGRES_HOST_AUTH_METHOD: "trust" POSTGRES_PASSWORD: mysecretpassword - ports: - - 5461:5432 + PGDATA: /postgres/data + expose: + - ${POSTGRES_PORT} + volumes: + - type: ${POSTGRES_LOGS_FS:-tmpfs} + source: ${POSTGRES4_DIR:-} + target: /postgres/ \ No newline at end of file diff --git a/docker/test/integration/runner/compose/docker_compose_postgesql.yml b/docker/test/integration/runner/compose/docker_compose_postgresql.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_postgesql.yml rename to docker/test/integration/runner/compose/docker_compose_postgresql.yml diff --git a/docker/test/integration/runner/compose/docker_compose_postgesql_java_client.yml b/docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml similarity index 100% rename from docker/test/integration/runner/compose/docker_compose_postgesql_java_client.yml rename to docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml diff --git a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml index 1e9c3777505..99e0ea8e280 100644 --- a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml +++ b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml @@ -2,11 +2,15 @@ version: '2.3' services: rabbitmq1: - image: rabbitmq:3-management + image: rabbitmq:3-management-alpine hostname: rabbitmq1 - ports: - - "5672:5672" - - "15672:15672" + expose: + - ${RABBITMQ_PORT} environment: RABBITMQ_DEFAULT_USER: "root" RABBITMQ_DEFAULT_PASS: "clickhouse" + RABBITMQ_LOG_BASE: /rabbitmq_logs/ + volumes: + - type: ${RABBITMQ_LOGS_FS:-tmpfs} + source: ${RABBITMQ_LOGS:-} + target: /rabbitmq_logs/ diff --git a/docker/test/integration/runner/compose/docker_compose_redis.yml b/docker/test/integration/runner/compose/docker_compose_redis.yml index 3d834aadaa4..1bf67a6c44e 100644 --- a/docker/test/integration/runner/compose/docker_compose_redis.yml +++ b/docker/test/integration/runner/compose/docker_compose_redis.yml @@ -4,5 +4,5 @@ services: image: redis restart: always ports: - - 6380:6379 + - ${REDIS_EXTERNAL_PORT}:${REDIS_INTERNAL_PORT} command: redis-server --requirepass "clickhouse" --databases 32 diff --git a/docker/test/integration/runner/compose/docker_compose_zookeeper_secure.yml b/docker/test/integration/runner/compose/docker_compose_zookeeper_secure.yml new file mode 100644 index 00000000000..7a1c32e0023 --- /dev/null +++ b/docker/test/integration/runner/compose/docker_compose_zookeeper_secure.yml @@ -0,0 +1,75 @@ +version: '2.3' +services: + zoo1: + image: zookeeper:3.6.2 + restart: always + environment: + ZOO_TICK_TIME: 500 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 + ZOO_MY_ID: 1 + JVMFLAGS: -Dzookeeper.forceSync=no + ZOO_SECURE_CLIENT_PORT: $ZOO_SECURE_CLIENT_PORT + command: ["zkServer.sh", "start-foreground"] + entrypoint: /zookeeper-ssl-entrypoint.sh + volumes: + - type: bind + source: /misc/zookeeper-ssl-entrypoint.sh + target: /zookeeper-ssl-entrypoint.sh + - type: bind + source: /misc/client.crt + target: /clickhouse-config/client.crt + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA1:-} + target: /data + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA_LOG1:-} + target: /datalog + zoo2: + image: zookeeper:3.6.2 + restart: always + environment: + ZOO_TICK_TIME: 500 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888 + ZOO_MY_ID: 2 + JVMFLAGS: -Dzookeeper.forceSync=no + ZOO_SECURE_CLIENT_PORT: $ZOO_SECURE_CLIENT_PORT + + command: ["zkServer.sh", "start-foreground"] + entrypoint: /zookeeper-ssl-entrypoint.sh + volumes: + - type: bind + source: /misc/zookeeper-ssl-entrypoint.sh + target: /zookeeper-ssl-entrypoint.sh + - type: bind + source: /misc/client.crt + target: /clickhouse-config/client.crt + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA2:-} + target: /data + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA_LOG2:-} + target: /datalog + zoo3: + image: zookeeper:3.6.2 + restart: always + environment: + ZOO_TICK_TIME: 500 + ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181 + ZOO_MY_ID: 3 + JVMFLAGS: -Dzookeeper.forceSync=no + ZOO_SECURE_CLIENT_PORT: $ZOO_SECURE_CLIENT_PORT + command: ["zkServer.sh", "start-foreground"] + entrypoint: /zookeeper-ssl-entrypoint.sh + volumes: + - type: bind + source: /misc/zookeeper-ssl-entrypoint.sh + target: /zookeeper-ssl-entrypoint.sh + - type: bind + source: /misc/client.crt + target: /clickhouse-config/client.crt + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA3:-} + target: /data + - type: ${ZK_FS:-tmpfs} + source: ${ZK_DATA_LOG3:-} + target: /datalog diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh index 4d25ca60c5e..626b6883b91 100755 --- a/docker/test/integration/runner/dockerd-entrypoint.sh +++ b/docker/test/integration/runner/dockerd-entrypoint.sh @@ -2,17 +2,17 @@ set -e mkdir -p /etc/docker/ -cat > /etc/docker/daemon.json << EOF -{ +echo '{ "ipv6": true, "fixed-cidr-v6": "fd00::/8", "ip-forward": true, + "log-level": "debug", + "storage-driver": "overlay2", "insecure-registries" : ["dockerhub-proxy.sas.yp-c.yandex.net:5000"], "registry-mirrors" : ["http://dockerhub-proxy.sas.yp-c.yandex.net:5000"] -} -EOF +}' | dd of=/etc/docker/daemon.json -dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 &>/var/log/somefile & +dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 --default-address-pool base=172.17.0.0/12,size=24 &>/ClickHouse/tests/integration/dockerd.log & set +e reties=0 diff --git a/docker/test/integration/runner/misc/client.crt b/docker/test/integration/runner/misc/client.crt new file mode 100644 index 00000000000..7ade2d96273 --- /dev/null +++ b/docker/test/integration/runner/misc/client.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/tests/integration/helpers/zookeeper-ssl-entrypoint.sh b/docker/test/integration/runner/misc/zookeeper-ssl-entrypoint.sh similarity index 97% rename from tests/integration/helpers/zookeeper-ssl-entrypoint.sh rename to docker/test/integration/runner/misc/zookeeper-ssl-entrypoint.sh index 3ddb21881d6..9748a5e8ce9 100755 --- a/tests/integration/helpers/zookeeper-ssl-entrypoint.sh +++ b/docker/test/integration/runner/misc/zookeeper-ssl-entrypoint.sh @@ -81,8 +81,8 @@ if [[ ! -f "$ZOO_DATA_DIR/myid" ]]; then echo "${ZOO_MY_ID:-1}" > "$ZOO_DATA_DIR/myid" fi -mkdir -p $(dirname $ZOO_SSL_KEYSTORE_LOCATION) -mkdir -p $(dirname $ZOO_SSL_TRUSTSTORE_LOCATION) +mkdir -p "$(dirname $ZOO_SSL_KEYSTORE_LOCATION)" +mkdir -p "$(dirname $ZOO_SSL_TRUSTSTORE_LOCATION)" if [[ ! -f "$ZOO_SSL_KEYSTORE_LOCATION" ]]; then keytool -genkeypair -alias zookeeper -keyalg RSA -validity 365 -keysize 2048 -dname "cn=zookeeper" -keypass password -keystore $ZOO_SSL_KEYSTORE_LOCATION -storepass password -deststoretype pkcs12 diff --git a/docs/en/engines/table-engines/special/dictionary.md b/docs/en/engines/table-engines/special/dictionary.md index a6d6f296673..d76adebe01e 100644 --- a/docs/en/engines/table-engines/special/dictionary.md +++ b/docs/en/engines/table-engines/special/dictionary.md @@ -94,4 +94,6 @@ select * from products limit 1; └───────────────┴─────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/dictionary/) +**See Also** + +- [Dictionary function](../../../sql-reference/table-functions/dictionary.md#dictionary-function) diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 10ea46098d4..fc5c887c92e 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -123,6 +123,19 @@ The `Insert` command creates one or more blocks (parts). When inserting into Rep A large number of `replicated_deduplication_window` slows down `Inserts` because it needs to compare more entries. The hash sum is calculated from the composition of the field names and types and the data of the inserted part (stream of bytes). +## non_replicated_deduplication_window {#non-replicated-deduplication-window} + +The number of the most recently inserted blocks in the non-replicated [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) table for which hash sums are stored to check for duplicates. + +Possible values: + +- Any positive integer. +- 0 (disable deduplication). + +Default value: 0. + +A deduplication mechanism is used, similar to replicated tables (see [replicated_deduplication_window](#replicated-deduplication-window) setting). The hash sums of the created parts are written to a local file on a disk. + ## replicated_deduplication_window_seconds {#replicated-deduplication-window-seconds} The number of seconds after which the hash sums of the inserted blocks are removed from Zookeeper. diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 8e97171d31b..e82cb4882a0 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -506,3 +506,256 @@ Solution: Write in the GROUP BY query SearchPhrase HAVING uniqUpTo(4)(UserID) >= ## sumMapFiltered(keys_to_keep)(keys, values) {#summapfilteredkeys-to-keepkeys-values} Same behavior as [sumMap](../../sql-reference/aggregate-functions/reference/summap.md#agg_functions-summap) except that an array of keys is passed as a parameter. This can be especially useful when working with a high cardinality of keys. + +## sequenceNextNode {#sequenceNextNode} + +Returns a value of next event that matched an event chain. + +_Experimental function, `SET allow_experimental_funnel_functions = 1` to enable it._ + +**Syntax** + +``` sql +sequenceNextNode(direction, base)(timestamp, event_column, base_condition, event1, event2, event3, ...) +``` + +**Parameters** +- `direction` - Used to navigate to directions. + - forward : Moving forward + - backward: Moving backward + +- `base` - Used to set the base point. + - head : Set the base point to the first event + - tail : Set the base point to the last event + - first_match : Set the base point to the first matched event1 + - last_match : Set the base point to the last matched event1 + +**Arguments** +- `timestamp` — Name of the column containing the timestamp. Data types supported: `Date`, `DateTime` and other unsigned integer types. +- `event_column` — Name of the column containing the value of the next event to be returned. Data types supported: `String` and `Nullable(String)` +- `base_condition` — Condition that the base point must fulfill. +- `cond` — Conditions describing the chain of events. `UInt8` + +**Returned value** +- `event_column[next_index]` - if the pattern is matched and next value exists. +- `NULL` - if the pattern isn’t matched or next value doesn't exist. + +Type: `Nullable(String)`. + +**Example** + +It can be used when events are A->B->C->E->F and you want to know the event following B->C, which is E. + +The query statement searching the event following A->B : + +``` sql +CREATE TABLE test_flow ( + dt DateTime, + id int, + page String) +ENGINE = MergeTree() +PARTITION BY toYYYYMMDD(dt) +ORDER BY id; + +INSERT INTO test_flow VALUES (1, 1, 'A') (2, 1, 'B') (3, 1, 'C') (4, 1, 'E') (5, 1, 'F'); + +SELECT id, sequenceNextNode('forward', 'head')(dt, page, page = 'A', page = 'A', page = 'B') as next_flow FROM test_flow GROUP BY id; +``` + +Result: + +``` text +┌─id─┬─next_flow─┐ +│ 1 │ C │ +└────┴───────────┘ +``` + +**Behavior for `forward` and `head`** + +```SQL +ALTER TABLE test_flow DELETE WHERE 1 = 1 settings mutations_sync = 1; + +INSERT INTO test_flow VALUES (1, 1, 'Home') (2, 1, 'Gift') (3, 1, 'Exit'); +INSERT INTO test_flow VALUES (1, 2, 'Home') (2, 2, 'Home') (3, 2, 'Gift') (4, 2, 'Basket'); +INSERT INTO test_flow VALUES (1, 3, 'Gift') (2, 3, 'Home') (3, 3, 'Gift') (4, 3, 'Basket'); +``` + +```SQL +SELECT id, sequenceNextNode('forward', 'head')(dt, page, page = 'Home', page = 'Home', page = 'Gift') FROM test_flow GROUP BY id; + + dt id page + 1970-01-01 09:00:01 1 Home // Base point, Matched with Home + 1970-01-01 09:00:02 1 Gift // Matched with Gift + 1970-01-01 09:00:03 1 Exit // The result + + 1970-01-01 09:00:01 2 Home // Base point, Matched with Home + 1970-01-01 09:00:02 2 Home // Unmatched with Gift + 1970-01-01 09:00:03 2 Gift + 1970-01-01 09:00:04 2 Basket + + 1970-01-01 09:00:01 3 Gift // Base point, Unmatched with Home + 1970-01-01 09:00:02 3 Home + 1970-01-01 09:00:03 3 Gift + 1970-01-01 09:00:04 3 Basket +``` + +**Behavior for `backward` and `tail`** + +```SQL +SELECT id, sequenceNextNode('backward', 'tail')(dt, page, page = 'Basket', page = 'Basket', page = 'Gift') FROM test_flow GROUP BY id; + + dt id page +1970-01-01 09:00:01 1 Home +1970-01-01 09:00:02 1 Gift +1970-01-01 09:00:03 1 Exit // Base point, Unmatched with Basket + +1970-01-01 09:00:01 2 Home +1970-01-01 09:00:02 2 Home // The result +1970-01-01 09:00:03 2 Gift // Matched with Gift +1970-01-01 09:00:04 2 Basket // Base point, Matched with Basket + +1970-01-01 09:00:01 3 Gift +1970-01-01 09:00:02 3 Home // The result +1970-01-01 09:00:03 3 Gift // Base point, Matched with Gift +1970-01-01 09:00:04 3 Basket // Base point, Matched with Basket +``` + + +**Behavior for `forward` and `first_match`** + +```SQL +SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', page = 'Gift') FROM test_flow GROUP BY id; + + dt id page +1970-01-01 09:00:01 1 Home +1970-01-01 09:00:02 1 Gift // Base point +1970-01-01 09:00:03 1 Exit // The result + +1970-01-01 09:00:01 2 Home +1970-01-01 09:00:02 2 Home +1970-01-01 09:00:03 2 Gift // Base point +1970-01-01 09:00:04 2 Basket The result + +1970-01-01 09:00:01 3 Gift // Base point +1970-01-01 09:00:02 3 Home // Thre result +1970-01-01 09:00:03 3 Gift +1970-01-01 09:00:04 3 Basket +``` + +```SQL +SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, page = 'Gift', page = 'Gift', page = 'Home') FROM test_flow GROUP BY id; + + dt id page +1970-01-01 09:00:01 1 Home +1970-01-01 09:00:02 1 Gift // Base point +1970-01-01 09:00:03 1 Exit // Unmatched with Home + +1970-01-01 09:00:01 2 Home +1970-01-01 09:00:02 2 Home +1970-01-01 09:00:03 2 Gift // Base point +1970-01-01 09:00:04 2 Basket // Unmatched with Home + +1970-01-01 09:00:01 3 Gift // Base point +1970-01-01 09:00:02 3 Home // Matched with Home +1970-01-01 09:00:03 3 Gift // The result +1970-01-01 09:00:04 3 Basket +``` + + +**Behavior for `backward` and `last_match`** + +```SQL +SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, page = 'Gift', page = 'Gift') FROM test_flow GROUP BY id; + + dt id page +1970-01-01 09:00:01 1 Home // The result +1970-01-01 09:00:02 1 Gift // Base point +1970-01-01 09:00:03 1 Exit + +1970-01-01 09:00:01 2 Home +1970-01-01 09:00:02 2 Home // The result +1970-01-01 09:00:03 2 Gift // Base point +1970-01-01 09:00:04 2 Basket + +1970-01-01 09:00:01 3 Gift +1970-01-01 09:00:02 3 Home // The result +1970-01-01 09:00:03 3 Gift // Base point +1970-01-01 09:00:04 3 Basket +``` + +```SQL +SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, page = 'Gift', page = 'Gift', page = 'Home') FROM test_flow GROUP BY id; + + dt id page +1970-01-01 09:00:01 1 Home // Matched with Home, the result is null +1970-01-01 09:00:02 1 Gift // Base point +1970-01-01 09:00:03 1 Exit + +1970-01-01 09:00:01 2 Home // The result +1970-01-01 09:00:02 2 Home // Matched with Home +1970-01-01 09:00:03 2 Gift // Base point +1970-01-01 09:00:04 2 Basket + +1970-01-01 09:00:01 3 Gift // The result +1970-01-01 09:00:02 3 Home // Matched with Home +1970-01-01 09:00:03 3 Gift // Base point +1970-01-01 09:00:04 3 Basket +``` + + +**Behavior for `base_condition`** + +```SQL +CREATE TABLE test_flow_basecond +( + `dt` DateTime, + `id` int, + `page` String, + `ref` String +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(dt) +ORDER BY id + +INSERT INTO test_flow_basecond VALUES (1, 1, 'A', 'ref4') (2, 1, 'A', 'ref3') (3, 1, 'B', 'ref2') (4, 1, 'B', 'ref1'); +``` + +```SQL +SELECT id, sequenceNextNode('forward', 'head')(dt, page, ref = 'ref1', page = 'A') FROM test_flow_basecond GROUP BY id; + + dt id page ref + 1970-01-01 09:00:01 1 A ref4 // The head can't be base point becasue the ref column of the head unmatched with 'ref1'. + 1970-01-01 09:00:02 1 A ref3 + 1970-01-01 09:00:03 1 B ref2 + 1970-01-01 09:00:04 1 B ref1 + ``` + +```SQL +SELECT id, sequenceNextNode('backward', 'tail')(dt, page, ref = 'ref4', page = 'B') FROM test_flow_basecond GROUP BY id; + + dt id page ref + 1970-01-01 09:00:01 1 A ref4 + 1970-01-01 09:00:02 1 A ref3 + 1970-01-01 09:00:03 1 B ref2 + 1970-01-01 09:00:04 1 B ref1 // The tail can't be base point becasue the ref column of the tail unmatched with 'ref4'. +``` + +```SQL +SELECT id, sequenceNextNode('forward', 'first_match')(dt, page, ref = 'ref3', page = 'A') FROM test_flow_basecond GROUP BY id; + + dt id page ref + 1970-01-01 09:00:01 1 A ref4 // This row can't be base point becasue the ref column unmatched with 'ref3'. + 1970-01-01 09:00:02 1 A ref3 // Base point + 1970-01-01 09:00:03 1 B ref2 // The result + 1970-01-01 09:00:04 1 B ref1 +``` + +```SQL +SELECT id, sequenceNextNode('backward', 'last_match')(dt, page, ref = 'ref2', page = 'B') FROM test_flow_basecond GROUP BY id; + + dt id page ref + 1970-01-01 09:00:01 1 A ref4 + 1970-01-01 09:00:02 1 A ref3 // The result + 1970-01-01 09:00:03 1 B ref2 // Base point + 1970-01-01 09:00:04 1 B ref1 // This row can't be base point becasue the ref column unmatched with 'ref2'. +``` diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 5a7efa37fd1..41da56362e7 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -98,6 +98,10 @@ Setting fields: When dictionary with source `FILE` is created via DDL command (`CREATE DICTIONARY ...`), the source file needs to be located in `user_files` directory, to prevent DB users accessing arbitrary file on ClickHouse node. +**See Also** + +- [Dictionary function](../../../sql-reference/table-functions/dictionary.md#dictionary-function) + ## Executable File {#dicts-external_dicts_dict_sources-executable} Working with executable files depends on [how the dictionary is stored in memory](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md). If the dictionary is stored using `cache` and `complex_key_cache`, ClickHouse requests the necessary keys by sending a request to the executable file’s STDIN. Otherwise, ClickHouse starts executable file and treats its output as dictionary data. diff --git a/docs/en/sql-reference/table-functions/dictionary.md b/docs/en/sql-reference/table-functions/dictionary.md new file mode 100644 index 00000000000..675fcb5bfdd --- /dev/null +++ b/docs/en/sql-reference/table-functions/dictionary.md @@ -0,0 +1,59 @@ +--- +toc_priority: 54 +toc_title: dictionary function +--- + +# dictionary {#dictionary-function} + +Displays the [dictionary](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) data as a ClickHouse table. Works the same way as [Dictionary](../../engines/table-engines/special/dictionary.md) engine. + +**Syntax** + +``` sql +dictionary('dict') +``` + +**Arguments** + +- `dict` — A dictionary name. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +A ClickHouse table. + +**Example** + +Input table `dictionary_source_table`: + +``` text +┌─id─┬─value─┐ +│ 0 │ 0 │ +│ 1 │ 1 │ +└────┴───────┘ +``` + +Create a dictionary: + +``` sql +CREATE DICTIONARY new_dictionary(id UInt64, value UInt64 DEFAULT 0) PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'dictionary_source_table')) LAYOUT(DIRECT()); +``` + +Query: + +``` sql +SELECT * FROM dictionary('new_dictionary'); +``` + +Result: + +``` text +┌─id─┬─value─┐ +│ 0 │ 0 │ +│ 1 │ 1 │ +└────┴───────┘ +``` + +**See Also** + +- [Dictionary engine](../../engines/table-engines/special/dictionary.md#dictionary) diff --git a/docs/ru/engines/table-engines/special/dictionary.md b/docs/ru/engines/table-engines/special/dictionary.md index 243fd5395c0..15d32419472 100644 --- a/docs/ru/engines/table-engines/special/dictionary.md +++ b/docs/ru/engines/table-engines/special/dictionary.md @@ -90,3 +90,6 @@ select * from products limit 1; └───────────────┴─────────────────┘ ``` +**Смотрите также** + +- [Функция dictionary](../../../sql-reference/table-functions/dictionary.md#dictionary-function) diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md index 2af99bb8026..4ef811eb1dc 100644 --- a/docs/ru/operations/settings/merge-tree-settings.md +++ b/docs/ru/operations/settings/merge-tree-settings.md @@ -120,6 +120,19 @@ Eсли суммарное число активных кусков во все Команда `Insert` создает один или несколько блоков (кусков). При вставке в Replicated таблицы ClickHouse для [дедупликации вставок](../../engines/table-engines/mergetree-family/replication.md) записывает в Zookeeper хеш-суммы созданных кусков. Но хранятся хеш-суммы не всех кусков, а только последние `replicated_deduplication_window`. Наиболее старые хеш-суммы удаляются из Zookeeper. Большое число `replicated_deduplication_window` замедляет `Insert`-ы. Хеш-сумма рассчитывается от композиции имен и типов полей, а также данных вставленного куска (потока байт). +## non_replicated_deduplication_window {#non-replicated-deduplication-window} + +Количество последних вставленных блоков в нереплицированной [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) таблице, для которых хранятся хеш-суммы для проверки дубликатов. + +Возможные значения: + +- Положительное целое число. +- 0 (дедупликация отключена). + +Значение по умолчанию: 0. + +Используется механизм дедупликации, аналогичный реплицированным таблицам (см. описание настройки [replicated_deduplication_window](#replicated-deduplication-window)). Хеш-суммы вставленных кусков записываются в локальный файл на диске. + ## replicated_deduplication_window_seconds {#replicated-deduplication-window-seconds} Число секунд, после которых хеш-суммы вставленных блоков удаляются из Zookeeper. diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index ff83eb425d0..a0378251ece 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -97,6 +97,10 @@ SOURCE(FILE(path './user_files/os.tsv' format 'TabSeparated')) Если словарь с источником `FILE` создается с помощью DDL-команды (`CREATE DICTIONARY ...`), источник словаря должен быть расположен в каталоге `user_files`. Иначе пользователи базы данных будут иметь доступ к произвольному файлу на узле ClickHouse. +**Смотрите также** + +- [Функция dictionary](../../../sql-reference/table-functions/dictionary.md#dictionary-function) + ## Исполняемый файл {#dicts-external_dicts_dict_sources-executable} Работа с исполняемым файлом зависит от [размещения словаря в памяти](external-dicts-dict-layout.md). Если тип размещения словаря `cache` и `complex_key_cache`, то ClickHouse запрашивает необходимые ключи, отправляя запрос в `STDIN` исполняемого файла. diff --git a/docs/ru/sql-reference/table-functions/dictionary.md b/docs/ru/sql-reference/table-functions/dictionary.md new file mode 100644 index 00000000000..d4909bf5d9f --- /dev/null +++ b/docs/ru/sql-reference/table-functions/dictionary.md @@ -0,0 +1,59 @@ +--- +toc_priority: 54 +toc_title: dictionary +--- + +# dictionary {#dictionary-function} + +Отображает данные [словаря](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) как таблицу ClickHouse. Работает аналогично движку [Dictionary](../../engines/table-engines/special/dictionary.md). + +**Синтаксис** + +``` sql +dictionary('dict') +``` + +**Аргументы** + +- `dict` — имя словаря. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +Таблица ClickHouse. + +**Пример** + +Входная таблица `dictionary_source_table`: + +``` text +┌─id─┬─value─┐ +│ 0 │ 0 │ +│ 1 │ 1 │ +└────┴───────┘ +``` + +Создаем словарь: + +``` sql +CREATE DICTIONARY new_dictionary(id UInt64, value UInt64 DEFAULT 0) PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'dictionary_source_table')) LAYOUT(DIRECT()); +``` + +Запрос: + +``` sql +SELECT * FROM dictionary('new_dictionary'); +``` + +Результат: + +``` text +┌─id─┬─value─┐ +│ 0 │ 0 │ +│ 1 │ 1 │ +└────┴───────┘ +``` + +**Смотрите также** + +- [Движок Dictionary](../../engines/table-engines/special/dictionary.md#dictionary) diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index facfc2d1ba1..9bb4f57e9e2 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -27,7 +27,7 @@ pymdown-extensions==8.0 python-slugify==4.0.1 PyYAML==5.4.1 repackage==0.7.3 -requests==2.24.0 +requests==2.25.1 singledispatch==3.4.0.3 six==1.15.0 soupsieve==2.0.1 diff --git a/programs/client/Suggest.cpp b/programs/client/Suggest.cpp index 8d4c0fdbd5a..d4db49a1598 100644 --- a/programs/client/Suggest.cpp +++ b/programs/client/Suggest.cpp @@ -3,6 +3,9 @@ #include #include #include +#include +#include + namespace DB { @@ -90,7 +93,7 @@ void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeo /// NOTE: Once you will update the completion list, /// do not forget to update 01676_clickhouse_client_autocomplete.sh - std::stringstream query; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + WriteBufferFromOwnString query; query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" "SELECT name FROM system.functions" " UNION ALL " diff --git a/src/AggregateFunctions/AggregateFunctionSequenceNextNode.cpp b/src/AggregateFunctions/AggregateFunctionSequenceNextNode.cpp new file mode 100644 index 00000000000..0c87a32a415 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionSequenceNextNode.cpp @@ -0,0 +1,144 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +constexpr size_t max_events_size = 64; + +constexpr size_t min_required_args = 3; + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; + extern const int UNKNOWN_AGGREGATE_FUNCTION; +} + +namespace +{ + +template +inline AggregateFunctionPtr createAggregateFunctionSequenceNodeImpl( + const DataTypePtr data_type, const DataTypes & argument_types, SequenceDirection direction, SequenceBase base) +{ + return std::make_shared>>( + data_type, argument_types, base, direction, min_required_args); +} + +AggregateFunctionPtr +createAggregateFunctionSequenceNode(const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings * settings) +{ + if (settings == nullptr || !settings->allow_experimental_funnel_functions) + { + throw Exception( + "Aggregate function " + name + " is experimental. Set `allow_experimental_funnel_functions` setting to enable it", + ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION); + } + + if (parameters.size() < 2) + throw Exception("Aggregate function '" + name + "' requires 2 parameters (direction, head)", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + auto expected_param_type = Field::Types::Which::String; + if (parameters.at(0).getType() != expected_param_type || parameters.at(1).getType() != expected_param_type) + throw Exception("Aggregate function '" + name + "' requires 'String' parameters", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + String param_dir = parameters.at(0).safeGet(); + std::unordered_map seq_dir_mapping{ + {"forward", SequenceDirection::Forward}, + {"backward", SequenceDirection::Backward}, + }; + if (!seq_dir_mapping.contains(param_dir)) + throw Exception{"Aggregate function " + name + " doesn't support a parameter: " + param_dir, ErrorCodes::BAD_ARGUMENTS}; + SequenceDirection direction = seq_dir_mapping[param_dir]; + + String param_base = parameters.at(1).safeGet(); + std::unordered_map seq_base_mapping{ + {"head", SequenceBase::Head}, + {"tail", SequenceBase::Tail}, + {"first_match", SequenceBase::FirstMatch}, + {"last_match", SequenceBase::LastMatch}, + }; + if (!seq_base_mapping.contains(param_base)) + throw Exception{"Aggregate function " + name + " doesn't support a parameter: " + param_base, ErrorCodes::BAD_ARGUMENTS}; + SequenceBase base = seq_base_mapping[param_base]; + + if ((base == SequenceBase::Head && direction == SequenceDirection::Backward) || + (base == SequenceBase::Tail && direction == SequenceDirection::Forward)) + throw Exception(fmt::format( + "Invalid argument combination of '{}' with '{}'", param_base, param_dir), ErrorCodes::BAD_ARGUMENTS); + + if (argument_types.size() < min_required_args) + throw Exception("Aggregate function " + name + " requires at least " + toString(min_required_args) + " arguments.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + bool is_base_match_type = base == SequenceBase::FirstMatch || base == SequenceBase::LastMatch; + if (is_base_match_type && argument_types.size() < min_required_args + 1) + throw Exception( + "Aggregate function " + name + " requires at least " + toString(min_required_args + 1) + " arguments when base is first_match or last_match.", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (argument_types.size() > max_events_size + min_required_args) + throw Exception(fmt::format( + "Aggregate function '{}' requires at most {} (timestamp, value_column, ...{} events) arguments.", + name, max_events_size + min_required_args, max_events_size), ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (const auto * cond_arg = argument_types[2].get(); cond_arg && !isUInt8(cond_arg)) + throw Exception("Illegal type " + cond_arg->getName() + " of third argument of aggregate function " + + name + ", must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + for (const auto i : ext::range(min_required_args, argument_types.size())) + { + const auto * cond_arg = argument_types[i].get(); + if (!isUInt8(cond_arg)) + throw Exception(fmt::format( + "Illegal type '{}' of {} argument of aggregate function '{}', must be UInt8", cond_arg->getName(), i + 1, name), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + if (WhichDataType(argument_types[1].get()).idx != TypeIndex::String) + throw Exception{"Illegal type " + argument_types[1].get()->getName() + + " of second argument of aggregate function " + name + ", must be String", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; + + DataTypePtr data_type = makeNullable(argument_types[1]); + + WhichDataType timestamp_type(argument_types[0].get()); + if (timestamp_type.idx == TypeIndex::UInt8) + return createAggregateFunctionSequenceNodeImpl(data_type, argument_types, direction, base); + if (timestamp_type.idx == TypeIndex::UInt16) + return createAggregateFunctionSequenceNodeImpl(data_type, argument_types, direction, base); + if (timestamp_type.idx == TypeIndex::UInt32) + return createAggregateFunctionSequenceNodeImpl(data_type, argument_types, direction, base); + if (timestamp_type.idx == TypeIndex::UInt64) + return createAggregateFunctionSequenceNodeImpl(data_type, argument_types, direction, base); + if (timestamp_type.isDate()) + return createAggregateFunctionSequenceNodeImpl(data_type, argument_types, direction, base); + if (timestamp_type.isDateTime()) + return createAggregateFunctionSequenceNodeImpl(data_type, argument_types, direction, base); + + throw Exception{"Illegal type " + argument_types.front().get()->getName() + + " of first argument of aggregate function " + name + ", must be Unsigned Number, Date, DateTime", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; +} + +} + +void registerAggregateFunctionSequenceNextNode(AggregateFunctionFactory & factory) +{ + AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = false }; + factory.registerFunction("sequenceNextNode", { createAggregateFunctionSequenceNode, properties }); +} + +} diff --git a/src/AggregateFunctions/AggregateFunctionSequenceNextNode.h b/src/AggregateFunctions/AggregateFunctionSequenceNextNode.h new file mode 100644 index 00000000000..116e53e95e8 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionSequenceNextNode.h @@ -0,0 +1,426 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include + +#include +#include + + +namespace DB +{ +struct Settings; + +enum class SequenceDirection +{ + Forward, + Backward, +}; + +enum SequenceBase +{ + Head, + Tail, + FirstMatch, + LastMatch, +}; + +/// NodeBase used to implement a linked list for storage of SequenceNextNodeImpl +template +struct NodeBase +{ + UInt64 size; /// size of payload + + DataTypeDateTime::FieldType event_time; + std::bitset events_bitset; + bool can_be_base; + + char * data() { return reinterpret_cast(this) + sizeof(Node); } + + const char * data() const { return reinterpret_cast(this) + sizeof(Node); } + + Node * clone(Arena * arena) const + { + return reinterpret_cast( + const_cast(arena->alignedInsert(reinterpret_cast(this), sizeof(Node) + size, alignof(Node)))); + } + + void write(WriteBuffer & buf) const + { + writeVarUInt(size, buf); + buf.write(data(), size); + + writeBinary(event_time, buf); + UInt64 ulong_bitset = events_bitset.to_ulong(); + writeBinary(ulong_bitset, buf); + writeBinary(can_be_base, buf); + } + + static Node * read(ReadBuffer & buf, Arena * arena) + { + UInt64 size; + readVarUInt(size, buf); + + Node * node = reinterpret_cast(arena->alignedAlloc(sizeof(Node) + size, alignof(Node))); + node->size = size; + buf.read(node->data(), size); + + readBinary(node->event_time, buf); + UInt64 ulong_bitset; + readBinary(ulong_bitset, buf); + node->events_bitset = ulong_bitset; + readBinary(node->can_be_base, buf); + + return node; + } +}; + +/// It stores String, timestamp, bitset of matched events. +template +struct NodeString : public NodeBase, MaxEventsSize> +{ + using Node = NodeString; + + static Node * allocate(const IColumn & column, size_t row_num, Arena * arena) + { + StringRef string = assert_cast(column).getDataAt(row_num); + + Node * node = reinterpret_cast(arena->alignedAlloc(sizeof(Node) + string.size, alignof(Node))); + node->size = string.size; + memcpy(node->data(), string.data, string.size); + + return node; + } + + void insertInto(IColumn & column) + { + assert_cast(column).insertData(this->data(), this->size); + } + + bool compare(const Node * rhs) const + { + auto cmp = strncmp(this->data(), rhs->data(), std::min(this->size, rhs->size)); + return (cmp == 0) ? this->size < rhs->size : cmp < 0; + } +}; + +/// TODO : Support other types than string +template +struct SequenceNextNodeGeneralData +{ + using Allocator = MixedAlignedArenaAllocator; + using Array = PODArray; + + Array value; + bool sorted = false; + + struct Comparator final + { + bool operator()(const Node * lhs, const Node * rhs) const + { + return lhs->event_time == rhs->event_time ? lhs->compare(rhs) : lhs->event_time < rhs->event_time; + } + }; + + void sort() + { + if (!sorted) + { + std::stable_sort(std::begin(value), std::end(value), Comparator{}); + sorted = true; + } + } +}; + +/// Implementation of sequenceFirstNode +template +class SequenceNextNodeImpl final + : public IAggregateFunctionDataHelper, SequenceNextNodeImpl> +{ + using Self = SequenceNextNodeImpl; + + using Data = SequenceNextNodeGeneralData; + static Data & data(AggregateDataPtr place) { return *reinterpret_cast(place); } + static const Data & data(ConstAggregateDataPtr place) { return *reinterpret_cast(place); } + + static constexpr size_t base_cond_column_idx = 2; + static constexpr size_t event_column_idx = 1; + + SequenceBase seq_base_kind; + SequenceDirection seq_direction; + const size_t min_required_args; + + DataTypePtr & data_type; + UInt8 events_size; + UInt64 max_elems; +public: + SequenceNextNodeImpl( + const DataTypePtr & data_type_, + const DataTypes & arguments, + SequenceBase seq_base_kind_, + SequenceDirection seq_direction_, + size_t min_required_args_, + UInt64 max_elems_ = std::numeric_limits::max()) + : IAggregateFunctionDataHelper, Self>({data_type_}, {}) + , seq_base_kind(seq_base_kind_) + , seq_direction(seq_direction_) + , min_required_args(min_required_args_) + , data_type(this->argument_types[0]) + , events_size(arguments.size() - min_required_args) + , max_elems(max_elems_) + { + } + + String getName() const override { return "sequenceNextNode"; } + + DataTypePtr getReturnType() const override { return data_type; } + + AggregateFunctionPtr getOwnNullAdapter( + const AggregateFunctionPtr & nested_function, const DataTypes & arguments, const Array & params, + const AggregateFunctionProperties &) const override + { + /// Even though some values are mapped to aggregating key, it could return nulls for the below case. + /// aggregated events: [A -> B -> C] + /// events to find: [C -> D] + /// [C -> D] is not matched to 'A -> B -> C' so that it returns null. + return std::make_shared>(nested_function, arguments, params); + } + + void insert(Data & a, const Node * v, Arena * arena) const + { + ++a.total_values; + a.value.push_back(v->clone(arena), arena); + } + + void create(AggregateDataPtr place) const override + { + new (place) Data; + } + + void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override + { + Node * node = Node::allocate(*columns[event_column_idx], row_num, arena); + + const auto timestamp = assert_cast *>(columns[0])->getData()[row_num]; + + /// The events_bitset variable stores matched events in the form of bitset. + /// Each Nth-bit indicates that the Nth-event are matched. + /// For example, event1 and event3 is matched then the values of events_bitset is 0x00000005. + /// 0x00000000 + /// + 1 (bit of event1) + /// + 4 (bit of event3) + node->events_bitset.reset(); + for (UInt8 i = 0; i < events_size; ++i) + if (assert_cast *>(columns[min_required_args + i])->getData()[row_num]) + node->events_bitset.set(i); + node->event_time = timestamp; + + node->can_be_base = assert_cast *>(columns[base_cond_column_idx])->getData()[row_num]; + + data(place).value.push_back(node, arena); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override + { + if (data(rhs).value.empty()) + return; + + if (data(place).value.size() >= max_elems) + return; + + auto & a = data(place).value; + auto & b = data(rhs).value; + const auto a_size = a.size(); + + const UInt64 new_elems = std::min(data(rhs).value.size(), static_cast(max_elems) - data(place).value.size()); + for (UInt64 i = 0; i < new_elems; ++i) + a.push_back(b[i]->clone(arena), arena); + + /// Either sort whole container or do so partially merging ranges afterwards + using Comparator = typename SequenceNextNodeGeneralData::Comparator; + + if (!data(place).sorted && !data(rhs).sorted) + std::stable_sort(std::begin(a), std::end(a), Comparator{}); + else + { + const auto begin = std::begin(a); + const auto middle = std::next(begin, a_size); + const auto end = std::end(a); + + if (!data(place).sorted) + std::stable_sort(begin, middle, Comparator{}); + + if (!data(rhs).sorted) + std::stable_sort(middle, end, Comparator{}); + + std::inplace_merge(begin, middle, end, Comparator{}); + } + + data(place).sorted = true; + } + + void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf) const override + { + /// Temporarily do a const_cast to sort the values. It helps to reduce the computational burden on the initiator node. + this->data(const_cast(place)).sort(); + + writeBinary(data(place).sorted, buf); + + auto & value = data(place).value; + + size_t size = std::min(static_cast(events_size + 1), value.size()); + switch (seq_base_kind) + { + case SequenceBase::Head: + writeVarUInt(size, buf); + for (size_t i = 0; i < size; ++i) + value[i]->write(buf); + break; + + case SequenceBase::Tail: + writeVarUInt(size, buf); + for (size_t i = 0; i < size; ++i) + value[value.size() - size + i]->write(buf); + break; + + case SequenceBase::FirstMatch: + case SequenceBase::LastMatch: + writeVarUInt(value.size(), buf); + for (auto & node : value) + node->write(buf); + break; + } + } + + void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, Arena * arena) const override + { + readBinary(data(place).sorted, buf); + + UInt64 size; + readVarUInt(size, buf); + + if (unlikely(size == 0)) + return; + + auto & value = data(place).value; + + value.resize(size, arena); + for (UInt64 i = 0; i < size; ++i) + value[i] = Node::read(buf, arena); + } + + inline std::optional getBaseIndex(Data & data) const + { + if (data.value.size() == 0) + return {}; + + switch (seq_base_kind) + { + case SequenceBase::Head: + if (data.value[0]->can_be_base) + return 0; + break; + + case SequenceBase::Tail: + if (data.value[data.value.size() - 1]->can_be_base) + return data.value.size() - 1; + break; + + case SequenceBase::FirstMatch: + for (size_t i = 0; i < data.value.size(); ++i) + { + if (data.value[i]->events_bitset.test(0) && data.value[i]->can_be_base) + return i; + } + break; + + case SequenceBase::LastMatch: + for (size_t i = 0; i < data.value.size(); ++i) + { + auto reversed_i = data.value.size() - i - 1; + if (data.value[reversed_i]->events_bitset.test(0) && data.value[reversed_i]->can_be_base) + return reversed_i; + } + break; + } + + return {}; + } + + /// This method returns an index of next node that matched the events. + /// matched events in the chain of events are represented as a bitmask. + /// The first matched event is 0x00000001, the second one is 0x00000002, the third one is 0x00000004, and so on. + UInt32 getNextNodeIndex(Data & data) const + { + const UInt32 unmatched_idx = data.value.size(); + + if (data.value.size() <= events_size) + return unmatched_idx; + + data.sort(); + + std::optional base_opt = getBaseIndex(data); + if (!base_opt.has_value()) + return unmatched_idx; + UInt32 base = static_cast(base_opt.value()); + + if (events_size == 0) + return data.value.size() > 0 ? base : unmatched_idx; + + UInt32 i = 0; + switch (seq_direction) + { + case SequenceDirection::Forward: + for (i = 0; i < events_size && base + i < data.value.size(); ++i) + if (!data.value[base + i]->events_bitset.test(i)) + break; + return (i == events_size) ? base + i : unmatched_idx; + + case SequenceDirection::Backward: + for (i = 0; i < events_size && i < base; ++i) + if (!data.value[base - i]->events_bitset.test(i)) + break; + return (i == events_size) ? base - i : unmatched_idx; + } + __builtin_unreachable(); + } + + void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override + { + auto & value = data(place).value; + + UInt32 event_idx = getNextNodeIndex(this->data(place)); + if (event_idx < value.size()) + { + ColumnNullable & to_concrete = assert_cast(to); + value[event_idx]->insertInto(to_concrete.getNestedColumn()); + to_concrete.getNullMapData().push_back(0); + } + else + { + to.insertDefault(); + } + } + + bool allocatesMemoryInArena() const override { return true; } +}; + +} diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp index 0c4e2d167a4..01ea2636434 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp @@ -17,7 +17,6 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int UNKNOWN_AGGREGATE_FUNCTION; } namespace @@ -25,15 +24,8 @@ namespace template