mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-27 18:12:02 +00:00
Merge pull request #72531 from ClickHouse/ci_remove_hdfs_tests
CI: Remove functional tests with hdfs
This commit is contained in:
Max Kainov
GitHub
commit
0eeb6c38dc
@ -78,10 +78,6 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
&& curl -L "https://dl.min.io/client/mc/release/linux-${arch}/archive/mc.RELEASE.${MINIO_CLIENT_VERSION}" -o ./mc \
|
||||
&& chmod +x ./mc ./minio
|
||||
|
||||
RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
|
||||
&& tar -xvf hadoop-3.3.1.tar.gz \
|
||||
&& rm -rf hadoop-3.3.1.tar.gz
|
||||
|
||||
ENV MINIO_ROOT_USER="clickhouse"
|
||||
ENV MINIO_ROOT_PASSWORD="clickhouse"
|
||||
ENV EXPORT_S3_STORAGE_POLICIES=1
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
#!/bin/bash
|
||||
# shellcheck disable=SC2024
|
||||
|
||||
set -e -x -a -u
|
||||
|
||||
ls -lha
|
||||
|
||||
cd /hadoop-3.3.1
|
||||
|
||||
export JAVA_HOME=/usr
|
||||
mkdir -p target/test/data
|
||||
chown clickhouse ./target/test/data
|
||||
sudo -E -u clickhouse bin/mapred minicluster -format -nomr -nnport 12222 >> /test_output/hdfs_minicluster.log 2>&1 &
|
||||
|
||||
while ! nc -z localhost 12222; do
|
||||
sleep 1
|
||||
done
|
||||
|
||||
lsof -i :12222
|
||||
|
||||
sleep 5
|
||||
@ -57,8 +57,6 @@ source /repo/tests/docker_scripts/utils.lib
|
||||
|
||||
/repo/tests/docker_scripts/setup_minio.sh stateless
|
||||
|
||||
/repo/tests/docker_scripts/setup_hdfs_minicluster.sh
|
||||
|
||||
config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml
|
||||
|
||||
if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; then
|
||||
|
||||
@ -1,10 +0,0 @@
|
||||
-- Tags: no-fasttest, use-hdfs
|
||||
|
||||
drop table if exists test_table_hdfs_syntax
|
||||
;
|
||||
create table test_table_hdfs_syntax (id UInt32) ENGINE = HDFS('')
|
||||
; -- { serverError BAD_ARGUMENTS }
|
||||
create table test_table_hdfs_syntax (id UInt32) ENGINE = HDFS('','','', '')
|
||||
; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
|
||||
drop table if exists test_table_hdfs_syntax
|
||||
;
|
||||
@ -16,5 +16,5 @@ for _ in {1..1000}; do
|
||||
if [[ $elapsed -gt 30 ]]; then
|
||||
break
|
||||
fi
|
||||
done 2>&1 | grep -o -P 'Query memory limit exceeded' | sed -r -e 's/(.*):([a-Z ]*)([mM]emory limit exceeded)(.*)/\2\3/' | uniq
|
||||
done 2>&1 | grep -o 'Query memory limit exceeded' | head -n1
|
||||
echo 'Ok'
|
||||
|
||||
@ -1 +0,0 @@
|
||||
OK
|
||||
@ -1,10 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest, use-hdfs
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
|
||||
TCP_PORT=$($CLICKHOUSE_CLIENT -q "SELECT tcpPort()")
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('hdfs://localhost:$TCP_PORT/data.csv', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "HDFS_ERROR" && echo 'OK' || echo 'FAIL';
|
||||
@ -1,17 +0,0 @@
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
@ -1,25 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest, use-hdfs
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('abcd', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('abcd/', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('//abcd', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('//abcd/', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('//abcd/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('://abcd', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('://abcd/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('abcd:9000', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('abcd:9000/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('//abcd:9000/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('://abcd:9000/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('abcd/', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('hdfs://abcd', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('hdfs1:9000/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('hdfs://hdfs1/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "HDFS_ERROR" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('http://hdfs1:9000/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('hdfs://hdfs1@nameservice/abcd/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "HDFS_ERROR" && echo 'OK' || echo 'FAIL';
|
||||
@ -1,48 +0,0 @@
|
||||
1 2 3
|
||||
4 5 6
|
||||
7 8 9
|
||||
1 2 3
|
||||
4 5 6
|
||||
7 8 9
|
||||
1 2 3
|
||||
4 5 6
|
||||
7 8 9
|
||||
1 2 3
|
||||
4 5 6
|
||||
7 8 9
|
||||
1 2 3
|
||||
4 5 6
|
||||
7 8 9
|
||||
1 2 3
|
||||
4 5 6
|
||||
7 8 9
|
||||
1 2 3
|
||||
4 5 6
|
||||
7 8 9
|
||||
1 2 3
|
||||
4 5 6
|
||||
7 8 9
|
||||
c1 Nullable(Int64)
|
||||
c2 Nullable(Int64)
|
||||
c3 Nullable(Int64)
|
||||
c1 Nullable(Int64)
|
||||
c2 Nullable(Int64)
|
||||
c3 Nullable(Int64)
|
||||
c1 UInt32
|
||||
c2 UInt32
|
||||
c3 UInt32
|
||||
c1 UInt32
|
||||
c2 UInt32
|
||||
c3 UInt32
|
||||
c1 Nullable(Int64)
|
||||
c2 Nullable(Int64)
|
||||
c3 Nullable(Int64)
|
||||
c1 Nullable(Int64)
|
||||
c2 Nullable(Int64)
|
||||
c3 Nullable(Int64)
|
||||
c1 UInt32
|
||||
c2 UInt32
|
||||
c3 UInt32
|
||||
c1 UInt32
|
||||
c2 UInt32
|
||||
c3 UInt32
|
||||
@ -1,26 +0,0 @@
|
||||
-- Tags: no-fasttest, no-parallel
|
||||
-- Tag no-fasttest: Depends on Java
|
||||
|
||||
insert into table function hdfs('hdfs://localhost:12222/test_1.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 1, 2, 3 settings hdfs_truncate_on_insert=1;
|
||||
insert into table function hdfs('hdfs://localhost:12222/test_2.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 4, 5, 6 settings hdfs_truncate_on_insert=1;
|
||||
insert into table function hdfs('hdfs://localhost:12222/test_3.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 7, 8, 9 settings hdfs_truncate_on_insert=1;
|
||||
|
||||
select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv') order by c1, c2, c3;
|
||||
select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV') order by c1, c2, c3;
|
||||
select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32') order by c1, c2, c3;
|
||||
select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto') order by c1, c2, c3;
|
||||
|
||||
select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv') order by c1, c2, c3;
|
||||
select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV') order by c1, c2, c3;
|
||||
select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32') order by c1, c2, c3;
|
||||
select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto') order by c1, c2, c3;
|
||||
|
||||
desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv');
|
||||
desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV');
|
||||
desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32');
|
||||
desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto');
|
||||
|
||||
desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv');
|
||||
desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV');
|
||||
desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32');
|
||||
desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto');
|
||||
@ -1,26 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest, no-asan, no-tsan, no-msan, no-ubsan, no-debug
|
||||
# FIXME https://github.com/ClickHouse/ClickHouse/issues/47207
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
|
||||
for i in $(seq 1 10);
|
||||
do
|
||||
$CLICKHOUSE_CLIENT --query_id="02368_$i" -q "insert into function hdfs('hdfs://localhost:12222/02368_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings hdfs_truncate_on_insert=1, output_format_parallel_formatting=1" 2> /dev/null &
|
||||
done
|
||||
|
||||
sleep 2
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null 2>&1
|
||||
|
||||
for i in $(seq 1 10);
|
||||
do
|
||||
$CLICKHOUSE_CLIENT --query_id="02368_$i" -q "insert into function hdfs('hdfs://localhost:12222/02368_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings hdfs_truncate_on_insert=1, output_format_parallel_formatting=0" 2> /dev/null &
|
||||
done
|
||||
|
||||
sleep 2
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null 2>&1
|
||||
@ -1,5 +0,0 @@
|
||||
-- Tags: no-fasttest
|
||||
SELECT * FROM hdfsCluster('test_shard_localhost', '', 'TSV'); -- { serverError BAD_ARGUMENTS }
|
||||
SELECT * FROM hdfsCluster('test_shard_localhost', ' ', 'TSV'); -- { serverError BAD_ARGUMENTS }
|
||||
SELECT * FROM hdfsCluster('test_shard_localhost', '/', 'TSV'); -- { serverError BAD_ARGUMENTS }
|
||||
SELECT * FROM hdfsCluster('test_shard_localhost', 'http/', 'TSV'); -- { serverError BAD_ARGUMENTS }
|
||||
@ -1,10 +0,0 @@
|
||||
c1 Nullable(Int64)
|
||||
c2 Nullable(Int64)
|
||||
c3 Nullable(Int64)
|
||||
c1 Nullable(Int64)
|
||||
c2 Nullable(Int64)
|
||||
c3 Nullable(Int64)
|
||||
1 2 3
|
||||
4 5 6
|
||||
1 2 3
|
||||
4 5 6
|
||||
@ -1,11 +0,0 @@
|
||||
-- Tags: no-fasttest, no-parallel
|
||||
-- Tag no-fasttest: Depends on Java
|
||||
|
||||
insert into table function hdfs('hdfs://localhost:12222/test_02458_1.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 1, 2, 3 settings hdfs_truncate_on_insert=1;
|
||||
insert into table function hdfs('hdfs://localhost:12222/test_02458_2.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 4, 5, 6 settings hdfs_truncate_on_insert=1;
|
||||
|
||||
desc hdfsCluster('test_cluster_one_shard_three_replicas_localhost', 'hdfs://localhost:12222/test_02458_{1,2}.tsv');
|
||||
desc hdfsCluster('test_cluster_one_shard_three_replicas_localhost', 'hdfs://localhost:12222/test_02458_{1,2}.tsv', 'TSV');
|
||||
|
||||
select * from hdfsCluster('test_cluster_one_shard_three_replicas_localhost', 'hdfs://localhost:12222/test_02458_{1,2}.tsv') order by c1, c2, c3;
|
||||
select * from hdfsCluster('test_cluster_one_shard_three_replicas_localhost', 'hdfs://localhost:12222/test_02458_{1,2}.tsv', 'TSV') order by c1, c2, c3;
|
||||
@ -1 +0,0 @@
|
||||
(1,2)
|
||||
@ -1,11 +0,0 @@
|
||||
-- Tags: no-fasttest, no-parallel
|
||||
-- Tag no-fasttest: Depends on Java
|
||||
|
||||
insert into table function hdfs('hdfs://localhost:12222/test_02536.jsonl', 'TSV') select '{"x" : {"a" : 1, "b" : 2}}' settings hdfs_truncate_on_insert=1;
|
||||
set input_format_json_try_infer_named_tuples_from_objects=0;
|
||||
drop table if exists test;
|
||||
create table test (x Tuple(a UInt32, b UInt32)) engine=Memory();
|
||||
insert into test select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_02536.jsonl') settings use_structure_from_insertion_table_in_table_functions=0; -- {serverError ILLEGAL_COLUMN}
|
||||
insert into test select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_02536.jsonl') settings use_structure_from_insertion_table_in_table_functions=1;
|
||||
select * from test;
|
||||
drop table test;
|
||||
@ -1,11 +0,0 @@
|
||||
Test 1: select from hdfs database
|
||||
1 2 3
|
||||
test_hdfs_1
|
||||
1 2 3
|
||||
test_hdfs_2
|
||||
Test 2: check exceptions
|
||||
BAD_ARGUMENTS
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
OK
|
||||
@ -1,72 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest, use-hdfs, no-parallel
|
||||
|
||||
CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# Prepare data
|
||||
${CLICKHOUSE_CLIENT} -q "insert into table function hdfs('hdfs://localhost:12222/test_02725_1.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 1, 2, 3 settings hdfs_truncate_on_insert=1;"
|
||||
ret=$?
|
||||
if [ $ret -ne 0 ]; then
|
||||
echo "Insert failed!"
|
||||
exit 1
|
||||
fi
|
||||
${CLICKHOUSE_CLIENT} -q "insert into table function hdfs('hdfs://localhost:12222/test_02725_2.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 4, 5, 6 settings hdfs_truncate_on_insert=1;"
|
||||
ret=$?
|
||||
if [ $ret -ne 0 ]; then
|
||||
echo "Insert failed!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
#################
|
||||
echo "Test 1: select from hdfs database"
|
||||
|
||||
# Database without specific host
|
||||
${CLICKHOUSE_CLIENT} --multiline -q """
|
||||
DROP DATABASE IF EXISTS test_hdfs_1;
|
||||
CREATE DATABASE test_hdfs_1 ENGINE = HDFS;
|
||||
USE test_hdfs_1;
|
||||
SELECT * FROM \"hdfs://localhost:12222/test_02725_1.tsv\"
|
||||
"""
|
||||
${CLICKHOUSE_CLIENT} -q "SHOW DATABASES;" | grep test_hdfs_1
|
||||
|
||||
# Database with host
|
||||
${CLICKHOUSE_CLIENT} --multiline -q """
|
||||
DROP DATABASE IF EXISTS test_hdfs_2;
|
||||
CREATE DATABASE test_hdfs_2 ENGINE = HDFS('hdfs://localhost:12222');
|
||||
USE test_hdfs_2;
|
||||
SELECT * FROM \"test_02725_1.tsv\"
|
||||
"""
|
||||
${CLICKHOUSE_CLIENT} -q "SHOW DATABASES;" | grep test_hdfs_2
|
||||
|
||||
#################
|
||||
echo "Test 2: check exceptions"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiline -q """
|
||||
DROP DATABASE IF EXISTS test_hdfs_3;
|
||||
CREATE DATABASE test_hdfs_3 ENGINE = HDFS('abacaba');
|
||||
""" 2>&1 | tr '\n' ' ' | grep -oF "BAD_ARGUMENTS"
|
||||
|
||||
${CLICKHOUSE_CLIENT} --multiline -q """
|
||||
DROP DATABASE IF EXISTS test_hdfs_4;
|
||||
CREATE DATABASE test_hdfs_4 ENGINE = HDFS;
|
||||
USE test_hdfs_4;
|
||||
SELECT * FROM \"abacaba/file.tsv\"
|
||||
""" 2>&1 | tr '\n' ' ' | grep -oF "CANNOT_EXTRACT_TABLE_STRUCTURE"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT * FROM test_hdfs_4.\`http://localhost:11111/test/a.tsv\`" 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "BAD_ARGUMENTS" > /dev/null && echo "OK" || echo 'FAIL' ||:
|
||||
${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_hdfs_4.\`hdfs://localhost:12222/file.myext\`" 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "The data format cannot be detected" > /dev/null && echo "OK" || echo 'FAIL' ||:
|
||||
${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_hdfs_4.\`hdfs://localhost:12222/test_02725_3.tsv\`" 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "The table structure cannot be extracted" > /dev/null && echo "OK" || echo 'FAIL' ||:
|
||||
${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_hdfs_4.\`hdfs://localhost:12222\`" 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "BAD_ARGUMENTS" > /dev/null && echo "OK" || echo 'FAIL' ||:
|
||||
|
||||
|
||||
# Cleanup
|
||||
${CLICKHOUSE_CLIENT} --multiline -q """
|
||||
DROP DATABASE IF EXISTS test_hdfs_1;
|
||||
DROP DATABASE IF EXISTS test_hdfs_2;
|
||||
DROP DATABASE IF EXISTS test_hdfs_3;
|
||||
DROP DATABASE IF EXISTS test_hdfs_4;
|
||||
"""
|
||||
@ -1,2 +0,0 @@
|
||||
1
|
||||
1
|
||||
@ -1,15 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest, use-hdfs
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data1.tsv') select 1 settings hdfs_truncate_on_insert=1;"
|
||||
$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data2.tsv') select 2 settings hdfs_truncate_on_insert=1;"
|
||||
$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data3.tsv') select 3 settings hdfs_truncate_on_insert=1;"
|
||||
|
||||
|
||||
$CLICKHOUSE_CLIENT --print-profile-events -q "select * from hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data*.tsv', auto, 'x UInt64') where _file like '%data1%' format Null" 2>&1 | grep -F -c "EngineFileLikeReadFiles: 1"
|
||||
|
||||
$CLICKHOUSE_CLIENT --print-profile-events -q "select * from hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data*.tsv', auto, 'x UInt64') where _path like '%data1%' format Null" 2>&1 | grep -F -c "EngineFileLikeReadFiles: 1"
|
||||
@ -1,6 +0,0 @@
|
||||
2
|
||||
3
|
||||
4
|
||||
2
|
||||
3
|
||||
4
|
||||
@ -1,15 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest, use-hdfs
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data1.tsv') select 1 settings hdfs_truncate_on_insert=1;"
|
||||
$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data2.tsv') select 11 settings hdfs_truncate_on_insert=1;"
|
||||
$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data3.tsv') select 111 settings hdfs_truncate_on_insert=1;"
|
||||
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "select _size from hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data*.tsv', auto, 'x UInt64') order by _size"
|
||||
$CLICKHOUSE_CLIENT -q "select _size from hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data*.tsv', auto, 'x UInt64') order by _size"
|
||||
|
||||
@ -1,2 +0,0 @@
|
||||
99999
|
||||
99999
|
||||
@ -1,11 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest, no-parallel, use-hdfs
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "insert into table function file('test_03243.parquet', 'Parquet') select number as i from numbers(100000) settings output_format_parquet_row_group_size=10000,engine_file_truncate_on_insert=1"
|
||||
$CLICKHOUSE_CLIENT -q "select max(i) from file('test_03243.parquet', 'Parquet') settings max_threads = 1;"
|
||||
$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/test_03243.parquet', 'Parquet') select number as i from numbers(100000) settings output_format_parquet_row_group_size=10000,hdfs_truncate_on_insert=1;"
|
||||
$CLICKHOUSE_CLIENT -q "select max(i) from hdfs('hdfs://localhost:12222/test_03243.parquet', 'Parquet') settings max_threads = 1;"
|
||||
Loading…
Reference in New Issue
Block a user