ClickHouse/tests/queries/0_stateless/02994_libarchive_compression.sh

#!/usr/bin/env bash
# Tags: no-fasttest

CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh

EPOCHTIME=$(date "+%s")

# Test tar format
$CLICKHOUSE_CLIENT --query  "INSERT INTO TABLE FUNCTION file('02994_file_$EPOCHTIME/test.tar', 'TabSeparatedRaw', 'column1 String') VALUES (char(116,101,115,116,46,99,115,118,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,48,48,48,48,54,54,52,0,48,48,48,49,55,53,48,0,48,48,48,49,55,53,48,0,48,48,48,48,48,48,48,48,48,49,52,0,49,52,53,54,53,53,50,50,54,49,55,0,48,49,50,50,49,50,0,32,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,117,115,116,97,114,32,32,0,117,98,117,110,116,117,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,117,98,117,110,116,117,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,44,50,44,51,10,52,44,53,44,54,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
$CLICKHOUSE_CLIENT --query   "SELECT c1 FROM file('02994_file_$EPOCHTIME/test.tar :: test.csv', CSV);"


# Test tar.xz format
$CLICKHOUSE_CLIENT --query  "INSERT INTO TABLE FUNCTION file('02994_file_$EPOCHTIME/test.tar.xz', 'TabSeparatedRaw', 'column1 String') VALUES (char(66,90,104,57,49,65,89,38,83,89,108,215,67,197,0,0,78,123,144,210,16,64,128,64,5,127,128,0,64,107,33,159,0,4,0,2,8,32,0,116,34,122,36,122,153,26,13,3,64,105,160,148,130,122,129,160,50,0,3,109,59,133,95,33,76,224,38,15,14,85,232,12,73,69,196,36,202,7,43,99,38,175,117,168,43,2,37,44,198,225,8,84,168,207,149,250,54,189,175,113,182,18,176,221,17,144,10,26,106,176,122,60,177,140,206,103,202,75,191,1,0,142,156,253,249,94,202,93,68,193,140,252,93,201,20,225,66,65,179,93,15,20));"
$CLICKHOUSE_CLIENT --query   "SELECT c1 FROM file('02994_file_$EPOCHTIME/test.tar.xz :: test.csv', CSV);"

# Test tar.gz format
$CLICKHOUSE_CLIENT --query  "INSERT INTO TABLE FUNCTION file('02994_file_$EPOCHTIME/test.tar.gz', 'TabSeparatedRaw', 'column1 String') VALUES (char(31,139,8,0,230,112,212,101,0,3,237,207,193,10,131,48,12,6,96,207,62,69,30,160,140,164,77,235,243,148,178,194,46,10,166,219,243,91,45,122,146,221,220,16,242,93,126,2,33,252,41,79,41,143,36,159,238,66,136,24,152,97,205,33,248,45,209,182,185,33,6,98,31,60,58,75,132,80,103,55,112,7,120,101,169,221,91,74,156,107,21,137,99,153,227,152,94,146,166,179,189,186,150,243,151,59,237,19,56,242,38,200,88,227,122,54,222,132,254,223,93,148,82,74,253,206,2,148,165,35,158,0,8,0,0));"
$CLICKHOUSE_CLIENT --query   "SELECT c1 FROM file('02994_file_$EPOCHTIME/test.tar.gz :: test.csv', CSV);"


# Test tar.bz2 format
$CLICKHOUSE_CLIENT --query  "INSERT INTO TABLE FUNCTION file('02994_file_$EPOCHTIME/test.tar.bz2', 'TabSeparatedRaw', 'column1 String') VALUES (char(66,90,104,57,49,65,89,38,83,89,108,215,67,197,0,0,78,123,144,210,16,64,128,64,5,127,128,0,64,107,33,159,0,4,0,2,8,32,0,116,34,122,36,122,153,26,13,3,64,105,160,148,130,122,129,160,50,0,3,109,59,133,95,33,76,224,38,15,14,85,232,12,73,69,196,36,202,7,43,99,38,175,117,168,43,2,37,44,198,225,8,84,168,207,149,250,54,189,175,113,182,18,176,221,17,144,10,26,106,176,122,60,177,140,206,103,202,75,191,1,0,142,156,253,249,94,202,93,68,193,140,252,93,201,20,225,66,65,179,93,15,20));"
$CLICKHOUSE_CLIENT --query   "SELECT c1 FROM file('02994_file_$EPOCHTIME/test.tar.bz2 :: test.csv', CSV);"


# Test 7z format
$CLICKHOUSE_CLIENT --query  "INSERT INTO TABLE FUNCTION file('02994_file_$EPOCHTIME/test.7z', 'TabSeparatedRaw', 'column1 String') VALUES (char(55,122,188,175,39,28,0,4,248,166,182,96,16,0,0,0,0,0,0,0,90,0,0,0,0,0,0,0,3,235,205,15,1,0,11,49,44,50,44,51,10,52,44,53,44,54,10,0,1,4,6,0,1,9,16,0,7,11,1,0,1,33,33,1,0,12,12,0,8,10,1,180,193,159,219,0,0,5,1,25,12,0,0,0,0,0,0,0,0,0,0,0,0,17,19,0,116,0,101,0,115,0,116,0,46,0,99,0,115,0,118,0,0,0,25,0,20,10,1,0,188,83,33,33,187,99,218,1,21,6,1,0,32,128,164,129,0,0));"
$CLICKHOUSE_CLIENT --query   "SELECT c1 FROM file('02994_file_$EPOCHTIME/test.7z :: test.csv', CSV);"


# Test zip format
$CLICKHOUSE_CLIENT --query  "INSERT INTO TABLE FUNCTION file('02994_file_$EPOCHTIME/test.zip', 'TabSeparatedRaw', 'column1 String') VALUES (char(80,75,3,4,10,0,0,0,0,0,80,129,84,88,180,193,159,219,12,0,0,0,12,0,0,0,8,0,28,0,116,101,115,116,46,99,115,118,85,84,9,0,3,72,52,212,101,74,52,212,101,117,120,11,0,1,4,245,1,0,0,4,20,0,0,0,49,44,50,44,51,10,52,44,53,44,54,10,80,75,1,2,30,3,10,0,0,0,0,0,80,129,84,88,180,193,159,219,12,0,0,0,12,0,0,0,8,0,24,0,0,0,0,0,1,0,0,0,164,129,0,0,0,0,116,101,115,116,46,99,115,118,85,84,5,0,3,72,52,212,101,117,120,11,0,1,4,245,1,0,0,4,20,0,0,0,80,75,5,6,0,0,0,0,1,0,1,0,78,0,0,0,78,0,0,0,0,0));"
$CLICKHOUSE_CLIENT --query   "SELECT c1 FROM file('02994_file_$EPOCHTIME/test.zip :: test.csv', CSV);"

# Test rar file with 7z extension - this should fail
$CLICKHOUSE_CLIENT --query  "INSERT INTO TABLE FUNCTION file('02994_file_$EPOCHTIME/test.rar.7z', 'TabSeparatedRaw', 'column1 String') VALUES (char(82,97,114,33,26,7,1,0,51,146,181,229,10,1,5,6,0,5,1,1,128,128,0,235,180,125,92,36,2,3,11,140,0,4,140,0,32,18,67,124,209,128,0,0,8,116,101,115,116,46,99,115,118,10,3,2,18,154,60,112,217,99,218,1,49,44,50,44,51,13,10,52,44,53,44,54,29,119,86,81,3,5,4,0));"
$CLICKHOUSE_CLIENT --query   "SELECT c1 FROM file('02994_file_$EPOCHTIME/test.rar.7z :: test.csv', CSV);" 2>&1 | grep -o "Unrecognized archive format"

# If the code reach here - return 0 otherwise we might confuse the test result with grep return value.
exit 0