ClickHouse/tests/queries/0_stateless/2010_lc_native.python

303 lines
8.3 KiB
Plaintext
Raw Normal View History

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import socket
2021-08-19 08:41:58 +00:00
import os
2021-08-19 08:41:58 +00:00
CLICKHOUSE_HOST = os.environ.get('CLICKHOUSE_HOST', '127.0.0.1')
2021-08-19 10:14:26 +00:00
CLICKHOUSE_PORT = int(os.environ.get('CLICKHOUSE_PORT_TCP', '900000'))
2021-08-19 11:44:51 +00:00
CLICKHOUSE_DATABASE = os.environ.get('CLICKHOUSE_DATABASE', 'default')
def writeVarUInt(x, ba):
for _ in range(0, 9):
byte = x & 0x7F
if x > 0x7F:
byte |= 0x80
ba.append(byte)
x >>= 7
if x == 0:
return
def writeStringBinary(s, ba):
b = bytes(s, 'utf-8')
writeVarUInt(len(s), ba)
ba.extend(b)
def readStrict(s, size = 1):
res = bytearray()
while size:
cur = s.recv(size)
# if not res:
# raise "Socket is closed"
size -= len(cur)
res.extend(cur)
return res
def readUInt(s, size=1):
res = readStrict(s, size)
val = 0
for i in range(len(res)):
val += res[i] << (i * 8)
return val
def readUInt8(s):
return readUInt(s)
def readUInt16(s):
return readUInt(s, 2)
def readUInt32(s):
return readUInt(s, 4)
def readUInt64(s):
return readUInt(s, 8)
def readVarUInt(s):
x = 0
for i in range(9):
byte = readStrict(s)[0]
x |= (byte & 0x7F) << (7 * i)
if not byte & 0x80:
return x
return x
def readStringBinary(s):
size = readVarUInt(s)
s = readStrict(s, size)
return s.decode('utf-8')
def sendHello(s):
ba = bytearray()
writeVarUInt(0, ba) # Hello
writeStringBinary('simple native protocol', ba)
writeVarUInt(21, ba)
writeVarUInt(9, ba)
writeVarUInt(54449, ba)
writeStringBinary('default', ba) # database
writeStringBinary('default', ba) # user
writeStringBinary('', ba) # pwd
s.sendall(ba)
def receiveHello(s):
p_type = readVarUInt(s)
assert (p_type == 0) # Hello
server_name = readStringBinary(s)
# print("Server name: ", server_name)
server_version_major = readVarUInt(s)
# print("Major: ", server_version_major)
server_version_minor = readVarUInt(s)
# print("Minor: ", server_version_minor)
server_revision = readVarUInt(s)
# print("Revision: ", server_revision)
server_timezone = readStringBinary(s)
# print("Timezone: ", server_timezone)
server_display_name = readStringBinary(s)
# print("Display name: ", server_display_name)
server_version_patch = readVarUInt(s)
# print("Version patch: ", server_version_patch)
def serializeClientInfo(ba):
writeStringBinary('default', ba) # initial_user
writeStringBinary('123456', ba) # initial_query_id
writeStringBinary('127.0.0.1:9000', ba) # initial_address
ba.extend([0] * 8) # initial_query_start_time_microseconds
ba.append(1) # TCP
writeStringBinary('os_user', ba) # os_user
writeStringBinary('client_hostname', ba) # client_hostname
writeStringBinary('client_name', ba) # client_name
writeVarUInt(21, ba)
writeVarUInt(9, ba)
writeVarUInt(54449, ba)
writeStringBinary('', ba) # quota_key
writeVarUInt(0, ba) # distributed_depth
writeVarUInt(1, ba) # client_version_patch
ba.append(0) # No telemetry
def sendQuery(s, query):
ba = bytearray()
writeVarUInt(1, ba) # query
writeStringBinary('123456', ba)
ba.append(1) # INITIAL_QUERY
# client info
serializeClientInfo(ba)
writeStringBinary('', ba) # No settings
writeStringBinary('', ba) # No interserver secret
writeVarUInt(2, ba) # Stage - Complete
ba.append(0) # No compression
writeStringBinary(query + ' settings input_format_defaults_for_omitted_fields=0', ba) # query, finally
s.sendall(ba)
def serializeBlockInfo(ba):
writeVarUInt(1, ba) # 1
ba.append(0) # is_overflows
writeVarUInt(2, ba) # 2
writeVarUInt(0, ba) # 0
ba.extend([0] * 4) # bucket_num
def sendEmptyBlock(s):
ba = bytearray()
writeVarUInt(2, ba) # Data
writeStringBinary('', ba)
serializeBlockInfo(ba)
writeVarUInt(0, ba) # rows
writeVarUInt(0, ba) # columns
s.sendall(ba)
def readHeader(s):
readVarUInt(s) # Data
readStringBinary(s) # external table name
# BlockInfo
readVarUInt(s) # 1
readUInt8(s) # is_overflows
readVarUInt(s) # 2
readUInt32(s) # bucket_num
readVarUInt(s) # 0
columns = readVarUInt(s) # rows
rows = readVarUInt(s) # columns
print("Rows {} Columns {}".format(rows, columns))
for _ in range(columns):
col_name = readStringBinary(s)
type_name = readStringBinary(s)
print("Column {} type {}".format(col_name, type_name))
def readException(s):
assert(readVarUInt(s) == 2)
code = readUInt32(s)
name = readStringBinary(s)
text = readStringBinary(s)
readStringBinary(s) # trace
assert(readUInt8(s) == 0) # has_nested
print("code {}: {}".format(code, text.replace('DB::Exception:', '')))
def insertValidLowCardinalityRow():
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
Increase timeout in 2010_lc_native 10 seconds it not enough [1]: 2010_lc_native: [ FAIL ] 15.53 sec. - having stderror: Traceback (most recent call last): File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 302, in <module> main() File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 298, in main insertLowCardinalityRowWithIndexOverflow() File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 259, in insertLowCardinalityRowWithIndexOverflow readException(s) File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 185, in readException assert(readVarUInt(s) == 2) File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 65, in readVarUInt byte = readStrict(s)[0] File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 34, in readStrict cur = s.recv(size) socket.timeout: timed out [1]: https://clickhouse-test-reports.s3.yandex.net/29060/7f1160c466a968c680d1bc73e0ad5e9419d0b6d0/functional_stateless_tests_(debug).html#fail1 Server log: 2021.09.17 21:02:53.734388 [ 682 ] {123456} <Trace> ContextAccess (default): Access granted: INSERT(x) ON test_0tmdh4.tab 2021.09.17 21:03:04.835044 [ 682 ] {123456} <Error> executeQuery: Code: 117. DB::Exception: Index for LowCardinality is out of range. Dictionary size is 1, but found index with value 72057594037927936. (INCORRECT_DATA) (version 21.11.1.8109) (from [::ffff:127.0.0.1]:59512) (in query: insert into test_0tmdh4.tab format TSV settings input_format_defaults_for_omitted_fields=0), Stack trace (when copying this message, always include the lines below): And according to trace_log the problem is slow unwind: 2021-09-17 21:02:55.103893 libunwind::LocalAddressSpace::get32 2021-09-17 21:02:56.151960 libunwind::LocalAddressSpace::get8 2021-09-17 21:03:03.168447 libunwind::LocalAddressSpace::getULEB128
2021-09-18 07:52:44 +00:00
s.settimeout(30)
2021-08-19 08:41:58 +00:00
s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT))
sendHello(s)
receiveHello(s)
2021-08-19 11:44:51 +00:00
sendQuery(s, 'insert into {}.tab format TSV'.format(CLICKHOUSE_DATABASE))
# external tables
sendEmptyBlock(s)
readHeader(s)
# Data
ba = bytearray()
writeVarUInt(2, ba) # Data
writeStringBinary('', ba)
serializeBlockInfo(ba)
writeVarUInt(1, ba) # rows
writeVarUInt(1, ba) # columns
writeStringBinary('x', ba)
writeStringBinary('LowCardinality(String)', ba)
ba.extend([1] + [0] * 7) # SharedDictionariesWithAdditionalKeys
ba.extend([3, 2] + [0] * 6) # indexes type: UInt64 [3], with additional keys [2]
ba.extend([1] + [0] * 7) # num_keys in dict
writeStringBinary('hello', ba) # key
ba.extend([1] + [0] * 7) # num_indexes
ba.extend([0] * 8) # UInt64 index (0 for 'hello')
s.sendall(ba)
# Fin block
sendEmptyBlock(s)
assert(readVarUInt(s) == 5) # End of stream
s.close()
def insertLowCardinalityRowWithIndexOverflow():
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
Increase timeout in 2010_lc_native 10 seconds it not enough [1]: 2010_lc_native: [ FAIL ] 15.53 sec. - having stderror: Traceback (most recent call last): File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 302, in <module> main() File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 298, in main insertLowCardinalityRowWithIndexOverflow() File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 259, in insertLowCardinalityRowWithIndexOverflow readException(s) File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 185, in readException assert(readVarUInt(s) == 2) File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 65, in readVarUInt byte = readStrict(s)[0] File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 34, in readStrict cur = s.recv(size) socket.timeout: timed out [1]: https://clickhouse-test-reports.s3.yandex.net/29060/7f1160c466a968c680d1bc73e0ad5e9419d0b6d0/functional_stateless_tests_(debug).html#fail1 Server log: 2021.09.17 21:02:53.734388 [ 682 ] {123456} <Trace> ContextAccess (default): Access granted: INSERT(x) ON test_0tmdh4.tab 2021.09.17 21:03:04.835044 [ 682 ] {123456} <Error> executeQuery: Code: 117. DB::Exception: Index for LowCardinality is out of range. Dictionary size is 1, but found index with value 72057594037927936. (INCORRECT_DATA) (version 21.11.1.8109) (from [::ffff:127.0.0.1]:59512) (in query: insert into test_0tmdh4.tab format TSV settings input_format_defaults_for_omitted_fields=0), Stack trace (when copying this message, always include the lines below): And according to trace_log the problem is slow unwind: 2021-09-17 21:02:55.103893 libunwind::LocalAddressSpace::get32 2021-09-17 21:02:56.151960 libunwind::LocalAddressSpace::get8 2021-09-17 21:03:03.168447 libunwind::LocalAddressSpace::getULEB128
2021-09-18 07:52:44 +00:00
s.settimeout(30)
2021-08-19 08:41:58 +00:00
s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT))
sendHello(s)
receiveHello(s)
2021-08-19 11:44:51 +00:00
sendQuery(s, 'insert into {}.tab format TSV'.format(CLICKHOUSE_DATABASE))
# external tables
sendEmptyBlock(s)
readHeader(s)
# Data
ba = bytearray()
writeVarUInt(2, ba) # Data
writeStringBinary('', ba)
serializeBlockInfo(ba)
writeVarUInt(1, ba) # rows
writeVarUInt(1, ba) # columns
writeStringBinary('x', ba)
writeStringBinary('LowCardinality(String)', ba)
ba.extend([1] + [0] * 7) # SharedDictionariesWithAdditionalKeys
ba.extend([3, 2] + [0] * 6) # indexes type: UInt64 [3], with additional keys [2]
ba.extend([1] + [0] * 7) # num_keys in dict
writeStringBinary('hello', ba) # key
ba.extend([1] + [0] * 7) # num_indexes
ba.extend([0] * 7 + [1]) # UInt64 index (overflow)
s.sendall(ba)
readException(s)
s.close()
def insertLowCardinalityRowWithIncorrectDictType():
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
Increase timeout in 2010_lc_native 10 seconds it not enough [1]: 2010_lc_native: [ FAIL ] 15.53 sec. - having stderror: Traceback (most recent call last): File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 302, in <module> main() File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 298, in main insertLowCardinalityRowWithIndexOverflow() File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 259, in insertLowCardinalityRowWithIndexOverflow readException(s) File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 185, in readException assert(readVarUInt(s) == 2) File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 65, in readVarUInt byte = readStrict(s)[0] File "/usr/share/clickhouse-test/queries/0_stateless/2010_lc_native.python", line 34, in readStrict cur = s.recv(size) socket.timeout: timed out [1]: https://clickhouse-test-reports.s3.yandex.net/29060/7f1160c466a968c680d1bc73e0ad5e9419d0b6d0/functional_stateless_tests_(debug).html#fail1 Server log: 2021.09.17 21:02:53.734388 [ 682 ] {123456} <Trace> ContextAccess (default): Access granted: INSERT(x) ON test_0tmdh4.tab 2021.09.17 21:03:04.835044 [ 682 ] {123456} <Error> executeQuery: Code: 117. DB::Exception: Index for LowCardinality is out of range. Dictionary size is 1, but found index with value 72057594037927936. (INCORRECT_DATA) (version 21.11.1.8109) (from [::ffff:127.0.0.1]:59512) (in query: insert into test_0tmdh4.tab format TSV settings input_format_defaults_for_omitted_fields=0), Stack trace (when copying this message, always include the lines below): And according to trace_log the problem is slow unwind: 2021-09-17 21:02:55.103893 libunwind::LocalAddressSpace::get32 2021-09-17 21:02:56.151960 libunwind::LocalAddressSpace::get8 2021-09-17 21:03:03.168447 libunwind::LocalAddressSpace::getULEB128
2021-09-18 07:52:44 +00:00
s.settimeout(30)
2021-08-19 08:41:58 +00:00
s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT))
sendHello(s)
receiveHello(s)
2021-08-19 11:44:51 +00:00
sendQuery(s, 'insert into {}.tab format TSV'.format(CLICKHOUSE_DATABASE))
# external tables
sendEmptyBlock(s)
readHeader(s)
# Data
ba = bytearray()
writeVarUInt(2, ba) # Data
writeStringBinary('', ba)
serializeBlockInfo(ba)
writeVarUInt(1, ba) # rows
writeVarUInt(1, ba) # columns
writeStringBinary('x', ba)
writeStringBinary('LowCardinality(String)', ba)
ba.extend([1] + [0] * 7) # SharedDictionariesWithAdditionalKeys
ba.extend([3, 3] + [0] * 6) # indexes type: UInt64 [3], with global dict and add keys [1 + 2]
ba.extend([1] + [0] * 7) # num_keys in dict
writeStringBinary('hello', ba) # key
ba.extend([1] + [0] * 7) # num_indexes
ba.extend([0] * 8) # UInt64 index (overflow)
s.sendall(ba)
readException(s)
s.close()
def main():
insertValidLowCardinalityRow()
insertLowCardinalityRowWithIndexOverflow()
insertLowCardinalityRowWithIncorrectDictType()
if __name__ == "__main__":
main()