Add test on virtual columns

dbms/tests/integration/test_storage_kafka/test.py

@@ -86,8 +86,8 @@ def kafka_produce_protobuf_messages(topic, start_index, num_messages):
 
 # Since everything is async and shaky when receiving messages from Kafka,
 # we may want to try and check results multiple times in a loop.
-def  kafka_check_result(result, check=False):
-    fpath = p.join(p.dirname(__file__), 'test_kafka_json.reference')
+def  kafka_check_result(result, check=False, ref_file='test_kafka_json.reference'):
+    fpath = p.join(p.dirname(__file__), ref_file)
     with open(fpath) as reference:
         if check:
             assert TSV(result) == TSV(reference)
@@ -365,6 +365,35 @@ def test_kafka_flush_on_big_message(kafka_cluster):
     assert int(result) == kafka_messages*batch_messages, 'ClickHouse lost some messages: {}'.format(result)
 
 
+def test_kafka_virtual_columns(kafka_cluster):
+    instance.query('''
+        CREATE TABLE test.kafka (key UInt64, value UInt64)
+            ENGINE = Kafka
+            SETTINGS
+                kafka_broker_list = 'kafka1:19092',
+                kafka_topic_list = 'json',
+                kafka_group_name = 'json',
+                kafka_format = 'JSONEachRow';
+        ''')
+
+    messages = ''
+    for i in range(25):
+        messages += json.dumps({'key': i, 'value': i}) + '\n'
+    kafka_produce('json', [messages])
+
+    messages = ''
+    for i in range(25, 50):
+        messages += json.dumps({'key': i, 'value': i}) + '\n'
+    kafka_produce('json', [messages])
+
+    result = ''
+    for i in range(50):
+        result += instance.query('SELECT _key, key, _topic, value, _offset FROM test.kafka')
+        if kafka_check_result(result):
+            break
+    kafka_check_result(result, True, 'test_kafka_virtual.reference')
+
+
 if __name__ == '__main__':
     cluster.start()
     raw_input("Cluster created, press any key to destroy...")

dbms/tests/integration/test_storage_kafka/test_kafka_virtual.reference

@@ -0,0 +1,50 @@
+	0	json	0	0
+	1	json	1	0
+	2	json	2	0
+	3	json	3	0
+	4	json	4	0
+	5	json	5	0
+	6	json	6	0
+	7	json	7	0
+	8	json	8	0
+	9	json	9	0
+	10	json	10	0
+	11	json	11	0
+	12	json	12	0
+	13	json	13	0
+	14	json	14	0
+	15	json	15	0
+	16	json	16	0
+	17	json	17	0
+	18	json	18	0
+	19	json	19	0
+	20	json	20	0
+	21	json	21	0
+	22	json	22	0
+	23	json	23	0
+	24	json	24	0
+	25	json	25	1
+	26	json	26	1
+	27	json	27	1
+	28	json	28	1
+	29	json	29	1
+	30	json	30	1
+	31	json	31	1
+	32	json	32	1
+	33	json	33	1
+	34	json	34	1
+	35	json	35	1
+	36	json	36	1
+	37	json	37	1
+	38	json	38	1
+	39	json	39	1
+	40	json	40	1
+	41	json	41	1
+	42	json	42	1
+	43	json	43	1
+	44	json	44	1
+	45	json	45	1
+	46	json	46	1
+	47	json	47	1
+	48	json	48	1
+	49	json	49	1