mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 16:50:48 +00:00
Add initial test for executable and http dictonaries [METR-23466]
This commit is contained in:
proller
parent
d1383c5f62
commit
22371fdead
@ -28,9 +28,11 @@ MSG_UNKNOWN = OP_SQUARE_BRACKET + colored(" UNKNOWN ", "yellow", attrs=['bold'])
|
||||
MSG_OK = OP_SQUARE_BRACKET + colored(" OK ", "green", attrs=['bold']) + CL_SQUARE_BRACKET
|
||||
MSG_SKIPPED = OP_SQUARE_BRACKET + colored(" SKIPPED ", "cyan", attrs=['bold']) + CL_SQUARE_BRACKET
|
||||
|
||||
#Not complete disable
|
||||
use_mysql = True
|
||||
use_mongo = True
|
||||
|
||||
wait_for_loading_sleep_time_sec = 3
|
||||
continue_on_error = False
|
||||
|
||||
failures = 0
|
||||
SERVER_DIED = False
|
||||
@ -46,35 +48,49 @@ dictionaries = [
|
||||
[ 'clickhouse_flat', 0, True ],
|
||||
[ 'mysql_flat', 0, True ],
|
||||
[ 'mongodb_flat', 0, True ],
|
||||
[ 'executable_flat', 0, True ],
|
||||
[ 'http_flat', 0, True ],
|
||||
|
||||
[ 'file_hashed', 0, True ],
|
||||
[ 'clickhouse_hashed', 0, True ],
|
||||
[ 'mysql_hashed', 0, True ],
|
||||
[ 'mongodb_hashed', 0, True ],
|
||||
[ 'executable_hashed', 0, True ],
|
||||
[ 'http_hashed', 0, True ],
|
||||
|
||||
[ 'clickhouse_cache', 0, True ],
|
||||
[ 'mysql_cache', 0, True ],
|
||||
[ 'mongodb_cache', 0, True ],
|
||||
[ 'executable_cache', 0, True ],
|
||||
[ 'http_cache', 0, True ],
|
||||
|
||||
# Complex key dictionaries with (UInt8, UInt8) key
|
||||
[ 'file_complex_integers_key_hashed', 1, False ],
|
||||
[ 'clickhouse_complex_integers_key_hashed', 1, False ],
|
||||
[ 'mysql_complex_integers_key_hashed', 1, False ],
|
||||
[ 'mongodb_complex_integers_key_hashed', 1, False ],
|
||||
[ 'executable_complex_integers_key_hashed', 1, False ],
|
||||
[ 'http_complex_integers_key_hashed', 1, False ],
|
||||
|
||||
[ 'clickhouse_complex_integers_key_cache', 1, False ],
|
||||
[ 'mysql_complex_integers_key_cache', 1, False ],
|
||||
[ 'mongodb_complex_integers_key_cache', 1, False ],
|
||||
[ 'executable_complex_integers_key_cache', 1, False ],
|
||||
[ 'http_complex_integers_key_cache', 1, False ],
|
||||
|
||||
# Complex key dictionaries with (String, UInt8) key
|
||||
[ 'file_complex_mixed_key_hashed', 2, False ],
|
||||
[ 'clickhouse_complex_mixed_key_hashed', 2, False ],
|
||||
[ 'mysql_complex_mixed_key_hashed', 2, False ],
|
||||
[ 'mongodb_complex_mixed_key_hashed', 2, False ],
|
||||
[ 'executable_complex_mixed_key_hashed', 2, False ],
|
||||
[ 'http_complex_mixed_key_hashed', 2, False ],
|
||||
|
||||
[ 'clickhouse_complex_mixed_key_cache', 2, False ],
|
||||
[ 'mysql_complex_mixed_key_cache', 2, False ],
|
||||
[ 'mongodb_complex_mixed_key_cache', 2, False ],
|
||||
[ 'executable_complex_mixed_key_hashed', 2, False ],
|
||||
[ 'http_complex_mixed_key_hashed', 2, False ],
|
||||
]
|
||||
|
||||
|
||||
@ -178,26 +194,27 @@ def generate_data(args):
|
||||
.format(prefix), shell=True)
|
||||
|
||||
# create MongoDB collection from complete_query via JSON file
|
||||
print 'Creating MongoDB collection'
|
||||
table_rows = json.loads(subprocess.check_output([
|
||||
args.client,
|
||||
'--port',
|
||||
args.port,
|
||||
'--output_format_json_quote_64bit_integers',
|
||||
'0',
|
||||
'--query',
|
||||
"select * from test.dictionary_source where not ignore(" \
|
||||
"concat('new Date(\\'', toString(Date_), '\\')') as Date_, " \
|
||||
"concat('new ISODate(\\'', replaceOne(toString(DateTime_, 'UTC'), ' ', 'T'), 'Z\\')') as DateTime_" \
|
||||
") format JSON"
|
||||
]))['data']
|
||||
if use_mongo:
|
||||
print 'Creating MongoDB collection'
|
||||
table_rows = json.loads(subprocess.check_output([
|
||||
args.client,
|
||||
'--port',
|
||||
args.port,
|
||||
'--output_format_json_quote_64bit_integers',
|
||||
'0',
|
||||
'--query',
|
||||
"select * from test.dictionary_source where not ignore(" \
|
||||
"concat('new Date(\\'', toString(Date_), '\\')') as Date_, " \
|
||||
"concat('new ISODate(\\'', replaceOne(toString(DateTime_, 'UTC'), ' ', 'T'), 'Z\\')') as DateTime_" \
|
||||
") format JSON"
|
||||
]))['data']
|
||||
|
||||
source_for_mongo = json.dumps(table_rows).replace(')"', ')').replace('"new', 'new')
|
||||
open('generated/full.json', 'w').write('db.dictionary_source.drop(); db.dictionary_source.insert(%s);' % source_for_mongo)
|
||||
result = system('cat {0}/full.json | mongo --quiet > /dev/null'.format(args.generated))
|
||||
if result != 0:
|
||||
print 'Could not create MongoDB collection'
|
||||
exit(-1)
|
||||
source_for_mongo = json.dumps(table_rows).replace(')"', ')').replace('"new', 'new')
|
||||
open('generated/full.json', 'w').write('db.dictionary_source.drop(); db.dictionary_source.insert(%s);' % source_for_mongo)
|
||||
result = system('cat {0}/full.json | mongo --quiet > /dev/null'.format(args.generated))
|
||||
if result != 0:
|
||||
print 'Could not create MongoDB collection'
|
||||
exit(-1)
|
||||
|
||||
|
||||
def generate_dictionaries(args):
|
||||
@ -280,6 +297,20 @@ def generate_dictionaries(args):
|
||||
</mongodb>
|
||||
'''
|
||||
|
||||
source_executable = '''
|
||||
<executable>
|
||||
<command>cat %s</command>
|
||||
<format>TabSeparated</format>
|
||||
</executable>
|
||||
'''
|
||||
|
||||
source_http = '''
|
||||
<http>
|
||||
<url>http://localhost:58000/generated/%s</url>
|
||||
<format>TabSeparated</format>
|
||||
</http>
|
||||
'''
|
||||
|
||||
layout_flat = '<flat />'
|
||||
layout_hashed = '<hashed />'
|
||||
layout_cache = '<cache><size_in_cells>128</size_in_cells></cache>'
|
||||
@ -335,35 +366,49 @@ def generate_dictionaries(args):
|
||||
[ source_clickhouse, layout_flat ],
|
||||
[ source_mysql, layout_flat ],
|
||||
[ source_mongodb, layout_flat ],
|
||||
[ source_executable % (generated_prefix + files[0]), layout_flat ],
|
||||
[ source_http % (files[0]), layout_flat ],
|
||||
|
||||
[ source_file % (generated_prefix + files[0]), layout_hashed],
|
||||
[ source_clickhouse, layout_hashed ],
|
||||
[ source_mysql, layout_hashed ],
|
||||
[ source_mongodb, layout_hashed ],
|
||||
[ source_executable % (generated_prefix + files[0]), layout_hashed ],
|
||||
[ source_http % (files[0]), layout_hashed ],
|
||||
|
||||
[ source_clickhouse, layout_cache ],
|
||||
[ source_mysql, layout_cache ],
|
||||
[ source_mongodb, layout_cache ],
|
||||
[ source_executable % (generated_prefix + files[0]), layout_cache ],
|
||||
[ source_http % (files[0]), layout_cache ],
|
||||
|
||||
# Complex key dictionaries with (UInt8, UInt8) key
|
||||
[ source_file % (generated_prefix + files[1]), layout_complex_key_hashed],
|
||||
[ source_clickhouse, layout_complex_key_hashed ],
|
||||
[ source_mysql, layout_complex_key_hashed ],
|
||||
[ source_mongodb, layout_complex_key_hashed ],
|
||||
[ source_executable % (generated_prefix + files[1]), layout_complex_key_hashed ],
|
||||
[ source_http % (files[1]), layout_complex_key_hashed ],
|
||||
|
||||
[ source_clickhouse, layout_complex_key_cache ],
|
||||
[ source_mysql, layout_complex_key_cache ],
|
||||
[ source_mongodb, layout_complex_key_cache ],
|
||||
[ source_executable % (generated_prefix + files[1]), layout_complex_key_cache ],
|
||||
[ source_http % (files[1]), layout_complex_key_cache ],
|
||||
|
||||
# Complex key dictionaries with (String, UInt8) key
|
||||
[ source_file % (generated_prefix + files[2]), layout_complex_key_hashed],
|
||||
[ source_clickhouse, layout_complex_key_hashed ],
|
||||
[ source_mysql, layout_complex_key_hashed ],
|
||||
[ source_mongodb, layout_complex_key_hashed ],
|
||||
[ source_executable % (generated_prefix + files[2]), layout_complex_key_hashed ],
|
||||
[ source_http % (files[2]), layout_complex_key_hashed ],
|
||||
|
||||
[ source_clickhouse, layout_complex_key_cache ],
|
||||
[ source_mysql, layout_complex_key_cache ],
|
||||
[ source_mongodb, layout_complex_key_cache ],
|
||||
[ source_executable % (generated_prefix + files[2]), layout_complex_key_cache ],
|
||||
[ source_http % (files[2]), layout_complex_key_cache ],
|
||||
]
|
||||
|
||||
for (name, key_idx, has_parent), (source, layout) in zip(dictionaries, sources_and_layouts):
|
||||
|
||||
35
dbms/tests/external_dictionaries/http_server.py
Executable file
35
dbms/tests/external_dictionaries/http_server.py
Executable file
@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python
|
||||
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
|
||||
from os import curdir, sep
|
||||
|
||||
PORT_NUMBER = 58000
|
||||
|
||||
class myHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path == "/":
|
||||
self.path = "/http_server.py"
|
||||
|
||||
try:
|
||||
f = open(curdir + sep + self.path)
|
||||
self.send_response(200)
|
||||
self.send_header('Content-type', 'text/plain')
|
||||
self.end_headers()
|
||||
self.wfile.write(f.read())
|
||||
f.close()
|
||||
return
|
||||
|
||||
except IOError:
|
||||
self.send_error(404,'File Not Found: %s' % self.path)
|
||||
|
||||
def do_POST(self):
|
||||
self.do_GET()
|
||||
return
|
||||
|
||||
try:
|
||||
server = HTTPServer(('', PORT_NUMBER), myHandler)
|
||||
print 'Started httpserver on port ' , PORT_NUMBER
|
||||
server.serve_forever()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print '^C received, shutting down the web server'
|
||||
server.socket.close()
|
||||
@ -1,5 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
|
||||
OS_NAME=`lsb_release -s -c`
|
||||
|
||||
if [ -z $(which python) ]; then
|
||||
sudo apt-get -y install python-lxml python-termcolor
|
||||
fi
|
||||
|
||||
# MySQL
|
||||
if [ -z $(which mysqld) ] || [ -z $(which mysqld) ]; then
|
||||
echo 'Installing MySQL'
|
||||
@ -31,13 +38,13 @@ if [ -z $LOCAL_INFILE_ENABLED ] || [ $LOCAL_INFILE_ENABLED != 1 ]; then
|
||||
sudo sed -i "$MY_CNF_PATTERN" $MY_CNF
|
||||
|
||||
echo 'Enabled local-infile support for mysql'
|
||||
sudo /etc/init.d/mysql stop
|
||||
sudo /etc/init.d/mysql start
|
||||
sudo service mysql stop
|
||||
sudo service mysql start
|
||||
else
|
||||
echo 'Support for local-infile already present'
|
||||
echo 'select 1;' | mysql $MYSQL_OPTIONS &>/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
sudo /etc/init.d/mysql start
|
||||
sudo service mysql start
|
||||
else
|
||||
echo 'MySQL already started'
|
||||
fi
|
||||
@ -46,18 +53,25 @@ fi
|
||||
# MongoDB
|
||||
if [ -z $(which mongod) ] || [ -z $(which mongo) ]; then
|
||||
echo 'Installing MongoDB'
|
||||
MONGODB_ORG_VERSION=3.0.6
|
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 &>/dev/null
|
||||
echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.0.list >/dev/null
|
||||
sudo apt-get update &>/dev/null
|
||||
sudo apt-get install -y mongodb-org=$MONGODB_ORG_VERSION >/dev/null
|
||||
which mongod >/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo 'Failed installing mongodb-org'
|
||||
exit -1
|
||||
|
||||
if [ $OS_NAME == "trusty" ]; then
|
||||
MONGODB_ORG_VERSION=3.0.6
|
||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 &>/dev/null
|
||||
#echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.0.list >/dev/null
|
||||
sudo apt-get update &>/dev/null
|
||||
sudo apt-get install -y mongodb-org=$MONGODB_ORG_VERSION >/dev/null
|
||||
|
||||
which mongod >/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo 'Failed installing mongodb-org'
|
||||
exit -1
|
||||
fi
|
||||
|
||||
echo "Installed mongodb-org $MONGODB_ORG_VERSION"
|
||||
else
|
||||
sudo apt-get install -y mongodb
|
||||
fi
|
||||
|
||||
echo "Installed mongodb-org $MONGODB_ORG_VERSION"
|
||||
fi
|
||||
|
||||
echo | mongo &>/dev/null
|
||||
@ -67,6 +81,9 @@ else
|
||||
echo 'MongoDB already started'
|
||||
fi
|
||||
|
||||
python http_server.py &
|
||||
http_pid=$!
|
||||
|
||||
# ClickHouse
|
||||
clickhouse-server &> clickhouse.log &
|
||||
sleep 3
|
||||
@ -84,3 +101,5 @@ fi
|
||||
kill -SIGTERM $PID
|
||||
#wait $PID
|
||||
echo 'Stopped ClickHouse server'
|
||||
|
||||
kill $http_pid
|
||||
|
||||
Loading…
Reference in New Issue
Block a user