diff --git a/dbms/tests/external_dictionaries/generate_and_test.py b/dbms/tests/external_dictionaries/generate_and_test.py index 903d6b1780a..34c5b5dc3ef 100755 --- a/dbms/tests/external_dictionaries/generate_and_test.py +++ b/dbms/tests/external_dictionaries/generate_and_test.py @@ -28,9 +28,11 @@ MSG_UNKNOWN = OP_SQUARE_BRACKET + colored(" UNKNOWN ", "yellow", attrs=['bold']) MSG_OK = OP_SQUARE_BRACKET + colored(" OK ", "green", attrs=['bold']) + CL_SQUARE_BRACKET MSG_SKIPPED = OP_SQUARE_BRACKET + colored(" SKIPPED ", "cyan", attrs=['bold']) + CL_SQUARE_BRACKET +#Not complete disable +use_mysql = True +use_mongo = True wait_for_loading_sleep_time_sec = 3 -continue_on_error = False failures = 0 SERVER_DIED = False @@ -46,35 +48,49 @@ dictionaries = [ [ 'clickhouse_flat', 0, True ], [ 'mysql_flat', 0, True ], [ 'mongodb_flat', 0, True ], + [ 'executable_flat', 0, True ], + [ 'http_flat', 0, True ], [ 'file_hashed', 0, True ], [ 'clickhouse_hashed', 0, True ], [ 'mysql_hashed', 0, True ], [ 'mongodb_hashed', 0, True ], + [ 'executable_hashed', 0, True ], + [ 'http_hashed', 0, True ], [ 'clickhouse_cache', 0, True ], [ 'mysql_cache', 0, True ], [ 'mongodb_cache', 0, True ], + [ 'executable_cache', 0, True ], + [ 'http_cache', 0, True ], # Complex key dictionaries with (UInt8, UInt8) key [ 'file_complex_integers_key_hashed', 1, False ], [ 'clickhouse_complex_integers_key_hashed', 1, False ], [ 'mysql_complex_integers_key_hashed', 1, False ], [ 'mongodb_complex_integers_key_hashed', 1, False ], + [ 'executable_complex_integers_key_hashed', 1, False ], + [ 'http_complex_integers_key_hashed', 1, False ], [ 'clickhouse_complex_integers_key_cache', 1, False ], [ 'mysql_complex_integers_key_cache', 1, False ], [ 'mongodb_complex_integers_key_cache', 1, False ], + [ 'executable_complex_integers_key_cache', 1, False ], + [ 'http_complex_integers_key_cache', 1, False ], # Complex key dictionaries with (String, UInt8) key [ 'file_complex_mixed_key_hashed', 2, False ], [ 'clickhouse_complex_mixed_key_hashed', 2, False ], [ 'mysql_complex_mixed_key_hashed', 2, False ], [ 'mongodb_complex_mixed_key_hashed', 2, False ], + [ 'executable_complex_mixed_key_hashed', 2, False ], + [ 'http_complex_mixed_key_hashed', 2, False ], [ 'clickhouse_complex_mixed_key_cache', 2, False ], [ 'mysql_complex_mixed_key_cache', 2, False ], [ 'mongodb_complex_mixed_key_cache', 2, False ], + [ 'executable_complex_mixed_key_hashed', 2, False ], + [ 'http_complex_mixed_key_hashed', 2, False ], ] @@ -178,26 +194,27 @@ def generate_data(args): .format(prefix), shell=True) # create MongoDB collection from complete_query via JSON file - print 'Creating MongoDB collection' - table_rows = json.loads(subprocess.check_output([ - args.client, - '--port', - args.port, - '--output_format_json_quote_64bit_integers', - '0', - '--query', - "select * from test.dictionary_source where not ignore(" \ - "concat('new Date(\\'', toString(Date_), '\\')') as Date_, " \ - "concat('new ISODate(\\'', replaceOne(toString(DateTime_, 'UTC'), ' ', 'T'), 'Z\\')') as DateTime_" \ - ") format JSON" - ]))['data'] + if use_mongo: + print 'Creating MongoDB collection' + table_rows = json.loads(subprocess.check_output([ + args.client, + '--port', + args.port, + '--output_format_json_quote_64bit_integers', + '0', + '--query', + "select * from test.dictionary_source where not ignore(" \ + "concat('new Date(\\'', toString(Date_), '\\')') as Date_, " \ + "concat('new ISODate(\\'', replaceOne(toString(DateTime_, 'UTC'), ' ', 'T'), 'Z\\')') as DateTime_" \ + ") format JSON" + ]))['data'] - source_for_mongo = json.dumps(table_rows).replace(')"', ')').replace('"new', 'new') - open('generated/full.json', 'w').write('db.dictionary_source.drop(); db.dictionary_source.insert(%s);' % source_for_mongo) - result = system('cat {0}/full.json | mongo --quiet > /dev/null'.format(args.generated)) - if result != 0: - print 'Could not create MongoDB collection' - exit(-1) + source_for_mongo = json.dumps(table_rows).replace(')"', ')').replace('"new', 'new') + open('generated/full.json', 'w').write('db.dictionary_source.drop(); db.dictionary_source.insert(%s);' % source_for_mongo) + result = system('cat {0}/full.json | mongo --quiet > /dev/null'.format(args.generated)) + if result != 0: + print 'Could not create MongoDB collection' + exit(-1) def generate_dictionaries(args): @@ -280,6 +297,20 @@ def generate_dictionaries(args): ''' + source_executable = ''' + + cat %s + TabSeparated + + ''' + + source_http = ''' + + http://localhost:58000/generated/%s + TabSeparated + + ''' + layout_flat = '' layout_hashed = '' layout_cache = '128' @@ -335,35 +366,49 @@ def generate_dictionaries(args): [ source_clickhouse, layout_flat ], [ source_mysql, layout_flat ], [ source_mongodb, layout_flat ], + [ source_executable % (generated_prefix + files[0]), layout_flat ], + [ source_http % (files[0]), layout_flat ], [ source_file % (generated_prefix + files[0]), layout_hashed], [ source_clickhouse, layout_hashed ], [ source_mysql, layout_hashed ], [ source_mongodb, layout_hashed ], + [ source_executable % (generated_prefix + files[0]), layout_hashed ], + [ source_http % (files[0]), layout_hashed ], [ source_clickhouse, layout_cache ], [ source_mysql, layout_cache ], [ source_mongodb, layout_cache ], + [ source_executable % (generated_prefix + files[0]), layout_cache ], + [ source_http % (files[0]), layout_cache ], # Complex key dictionaries with (UInt8, UInt8) key [ source_file % (generated_prefix + files[1]), layout_complex_key_hashed], [ source_clickhouse, layout_complex_key_hashed ], [ source_mysql, layout_complex_key_hashed ], [ source_mongodb, layout_complex_key_hashed ], + [ source_executable % (generated_prefix + files[1]), layout_complex_key_hashed ], + [ source_http % (files[1]), layout_complex_key_hashed ], [ source_clickhouse, layout_complex_key_cache ], [ source_mysql, layout_complex_key_cache ], [ source_mongodb, layout_complex_key_cache ], + [ source_executable % (generated_prefix + files[1]), layout_complex_key_cache ], + [ source_http % (files[1]), layout_complex_key_cache ], # Complex key dictionaries with (String, UInt8) key [ source_file % (generated_prefix + files[2]), layout_complex_key_hashed], [ source_clickhouse, layout_complex_key_hashed ], [ source_mysql, layout_complex_key_hashed ], [ source_mongodb, layout_complex_key_hashed ], + [ source_executable % (generated_prefix + files[2]), layout_complex_key_hashed ], + [ source_http % (files[2]), layout_complex_key_hashed ], [ source_clickhouse, layout_complex_key_cache ], [ source_mysql, layout_complex_key_cache ], [ source_mongodb, layout_complex_key_cache ], + [ source_executable % (generated_prefix + files[2]), layout_complex_key_cache ], + [ source_http % (files[2]), layout_complex_key_cache ], ] for (name, key_idx, has_parent), (source, layout) in zip(dictionaries, sources_and_layouts): diff --git a/dbms/tests/external_dictionaries/http_server.py b/dbms/tests/external_dictionaries/http_server.py new file mode 100755 index 00000000000..4310985575c --- /dev/null +++ b/dbms/tests/external_dictionaries/http_server.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +from os import curdir, sep + +PORT_NUMBER = 58000 + +class myHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/": + self.path = "/http_server.py" + + try: + f = open(curdir + sep + self.path) + self.send_response(200) + self.send_header('Content-type', 'text/plain') + self.end_headers() + self.wfile.write(f.read()) + f.close() + return + + except IOError: + self.send_error(404,'File Not Found: %s' % self.path) + + def do_POST(self): + self.do_GET() + return + +try: + server = HTTPServer(('', PORT_NUMBER), myHandler) + print 'Started httpserver on port ' , PORT_NUMBER + server.serve_forever() + +except KeyboardInterrupt: + print '^C received, shutting down the web server' + server.socket.close() diff --git a/dbms/tests/external_dictionaries/run.sh b/dbms/tests/external_dictionaries/run.sh index 763b6ff6fcd..a846095672b 100755 --- a/dbms/tests/external_dictionaries/run.sh +++ b/dbms/tests/external_dictionaries/run.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash + +OS_NAME=`lsb_release -s -c` + +if [ -z $(which python) ]; then + sudo apt-get -y install python-lxml python-termcolor +fi + # MySQL if [ -z $(which mysqld) ] || [ -z $(which mysqld) ]; then echo 'Installing MySQL' @@ -31,13 +38,13 @@ if [ -z $LOCAL_INFILE_ENABLED ] || [ $LOCAL_INFILE_ENABLED != 1 ]; then sudo sed -i "$MY_CNF_PATTERN" $MY_CNF echo 'Enabled local-infile support for mysql' - sudo /etc/init.d/mysql stop - sudo /etc/init.d/mysql start + sudo service mysql stop + sudo service mysql start else echo 'Support for local-infile already present' echo 'select 1;' | mysql $MYSQL_OPTIONS &>/dev/null if [ $? -ne 0 ]; then - sudo /etc/init.d/mysql start + sudo service mysql start else echo 'MySQL already started' fi @@ -46,18 +53,25 @@ fi # MongoDB if [ -z $(which mongod) ] || [ -z $(which mongo) ]; then echo 'Installing MongoDB' - MONGODB_ORG_VERSION=3.0.6 - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 &>/dev/null - echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.0.list >/dev/null - sudo apt-get update &>/dev/null - sudo apt-get install -y mongodb-org=$MONGODB_ORG_VERSION >/dev/null - which mongod >/dev/null - if [ $? -ne 0 ]; then - echo 'Failed installing mongodb-org' - exit -1 + + if [ $OS_NAME == "trusty" ]; then + MONGODB_ORG_VERSION=3.0.6 + sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 &>/dev/null + #echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.0.list >/dev/null + sudo apt-get update &>/dev/null + sudo apt-get install -y mongodb-org=$MONGODB_ORG_VERSION >/dev/null + + which mongod >/dev/null + if [ $? -ne 0 ]; then + echo 'Failed installing mongodb-org' + exit -1 + fi + + echo "Installed mongodb-org $MONGODB_ORG_VERSION" + else + sudo apt-get install -y mongodb fi - echo "Installed mongodb-org $MONGODB_ORG_VERSION" fi echo | mongo &>/dev/null @@ -67,6 +81,9 @@ else echo 'MongoDB already started' fi +python http_server.py & +http_pid=$! + # ClickHouse clickhouse-server &> clickhouse.log & sleep 3 @@ -84,3 +101,5 @@ fi kill -SIGTERM $PID #wait $PID echo 'Stopped ClickHouse server' + +kill $http_pid