ClickHouse/dbms/scripts/merge_algorithm/stats.py

import time
import ast
from datetime import datetime

FILE='data.tsv'

def get_metrix():
	data = []
	time_to_merge = 0
	count_of_parts = 0
	max_count_of_parts = 0
	parts_in_time = []
	last_date = 0
	for line in open(FILE):
		fields = line.split('\t')
		last_date = datetime.strptime(fields[2], '%Y-%m-%d %H:%M:%S')
		break

	for line in open(FILE):
		fields = line.split('\t')
		cur_date = datetime.strptime(fields[2], '%Y-%m-%d %H:%M:%S')
		if fields[0] == '2':
			time_to_merge += int(fields[4])
			list = ast.literal_eval(fields[-1])
			count_of_parts -= len(list) - 1
		else:
			count_of_parts += 1

		if max_count_of_parts < count_of_parts:
			max_count_of_parts = count_of_parts

		parts_in_time.append([(cur_date-last_date).total_seconds(), count_of_parts])
		last_date = cur_date

	stats_parts_in_time = []
	global_time = 0
	average_parts = 0
	for i in range(max_count_of_parts + 1):
		stats_parts_in_time.append(0)

	for elem in parts_in_time:
		stats_parts_in_time[elem[1]] += elem[0]
		global_time += elem[0]
		average_parts += elem[0] * elem[1]

	for i in range(max_count_of_parts):
		stats_parts_in_time[i] /= global_time
	average_parts /= global_time

	return time_to_merge, max_count_of_parts, average_parts, stats_parts_in_time

def main():
   time_to_merge, max_parts, average_parts, stats_parts = get_metrix()
   print('time_to_merge=', time_to_merge)
   print('max_parts=', max_parts)
   print('average_parts=', average_parts)
   print('stats_parts=', stats_parts)


if __name__ == '__main__':
	main()