Add Pinot

2024-11-09 17:14:47 +00:00 · 2022-07-07 17:05:11 +00:00 · 2022-07-07 17:05:11 +00:00 · 2e84e22bed
commit 2e84e22bed
parent 2bc099e03a
8 changed files with 674 additions and 0 deletions
--- a/benchmark/pinot/benchmark.sh
+++ b/benchmark/pinot/benchmark.sh
@ -0,0 +1,44 @@
+#!/bin/bash
+
+sudo apt-get update
+sudo apt install openjdk-11-jdk jq -y
+sudo update-alternatives --config java
+
+# Install
+
+PINOT_VERSION=0.10.0
+
+wget https://downloads.apache.org/pinot/apache-pinot-$PINOT_VERSION/apache-pinot-$PINOT_VERSION-bin.tar.gz
+tar -zxvf apache-pinot-$PINOT_VERSION-bin.tar.gz
+
+./apache-pinot-$PINOT_VERSION-bin/bin/pinot-admin.sh QuickStart -type batch &
+sleep 30
+./apache-pinot-$PINOT_VERSION-bin/bin/pinot-admin.sh AddTable -tableConfigFile offline_table.json -schemaFile schema.json -exec
+
+# Load the data
+
+wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+gzip -d hits.tsv.gz
+
+# Pinot was unable to load data as a single file wihout any errors returned. We have to split the data
+split -d --additional-suffix .tsv --verbose -n l/100 hits.tsv parts
+
+# Pinot can't load value '"tatuirovarki_redmond' so we need to fix this row to make it work
+sed parts93.tsv -e 's "tatuirovarki_redmond tatuirovarki_redmond g' -i
+
+# Fix path to local directory
+sed splitted.yaml 's PWD_DIR_PLACEHOLDER '$PWD' g' -i
+sed local.yaml 's PWD_DIR_PLACEHOLDER '$PWD' g' -i
+
+# Load data
+./apache-pinot-$PINOT_VERSION-bin/bin/pinot-admin.sh LaunchDataIngestionJob -jobSpecFile splitted.yaml
+
+# After upload it shows 94465149 rows instead of 99997497 in the dataset
+
+# Run the queries
+./run.sh
+
+# stop Druid services
+kill %1
+
+du -bcs ./batch
--- a/benchmark/pinot/local.yaml
+++ b/benchmark/pinot/local.yaml
@ -0,0 +1,38 @@
+executionFrameworkSpec:
+  name: 'standalone'
+  segmentGenerationJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
+  segmentTarPushJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
+  segmentUriPushJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+  segmentMetadataPushJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
+jobType: SegmentCreationAndTarPush
+inputDirURI: 'PWD_DIR_PLACEHOLDER'
+includeFileNamePattern: 'glob:PWD_DIR_PLACEHOLDER/hits.tsv'
+outputDirURI: 'batch/hits/segments'
+overwriteOutput: true
+segmentCreationJobParallelism: 10
+pinotFSSpecs:
+  - scheme: file
+    className: org.apache.pinot.spi.filesystem.LocalPinotFS
+recordReaderSpec:
+  dataFormat: 'csv'
+  className: 'org.apache.pinot.plugin.inputformat.csv.CSVRecordReader'
+  configClassName: 'org.apache.pinot.plugin.inputformat.csv.CSVRecordReaderConfig'
+  configs:
+    fileFormat: 'default'
+    delimiter: '	'
+    multiValueDelimiter: ''
+    header: 'WatchID	JavaEnable	Title	GoodEvent	EventTime	EventDate	CounterID	ClientIP	RegionID	UserID	CounterClass	OS	UserAgent	URL	Referer	IsRefresh	RefererCategoryID	RefererRegionID	URLCategoryID	URLRegionID	ResolutionWidth	ResolutionHeight	ResolutionDepth	FlashMajor	FlashMinor	FlashMinor2	NetMajor	NetMinor	UserAgentMajor	UserAgentMinor	CookieEnable	JavascriptEnable	IsMobile	MobilePhone	MobilePhoneModel	Params	IPNetworkID	TraficSourceID	SearchEngineID	SearchPhrase	AdvEngineID	IsArtifical	WindowClientWidth	WindowClientHeight	ClientTimeZone	ClientEventTime	SilverlightVersion1	SilverlightVersion2	SilverlightVersion3	SilverlightVersion4	PageCharset	CodeVersion	IsLink	IsDownload	IsNotBounce	FUniqID	OriginalURL	HID	IsOldCounter	IsEvent	IsParameter	DontCountHits	WithHash	HitColor	LocalEventTime	Age	Sex	Income	Interests	Robotness	RemoteIP	WindowName	OpenerName	HistoryLength	BrowserLanguage	BrowserCountry	SocialNetwork	SocialAction	HTTPError	SendTiming	DNSTiming	ConnectTiming	ResponseStartTiming	ResponseEndTiming	FetchTiming	SocialSourceNetworkID	SocialSourcePage	ParamPrice	ParamOrderID	ParamCurrency	ParamCurrencyID	OpenstatServiceName	OpenstatCampaignID	OpenstatAdID	OpenstatSourceID	UTMSource	UTMMedium	UTMCampaign	UTMContent	UTMTerm	FromTag	HasGCLID	RefererHash	URLHash'
+
+
+tableSpec:
+  tableName: 'hits'
+  schemaURI: 'http://localhost:9000/tables/hits/schema'
+  tableConfigURI: 'http://localhost:9000/tables/hits'
+
+pinotClusterSpecs:
+  - controllerURI: 'http://localhost:9000'
+
+pushJobSpec:
+  pushAttempts: 10
+  pushRetryIntervalMillis: 1000
+  segmentUriPrefix: 'file://'
--- a/benchmark/pinot/offline_table.json
+++ b/benchmark/pinot/offline_table.json
@ -0,0 +1,17 @@
+{
+  "tableName": "hits",
+  "tableType": "OFFLINE",
+  "segmentsConfig": {
+    "segmentPushType": "APPEND",
+    "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy",
+    "schemaName": "hitsSchema",
+    "replication": "1"
+  },
+  "tableIndexConfig":{},
+  "metadata":{},
+  "tenants":
+  {
+	  "server": "DefaultTenant",
+	  "broker": "DefaultTenant"
+  }
+}
--- a/benchmark/pinot/queries.sql
+++ b/benchmark/pinot/queries.sql
@ -0,0 +1,43 @@
+SELECT COUNT(*) FROM hits;
+SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0;
+SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits;
+SELECT AVG(UserID) FROM hits;
+SELECT COUNT(DISTINCT UserID) FROM hits;
+SELECT COUNT(DISTINCT SearchPhrase) FROM hits;
+SELECT MIN(EventDate), MAX(EventDate) FROM hits;
+SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC;
+SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10;
+SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10;
+SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10;
+SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u FROM hits WHERE MobilePhoneModel <> '' GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10;
+SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;
+SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u FROM hits WHERE SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10;
+SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10;
+SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10;
+SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;
+SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10;
+SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10;
+SELECT UserID FROM hits WHERE UserID = 435090932899640449;
+SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%';
+SELECT SearchPhrase, MIN(URL), COUNT(*) AS c FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;
+SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10;
+SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10;
+SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10;
+SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10;
+SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10;
+SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;
+SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;
+SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;
+SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;
+SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;
+SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;
+SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;
+SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;
+SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;
+SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;
+SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;
+SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;
+SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;
+SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;
+SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;
+SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;
--- a/benchmark/pinot/results/c6a.4xlarge.txt
+++ b/benchmark/pinot/results/c6a.4xlarge.txt
@ -0,0 +1,43 @@
+[0.002, 0.001, 0.001],
+[0.176, 0.176, 0.175],
+[0.237, 0.261, 0.244],
+[0.449, 0.265, 0.225],
+[3.691, 3.453, 3.432],
+[28.785, 13.874, 14.104],
+["-", "-", "-"],
+[0.128, 0.127, 0.14],
+[2.871, 2.741, 2.775],
+[2.984, 3.034, 3.047],
+[3.984, 3.965, 4.141],
+[3.916, 3.896, 3.893],
+[2.824, 2.971, 3.004],
+[6.048, 6.505, 6.021],
+[3.065, 3.165, 2.956],
+[4.829, 4.749, 4.857],
+[5.204, 4.987, 4.966],
+[0.817, 0.809, 0.783],
+["-", "-", "-"],
+[0.016, 0.014, 0.014],
+[51.341, 18.48, 18.07],
+["-", "-", "-"],
+[72.358, 70.585, 13.441],
+[7.029, 5.451, 5.545],
+[0.355, 0.309, 0.27],
+[7.264, 0.373, 1.21],
+[3.244, 0.471, 0.472],
+[26.148, 2.246, 2.261],
+["-", "-", "-"],
+[2.098, 2.104, 2.047],
+[4.668, 4.382, 4.312],
+[5.329, 5.073, 4.73],
+[5.002, 4.729, 5.061],
+[4.988, 4.922, 4.823],
+[5.886, 5.893, 6.032],
+[5.591, 5.605, 5.606],
+[0.246, 0.191, 0.198],
+[0.344, 0.068, 0.065],
+[0.04, 0.032, 0.033],
+[1.401, 0.648, 0.615],
+[0.107, 0.067, 0.075],
+[0.04, 0.048, 0.035],
+["-", "-", "-"]
--- a/benchmark/pinot/run.sh
+++ b/benchmark/pinot/run.sh
@ -0,0 +1,15 @@
+#!/bin/bash
+
+TRIES=3
+cat queries.sql | while read query; do
+    sync
+    echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null
+    echo -n "["
+    for i in $(seq 1 $TRIES); do
+        echo "{\"sql\":\"$query  option(timeoutMs=300000)\"}"| tr -d ';' > query.json
+        RES=$(curl -s -XPOST -H'Content-Type: application/json' http://localhost:8000/query/sql/ -d @query.json | jq 'if .exceptions == [] then .timeUsedMs/1000 else "-" end' )
+        [[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null"
+        [[ "$i" != $TRIES ]] && echo -n ", "
+    done
+    echo "],"
+done
--- a/benchmark/pinot/schema.json
+++ b/benchmark/pinot/schema.json
@ -0,0 +1,437 @@
+{
+  "metricFieldSpecs": [
+{
+    "name": "AdvEngineID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "ClientIP",
+    "dataType": "LONG"
+  },
+  {
+    "name": "RegionID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "IsRefresh",
+    "dataType": "LONG"
+  },
+  {
+    "name": "ResolutionWidth",
+    "dataType": "LONG"
+  },
+  {
+    "name": "ResolutionHeight",
+    "dataType": "LONG"
+  },
+  {
+    "name": "WindowClientWidth",
+    "dataType": "LONG"
+  },
+  {
+    "name": "WindowClientHeight",
+    "dataType": "LONG"
+  },
+  {
+    "name": "URLHash",
+    "dataType": "LONG"
+  },
+  {
+    "name": "JavaEnable",
+    "dataType": "LONG"
+  },
+  {
+    "name": "GoodEvent",
+    "dataType": "LONG"
+  },
+  {
+    "name": "CounterClass",
+    "dataType": "LONG"
+  },
+  {
+    "name": "OS",
+    "dataType": "LONG"
+  },
+  {
+    "name": "UserAgent",
+    "dataType": "LONG"
+  },
+  {
+    "name": "RefererCategoryID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "RefererRegionID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "URLCategoryID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "URLRegionID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "ResolutionDepth",
+    "dataType": "LONG"
+  },
+  {
+    "name": "FlashMajor",
+    "dataType": "LONG"
+  },
+  {
+    "name": "FlashMinor",
+    "dataType": "LONG"
+  },
+  {
+    "name": "NetMajor",
+    "dataType": "LONG"
+  },
+  {
+    "name": "NetMinor",
+    "dataType": "LONG"
+  },
+  {
+    "name": "UserAgentMajor",
+    "dataType": "LONG"
+  },
+  {
+    "name": "CookieEnable",
+    "dataType": "LONG"
+  },
+  {
+    "name": "JavascriptEnable",
+    "dataType": "LONG"
+  },
+  {
+    "name": "IsMobile",
+    "dataType": "LONG"
+  },
+  {
+    "name": "MobilePhone",
+    "dataType": "LONG"
+  },
+  {
+    "name": "IPNetworkID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "TraficSourceID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "SearchEngineID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "IsArtifical",
+    "dataType": "LONG"
+  },
+  {
+    "name": "ClientTimeZone",
+    "dataType": "LONG"
+  },
+  {
+    "name": "SilverlightVersion1",
+    "dataType": "LONG"
+  },
+  {
+    "name": "SilverlightVersion2",
+    "dataType": "LONG"
+  },
+  {
+    "name": "SilverlightVersion3",
+    "dataType": "LONG"
+  },
+  {
+    "name": "SilverlightVersion4",
+    "dataType": "LONG"
+  },
+  {
+    "name": "CodeVersion",
+    "dataType": "LONG"
+  },
+  {
+    "name": "IsLink",
+    "dataType": "LONG"
+  },
+  {
+    "name": "IsDownload",
+    "dataType": "LONG"
+  },
+  {
+    "name": "IsNotBounce",
+    "dataType": "LONG"
+  },
+  {
+    "name": "FUniqID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "HID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "IsOldCounter",
+    "dataType": "LONG"
+  },
+  {
+    "name": "IsEvent",
+    "dataType": "LONG"
+  },
+  {
+    "name": "IsParameter",
+    "dataType": "LONG"
+  },
+  {
+    "name": "DontCountHits",
+    "dataType": "LONG"
+  },
+  {
+    "name": "WithHash",
+    "dataType": "LONG"
+  },
+  {
+    "name": "Age",
+    "dataType": "LONG"
+  },
+  {
+    "name": "Sex",
+    "dataType": "LONG"
+  },
+  {
+    "name": "Income",
+    "dataType": "LONG"
+  },
+  {
+    "name": "Interests",
+    "dataType": "LONG"
+  },
+  {
+    "name": "Robotness",
+    "dataType": "LONG"
+  },
+  {
+    "name": "RemoteIP",
+    "dataType": "LONG"
+  },
+  {
+    "name": "WindowName",
+    "dataType": "LONG"
+  },
+  {
+    "name": "OpenerName",
+    "dataType": "LONG"
+  },
+  {
+    "name": "HistoryLength",
+    "dataType": "LONG"
+  },
+  {
+    "name": "HTTPError",
+    "dataType": "LONG"
+  },
+  {
+    "name": "SendTiming",
+    "dataType": "LONG"
+  },
+  {
+    "name": "DNSTiming",
+    "dataType": "LONG"
+  },
+  {
+    "name": "ConnectTiming",
+    "dataType": "LONG"
+  },
+  {
+    "name": "ResponseStartTiming",
+    "dataType": "LONG"
+  },
+  {
+    "name": "ResponseEndTiming",
+    "dataType": "LONG"
+  },
+  {
+    "name": "FetchTiming",
+    "dataType": "LONG"
+  },
+  {
+    "name": "SocialSourceNetworkID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "ParamPrice",
+    "dataType": "LONG"
+  },
+  {
+    "name": "ParamCurrencyID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "HasGCLID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "RefererHash",
+    "dataType": "LONG"
+  },
+  {
+    "name": "CLID",
+    "dataType": "LONG"
+  }
+
+  ],
+  "dimensionFieldSpecs": [
+  {
+    "name": "CounterID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "UserID",
+    "dataType": "LONG"
+  },
+  {
+    "name": "URL",
+    "dataType": "STRING"
+  },
+  {
+    "name": "OpenstatServiceName",
+    "dataType": "STRING"
+  },
+  {
+    "name": "OpenstatCampaignID",
+    "dataType": "STRING"
+  },
+  {
+    "name": "OpenstatAdID",
+    "dataType": "STRING"
+  },
+  {
+    "name": "UserAgentMinor",
+    "dataType": "STRING"
+  },
+  {
+    "name": "OpenstatSourceID",
+    "dataType": "STRING"
+  },
+  {
+    "name": "MobilePhoneModel",
+    "dataType": "STRING"
+  },
+  {
+    "name": "Params",
+    "dataType": "STRING"
+  },
+  {
+    "name": "UTMSource",
+    "dataType": "STRING"
+  },
+  {
+    "name": "PageCharset",
+    "dataType": "STRING"
+  },
+  {
+    "name": "SearchPhrase",
+    "dataType": "STRING"
+  },
+  {
+    "name": "UTMMedium",
+    "dataType": "STRING"
+  },
+  {
+    "name": "OriginalURL",
+    "dataType": "STRING"
+  },
+  {
+    "name": "UTMCampaign",
+    "dataType": "STRING"
+  },
+  {
+    "name": "ClientEventTime",
+    "dataType": "STRING"
+  },
+  {
+    "name": "ParamOrderID",
+    "dataType": "STRING"
+  },
+  {
+    "name": "ParamCurrency",
+    "dataType": "STRING"
+  },
+  {
+    "name": "UTMContent",
+    "dataType": "STRING"
+  },
+  {
+    "name": "UTMTerm",
+    "dataType": "STRING"
+  },
+  {
+    "name": "FlashMinor2",
+    "dataType": "STRING"
+  },
+  {
+    "name": "FromTag",
+    "dataType": "STRING"
+  },
+
+  {
+    "name": "Referer",
+    "dataType": "STRING"
+  },
+  {
+    "name": "Title",
+    "dataType": "STRING"
+  },
+  {
+    "name": "HitColor",
+    "dataType": "STRING"
+  },
+  {
+    "name": "LocalEventTime",
+    "dataType": "STRING"
+  },
+  {
+    "name": "BrowserLanguage",
+    "dataType": "STRING"
+  },
+  {
+    "name": "SocialSourcePage",
+    "dataType": "STRING"
+  },
+  {
+    "name": "BrowserCountry",
+    "dataType": "STRING"
+  },
+  {
+    "name": "SocialNetwork",
+    "dataType": "STRING"
+  },
+  {
+    "name": "SocialAction",
+    "dataType": "STRING"
+  },
+
+ {
+    "name": "WatchID",
+    "dataType": "LONG"
+  }
+
+  ],
+  "dateTimeFieldSpecs": [
+  {
+    "name": "EventTime",
+    "dataType": "STRING",
+    "format": "1:SECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss",
+    "granularity": "1:SECONDS"
+  },
+  {
+    "name": "EventDate",
+    "dataType": "STRING",
+    "format": "1:DAYS:SIMPLE_DATE_FORMAT:yyyy-MM-dd",
+    "granularity": "1:DAYS"
+  }
+  ],
+  "schemaName": "hitsSchema"
+}
--- a/benchmark/pinot/splitted.yaml
+++ b/benchmark/pinot/splitted.yaml
@ -0,0 +1,37 @@
+executionFrameworkSpec:
+  name: 'standalone'
+  segmentGenerationJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
+  segmentTarPushJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
+  segmentUriPushJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+  segmentMetadataPushJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
+jobType: SegmentCreationAndTarPush
+inputDirURI: 'PWD_DIR_PLACEHOLDER'
+includeFileNamePattern: 'glob:PWD_DIR_PLACEHOLDER/parts??.tsv'
+outputDirURI: 'batch/hits/segments'
+overwriteOutput: true
+segmentCreationJobParallelism: 16
+pinotFSSpecs:
+  - scheme: file
+    className: org.apache.pinot.spi.filesystem.LocalPinotFS
+recordReaderSpec:
+  dataFormat: 'csv'
+  className: 'org.apache.pinot.plugin.inputformat.csv.CSVRecordReader'
+  configClassName: 'org.apache.pinot.plugin.inputformat.csv.CSVRecordReaderConfig'
+  configs:
+    fileFormat: 'default'
+    delimiter: '	'
+    multiValueDelimiter: ''
+    header: 'WatchID	JavaEnable	Title	GoodEvent	EventTime	EventDate	CounterID	ClientIP	RegionID	UserID	CounterClass	OS	UserAgent	URL	Referer	IsRefresh	RefererCategoryID	RefererRegionID	URLCategoryID	URLRegionID	ResolutionWidth	ResolutionHeight	ResolutionDepth	FlashMajor	FlashMinor	FlashMinor2	NetMajor	NetMinor	UserAgentMajor	UserAgentMinor	CookieEnable	JavascriptEnable	IsMobile	MobilePhone	MobilePhoneModel	Params	IPNetworkID	TraficSourceID	SearchEngineID	SearchPhrase	AdvEngineID	IsArtifical	WindowClientWidth	WindowClientHeight	ClientTimeZone	ClientEventTime	SilverlightVersion1	SilverlightVersion2	SilverlightVersion3	SilverlightVersion4	PageCharset	CodeVersion	IsLink	IsDownload	IsNotBounce	FUniqID	OriginalURL	HID	IsOldCounter	IsEvent	IsParameter	DontCountHits	WithHash	HitColor	LocalEventTime	Age	Sex	Income	Interests	Robotness	RemoteIP	WindowName	OpenerName	HistoryLength	BrowserLanguage	BrowserCountry	SocialNetwork	SocialAction	HTTPError	SendTiming	DNSTiming	ConnectTiming	ResponseStartTiming	ResponseEndTiming	FetchTiming	SocialSourceNetworkID	SocialSourcePage	ParamPrice	ParamOrderID	ParamCurrency	ParamCurrencyID	OpenstatServiceName	OpenstatCampaignID	OpenstatAdID	OpenstatSourceID	UTMSource	UTMMedium	UTMCampaign	UTMContent	UTMTerm	FromTag	HasGCLID	RefererHash	URLHash'
+
+
+tableSpec:
+  tableName: 'hits'
+  schemaURI: 'http://localhost:9000/tables/hits/schema'
+  tableConfigURI: 'http://localhost:9000/tables/hits'
+
+pinotClusterSpecs:
+  - controllerURI: 'http://localhost:9000'
+
+pushJobSpec:
+  pushAttempts: 10
+  pushRetryIntervalMillis: 1000