thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 08:32:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Extended date parsing (#1495)

Browse Source 
* Allowed to parse Date in YYYY-mM-dD format [#CLICKHOUSE-3].

* Allowed to parse Date in YYYY-mM-dD format [#CLICKHOUSE-3].
This commit is contained in:
alexey-milovidov 2017-11-15 05:08:55 +03:00 committed by GitHub
parent dfdc3ba680
commit 19e0cd879d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 145 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
dbms/src/IO/ReadHelpers.cpp
View File

@ -633,6 +633,36 @@ template bool readJSONStringInto<PaddedPODArray<UInt8>, bool>(PaddedPODArray<UIn
template void readJSONStringInto<NullSink>(NullSink & s, ReadBuffer & buf);
void readDateTextFallback(LocalDate & date, ReadBuffer & buf)
{
char chars_year[4];
readPODBinary(chars_year, buf);
UInt16 year = (chars_year[0] - '0') * 1000 + (chars_year[1] - '0') * 100 + (chars_year[2] - '0') * 10 + (chars_year[3] - '0');
buf.ignore();
char chars_month[2];
readPODBinary(chars_month, buf);
UInt8 month = chars_month[0] - '0';
if (isNumericASCII(chars_month[1]))
{
month = month * 10 + chars_month[1] - '0';
buf.ignore();
}
char char_day;
readChar(char_day, buf);
UInt8 day = char_day - '0';
if (!buf.eof() && isNumericASCII(*buf.position()))
{
day = day * 10 + *buf.position() - '0';
++buf.position();
}
date = LocalDate(year, month, day);
}
void readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut)
{
static constexpr auto DATE_TIME_BROKEN_DOWN_LENGTH = 19;

67
dbms/src/IO/ReadHelpers.h
View File

@ -572,39 +572,52 @@ void parseUUID(const UInt8 * src36, std::reverse_iterator<UInt8 *> dst16);
template <typename IteratorSrc, typename IteratorDst>
void formatHex(IteratorSrc src, IteratorDst dst, const size_t num_bytes);
/// In YYYY-MM-DD format
inline void readDateText(DayNum_t & date, ReadBuffer & buf)
{
char s[10];
size_t size = buf.read(s, 10);
if (10 != size)
{
s[size] = 0;
throw Exception(std::string("Cannot parse date ") + s, ErrorCodes::CANNOT_PARSE_DATE);
}
UInt16 year = (s[0] - '0') * 1000 + (s[1] - '0') * 100 + (s[2] - '0') * 10 + (s[3] - '0');
UInt8 month = (s[5] - '0') * 10 + (s[6] - '0');
UInt8 day = (s[8] - '0') * 10 + (s[9] - '0');
date = DateLUT::instance().makeDayNum(year, month, day);
}
void readDateTextFallback(LocalDate & date, ReadBuffer & buf);
/// In YYYY-MM-DD format.
/// For convenience, Month and Day parts can have single digit instead of two digits.
/// Any separators other than '-' are supported.
inline void readDateText(LocalDate & date, ReadBuffer & buf)
{
char s[10];
size_t size = buf.read(s, 10);
if (10 != size)
/// Optimistic path, when whole value is in buffer.
if (buf.position() + 10 <= buf.buffer().end())
{
s[size] = 0;
throw Exception(std::string("Cannot parse date ") + s, ErrorCodes::CANNOT_PARSE_DATE);
}
UInt16 year = (buf.position()[0] - '0') * 1000 + (buf.position()[1] - '0') * 100 + (buf.position()[2] - '0') * 10 + (buf.position()[3] - '0');
buf.position() += 5;
date.year((s[0] - '0') * 1000 + (s[1] - '0') * 100 + (s[2] - '0') * 10 + (s[3] - '0'));
date.month((s[5] - '0') * 10 + (s[6] - '0'));
date.day((s[8] - '0') * 10 + (s[9] - '0'));
UInt8 month = buf.position()[0] - '0';
if (isNumericASCII(buf.position()[1]))
{
month = month * 10 + buf.position()[1] - '0';
buf.position() += 3;
}
else
buf.position() += 2;
UInt8 day = buf.position()[0] - '0';
if (isNumericASCII(buf.position()[1]))
{
day = day * 10 + buf.position()[1] - '0';
buf.position() += 2;
}
else
buf.position() += 1;
date = LocalDate(year, month, day);
}
else
readDateTextFallback(date, buf);
}
inline void readDateText(DayNum_t & date, ReadBuffer & buf)
{
LocalDate local_date;
readDateText(local_date, buf);
date = DateLUT::instance().makeDayNum(local_date.year(), local_date.month(), local_date.day());
}
inline void readUUIDText(UUID & uuid, ReadBuffer & buf)
{
char s[36];
@ -637,9 +650,9 @@ inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTI
* If it is not a number - then parse datetime in YYYY-MM-DD hh:mm:ss format.
*/
/// Optimistic path, when whole value are in buffer.
/// Optimistic path, when whole value is in buffer.
const char * s = buf.position();
if (s + 19 < buf.buffer().end())
if (s + 19 <= buf.buffer().end())
{
if (s[4] < '0' || s[4] > '9')
{

25
dbms/tests/performance/date_parsing/date_parsing.xml Normal file
View File

@ -0,0 +1,25 @@
<test>
<name>date_parsing</name>
<type>loop</type>
<stop_conditions>
<all_of>
<iterations>5</iterations>
<min_time_not_changing_for_ms>10000</min_time_not_changing_for_ms>
</all_of>
<any_of>
<iterations>50</iterations>
<total_time_ms>60000</total_time_ms>
</any_of>
</stop_conditions>
<main_metric>
<min_time/>
</main_metric>
<preconditions>
<table_exists>test.hits</table_exists>
</preconditions>
<query>SELECT count() FROM test.hits WHERE NOT ignore(toDate(toString(EventDate)))</query>
</test>

36
dbms/tests/queries/0_stateless/00517_date_parsing.reference Normal file
View File

@ -0,0 +1,36 @@
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-11-12
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-11-12
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-01-02
2017-11-12
2017-11-12

14
dbms/tests/queries/0_stateless/00517_date_parsing.sql Normal file
View File

@ -0,0 +1,14 @@
SELECT toDate(s) FROM (SELECT arrayJoin(['2017-01-02', '2017-1-02', '2017-01-2', '2017-1-2', '2017/01/02', '2017/1/02', '2017/01/2', '2017/1/2', '2017-11-12']) AS s);
DROP TABLE IF EXISTS test.date;
CREATE TABLE test.date (d Date) ENGINE = Memory;
INSERT INTO test.date VALUES ('2017-01-02'), ('2017-1-02'), ('2017-01-2'), ('2017-1-2'), ('2017/01/02'), ('2017/1/02'), ('2017/01/2'), ('2017/1/2'), ('2017-11-12');
SELECT * FROM test.date;
INSERT INTO test.date FORMAT JSONEachRow {"d": "2017-01-02"}, {"d": "2017-1-02"}, {"d": "2017-01-2"}, {"d": "2017-1-2"}, {"d": "2017/01/02"}, {"d": "2017/1/02"}, {"d": "2017/01/2"}, {"d": "2017/1/2"}, {"d": "2017-11-12"};
SELECT * FROM test.date ORDER BY d;
DROP TABLE test.date;
WITH toDate('2000-01-01') + rand() % (30000) AS EventDate SELECT * FROM numbers(1000000) WHERE EventDate != toDate(concat(toString(toYear(EventDate)), '-', toString(toMonth(EventDate)), '-', toString(toDayOfMonth(EventDate))));