thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update logic and tests

Browse Source
This commit is contained in:
Alexey Milovidov 2021-02-27 22:33:55 +03:00
parent fd1cf49e92
commit 2ac673b12a
4 changed files with 22 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/Functions/extractTextFromHTML.cpp
View File

@ -61,7 +61,7 @@ bool processCDATA(const char * __restrict & src, const char * end, char * __rest
if (!checkAndSkip(src, end, "<![CDATA["))
return false;
if (pending_whitespace && src < end)
if (dst && pending_whitespace && src < end)
{
pending_whitespace = false;
*dst = ' ';
@ -76,10 +76,13 @@ bool processCDATA(const char * __restrict & src, const char * end, char * __rest
break;
if (gt[-1] == ']' && gt[-2] == ']')
{
if (dst)
{
size_t bytes_to_copy = gt - src - strlen("]]");
memcpy(dst, src, bytes_to_copy);
dst += bytes_to_copy;
}
src = gt + 1;
break;
}
@ -127,6 +130,17 @@ bool processElementAndSkipContent(const char * __restrict & src, const char * en
break;
++src;
/// Skip comments and CDATA
if (*src == '!')
{
--src;
bool pending_whitespace = false;
char * dst = nullptr;
processComment(src, end) || processCDATA(src, end, dst, pending_whitespace);
continue;
}
if (*src != '/')
continue;
++src;

57
tests/queries/0_stateless/01746_extract_text_from_html.reference
View File

@ -1,106 +1,53 @@
-- { echo }
SELECT extractTextFromHTML('');
SELECT extractTextFromHTML(' ');
SELECT extractTextFromHTML(' ');
SELECT extractTextFromHTML('Hello');
Hello
SELECT extractTextFromHTML('Hello, world');
Hello, world
SELECT extractTextFromHTML('Hello, world');
Hello, world
SELECT extractTextFromHTML(' Hello, world');
Hello, world
SELECT extractTextFromHTML(' Hello, world ');
Hello, world
SELECT extractTextFromHTML(' \t Hello,\rworld \n ');
Hello, world
SELECT extractTextFromHTML('Hello<world');
Hello
SELECT extractTextFromHTML('Hello < world');
Hello
SELECT extractTextFromHTML('Hello > world');
Hello > world
SELECT extractTextFromHTML('Hello<world>');
Hello
SELECT extractTextFromHTML('Hello<>world');
Helloworld
SELECT extractTextFromHTML('Hello<!>world');
Helloworld
SELECT extractTextFromHTML('Hello<!->world');
Helloworld
SELECT extractTextFromHTML('Hello<!-->world');
Helloworld
SELECT extractTextFromHTML('Hello<!--->world');
Helloworld
SELECT extractTextFromHTML('Hello<!---->world');
Helloworld
SELECT extractTextFromHTML('Hello <!-- --> World');
Hello World
SELECT extractTextFromHTML('Hello<!-- --> World');
Hello World
SELECT extractTextFromHTML('Hello<!-- -->World');
HelloWorld
SELECT extractTextFromHTML('Hello <!-- -->World');
Hello World
SELECT extractTextFromHTML('Hello <u> World</u>');
Hello World
SELECT extractTextFromHTML('Hello <u>World</u>');
Hello World
SELECT extractTextFromHTML('Hello<u>World</u>');
HelloWorld
SELECT extractTextFromHTML('Hello<u> World</u>');
Hello World
SELECT extractTextFromHTML('<![CDATA[ \t Hello,\rworld \n ]]>');
\t Hello,\rworld \n
SELECT extractTextFromHTML('Hello <![CDATA[Hello\tworld]]> world!');
Hello Hello\tworld world!
SELECT extractTextFromHTML('Hello<![CDATA[Hello\tworld]]>world!');
HelloHello\tworldworld!
SELECT extractTextFromHTML('Hello <![CDATA[Hello <b>world</b>]]> world!');
Hello Hello <b>world</b> world!
SELECT extractTextFromHTML('<![CDATA[<sender>John Smith</sender>]]>');
<sender>John Smith</sender>
SELECT extractTextFromHTML('<![CDATA[<sender>John <![CDATA[Smith</sender>]]>');
<sender>John <![CDATA[Smith</sender>
SELECT extractTextFromHTML('<![CDATA[<sender>John <![CDATA[]]>Smith</sender>]]>');
<sender>John <![CDATA[Smith]]>
SELECT extractTextFromHTML('<![CDATA[<sender>John ]]><![CDATA[Smith</sender>]]>');
<sender>John Smith</sender>
SELECT extractTextFromHTML('<![CDATA[<sender>John ]]> <![CDATA[Smith</sender>]]>');
<sender>John Smith</sender>
SELECT extractTextFromHTML('<![CDATA[<sender>John]]> <![CDATA[Smith</sender>]]>');
<sender>John Smith</sender>
SELECT extractTextFromHTML('<![CDATA[<sender>John ]]>]]><![CDATA[Smith</sender>]]>');
<sender>John ]]>Smith</sender>
SELECT extractTextFromHTML('Hello<script>World</script> goodbye');
Hello goodbye
SELECT extractTextFromHTML('Hello<script >World</script> goodbye');
Hello goodbye
SELECT extractTextFromHTML('Hello<scripta>World</scripta> goodbye');
HelloWorld goodbye
SELECT extractTextFromHTML('Hello<script type="text/javascript">World</script> goodbye');
Hello goodbye
SELECT extractTextFromHTML('Hello<style type="text/css">World</style> goodbye');
Hello goodbye
SELECT extractTextFromHTML('Hello<script:p>World</script:p> goodbye');
HelloWorld goodbye
SELECT extractTextFromHTML('Hello<script:p type="text/javascript">World</script:p> goodbye');
HelloWorld goodbye
SELECT extractTextFromHTML('Hello<style type="text/css">World <!-- abc --> </style> goodbye');
Hello goodbye
SELECT extractTextFromHTML('Hello<style type="text/css">World <!-- abc --> </style \n > goodbye');
Hello goodbye
SELECT extractTextFromHTML('Hello<style type="text/css">World <!-- abc --> </ style> goodbye');
Hello
SELECT extractTextFromHTML('Hello<style type="text/css">World <!-- abc --> </stylea> goodbye');
Hello
SELECT extractTextFromHTML('Hello<style type="text/css">World <![CDATA[</style>]]> </stylea> goodbye');
Hello]]> goodbye
SELECT extractTextFromHTML('Hello<style type="text/css">World <script>abc</script> </stylea> goodbye');
Hello
SELECT extractTextFromHTML('Hello<style type="text/css">World <script>abc</script> </style> goodbye');
Hello goodbye
Hello
Hello goodbye

1
tests/queries/0_stateless/01746_extract_text_from_html.sql
View File

@ -57,5 +57,6 @@ SELECT extractTextFromHTML('Hello<style type="text/css">World <!-- abc --> </ st
SELECT extractTextFromHTML('Hello<style type="text/css">World <!-- abc --> </stylea> goodbye');
SELECT extractTextFromHTML('Hello<style type="text/css">World <![CDATA[</style>]]> </stylea> goodbye');
SELECT extractTextFromHTML('Hello<style type="text/css">World <![CDATA[</style>]]> </style> goodbye');
SELECT extractTextFromHTML('Hello<style type="text/css">World <script>abc</script> </stylea> goodbye');
SELECT extractTextFromHTML('Hello<style type="text/css">World <script>abc</script> </style> goodbye');