/* * MIT License * * Copyright © 2019 Yibo Cai * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * * Source: https://github.com/cyb70289/utf8 */ #include #include #include #include "unicode.h" #pragma GCC diagnostic ignored "-Woverflow" static const int8_t _first_len_tbl[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3, }; static const int8_t _first_range_tbl[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, }; static const int8_t _range_min_tbl[] = { 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80, 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, }; static const int8_t _range_max_tbl[] = { 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F, 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, }; static const int8_t _df_ee_tbl[] = { 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, }; static const int8_t _ef_fe_tbl[] = { 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; /* Return 0 on success, -1 on error */ bool utf8_validate_simd(const unsigned char *s, size_t len) { if (len >= 32) { __m128i prev_input = _mm_set1_epi8(0); __m128i prev_first_len = _mm_set1_epi8(0); const __m128i first_len_tbl = _mm_loadu_si128( (const __m128i *)_first_len_tbl); const __m128i first_range_tbl = _mm_loadu_si128( (const __m128i *)_first_range_tbl); const __m128i range_min_tbl = _mm_loadu_si128( (const __m128i *)_range_min_tbl); const __m128i range_max_tbl = _mm_loadu_si128( (const __m128i *)_range_max_tbl); const __m128i df_ee_tbl = _mm_loadu_si128((const __m128i *)_df_ee_tbl); const __m128i ef_fe_tbl = _mm_loadu_si128((const __m128i *)_ef_fe_tbl); __m128i error = _mm_set1_epi8(0); while (len >= 32) { /***************************** block 1 ****************************/ const __m128i input_a = _mm_loadu_si128((const __m128i *)s); __m128i high_nibbles = _mm_and_si128(_mm_srli_epi16(input_a, 4), _mm_set1_epi8(0x0F)); __m128i first_len_a = _mm_shuffle_epi8(first_len_tbl, high_nibbles); __m128i range_a = _mm_shuffle_epi8(first_range_tbl, high_nibbles); range_a = _mm_or_si128( range_a, _mm_alignr_epi8(first_len_a, prev_first_len, 15)); __m128i tmp; tmp = _mm_alignr_epi8(first_len_a, prev_first_len, 14); tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(1)); range_a = _mm_or_si128(range_a, tmp); tmp = _mm_alignr_epi8(first_len_a, prev_first_len, 13); tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(2)); range_a = _mm_or_si128(range_a, tmp); __m128i shift1, pos, range2; shift1 = _mm_alignr_epi8(input_a, prev_input, 15); pos = _mm_sub_epi8(shift1, _mm_set1_epi8(0xEF)); tmp = _mm_subs_epu8(pos, _mm_set1_epi8(0xF0)); range2 = _mm_shuffle_epi8(df_ee_tbl, tmp); tmp = _mm_adds_epu8(pos, _mm_set1_epi8(0x70)); range2 = _mm_add_epi8(range2, _mm_shuffle_epi8(ef_fe_tbl, tmp)); range_a = _mm_add_epi8(range_a, range2); __m128i minv = _mm_shuffle_epi8(range_min_tbl, range_a); __m128i maxv = _mm_shuffle_epi8(range_max_tbl, range_a); tmp = _mm_or_si128(_mm_cmplt_epi8(input_a, minv), _mm_cmpgt_epi8(input_a, maxv)); error = _mm_or_si128(error, tmp); /***************************** block 2 ****************************/ const __m128i input_b = _mm_loadu_si128((const __m128i *)(s + 16)); high_nibbles = _mm_and_si128(_mm_srli_epi16(input_b, 4), _mm_set1_epi8(0x0F)); __m128i first_len_b = _mm_shuffle_epi8(first_len_tbl, high_nibbles); __m128i range_b = _mm_shuffle_epi8(first_range_tbl, high_nibbles); range_b = _mm_or_si128( range_b, _mm_alignr_epi8(first_len_b, first_len_a, 15)); tmp = _mm_alignr_epi8(first_len_b, first_len_a, 14); tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(1)); range_b = _mm_or_si128(range_b, tmp); tmp = _mm_alignr_epi8(first_len_b, first_len_a, 13); tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(2)); range_b = _mm_or_si128(range_b, tmp); shift1 = _mm_alignr_epi8(input_b, input_a, 15); pos = _mm_sub_epi8(shift1, _mm_set1_epi8(0xEF)); tmp = _mm_subs_epu8(pos, _mm_set1_epi8(0xF0)); range2 = _mm_shuffle_epi8(df_ee_tbl, tmp); tmp = _mm_adds_epu8(pos, _mm_set1_epi8(0x70)); range2 = _mm_add_epi8(range2, _mm_shuffle_epi8(ef_fe_tbl, tmp)); range_b = _mm_add_epi8(range_b, range2); minv = _mm_shuffle_epi8(range_min_tbl, range_b); maxv = _mm_shuffle_epi8(range_max_tbl, range_b); tmp = _mm_or_si128(_mm_cmplt_epi8(input_b, minv), _mm_cmpgt_epi8(input_b, maxv)); error = _mm_or_si128(error, tmp); /************************ next iteration **************************/ prev_input = input_b; prev_first_len = first_len_b; s += 32; len -= 32; } if (!_mm_testz_si128(error, error)) return false; int32_t token4 = _mm_extract_epi32(prev_input, 3); const int8_t *token = (const int8_t *)&token4; int lookahead = 0; if (token[3] > (int8_t)0xBF) lookahead = 1; else if (token[2] > (int8_t)0xBF) lookahead = 2; else if (token[1] > (int8_t)0xBF) lookahead = 3; s -= lookahead; len += lookahead; } return utf8_validate_off(s, len) == 0; }