代码拉取完成，页面将自动刷新
From e95491b3a2261aecdc5576a7e507b4f4ace88cbc Mon Sep 17 00:00:00 2001
From: Yikun Jiang <yikunkero@gmail.com>
Date: Mon, 20 Jul 2020 17:20:15 +0800
Subject: [PATCH] Add aarch64 support
Signed-off-by: Liu Zixian <liuzixian4@huawei.com>
---
CMakeLists.txt | 108 +-
cmake/config.h.in | 9 +
cmake/platform.cmake | 13 +-
cmake/ragel.cmake | 20 +
src/crc32.c | 43 +
src/fdr/fdr.c | 136 ++-
src/hs_valid_platform.c | 9 +-
src/nfa/limex_exceptional.h | 22 +-
src/nfa/limex_internal.h | 2 +-
src/nfa/limex_native.c | 10 +-
src/nfa/shufti.c | 18 +-
src/nfa/truffle.c | 10 +-
src/parser/control_verbs.cpp | 340 +++++++
src/rose/counting_miracle.h | 2 +-
src/util/arch.h | 11 +
src/util/cpuid_flags.c | 6 +
src/util/cpuid_flags.h | 2 +
src/util/cpuid_inline.h | 17 +-
src/util/intrinsics.h | 12 +
src/util/popcount.h | 6 +-
src/util/simd_arm.h | 1069 ++++++++++++++++++++
src/util/simd_types.h | 17 +
src/util/simd_utils.h | 13 +
src/util/simd_x86.h | 10 +
tools/hscollider/CMakeLists.txt | 9 +-
tools/hscollider/ColliderCorporaParser.cpp | 474 +++++++++
unit/internal/simd_utils.cpp | 2 +-
util/CMakeLists.txt | 8 +-
util/ExpressionParser.cpp | 397 ++++++++
29 files changed, 2717 insertions(+), 78 deletions(-)
create mode 100644 src/parser/control_verbs.cpp
create mode 100644 src/util/simd_arm.h
create mode 100644 src/util/simd_utils.h
create mode 100644 tools/hscollider/ColliderCorporaParser.cpp
create mode 100644 util/ExpressionParser.cpp
diff --git a/CMakeLists.txt b/CMakeLists.txt
index bd6d2de..8dbcb72 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -74,6 +74,7 @@ include (${CMAKE_MODULE_PATH}/boost.cmake)
# -- make this work? set(python_ADDITIONAL_VERSIONS 2.7 2.6)
find_package(PythonInterp)
find_program(RAGEL ragel)
+find_program(COPY cp)
if(PYTHONINTERP_FOUND)
set(PYTHON ${PYTHON_EXECUTABLE})
@@ -189,24 +190,30 @@ else()
# cpuid info and then chooses the best microarch it can (and replaces
# the flag), so use that for tune.
- # arg1 might exist if using ccache
- string (STRIP "${CMAKE_C_COMPILER_ARG1}" CC_ARG1)
- set (EXEC_ARGS ${CC_ARG1} -c -Q --help=target -march=native -mtune=native)
- execute_process(COMMAND ${CMAKE_C_COMPILER} ${EXEC_ARGS}
- OUTPUT_VARIABLE _GCC_OUTPUT)
- string(FIND "${_GCC_OUTPUT}" "march" POS)
- string(SUBSTRING "${_GCC_OUTPUT}" ${POS} -1 _GCC_OUTPUT)
- string(REGEX REPLACE "march=[ \t]*([^ \n]*)[ \n].*" "\\1"
- GNUCC_ARCH "${_GCC_OUTPUT}")
-
- # test the parsed flag
- set (EXEC_ARGS ${CC_ARG1} -E - -mtune=${GNUCC_ARCH})
- execute_process(COMMAND ${CMAKE_C_COMPILER} ${EXEC_ARGS}
- OUTPUT_QUIET ERROR_QUIET
- INPUT_FILE /dev/null
- RESULT_VARIABLE GNUCC_TUNE_TEST)
- if (NOT GNUCC_TUNE_TEST EQUAL 0)
- message(SEND_ERROR "Something went wrong determining gcc tune: -mtune=${GNUCC_ARCH} not valid")
+ if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i386")
+ # arg1 might exist if using ccache
+ string (STRIP "${CMAKE_C_COMPILER_ARG1}" CC_ARG1)
+ set (EXEC_ARGS ${CC_ARG1} -c -Q --help=target -march=native -mtune=native)
+ execute_process(COMMAND ${CMAKE_C_COMPILER} ${EXEC_ARGS}
+ OUTPUT_VARIABLE _GCC_OUTPUT)
+ string(FIND "${_GCC_OUTPUT}" "march" POS)
+ string(SUBSTRING "${_GCC_OUTPUT}" ${POS} -1 _GCC_OUTPUT)
+ string(REGEX REPLACE "march=[ \t]*([^ \n]*)[ \n].*" "\\1"
+ GNUCC_ARCH "${_GCC_OUTPUT}")
+
+ # test the parsed flag
+ set (EXEC_ARGS ${CC_ARG1} -E - -mtune=${GNUCC_ARCH})
+ execute_process(COMMAND ${CMAKE_C_COMPILER} ${EXEC_ARGS}
+ OUTPUT_QUIET ERROR_QUIET
+ INPUT_FILE /dev/null
+ RESULT_VARIABLE GNUCC_TUNE_TEST)
+ if (NOT GNUCC_TUNE_TEST EQUAL 0)
+ message(SEND_ERROR "Something went wrong determining gcc tune: -mtune=${GNUCC_ARCH} not valid")
+ endif()
+ endif()
+
+ if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+ set (EXEC_ARGS ${CC_ARG1} -c -Q --help=target -march=armv8-a -mtune=armv8-a)
endif()
set(TUNE_FLAG ${GNUCC_ARCH})
else ()
@@ -239,6 +246,13 @@ else()
set(EXTRA_C_FLAGS "${OPT_C_FLAG} -std=c99 -Wall -Wextra -Wshadow -Wcast-qual -fno-strict-aliasing")
set(EXTRA_CXX_FLAGS "${OPT_CXX_FLAG} -std=c++11 -Wall -Wextra -Wshadow -Wswitch -Wreturn-type -Wcast-qual -Wno-deprecated -Wnon-virtual-dtor -fno-strict-aliasing")
+ if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+ set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} -fsigned-char")
+ set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -fsigned-char")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc")
+ endif()
+
if (NOT RELEASE_BUILD)
# -Werror is most useful during development, don't potentially break
# release builds
@@ -252,11 +266,19 @@ else()
endif()
if (NOT CMAKE_C_FLAGS MATCHES .*march.* AND NOT CMAKE_C_FLAGS MATCHES .*mtune.*)
- set(ARCH_C_FLAGS "-march=native -mtune=${TUNE_FLAG}")
+ if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i386")
+ set(ARCH_C_FLAGS "-march=native -mtune=${TUNE_FLAG}")
+ elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+ set(ARCH_C_FLAGS "-march=armv8-a -mtune=${TUNE_FLAG}")
+ endif ()
endif()
if (NOT CMAKE_CXX_FLAGS MATCHES .*march.* AND NOT CMAKE_CXX_FLAGS MATCHES .*mtune.*)
- set(ARCH_CXX_FLAGS "-march=native -mtune=${TUNE_FLAG}")
+ if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i386")
+ set(ARCH_CXX_FLAGS "-march=native -mtune=${TUNE_FLAG}")
+ elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+ set(ARCH_CXX_FLAGS "-march=armv8-a -mtune=${TUNE_FLAG}")
+ endif()
endif()
if(CMAKE_COMPILER_IS_GNUCC)
@@ -289,10 +311,18 @@ else()
endif()
CHECK_INCLUDE_FILES(unistd.h HAVE_UNISTD_H)
-CHECK_INCLUDE_FILES(intrin.h HAVE_C_INTRIN_H)
-CHECK_INCLUDE_FILE_CXX(intrin.h HAVE_CXX_INTRIN_H)
-CHECK_INCLUDE_FILES(x86intrin.h HAVE_C_X86INTRIN_H)
-CHECK_INCLUDE_FILE_CXX(x86intrin.h HAVE_CXX_X86INTRIN_H)
+
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i386")
+ CHECK_INCLUDE_FILES(intrin.h HAVE_C_INTRIN_H)
+ CHECK_INCLUDE_FILE_CXX(intrin.h HAVE_CXX_INTRIN_H)
+ CHECK_INCLUDE_FILES(x86intrin.h HAVE_C_X86INTRIN_H)
+ CHECK_INCLUDE_FILE_CXX(x86intrin.h HAVE_CXX_X86INTRIN_H)
+endif()
+
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+ CHECK_INCLUDE_FILES(arm_neon.h HAVE_C_ARM_NEON_H)
+ CHECK_INCLUDE_FILE_CXX(arm_neon.h HAVE_CXX_ARM_NEON_H)
+endif()
CHECK_FUNCTION_EXISTS(posix_memalign HAVE_POSIX_MEMALIGN)
CHECK_FUNCTION_EXISTS(_aligned_malloc HAVE__ALIGNED_MALLOC)
@@ -325,6 +355,9 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux")
(CMAKE_VERSION VERSION_GREATER "3.0" AND CMAKE_GENERATOR MATCHES "Ninja")))
message (STATUS "Building the fat runtime requires the Unix Makefiles generator, or Ninja with CMake v3.0 or higher")
set (FAT_RUNTIME_REQUISITES FALSE)
+ elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+ message(STATUS "AARCH64 platform don't support fat runtime")
+ set (FAT_RUNTIME_REQUISITES FALSE)
else()
include (${CMAKE_MODULE_PATH}/attrib.cmake)
if (NOT HAS_C_ATTR_IFUNC)
@@ -337,7 +370,9 @@ if (CMAKE_SYSTEM_NAME MATCHES "Linux")
CMAKE_DEPENDENT_OPTION(FAT_RUNTIME "Build a library that supports multiple microarchitectures" ${RELEASE_BUILD} "FAT_RUNTIME_REQUISITES" OFF)
endif ()
-include (${CMAKE_MODULE_PATH}/arch.cmake)
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i386")
+ include (${CMAKE_MODULE_PATH}/arch.cmake)
+endif()
# testing a builtin takes a little more work
CHECK_C_SOURCE_COMPILES("void *aa_test(void *x) { return __builtin_assume_aligned(x, 16);}\nint main(void) { return 0; }" HAVE_CC_BUILTIN_ASSUME_ALIGNED)
@@ -415,12 +450,6 @@ if (CXX_IGNORED_ATTR)
set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -Wno-ignored-attributes")
endif()
-# gcc 9 complains about redundant move for returned variable
-CHECK_CXX_COMPILER_FLAG("-Wredundant-move" CXX_REDUNDANT_MOVE)
-if (CXX_REDUNDANT_MOVE)
- set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -Wno-redundant-move")
-endif()
-
# note this for later
# g++ doesn't have this flag but clang does
CHECK_CXX_COMPILER_FLAG("-Wweak-vtables" CXX_WEAK_VTABLES)
@@ -477,6 +506,14 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
endif()
+# Test case for neon function.
+option(UNIT_SIMD "Simd funtion test case, default is OFF" OFF)
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+ if (UNIT_SIMD)
+ add_subdirectory(unit-simd)
+ endif()
+endif()
+
add_subdirectory(util)
add_subdirectory(doc/dev-reference)
@@ -573,7 +610,14 @@ set_source_files_properties(
PROPERTIES
COMPILE_FLAGS "${RAGEL_C_FLAGS}")
-ragelmaker(src/parser/control_verbs.rl)
+
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i386")
+ ragelmaker(src/parser/control_verbs.rl)
+endif()
+
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+ ragelcopyer(src/parser/control_verbs.rl)
+endif()
SET(hs_HEADERS
src/hs.h
diff --git a/cmake/config.h.in b/cmake/config.h.in
index 5454643..336cf19 100644
--- a/cmake/config.h.in
+++ b/cmake/config.h.in
@@ -15,6 +15,9 @@
/* "Define if building for EM64T" */
#cmakedefine ARCH_X86_64
+/* "Define if building for aarch64" */
+#cmakedefine ARCH_AARCH64
+
/* internal build, switch on dump support. */
#cmakedefine DUMP_SUPPORT
@@ -48,6 +51,12 @@
/* C compiler has intrin.h */
#cmakedefine HAVE_C_INTRIN_H
+/* C++ compiler has arm_neon.h */
+#cmakedefine HAVE_CXX_ARM_NEON_H
+
+/* C compiler has arm_neon.h */
+#cmakedefine HAVE_C_ARM_NEON_H
+
/* Define to 1 if you have the declaration of `pthread_setaffinity_np', and to
0 if you don't. */
#cmakedefine HAVE_DECL_PTHREAD_SETAFFINITY_NP
diff --git a/cmake/platform.cmake b/cmake/platform.cmake
index 593c544..213dcc5 100644
--- a/cmake/platform.cmake
+++ b/cmake/platform.cmake
@@ -1,9 +1,14 @@
# determine the target arch
# really only interested in the preprocessor here
-CHECK_C_SOURCE_COMPILES("#if !(defined(__x86_64__) || defined(_M_X64))\n#error not 64bit\n#endif\nint main(void) { return 0; }" ARCH_64_BIT)
+CHECK_C_SOURCE_COMPILES("#if !(defined(__x86_64__) || defined(_M_X64))\n#error not 64bit\n#endif\nint main(void) { return 0; }" ARCH_X86_64)
-CHECK_C_SOURCE_COMPILES("#if !(defined(__i386__) || defined(_M_IX86))\n#error not 64bit\n#endif\nint main(void) { return 0; }" ARCH_32_BIT)
+CHECK_C_SOURCE_COMPILES("#if !(defined(__i386__) || defined(_M_IX86))\n#error not 32bit\n#endif\nint main(void) { return 0; }" ARCH_IA32)
-set(ARCH_X86_64 ${ARCH_64_BIT})
-set(ARCH_IA32 ${ARCH_32_BIT})
+CHECK_C_SOURCE_COMPILES("#if !(defined(__aarch64__))\n#error not 64bit\n#endif\nint main(void) { return 0; }" ARCH_AARCH64)
+
+if (ARCH_X86_64 OR ARCH_AARCH64)
+ set(ARCH_64_BIT 1)
+elseif (ARCH_IA32)
+ set(ARCH_32_BIT 1)
+endif()
\ No newline at end of file
diff --git a/cmake/ragel.cmake b/cmake/ragel.cmake
index d3f0b92..3356cb9 100644
--- a/cmake/ragel.cmake
+++ b/cmake/ragel.cmake
@@ -14,3 +14,23 @@ function(ragelmaker src_rl)
set_source_files_properties(${rl_out} PROPERTIES GENERATED TRUE)
endfunction(ragelmaker)
+ # On the aarch64 platform, char is unsigned by default, so in order to be consistent with
+ # the x86 platform, we will add -fsigned-char to the compile option to force the char type.
+ # However, when the ragel generates c++ code, the char variable used will still be considered
+ # unsigned, resulting in the overflow of the char variable value in the generated code,
+ # resulting in some errors.
+ # function for copying the previously modified code to the specified path
+
+ function(ragelcopyer src_rl)
+ get_filename_component(src_dir ${src_rl} PATH) # old cmake needs PATH
+ get_filename_component(src_file ${src_rl} NAME_WE)
+ set(rl_out ${CMAKE_CURRENT_BINARY_DIR}/${src_dir}/${src_file}.cpp)
+ add_custom_command(
+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${src_dir}/${src_file}.cpp
+ COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${src_dir}
+ COMMAND ${COPY} -f ${CMAKE_CURRENT_SOURCE_DIR}/${src_dir}/${src_file}.cpp ${rl_out} 2>/dev/null ||:
+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${src_dir}/${src_file}.cpp
+ )
+ add_custom_target(ragel_${src_file} DEPENDS ${rl_out})
+ set_source_files_properties(${rl_out} PROPERTIES GENERATED TRUE)
+ endfunction(ragelcopyer)
\ No newline at end of file
diff --git a/src/crc32.c b/src/crc32.c
index 1dae47b..4609c5d 100644
--- a/src/crc32.c
+++ b/src/crc32.c
@@ -32,6 +32,47 @@
#include "util/arch.h"
#include "util/intrinsics.h"
+#if defined(HAVE_NEON)
+
+#define CRC32CX(crc, value) __asm__("crc32cx %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
+#define CRC32CW(crc, value) __asm__("crc32cw %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
+#define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
+#define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
+#define CRC_WORD 8
+#define CRC_TYPE u64a
+static really_inline
+u32 crc32c_neon(u32 running_crc, const unsigned char * p_buf, const size_t length)
+{
+ u32 crc=running_crc;
+
+ //Processbyte-by-byteuntilp_bufisaligned
+ const unsigned char * aligned_buf = ROUNDUP_PTR(p_buf, CRC_WORD);
+ size_t init_bytes = aligned_buf - p_buf;
+ size_t running_length = ((length - init_bytes) / CRC_WORD) * CRC_WORD;
+ size_t end_bytes = length - init_bytes - running_length;
+
+ while(p_buf < aligned_buf){
+ CRC32CB(crc, *p_buf);
+ p_buf++;
+ }
+
+ //Main aligned loop, processes a word at a time.
+ for(size_t li = 0; li < running_length / CRC_WORD; li++){
+ CRC_TYPE block = *(const CRC_TYPE *)p_buf;
+ CRC32CX(crc,block);
+ p_buf += CRC_WORD;
+ }
+
+ //Remainingbytes
+ for(size_t li = 0; li < end_bytes; li++){
+ CRC32CB(crc,*p_buf);
+ p_buf++;
+ }
+ return crc;
+}
+#endif
+
+
#if !defined(HAVE_SSE42)
/***
@@ -636,6 +677,8 @@ u32 crc32c_sse42(u32 running_crc, const unsigned char* p_buf,
u32 Crc32c_ComputeBuf(u32 inCrc32, const void *buf, size_t bufLen) {
#if defined(HAVE_SSE42)
u32 crc = crc32c_sse42(inCrc32, (const unsigned char *)buf, bufLen);
+#elif defined(HAVE_NEON)
+ u32 crc = crc32c_neon(inCrc32, (const unsigned char *)buf, bufLen);
#else
u32 crc = crc32c_sb8_64_bit(inCrc32, (const unsigned char *)buf, bufLen);
#endif
diff --git a/src/fdr/fdr.c b/src/fdr/fdr.c
index d33756d..718f169 100644
--- a/src/fdr/fdr.c
+++ b/src/fdr/fdr.c
@@ -127,6 +127,13 @@ u64a andn(const u32 a, const u8 *b) {
u64a r;
#if defined(HAVE_BMI) && !defined(NO_ASM)
__asm__ ("andn\t%2,%1,%k0" : "=r"(r) : "r"(a), "m"(*(const u32 *)b));
+#elif defined(HAVE_NEON)
+ __asm__ __volatile__("ldr w0, %w2 \n\t"
+ "bic %w0,w0,%w1 \n\t"
+ : "=r"(r)
+ : "r"(a), "m"(*(const u32 *)b)
+ : "w0"
+ );
#else
r = unaligned_load_u32(b) & ~a;
#endif
@@ -159,7 +166,104 @@ void get_conf_stride_1(const u8 *itPtr, UNUSED const u8 *start_ptr,
UNUSED const u8 *end_ptr, u32 domain_mask_flipped,
const u64a *ft, u64a *conf0, u64a *conf8, m128 *s) {
/* +1: the zones ensure that we can read the byte at z->end */
- assert(itPtr >= start_ptr && itPtr + ITER_BYTES <= end_ptr);
+ assert(itPtr >= start_ptr && itPtr <= end_ptr);
+#if defined(HAVE_NEON)
+ domain_mask_flipped = ~domain_mask_flipped;
+
+ u32 reach0, reach1, reach2, reach3;
+ u64a ptr = unaligned_load_u64a(itPtr);
+
+ reach0 = ptr & domain_mask_flipped;
+ reach1 = ptr >> 8 & domain_mask_flipped;
+ reach2 = ptr >> 16 & domain_mask_flipped;
+ reach3 = ptr >> 24 & domain_mask_flipped;
+
+ m128 st0 = load_m128_from_u64a(ft + reach0);
+ m128 st1 = load_m128_from_u64a(ft + reach1);
+ m128 st2 = load_m128_from_u64a(ft + reach2);
+ m128 st3 = load_m128_from_u64a(ft + reach3);
+
+ u32 reach4, reach5, reach6, reach7;
+ ptr = unaligned_load_u64a(itPtr + 4);
+ reach4 = ptr & domain_mask_flipped;
+ reach5 = ptr >> 8 & domain_mask_flipped;
+ reach6 = ptr >> 16 & domain_mask_flipped;
+ reach7 = ptr >> 24 & domain_mask_flipped;
+
+ m128 st4 = load_m128_from_u64a(ft + reach4);
+ m128 st5 = load_m128_from_u64a(ft + reach5);
+ m128 st6 = load_m128_from_u64a(ft + reach6);
+ m128 st7 = load_m128_from_u64a(ft + reach7);
+
+ m128 zero = zeroes128();
+
+ st1.vect_s8 = vextq_s8(zero.vect_s8, st1.vect_s8, 15);
+ st2.vect_s8 = vextq_s8(zero.vect_s8, st2.vect_s8, 14);
+ st3.vect_s8 = vextq_s8(zero.vect_s8, st3.vect_s8, 13);
+ st4.vect_s8 = vextq_s8(zero.vect_s8, st4.vect_s8, 12);
+ st5.vect_s8 = vextq_s8(zero.vect_s8, st5.vect_s8, 11);
+ st6.vect_s8 = vextq_s8(zero.vect_s8, st6.vect_s8, 10);
+ st7.vect_s8 = vextq_s8(zero.vect_s8, st7.vect_s8, 9);
+
+ st0 = or128(st0, st1);
+ st2 = or128(st2, st3);
+ st4 = or128(st4, st5);
+ st6 = or128(st6, st7);
+ st0 = or128(st0, st2);
+ st4 = or128(st4, st6);
+ st0 = or128(st0, st4);
+ *s = or128(*s, st0);
+
+ *conf0 = movq(*s);
+ *s = rshiftbyte_m128(*s, 8);
+ *conf0 = ~(*conf0);
+
+ u32 reach8, reach9, reach10, reach11;
+ ptr = unaligned_load_u64a(itPtr + 8);
+ reach8 = ptr & domain_mask_flipped;
+ reach9 = ptr >> 8 & domain_mask_flipped;
+ reach10 = ptr >> 16 & domain_mask_flipped;
+ reach11 = ptr >> 24 & domain_mask_flipped;
+
+ m128 st8 = load_m128_from_u64a(ft + reach8);
+ m128 st9 = load_m128_from_u64a(ft + reach9);
+ m128 st10 = load_m128_from_u64a(ft + reach10);
+ m128 st11 = load_m128_from_u64a(ft + reach11);
+
+ u32 reach12, reach13, reach14, reach15;
+ ptr = unaligned_load_u64a(itPtr + 12);
+ reach12 = ptr & domain_mask_flipped;
+ reach13 = ptr >> 8 & domain_mask_flipped;
+ reach14 = ptr >> 16 & domain_mask_flipped;
+ reach15 = ptr >> 24 & domain_mask_flipped;
+
+ m128 st12 = load_m128_from_u64a(ft + reach12);
+ m128 st13 = load_m128_from_u64a(ft + reach13);
+ m128 st14 = load_m128_from_u64a(ft + reach14);
+ m128 st15 = load_m128_from_u64a(ft + reach15);
+
+ st9.vect_s8 = vextq_s8(zero.vect_s8, st9.vect_s8, 15);
+ st10.vect_s8 = vextq_s8(zero.vect_s8, st10.vect_s8, 14);
+ st11.vect_s8 = vextq_s8(zero.vect_s8, st11.vect_s8, 13);
+ st12.vect_s8 = vextq_s8(zero.vect_s8, st12.vect_s8, 12);
+ st13.vect_s8 = vextq_s8(zero.vect_s8, st13.vect_s8, 11);
+ st14.vect_s8 = vextq_s8(zero.vect_s8, st14.vect_s8, 10);
+ st15.vect_s8 = vextq_s8(zero.vect_s8, st15.vect_s8, 9);
+
+ st8 = or128(st8, st9);
+ st10 = or128(st10, st11);
+ st12 = or128(st12, st13);
+ st14 = or128(st14, st15);
+ st8 = or128(st8, st10);
+ st12 = or128(st12, st14);
+ st8 = or128(st8, st12);
+ *s = or128(*s, st8);
+
+ *conf8 = movq(*s);
+ *s = rshiftbyte_m128(*s, 8);
+ *conf8 = ~(*conf8);
+
+#else
u64a reach0 = andn(domain_mask_flipped, itPtr);
u64a reach1 = andn(domain_mask_flipped, itPtr + 1);
u64a reach2 = andn(domain_mask_flipped, itPtr + 2);
@@ -241,6 +345,8 @@ void get_conf_stride_1(const u8 *itPtr, UNUSED const u8 *start_ptr,
*conf8 = movq(*s);
*s = rshiftbyte_m128(*s, 8);
*conf8 ^= ~0ULL;
+
+#endif
}
static really_inline
@@ -349,12 +455,12 @@ void do_confirm_fdr(u64a *conf, u8 offset, hwlmcb_rv_t *control,
u32 bitRem = bit % bucket;
u32 idx = bitRem;
u32 cf = confBase[idx];
- if (!cf) {
+ if (unlikely(!cf)) {
continue;
}
const struct FDRConfirm *fdrc = (const struct FDRConfirm *)
((const u8 *)confBase + cf);
- if (!(fdrc->groups & *control)) {
+ if (unlikely(!(fdrc->groups & *control))) {
continue;
}
u64a confVal = unaligned_load_u64a(confLoc + byte - sizeof(u64a) + 1);
@@ -603,7 +709,7 @@ void createEndZone(const u8 *buf, const u8 *begin, const u8 *end,
assert(z_len > 0);
size_t iter_bytes_second = 0;
size_t z_len_first = z_len;
- if (z_len > ITER_BYTES) {
+ if (unlikely(z_len > ITER_BYTES)) {
z_len_first = z_len - ITER_BYTES;
iter_bytes_second = ITER_BYTES;
}
@@ -637,7 +743,7 @@ void createEndZone(const u8 *buf, const u8 *begin, const u8 *end,
/* copy the last 16 bytes, may overlap with the previous 8 byte write */
storeu128(z_end_first - sizeof(m128), loadu128(end_first - sizeof(m128)));
- if (iter_bytes_second) {
+ if (unlikely(iter_bytes_second)) {
storeu128(z_end - sizeof(m128), loadu128(end - sizeof(m128)));
}
@@ -658,7 +764,7 @@ size_t prepareZones(const u8 *buf, size_t len, const u8 *hend,
const u8 *ptr = buf + start;
size_t remaining = len - start;
- if (remaining <= ITER_BYTES) {
+ if (unlikely(remaining <= ITER_BYTES)) {
/* enough bytes to make only one zone */
createShortZone(buf, hend, ptr, buf + len, &zoneArr[0]);
return 1;
@@ -691,13 +797,25 @@ size_t prepareZones(const u8 *buf, size_t len, const u8 *hend,
#define INVALID_MATCH_ID (~0U)
+/* add prefetch for aarch64,
+ *- due to gcc4.8.5 do not support builtin_prefetch.
+ */
+#if defined(HAVE_NEON)
+#define PREFETCH __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(itPtr + 256)))
+#define P2ALIGN __asm__ __volatile__(".p2align 6")
+#else
+#define PREFETCH __builtin_prefetch(itPtr + ITER_BYTES)
+#define P2ALIGN
+#endif
+
#define FDR_MAIN_LOOP(zz, s, get_conf_fn) \
do { \
+ P2ALIGN; \
const u8 *tryFloodDetect = zz->floodPtr; \
const u8 *start_ptr = zz->start; \
- const u8 *end_ptr = zz->end; \
+ const u8 *end_ptr = zz->end - ITER_BYTES; \
\
- for (const u8 *itPtr = start_ptr; itPtr + ITER_BYTES <= end_ptr; \
+ for (const u8 *itPtr = start_ptr; itPtr <= end_ptr; \
itPtr += ITER_BYTES) { \
if (unlikely(itPtr > tryFloodDetect)) { \
tryFloodDetect = floodDetect(fdr, a, &itPtr, tryFloodDetect,\
@@ -707,7 +825,7 @@ size_t prepareZones(const u8 *buf, size_t len, const u8 *hend,
return HWLM_TERMINATED; \
} \
} \
- __builtin_prefetch(itPtr + ITER_BYTES); \
+ PREFETCH; \
u64a conf0; \
u64a conf8; \
get_conf_fn(itPtr, start_ptr, end_ptr, domain_mask_flipped, \
diff --git a/src/hs_valid_platform.c b/src/hs_valid_platform.c
index 59ad3f3..035d3ff 100644
--- a/src/hs_valid_platform.c
+++ b/src/hs_valid_platform.c
@@ -33,9 +33,16 @@
HS_PUBLIC_API
hs_error_t HS_CDECL hs_valid_platform(void) {
/* Hyperscan requires SSSE3, anything else is a bonus */
+#if defined(__x86_64__)
if (check_ssse3()) {
return HS_SUCCESS;
- } else {
+ }
+#else
+ if (check_neon()) {
+ return HS_SUCCESS;
+ }
+#endif
+ else {
return HS_ARCH_ERROR;
}
}
diff --git a/src/nfa/limex_exceptional.h b/src/nfa/limex_exceptional.h
index 6c7335f..8304215 100644
--- a/src/nfa/limex_exceptional.h
+++ b/src/nfa/limex_exceptional.h
@@ -131,7 +131,7 @@ int RUN_EXCEPTION_FN(const EXCEPTION_T *e, STATE_ARG,
union RepeatControl *repeat_ctrl = ctx->repeat_ctrl + info->ctrlIndex;
char *repeat_state = ctx->repeat_state + info->stateOffset;
- if (e->trigger == LIMEX_TRIGGER_POS) {
+ if (unlikely(e->trigger == LIMEX_TRIGGER_POS)) {
char cyclic_on = TESTBIT_STATE(*STATE_ARG_P, info->cyclicState);
processPosTrigger(repeat, repeat_ctrl, repeat_state, offset,
cyclic_on);
@@ -140,7 +140,7 @@ int RUN_EXCEPTION_FN(const EXCEPTION_T *e, STATE_ARG,
assert(e->trigger == LIMEX_TRIGGER_TUG);
enum TriggerResult rv =
processTugTrigger(repeat, repeat_ctrl, repeat_state, offset);
- if (rv == TRIGGER_FAIL) {
+ if (likely(rv == TRIGGER_FAIL)) {
*cacheable = DO_NOT_CACHE_RESULT_AND_FLUSH_BR_ENTRIES;
DEBUG_PRINTF("tug found no valid matches in repeat state\n");
return 1; // continue
@@ -150,7 +150,7 @@ int RUN_EXCEPTION_FN(const EXCEPTION_T *e, STATE_ARG,
assert(e->hasSquash == LIMEX_SQUASH_TUG);
*succ = AND_STATE(*succ, LOAD_FROM_ENG(&e->squash));
return 1; // continue
- } else if (rv == TRIGGER_SUCCESS_CACHE) {
+ } else if (unlikely(rv == TRIGGER_SUCCESS_CACHE)) {
new_cache->br = 1;
} else {
assert(rv == TRIGGER_SUCCESS);
@@ -160,7 +160,7 @@ int RUN_EXCEPTION_FN(const EXCEPTION_T *e, STATE_ARG,
}
// Some exceptions fire accepts.
- if (e->reports != MO_INVALID_IDX) {
+ if (unlikely(e->reports != MO_INVALID_IDX)) {
if (flags & CALLBACK_OUTPUT) {
const ReportID *reports =
(const ReportID *)((const char *)limex + e->reports);
@@ -171,7 +171,7 @@ int RUN_EXCEPTION_FN(const EXCEPTION_T *e, STATE_ARG,
return 0; // halt
}
if (*cacheable == CACHE_RESULT) {
- if (!new_cache->reports || new_cache->reports == reports) {
+ if (likely(!new_cache->reports || new_cache->reports == reports)) {
new_cache->reports = reports;
} else {
*cacheable = DO_NOT_CACHE_RESULT;
@@ -194,8 +194,8 @@ int RUN_EXCEPTION_FN(const EXCEPTION_T *e, STATE_ARG,
// Some exceptions squash states behind them. Note that we squash states in
// 'succ', not local_succ.
- if (e->hasSquash == LIMEX_SQUASH_CYCLIC
- || e->hasSquash == LIMEX_SQUASH_REPORT) {
+ if (unlikely(e->hasSquash == LIMEX_SQUASH_CYCLIC
+ || e->hasSquash == LIMEX_SQUASH_REPORT)) {
*succ = AND_STATE(*succ, LOAD_FROM_ENG(&e->squash));
if (*cacheable == CACHE_RESULT) {
*cacheable = DO_NOT_CACHE_RESULT;
@@ -331,12 +331,12 @@ int PE_FN(STATE_ARG, ESTATE_ARG, UNUSED u32 diffmask, STATE_T *succ,
u32 idx = local_index + base_index[t];
const EXCEPTION_T *e = &exceptions[idx];
- if (!RUN_EXCEPTION_FN(e, STATE_ARG_NAME, succ,
+ if (unlikely(!RUN_EXCEPTION_FN(e, STATE_ARG_NAME, succ,
#ifndef BIG_MODEL
&local_succ,
#endif
limex, offset, ctx, &new_cache, &cacheable,
- in_rev, flags)) {
+ in_rev, flags))) {
return PE_RV_HALT;
}
} while (word);
@@ -349,7 +349,7 @@ int PE_FN(STATE_ARG, ESTATE_ARG, UNUSED u32 diffmask, STATE_T *succ,
*succ = OR_STATE(*succ, ctx->local_succ);
#endif
- if (cacheable == CACHE_RESULT) {
+ if (likely(cacheable == CACHE_RESULT)) {
ctx->cached_estate = estate;
#ifndef BIG_MODEL
ctx->cached_esucc = local_succ;
@@ -359,7 +359,7 @@ int PE_FN(STATE_ARG, ESTATE_ARG, UNUSED u32 diffmask, STATE_T *succ,
ctx->cached_reports = new_cache.reports;
ctx->cached_br = new_cache.br;
} else if (cacheable == DO_NOT_CACHE_RESULT_AND_FLUSH_BR_ENTRIES) {
- if (ctx->cached_br) {
+ if (unlikely(ctx->cached_br)) {
ctx->cached_estate = ZERO_STATE;
}
}
diff --git a/src/nfa/limex_internal.h b/src/nfa/limex_internal.h
index 23b1bd9..0e27c79 100644
--- a/src/nfa/limex_internal.h
+++ b/src/nfa/limex_internal.h
@@ -119,7 +119,7 @@ struct NFAException##size { \
u32 repeatOffset; /**< offset to NFARepeatInfo, or MO_INVALID_IDX */ \
u8 hasSquash; /**< from enum LimExSquash */ \
u8 trigger; /**< from enum LimExTrigger */ \
-}; \
+}__attribute__ ((aligned (16))); \
\
struct LimExNFA##size { \
u8 reachMap[N_CHARS]; /**< map of char -> entry in reach[] */ \
diff --git a/src/nfa/limex_native.c b/src/nfa/limex_native.c
index f6f5809..8998830 100644
--- a/src/nfa/limex_native.c
+++ b/src/nfa/limex_native.c
@@ -77,7 +77,7 @@ int processExceptional32(u32 s, u32 estate, UNUSED u32 diffmask, u32 *succ,
struct NFAContext32 *ctx, char in_rev, char flags) {
assert(estate != 0); // guaranteed by calling macro
- if (estate == ctx->cached_estate) {
+ if (unlikely(estate == ctx->cached_estate)) {
DEBUG_PRINTF("using cached succ from previous state\n");
*succ |= ctx->cached_esucc;
if (ctx->cached_reports && (flags & CALLBACK_OUTPUT)) {
@@ -103,21 +103,21 @@ int processExceptional32(u32 s, u32 estate, UNUSED u32 diffmask, u32 *succ,
u32 bit = findAndClearLSB_32(&estate);
u32 idx = rank_in_mask32(limex->exceptionMask, bit);
const struct NFAException32 *e = &exceptions[idx];
- if (!runException32(e, s, succ, &local_succ, limex, offset, ctx,
- &new_cache, &cacheable, in_rev, flags)) {
+ if (unlikely(!runException32(e, s, succ, &local_succ, limex, offset, ctx,
+ &new_cache, &cacheable, in_rev, flags))) {
return PE_RV_HALT;
}
} while (estate != 0);
*succ |= local_succ;
- if (cacheable == CACHE_RESULT) {
+ if (unlikely(cacheable == CACHE_RESULT)) {
ctx->cached_estate = orig_estate;
ctx->cached_esucc = local_succ;
ctx->cached_reports = new_cache.reports;
ctx->cached_br = new_cache.br;
} else if (cacheable == DO_NOT_CACHE_RESULT_AND_FLUSH_BR_ENTRIES) {
- if (ctx->cached_br) {
+ if (unlikely(ctx->cached_br)) {
ctx->cached_estate = 0U;
}
}
diff --git a/src/nfa/shufti.c b/src/nfa/shufti.c
index 09ffc0c..2cb74f0 100644
--- a/src/nfa/shufti.c
+++ b/src/nfa/shufti.c
@@ -153,13 +153,13 @@ const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
assert(buf < buf_end);
// Slow path for small cases.
- if (buf_end - buf < 16) {
+ if (unlikely(buf_end - buf < 16)) {
return shuftiFwdSlow((const u8 *)&mask_lo, (const u8 *)&mask_hi,
buf, buf_end);
}
const m128 zeroes = zeroes128();
- const m128 low4bits = _mm_set1_epi8(0xf);
+ const m128 low4bits = set16x8(0xf);
const u8 *rv;
size_t min = (size_t)buf % 16;
@@ -179,6 +179,11 @@ const u8 *shuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
const u8 *last_block = buf_end - 16;
while (buf < last_block) {
m128 lchars = load128(buf);
+
+#if defined(HAVE_NEON)
+ __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(buf + 256)));
+#endif
+
rv = fwdBlock(mask_lo, mask_hi, lchars, buf, low4bits, zeroes);
if (rv) {
return rv;
@@ -246,7 +251,7 @@ const u8 *rshuftiExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
}
const m128 zeroes = zeroes128();
- const m128 low4bits = _mm_set1_epi8(0xf);
+ const m128 low4bits = set16x8(0xf);
const u8 *rv;
assert(buf_end - buf >= 16);
@@ -320,7 +325,7 @@ const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi,
m128 mask2_lo, m128 mask2_hi,
const u8 *buf, const u8 *buf_end) {
const m128 ones = ones128();
- const m128 low4bits = _mm_set1_epi8(0xf);
+ const m128 low4bits = set16x8(0xf);
const u8 *rv;
size_t min = (size_t)buf % 16;
@@ -340,6 +345,11 @@ const u8 *shuftiDoubleExec(m128 mask1_lo, m128 mask1_hi,
const u8 *last_block = buf_end - 16;
while (buf < last_block) {
m128 lchars = load128(buf);
+
+#if defined(HAVE_NEON)
+ __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(buf + 256)));
+#endif
+
rv = fwdBlock2(mask1_lo, mask1_hi, mask2_lo, mask2_hi,
lchars, buf, low4bits, ones);
if (rv) {
diff --git a/src/nfa/truffle.c b/src/nfa/truffle.c
index be6b312..c05d778 100644
--- a/src/nfa/truffle.c
+++ b/src/nfa/truffle.c
@@ -41,7 +41,7 @@
static really_inline
const u8 *lastMatch(const u8 *buf, u32 z) {
- if (unlikely(z != 0xffff)) {
+ if (z != 0xffff) {
u32 pos = clz32(~z & 0xffff);
assert(pos >= 16 && pos < 32);
return buf + (31 - pos);
@@ -52,7 +52,7 @@ const u8 *lastMatch(const u8 *buf, u32 z) {
static really_inline
const u8 *firstMatch(const u8 *buf, u32 z) {
- if (unlikely(z != 0xffff)) {
+ if (likely(z != 0xffff)) {
u32 pos = ctz32(~z & 0xffff);
assert(pos < 16);
return buf + pos;
@@ -64,8 +64,8 @@ const u8 *firstMatch(const u8 *buf, u32 z) {
static really_inline
u32 block(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highset, m128 v) {
- m128 highconst = _mm_set1_epi8(0x80);
- m128 shuf_mask_hi = _mm_set1_epi64x(0x8040201008040201);
+ m128 highconst = set16x8(0x80);
+ m128 shuf_mask_hi = set2x64(0x8040201008040201);
// and now do the real work
m128 shuf1 = pshufb_m128(shuf_mask_lo_highclear, v);
@@ -124,7 +124,7 @@ const u8 *truffleExec(m128 shuf_mask_lo_highclear,
assert(buf < buf_end);
const u8 *rv;
- if (buf_end - buf < 16) {
+ if (unlikely(buf_end - buf < 16)) {
return truffleMini(shuf_mask_lo_highclear, shuf_mask_lo_highset, buf,
buf_end);
}
diff --git a/src/parser/control_verbs.cpp b/src/parser/control_verbs.cpp
new file mode 100644
index 0000000..482004d
--- /dev/null
+++ b/src/parser/control_verbs.cpp
@@ -0,0 +1,340 @@
+
+/*
+ * Copyright (c) 2017, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Intel Corporation nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * \file
+ * \brief Parser for control verbs that can occur at the beginning of a pattern.
+ */
+
+#include "parser/control_verbs.h"
+
+#include "parser/Parser.h"
+#include "parser/parse_error.h"
+
+#include <cstring>
+#include <sstream>
+
+using namespace std;
+
+namespace ue2 {
+
+const char *read_control_verbs(const char *ptr, const char *end, size_t start,
+ ParseMode &mode) {
+ const char *p = ptr;
+ const char *pe = end;
+ const char *eof = pe;
+ const char *ts, *te;
+ int cs;
+ UNUSED int act;
+
+ static const char _ControlVerbs_actions[] = {
+ 0, 1, 0, 1, 1, 1, 2, 1, 3, 1, 4, 1, 5, 1, 6, 1, 7, 1, 8, 1, 9};
+
+ static const unsigned char _ControlVerbs_key_offsets[] = {
+ 0, 7, 8, 10, 12, 14, 16, 18, 20, 21, 23, 25, 27,
+ 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 55,
+ 57, 59, 61, 63, 66, 68, 70, 72, 74, 76, 79, 82, 84,
+ 86, 88, 90, 92, 94, 96, 98, 100, 102, 105, 107, 109, 111,
+ 113, 115, 117, 119, 121, 123, 125, 127, 129, 131, 133, 135, 137,
+ 139, 141, 143, 146, 148, 149, 151, 155, 157, 159, 160, 161};
+
+ static const char _ControlVerbs_trans_keys[] = {
+ 41, 65, 66, 67, 76, 78, 85, 41, 41, 78, 41, 89, 41, 67, 41, 82, 41,
+ 76, 41, 70, 41, 41, 83, 41, 82, 41, 95, 41, 65, 85, 41, 78, 41, 89,
+ 41, 67, 41, 78, 41, 73, 41, 67, 41, 79, 41, 68, 41, 69, 41, 82, 41,
+ 76, 41, 70, 73, 41, 77, 41, 73, 41, 84, 41, 95, 41, 77, 82, 41, 65,
+ 41, 84, 41, 67, 41, 72, 41, 61, 41, 48, 57, 41, 48, 57, 41, 69, 41,
+ 67, 41, 85, 41, 82, 41, 83, 41, 73, 41, 79, 41, 78, 41, 79, 41, 95,
+ 41, 65, 83, 41, 85, 41, 84, 41, 79, 41, 95, 41, 80, 41, 79, 41, 83,
+ 41, 83, 41, 69, 41, 83, 41, 83, 41, 84, 41, 65, 41, 82, 41, 84, 41,
+ 95, 41, 79, 41, 80, 41, 84, 41, 67, 84, 41, 80, 41, 41, 70, 41, 49,
+ 51, 56, 41, 54, 41, 50, 41, 40, 42, 0};
+
+ static const char _ControlVerbs_single_lengths[] = {
+ 7, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 2, 4, 2, 2, 1, 1, 1};
+
+ static const char _ControlVerbs_range_lengths[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+ static const short _ControlVerbs_index_offsets[] = {
+ 0, 8, 10, 13, 16, 19, 22, 25, 28, 30, 33, 36, 39,
+ 43, 46, 49, 52, 55, 58, 61, 64, 67, 70, 73, 76, 80,
+ 83, 86, 89, 92, 96, 99, 102, 105, 108, 111, 114, 117, 120,
+ 123, 126, 129, 132, 135, 138, 141, 144, 147, 151, 154, 157, 160,
+ 163, 166, 169, 172, 175, 178, 181, 184, 187, 190, 193, 196, 199,
+ 202, 205, 208, 212, 215, 217, 220, 225, 228, 231, 233, 235};
+
+ static const char _ControlVerbs_indicies[] = {
+ 0, 2, 3, 4, 5, 6, 7, 1, 8, 1, 8, 9, 1, 8, 10, 1, 11,
+ 12, 1, 8, 13, 1, 8, 14, 1, 8, 15, 1, 11, 1, 8, 16, 1, 8,
+ 17, 1, 8, 18, 1, 8, 19, 20, 1, 8, 21, 1, 8, 22, 1, 8, 12,
+ 1, 8, 23, 1, 8, 24, 1, 8, 25, 1, 8, 26, 1, 8, 27, 1, 8,
+ 15, 1, 8, 28, 1, 11, 14, 1, 8, 15, 29, 1, 8, 30, 1, 8, 31,
+ 1, 8, 32, 1, 8, 33, 1, 8, 34, 35, 1, 8, 36, 1, 8, 37, 1,
+ 8, 38, 1, 8, 39, 1, 8, 40, 1, 8, 41, 1, 11, 41, 1, 8, 42,
+ 1, 8, 43, 1, 8, 44, 1, 8, 45, 1, 8, 46, 1, 8, 47, 1, 8,
+ 48, 1, 8, 39, 1, 8, 49, 1, 8, 50, 1, 8, 51, 52, 1, 8, 53,
+ 1, 8, 54, 1, 8, 55, 1, 8, 56, 1, 8, 57, 1, 8, 58, 1, 8,
+ 59, 1, 8, 60, 1, 8, 61, 1, 8, 62, 1, 8, 15, 1, 8, 63, 1,
+ 8, 64, 1, 8, 65, 1, 8, 66, 1, 8, 67, 1, 8, 68, 1, 8, 69,
+ 1, 8, 15, 1, 8, 70, 71, 1, 8, 72, 1, 73, 1, 8, 74, 1, 75,
+ 76, 77, 78, 1, 8, 15, 1, 8, 15, 1, 75, 1, 80, 79, 82, 81, 0};
+
+ static const char _ControlVerbs_trans_targs[] = {
+ 75, 1, 2, 9, 22, 24, 45, 67, 75, 3, 4, 75, 5, 6, 7, 8, 10,
+ 11, 12, 13, 16, 14, 15, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29,
+ 30, 37, 31, 32, 33, 34, 35, 36, 38, 39, 40, 41, 42, 43, 44, 46, 47,
+ 48, 59, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 60, 61, 62, 63, 64,
+ 65, 66, 68, 70, 69, 75, 71, 75, 72, 73, 74, 75, 76, 75, 0};
+
+ static const char _ControlVerbs_trans_actions[] = {
+ 19, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7, 0, 0, 0, 15, 5, 17, 0};
+
+ static const char _ControlVerbs_to_state_actions[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0};
+
+ static const char _ControlVerbs_from_state_actions[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0};
+
+ static const short _ControlVerbs_eof_trans[] = {
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 82};
+
+ static const int ControlVerbs_start = 75;
+ static const int ControlVerbs_first_final = 75;
+ static const int ControlVerbs_error = -1;
+
+ static const int ControlVerbs_en_main = 75;
+
+ {
+ cs = ControlVerbs_start;
+ ts = 0;
+ te = 0;
+ act = 0;
+ }
+
+ try {
+
+ {
+ int _klen;
+ unsigned int _trans;
+ const char *_acts;
+ unsigned int _nacts;
+ const char *_keys;
+
+ if (p == pe)
+ goto _test_eof;
+ _resume:
+ _acts =
+ _ControlVerbs_actions + _ControlVerbs_from_state_actions[cs];
+ _nacts = (unsigned int)*_acts++;
+ while (_nacts-- > 0) {
+ switch (*_acts++) {
+ case 1: {
+ ts = p;
+ } break;
+ }
+ }
+
+ _keys = _ControlVerbs_trans_keys + _ControlVerbs_key_offsets[cs];
+ _trans = _ControlVerbs_index_offsets[cs];
+
+ _klen = _ControlVerbs_single_lengths[cs];
+ if (_klen > 0) {
+ const char *_lower = _keys;
+ const char *_mid;
+ const char *_upper = _keys + _klen - 1;
+ while (1) {
+ if (_upper < _lower)
+ break;
+
+ _mid = _lower + ((_upper - _lower) >> 1);
+ if ((*p) < *_mid)
+ _upper = _mid - 1;
+ else if ((*p) > *_mid)
+ _lower = _mid + 1;
+ else {
+ _trans += (unsigned int)(_mid - _keys);
+ goto _match;
+ }
+ }
+ _keys += _klen;
+ _trans += _klen;
+ }
+
+ _klen = _ControlVerbs_range_lengths[cs];
+ if (_klen > 0) {
+ const char *_lower = _keys;
+ const char *_mid;
+ const char *_upper = _keys + (_klen << 1) - 2;
+ while (1) {
+ if (_upper < _lower)
+ break;
+
+ _mid = _lower + (((_upper - _lower) >> 1) & ~1);
+ if ((*p) < _mid[0])
+ _upper = _mid - 2;
+ else if ((*p) > _mid[1])
+ _lower = _mid + 2;
+ else {
+ _trans += (unsigned int)((_mid - _keys) >> 1);
+ goto _match;
+ }
+ }
+ _trans += _klen;
+ }
+
+ _match:
+ _trans = _ControlVerbs_indicies[_trans];
+ _eof_trans:
+ cs = _ControlVerbs_trans_targs[_trans];
+
+ if (_ControlVerbs_trans_actions[_trans] == 0)
+ goto _again;
+
+ _acts = _ControlVerbs_actions + _ControlVerbs_trans_actions[_trans];
+ _nacts = (unsigned int)*_acts++;
+ while (_nacts-- > 0) {
+ switch (*_acts++) {
+ case 2: {
+ te = p + 1;
+ } break;
+ case 3: {
+ te = p + 1;
+ { mode.utf8 = true; }
+ } break;
+ case 4: {
+ te = p + 1;
+ { mode.ucp = true; }
+ } break;
+ case 5: {
+ te = p + 1;
+ {
+ ostringstream str;
+ str << "Unsupported control verb "
+ << string(ts, te - ts);
+ throw LocatedParseError(str.str());
+ }
+ } break;
+ case 6: {
+ te = p + 1;
+ {
+ ostringstream str;
+ str << "Unknown control verb " << string(ts, te - ts);
+ throw LocatedParseError(str.str());
+ }
+ } break;
+ case 7: {
+ te = p + 1;
+ {
+ p--;
+ {
+ p++;
+ goto _out;
+ }
+ }
+ } break;
+ case 8: {
+ te = p;
+ p--;
+ {
+ p--;
+ {
+ p++;
+ goto _out;
+ }
+ }
+ } break;
+ case 9: {
+ { p = ((te)) - 1; }
+ {
+ p--;
+ {
+ p++;
+ goto _out;
+ }
+ }
+ } break;
+ }
+ }
+
+ _again:
+ _acts = _ControlVerbs_actions + _ControlVerbs_to_state_actions[cs];
+ _nacts = (unsigned int)*_acts++;
+ while (_nacts-- > 0) {
+ switch (*_acts++) {
+ case 0: {
+ ts = 0;
+ } break;
+ }
+ }
+
+ if (++p != pe)
+ goto _resume;
+ _test_eof : {}
+ if (p == eof) {
+ if (_ControlVerbs_eof_trans[cs] > 0) {
+ _trans = _ControlVerbs_eof_trans[cs] - 1;
+ goto _eof_trans;
+ }
+ }
+
+ _out : {}
+ }
+
+ } catch (LocatedParseError &error) {
+ if (ts >= ptr && ts <= pe) {
+ error.locate(ts - ptr + start);
+ } else {
+ error.locate(0);
+ }
+ throw;
+ }
+
+ return p;
+}
+
+} // namespace ue2
diff --git a/src/rose/counting_miracle.h b/src/rose/counting_miracle.h
index 976208b..4456679 100644
--- a/src/rose/counting_miracle.h
+++ b/src/rose/counting_miracle.h
@@ -94,7 +94,7 @@ u32 roseCountingMiracleScanShufti(m128 mask_lo, m128 mask_hi, u8 poison,
u32 count = *count_inout;
const m128 zeroes = zeroes128();
- const m128 low4bits = _mm_set1_epi8(0xf);
+ const m128 low4bits = set16x8(0xf);
for (; d + 16 <= d_end; d_end -= 16) {
m128 data = loadu128(d_end - 16);
diff --git a/src/util/arch.h b/src/util/arch.h
index 985fec6..fe4a910 100644
--- a/src/util/arch.h
+++ b/src/util/arch.h
@@ -61,6 +61,10 @@
#define HAVE_AVX512VBMI
#endif
+#if defined(__aarch64__)
+#define HAVE_NEON
+#endif
+
/*
* ICC and MSVC don't break out POPCNT or BMI/2 as separate pre-def macros
*/
@@ -87,4 +91,11 @@
#define NO_ASM
#endif
+/*
+ * AARCH64 uses a different form of inline asm
+ */
+#if defined(__aarch64__)
+#define NO_ASM
+#endif
+
#endif // UTIL_ARCH_H_
diff --git a/src/util/cpuid_flags.c b/src/util/cpuid_flags.c
index c00ce58..96286ee 100644
--- a/src/util/cpuid_flags.c
+++ b/src/util/cpuid_flags.c
@@ -40,6 +40,7 @@
u64a cpuid_flags(void) {
u64a cap = 0;
+#if defined(__X86_64__)
if (check_avx2()) {
DEBUG_PRINTF("AVX2 enabled\n");
cap |= HS_CPU_FEATURES_AVX2;
@@ -67,6 +68,7 @@ u64a cpuid_flags(void) {
#if (!defined(FAT_RUNTIME) && !defined(HAVE_AVX512VBMI)) || \
(defined(FAT_RUNTIME) && !defined(BUILD_AVX512VBMI))
cap &= ~HS_CPU_FEATURES_AVX512VBMI;
+#endif
#endif
return cap;
@@ -78,6 +80,7 @@ struct family_id {
u32 tune;
};
+#if defined(__X86_64__)
/* from table 35-1 of the Intel 64 and IA32 Arch. Software Developer's Manual
* and "Intel Architecture and Processor Identification With CPUID Model and
* Family Numbers" */
@@ -121,6 +124,7 @@ static const struct family_id known_microarch[] = {
{ 0x6, 0x6C, HS_TUNE_FAMILY_ICX }, /* Icelake Xeon */
};
+#endif
#ifdef DUMP_SUPPORT
static UNUSED
@@ -144,6 +148,7 @@ const char *dumpTune(u32 tune) {
#endif
u32 cpuid_tune(void) {
+#if defined(__X86_64__)
unsigned int eax, ebx, ecx, edx;
cpuid(1, 0, &eax, &ebx, &ecx, &edx);
@@ -171,6 +176,7 @@ u32 cpuid_tune(void) {
DEBUG_PRINTF("found tune flag %s\n", dumpTune(tune) );
return tune;
}
+#endif
return HS_TUNE_FAMILY_GENERIC;
}
diff --git a/src/util/cpuid_flags.h b/src/util/cpuid_flags.h
index 527c6d5..3125bd1 100644
--- a/src/util/cpuid_flags.h
+++ b/src/util/cpuid_flags.h
@@ -32,7 +32,9 @@
#include "ue2common.h"
#if !defined(_WIN32) && !defined(CPUID_H_)
+#if defined(__x86_64__)
#include <cpuid.h>
+#endif
/* system header doesn't have a header guard */
#define CPUID_H_
#endif
diff --git a/src/util/cpuid_inline.h b/src/util/cpuid_inline.h
index b7b4245..b228c1d 100644
--- a/src/util/cpuid_inline.h
+++ b/src/util/cpuid_inline.h
@@ -32,17 +32,20 @@
#include "ue2common.h"
#include "cpuid_flags.h"
+#if defined(__x86_64__) || defined(_M_X64)
#if !defined(_WIN32) && !defined(CPUID_H_)
#include <cpuid.h>
/* system header doesn't have a header guard */
#define CPUID_H_
#endif
+#endif
#ifdef __cplusplus
extern "C"
{
#endif
+#if defined(__x86_64__) || defined(_M_X64)
static inline
void cpuid(unsigned int op, unsigned int leaf, unsigned int *eax,
unsigned int *ebx, unsigned int *ecx, unsigned int *edx) {
@@ -57,6 +60,7 @@ void cpuid(unsigned int op, unsigned int leaf, unsigned int *eax,
*edx = a[3];
#endif
}
+#endif
// ECX
#define CPUID_SSE3 (1 << 0)
@@ -93,11 +97,12 @@ void cpuid(unsigned int op, unsigned int leaf, unsigned int *eax,
#define CPUID_XCR0_AVX512 \
(CPUID_XCR0_OPMASK | CPUID_XCR0_ZMM_Hi256 | CPUID_XCR0_Hi16_ZMM)
+#if defined(__x86_64__)
static inline
u64a xgetbv(u32 op) {
#if defined(_WIN32) || defined(__INTEL_COMPILER)
return _xgetbv(op);
-#else
+#elif defined(__x86_64__)
u32 a, d;
__asm__ volatile (
"xgetbv\n"
@@ -252,6 +257,16 @@ int check_popcnt(void) {
cpuid(1, 0, &eax, &ebx, &ecx, &edx);
return !!(ecx & CPUID_POPCNT);
}
+#endif //__x86_64__
+
+static inline
+int check_neon(void) {
+#if defined(__aarch64__)
+ return 1;
+#else
+ return 0;
+#endif
+}
#ifdef __cplusplus
} /* extern "C" */
diff --git a/src/util/intrinsics.h b/src/util/intrinsics.h
index edc4f6e..ece3b1a 100644
--- a/src/util/intrinsics.h
+++ b/src/util/intrinsics.h
@@ -55,10 +55,22 @@
# endif
#endif
+#ifdef __cplusplus
+# if defined(HAVE_CXX_ARM_NEON_H)
+# define USE_ARM_NEON_H
+# endif
+#else // C
+# if defined(HAVE_C_ARM_NEON_H)
+# define USE_ARM_NEON_H
+# endif
+#endif
+
#if defined(USE_X86INTRIN_H)
#include <x86intrin.h>
#elif defined(USE_INTRIN_H)
#include <intrin.h>
+#elif defined(USE_ARM_NEON_H)
+#include <arm_neon.h>
#else
#error no intrinsics file
#endif
diff --git a/src/util/popcount.h b/src/util/popcount.h
index eb08f6b..7d794d1 100644
--- a/src/util/popcount.h
+++ b/src/util/popcount.h
@@ -41,6 +41,8 @@ u32 popcount32(u32 x) {
#if defined(HAVE_POPCOUNT_INSTR)
// Single-instruction builtin.
return _mm_popcnt_u32(x);
+#elif defined(HAVE_NEON)
+ return (u32)vaddlv_u8(vcnt_u8(vcreate_u8((u64a)x)));
#else
// Fast branch-free version from bit-twiddling hacks as older Intel
// processors do not have a POPCNT instruction.
@@ -63,7 +65,9 @@ u32 popcount64(u64a x) {
x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333);
x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f;
return (x * 0x0101010101010101) >> 56;
-# endif
+#endif
+#elif defined(HAVE_NEON)
+ return (u32)vaddlv_u8(vcnt_u8(vcreate_u8((u64a)x)));
#else
// Synthesise from two 32-bit cases.
return popcount32(x >> 32) + popcount32(x);
diff --git a/src/util/simd_arm.h b/src/util/simd_arm.h
new file mode 100644
index 0000000..cce119f
--- /dev/null
+++ b/src/util/simd_arm.h
@@ -0,0 +1,1069 @@
+/*
+ * Copyright (c) 2015-2017, Intel Corporation
+ * 2020.01 - Use the neon instruction to implement the function of 128-bit operation.
+ * Huawei Technologies Co., Ltd.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Intel Corporation nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief SIMD types and primitive operations.
+ */
+
+#ifndef SIMD_ARM
+#define SIMD_ARM
+
+#include "config.h"
+#include "simd_types.h"
+#include "ue2common.h"
+#include "unaligned.h"
+#include "util/arch.h"
+#include "util/intrinsics.h"
+
+#include <string.h> // for memcpy
+
+// Define a common assume_aligned using an appropriate compiler built-in, if
+// it's available. Note that we need to handle C or C++ compilation.
+#ifdef __cplusplus
+#ifdef HAVE_CXX_BUILTIN_ASSUME_ALIGNED
+#define assume_aligned(x, y) __builtin_assume_aligned((x), (y))
+#endif
+#else
+#ifdef HAVE_CC_BUILTIN_ASSUME_ALIGNED
+#define assume_aligned(x, y) __builtin_assume_aligned((x), (y))
+#endif
+#endif
+
+// Fallback to identity case.
+#ifndef assume_aligned
+#define assume_aligned(x, y) (x)
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern const char vbs_mask_data[];
+#ifdef __cplusplus
+}
+#endif
+
+/*
+** extend 4.8.5 neon inline assembly functions
+*/
+__extension__ static __inline uint64x2_t __attribute__((__always_inline__))
+vmvnq_u64(uint64x2_t a) {
+ uint64x2_t result;
+ __asm__("mvn %0.16b,%1.16b" : "=w"(result) : "w"(a) : /* No clobbers */);
+ return result;
+}
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wshadow"
+
+static really_inline m128 ones128(void) {
+ m128 result;
+ result.vect_s32 = vdupq_n_s32(0xFFFFFFFF);
+ return result;
+}
+
+static really_inline m128 zeroes128(void) {
+ m128 result;
+ result.vect_s32 = vdupq_n_s32(0x0);
+ return result;
+}
+
+/** \brief Return 1 if a and b are different otherwise 0 */
+static really_inline int diff128(m128 a, m128 b) {
+ return !!vaddlvq_s16(veorq_s16(a.vect_s16, b.vect_s16));
+}
+
+static really_inline int isnonzero128(m128 a) {
+ return !!diff128(a, zeroes128());
+}
+
+/**
+ * "Rich" version of diff128(). Takes two vectors a and b and returns a 4-bit
+ * mask indicating which 32-bit words contain differences.
+ */
+static really_inline u32 diffrich128(m128 a, m128 b) {
+ m128 tmp;
+ tmp.vect_u32 = vmvnq_u32(vceqq_u32(a.vect_u32, b.vect_u32));
+ return ((vgetq_lane_u32(tmp.vect_u32, 3) & 0x8) |
+ (vgetq_lane_u32(tmp.vect_u32, 2) & 0x4) |
+ (vgetq_lane_u32(tmp.vect_u32, 1) & 0x2) |
+ (vgetq_lane_u32(tmp.vect_u32, 0) & 0x1));
+}
+
+/**
+ * "Rich" version of diff128(), 64-bit variant. Takes two vectors a and b and
+ * returns a 4-bit mask indicating which 64-bit words contain differences.
+ */
+static really_inline u32 diffrich64_128(m128 a, m128 b) {
+ m128 tmp;
+ tmp.vect_u64 = vmvnq_u64(vceqq_u64(a.vect_u64, b.vect_u64));
+ return (u32)((vgetq_lane_u64(tmp.vect_u64, 1) & 0x4) |
+ (vgetq_lane_u64(tmp.vect_u64, 0) & 0x1));
+}
+
+static really_really_inline m128 lshift64_m128(m128 a, unsigned b) {
+ assert(b <= 63);
+ m128 result;
+ result.vect_s64 = vshlq_n_s64(a.vect_s64, b);
+ return result;
+}
+
+static really_really_inline m128 rshift64_m128(m128 a, int imm8) {
+ assert(imm8 >= 0 && imm8 <= 63);
+ if (unlikely(imm8 == 0)) {
+ return a;
+ }
+ m128 result;
+ result.vect_u64 = vshrq_n_u64(a.vect_u64, imm8);
+ return result;
+}
+
+static really_really_inline m128 eq128(m128 a, m128 b) {
+ m128 result;
+ result.vect_u8 = vceqq_s8(a.vect_s8, b.vect_s8);
+ return result;
+}
+
+static really_really_inline u32 movemask128(m128 a) {
+ m128 result;
+ result.vect_u8 = vshrq_n_u8(a.vect_u8, 7);
+ result.vect_u16 = vsraq_n_u16(result.vect_u16, result.vect_u16, 7);
+ result.vect_u32 = vsraq_n_u32(result.vect_u32, result.vect_u32, 14);
+ result.vect_u64 = vsraq_n_u64(result.vect_u64, result.vect_u64, 28);
+ return (u32)(vgetq_lane_u8(result.vect_u8, 0) |
+ ((u32)vgetq_lane_u8(result.vect_u8, 8) << 8));
+}
+
+static really_really_inline m128 rshiftbyte_m128(m128 a, int imm8) {
+ assert(imm8 >= 0 && imm8 <= 15);
+ m128 result;
+ result.vect_s8 = vextq_s8(a.vect_s8, vdupq_n_s8(0), imm8);
+ return result;
+}
+
+static really_really_inline m128 lshiftbyte_m128(m128 a, int imm8) {
+ assert(imm8 >= 0 && imm8 <= 15);
+ m128 result;
+ if (unlikely(imm8 == 0)) {
+ return a;
+ }
+ result.vect_s8 = vextq_s8(vdupq_n_s8(0), a.vect_s8, (16 - imm8));
+ return result;
+}
+
+static really_inline m128 set16x8(u8 c) {
+ m128 result;
+ result.vect_s8 = vdupq_n_s8(c);
+ return result;
+}
+
+static really_inline m128 set4x32(u32 c) {
+ m128 result;
+ result.vect_s32 = vdupq_n_s32(c);
+ return result;
+}
+
+static really_inline m128 set2x64(u64a c) {
+ m128 result;
+ result.vect_u64 = vdupq_n_u64(c);
+ return result;
+}
+
+static really_inline u32 movd(const m128 in) {
+ u32 result;
+ result = vgetq_lane_u32(in.vect_u32, 0);
+ return result;
+}
+
+static really_inline u64a movq(const m128 in) {
+ return vgetq_lane_u64(in.vect_u64, 0);
+}
+
+/* another form of movq */
+static really_inline m128 load_m128_from_u64a(const u64a *p) {
+ m128 result;
+ __asm__ __volatile__("ldr %d0, %1 \n\t"
+ : "=w"(result)
+ : "Utv"(*p)
+ : /* No clobbers */
+ );
+ return result;
+}
+
+/*The x86 platform does not perform the lower 2 bit operation.
+If the value of imm exceeds 2 bit, a compilation error occurs.*/
+static really_inline u32 extract32from128(m128 a, int imm) {
+ return vgetq_lane_s32(a.vect_s32, imm & 0x0003);
+}
+
+/*The x86 platform does not perform the lower 1 bit operation.
+If the value of imm exceeds 1 bit, a compilation error occurs.*/
+static really_inline u64a extract64from128(m128 a, int imm) {
+ return vgetq_lane_s64(a.vect_s64, imm & 0x0001);
+}
+
+#define extractlow64from256(a) movq(a.lo)
+#define extractlow32from256(a) movd(a.lo)
+
+/*The x86 platform does not perform the lower 2 bit operation.
+If the value of imm exceeds 2 bit, a compilation error occurs.*/
+static really_inline u32 extract32from256(m256 a, int imm) {
+ return vgetq_lane_s32((imm >> 2) ? a.hi.vect_s32 : a.lo.vect_s32,
+ imm & 0x0003);
+}
+
+/*The x86 platform does not perform the lower 1 bit operation.
+If the value of imm exceeds 1 bit, a compilation error occurs.*/
+static really_inline u64a extract64from256(m256 a, int imm) {
+ return vgetq_lane_s64((imm >> 1) ? a.hi.vect_s64 : a.lo.vect_s64,
+ imm & 0x0001);
+}
+
+static really_inline m128 and128(m128 a, m128 b) {
+ m128 result;
+ result.vect_s32 = vandq_s32(a.vect_s32, b.vect_s32);
+ return result;
+}
+
+static really_inline m128 not128(m128 a) {
+ m128 result;
+ result.vect_s32 = vmvnq_s32(a.vect_s32);
+ return result;
+}
+
+static really_inline m128 xor128(m128 a, m128 b) {
+ m128 result;
+ result.vect_s32 = veorq_s32(a.vect_s32, b.vect_s32);
+ return result;
+}
+
+static really_inline m128 or128(m128 a, m128 b) {
+ m128 result;
+ result.vect_s32 = vorrq_s32(a.vect_s32, b.vect_s32);
+ return result;
+}
+
+static really_inline m128 andnot128(m128 a, m128 b) {
+ m128 result;
+ result.vect_s32 = vbicq_s32(b.vect_s32, a.vect_s32);
+ return result;
+}
+
+// aligned load
+static really_inline m128 load128(const void *ptr) {
+ assert(ISALIGNED_N(ptr, alignof(m128)));
+ ptr = assume_aligned(ptr, 16);
+ m128 result;
+ result.vect_s32 = vld1q_s32((const int32_t *)ptr);
+ return result;
+}
+
+// aligned store
+static really_inline void store128(void *ptr, m128 a) {
+ assert(ISALIGNED_N(ptr, alignof(m128)));
+ ptr = assume_aligned(ptr, 16);
+ *(m128 *)ptr = a;
+}
+
+// unaligned load
+static really_inline m128 loadu128(const void *ptr) {
+ m128 result;
+ result.vect_s32 = vld1q_s32((const int32_t *)ptr);
+ return result;
+}
+
+// unaligned store
+static really_inline void storeu128(void *ptr, m128 a) {
+ vst1q_s32((int32_t *)ptr, a.vect_s32);
+}
+
+// packed unaligned store of first N bytes
+static really_inline void storebytes128(void *ptr, m128 a, unsigned int n) {
+ assert(n <= sizeof(a));
+ memcpy(ptr, &a, n);
+}
+
+// packed unaligned load of first N bytes, pad with zero
+static really_inline m128 loadbytes128(const void *ptr, unsigned int n) {
+ m128 a = zeroes128();
+ assert(n <= sizeof(a));
+ memcpy(&a, ptr, n);
+ return a;
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern const u8 simd_onebit_masks[];
+#ifdef __cplusplus
+}
+#endif
+
+static really_inline m128 mask1bit128(unsigned int n) {
+ assert(n < sizeof(m128) * 8);
+ u32 mask_idx = ((n % 8) * 64) + 95;
+ mask_idx -= n / 8;
+ return loadu128(&simd_onebit_masks[mask_idx]);
+}
+
+// switches on bit N in the given vector.
+static really_inline void setbit128(m128 *ptr, unsigned int n) {
+ *ptr = or128(mask1bit128(n), *ptr);
+}
+
+// switches off bit N in the given vector.
+static really_inline void clearbit128(m128 *ptr, unsigned int n) {
+ *ptr = andnot128(mask1bit128(n), *ptr);
+}
+
+// tests bit N in the given vector.
+static really_inline char testbit128(m128 val, unsigned int n) {
+ const m128 mask = mask1bit128(n);
+ return isnonzero128(and128(mask, val));
+}
+
+// offset must be an immediate
+/*The x86 platform does not perform the lower 8 bit operation.
+If the value of imm exceeds 8 bit, a compilation error occurs.*/
+static really_inline m128 palignr(m128 a, m128 b, int count) {
+ m128 result;
+ count = count & 0xff;
+ if (likely(count < 16)) {
+ result.vect_s8 = vextq_s8(b.vect_s8, a.vect_s8, count);
+ } else if (count < 32) {
+ result.vect_s8 = vextq_s8(a.vect_s8, vdupq_n_s8(0x0), count - 16);
+ } else {
+ result.vect_s32 = vdupq_n_s32(0);
+ }
+ return result;
+}
+
+static really_inline m128 pshufb_m128(m128 a, m128 b) {
+ m128 result;
+ __asm__ __volatile__("movi v3.16b, 0x8f \n\t"
+ "and v3.16b, v3.16b, %2.16b \n\t"
+ "tbl %0.16b, {%1.16b}, v3.16b \n\t"
+ : "=w"(result)
+ : "w"(a), "w"(b)
+ : "v3");
+ return result;
+}
+
+static really_inline m256 pshufb_m256(m256 a, m256 b) {
+ m256 rv;
+ rv.lo = pshufb_m128(a.lo, b.lo);
+ rv.hi = pshufb_m128(a.hi, b.hi);
+ return rv;
+}
+
+static really_inline m128 variable_byte_shift_m128(m128 in, s32 amount) {
+ assert(amount >= -16 && amount <= 16);
+ m128 shift_mask = loadu128(vbs_mask_data + 16 - amount);
+ return pshufb_m128(in, shift_mask);
+}
+
+static really_inline m128 max_u8_m128(m128 a, m128 b) {
+ m128 result;
+ result.vect_u8 = vmaxq_u8(a.vect_u8, b.vect_u8);
+ return result;
+}
+
+static really_inline m128 min_u8_m128(m128 a, m128 b) {
+ m128 result;
+ result.vect_u8 = vminq_u8(a.vect_u8, b.vect_u8);
+ return result;
+}
+
+static really_inline m128 sadd_u8_m128(m128 a, m128 b) {
+ m128 result;
+ result.vect_u8 = vqaddq_u8(a.vect_u8, b.vect_u8);
+ return result;
+}
+
+static really_inline m128 sub_u8_m128(m128 a, m128 b) {
+ m128 result;
+ result.vect_u8 = vsubq_u8(a.vect_u8, b.vect_u8);
+ return result;
+}
+
+static really_inline m128 set64x2(int64_t hi, int64_t lo) {
+ m128 result;
+ result.vect_s64 = vsetq_lane_s64(hi, vdupq_n_s64(lo), 1);
+ return result;
+}
+
+static really_inline m128 set32x4(int i3, int i2, int i1, int i0) {
+ m128 result;
+ result.vect_s32 = vsetq_lane_s32(
+ i3, vsetq_lane_s32(i2, vsetq_lane_s32(i1, vdupq_n_s32(i0), 1), 2), 3);
+ return result;
+}
+
+/****
+ **** 256-bit Primitives
+ ****/
+
+static really_really_inline m256 lshift64_m256(m256 a, int b) {
+ m256 rv = a;
+ rv.lo = lshift64_m128(rv.lo, b);
+ rv.hi = lshift64_m128(rv.hi, b);
+ return rv;
+}
+
+static really_inline m256 rshift64_m256(m256 a, int b) {
+ m256 rv = a;
+ rv.lo = rshift64_m128(rv.lo, b);
+ rv.hi = rshift64_m128(rv.hi, b);
+ return rv;
+}
+static really_inline m256 set32x8(u32 in) {
+ m256 rv;
+ rv.lo = set16x8((u8)in);
+ rv.hi = rv.lo;
+ return rv;
+}
+
+static really_inline m256 eq256(m256 a, m256 b) {
+ m256 rv;
+ rv.lo = eq128(a.lo, b.lo);
+ rv.hi = eq128(a.hi, b.hi);
+ return rv;
+}
+
+static really_inline u32 movemask256(m256 a) {
+ u32 lo_mask = movemask128(a.lo);
+ u32 hi_mask = movemask128(a.hi);
+ return lo_mask | (hi_mask << 16);
+}
+
+static really_inline m256 set2x128(m128 a) {
+ m256 rv = {a, a};
+ return rv;
+}
+
+static really_inline m256 zeroes256(void) {
+ m256 rv = {zeroes128(), zeroes128()};
+ return rv;
+}
+
+static really_inline m256 ones256(void) {
+ m256 rv = {ones128(), ones128()};
+ return rv;
+}
+
+static really_inline m256 and256(m256 a, m256 b) {
+ m256 rv;
+ rv.lo = and128(a.lo, b.lo);
+ rv.hi = and128(a.hi, b.hi);
+ return rv;
+}
+
+static really_inline m256 or256(m256 a, m256 b) {
+ m256 rv;
+ rv.lo = or128(a.lo, b.lo);
+ rv.hi = or128(a.hi, b.hi);
+ return rv;
+}
+
+static really_inline m256 xor256(m256 a, m256 b) {
+ m256 rv;
+ rv.lo = xor128(a.lo, b.lo);
+ rv.hi = xor128(a.hi, b.hi);
+ return rv;
+}
+
+static really_inline m256 not256(m256 a) {
+ m256 rv;
+ rv.lo = not128(a.lo);
+ rv.hi = not128(a.hi);
+ return rv;
+}
+
+static really_inline m256 andnot256(m256 a, m256 b) {
+ m256 rv;
+ rv.lo = andnot128(a.lo, b.lo);
+ rv.hi = andnot128(a.hi, b.hi);
+ return rv;
+}
+
+static really_inline int diff256(m256 a, m256 b) {
+ return diff128(a.lo, b.lo) || diff128(a.hi, b.hi);
+}
+
+static really_inline int isnonzero256(m256 a) {
+ return isnonzero128(or128(a.lo, a.hi));
+}
+
+/**
+ * "Rich" version of diff256(). Takes two vectors a and b and returns an 8-bit
+ * mask indicating which 32-bit words contain differences.
+ */
+static really_inline u32 diffrich256(m256 a, m256 b) {
+ uint32x4_t x = vceqq_s32(a.lo.vect_s32, b.lo.vect_s32);
+ uint32x4_t y = vceqq_s32(a.hi.vect_s32, b.hi.vect_s32);
+ uint8x8_t lo = vqmovn_u16(vcombine_u16(vqmovn_u32(x), vqmovn_u32(y)));
+
+ static const int8_t __attribute__((aligned(16)))
+ xr[8] = {-7, -6, -5, -4, -3, -2, -1, 0};
+ uint8x8_t mask_and = vdup_n_u8(0x80);
+ int8x8_t mask_shift = vld1_s8(xr);
+
+ lo = vand_u8(lo, mask_and);
+ lo = vshl_u8(lo, mask_shift);
+
+ lo = vpadd_u8(lo, lo);
+ lo = vpadd_u8(lo, lo);
+ lo = vpadd_u8(lo, lo);
+
+ return ~(lo[0] & 0xFF) & 0xff;
+}
+
+/**
+ * "Rich" version of diff256(), 64-bit variant. Takes two vectors a and b and
+ * returns an 8-bit mask indicating which 64-bit words contain differences.
+ */
+static really_inline u32 diffrich64_256(m256 a, m256 b) {
+ u32 d = diffrich256(a, b);
+ return (d | (d >> 1)) & 0x55555555;
+}
+
+// aligned load
+static really_inline m256 load256(const void *ptr) {
+ assert(ISALIGNED_N(ptr, alignof(m256)));
+ m256 rv = {load128(ptr), load128((const char *)ptr + 16)};
+ return rv;
+}
+
+// aligned load of 128-bit value to low and high part of 256-bit value
+static really_inline m256 load2x128(const void *ptr) {
+ assert(ISALIGNED_N(ptr, alignof(m128)));
+ m256 rv;
+ rv.hi = rv.lo = load128(ptr);
+ return rv;
+}
+
+static really_inline m256 loadu2x128(const void *ptr) {
+ return set2x128(loadu128(ptr));
+}
+
+// aligned store
+static really_inline void store256(void *ptr, m256 a) {
+ assert(ISALIGNED_N(ptr, alignof(m256)));
+ ptr = assume_aligned(ptr, 16);
+ *(m256 *)ptr = a;
+}
+
+// unaligned load
+static really_inline m256 loadu256(const void *ptr) {
+ m256 rv = {loadu128(ptr), loadu128((const char *)ptr + 16)};
+ return rv;
+}
+
+// unaligned store
+static really_inline void storeu256(void *ptr, m256 a) {
+ storeu128(ptr, a.lo);
+ storeu128((char *)ptr + 16, a.hi);
+}
+
+// packed unaligned store of first N bytes
+static really_inline void storebytes256(void *ptr, m256 a, unsigned int n) {
+ assert(n <= sizeof(a));
+ memcpy(ptr, &a, n);
+}
+
+// packed unaligned load of first N bytes, pad with zero
+static really_inline m256 loadbytes256(const void *ptr, unsigned int n) {
+ m256 a = zeroes256();
+ assert(n <= sizeof(a));
+ memcpy(&a, ptr, n);
+ return a;
+}
+
+static really_inline m256 mask1bit256(unsigned int n) {
+ assert(n < sizeof(m256) * 8);
+ u32 mask_idx = ((n % 8) * 64) + 95;
+ mask_idx -= n / 8;
+ return loadu256(&simd_onebit_masks[mask_idx]);
+}
+
+static really_inline m256 set64x4(u64a hi_1, u64a hi_0, u64a lo_1, u64a lo_0) {
+ m256 rv;
+ rv.hi = set64x2(hi_1, hi_0);
+ rv.lo = set64x2(lo_1, lo_0);
+ return rv;
+}
+
+// switches on bit N in the given vector.
+static really_inline void setbit256(m256 *ptr, unsigned int n) {
+ assert(n < sizeof(*ptr) * 8);
+ m128 *sub;
+ if (n < 128) {
+ sub = &ptr->lo;
+ } else {
+ sub = &ptr->hi;
+ n -= 128;
+ }
+ setbit128(sub, n);
+}
+
+// switches off bit N in the given vector.
+static really_inline void clearbit256(m256 *ptr, unsigned int n) {
+ assert(n < sizeof(*ptr) * 8);
+ m128 *sub;
+ if (n < 128) {
+ sub = &ptr->lo;
+ } else {
+ sub = &ptr->hi;
+ n -= 128;
+ }
+ clearbit128(sub, n);
+}
+
+// tests bit N in the given vector.
+static really_inline char testbit256(m256 val, unsigned int n) {
+ assert(n < sizeof(val) * 8);
+ m128 sub;
+ if (n < 128) {
+ sub = val.lo;
+ } else {
+ sub = val.hi;
+ n -= 128;
+ }
+ return testbit128(sub, n);
+}
+
+static really_really_inline m128 movdq_hi(m256 x) { return x.hi; }
+
+static really_really_inline m128 movdq_lo(m256 x) { return x.lo; }
+
+static really_inline m256 combine2x128(m128 hi, m128 lo) {
+ m256 rv = {lo, hi};
+ return rv;
+}
+
+/****
+ **** 384-bit Primitives
+ ****/
+
+static really_inline m384 and384(m384 a, m384 b) {
+ m384 rv;
+ rv.lo = and128(a.lo, b.lo);
+ rv.mid = and128(a.mid, b.mid);
+ rv.hi = and128(a.hi, b.hi);
+ return rv;
+}
+
+static really_inline m384 or384(m384 a, m384 b) {
+ m384 rv;
+ rv.lo = or128(a.lo, b.lo);
+ rv.mid = or128(a.mid, b.mid);
+ rv.hi = or128(a.hi, b.hi);
+ return rv;
+}
+
+static really_inline m384 xor384(m384 a, m384 b) {
+ m384 rv;
+ rv.lo = xor128(a.lo, b.lo);
+ rv.mid = xor128(a.mid, b.mid);
+ rv.hi = xor128(a.hi, b.hi);
+ return rv;
+}
+static really_inline m384 not384(m384 a) {
+ m384 rv;
+ rv.lo = not128(a.lo);
+ rv.mid = not128(a.mid);
+ rv.hi = not128(a.hi);
+ return rv;
+}
+static really_inline m384 andnot384(m384 a, m384 b) {
+ m384 rv;
+ rv.lo = andnot128(a.lo, b.lo);
+ rv.mid = andnot128(a.mid, b.mid);
+ rv.hi = andnot128(a.hi, b.hi);
+ return rv;
+}
+
+static really_really_inline m384 lshift64_m384(m384 a, unsigned b) {
+ m384 rv;
+ rv.lo = lshift64_m128(a.lo, b);
+ rv.mid = lshift64_m128(a.mid, b);
+ rv.hi = lshift64_m128(a.hi, b);
+ return rv;
+}
+
+static really_inline m384 zeroes384(void) {
+ m384 rv = {zeroes128(), zeroes128(), zeroes128()};
+ return rv;
+}
+
+static really_inline m384 ones384(void) {
+ m384 rv = {ones128(), ones128(), ones128()};
+ return rv;
+}
+
+static really_inline int diff384(m384 a, m384 b) {
+ return diff128(a.lo, b.lo) || diff128(a.mid, b.mid) || diff128(a.hi, b.hi);
+}
+
+static really_inline int isnonzero384(m384 a) {
+ return isnonzero128(or128(or128(a.lo, a.mid), a.hi));
+}
+
+/**
+ * "Rich" version of diff384(). Takes two vectors a and b and returns a 12-bit
+ * mask indicating which 32-bit words contain differences.
+ */
+static really_inline u32 diffrich384(m384 a, m384 b) {
+ m128 z = zeroes128();
+ uint32x4_t x = vceqq_s32(a.lo.vect_s32, b.lo.vect_s32);
+ uint32x4_t y = vceqq_s32(a.mid.vect_s32, b.mid.vect_s32);
+ uint32x4_t w = vceqq_s32(a.hi.vect_s32, b.hi.vect_s32);
+
+ uint16x8_t q = vcombine_u16(vqmovn_u32(x), vqmovn_u32(y));
+ uint16x8_t p = vcombine_u16(vqmovn_u32(w), vqmovn_u32(z.vect_u32));
+
+ uint8x16_t input = vcombine_u8(vqmovn_u16(q), vqmovn_u16(p));
+
+ static const int8_t __attribute__((aligned(16)))
+ xr[8] = {-7, -6, -5, -4, -3, -2, -1, 0};
+ uint8x8_t mask_and = vdup_n_u8(0x80);
+ int8x8_t mask_shift = vld1_s8(xr);
+
+ uint8x8_t lo = vget_low_u8(input);
+ uint8x8_t hi = vget_high_u8(input);
+
+ lo = vand_u8(lo, mask_and);
+ lo = vshl_u8(lo, mask_shift);
+
+ hi = vand_u8(hi, mask_and);
+ hi = vshl_u8(hi, mask_shift);
+
+ lo = vpadd_u8(lo, lo);
+ lo = vpadd_u8(lo, lo);
+ lo = vpadd_u8(lo, lo);
+
+ hi = vpadd_u8(hi, hi);
+ hi = vpadd_u8(hi, hi);
+ hi = vpadd_u8(hi, hi);
+
+ return ~((hi[0] << 8) | (lo[0] & 0xFF)) & 0xfff;
+}
+
+/**
+ * "Rich" version of diff384(), 64-bit variant. Takes two vectors a and b and
+ * returns a 12-bit mask indicating which 64-bit words contain differences.
+ */
+static really_inline u32 diffrich64_384(m384 a, m384 b) {
+ u32 d = diffrich384(a, b);
+ return (d | (d >> 1)) & 0x55555555;
+}
+
+// aligned load
+static really_inline m384 load384(const void *ptr) {
+ assert(ISALIGNED_16(ptr));
+ m384 rv = {load128(ptr), load128((const char *)ptr + 16),
+ load128((const char *)ptr + 32)};
+ return rv;
+}
+
+// aligned store
+static really_inline void store384(void *ptr, m384 a) {
+ assert(ISALIGNED_16(ptr));
+ ptr = assume_aligned(ptr, 16);
+ *(m384 *)ptr = a;
+}
+
+// unaligned load
+static really_inline m384 loadu384(const void *ptr) {
+ m384 rv = {loadu128(ptr), loadu128((const char *)ptr + 16),
+ loadu128((const char *)ptr + 32)};
+ return rv;
+}
+
+// packed unaligned store of first N bytes
+static really_inline void storebytes384(void *ptr, m384 a, unsigned int n) {
+ assert(n <= sizeof(a));
+ memcpy(ptr, &a, n);
+}
+
+// packed unaligned load of first N bytes, pad with zero
+static really_inline m384 loadbytes384(const void *ptr, unsigned int n) {
+ m384 a = zeroes384();
+ assert(n <= sizeof(a));
+ memcpy(&a, ptr, n);
+ return a;
+}
+
+// switches on bit N in the given vector.
+static really_inline void setbit384(m384 *ptr, unsigned int n) {
+ assert(n < sizeof(*ptr) * 8);
+ m128 *sub;
+ if (n < 128) {
+ sub = &ptr->lo;
+ } else if (n < 256) {
+ sub = &ptr->mid;
+ } else {
+ sub = &ptr->hi;
+ }
+ setbit128(sub, n % 128);
+}
+
+// switches off bit N in the given vector.
+static really_inline void clearbit384(m384 *ptr, unsigned int n) {
+ assert(n < sizeof(*ptr) * 8);
+ m128 *sub;
+ if (n < 128) {
+ sub = &ptr->lo;
+ } else if (n < 256) {
+ sub = &ptr->mid;
+ } else {
+ sub = &ptr->hi;
+ }
+ clearbit128(sub, n % 128);
+}
+
+// tests bit N in the given vector.
+static really_inline char testbit384(m384 val, unsigned int n) {
+ assert(n < sizeof(val) * 8);
+ m128 sub;
+ if (n < 128) {
+ sub = val.lo;
+ } else if (n < 256) {
+ sub = val.mid;
+ } else {
+ sub = val.hi;
+ }
+ return testbit128(sub, n % 128);
+}
+
+/****
+ **** 512-bit Primitives
+ ****/
+
+static really_inline m512 zeroes512(void) {
+ m512 rv = {zeroes256(), zeroes256()};
+ return rv;
+}
+
+static really_inline m512 ones512(void) {
+ m512 rv = {ones256(), ones256()};
+ return rv;
+}
+
+static really_inline m512 and512(m512 a, m512 b) {
+ m512 rv;
+ rv.lo = and256(a.lo, b.lo);
+ rv.hi = and256(a.hi, b.hi);
+ return rv;
+}
+
+static really_inline m512 or512(m512 a, m512 b) {
+ m512 rv;
+ rv.lo = or256(a.lo, b.lo);
+ rv.hi = or256(a.hi, b.hi);
+ return rv;
+}
+
+static really_inline m512 xor512(m512 a, m512 b) {
+ m512 rv;
+ rv.lo = xor256(a.lo, b.lo);
+ rv.hi = xor256(a.hi, b.hi);
+ return rv;
+}
+
+static really_inline m512 not512(m512 a) {
+ m512 rv;
+ rv.lo = not256(a.lo);
+ rv.hi = not256(a.hi);
+ return rv;
+}
+
+static really_inline m512 andnot512(m512 a, m512 b) {
+ m512 rv;
+ rv.lo = andnot256(a.lo, b.lo);
+ rv.hi = andnot256(a.hi, b.hi);
+ return rv;
+}
+
+static really_really_inline m512 lshift64_m512(m512 a, unsigned b) {
+ m512 rv;
+ rv.lo = lshift64_m256(a.lo, b);
+ rv.hi = lshift64_m256(a.hi, b);
+ return rv;
+}
+
+static really_inline int diff512(m512 a, m512 b) {
+ return diff256(a.lo, b.lo) || diff256(a.hi, b.hi);
+}
+
+static really_inline int isnonzero512(m512 a) {
+ m128 x = or128(a.lo.lo, a.lo.hi);
+ m128 y = or128(a.hi.lo, a.hi.hi);
+ return isnonzero128(or128(x, y));
+}
+
+/**
+ * "Rich" version of diff512(). Takes two vectors a and b and returns a 16-bit
+ * mask indicating which 32-bit words contain differences.
+ */
+static really_inline u32 diffrich512(m512 a, m512 b) {
+ uint32x4_t x = vceqq_s32(a.lo.lo.vect_s32, b.lo.lo.vect_s32);
+ uint32x4_t y = vceqq_s32(a.lo.hi.vect_s32, b.lo.hi.vect_s32);
+ uint32x4_t z = vceqq_s32(a.hi.lo.vect_s32, b.hi.lo.vect_s32);
+ uint32x4_t w = vceqq_s32(a.hi.hi.vect_s32, b.hi.hi.vect_s32);
+ uint16x8_t p = vcombine_u16(vqmovn_u32(x), vqmovn_u32(y));
+ uint16x8_t q = vcombine_u16(vqmovn_u32(z), vqmovn_u32(w));
+
+ uint8x16_t input = vcombine_u8(vqmovn_u16(p), vqmovn_u16(q));
+
+ static const int8_t __attribute__((aligned(16)))
+ xr[8] = {-7, -6, -5, -4, -3, -2, -1, 0};
+ uint8x8_t mask_and = vdup_n_u8(0x80);
+ int8x8_t mask_shift = vld1_s8(xr);
+
+ uint8x8_t lo = vget_low_u8(input);
+ uint8x8_t hi = vget_high_u8(input);
+
+ lo = vand_u8(lo, mask_and);
+ lo = vshl_u8(lo, mask_shift);
+
+ hi = vand_u8(hi, mask_and);
+ hi = vshl_u8(hi, mask_shift);
+
+ lo = vpadd_u8(lo, lo);
+ lo = vpadd_u8(lo, lo);
+ lo = vpadd_u8(lo, lo);
+
+ hi = vpadd_u8(hi, hi);
+ hi = vpadd_u8(hi, hi);
+ hi = vpadd_u8(hi, hi);
+
+ return ~((hi[0] << 8) | (lo[0] & 0xFF)) & 0xffff;
+}
+
+/**
+ * "Rich" version of diffrich(), 64-bit variant. Takes two vectors a and b and
+ * returns a 16-bit mask indicating which 64-bit words contain differences.
+ */
+static really_inline u32 diffrich64_512(m512 a, m512 b) {
+ u32 d = diffrich512(a, b);
+ return (d | (d >> 1)) & 0x55555555;
+}
+
+// aligned load
+static really_inline m512 load512(const void *ptr) {
+ assert(ISALIGNED_N(ptr, alignof(m256)));
+ m512 rv = {load256(ptr), load256((const char *)ptr + 32)};
+ return rv;
+}
+
+// aligned store
+static really_inline void store512(void *ptr, m512 a) {
+ assert(ISALIGNED_N(ptr, alignof(m512)));
+ ptr = assume_aligned(ptr, 16);
+ *(m512 *)ptr = a;
+}
+
+// unaligned load
+static really_inline m512 loadu512(const void *ptr) {
+ m512 rv = {loadu256(ptr), loadu256((const char *)ptr + 32)};
+ return rv;
+}
+
+// packed unaligned store of first N bytes
+static really_inline void storebytes512(void *ptr, m512 a, unsigned int n) {
+ assert(n <= sizeof(a));
+ memcpy(ptr, &a, n);
+}
+
+// packed unaligned load of first N bytes, pad with zero
+static really_inline m512 loadbytes512(const void *ptr, unsigned int n) {
+ m512 a = zeroes512();
+ assert(n <= sizeof(a));
+ memcpy(&a, ptr, n);
+ return a;
+}
+
+static really_inline m512 mask1bit512(unsigned int n) {
+ assert(n < sizeof(m512) * 8);
+ u32 mask_idx = ((n % 8) * 64) + 95;
+ mask_idx -= n / 8;
+ return loadu512(&simd_onebit_masks[mask_idx]);
+}
+
+// switches on bit N in the given vector.
+static really_inline void setbit512(m512 *ptr, unsigned int n) {
+ assert(n < sizeof(*ptr) * 8);
+ m128 *sub;
+ if (n < 128) {
+ sub = &ptr->lo.lo;
+ } else if (n < 256) {
+ sub = &ptr->lo.hi;
+ } else if (n < 384) {
+ sub = &ptr->hi.lo;
+ } else {
+ sub = &ptr->hi.hi;
+ }
+ setbit128(sub, n % 128);
+}
+
+// switches off bit N in the given vector.
+static really_inline void clearbit512(m512 *ptr, unsigned int n) {
+ assert(n < sizeof(*ptr) * 8);
+ m128 *sub;
+ if (n < 128) {
+ sub = &ptr->lo.lo;
+ } else if (n < 256) {
+ sub = &ptr->lo.hi;
+ } else if (n < 384) {
+ sub = &ptr->hi.lo;
+ } else {
+ sub = &ptr->hi.hi;
+ }
+ clearbit128(sub, n % 128);
+}
+
+// tests bit N in the given vector.
+static really_inline char testbit512(m512 val, unsigned int n) {
+ assert(n < sizeof(val) * 8);
+ m128 sub;
+ if (n < 128) {
+ sub = val.lo.lo;
+ } else if (n < 256) {
+ sub = val.lo.hi;
+ } else if (n < 384) {
+ sub = val.hi.lo;
+ } else {
+ sub = val.hi.hi;
+ }
+ return testbit128(sub, n % 128);
+}
+#pragma GCC diagnostic pop
+
+#endif
diff --git a/src/util/simd_types.h b/src/util/simd_types.h
index 962cad6..b3f96ea 100644
--- a/src/util/simd_types.h
+++ b/src/util/simd_types.h
@@ -35,6 +35,23 @@
#include "ue2common.h"
#if defined(HAVE_SSE2)
+typedef __m128i m128;
+#elif defined(HAVE_NEON)
+#include "arm_neon.h"
+
+typedef union {
+ int8x16_t vect_s8;
+ int16x8_t vect_s16;
+ int32x4_t vect_s32;
+ int64x2_t vect_s64;
+ uint8x16_t vect_u8;
+ uint16x8_t vect_u16;
+ uint32x4_t vect_u32;
+ uint64x2_t vect_u64;
+} __m128i;
+typedef float32x4_t __m128;
+typedef float64x2_t __m128d;
+
typedef __m128i m128;
#else
typedef struct ALIGN_DIRECTIVE {u64a hi; u64a lo;} m128;
diff --git a/src/util/simd_utils.h b/src/util/simd_utils.h
new file mode 100644
index 0000000..9588d97
--- /dev/null
+++ b/src/util/simd_utils.h
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0-only
+// Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+
+#ifndef SIMD_UTILS
+#define SIMD_UTILS
+
+#if defined(__x86_64__)
+#include "simd_x86.h"
+#elif defined(__aarch64__)
+#include "simd_arm.h"
+#endif
+
+#endif
diff --git a/src/util/simd_x86.h b/src/util/simd_x86.h
index 5fa727e..5daaa74 100644
--- a/src/util/simd_x86.h
+++ b/src/util/simd_x86.h
@@ -1417,4 +1417,14 @@ char testbit512(m512 val, unsigned int n) {
#endif
}
+static really_inline m128 set2x64(u64a c)
+{
+ return _mm_set1_epi32(c);
+}
+
+static really_inline m128 set32x4(int i3, int i2, int i1, int i0)
+{
+ return _mm_set_epi32(i3, i2, i1, i0);
+}
+
#endif
diff --git a/tools/hscollider/CMakeLists.txt b/tools/hscollider/CMakeLists.txt
index a4d71b2..0c41ab9 100644
--- a/tools/hscollider/CMakeLists.txt
+++ b/tools/hscollider/CMakeLists.txt
@@ -21,7 +21,14 @@ set_source_files_properties(
PROPERTIES
COMPILE_FLAGS "${RAGEL_C_FLAGS} -I${CMAKE_CURRENT_SOURCE_DIR}")
-ragelmaker(ColliderCorporaParser.rl)
+
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i386")
+ ragelmaker(ColliderCorporaParser.rl)
+endif()
+
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+ ragelcopyer(ColliderCorporaParser.rl)
+endif()
if (BUILD_CHIMERA)
add_definitions(-DHS_HYBRID)
diff --git a/tools/hscollider/ColliderCorporaParser.cpp b/tools/hscollider/ColliderCorporaParser.cpp
new file mode 100644
index 0000000..5391473
--- /dev/null
+++ b/tools/hscollider/ColliderCorporaParser.cpp
@@ -0,0 +1,474 @@
+
+
+/*
+ * Copyright (c) 2015-2017, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Intel Corporation nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include "ColliderCorporaParser.h"
+#include "Corpora.h"
+
+#include "ue2common.h"
+
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+
+using namespace std;
+
+namespace /* anonymous */ {
+
+// Take a string like '\xFF' and convert it to the character it represents
+char unhex(const char *start, UNUSED const char *end) {
+ assert(start + 4 == end);
+ assert(start[0] == '\\');
+ assert(start[1] == 'x');
+ assert(isxdigit(start[2]));
+ assert(isxdigit(start[2]));
+
+ char temp[3] = {start[2], start[3], 0};
+
+ return strtol(temp, nullptr, 16);
+}
+
+static const char _FileCorporaParser_actions[] = {
+ 0, 1, 0, 1, 3, 1, 4, 1, 5, 1, 6, 1, 7, 1, 8, 1, 9, 1, 10,
+ 1, 11, 1, 12, 1, 13, 1, 14, 1, 15, 1, 16, 1, 17, 1, 18, 1, 19, 1,
+ 20, 1, 21, 1, 22, 1, 23, 1, 24, 2, 0, 2, 2, 3, 0, 3, 1, 0, 2};
+
+static const char _FileCorporaParser_key_offsets[] = {
+ 0, 0, 2, 6, 7, 13, 19, 25, 31, 34, 34, 35, 52, 54, 71, 72, 75, 79};
+
+static const char _FileCorporaParser_trans_keys[] = {
+ 48, 57, 58, 61, 48, 57, 34, 48, 57, 65, 70, 97, 102, 48,
+ 57, 65, 70, 97, 102, 48, 57, 65, 70, 97, 102, 48, 57, 65,
+ 70, 97, 102, 32, 48, 57, 92, 48, 97, 110, 114, 116, 118, 120,
+ 49, 57, 65, 90, 98, 100, 101, 102, 103, 122, 34, 92, 48, 97,
+ 110, 114, 116, 118, 120, 49, 57, 65, 90, 98, 100, 101, 102, 103,
+ 122, 58, 32, 48, 57, 32, 44, 48, 57, 32, 44, 0};
+
+static const char _FileCorporaParser_single_lengths[] = {
+ 0, 0, 2, 1, 0, 0, 0, 0, 1, 0, 1, 7, 2, 7, 1, 1, 2, 2};
+
+static const char _FileCorporaParser_range_lengths[] = {
+ 0, 1, 1, 0, 3, 3, 3, 3, 1, 0, 0, 5, 0, 5, 0, 1, 1, 0};
+
+static const char _FileCorporaParser_index_offsets[] = {
+ 0, 0, 2, 6, 8, 12, 16, 20, 24, 27, 28, 30, 43, 46, 59, 61, 64, 68};
+
+static const char _FileCorporaParser_indicies[] = {
+ 0, 1, 3, 4, 2, 1, 5, 1, 7, 7, 7, 6, 8, 8, 8, 6, 10, 10,
+ 10, 9, 11, 11, 11, 9, 12, 13, 1, 1, 15, 14, 18, 18, 18, 18, 18, 18,
+ 19, 16, 16, 16, 18, 16, 17, 21, 22, 20, 25, 25, 25, 25, 25, 25, 26, 23,
+ 23, 23, 25, 23, 24, 27, 1, 28, 29, 1, 31, 32, 13, 30, 31, 32, 30, 0};
+
+static const char _FileCorporaParser_trans_targs[] = {
+ 2, 0, 2, 9, 3, 9, 10, 5, 10, 12, 7, 12, 8, 16, 10, 11, 10,
+ 10, 10, 4, 12, 12, 13, 12, 12, 12, 6, 14, 8, 16, 15, 17, 15};
+
+static const char _FileCorporaParser_trans_actions[] = {
+ 53, 0, 47, 5, 0, 7, 25, 0, 15, 39, 0, 27, 0, 1, 21, 13, 23,
+ 19, 17, 0, 33, 35, 13, 37, 31, 29, 0, 41, 3, 50, 45, 0, 43};
+
+static const char _FileCorporaParser_to_state_actions[] = {
+ 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 9, 0, 9, 9, 0, 0};
+
+static const char _FileCorporaParser_from_state_actions[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 11, 0, 11, 11, 0, 0};
+
+static const char _FileCorporaParser_eof_trans[] = {
+ 0, 0, 0, 0, 7, 7, 10, 10, 0, 0, 0, 17, 0, 24, 0, 0, 31, 31};
+
+static const int FileCorporaParser_start = 1;
+static const int FileCorporaParser_first_final = 9;
+static const int FileCorporaParser_error = 0;
+
+static const int FileCorporaParser_en_corpus_old = 10;
+static const int FileCorporaParser_en_corpus_new = 12;
+static const int FileCorporaParser_en_colon_sep = 14;
+static const int FileCorporaParser_en_match_list = 15;
+static const int FileCorporaParser_en_main = 1;
+
+} // namespace
+
+bool parseCorpus(const string &line, Corpus &c, unsigned int &id) {
+ const char *p = line.c_str();
+ const char *pe = p + line.size();
+ const char *eof = pe;
+ const char *ts;
+ const char *te;
+ int cs;
+ UNUSED int act;
+
+ // For storing integers as they're scanned
+ unsigned int num = 0;
+
+ string &sout = c.data;
+
+ {
+ cs = FileCorporaParser_start;
+ ts = 0;
+ te = 0;
+ act = 0;
+ }
+
+ {
+ int _klen;
+ unsigned int _trans;
+ const char *_acts;
+ unsigned int _nacts;
+ const char *_keys;
+
+ if (p == pe)
+ goto _test_eof;
+ if (cs == 0)
+ goto _out;
+ _resume:
+ _acts = _FileCorporaParser_actions +
+ _FileCorporaParser_from_state_actions[cs];
+ _nacts = (unsigned int)*_acts++;
+ while (_nacts-- > 0) {
+ switch (*_acts++) {
+ case 7:
+
+ {
+ ts = p;
+ } break;
+ }
+ }
+
+ _keys =
+ _FileCorporaParser_trans_keys + _FileCorporaParser_key_offsets[cs];
+ _trans = _FileCorporaParser_index_offsets[cs];
+
+ _klen = _FileCorporaParser_single_lengths[cs];
+ if (_klen > 0) {
+ const char *_lower = _keys;
+ const char *_mid;
+ const char *_upper = _keys + _klen - 1;
+ while (1) {
+ if (_upper < _lower)
+ break;
+
+ _mid = _lower + ((_upper - _lower) >> 1);
+ if ((*p) < *_mid)
+ _upper = _mid - 1;
+ else if ((*p) > *_mid)
+ _lower = _mid + 1;
+ else {
+ _trans += (unsigned int)(_mid - _keys);
+ goto _match;
+ }
+ }
+ _keys += _klen;
+ _trans += _klen;
+ }
+
+ _klen = _FileCorporaParser_range_lengths[cs];
+ if (_klen > 0) {
+ const char *_lower = _keys;
+ const char *_mid;
+ const char *_upper = _keys + (_klen << 1) - 2;
+ while (1) {
+ if (_upper < _lower)
+ break;
+
+ _mid = _lower + (((_upper - _lower) >> 1) & ~1);
+ if ((*p) < _mid[0])
+ _upper = _mid - 2;
+ else if ((*p) > _mid[1])
+ _lower = _mid + 2;
+ else {
+ _trans += (unsigned int)((_mid - _keys) >> 1);
+ goto _match;
+ }
+ }
+ _trans += _klen;
+ }
+
+ _match:
+ _trans = _FileCorporaParser_indicies[_trans];
+ _eof_trans:
+ cs = _FileCorporaParser_trans_targs[_trans];
+
+ if (_FileCorporaParser_trans_actions[_trans] == 0)
+ goto _again;
+
+ _acts = _FileCorporaParser_actions +
+ _FileCorporaParser_trans_actions[_trans];
+ _nacts = (unsigned int)*_acts++;
+ while (_nacts-- > 0) {
+ switch (*_acts++) {
+ case 0:
+
+ {
+ num = (num * 10) + ((*p) - '0');
+ } break;
+ case 1:
+
+ {
+ num = 0;
+ } break;
+ case 2:
+
+ {
+ id = num;
+ } break;
+ case 3:
+
+ {
+ num = 0;
+ } break;
+ case 4:
+
+ {
+ {
+ cs = 10;
+ goto _again;
+ }
+ } break;
+ case 5:
+
+ {
+ c.hasMatches = true;
+ {
+ cs = 12;
+ goto _again;
+ }
+ } break;
+ case 8:
+
+ {
+ te = p + 1;
+ } break;
+ case 9:
+
+ {
+ te = p + 1;
+ { sout.push_back(unhex(ts, te)); }
+ } break;
+ case 10:
+
+ {
+ te = p + 1;
+ {
+ switch (*(ts + 1)) {
+ case '0':
+ sout.push_back('\x00');
+ break;
+ case 'a':
+ sout.push_back('\x07');
+ break;
+ case 'e':
+ sout.push_back('\x1b');
+ break;
+ case 'f':
+ sout.push_back('\x0c');
+ break;
+ case 'n':
+ sout.push_back('\x0a');
+ break;
+ case 'v':
+ sout.push_back('\x0b');
+ break;
+ case 'r':
+ sout.push_back('\x0d');
+ break;
+ case 't':
+ sout.push_back('\x09');
+ break;
+ default: {
+ p++;
+ goto _out;
+ }
+ }
+ }
+ } break;
+ case 11:
+
+ {
+ te = p + 1;
+ { sout.push_back(*(ts + 1)); }
+ } break;
+ case 12:
+
+ {
+ te = p + 1;
+ { sout.push_back(*ts); }
+ } break;
+ case 13:
+
+ {
+ te = p;
+ p--;
+ { sout.push_back(*ts); }
+ } break;
+ case 14:
+
+ {
+ { p = ((te)) - 1; }
+ { sout.push_back(*ts); }
+ } break;
+ case 15:
+
+ {
+ te = p + 1;
+ { sout.push_back(unhex(ts, te)); }
+ } break;
+ case 16:
+
+ {
+ te = p + 1;
+ {
+ switch (*(ts + 1)) {
+ case '0':
+ sout.push_back('\x00');
+ break;
+ case 'a':
+ sout.push_back('\x07');
+ break;
+ case 'e':
+ sout.push_back('\x1b');
+ break;
+ case 'f':
+ sout.push_back('\x0c');
+ break;
+ case 'n':
+ sout.push_back('\x0a');
+ break;
+ case 'v':
+ sout.push_back('\x0b');
+ break;
+ case 'r':
+ sout.push_back('\x0d');
+ break;
+ case 't':
+ sout.push_back('\x09');
+ break;
+ default: {
+ p++;
+ goto _out;
+ }
+ }
+ }
+ } break;
+ case 17:
+
+ {
+ te = p + 1;
+ { sout.push_back(*(ts + 1)); }
+ } break;
+ case 18:
+
+ {
+ te = p + 1;
+ { sout.push_back(*ts); }
+ } break;
+ case 19:
+
+ {
+ te = p + 1;
+ {
+ {
+ cs = 14;
+ goto _again;
+ }
+ }
+ } break;
+ case 20:
+
+ {
+ te = p;
+ p--;
+ { sout.push_back(*ts); }
+ } break;
+ case 21:
+
+ {
+ { p = ((te)) - 1; }
+ { sout.push_back(*ts); }
+ } break;
+ case 22:
+
+ {
+ te = p + 1;
+ {
+ {
+ cs = 15;
+ goto _again;
+ }
+ }
+ } break;
+ case 23:
+
+ {
+ te = p + 1;
+ { c.matches.insert(num); }
+ } break;
+ case 24:
+
+ {
+ te = p;
+ p--;
+ { c.matches.insert(num); }
+ } break;
+ }
+ }
+
+ _again:
+ _acts = _FileCorporaParser_actions +
+ _FileCorporaParser_to_state_actions[cs];
+ _nacts = (unsigned int)*_acts++;
+ while (_nacts-- > 0) {
+ switch (*_acts++) {
+ case 6:
+
+ {
+ ts = 0;
+ } break;
+ }
+ }
+
+ if (cs == 0)
+ goto _out;
+ if (++p != pe)
+ goto _resume;
+ _test_eof : {}
+ if (p == eof) {
+ if (_FileCorporaParser_eof_trans[cs] > 0) {
+ _trans = _FileCorporaParser_eof_trans[cs] - 1;
+ goto _eof_trans;
+ }
+ }
+
+ _out : {}
+ }
+
+ return (cs != FileCorporaParser_error) && (p == pe);
+}
diff --git a/unit/internal/simd_utils.cpp b/unit/internal/simd_utils.cpp
index 623c2c9..22945d6 100644
--- a/unit/internal/simd_utils.cpp
+++ b/unit/internal/simd_utils.cpp
@@ -663,7 +663,7 @@ TEST(SimdUtilsTest, movq) {
ASSERT_EQ(0, memcmp(cmp, &simd, sizeof(simd)));
ASSERT_EQ(0, memcmp(cmp, &r, sizeof(r)));
- simd = _mm_set_epi64x(~0LL, 0x123456789abcdef);
+ simd = set64x2(~0LL, 0x123456789abcdef);
r = movq(simd);
ASSERT_EQ(r, 0x123456789abcdef);
}
diff --git a/util/CMakeLists.txt b/util/CMakeLists.txt
index ea942ef..d7bef50 100644
--- a/util/CMakeLists.txt
+++ b/util/CMakeLists.txt
@@ -11,7 +11,13 @@ set_source_files_properties(
PROPERTIES
COMPILE_FLAGS "${RAGEL_C_FLAGS}")
-ragelmaker(ExpressionParser.rl)
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i386")
+ ragelmaker(ExpressionParser.rl)
+endif()
+
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+ ragelcopyer(ExpressionParser.rl)
+endif()
set(expressionutil_SRCS
expressions.cpp
diff --git a/util/ExpressionParser.cpp b/util/ExpressionParser.cpp
new file mode 100644
index 0000000..687fc39
--- /dev/null
+++ b/util/ExpressionParser.cpp
@@ -0,0 +1,397 @@
+
+
+/*
+ * Copyright (c) 2015-2018, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Intel Corporation nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include "ExpressionParser.h"
+
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+
+#include "hs_compile.h"
+#include "ue2common.h"
+
+using std::string;
+
+namespace { // anon
+
+enum ParamKey {
+ PARAM_NONE,
+ PARAM_MIN_OFFSET,
+ PARAM_MAX_OFFSET,
+ PARAM_MIN_LENGTH,
+ PARAM_EDIT_DISTANCE,
+ PARAM_HAMM_DISTANCE
+};
+
+static const char _ExpressionParser_actions[] = {0, 1, 0, 1, 1, 1, 2, 1, 3,
+ 1, 4, 1, 5, 1, 6, 1, 7, 1,
+ 9, 1, 10, 2, 8, 0
+
+};
+
+static const char _ExpressionParser_key_offsets[] = {
+ 0, 0, 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+ 20, 21, 23, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41,
+ 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57,
+ 58, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 82};
+
+static const char _ExpressionParser_trans_keys[] = {
+ 32, 101, 104, 109, 32, 101, 104, 109, 100, 105, 116, 95, 100, 105,
+ 115, 116, 97, 110, 99, 101, 61, 48, 57, 32, 44, 125, 48, 57,
+ 32, 44, 125, 97, 109, 109, 105, 110, 103, 95, 100, 105, 115, 116,
+ 97, 110, 99, 101, 97, 105, 120, 95, 111, 102, 102, 115, 101, 116,
+ 110, 95, 108, 111, 101, 110, 103, 116, 104, 102, 102, 115, 101, 116,
+ 56, 67, 72, 76, 105, 109, 115, 123, 79, 81, 86, 87, 0};
+
+static const char _ExpressionParser_single_lengths[] = {
+ 0, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 3, 3, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 8, 0};
+
+static const char _ExpressionParser_range_lengths[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0};
+
+static const unsigned char _ExpressionParser_index_offsets[] = {
+ 0, 0, 5, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32,
+ 34, 36, 38, 43, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67,
+ 69, 71, 73, 75, 77, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98,
+ 100, 103, 105, 107, 109, 111, 113, 115, 117, 119, 121, 123, 134};
+
+static const char _ExpressionParser_trans_targs[] = {
+ 2, 3, 19, 34, 0, 2, 3, 19, 34, 0, 4, 0, 5, 0, 6, 0, 7,
+ 0, 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 0,
+ 16, 0, 17, 0, 18, 1, 57, 17, 0, 18, 1, 57, 0, 20, 0, 21, 0,
+ 22, 0, 23, 0, 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30,
+ 0, 31, 0, 32, 0, 33, 0, 15, 0, 35, 43, 0, 36, 0, 37, 0, 38,
+ 0, 39, 0, 40, 0, 41, 0, 42, 0, 15, 0, 44, 0, 45, 0, 46, 51,
+ 0, 47, 0, 48, 0, 49, 0, 50, 0, 15, 0, 52, 0, 53, 0, 54, 0,
+ 55, 0, 15, 0, 56, 56, 56, 56, 56, 56, 56, 1, 56, 56, 0, 0, 0};
+
+static const char _ExpressionParser_trans_actions[] = {
+ 17, 17, 17, 17, 19, 0, 0, 0, 0, 19, 0, 19, 0, 19, 0, 19, 0,
+ 19, 0, 19, 0, 19, 0, 19, 0, 19, 0, 19, 0, 19, 0, 19, 13, 19,
+ 0, 19, 21, 19, 0, 5, 5, 1, 19, 0, 5, 5, 19, 0, 19, 0, 19,
+ 0, 19, 0, 19, 0, 19, 0, 19, 0, 19, 0, 19, 0, 19, 0, 19, 0,
+ 19, 0, 19, 0, 19, 0, 19, 15, 19, 0, 0, 19, 0, 19, 0, 19, 0,
+ 19, 0, 19, 0, 19, 0, 19, 0, 19, 9, 19, 0, 19, 0, 19, 0, 0,
+ 19, 0, 19, 0, 19, 0, 19, 0, 19, 11, 19, 0, 19, 0, 19, 0, 19,
+ 0, 19, 7, 19, 3, 3, 3, 3, 3, 3, 3, 0, 3, 3, 19, 19, 0};
+
+static const char _ExpressionParser_eof_actions[] = {
+ 0, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
+ 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 0, 0};
+
+static const int ExpressionParser_start = 56;
+static const int ExpressionParser_first_final = 56;
+static const int ExpressionParser_error = 0;
+
+static const int ExpressionParser_en_main = 56;
+
+} // namespace
+
+static void initExt(hs_expr_ext *ext) {
+ memset(ext, 0, sizeof(*ext));
+ ext->max_offset = MAX_OFFSET;
+}
+
+bool HS_CDECL readExpression(const std::string &input, std::string &expr,
+ unsigned int *flags, hs_expr_ext *ext,
+ bool *must_be_ordered) {
+ assert(flags);
+ assert(ext);
+
+ // Init flags and ext params.
+ *flags = 0;
+ initExt(ext);
+ if (must_be_ordered) {
+ *must_be_ordered = false;
+ }
+
+ // Extract expr, which is easier to do in straight C++ than with Ragel.
+ if (input.empty() || input[0] != '/') {
+ return false;
+ }
+ size_t end = input.find_last_of('/');
+ if (end == string::npos || end == 0) {
+ return false;
+ }
+ expr = input.substr(1, end - 1);
+
+ // Use a Ragel scanner to handle flags and params.
+ const char *p = input.c_str() + end + 1;
+ const char *pe = input.c_str() + input.size();
+ UNUSED const char *eof = pe;
+ UNUSED const char *ts = p, *te = p;
+ int cs;
+ UNUSED int act;
+
+ assert(p);
+ assert(pe);
+
+ // For storing integers as they're scanned.
+ u64a num = 0;
+ enum ParamKey key = PARAM_NONE;
+
+ { cs = ExpressionParser_start; }
+
+ {
+ int _klen;
+ unsigned int _trans;
+ const char *_acts;
+ unsigned int _nacts;
+ const char *_keys;
+
+ if (p == pe)
+ goto _test_eof;
+ if (cs == 0)
+ goto _out;
+ _resume:
+ _keys =
+ _ExpressionParser_trans_keys + _ExpressionParser_key_offsets[cs];
+ _trans = _ExpressionParser_index_offsets[cs];
+
+ _klen = _ExpressionParser_single_lengths[cs];
+ if (_klen > 0) {
+ const char *_lower = _keys;
+ const char *_mid;
+ const char *_upper = _keys + _klen - 1;
+ while (1) {
+ if (_upper < _lower)
+ break;
+
+ _mid = _lower + ((_upper - _lower) >> 1);
+ if ((*p) < *_mid)
+ _upper = _mid - 1;
+ else if ((*p) > *_mid)
+ _lower = _mid + 1;
+ else {
+ _trans += (unsigned int)(_mid - _keys);
+ goto _match;
+ }
+ }
+ _keys += _klen;
+ _trans += _klen;
+ }
+
+ _klen = _ExpressionParser_range_lengths[cs];
+ if (_klen > 0) {
+ const char *_lower = _keys;
+ const char *_mid;
+ const char *_upper = _keys + (_klen << 1) - 2;
+ while (1) {
+ if (_upper < _lower)
+ break;
+
+ _mid = _lower + (((_upper - _lower) >> 1) & ~1);
+ if ((*p) < _mid[0])
+ _upper = _mid - 2;
+ else if ((*p) > _mid[1])
+ _lower = _mid + 2;
+ else {
+ _trans += (unsigned int)((_mid - _keys) >> 1);
+ goto _match;
+ }
+ }
+ _trans += _klen;
+ }
+
+ _match:
+ cs = _ExpressionParser_trans_targs[_trans];
+
+ if (_ExpressionParser_trans_actions[_trans] == 0)
+ goto _again;
+
+ _acts =
+ _ExpressionParser_actions + _ExpressionParser_trans_actions[_trans];
+ _nacts = (unsigned int)*_acts++;
+ while (_nacts-- > 0) {
+ switch (*_acts++) {
+ case 0:
+
+ {
+ num = (num * 10) + ((*p) - '0');
+ } break;
+ case 1:
+
+ {
+ switch ((*p)) {
+ case 'i':
+ *flags |= HS_FLAG_CASELESS;
+ break;
+ case 's':
+ *flags |= HS_FLAG_DOTALL;
+ break;
+ case 'm':
+ *flags |= HS_FLAG_MULTILINE;
+ break;
+ case 'H':
+ *flags |= HS_FLAG_SINGLEMATCH;
+ break;
+ case 'O':
+ if (must_be_ordered) {
+ *must_be_ordered = true;
+ }
+ break;
+ case 'V':
+ *flags |= HS_FLAG_ALLOWEMPTY;
+ break;
+ case 'W':
+ *flags |= HS_FLAG_UCP;
+ break;
+ case '8':
+ *flags |= HS_FLAG_UTF8;
+ break;
+ case 'P':
+ *flags |= HS_FLAG_PREFILTER;
+ break;
+ case 'L':
+ *flags |= HS_FLAG_SOM_LEFTMOST;
+ break;
+ case 'C':
+ *flags |= HS_FLAG_COMBINATION;
+ break;
+ case 'Q':
+ *flags |= HS_FLAG_QUIET;
+ break;
+ default: {
+ p++;
+ goto _out;
+ }
+ }
+ } break;
+ case 2:
+
+ {
+ switch (key) {
+ case PARAM_MIN_OFFSET:
+ ext->flags |= HS_EXT_FLAG_MIN_OFFSET;
+ ext->min_offset = num;
+ break;
+ case PARAM_MAX_OFFSET:
+ ext->flags |= HS_EXT_FLAG_MAX_OFFSET;
+ ext->max_offset = num;
+ break;
+ case PARAM_MIN_LENGTH:
+ ext->flags |= HS_EXT_FLAG_MIN_LENGTH;
+ ext->min_length = num;
+ break;
+ case PARAM_EDIT_DISTANCE:
+ ext->flags |= HS_EXT_FLAG_EDIT_DISTANCE;
+ ext->edit_distance = num;
+ break;
+ case PARAM_HAMM_DISTANCE:
+ ext->flags |= HS_EXT_FLAG_HAMMING_DISTANCE;
+ ext->hamming_distance = num;
+ break;
+ case PARAM_NONE:
+ default:
+ // No key specified, syntax invalid.
+ return false;
+ }
+ } break;
+ case 3:
+
+ {
+ key = PARAM_MIN_OFFSET;
+ } break;
+ case 4:
+
+ {
+ key = PARAM_MAX_OFFSET;
+ } break;
+ case 5:
+
+ {
+ key = PARAM_MIN_LENGTH;
+ } break;
+ case 6:
+
+ {
+ key = PARAM_EDIT_DISTANCE;
+ } break;
+ case 7:
+
+ {
+ key = PARAM_HAMM_DISTANCE;
+ } break;
+ case 8:
+
+ {
+ num = 0;
+ } break;
+ case 9:
+
+ {
+ key = PARAM_NONE;
+ } break;
+ case 10:
+
+ {
+ return false;
+ } break;
+ }
+ }
+
+ _again:
+ if (cs == 0)
+ goto _out;
+ if (++p != pe)
+ goto _resume;
+ _test_eof : {}
+ if (p == eof) {
+ const char *__acts =
+ _ExpressionParser_actions + _ExpressionParser_eof_actions[cs];
+ unsigned int __nacts = (unsigned int)*__acts++;
+ while (__nacts-- > 0) {
+ switch (*__acts++) {
+ case 10:
+
+ {
+ return false;
+ } break;
+ }
+ }
+ }
+
+ _out : {}
+ }
+
+ DEBUG_PRINTF("expr='%s', flags=%u\n", expr.c_str(), *flags);
+
+ return (cs != ExpressionParser_error) && (p == pe);
+}
--
2.39.0
此处可能存在不合适展示的内容，页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容，可点击提交进行申诉，我们将尽快为您处理。
