From: Timo Röhling Date: Mon, 22 Sep 2025 19:05:28 +0000 (+0200) Subject: New upstream version 0.9 X-Git-Tag: archive/raspbian/0.9.4-1+rpi1~1^2~18^2 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=25ffa9a8c07736a149413c1abc2814c6ebd38631;p=libfyaml.git New upstream version 0.9 --- diff --git a/.gitignore b/.gitignore index 3f93e1b..2378768 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,9 @@ output/ sphinx/ src/fy-tool src/libfyaml-parser +src/fy-thread +src/fy-b3sum +src/*.a stamp-h1 tags TAGS diff --git a/.libtool-version b/.libtool-version index d444a18..2531082 100644 --- a/.libtool-version +++ b/.libtool-version @@ -1 +1 @@ -1:3:1 +1:4:2 diff --git a/.tarball-version b/.tarball-version index aec258d..b63ba69 100644 --- a/.tarball-version +++ b/.tarball-version @@ -1 +1 @@ -0.8 +0.9 diff --git a/CMakeLists.txt b/CMakeLists.txt index 11aaf84..a77beaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,38 +13,38 @@ cmake_minimum_required(VERSION 3.0) include(GNUInstallDirs) set(LIBHDRS - src/valgrind/fy-valgrind.h - src/xxhash/xxhash.h src/lib/fy-accel.h src/lib/fy-atom.h src/lib/fy-composer.h - src/lib/fy-ctype.h src/lib/fy-diag.h src/lib/fy-doc.h - src/lib/fy-docbuilder.c + src/lib/fy-docbuilder.h src/lib/fy-docstate.h src/lib/fy-dump.h - src/lib/fy-emit-accum.h - src/lib/fy-emit.h - src/lib/fy-event.h + src/lib/fy-emit.h src/lib/fy-emit-accum.h src/lib/fy-input.h - src/lib/fy-list.h src/lib/fy-parse.h src/lib/fy-path.h src/lib/fy-token.h - src/lib/fy-typelist.h src/lib/fy-types.h - src/lib/fy-utf8.h - src/lib/fy-utils.h src/lib/fy-walk.h + src/util/fy-blob.h + src/util/fy-ctype.h + src/util/fy-endian.h + src/util/fy-id.h + src/util/fy-list.h + src/util/fy-typelist.h + src/util/fy-utf8.h + src/util/fy-utils.h + src/xxhash/xxhash.h ) set(LIBSRCS src/lib/fy-accel.c src/lib/fy-atom.c src/lib/fy-composer.c - src/lib/fy-ctype.c src/lib/fy-diag.c src/lib/fy-doc.c + src/lib/fy-docbuilder.c src/lib/fy-docstate.c src/lib/fy-dump.c src/lib/fy-emit.c @@ -54,9 +54,11 @@ set(LIBSRCS src/lib/fy-path.c src/lib/fy-token.c src/lib/fy-types.c - src/lib/fy-utf8.c - src/lib/fy-utils.c src/lib/fy-walk.c + src/util/fy-blob.c + src/util/fy-ctype.c + src/util/fy-utf8.c + src/util/fy-utils.c src/xxhash/xxhash.c ) set(LIBHDRSPUB @@ -69,6 +71,13 @@ add_library(${PROJECT_NAME} ${LIBHDRSPUB} ) +include(CheckIncludeFile) +check_include_file("byteswap.h" HAVE_BYTESWAP_H) +include(CheckFunctionExists) +check_function_exists(__builtin_bswap16 HAVE___BUILTIN_BSWAP16) +check_function_exists(__builtin_bswap32 HAVE___BUILTIN_BSWAP32) +check_function_exists(__builtin_bswap64 HAVE___BUILTIN_BSWAP64) + add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) set_target_properties(${PROJECT_NAME} PROPERTIES FOLDER "Libraries") @@ -79,7 +88,7 @@ target_include_directories(${PROJECT_NAME} $ PRIVATE ${${PROJECT_NAME}_SOURCE_DIR}/src/lib - ${${PROJECT_NAME}_SOURCE_DIR}/src/lib/internal + ${${PROJECT_NAME}_SOURCE_DIR}/src/util ${${PROJECT_NAME}_SOURCE_DIR}/src/xxhash ) diff --git a/configure.ac b/configure.ac index 975fc19..fd3d318 100644 --- a/configure.ac +++ b/configure.ac @@ -1,14 +1,24 @@ -AC_PREREQ(2.61) +AC_PREREQ(2.69) AC_INIT([libfyaml], m4_esyscmd([build-aux/git-version-gen .tarball-version]), [pantelis.antoniou@konsulko.com]) +AC_CONFIG_AUX_DIR([build-aux]) + +AC_PATH_PROG([M4], [m4 gm4], [no]) +if test "x$M4" = xno ; then + AC_MSG_ERROR([m4 missing]) +fi + +AC_SUBST(ACLOCAL_AMFLAGS, "-I m4") + +AC_CANONICAL_TARGET + AC_CONFIG_SRCDIR([src/lib/fy-parse.c]) -AC_CONFIG_HEADER([config.h]) +AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) -AC_CONFIG_AUX_DIR([build-aux]) -AM_INIT_AUTOMAKE([foreign 1.8.5 -Wall subdir-objects ]) +AM_INIT_AUTOMAKE([foreign 1.8.5 subdir-objects -Wno-portability]) m4_pattern_allow([^(AM_EXTRA_RECURSIVE_TARGETS|AM_PROG_AR)$])dnl @@ -72,8 +82,8 @@ m4_version_prereq(2.64, [AX_CHECK_ENABLE_DEBUG()], [true]) AC_PROG_MKDIR_P AC_PROG_CC -AC_PROG_CC_C99 AM_PROG_CC_C_O +AM_PROG_AS AC_PROG_GCC_TRADITIONAL AC_USE_SYSTEM_EXTENSIONS @@ -84,28 +94,25 @@ AC_PROG_CXX AC_PROG_AWK AC_PROG_LN_S -AX_PTHREAD +AX_PTHREAD([], [AC_MSG_ERROR("Missing required pthread support")]) -AC_PATH_PROG([M4], [m4 gm4], [no]) -if test "x$M4" = xno ; then - AC_MSG_ERROR([m4 missing]) +# in some cases PTHREAD_LIBS is empty - force -lpthread */ +if test "x$PTHREAD_LIBS" = "x"; then + PTHREAD_LIBS="-lpthread" fi -AC_SUBST(ACLOCAL_AMFLAGS, "-I m4") +CFLAGS="$CFLAGS $PTHREAD_CFLAGS" +LIBS="$LIBS $PTHREAD_LIBS" # pkg-config PKG_PROG_PKG_CONFIG AC_LTDL_ENABLE_INSTALL -AC_LIBLTDL_INSTALLABLE -AC_LIBTOOL_DLOPEN -AC_LIBTOOL_WIN32_DLL -AC_PROG_LIBTOOL +LT_INIT([dlopen win32-dll]) AC_SUBST(LTDLINCL) AC_SUBST(LIBLTDL) AC_CONFIG_SUBDIRS(libltdl) -AC_HEADER_STDC AC_C_CONST AC_C_BIGENDIAN AC_TYPE_PID_T @@ -113,10 +120,12 @@ AC_TYPE_SIZE_T AC_CHECK_TYPES(ssize_t, , [AC_DEFINE([ssize_t], [signed long], [Define ssize_t if it is not done by the standard libs.])]) AC_TYPE_OFF_T -AC_TYPE_SIGNAL AC_TYPE_UID_T AC_CHECK_DECLS(environ) +AC_CHECK_HEADERS([byteswap.h]) +AC_CHECK_FUNCS([__builtin_bswap16 __builtin_bswap32 __builtin_bswap64]) + dnl for old autoconf version AX_APPEND_COMPILE_FLAGS does not work m4_version_prereq(2.64, [AX_APPEND_COMPILE_FLAGS([-Wall -Wsign-compare -Wno-stringop-overflow -fvisibility=hidden], @@ -139,11 +148,57 @@ AC_SUBST(HAVE_STATIC) AC_DEFINE_UNQUOTED([HAVE_STATIC], [$HAVE_STATIC], [Define to 1 if static linking is available]) AM_CONDITIONAL([HAVE_STATIC], [ test x$HAVE_STATIC = x1 ]) -dnl for old autoconf version AX_APPEND_COMPILE_FLAGS does not work -m4_version_prereq(2.64, - [AX_APPEND_COMPILE_FLAGS([-Wall -Wsign-compare -Wno-stringop-overflow -fvisibility=hidden], - [CFLAGS], [-pedantic -Werror])], - [CFLAGS="$CFLAGS -Wall -Wsign-compare"]) +dnl check whether warning flags are supported +AX_CHECK_COMPILE_FLAG([-Wstringop-overread], , , [-Werror]) +AS_IF([test "x$ax_cv_check_cflags__Werror__wstringop_overread" = x"yes"], + [HAVE_WSTRINGOP_OVERREAD=1], [HAVE_WSTRINGOP_OVERREAD=0]) +AC_DEFINE_UNQUOTED([HAVE_WSTRINGOP_OVERREAD], [$HAVE_WSTRINGOP_OVERREAD], [Define to 1 if -Wstringop-overread is supported]) + +dnl Per target optimizations +AC_ARG_ENABLE([portable-target], + AS_HELP_STRING([--enable-portable-target], + [Enable portable mode (disable per-target optimizations)])) +HAVE_PORTABLE_TARGET=0 +if test "x$enable_portable_target" == "xyes"; then + HAVE_PORTABLE_TARGET=1 +fi +AC_SUBST(HAVE_PORTABLE_TARGET) +AC_DEFINE_UNQUOTED([HAVE_PORTABLE_TARGET], [$HAVE_PORTABLE_TARGET], [Define to 1 if PORTABLE_TARGET is enabled]) +AM_CONDITIONAL([HAVE_PORTABLE_TARGET], [ test x$HAVE_PORTABLE_TARGET == x1 ]) + +AM_CONDITIONAL([TARGET_CPU_X86], [ test x$target_cpu = xx86 ]) +AM_CONDITIONAL([TARGET_CPU_X86_64], [ test x$target_cpu = xx86_64 ]) +AM_CONDITIONAL([TARGET_CPU_ANY_X86], [ test x$target_cpu = xx86 -o x$target_cpu = xx86_64 ]) +AM_CONDITIONAL([TARGET_CPU_ARM], [ test x$target_cpu = xarm ]) +AM_CONDITIONAL([TARGET_CPU_ARM64], [ test x$target_cpu = xaarch64 ]) +AM_CONDITIONAL([TARGET_CPU_ANY_ARM], [ test x$target_cpu = xarm -o x$target_cpu = xaarch64 ]) + +AM_COND_IF([TARGET_CPU_ANY_X86], [m4_version_prereq(2.64, [ + AX_CHECK_COMPILE_FLAG([-msse2], , , [-Werror]) + AX_CHECK_COMPILE_FLAG([-msse4.1], , , [-Werror]) + AX_CHECK_COMPILE_FLAG([-mavx2], , , [-Werror]) + AX_CHECK_COMPILE_FLAG([-mavx512f -mavx512vl], , , [-Werror]) + ], [true]) +]) + +AM_COND_IF([TARGET_CPU_ARM], [m4_version_prereq(2.64, [ + AX_CHECK_COMPILE_FLAG([-mfpu=neon], , , [-Werror]) + ], [true]) +]) + +AM_CONDITIONAL([TARGET_HAS_SSE2], [ test "x$ax_cv_check_cflags__Werror__msse2" = x"yes" -a "x$enable_portable_target" != "xyes" ]) +AM_CONDITIONAL([TARGET_HAS_SSE41], [ test "x$ax_cv_check_cflags__Werror__msse4_1" = x"yes" -a "x$enable_portable_target" != "xyes" ]) +AM_CONDITIONAL([TARGET_HAS_AVX2], [ test "x$ax_cv_check_cflags__Werror__mavx2" = x"yes" -a "x$enable_portable_target" != "xyes" ]) +AM_CONDITIONAL([TARGET_HAS_AVX512], [ test "x$ax_cv_check_cflags__Werror__mavx512f__mavx512vl" = x"yes" -a "x$enable_portable_target" != "xyes" ]) + +AM_CONDITIONAL([TARGET_HAS_NEON], [ test \( x$target_cpu = xaarch64 -o "x$ax_cv_check_cflags__Werror__mcpu_neon" = x"yes" \) -a "x$enable_portable_target" != "xyes" ]) + +AM_COND_IF([TARGET_HAS_SSE2], AC_DEFINE([TARGET_HAS_SSE2], [1], [SSE2 target support])) +AM_COND_IF([TARGET_HAS_SSE41], AC_DEFINE([TARGET_HAS_SSE41], [1], [SSE41 target support])) +AM_COND_IF([TARGET_HAS_AVX2], AC_DEFINE([TARGET_HAS_AVX2], [1], [AVX2 target support])) +AM_COND_IF([TARGET_HAS_AVX512], AC_DEFINE([TARGET_HAS_AVX512], [1], [AVX512 target support])) + +AM_COND_IF([TARGET_HAS_NEON], AC_DEFINE([TARGET_HAS_NEON], [1], [NEON target support])) dnl ASAN enable switch AC_ARG_ENABLE([asan], @@ -176,9 +231,6 @@ AM_CONDITIONAL([HAVE_ASAN], [ test x$HAVE_ASAN = x1 ]) AC_DEFINE_UNQUOTED([HAVE_ASAN], [$HAVE_ASAN], [Define to 1 if ASAN is enabled]) -# include -lm in the link -AC_SEARCH_LIBS([llrintf], [m], [], [AC_MSG_ERROR([unable to find the llrintf() function])]) - # check if there's a qsort_r available (musl does not have it) AC_CHECK_FUNC([qsort_r], HAVE_QSORT_R=1, @@ -369,6 +421,105 @@ AC_DEFINE_UNQUOTED([HAVE_JQ], [$HAVE_JQ], [Define to 1 if you have jq available] AM_CONDITIONAL([HAVE_JQ], [ test x$HAVE_JQ = x1 ]) AC_SUBST(JQ, "$JQ") +# check for libclang +AC_ARG_WITH([libclang], + dnl enable internet available when checking +AC_ARG_ENABLE([network], + AS_HELP_STRING([--disable-network], + [Disable tests requiring network access])) + + [AS_HELP_STRING([--with-libclang=[]], + [Enable support for libclang @<:@default=auto@:>@])], + [with_libclang="$withval"], + [with_libclang=auto]) + +HAVE_LIBCLANG=0 +LIBCLANG_CFLAGS="" +LIBCLANG_CPPFLAGS="" +LIBCLANG_LDFLAGS="" +LIBCLANG_LIBS="" +dnl TODO think about cross compilation +if test "x$with_libclang" != "xno"; then + if test "x$with_libclang" == "xauto"; then + # try to auto detect (llvm 15 down to 8) + check="llvm-config llvm-config-15 llvm-config-14 llvm-config-13 llvm-config-12 llvm-config-11 llvm-config-10" + else + if test "x$with_libclang" == "xlatest"; then + check="llvm-config" + else + check="llvm-config-$with_libclang" + fi + fi + orig_CFLAGS="$CFLAGS" + orig_CPPFLAGS="$CPPFLAGS" + orig_LDFLAGS="$LDFLAGS" + orig_LIBS="$LIBS" + for cfg in $check ; do + # no caching + AC_PATH_PROG([LLVM_CONFIG], [$cfg]) + if test "x$LLVM_CONFIG" != "x" ; then + LIBCLANG_CFLAGS=`$LLVM_CONFIG --cflags` + LIBCLANG_CPPFLAGS=`$LLVM_CONFIG --cppflags` + LIBCLANG_LDFLAGS=`$LLVM_CONFIG --ldflags` + LIBCLANG_LIBS="-lclang" + CPPFLAGS="$CPPFLAGS $LIBCLANG_CPPFLAGS" + CFLAGS="$CFLAGS $LIBCLANG_CFLAGS" + LDFLAGS="$LDFLAGS $LIBCLANG_LDFLAGS" + LIBS="$LIBS $LIBCLANG_LIBS" + AC_CHECK_LIB([clang], [clang_createIndex], + [HAVE_LIBCLANG_LIB=1], [HAVE_LIBCLANG_LIB=0]) + AC_CHECK_HEADER([clang-c/Index.h], + [HAVE_LIBCLANG_HDR=1], [HAVE_LIBCLANG_HDR=0]) + # if we found it, break out + if test "x$HAVE_LIBCLANG_LIB" = "x1" -a "x$HAVE_LIBCLANG_HDR" = "x1" ; then + HAVE_LIBCLANG=1 + break + fi + $as_unset ac_cv_lib_clang_clang_createIndex $as_unset ac_cv_header_clang_c_Index_h \ + LIBCLANG_CFLAGS LIBCLANG_CPPFLAGS LIBCLANG_LIBS HAVE_LIBCLANG_LIB HAVE_LIBCLANG_HDR + fi + CFLAGS=$orig_CFLAGS + LDFLAGS=$orig_LDFLAGS + LIBS=$orig_LIBS + if test "x$HAVE_LIBCLANG" = "x1"; then + break + fi + $as_unset ac_cv_path_LLVM_CONFIG LLVM_CONFIG + done + CFLAGS=$orig_CFLAGS + CPPFLAGS=$orig_CPPFLAGS + LDFLAGS=$orig_LDFLAGS + LIBS=$orig_LIBS +fi + +dnl it is safe to update CFLAGS, CPPFLAGS and LDFLAGS from clang +dnl we will only link against libclang when we have to + +AC_SUBST(HAVE_LIBCLANG) +AC_SUBST(LIBCLANG_CFLAGS) +AC_SUBST(LIBCLANG_CPPFLAGS) +AC_SUBST(LIBCLANG_LDFLAGS) +AC_SUBST(LIBCLANG_LIBS) +AC_DEFINE_UNQUOTED([HAVE_LIBCLANG], [$HAVE_LIBCLANG], [Define to 1 if libclang is available]) +AM_CONDITIONAL([HAVE_LIBCLANG], [ test x$HAVE_LIBCLANG = x1 ]) + +dnl AS_IF([test "x$with_libclang" != xno], +dnl [AC_CHECK_LIB([libclang], [main], +dnl [AC_SUBST([LIBREADLINE], ["-lreadline -lncurses"]) +dnl AC_DEFINE([HAVE_LIBREADLINE], [1], +dnl [Define if you have libreadline]) +dnl ], +dnl [if test "x$with_readline" != xcheck; then +dnl AC_MSG_FAILURE( +dnl [--with-readline was given, but test for readline failed]) +dnl fi +dnl ], -lncurses)]) + +dnl trim cflags, cppflags, ldflags +CFLAGS=`echo $CFLAGS | sed -e 's/^[ ]*//'` +CPPFLAGS=`echo $CPPFLAGS | sed -e 's/^[ ]*//'` +LDFLAGS=`echo $LDFLAGS | sed -e 's/^[ ]*//'` + # Shave by default on supported autoconf versions dnl m4_version_prereq(2.64, SHAVE_INIT([build-aux], [enable]), [true]) m4_ifdef([SHAVE_INIT], [SHAVE_INIT([build-aux], [enable])]) @@ -397,8 +548,8 @@ echo " LIBTOOL_VERSION: ${LIBTOOL_VERSION} prefix: ${prefix} C compiler: ${CC} - CFLAGS: ${CFLAGS} Linker: ${LD} + CFLAGS: ${CFLAGS} LDFLAGS: ${LDFLAGS} LIBS: ${LIBS} HAVE_ASAN: ${HAVE_ASAN} @@ -407,8 +558,10 @@ echo " HAVE_NETWORK: ${HAVE_NETWORK} HAVE_DEVMODE: ${HAVE_DEVMODE} HAVE_SPHINX: ${HAVE_SPHINX} - GIT: $GIT - DOCKER: $DOCKER + GIT: ${GIT} + DOCKER: ${DOCKER} + HAVE_LIBCLANG: ${HAVE_LIBCLANG} + LIBCLANG_LIBS: ${LIBCLANG_LIBS} TESTSUITEURL: $TESTSUITEURL TESTSUITECHECKOUT: $TESTSUITECHECKOUT JSONTESTSUITEURL: $JSONTESTSUITEURL diff --git a/include/libfyaml.h b/include/libfyaml.h index 16d8971..e1c03c4 100644 --- a/include/libfyaml.h +++ b/include/libfyaml.h @@ -73,9 +73,11 @@ struct fy_document_iterator; #if defined(__GNUC__) && __GNUC__ >= 4 #define FY_EXPORT __attribute__ ((visibility ("default"))) #define FY_DEPRECATED __attribute__ ((deprecated)) +#define FY_FORMAT(_t, _x, _y) __attribute__ ((format(_t, _x, _y))) #else #define FY_EXPORT /* nothing */ #define FY_DEPRECATED /* nothing */ +#define FY_FORMAT(_t, x, y) /* nothing */ #endif /* make a copy of an allocated string and return it on stack @@ -992,6 +994,108 @@ bool fy_parser_get_stream_error(struct fy_parser *fyp) FY_EXPORT; +/** + * fy_parser_vlog() - Log using the parsers diagnostics printf style (va_arg) + * + * Output a log on the parser diagnostic output + * + * @fyp: The parser + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_parser_vlog(struct fy_parser *fyp, enum fy_error_type type, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_parser_log() - Log using the parsers diagnostics printf style + * + * Output a report on the parser's diagnostics + * + * @fyp: The parser + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments + */ +void +fy_parser_log(struct fy_parser *fyp, enum fy_error_type type, const char *fmt, ...) + FY_FORMAT(printf, 3, 4) + FY_EXPORT; + +#ifndef NDEBUG + +#define fy_parser_debug(_fyp, _fmt, ...) \ + fy_parser_log((_fyp), FYET_DEBUG, (_fmt) , ## __VA_ARGS__) +#else + +#define fy_parser_debug(_fyp, _fmt, ...) \ + do { } while(0) + +#endif + +#define fy_parser_info(_fyp, _fmt, ...) \ + fy_parser_log((_fyp), FYET_INFO, (_fmt) , ## __VA_ARGS__) +#define fy_parser_notice(_fyp, _fmt, ...) \ + fy_parser_log((_fyp), FYET_NOTICE, (_fmt) , ## __VA_ARGS__) +#define fy_parser_warning(_fyp, _fmt, ...) \ + fy_parser_log((_fyp), FYET_WARNING, (_fmt) , ## __VA_ARGS__) +#define fy_parser_error(_fyp, _fmt, ...) \ + fy_parser_log((_fyp), FYET_ERROR, (_fmt) , ## __VA_ARGS__) + +/** + * fy_parser_vreport() - Report using the parsers diagnostics printf style and mark token (va_arg) + * + * Output a report on the parser and indicate the token's position + * + * @fyp: The parser + * @type: The error type + * @fyt: The token + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_parser_vreport(struct fy_parser *fyp, enum fy_error_type type, struct fy_token *fyt, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_parser_report() - Report using the parsers diagnostics printf style and mark token + * + * Output a report on the parser and indicate the token's position + * + * @fyp: The parser + * @type: The error type + * @fyt: The token + * @fmt: The printf format string + * @..: The extra arguments + */ +void +fy_parser_report(struct fy_parser *fyp, enum fy_error_type type, struct fy_token *fyt, + const char *fmt, ...) + FY_FORMAT(printf, 4, 5) + FY_EXPORT; + +#ifndef NDEBUG + +#define fy_parser_report_debug(_fyp, _fyt, _fmt, ...) \ + fy_parser_report((_fyp), FYET_DEBUG, (_fyt), (_fmt) , ## __VA_ARGS__) +#else + +#define fy_parser_report_debug(_fyp, _fyt, _fmt, ...) \ + do { } while(0) + +#endif + +#define fy_parser_report_info(_fyp, _fyt, _fmt, ...) \ + fy_parser_report((_fyp), FYET_INFO, (_fyt), (_fmt) , ## __VA_ARGS__) +#define fy_parser_report_notice(_fyp, _fyt, _fmt, ...) \ + fy_parser_report((_fyp), FYET_NOTICE, (_fyt), (_fmt) , ## __VA_ARGS__) +#define fy_parser_report_warning(_fyp, _fyt, _fmt, ...) \ + fy_parser_report((_fyp), FYET_WARNING, (_fyt), (_fmt) , ## __VA_ARGS__) +#define fy_parser_report_error(_fyp, _fyt, _fmt, ...) \ + fy_parser_report((_fyp), FYET_ERROR, (_fyt), (_fmt) , ## __VA_ARGS__) + /** * fy_token_scalar_style() - Get the style of a scalar token * @@ -1009,7 +1113,7 @@ fy_token_scalar_style(struct fy_token *fyt) * fy_token_scalar_is_null() - Test whether the scalar is null (content) * * @fyt: The scalar token to check for NULLity. - * + * * Note that this is different than null of the YAML type system. * It is null as in null content. It is also different than an * empty scalar. @@ -1835,6 +1939,107 @@ int fy_emit_event(struct fy_emitter *emit, struct fy_event *fye) FY_EXPORT; +/** + * fy_emit_vevent() - Queue (and possibly emit) an event using varargs + * + * Queue and output using the emitter. The format is identical to + * the one used in fy_emit_event_vcreate(). + * + * @emit: The emitter to use + * @type: The event type to create + * @ap: The variable argument list pointer. + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_vevent(struct fy_emitter *emit, enum fy_event_type type, va_list ap) + FY_EXPORT; + +/** + * fy_emit_eventf() - Queue (and possibly emit) an event using variable args + * + * Queue and output using the emitter. The format is identical to + * the one used in fy_emit_event_create(). + * + * @emit: The emitter to use + * @type: The event type to create + * @...: The optional extra arguments + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_eventf(struct fy_emitter *emit, enum fy_event_type type, ...) + FY_EXPORT; + +/** + * fy_emit_scalar_write() - Emit a scalar with write semantics + * + * Queue and output using the emitter a scalar using a standard + * write interface. + * + * @emit: The emitter to use + * @style: The scalar style to use + * @anchor: The anchor or NULL + * @tag: The tag or NULL + * @buf: Pointer to the buffer + * @count: The number of bytes to write + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_scalar_write(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *buf, size_t count) + FY_EXPORT; + +/** + * fy_emit_scalar_vprintf() - Emit a scalar with vprintf semantics + * + * Queue and output using the emitter a scalar using a standard + * vprintf interface. + * + * @emit: The emitter to use + * @style: The scalar style to use + * @anchor: The anchor or NULL + * @tag: The tag or NULL + * @fmt: The printf like format string + * @ap: The argument list + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_scalar_vprintf(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_emit_scalar_printf() - Emit a scalar with printf semantics + * + * Queue and output using the emitter a scalar using a standard + * printf interface. + * + * @emit: The emitter to use + * @style: The scalar style to use + * @anchor: The anchor or NULL + * @tag: The tag or NULL + * @fmt: The printf like format string + * @...: The extra arguments + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_scalar_printf(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *fmt, ...) + FY_FORMAT(printf, 5, 6) + FY_EXPORT; + /** * fy_emit_event_from_parser() - Queue (and possibly emit) an event * generated by the parser. @@ -2687,7 +2892,7 @@ fy_document_vbuildf(const struct fy_parse_cfg *cfg, */ struct fy_document * fy_document_buildf(const struct fy_parse_cfg *cfg, const char *fmt, ...) - __attribute__((format(printf, 2, 3))) + FY_FORMAT(printf, 2, 3) FY_EXPORT; /** @@ -2722,6 +2927,36 @@ fy_flow_document_build_from_string(const struct fy_parse_cfg *cfg, const char *str, size_t len, size_t *consumed) FY_EXPORT; +/** + * fy_block_document_build_from_string() - Create a document using the provided YAML source. + * + * Create a document parsing the provided string as a YAML source. + * + * The document is a block document, and it terminates when indentation + * appears to do so. + * + * Example of block documents: + * + * this-is-yaml + * foo: bar <- starts here + * baz: + * - 1 + * - 2 + * this-is-yaml-no-more + * + * @cfg: The parse configuration to use or NULL for the default. + * @str: The YAML source to use. + * @len: The length of the string (or -1 if '\0' terminated) + * @consumed: A pointer to the consumed amount + * + * Returns: + * The created document, or NULL on error. + */ +struct fy_document * +fy_block_document_build_from_string(const struct fy_parse_cfg *cfg, + const char *str, size_t len, size_t *consumed) + FY_EXPORT; + /** * fy_document_root() - Return the root node of the document * @@ -2784,6 +3019,36 @@ enum fy_node_style fy_node_get_style(struct fy_node *fyn) FY_EXPORT; +/** + * fy_node_get_start_token() - Get the node start token + * + * Retrieve the node start token. + * + * For scalars, this is the same as the end token. + * + * @fyn: The node + * + * Returns: + * The node start token + */ +struct fy_token * fy_node_get_start_token(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_end_token() - Get the node end token + * + * Retrieve the node end token. + * + * For scalars, this is the same as the start token. + * + * @fyn: The node + * + * Returns: + * The node end token + */ + struct fy_token * fy_node_get_end_token(struct fy_node *fyn) + FY_EXPORT; + /** * fy_node_is_scalar() - Check whether the node is a scalar * @@ -3072,7 +3337,7 @@ fy_node_vbuildf(struct fy_document *fyd, const char *fmt, va_list ap) */ struct fy_node * fy_node_buildf(struct fy_document *fyd, const char *fmt, ...) - __attribute__((format(printf, 2, 3))) + FY_FORMAT(printf, 2, 3) FY_EXPORT; /** @@ -3406,8 +3671,8 @@ fy_node_create_vscalarf(struct fy_document *fyd, const char *fmt, va_list ap) */ struct fy_node * fy_node_create_scalarf(struct fy_document *fyd, const char *fmt, ...) - FY_EXPORT - __attribute__((format(printf, 2, 3))); + FY_FORMAT(printf, 2, 3) + FY_EXPORT; /** * fy_node_create_sequence() - Create an empty sequence node. @@ -4256,7 +4521,7 @@ int fy_node_vscanf(struct fy_node *fyn, const char *fmt, va_list ap); */ int fy_node_scanf(struct fy_node *fyn, const char *fmt, ...) - __attribute__((format(scanf, 2, 3))) + FY_FORMAT(scanf, 2, 3) FY_EXPORT; /** @@ -4307,7 +4572,7 @@ fy_document_vscanf(struct fy_document *fyd, const char *fmt, va_list ap) */ int fy_document_scanf(struct fy_document *fyd, const char *fmt, ...) - __attribute__((format(scanf, 2, 3))) + FY_FORMAT(scanf, 2, 3) FY_EXPORT; /** @@ -4603,7 +4868,8 @@ fy_node_set_vanchorf(struct fy_node *fyn, const char *fmt, va_list ap) */ int fy_node_set_anchorf(struct fy_node *fyn, const char *fmt, ...) - FY_EXPORT __attribute__((format(printf, 2, 3))); + FY_FORMAT(printf, 2, 3) + FY_EXPORT; /** * fy_node_remove_anchor() - Remove an anchor @@ -4871,7 +5137,7 @@ fy_node_vreport(struct fy_node *fyn, enum fy_error_type type, void fy_node_report(struct fy_node *fyn, enum fy_error_type type, const char *fmt, ...) - __attribute__((format(printf, 3, 4))) + FY_FORMAT(printf, 3, 4) FY_EXPORT; /** @@ -4922,7 +5188,7 @@ void fy_node_override_report(struct fy_node *fyn, enum fy_error_type type, const char *file, int line, int column, const char *fmt, ...) - __attribute__((format(printf, 6, 7))) + FY_FORMAT(printf, 6, 7) FY_EXPORT; typedef void (*fy_diag_output_fn)(struct fy_diag *diag, void *user, @@ -5201,8 +5467,8 @@ fy_diag_vprintf(struct fy_diag *diag, const char *fmt, va_list ap) */ int fy_diag_printf(struct fy_diag *diag, const char *fmt, ...) - FY_EXPORT - __attribute__((format(printf, 2, 3))); + FY_FORMAT(printf, 2, 3) + FY_EXPORT; /** * struct fy_diag_ctx - The diagnostics context @@ -5268,8 +5534,8 @@ fy_vdiag(struct fy_diag *diag, const struct fy_diag_ctx *fydc, int fy_diagf(struct fy_diag *diag, const struct fy_diag_ctx *fydc, const char *fmt, ...) - FY_EXPORT - __attribute__((format(printf, 3, 4))); + FY_FORMAT(printf, 3, 4) + FY_EXPORT; #define fy_diag_diag(_diag, _level, _fmt, ...) \ ({ \ @@ -5306,6 +5572,100 @@ fy_diagf(struct fy_diag *diag, const struct fy_diag_ctx *fydc, #define fy_error(_diag, _fmt, ...) \ fy_diag_diag((_diag), FYET_ERROR, (_fmt) , ## __VA_ARGS__) +/** + * fy_diag_token_vreport() - Report about a token vprintf style using + * the given diagnostic object + * + * Output a report about the given token via the specific + * error type, and using the reporting configuration of the token's + * document. + * + * @diag: The diag object + * @fyt: The token + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_diag_token_vreport(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_diag_token_report() - Report about a token printf style using + * the given diagnostic object + * + * Output a report about the given token via the specific + * error type, and using the reporting configuration of the token's + * document. + * + * @diag: The diag object + * @fyt: The token + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_diag_token_report(struct fy_diag *diag, struct fy_token *fye, + enum fy_error_type type, const char *fmt, ...) + FY_FORMAT(printf, 4, 5) + FY_EXPORT; + +/** + * fy_diag_token_override_vreport() - Report about a token vprintf style, + * overriding file, line and column info using + * the given diagnostic object + * + * Output a report about the given token via the specific + * error type, and using the reporting configuration of the token's + * document. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @diag: The diag object + * @fyt: The token + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_diag_token_override_vreport(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *file, + int line, int column, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_diag_token_override_report() - Report about a token printf style, + * overriding file, line and column info using + * the given diagnostic object + * + * Output a report about the given token via the specific + * error type, and using the reporting configuration of the token's + * document. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @diag: The diag object + * @fyt: The token + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_diag_token_override_report(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *file, + int line, int column, const char *fmt, ...) + FY_FORMAT(printf, 7, 8) + FY_EXPORT; + /** * fy_diag_node_vreport() - Report about a node vprintf style using * the given diagnostic object @@ -5342,7 +5702,7 @@ fy_diag_node_vreport(struct fy_diag *diag, struct fy_node *fyn, void fy_diag_node_report(struct fy_diag *diag, struct fy_node *fyn, enum fy_error_type type, const char *fmt, ...) - __attribute__((format(printf, 4, 5))) + FY_FORMAT(printf, 4, 5) FY_EXPORT; /** @@ -5397,7 +5757,7 @@ void fy_diag_node_override_report(struct fy_diag *diag, struct fy_node *fyn, enum fy_error_type type, const char *file, int line, int column, const char *fmt, ...) - __attribute__((format(printf, 7, 8))) + FY_FORMAT(printf, 7, 8) FY_EXPORT; /** @@ -6063,6 +6423,26 @@ const char * fy_tag_token_suffix(struct fy_token *fyt, size_t *lenp) FY_EXPORT; +/** + * fy_tag_token_short() - Get the short tag of the tag token + * + * Retrieve the short tag contents. The short tag is the same + * one that will need to be emitted. + * Will fail if token is not a tag token, or if a memory error happens. + * + * @fyt: The tag token out of which the short pointer + * will be returned. + * @lenp: Pointer to a variable that will hold the returned length + * + * Returns: + * A pointer to the text representation of the short tag, while + * @lenp will be assigned the character length of said representation. + * NULL in case of an error. + */ +const char * +fy_tag_token_short(struct fy_token *fyt, size_t *lenp) + FY_EXPORT; + /** * fy_tag_token_handle0() - Get the handle contained in the * tag token as zero terminated string @@ -6082,6 +6462,24 @@ const char * fy_tag_token_handle0(struct fy_token *fyt) FY_EXPORT; +/** + * fy_tag_token_short0() - Get the short tag of the tag token as zero + * terminated string. + * + * Retrieve the short tag contents. The short tag is the same + * one that will need to be emitted. + * Will fail if token is not a tag token, or if a memory error happens. + * + * @fyt: The tag token out of which the short pointer will be returned. + * + * Returns: + * A pointer to the null terminated text representation of the short tag. + * NULL in case of an error. + */ +const char * +fy_tag_token_short0(struct fy_token *fyt) + FY_EXPORT; + /** * fy_tag_token_suffix0() - Get the suffix contained in the * tag token as zero terminated string @@ -6412,6 +6810,23 @@ const struct fy_tag * fy_document_state_tag_directive_iterate(struct fy_document_state *fyds, void **prevp) FY_EXPORT; +/** + * fy_document_state_tag_directives() - Get all the tag directives in a + * malloc'ed array + * + * Return all the tag directives in a dynamically allocated area. + * Must be free()'d when not in use. + * + * @fyds: The document state + * + * Returns: + * An array of fy_tag pointer structures, terminated with a NULL pointer + * NULL on error + */ +struct fy_tag ** +fy_document_state_tag_directives(struct fy_document_state *fyds) + FY_EXPORT; + /** * fy_document_state_tag_is_default() - Test whether the given tag is a default one * @@ -6679,6 +7094,7 @@ fy_emitter_get_document_state(struct fy_emitter *emit) * * @emit: The emitter * @type: The event type to create + * @...: The optional extra arguments * * Returns: * The created event or NULL in case of an error @@ -7145,6 +7561,54 @@ void fy_path_component_set_sequence_user_data(struct fy_path_component *fypc, void *data) FY_EXPORT; +/** + * fy_path_get_parent_user_data() - Return the userdata of the parent collection + * + * @fypp: The path + * + * Returns: + * The user data associated with the parent collection of the path, or NULL if no path + */ +void * +fy_path_get_parent_user_data(struct fy_path *path) + FY_EXPORT; + +/** + * fy_path_set_parent_user_data() - Set the user data associated with the parent collection + * + * Note, no error condition if not a path + * + * @fypp: The path + * @data: The data to set as parent collection data + */ +void +fy_path_set_parent_user_data(struct fy_path *path, void *data) + FY_EXPORT; + +/** + * fy_path_get_last_user_data() - Return the userdata of the last collection + * + * @fypp: The path + * + * Returns: + * The user data associated with the last collection of the path, or NULL if no path + */ +void * +fy_path_get_last_user_data(struct fy_path *path) + FY_EXPORT; + +/** + * fy_path_set_last_user_data() - Set the user data associated with the last collection + * + * Note, no error condition if not a path + * + * @fypp: The path + * @data: The data to set as last collection data + */ +void +fy_path_set_last_user_data(struct fy_path *path, void *data) + FY_EXPORT; + /** * fy_path_last_component() - Get the very last component of a path * @@ -7345,6 +7809,445 @@ bool fy_document_iterator_get_error(struct fy_document_iterator *fydi) FY_EXPORT; +/* + * The libfyaml's push-pull thread implementation + * + */ + +/* opaque types for the user */ +struct fy_thread_pool; +struct fy_thread; +struct fy_work_pool; + +/** + * typedef fy_work_exec_fn - Work exec function + * + * The callback executed on work submission + * + * @arg: The argument to the method + * + */ +typedef void (*fy_work_exec_fn)(void *arg); + +/** + * typedef fy_work_check_fn - Work check function + * + * Work checker function to decide if it's worth to + * offload to a thread. + * + * @arg: The argument to the method + * + * Returns: + * true if it should offload to thread, false otherwise + * + */ +typedef bool (*fy_work_check_fn)(const void *arg); + +/** + * struct fy_thread_work - Work submitted to a thread for execution + * + * @fn: The execution function for this work + * @arg: The argument to the fn + * @wp: Used internally, must be set to NULL on entry + * + * This is the structure describing the work submitted + * to a thread for execution. + */ +struct fy_thread_work { + fy_work_exec_fn fn; + void *arg; + struct fy_work_pool *wp; +}; + +/* + * enum fy_thread_pool_cfg_flags - Thread pool configuration flags + * + * These flags control the operation of the thread pool. + * For now only the steal mode flag is defined. + * + * @FYTPCF_STEAL_MODE: Enable steal mode for the thread pool + */ +enum fy_thread_pool_cfg_flags { + FYTPCF_STEAL_MODE = FY_BIT(0), +}; + +/** + * struct fy_thread_pool_cfg - thread pool configuration structure. + * + * Argument to the fy_thread_pool_create() method. + * + * @flags: Thread pool configuration flags + * @num_threads: Number of threads, if 0 == online CPUs + * @userdata: A userdata pointer + */ +struct fy_thread_pool_cfg { + enum fy_thread_pool_cfg_flags flags; + unsigned int num_threads; + void *userdata; +}; + +/** + * fy_thread_pool_create() - Create a thread pool + * + * Creates a thread pool with its configuration @cfg + * The thread pool may be destroyed by a corresponding call to + * fy_thread_pool_destroy(). + * + * @cfg: The configuration for the thread pool + * + * Returns: + * A pointer to the thread pool or NULL in case of an error. + */ +struct fy_thread_pool * +fy_thread_pool_create(const struct fy_thread_pool_cfg *cfg) + FY_EXPORT; + +/** + * fy_thread_pool_destroy() - Destroy the given thread pool + * + * Destroy a thread pool created earlier via fy_thread_pool_create(). + * Note that this function will block until all threads + * of the pool are destroyed. + * + * @tp: The thread pool to destroy + */ +void +fy_thread_pool_destroy(struct fy_thread_pool *tp) + FY_EXPORT; + +/** + * fy_thread_pool_get_num_threads() - Get the number of threads + * + * Returns the actual number of created threads. + * + * @tp: The thread pool + * + * Returns: + * > 0 for the number of actual threads created, + * -1 on error + */ +int +fy_thread_pool_get_num_threads(struct fy_thread_pool *tp) + FY_EXPORT; + +/** + * fy_thread_pool_get_cfg() - Get the configuration of a thread pool + * + * @tp: The thread pool + * + * Returns: + * The configuration of the thread pool + */ +const struct fy_thread_pool_cfg * +fy_thread_pool_get_cfg(struct fy_thread_pool *tp) + FY_EXPORT; + +/* + * fy_thread_reserve() - Reserve a thread from the pool. + * + * Reserve a thread from the pool and return it. + * Note this is only valid for a non-work stealing thread pool. + * You release the thread again via a call to fy_thread_unreserve. + * + * @tp: The thread pool + * + * Returns: + * A reserved thread if not NULL, NULL if no threads are available. + */ +struct fy_thread * +fy_thread_reserve(struct fy_thread_pool *tp) + FY_EXPORT; + +/* + * fy_thread_unreserve() - Unreserve a previously reserved thread + * + * Unreserve a thread previously reserved via a call to fy_thread_reserve() + * Note this is only valid for a non-work stealing thread pool. + * + * @t: The thread + */ +void +fy_thread_unreserve(struct fy_thread *t) + FY_EXPORT; + +/* + * fy_thread_submit_work() - Submit work for execution + * + * Submit work for execution. If successful the thread + * will start executing the work in parallel with the + * calling thread. You can wait for the thread to + * terminate via a call to fy_thread_wait_work(). + * The thread must have been reserved earlier via fy_thread_reserve() + * + * Note this is only valid for a non-work stealing thread pool. + * + * @t: The thread + * @work: The work + * + * Returns: + * 0 if work has been submitted, -1 otherwise. + */ +int +fy_thread_submit_work(struct fy_thread *t, struct fy_thread_work *work) + FY_EXPORT; + +/* + * fy_thread_wait_work() - Wait for completion of submitted work + * + * Wait until submitted work to the thread has finished. + * Note this is only valid for a non-work stealing thread pool. + * + * @t: The thread + * + * Returns: + * 0 if work finished, -1 on error. + */ +int +fy_thread_wait_work(struct fy_thread *t) + FY_EXPORT; + +/* + * fy_thread_work_join() - Submit works for execution and wait + * + * Submit works for possible parallel execution. If no offloading + * is possible at the time execute in the current context. + * It is possible to use in both stealing and non-stealing mode + * with the difference being that stealing mode is about 30% faster. + * + * @tp: The thread pool + * @works: Pointer to an array of works sized @work_count + * @work_count: The size of the @works array + * @check_fn: Pointer to a check function, or NULL for no checks + */ +void +fy_thread_work_join(struct fy_thread_pool *tp, + struct fy_thread_work *works, size_t work_count, + fy_work_check_fn check_fn) + FY_EXPORT; + +/* + * fy_thread_args_join() - Execute function in parallel using arguments as pointers + * + * Execute @fn possibly in parallel using the threads in the thread pool. + * The arguments of the function are provided by the args array. + * + * @tp: The thread pool + * @fn: The function to execute in parallel + * @check_fn: Pointer to a check function, or NULL for no checks + * @args: An args array sized @count of argument pointers + * @count: The count of the args array items + */ +void +fy_thread_args_join(struct fy_thread_pool *tp, + fy_work_exec_fn, fy_work_check_fn check_fn, + void **args, size_t count) + FY_EXPORT; + +/* + * fy_thread_arg_array_join() - Execute function in parallel using argument array + * + * Execute @fn possibly in parallel using the threads in the thread pool. + * The arguments of the function are provided by the args array. + * + * @tp: The thread pool + * @fn: The function to execute in parallel + * @check_fn: Pointer to a check function, or NULL for no checks + * @args: An args array of @argsize items + * @argsize: The size of each argument array item + * @count: The count of the args array items + */ +void +fy_thread_arg_array_join(struct fy_thread_pool *tp, + fy_work_exec_fn fn, fy_work_check_fn check_fn, + void *args, size_t argsize, size_t count) + FY_EXPORT; + +/* + * fy_thread_arg_array_join() - Execute function in parallel with the same argument + * + * Execute @fn possibly in parallel using the threads in the thread pool. + * The argument of the functions is the same. + * + * @tp: The thread pool + * @fn: The function to execute in parallel + * @arg: The common argument + * @count: The count of executions + */ +void +fy_thread_arg_join(struct fy_thread_pool *tp, + fy_work_exec_fn fn, fy_work_check_fn check_fn, + void *arg, size_t count) + FY_EXPORT; + +/* + * Minimal exposing of internal BLAKE3 implementation + * + */ + +/* BLAKE3 key length */ +#define FY_BLAKE3_KEY_LEN 32 +/* BLAKE3 output length */ +#define FY_BLAKE3_OUT_LEN 32 + +/* opaque BLAKE3 hasher type for the user*/ +struct fy_blake3_hasher; + +/** + * fy_blake3_backend_iterate() - Iterate over the supported BLAKE3 backends + * + * This method iterates over the supported BLAKE3 backends. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * The default backend is always the last in sequence, so for + * example if the order is [ "portable", "sse2", NULL ] the + * default is "sse2". + * + * @prevp: The previous backend pointer, or NULL at start + * + * Returns: + * The next backend or NULL at the end. + */ +const char * +fy_blake3_backend_iterate(const char **prevp) + FY_EXPORT; + +/** + * struct fy_blake3_hasher_cfg - BLAKE3 hasher configuration + * + * Argument to the fy_blake3_hasher_create() method which + * is the fyaml's user facing BLAKE3 API. + * It is very minimal, on purpose, since it's meant to be + * exposing a full blown BLAKE3 API. + * + * @backend: NULL for default, or a specific backend name + * @file_buffer: Use this amount of buffer for buffering, zero for default + * @mmap_min_chunk: Minimum chunk size for mmap case + * @mmap_max_chunk: Maximum chunk size for mmap case + * @no_mmap: Disable mmap for file access + * @key: pointer to a FY_BLAKE3_KEY_LEN area when in keyed mode. + * NULL otherwise. + * @context: pointer to a context when in key derivation mode. + * NULL otherwise. + * @context_len: The size of the context when in key derivation mode. + * 0 otherwise. + * @tp: The thread pool to use, if NULL, create a private one + * @num_threads: Number of threads to use + * - 0 means default: NUM_CPUS * 3 / 2 + * - > 0 specific number of threads + * - -1 disable threading entirely + */ +struct fy_blake3_hasher_cfg { + const char *backend; + size_t file_buffer; + size_t mmap_min_chunk; + size_t mmap_max_chunk; + bool no_mmap; + const uint8_t *key; + const void *context; + size_t context_len; + struct fy_thread_pool *tp; + int num_threads; +}; + +/** + * fy_blake3_hasher_create() - Create a BLAKE3 hasher object. + * + * Creates a BLAKE3 hasher with its configuration @cfg + * The hasher may be destroyed by a corresponding call to + * fy_blake3_hasher_destroy(). + * + * @cfg: The configuration for the BLAKE3 hasher + * + * Returns: + * A pointer to the BLAKE3 hasher or NULL in case of an error. + */ +struct fy_blake3_hasher * +fy_blake3_hasher_create(const struct fy_blake3_hasher_cfg *cfg) + FY_EXPORT; + +/** + * fy_blake3_hasher_destroy() - Destroy the given BLAKE3 hasher + * + * Destroy a BLAKE3 hasher created earlier via fy_blake3_hasher_create(). + * + * @fyh: The BLAKE3 hasher to destroy + */ +void +fy_blake3_hasher_destroy(struct fy_blake3_hasher *fyh) + FY_EXPORT; + +/** + * fy_blake3_hasher_update() - Update the BLAKE3 hasher state with the given input + * + * Updates the BLAKE3 hasher state by hashing the given input. + * + * @fyh: The BLAKE3 hasher + * @input: Pointer to the input + * @input_len: Size of the input in bytes + */ +void +fy_blake3_hasher_update(struct fy_blake3_hasher *fyh, const void *input, size_t input_len) + FY_EXPORT; + +/** + * fy_blake3_hasher_finalize() - Finalize the hash and get output + * + * Finalizes the BLAKE3 hasher and returns the output + * + * @fyh: The BLAKE3 hasher + * + * Returns: + * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL + * in case of an error. + */ +const uint8_t * +fy_blake3_hasher_finalize(struct fy_blake3_hasher *fyh) + FY_EXPORT; + +/** + * fy_blake3_hasher_reset() - Resets the hasher + * + * Resets the hasher for re-use + * + * @fyh: The BLAKE3 hasher + */ +void +fy_blake3_hasher_reset(struct fy_blake3_hasher *fyh) + FY_EXPORT; + +/** + * fy_blake3_hash() - BLAKE3 hash a memory area + * + * Hash a memory area and return the BLAKE3 output. + * + * @fyh: The BLAKE3 hasher + * @mem: Pointer to the memory to use + * @size: The size of the memory in bytes + * + * Returns: + * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL + * in case of an error. + */ +const uint8_t * +fy_blake3_hash(struct fy_blake3_hasher *fyh, const void *mem, size_t size) + FY_EXPORT; + +/** + * fy_blake3_hash_file() - BLAKE3 hash a file. + * + * Hash the given file (possibly using mmap) + * + * @fyh: The BLAKE3 hasher + * @filename: The filename + * + * Returns: + * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL + * in case of an error. + */ +const uint8_t * +fy_blake3_hash_file(struct fy_blake3_hasher *fyh, const char *filename) + FY_EXPORT; + #ifdef __cplusplus } #endif diff --git a/libfyaml.pc.in b/libfyaml.pc.in index 299d11e..3ca5d42 100644 --- a/libfyaml.pc.in +++ b/libfyaml.pc.in @@ -6,5 +6,5 @@ includedir=@includedir@ Name: libfyaml Description: Fancy YAML 1.3 parser library Version: @PACKAGE_VERSION@ -Libs: -L${libdir} @ASAN_LIBS@ -lfyaml @PTHREAD_LIBS@ +Libs: -L${libdir} @ASAN_LIBS@ @PTHREAD_LIBS@ -lfyaml Cflags: -I${includedir} @ASAN_CFLAGS@ @PTHREAD_CFLAGS@ diff --git a/src/Makefile.am b/src/Makefile.am index b9faeda..141209f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,42 +1,118 @@ +bin_PROGRAMS = +noinst_PROGRAMS = +noinst_LIBRARIES = +lib_LTLIBRARIES = +noinst_LTLIBRARIES = + AM_CPPFLAGS = \ -I$(top_srcdir)/src \ -I$(top_srcdir)/include AM_CFLAGS = -lib_LTLIBRARIES = libfyaml.la +lib_LTLIBRARIES += libfyaml.la libfyaml_la_SOURCES = \ lib/fy-parse.c lib/fy-parse.h \ - lib/fy-utf8.c lib/fy-utf8.h \ lib/fy-types.c lib/fy-types.h \ - lib/fy-list.h \ - lib/fy-typelist.h \ lib/fy-diag.c lib/fy-diag.h \ lib/fy-dump.c lib/fy-dump.h \ lib/fy-atom.c lib/fy-atom.h \ - lib/fy-ctype.c lib/fy-ctype.h \ lib/fy-token.c lib/fy-token.h \ lib/fy-input.c lib/fy-input.h \ lib/fy-docstate.c lib/fy-docstate.h \ lib/fy-doc.c lib/fy-doc.h \ lib/fy-docbuilder.c lib/fy-docbuilder.h \ lib/fy-emit.c lib/fy-emit.h lib/fy-emit-accum.h \ - lib/fy-utils.c lib/fy-utils.h \ lib/fy-event.h lib/fy-event.c \ - xxhash/xxhash.c xxhash/xxhash.h \ lib/fy-accel.c lib/fy-accel.h \ lib/fy-walk.c lib/fy-walk.h \ lib/fy-path.c lib/fy-path.h \ - lib/fy-composer.c lib/fy-composer.h + lib/fy-composer.c lib/fy-composer.h \ + xxhash/xxhash.c xxhash/xxhash.h \ + util/fy-list.h \ + util/fy-typelist.h \ + util/fy-ctype.c util/fy-ctype.h \ + util/fy-utf8.c util/fy-utf8.h \ + util/fy-utils.c util/fy-utils.h \ + util/fy-endian.h \ + util/fy-blob.c util/fy-blob.h \ + util/fy-id.h \ + util/fy-align.h \ + util/fy-bit64.h \ + thread/fy-thread.h thread/fy-thread.c libfyaml_la_CPPFLAGS = $(AM_CPPFLAGS) \ - -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/xxhash + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/xxhash \ + -I$(top_srcdir)/src/util \ + -I$(top_srcdir)/src/thread libfyaml_la_CFLAGS = $(AM_CFLAGS) -libfyaml_la_LDFLAGS = $(AM_LDFLAGS) $(AM_LIBLDFLAGS) \ +libfyaml_la_LDFLAGS = -no-undefined $(AM_LDFLAGS) $(AM_LIBLDFLAGS) \ -version $(LIBTOOL_VERSION) +libfyaml_la_LIBADD = -bin_PROGRAMS = -noinst_PROGRAMS = +# blake3 section +libfyaml_la_CPPFLAGS += -I$(top_srcdir)/src/blake3 +libfyaml_la_SOURCES += blake3/blake3.h \ + blake3/blake3_impl.h \ + blake3/blake3_internal.h \ + blake3/blake3_host_state.c \ + blake3/blake3_backend.c \ + blake3/blake3_be_cpusimd.c \ + blake3/fy-blake3.c + +LIBB3_COMMON_CPPFLAGS=$(AM_CPPFLAGS) -I$(top_srcdir)/src/util -I$(top_srcdir)/src/thread + +# the portable implementation +noinst_LTLIBRARIES += libb3portable.la +libb3portable_la_SOURCES = blake3/blake3_portable.c blake3/blake3.c +libb3portable_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -DHASHER_SUFFIX=portable -DSIMD_DEGREE=1 +libb3portable_la_CFLAGS = $(AM_CPPFLAGS) +libfyaml_la_LIBADD += libb3portable.la + +# note that each target lib has blake3 compiled again but with a suffix +if TARGET_HAS_SSE2 +noinst_LTLIBRARIES += libb3sse2.la +libb3sse2_la_SOURCES = blake3/blake3_sse2.c blake3/blake3_sse2_x86-64_unix.S blake3/blake3.c +libb3sse2_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -msse2 -DHASHER_SUFFIX=sse2 -DSIMD_DEGREE=4 +libb3sse2_la_CFLAGS = $(AM_CPPFLAGS) -msse2 +libfyaml_la_LIBADD += libb3sse2.la +endif + +if TARGET_HAS_SSE41 +noinst_LTLIBRARIES += libb3sse41.la +libb3sse41_la_SOURCES = blake3/blake3_sse41.c blake3/blake3_sse41_x86-64_unix.S blake3/blake3.c +libb3sse41_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -msse4.1 -DHASHER_SUFFIX=sse41 -DSIMD_DEGREE=4 +libb3sse41_la_CFLAGS = $(AM_CPPFLAGS) -msse4.1 +libfyaml_la_LIBADD += libb3sse41.la +endif + +if TARGET_HAS_AVX2 +noinst_LTLIBRARIES += libb3avx2.la +libb3avx2_la_SOURCES = blake3/blake3_avx2.c blake3/blake3_avx2_x86-64_unix.S blake3/blake3.c +libb3avx2_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -mavx2 -DHASHER_SUFFIX=avx2 -DSIMD_DEGREE=8 +libb3avx2_la_CFLAGS = $(AM_CPPFLAGS) -mavx2 +libfyaml_la_LIBADD += libb3avx2.la +endif + +if TARGET_HAS_AVX512 +noinst_LTLIBRARIES += libb3avx512.la +libb3avx512_la_SOURCES = blake3/blake3_avx512.c blake3/blake3_avx512_x86-64_unix.S blake3/blake3.c +libb3avx512_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -mavx512f -mavx512vl -DHASHER_SUFFIX=avx512 -DSIMD_DEGREE=16 +libb3avx512_la_CFLAGS = $(AM_CPPFLAGS) -mavx512f -mavx512vl +libfyaml_la_LIBADD += libb3avx512.la +endif + +if TARGET_HAS_NEON +noinst_LTLIBRARIES += libb3neon.la +libb3neon_la_SOURCES = blake3/blake3_neon.c blake3/blake3.c +libb3neon_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -DHASHER_SUFFIX=neon -DSIMD_DEGREE=4 +libb3neon_la_CFLAGS = $(AM_CPPFLAGS) +if TARGET_CPU_ARM +libb3neon_la_CPPFLAGS += -mfpu=neon +libb3neon_la_CFLAGS += -mfpu=neon +endif +libfyaml_la_LIBADD += libb3neon.la +endif # libfyaml-parser needs both LIBYAML and static if HAVE_LIBYAML @@ -51,7 +127,9 @@ libfyaml_parser_SOURCES = \ libfyaml_parser_CPPFLAGS = $(AM_CPPFLAGS) \ -I$(top_srcdir)/src/valgrind \ -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/xxhash + -I$(top_srcdir)/src/xxhash \ + -I$(top_srcdir)/src/util \ + -I$(top_srcdir)/src/thread libfyaml_parser_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la libfyaml_parser_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS) @@ -59,6 +137,49 @@ libfyaml_parser_LDFLAGS = $(AM_LDFLAGS) -static endif endif +# fy-thread +if HAVE_STATIC + +noinst_PROGRAMS += fy-thread + +fy_thread_SOURCES = \ + internal/fy-thread.c \ + valgrind/fy-valgrind.h + +fy_thread_CPPFLAGS = $(AM_CPPFLAGS) \ + -I$(top_srcdir)/src/valgrind \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/xxhash \ + -I$(top_srcdir)/src/util \ + -I$(top_srcdir)/src/thread +fy_thread_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la +fy_thread_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS) + +fy_thread_LDFLAGS = $(AM_LDFLAGS) -static +endif + +# fy-b3sum +if HAVE_STATIC + +noinst_PROGRAMS += fy-b3sum + +fy_b3sum_SOURCES = \ + internal/fy-b3sum.c \ + valgrind/fy-valgrind.h + +fy_b3sum_CPPFLAGS = $(AM_CPPFLAGS) \ + -I$(top_srcdir)/src/valgrind \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/xxhash \ + -I$(top_srcdir)/src/util \ + -I$(top_srcdir)/src/thread \ + -I$(top_srcdir)/src/blake3 +fy_b3sum_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la +fy_b3sum_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS) + +fy_b3sum_LDFLAGS = $(AM_LDFLAGS) -static +endif + bin_PROGRAMS += fy-tool fy_tool_SOURCES = \ diff --git a/src/blake3/blake3.c b/src/blake3/blake3.c new file mode 100644 index 0000000..ff1bbd5 --- /dev/null +++ b/src/blake3/blake3.c @@ -0,0 +1,741 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "blake3.h" +#include "blake3_impl.h" +#include "blake3_internal.h" + +#include "fy-thread.h" + +// GCC gets confused sometimes +#ifdef GCC_DISABLE_WSTRING_OP_OVERREAD +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-overread" +#endif + +#undef HASHER_OP +#ifdef HASHER_SUFFIX +// two levels of indirection required +#define __HASHER_OP(_name, _sfx) _name ##_ ## _sfx +#define _HASHER_OP(_name, _sfx) __HASHER_OP(_name, _sfx) +#define HASHER_OP(_name) _HASHER_OP(_name, HASHER_SUFFIX) +#else +#define HASHER_OP(_name) _name +#endif + +#ifndef SIMD_DEGREE +#define SIMD_DEGREE (hs->simd_degree) +#endif + +#ifndef SIMD_DEGREE_OR_2 +#define SIMD_DEGREE_OR_2 (SIMD_DEGREE >= 2 ? SIMD_DEGREE : 2) +#endif + +// keep the same naming +#define MAX_SIMD_DEGREE SIMD_DEGREE +#define MAX_SIMD_DEGREE_OR_2 SIMD_DEGREE_OR_2 +#define blake3_simd_degree() SIMD_DEGREE + +// rename to keep the same +#define blake3_compress_in_place hasher->hs->compress_in_place +#define blake3_compress_xof hasher->hs->compress_xof +#define blake3_hash_many self->hs->hash_many + +INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8], + uint8_t flags) { + memcpy(self->cv, key, BLAKE3_KEY_LEN); + self->chunk_counter = 0; + self->buf_len = 0; + self->blocks_compressed = 0; + self->flags = flags; +} + +INLINE void chunk_state_reset(blake3_chunk_state *self, const uint32_t key[8], + uint64_t chunk_counter) { + memcpy(self->cv, key, BLAKE3_KEY_LEN); + self->chunk_counter = chunk_counter; + self->blocks_compressed = 0; + self->buf_len = 0; +} + +INLINE size_t chunk_state_len(const blake3_chunk_state *self) { + return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) + + ((size_t)self->buf_len); +} + +INLINE size_t chunk_state_fill_buf(blake3_chunk_state *self, + const uint8_t *input, size_t input_len) { + size_t take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len); + if (take > input_len) { + take = input_len; + } + uint8_t *dest = self->buf + ((size_t)self->buf_len); + memcpy(dest, input, take); + self->buf_len += (uint8_t)take; + return take; +} + +INLINE uint8_t chunk_state_maybe_start_flag(const blake3_chunk_state *self) { + if (self->blocks_compressed == 0) { + return CHUNK_START; + } else { + return 0; + } +} + +typedef struct { + uint32_t input_cv[8] BLAKE3_ALIGN; + uint8_t block[BLAKE3_BLOCK_LEN] BLAKE3_ALIGN; + uint64_t counter; + uint8_t block_len; + uint8_t flags; +} output_t; + +INLINE output_t make_output(const uint32_t input_cv[BLAKE3_OUT_WORDS], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + output_t ret; + memcpy(ret.input_cv, input_cv, BLAKE3_OUT_LEN); + memcpy(ret.block, block, block_len); + ret.block_len = block_len; + ret.counter = counter; + ret.flags = flags; + return ret; +} + +// Chaining values within a given chunk (specifically the compress_in_place +// interface) are represented as words. This avoids unnecessary bytes<->words +// conversion overhead in the portable implementation. However, the hash_many +// interface handles both user input and parent node blocks, so it accepts +// bytes. For that reason, chaining values in the CV stack are represented as +// bytes. +INLINE void output_chaining_value(const blake3_hasher *hasher, output_t *self, uint8_t cv[BLAKE3_OUT_LEN]) { + uint32_t cv_words[BLAKE3_OUT_WORDS] BLAKE3_ALIGN; + memcpy(cv_words, self->input_cv, BLAKE3_OUT_LEN); + memset(self->block + self->block_len, 0, BLAKE3_BLOCK_LEN - self->block_len); + blake3_compress_in_place(cv_words, self->block, self->block_len, + self->counter, self->flags); + store_cv_words(cv, cv_words); +} + +INLINE void output_root_bytes(const blake3_hasher *hasher, output_t *self, uint64_t seek, uint8_t *out, + size_t out_len) { + uint64_t output_block_counter = seek / 64; + size_t offset_within_block = seek % 64; + uint8_t wide_buf[64] BLAKE3_ALIGN; + while (out_len > 0) { + memset(self->block + self->block_len, 0, BLAKE3_BLOCK_LEN - self->block_len); + blake3_compress_xof(self->input_cv, self->block, self->block_len, + output_block_counter, self->flags | ROOT, wide_buf); + size_t available_bytes = 64 - offset_within_block; + size_t memcpy_len; + if (out_len > available_bytes) { + memcpy_len = available_bytes; + } else { + memcpy_len = out_len; + } + memcpy(out, wide_buf + offset_within_block, memcpy_len); + out += memcpy_len; + out_len -= memcpy_len; + output_block_counter += 1; + offset_within_block = 0; + } +} + +INLINE void chunk_state_update(blake3_hasher *hasher, blake3_chunk_state *self, const uint8_t *input, + size_t input_len) { + if (self->buf_len > 0) { + size_t take = chunk_state_fill_buf(self, input, input_len); + input += take; + input_len -= take; + if (input_len > 0) { + blake3_compress_in_place( + self->cv, self->buf, BLAKE3_BLOCK_LEN, self->chunk_counter, + self->flags | chunk_state_maybe_start_flag(self)); + self->blocks_compressed += 1; + self->buf_len = 0; + } + } + + while (input_len > BLAKE3_BLOCK_LEN) { + blake3_compress_in_place(self->cv, input, BLAKE3_BLOCK_LEN, + self->chunk_counter, + self->flags | chunk_state_maybe_start_flag(self)); + self->blocks_compressed += 1; + input += BLAKE3_BLOCK_LEN; + input_len -= BLAKE3_BLOCK_LEN; + } + + size_t take = chunk_state_fill_buf(self, input, input_len); + input += take; + input_len -= take; +} + +INLINE output_t chunk_state_output(const blake3_chunk_state *self) { + uint8_t block_flags = + self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END; + return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter, + block_flags); +} + +INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], + const uint32_t key[8], uint8_t flags) { + return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT); +} + +// Given some input larger than one chunk, return the number of bytes that +// should go in the left subtree. This is the largest power-of-2 number of +// chunks that leaves at least 1 byte for the right subtree. +INLINE size_t left_len(size_t content_len) { + // Subtract 1 to reserve at least one byte for the right side. content_len + // should always be greater than BLAKE3_CHUNK_LEN. + size_t full_chunks = (content_len - 1) / BLAKE3_CHUNK_LEN; + return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; +} + +// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE chunks at the same time +// on a single thread. Write out the chunk chaining values and return the +// number of chunks hashed. These chunks are never the root and never empty; +// those cases use a different codepath. +INLINE size_t compress_chunks_parallel(blake3_hasher *self, + const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out) { +#if defined(BLAKE3_TESTING) + assert(0 < input_len); + assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN); +#endif + + const uint8_t *chunks_array[MAX_SIMD_DEGREE]; + size_t input_position = 0; + size_t chunks_array_len = 0; + while (input_len - input_position >= BLAKE3_CHUNK_LEN) { + chunks_array[chunks_array_len] = &input[input_position]; + input_position += BLAKE3_CHUNK_LEN; + chunks_array_len += 1; + } + size_t i = chunks_array_len; + while (i < MAX_SIMD_DEGREE) + chunks_array[i++] = NULL; + + blake3_hash_many(chunks_array, chunks_array_len, + BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN, key, chunk_counter, + true, flags, CHUNK_START, CHUNK_END, out); + + // Hash the remaining partial chunk, if there is one. Note that the empty + // chunk (meaning the empty message) is a different codepath. + if (input_len > input_position) { + uint64_t counter = chunk_counter + (uint64_t)chunks_array_len; + blake3_chunk_state chunk_state; + chunk_state_init(&chunk_state, key, flags); + chunk_state.chunk_counter = counter; + chunk_state_update(self, &chunk_state, &input[input_position], + input_len - input_position); + output_t output = chunk_state_output(&chunk_state); + output_chaining_value(self, &output, &out[chunks_array_len * BLAKE3_OUT_LEN]); + return chunks_array_len + 1; + } else { + return chunks_array_len; + } +} + +// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE parents at the same time +// on a single thread. Write out the parent chaining values and return the +// number of parents hashed. (If there's an odd input chaining value left over, +// return it as an additional output.) These parents are never the root and +// never empty; those cases use a different codepath. +INLINE size_t compress_parents_parallel(blake3_hasher *self, + const uint8_t *child_chaining_values, + size_t num_chaining_values, + const uint32_t key[8], uint8_t flags, + uint8_t *out) { +#if defined(BLAKE3_TESTING) + assert(2 <= num_chaining_values); + assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2); +#endif + + const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2]; + size_t parents_array_len = 0; + while (num_chaining_values - (2 * parents_array_len) >= 2) { + parents_array[parents_array_len] = + &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN]; + parents_array_len += 1; + } + + blake3_hash_many(parents_array, parents_array_len, 1, key, + 0, // Parents always use counter 0. + false, flags | PARENT, + 0, // Parents have no start flags. + 0, // Parents have no end flags. + out); + + // If there's an odd child left over, it becomes an output. + if (num_chaining_values > 2 * parents_array_len) { + memcpy(&out[parents_array_len * BLAKE3_OUT_LEN], + &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN], + BLAKE3_OUT_LEN); + return parents_array_len + 1; + } else { + return parents_array_len; + } +} + +static size_t blake3_compress_subtree_wide(blake3_hasher *self, + const uint8_t *input, + size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, + uint8_t flags, uint8_t *out); + +static bool blake3_compress_subtree_wide_work_check(const void *arg) { + const blake3_compress_subtree_state *s = arg; + + /* only off-load to thread if we have enough input */ + return s->input_len >= s->self->hs->mt_degree * BLAKE3_CHUNK_LEN; +} + +static void blake3_compress_subtree_wide_thread(void *arg) { + blake3_compress_subtree_state *s = arg; + + s->n = blake3_compress_subtree_wide(s->self, + s->input, s->input_len, + s->key, + s->chunk_counter, + s->flags, + s->out); +} + +// The wide helper function returns (writes out) an array of chaining values +// and returns the length of that array. The number of chaining values returned +// is the dynamically detected SIMD degree, at most MAX_SIMD_DEGREE. Or fewer, +// if the input is shorter than that many chunks. The reason for maintaining a +// wide array of chaining values going back up the tree, is to allow the +// implementation to hash as many parents in parallel as possible. +// +// As a special case when the SIMD degree is 1, this function will still return +// at least 2 outputs. This guarantees that this function doesn't perform the +// root compression. (If it did, it would use the wrong flags, and also we +// wouldn't be able to implement extendable output.) Note that this function is +// not used when the whole input is only 1 chunk long; that's a different +// codepath. +// +// Why not just have the caller split the input on the first update(), instead +// of implementing this special rule? Because we don't want to limit SIMD or +// multi-threading parallelism for that update(). +static size_t blake3_compress_subtree_wide(blake3_hasher *self, const uint8_t *input, + size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, + uint8_t flags, uint8_t *out) { + // Note that the single chunk case does *not* bump the SIMD degree up to 2 + // when it is 1. If this implementation adds multi-threading in the future, + // this gives us the option of multi-threading even the 2-chunk case, which + // can help performance on smaller platforms. + if (input_len <= blake3_simd_degree() * BLAKE3_CHUNK_LEN) { + return compress_chunks_parallel(self, input, input_len, key, chunk_counter, flags, + out); + } + + // With more than simd_degree chunks, we need to recurse. Start by dividing + // the input into left and right subtrees. (Note that this is only optimal + // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree + // of 3 or something, we'll need a more complicated strategy.) + size_t left_input_len = left_len(input_len); + size_t right_input_len = input_len - left_input_len; + const uint8_t *right_input = &input[left_input_len]; + uint64_t right_chunk_counter = + chunk_counter + (uint64_t)(left_input_len / BLAKE3_CHUNK_LEN); + + // Make space for the child outputs. Here we use MAX_SIMD_DEGREE_OR_2 to + // account for the special case of returning 2 outputs when the SIMD degree + // is 1. + uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; + size_t degree = blake3_simd_degree(); + if (left_input_len > BLAKE3_CHUNK_LEN && degree == 1) { + // The special case: We always use a degree of at least two, to make + // sure there are two outputs. Except, as noted above, at the chunk + // level, where we allow degree=1. (Note that the 1-chunk-input case is + // a different codepath.) + degree = 2; + } + uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; + + // Recurse + size_t left_n, right_n; + if (self->hs->cfg.no_mthread) { + left_n = blake3_compress_subtree_wide(self, input, left_input_len, key, + chunk_counter, flags, cv_array); + right_n = blake3_compress_subtree_wide(self, right_input, right_input_len, key, + right_chunk_counter, flags, right_cvs); + } else { + + // this is the multi-threaded implementation + blake3_compress_subtree_state states[2]; + + /* common */ + states[0].self = states[1].self = self; + states[0].key = states[1].key = key; + states[0].flags = states[1].flags = flags; + states[0].n = states[1].n = 0; + + /* left */ + states[0].input = input; + states[0].input_len = left_input_len; + states[0].chunk_counter = chunk_counter; + states[0].out = cv_array; + + /* right */ + states[1].input = right_input; + states[1].input_len = right_input_len; + states[1].chunk_counter = right_chunk_counter; + states[1].out = right_cvs; + + fy_thread_arg_array_join(self->hs->tp, + blake3_compress_subtree_wide_thread, + blake3_compress_subtree_wide_work_check, + states, sizeof(states[0]), 2); + + left_n = states[0].n; + right_n = states[1].n; + } + + // The special case again. If SIMD_DEGREE=1, then we'll have left_n=1 and + // right_n=1. Rather than compressing them into a single output, return + // them directly, to make sure we always have at least two outputs. + if (left_n == 1) { + memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); + return 2; + } + + // Otherwise, do one layer of parent node compression. + size_t num_chaining_values = left_n + right_n; + return compress_parents_parallel(self, cv_array, num_chaining_values, key, flags, + out); +} + +// Hash a subtree with compress_subtree_wide(), and then condense the resulting +// list of chaining values down to a single parent node. Don't compress that +// last parent node, however. Instead, return its message bytes (the +// concatenated chaining values of its children). This is necessary when the +// first call to update() supplies a complete subtree, because the topmost +// parent node of that subtree could end up being the root. It's also necessary +// for extended output in the general case. +// +// As with compress_subtree_wide(), this function is not used on inputs of 1 +// chunk or less. That's a different codepath. +INLINE void compress_subtree_to_parent_node(blake3_hasher *self, + const uint8_t *input, size_t input_len, const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN]) { +#if defined(BLAKE3_TESTING) + assert(input_len > BLAKE3_CHUNK_LEN); +#endif + + uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; + size_t num_cvs = blake3_compress_subtree_wide(self, input, input_len, key, + chunk_counter, flags, cv_array); + assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); + + // If MAX_SIMD_DEGREE is greater than 2 and there's enough input, + // compress_subtree_wide() returns more than 2 chaining values. Condense + // them into 2 by forming parent nodes repeatedly. + uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; + // The second half of this loop condition is always true, and we just + // asserted it above. But GCC can't tell that it's always true, and if NDEBUG + // is set on platforms where MAX_SIMD_DEGREE_OR_2 == 2, GCC emits spurious + // warnings here. GCC 8.5 is particularly sensitive, so if you're changing + // this code, test it against that version. + while (num_cvs > 2 && num_cvs <= MAX_SIMD_DEGREE_OR_2) { + num_cvs = + compress_parents_parallel(self, cv_array, num_cvs, key, flags, out_array); + memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); + } + memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); +} + +INLINE void hasher_init_base(blake3_host_state *hs, blake3_hasher *self, const uint32_t key[8], + uint8_t flags) { + memset(self, 0, sizeof(*self)); + self->hs = hs; + memcpy(self->key, key, BLAKE3_KEY_LEN); + chunk_state_init(&self->chunk, key, flags); + self->cv_stack_len = 0; +} + +static const uint32_t IV[8] BLAKE3_ALIGN = { + B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3, + B3_IV_4, B3_IV_5, B3_IV_6, B3_IV_7, +}; + +void HASHER_OP(blake3_hasher_init) (blake3_host_state *hs, blake3_hasher *self) { hasher_init_base(hs, self, IV, 0); } + +void HASHER_OP(blake3_hasher_init_keyed) (blake3_host_state *hs, blake3_hasher *self, + const uint8_t key[BLAKE3_KEY_LEN]) { + uint32_t key_words[8] BLAKE3_ALIGN; + load_key_words(key, key_words); + hasher_init_base(hs, self, key_words, KEYED_HASH); +} + +void HASHER_OP(blake3_hasher_init_derive_key_raw) (blake3_host_state *hs, blake3_hasher *self, const void *context, + size_t context_len) { + blake3_hasher context_hasher; + hasher_init_base(hs, &context_hasher, IV, DERIVE_KEY_CONTEXT); + blake3_hasher_update(&context_hasher, context, context_len); + uint8_t context_key[BLAKE3_KEY_LEN]; + blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN); + uint32_t context_key_words[8]; + load_key_words(context_key, context_key_words); + hasher_init_base(hs, self, context_key_words, DERIVE_KEY_MATERIAL); +} + +void HASHER_OP(blake3_hasher_init_derive_key) (blake3_host_state *hs, blake3_hasher *self, const char *context) { + blake3_hasher_init_derive_key_raw(hs, self, context, strlen(context)); +} + +// As described in hasher_push_cv() below, we do "lazy merging", delaying +// merges until right before the next CV is about to be added. This is +// different from the reference implementation. Another difference is that we +// aren't always merging 1 chunk at a time. Instead, each CV might represent +// any power-of-two number of chunks, as long as the smaller-above-larger stack +// order is maintained. Instead of the "count the trailing 0-bits" algorithm +// described in the spec, we use a "count the total number of 1-bits" variant +// that doesn't require us to retain the subtree size of the CV on top of the +// stack. The principle is the same: each CV that should remain in the stack is +// represented by a 1-bit in the total number of chunks (or bytes) so far. +INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { + size_t post_merge_stack_len = (size_t)popcnt(total_len); + while (self->cv_stack_len > post_merge_stack_len) { + uint8_t *parent_node = + &self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN]; + output_t output = parent_output(parent_node, self->key, self->chunk.flags); + output_chaining_value(self, &output, parent_node); + self->cv_stack_len -= 1; + } +} + +// In reference_impl.rs, we merge the new CV with existing CVs from the stack +// before pushing it. We can do that because we know more input is coming, so +// we know none of the merges are root. +// +// This setting is different. We want to feed as much input as possible to +// compress_subtree_wide(), without setting aside anything for the chunk_state. +// If the user gives us 64 KiB, we want to parallelize over all 64 KiB at once +// as a single subtree, if at all possible. +// +// This leads to two problems: +// 1) This 64 KiB input might be the only call that ever gets made to update. +// In this case, the root node of the 64 KiB subtree would be the root node +// of the whole tree, and it would need to be ROOT finalized. We can't +// compress it until we know. +// 2) This 64 KiB input might complete a larger tree, whose root node is +// similarly going to be the the root of the whole tree. For example, maybe +// we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the +// node at the root of the 256 KiB subtree until we know how to finalize it. +// +// The second problem is solved with "lazy merging". That is, when we're about +// to add a CV to the stack, we don't merge it with anything first, as the +// reference impl does. Instead we do merges using the *previous* CV that was +// added, which is sitting on top of the stack, and we put the new CV +// (unmerged) on top of the stack afterwards. This guarantees that we never +// merge the root node until finalize(). +// +// Solving the first problem requires an additional tool, +// compress_subtree_to_parent_node(). That function always returns the top +// *two* chaining values of the subtree it's compressing. We then do lazy +// merging with each of them separately, so that the second CV will always +// remain unmerged. (That also helps us support extendable output when we're +// hashing an input all-at-once.) +INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], + uint64_t chunk_counter) { + hasher_merge_cv_stack(self, chunk_counter); + memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv, + BLAKE3_OUT_LEN); + self->cv_stack_len += 1; +} + +void HASHER_OP(blake3_hasher_update) (blake3_hasher *self, const void *input, + size_t input_len) { + // Explicitly checking for zero avoids causing UB by passing a null pointer + // to memcpy. This comes up in practice with things like: + // std::vector v; + // blake3_hasher_update(&hasher, v.data(), v.size()); + if (input_len == 0) { + return; + } + + const uint8_t *input_bytes = (const uint8_t *)input; + + // If we have some partial chunk bytes in the internal chunk_state, we need + // to finish that chunk first. + if (chunk_state_len(&self->chunk) > 0) { + size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk); + if (take > input_len) { + take = input_len; + } + chunk_state_update(self, &self->chunk, input_bytes, take); + input_bytes += take; + input_len -= take; + // If we've filled the current chunk and there's more coming, finalize this + // chunk and proceed. In this case we know it's not the root. + if (input_len > 0) { + output_t output = chunk_state_output(&self->chunk); + uint8_t chunk_cv[32] BLAKE3_ALIGN; + output_chaining_value(self, &output, chunk_cv); + hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter); + chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1); + } else { + return; + } + } + + // Now the chunk_state is clear, and we have more input. If there's more than + // a single chunk (so, definitely not the root chunk), hash the largest whole + // subtree we can, with the full benefits of SIMD (and maybe in the future, + // multi-threading) parallelism. Two restrictions: + // - The subtree has to be a power-of-2 number of chunks. Only subtrees along + // the right edge can be incomplete, and we don't know where the right edge + // is going to be until we get to finalize(). + // - The subtree must evenly divide the total number of chunks up until this + // point (if total is not 0). If the current incomplete subtree is only + // waiting for 1 more chunk, we can't hash a subtree of 4 chunks. We have + // to complete the current subtree first. + // Because we might need to break up the input to form powers of 2, or to + // evenly divide what we already have, this part runs in a loop. + while (input_len > BLAKE3_CHUNK_LEN) { + size_t subtree_len = round_down_to_power_of_2(input_len); + uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN; + // Shrink the subtree_len until it evenly divides the count so far. We know + // that subtree_len itself is a power of 2, so we can use a bitmasking + // trick instead of an actual remainder operation. (Note that if the caller + // consistently passes power-of-2 inputs of the same size, as is hopefully + // typical, this loop condition will always fail, and subtree_len will + // always be the full length of the input.) + // + // An aside: We don't have to shrink subtree_len quite this much. For + // example, if count_so_far is 1, we could pass 2 chunks to + // compress_subtree_to_parent_node. Since we'll get 2 CVs back, we'll still + // get the right answer in the end, and we might get to use 2-way SIMD + // parallelism. The problem with this optimization, is that it gets us + // stuck always hashing 2 chunks. The total number of chunks will remain + // odd, and we'll never graduate to higher degrees of parallelism. See + // https://github.com/BLAKE3-team/BLAKE3/issues/69. + while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) { + subtree_len /= 2; + } + // The shrunken subtree_len might now be 1 chunk long. If so, hash that one + // chunk by itself. Otherwise, compress the subtree into a pair of CVs. + uint64_t subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN; + if (subtree_len <= BLAKE3_CHUNK_LEN) { + blake3_chunk_state chunk_state; + chunk_state_init(&chunk_state, self->key, self->chunk.flags); + chunk_state.chunk_counter = self->chunk.chunk_counter; + chunk_state_update(self, &chunk_state, input_bytes, subtree_len); + output_t output = chunk_state_output(&chunk_state); + uint8_t cv[BLAKE3_OUT_LEN] BLAKE3_ALIGN; + output_chaining_value(self, &output, cv); + hasher_push_cv(self, cv, chunk_state.chunk_counter); + } else { + // This is the high-performance happy path, though getting here depends + // on the caller giving us a long enough input. + uint8_t cv_pair[2 * BLAKE3_OUT_LEN] BLAKE3_ALIGN; + compress_subtree_to_parent_node(self, input_bytes, subtree_len, self->key, + self->chunk.chunk_counter, + self->chunk.flags, cv_pair); + hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); + hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], + self->chunk.chunk_counter + (subtree_chunks / 2)); + } + self->chunk.chunk_counter += subtree_chunks; + input_bytes += subtree_len; + input_len -= subtree_len; + } + + // If there's any remaining input less than a full chunk, add it to the chunk + // state. In that case, also do a final merge loop to make sure the subtree + // stack doesn't contain any unmerged pairs. The remaining input means we + // know these merges are non-root. This merge loop isn't strictly necessary + // here, because hasher_push_chunk_cv already does its own merge loop, but it + // simplifies blake3_hasher_finalize below. + if (input_len > 0) { + chunk_state_update(self, &self->chunk, input_bytes, input_len); + hasher_merge_cv_stack(self, self->chunk.chunk_counter); + } +} + +void HASHER_OP(blake3_hasher_finalize_seek) (const blake3_hasher *self, uint64_t seek, + uint8_t *out, size_t out_len) { + // Explicitly checking for zero avoids causing UB by passing a null pointer + // to memcpy. This comes up in practice with things like: + // std::vector v; + // blake3_hasher_finalize(&hasher, v.data(), v.size()); + if (out_len == 0) { + return; + } + + // If the subtree stack is empty, then the current chunk is the root. + if (self->cv_stack_len == 0) { + output_t output = chunk_state_output(&self->chunk); + output_root_bytes(self, &output, seek, out, out_len); + return; + } + // If there are any bytes in the chunk state, finalize that chunk and do a + // roll-up merge between that chunk hash and every subtree in the stack. In + // this case, the extra merge loop at the end of blake3_hasher_update + // guarantees that none of the subtrees in the stack need to be merged with + // each other first. Otherwise, if there are no bytes in the chunk state, + // then the top of the stack is a chunk hash, and we start the merge from + // that. + output_t output; + size_t cvs_remaining; + if (chunk_state_len(&self->chunk) > 0) { + cvs_remaining = self->cv_stack_len; + output = chunk_state_output(&self->chunk); + } else { + // There are always at least 2 CVs in the stack in this case. + cvs_remaining = self->cv_stack_len - 2; + output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key, + self->chunk.flags); + } + while (cvs_remaining > 0) { + cvs_remaining -= 1; + uint8_t parent_block[BLAKE3_BLOCK_LEN] BLAKE3_ALIGN; + memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32); + output_chaining_value(self, &output, &parent_block[32]); + output = parent_output(parent_block, self->key, self->chunk.flags); + } + output_root_bytes(self, &output, seek, out, out_len); +} + +void HASHER_OP(blake3_hasher_finalize) (const blake3_hasher *self, uint8_t *out, + size_t out_len) { + HASHER_OP(blake3_hasher_finalize_seek) (self, 0, out, out_len); +} + +void HASHER_OP(blake3_hasher_reset) (blake3_hasher *self) { + chunk_state_reset(&self->chunk, self->key, 0); + self->cv_stack_len = 0; +} +#ifdef GCC_DISABLE_WSTRING_OP_OVERREAD +#pragma GCC diagnostic pop +#endif + +const blake3_hasher_ops HASHER_OP(blake3_hasher_op) = { + .hasher_init = HASHER_OP(blake3_hasher_init), + .hasher_init_keyed = HASHER_OP(blake3_hasher_init_keyed), + .hasher_init_derive_key = HASHER_OP(blake3_hasher_init_derive_key), + .hasher_init_derive_key_raw = HASHER_OP(blake3_hasher_init_derive_key_raw), + .hasher_update = HASHER_OP(blake3_hasher_update), + .hasher_finalize = HASHER_OP(blake3_hasher_finalize), + .hasher_finalize_seek = HASHER_OP(blake3_hasher_finalize_seek), + .hasher_reset = HASHER_OP(blake3_hasher_reset), +}; diff --git a/src/blake3/blake3.h b/src/blake3/blake3.h new file mode 100644 index 0000000..e13d114 --- /dev/null +++ b/src/blake3/blake3.h @@ -0,0 +1,164 @@ +#ifndef BLAKE3_H +#define BLAKE3_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#define BLAKE3_VERSION_STRING "1.4.1" + +#define BLAKE3_KEY_LEN 32 +#define BLAKE3_KEY_WORDS (BLAKE3_KEY_LEN / 4) // words are 4 bytes +#define BLAKE3_OUT_LEN 32 +#define BLAKE3_OUT_WORDS (BLAKE3_OUT_LEN / 4) // words are 4 bytes +#define BLAKE3_BLOCK_LEN 64 +#define BLAKE3_CHUNK_LEN 1024 + +struct blake_host_state; +struct blake_hasher; + +const char *blake3_version(void); + +/* NOTE: maximum supported backends is 64 + * and backends are ordered by reverse order + * of preference. + * + * PORTABLE is always available and is the fall-back. + */ + +typedef enum { + + /* just the portable implementation */ + B3BID_PORTABLE, + + /* x86 */ + B3BID_SSE2, + B3BID_SSE2_ASM, + B3BID_SSE41, + B3BID_SSE41_ASM, + B3BID_AVX2, + B3BID_AVX2_ASM, + B3BID_AVX512, + B3BID_AVX512_ASM, + + /* arm */ + B3BID_NEON, + + /* gpu */ + B3BID_VULKAN, + B3BID_OPENCL, + B3BID_CUDA, + + /* CPU simd (experimental) */ + B3BID_CPUSIMD, + +} blake3_backend_id; + +#define B3BID_COUNT (B3BID_CPUSIMD+1) +#define B3BID_INVALID ((blake3_backend_id)-1) + +#define B3BF_BIT(_x) ((uint64_t)1 << (_x)) + +#define B3BF_PORTABLE B3BF_BIT(B3BID_PORTABLE) + +#define B3BF_SSE2 B3BF_BIT(B3BID_SSE2) +#define B3BF_SSE2_ASM B3BF_BIT(B3BID_SSE2_ASM) +#define B3BF_SSE41 B3BF_BIT(B3BID_SSE41) +#define B3BF_SSE41_ASM B3BF_BIT(B3BID_SSE41_ASM) +#define B3BF_AVX2 B3BF_BIT(B3BID_AVX2) +#define B3BF_AVX2_ASM B3BF_BIT(B3BID_AVX2_ASM) +#define B3BF_AVX512 B3BF_BIT(B3BID_AVX512) +#define B3BF_AVX512_ASM B3BF_BIT(B3BID_AVX512_ASM) + +#define B3BF_NEON B3BF_BIT(B3BID_NEON) + +#define B3BF_VULKAN B3BF_BIT(B3BID_VULKAN) +#define B3BF_OPENCL B3BF_BIT(B3BID_OPENCL) +#define B3BF_CUDA B3BF_BIT(B3BID_CUDA) + +#define B3BF_CPUSIMD B3BF_BIT(B3BID_CPUSIMD) + +uint64_t blake3_get_supported_backends(void); +uint64_t blake3_get_detected_backends(void); +uint64_t blake3_get_selectable_backends(void); + +typedef enum { + B3FID_HASH_MANY, + B3FID_COMPRESS_XOF, + B3FID_COMPRESS_IN_PLACE, +} blake3_func_id; + +#define B3FID_COUNT (B3FID_COMPRESS_IN_PLACE+1) +#define B3FID_INVALID ((blake3_func_id)-1) + +#define B3FF_BIT(_x) ((uint64_t)1 << (_x)) + +#define B3FF_HASH_MANY B3FF_BIT(B3FID_HASH_MANY) +#define B3FF_COMPRESS_XOF B3FF_BIT(B3FID_COMPRESS_XOF) +#define B3FF_COMPRESS_IN_PLACE B3FF_BIT(B3FID_COMPRESS_IN_PLACE) + +typedef struct blake3_backend_info { + blake3_backend_id id; + const char *name; + const char *description; + unsigned int simd_degree; + uint64_t funcs; +} blake3_backend_info; + +const blake3_backend_info *blake3_get_backend_info(blake3_backend_id id); + +typedef struct blake3_host_config { + bool debug; + bool no_mthread; + bool no_mmap; + unsigned int num_threads; + unsigned int mt_degree; // number of chunks to be worth spinning up a thread + const char *backend; // backend selection string, or NULL for default + size_t file_io_bufsz; // buffer size when doing file I/O + size_t mmap_min_chunk; // minimum chunk size for mmap + size_t mmap_max_chunk; // maximum chunk size for mmap + struct fy_thread_pool *tp; // use this thread pool instead of spinning one up +} blake3_host_config; + +struct blake3_host_state *blake3_host_state_create(const blake3_host_config *cfg); +void blake3_host_state_destroy(struct blake3_host_state *hs); + +struct blake3_hasher *blake3_hasher_create(struct blake3_host_state *hs, const uint8_t *key, const void *context, size_t context_len); +void blake3_hasher_destroy(struct blake3_hasher *self); + +/* simple optimized method for file hashing */ +int blake3_hash_file(struct blake3_hasher *hasher, const char *filename, uint8_t output[BLAKE3_OUT_LEN]); +void blake3_hash(struct blake3_hasher *hasher, const void *mem, size_t size, uint8_t output[BLAKE3_OUT_LEN]); + +/* experimental CPUSIMD enable */ +int blake3_backend_cpusimd_setup(unsigned int num_cpus, unsigned int mult_fact); +void blake3_backend_cpusimd_cleanup(void); + +void blake3_hasher_init(struct blake3_host_state *hs, struct blake3_hasher *self); +void blake3_hasher_init_keyed(struct blake3_host_state *hs, struct blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]); +void blake3_hasher_init_derive_key(struct blake3_host_state *hs, struct blake3_hasher *self, const char *context); +void blake3_hasher_init_derive_key_raw(struct blake3_host_state *hs, struct blake3_hasher *self, const void *context, size_t context_len); + +void blake3_hasher_update(struct blake3_hasher *self, const void *input, size_t input_len); +void blake3_hasher_finalize(const struct blake3_hasher *self, uint8_t *out, size_t out_len); +void blake3_hasher_finalize_seek(const struct blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len); +void blake3_hasher_reset(struct blake3_hasher *self); + +typedef struct blake3_hasher_ops { + void (*hasher_init)(struct blake3_host_state *hs, struct blake3_hasher *self); + void (*hasher_init_keyed)(struct blake3_host_state *hs, struct blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]); + void (*hasher_init_derive_key)(struct blake3_host_state *hs, struct blake3_hasher *self, const char *context); + void (*hasher_init_derive_key_raw)(struct blake3_host_state *hs, struct blake3_hasher *self, const void *context, size_t context_len); + + void (*hasher_update)(struct blake3_hasher *self, const void *input, size_t input_len); + void (*hasher_finalize)(const struct blake3_hasher *self, uint8_t *out, size_t out_len); + void (*hasher_finalize_seek)(const struct blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len); + void (*hasher_reset)(struct blake3_hasher *self); +} blake3_hasher_ops; + +#endif /* BLAKE3_H */ diff --git a/src/blake3/blake3_avx2.c b/src/blake3/blake3_avx2.c new file mode 100644 index 0000000..8d5b427 --- /dev/null +++ b/src/blake3/blake3_avx2.c @@ -0,0 +1,333 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" + +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +#define DEGREE 8 + +INLINE __m256i loadu(const uint8_t src[32]) { + return _mm256_loadu_si256((const __m256i *)src); +} + +INLINE void storeu(__m256i src, uint8_t dest[16]) { + _mm256_storeu_si256((__m256i *)dest, src); +} + +INLINE __m256i addv(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } + +// Note that clang-format doesn't like the name "xor" for some reason. +INLINE __m256i xorv(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } + +INLINE __m256i set1(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } + +INLINE __m256i rot16(__m256i x) { + return _mm256_shuffle_epi8( + x, _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, + 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); +} + +INLINE __m256i rot12(__m256i x) { + return _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 32 - 12)); +} + +INLINE __m256i rot8(__m256i x) { + return _mm256_shuffle_epi8( + x, _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1, + 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); +} + +INLINE __m256i rot7(__m256i x) { + return _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 32 - 7)); +} + +INLINE void round_fn(__m256i v[16], __m256i m[16], size_t r) { + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[15] = rot16(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot12(v[4]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[15] = rot8(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot7(v[4]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot16(v[15]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[4] = rot12(v[4]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot8(v[15]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + v[4] = rot7(v[4]); +} + +INLINE void transpose_vecs(__m256i vecs[DEGREE]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high + // is 22/33/66/77. + __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); + __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); + __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); + __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); + __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); + __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); + __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); + __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); + + // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is + // 11/33. + __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); + __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); + __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); + __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); + __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); + __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); + __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); + __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); + + // Interleave 128-bit lanes. + vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); + vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); + vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); + vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); + vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); + vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); + vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); + vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); +} + +INLINE void transpose_msg_vecs(const uint8_t *const *inputs, + size_t block_offset, __m256i out[16]) { + out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m256i)]); + out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m256i)]); + out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m256i)]); + out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m256i)]); + out[4] = loadu(&inputs[4][block_offset + 0 * sizeof(__m256i)]); + out[5] = loadu(&inputs[5][block_offset + 0 * sizeof(__m256i)]); + out[6] = loadu(&inputs[6][block_offset + 0 * sizeof(__m256i)]); + out[7] = loadu(&inputs[7][block_offset + 0 * sizeof(__m256i)]); + out[8] = loadu(&inputs[0][block_offset + 1 * sizeof(__m256i)]); + out[9] = loadu(&inputs[1][block_offset + 1 * sizeof(__m256i)]); + out[10] = loadu(&inputs[2][block_offset + 1 * sizeof(__m256i)]); + out[11] = loadu(&inputs[3][block_offset + 1 * sizeof(__m256i)]); + out[12] = loadu(&inputs[4][block_offset + 1 * sizeof(__m256i)]); + out[13] = loadu(&inputs[5][block_offset + 1 * sizeof(__m256i)]); + out[14] = loadu(&inputs[6][block_offset + 1 * sizeof(__m256i)]); + out[15] = loadu(&inputs[7][block_offset + 1 * sizeof(__m256i)]); + for (size_t i = 0; i < 8; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs(&out[0]); + transpose_vecs(&out[8]); +} + +INLINE void load_counters(uint64_t counter, bool increment_counter, + __m256i *out_lo, __m256i *out_hi) { + const __m256i mask = _mm256_set1_epi32(-(int32_t)increment_counter); + const __m256i add0 = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + const __m256i add1 = _mm256_and_si256(mask, add0); + __m256i l = _mm256_add_epi32(_mm256_set1_epi32((int32_t)counter), add1); + __m256i carry = _mm256_cmpgt_epi32(_mm256_xor_si256(add1, _mm256_set1_epi32(0x80000000)), + _mm256_xor_si256( l, _mm256_set1_epi32(0x80000000))); + __m256i h = _mm256_sub_epi32(_mm256_set1_epi32((int32_t)(counter >> 32)), carry); + *out_lo = l; + *out_hi = h; +} + +static +void blake3_hash8_avx2(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m256i h_vecs[8] = { + set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), + set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), + }; + __m256i counter_low_vec, counter_high_vec; + load_counters(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m256i block_len_vec = set1(BLAKE3_BLOCK_LEN); + __m256i block_flags_vec = set1(block_flags); + __m256i msg_vecs[16]; + transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m256i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1(B3_IV_0), set1(B3_IV_1), set1(B3_IV_2), set1(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn(v, msg_vecs, 0); + round_fn(v, msg_vecs, 1); + round_fn(v, msg_vecs, 2); + round_fn(v, msg_vecs, 3); + round_fn(v, msg_vecs, 4); + round_fn(v, msg_vecs, 5); + round_fn(v, msg_vecs, 6); + h_vecs[0] = xorv(v[0], v[8]); + h_vecs[1] = xorv(v[1], v[9]); + h_vecs[2] = xorv(v[2], v[10]); + h_vecs[3] = xorv(v[3], v[11]); + h_vecs[4] = xorv(v[4], v[12]); + h_vecs[5] = xorv(v[5], v[13]); + h_vecs[6] = xorv(v[6], v[14]); + h_vecs[7] = xorv(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs(h_vecs); + storeu(h_vecs[0], &out[0 * sizeof(__m256i)]); + storeu(h_vecs[1], &out[1 * sizeof(__m256i)]); + storeu(h_vecs[2], &out[2 * sizeof(__m256i)]); + storeu(h_vecs[3], &out[3 * sizeof(__m256i)]); + storeu(h_vecs[4], &out[4 * sizeof(__m256i)]); + storeu(h_vecs[5], &out[5 * sizeof(__m256i)]); + storeu(h_vecs[6], &out[6 * sizeof(__m256i)]); + storeu(h_vecs[7], &out[7 * sizeof(__m256i)]); +} + +#if !defined(BLAKE3_NO_SSE41) +void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#else +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif + +void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= DEGREE) { + blake3_hash8_avx2(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += DEGREE; + } + inputs += DEGREE; + num_inputs -= DEGREE; + out = &out[DEGREE * BLAKE3_OUT_LEN]; + } +#if !defined(BLAKE3_NO_SSE41) + blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, out); +#else + blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); +#endif +} diff --git a/src/blake3/blake3_avx2_x86-64_unix.S b/src/blake3/blake3_avx2_x86-64_unix.S new file mode 100644 index 0000000..29729ec --- /dev/null +++ b/src/blake3/blake3_avx2_x86-64_unix.S @@ -0,0 +1,1815 @@ +#if defined(__ELF__) && defined(__linux__) +.section .note.GNU-stack,"",%progbits +#endif + +#if defined(__ELF__) && defined(__CET__) && defined(__has_include) +#if __has_include() +#include +#endif +#endif + +#if !defined(_CET_ENDBR) +#define _CET_ENDBR +#endif + +.intel_syntax noprefix +.global _blake3_hash_many_avx2_asm +.global blake3_hash_many_avx2_asm +#ifdef __APPLE__ +.text +#else +.section .text +#endif + .p2align 6 +_blake3_hash_many_avx2_asm: +blake3_hash_many_avx2_asm: + _CET_ENDBR + push r15 + push r14 + push r13 + push r12 + push rbx + push rbp + mov rbp, rsp + sub rsp, 680 + and rsp, 0xFFFFFFFFFFFFFFC0 + neg r9d + vmovd xmm0, r9d + vpbroadcastd ymm0, xmm0 + vmovdqa ymmword ptr [rsp+0x280], ymm0 + vpand ymm1, ymm0, ymmword ptr [ADD0+rip] + vpand ymm2, ymm0, ymmword ptr [ADD1+rip] + vmovdqa ymmword ptr [rsp+0x220], ymm2 + vmovd xmm2, r8d + vpbroadcastd ymm2, xmm2 + vpaddd ymm2, ymm2, ymm1 + vmovdqa ymmword ptr [rsp+0x240], ymm2 + vpxor ymm1, ymm1, ymmword ptr [CMP_MSB_MASK+rip] + vpxor ymm2, ymm2, ymmword ptr [CMP_MSB_MASK+rip] + vpcmpgtd ymm2, ymm1, ymm2 + shr r8, 32 + vmovd xmm3, r8d + vpbroadcastd ymm3, xmm3 + vpsubd ymm3, ymm3, ymm2 + vmovdqa ymmword ptr [rsp+0x260], ymm3 + shl rdx, 6 + mov qword ptr [rsp+0x2A0], rdx + cmp rsi, 8 + jc 3f +2: + vpbroadcastd ymm0, dword ptr [rcx] + vpbroadcastd ymm1, dword ptr [rcx+0x4] + vpbroadcastd ymm2, dword ptr [rcx+0x8] + vpbroadcastd ymm3, dword ptr [rcx+0xC] + vpbroadcastd ymm4, dword ptr [rcx+0x10] + vpbroadcastd ymm5, dword ptr [rcx+0x14] + vpbroadcastd ymm6, dword ptr [rcx+0x18] + vpbroadcastd ymm7, dword ptr [rcx+0x1C] + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + mov r12, qword ptr [rdi+0x20] + mov r13, qword ptr [rdi+0x28] + mov r14, qword ptr [rdi+0x30] + mov r15, qword ptr [rdi+0x38] + movzx eax, byte ptr [rbp+0x38] + movzx ebx, byte ptr [rbp+0x40] + or eax, ebx + xor edx, edx +.p2align 5 +9: + movzx ebx, byte ptr [rbp+0x48] + or ebx, eax + add rdx, 64 + cmp rdx, qword ptr [rsp+0x2A0] + cmove eax, ebx + mov dword ptr [rsp+0x200], eax + vmovups xmm8, xmmword ptr [r8+rdx-0x40] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x40], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x40] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x40], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x40] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x40], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x40] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x40], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm8, ymm12, ymm14, 136 + vmovaps ymmword ptr [rsp], ymm8 + vshufps ymm9, ymm12, ymm14, 221 + vmovaps ymmword ptr [rsp+0x20], ymm9 + vshufps ymm10, ymm13, ymm15, 136 + vmovaps ymmword ptr [rsp+0x40], ymm10 + vshufps ymm11, ymm13, ymm15, 221 + vmovaps ymmword ptr [rsp+0x60], ymm11 + vmovups xmm8, xmmword ptr [r8+rdx-0x30] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x30], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x30] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x30], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x30] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x30], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x30] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x30], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm8, ymm12, ymm14, 136 + vmovaps ymmword ptr [rsp+0x80], ymm8 + vshufps ymm9, ymm12, ymm14, 221 + vmovaps ymmword ptr [rsp+0xA0], ymm9 + vshufps ymm10, ymm13, ymm15, 136 + vmovaps ymmword ptr [rsp+0xC0], ymm10 + vshufps ymm11, ymm13, ymm15, 221 + vmovaps ymmword ptr [rsp+0xE0], ymm11 + vmovups xmm8, xmmword ptr [r8+rdx-0x20] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x20], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x20] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x20], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x20] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x20], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x20] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x20], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm8, ymm12, ymm14, 136 + vmovaps ymmword ptr [rsp+0x100], ymm8 + vshufps ymm9, ymm12, ymm14, 221 + vmovaps ymmword ptr [rsp+0x120], ymm9 + vshufps ymm10, ymm13, ymm15, 136 + vmovaps ymmword ptr [rsp+0x140], ymm10 + vshufps ymm11, ymm13, ymm15, 221 + vmovaps ymmword ptr [rsp+0x160], ymm11 + vmovups xmm8, xmmword ptr [r8+rdx-0x10] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x10], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x10] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x10], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x10] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x10], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x10] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x10], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm8, ymm12, ymm14, 136 + vmovaps ymmword ptr [rsp+0x180], ymm8 + vshufps ymm9, ymm12, ymm14, 221 + vmovaps ymmword ptr [rsp+0x1A0], ymm9 + vshufps ymm10, ymm13, ymm15, 136 + vmovaps ymmword ptr [rsp+0x1C0], ymm10 + vshufps ymm11, ymm13, ymm15, 221 + vmovaps ymmword ptr [rsp+0x1E0], ymm11 + vpbroadcastd ymm15, dword ptr [rsp+0x200] + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r12+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r13+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r14+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + prefetcht0 [r15+rdx+0x80] + vpaddd ymm0, ymm0, ymmword ptr [rsp] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x40] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x80] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm0, ymmword ptr [rsp+0x240] + vpxor ymm13, ymm1, ymmword ptr [rsp+0x260] + vpxor ymm14, ymm2, ymmword ptr [BLAKE3_BLOCK_LEN+rip] + vpxor ymm15, ymm3, ymm15 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [BLAKE3_IV_0+rip] + vpaddd ymm9, ymm13, ymmword ptr [BLAKE3_IV_1+rip] + vpaddd ymm10, ymm14, ymmword ptr [BLAKE3_IV_2+rip] + vpaddd ymm11, ymm15, ymmword ptr [BLAKE3_IV_3+rip] + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x20] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x60] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0xA0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x100] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x180] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1C0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x120] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1A0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x40] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x60] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0xE0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x80] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0xC0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140] + vpaddd ymm2, ymm2, ymmword ptr [rsp] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1A0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x20] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x120] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x160] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1C0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x60] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1A0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x80] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x40] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1C0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0xC0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x120] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x160] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0xA0] + vpaddd ymm1, ymm1, ymmword ptr [rsp] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1E0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x20] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x140] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1C0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1A0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0xE0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x120] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x60] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x80] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0xA0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x20] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x40] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x100] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x180] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x120] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1E0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1C0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1A0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x140] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0xE0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0] + vpaddd ymm2, ymm2, ymmword ptr [rsp] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x40] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x60] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x20] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x80] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x120] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x100] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1C0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x180] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x20] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1A0] + vpaddd ymm1, ymm1, ymmword ptr [rsp] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x40] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x80] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x60] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0xC0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x160] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x20] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1E0] + vpaddd ymm1, ymm1, ymmword ptr [rsp] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x120] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1C0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x40] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x60] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x140] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x80] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1A0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vpxor ymm0, ymm0, ymm8 + vpxor ymm1, ymm1, ymm9 + vpxor ymm2, ymm2, ymm10 + vpxor ymm3, ymm3, ymm11 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpxor ymm4, ymm4, ymm12 + vpxor ymm5, ymm5, ymm13 + vpxor ymm6, ymm6, ymm14 + vpxor ymm7, ymm7, ymm15 + movzx eax, byte ptr [rbp+0x38] + jne 9b + mov rbx, qword ptr [rbp+0x50] + vunpcklps ymm8, ymm0, ymm1 + vunpcklps ymm9, ymm2, ymm3 + vunpckhps ymm10, ymm0, ymm1 + vunpcklps ymm11, ymm4, ymm5 + vunpcklps ymm0, ymm6, ymm7 + vshufps ymm12, ymm8, ymm9, 78 + vblendps ymm1, ymm8, ymm12, 0xCC + vshufps ymm8, ymm11, ymm0, 78 + vunpckhps ymm13, ymm2, ymm3 + vblendps ymm2, ymm11, ymm8, 0xCC + vblendps ymm3, ymm12, ymm9, 0xCC + vperm2f128 ymm12, ymm1, ymm2, 0x20 + vmovups ymmword ptr [rbx], ymm12 + vunpckhps ymm14, ymm4, ymm5 + vblendps ymm4, ymm8, ymm0, 0xCC + vunpckhps ymm15, ymm6, ymm7 + vperm2f128 ymm7, ymm3, ymm4, 0x20 + vmovups ymmword ptr [rbx+0x20], ymm7 + vshufps ymm5, ymm10, ymm13, 78 + vblendps ymm6, ymm5, ymm13, 0xCC + vshufps ymm13, ymm14, ymm15, 78 + vblendps ymm10, ymm10, ymm5, 0xCC + vblendps ymm14, ymm14, ymm13, 0xCC + vperm2f128 ymm8, ymm10, ymm14, 0x20 + vmovups ymmword ptr [rbx+0x40], ymm8 + vblendps ymm15, ymm13, ymm15, 0xCC + vperm2f128 ymm13, ymm6, ymm15, 0x20 + vmovups ymmword ptr [rbx+0x60], ymm13 + vperm2f128 ymm9, ymm1, ymm2, 0x31 + vperm2f128 ymm11, ymm3, ymm4, 0x31 + vmovups ymmword ptr [rbx+0x80], ymm9 + vperm2f128 ymm14, ymm10, ymm14, 0x31 + vperm2f128 ymm15, ymm6, ymm15, 0x31 + vmovups ymmword ptr [rbx+0xA0], ymm11 + vmovups ymmword ptr [rbx+0xC0], ymm14 + vmovups ymmword ptr [rbx+0xE0], ymm15 + vmovdqa ymm0, ymmword ptr [rsp+0x220] + vpaddd ymm1, ymm0, ymmword ptr [rsp+0x240] + vmovdqa ymmword ptr [rsp+0x240], ymm1 + vpxor ymm0, ymm0, ymmword ptr [CMP_MSB_MASK+rip] + vpxor ymm2, ymm1, ymmword ptr [CMP_MSB_MASK+rip] + vpcmpgtd ymm2, ymm0, ymm2 + vmovdqa ymm0, ymmword ptr [rsp+0x260] + vpsubd ymm2, ymm0, ymm2 + vmovdqa ymmword ptr [rsp+0x260], ymm2 + add rdi, 64 + add rbx, 256 + mov qword ptr [rbp+0x50], rbx + sub rsi, 8 + cmp rsi, 8 + jnc 2b + test rsi, rsi + jnz 3f +4: + vzeroupper + mov rsp, rbp + pop rbp + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + ret +.p2align 5 +3: + mov rbx, qword ptr [rbp+0x50] + mov r15, qword ptr [rsp+0x2A0] + movzx r13d, byte ptr [rbp+0x38] + movzx r12d, byte ptr [rbp+0x48] + test rsi, 0x4 + je 3f + vbroadcasti128 ymm0, xmmword ptr [rcx] + vbroadcasti128 ymm1, xmmword ptr [rcx+0x10] + vmovdqa ymm8, ymm0 + vmovdqa ymm9, ymm1 + vbroadcasti128 ymm12, xmmword ptr [rsp+0x240] + vbroadcasti128 ymm13, xmmword ptr [rsp+0x260] + vpunpckldq ymm14, ymm12, ymm13 + vpunpckhdq ymm15, ymm12, ymm13 + vpermq ymm14, ymm14, 0x50 + vpermq ymm15, ymm15, 0x50 + vbroadcasti128 ymm12, xmmword ptr [BLAKE3_BLOCK_LEN+rip] + vpblendd ymm14, ymm14, ymm12, 0x44 + vpblendd ymm15, ymm15, ymm12, 0x44 + vmovdqa ymmword ptr [rsp], ymm14 + vmovdqa ymmword ptr [rsp+0x20], ymm15 + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + mov dword ptr [rsp+0x200], eax + vmovups ymm2, ymmword ptr [r8+rdx-0x40] + vinsertf128 ymm2, ymm2, xmmword ptr [r9+rdx-0x40], 0x01 + vmovups ymm3, ymmword ptr [r8+rdx-0x30] + vinsertf128 ymm3, ymm3, xmmword ptr [r9+rdx-0x30], 0x01 + vshufps ymm4, ymm2, ymm3, 136 + vshufps ymm5, ymm2, ymm3, 221 + vmovups ymm2, ymmword ptr [r8+rdx-0x20] + vinsertf128 ymm2, ymm2, xmmword ptr [r9+rdx-0x20], 0x01 + vmovups ymm3, ymmword ptr [r8+rdx-0x10] + vinsertf128 ymm3, ymm3, xmmword ptr [r9+rdx-0x10], 0x01 + vshufps ymm6, ymm2, ymm3, 136 + vshufps ymm7, ymm2, ymm3, 221 + vpshufd ymm6, ymm6, 0x93 + vpshufd ymm7, ymm7, 0x93 + vmovups ymm10, ymmword ptr [r10+rdx-0x40] + vinsertf128 ymm10, ymm10, xmmword ptr [r11+rdx-0x40], 0x01 + vmovups ymm11, ymmword ptr [r10+rdx-0x30] + vinsertf128 ymm11, ymm11, xmmword ptr [r11+rdx-0x30], 0x01 + vshufps ymm12, ymm10, ymm11, 136 + vshufps ymm13, ymm10, ymm11, 221 + vmovups ymm10, ymmword ptr [r10+rdx-0x20] + vinsertf128 ymm10, ymm10, xmmword ptr [r11+rdx-0x20], 0x01 + vmovups ymm11, ymmword ptr [r10+rdx-0x10] + vinsertf128 ymm11, ymm11, xmmword ptr [r11+rdx-0x10], 0x01 + vshufps ymm14, ymm10, ymm11, 136 + vshufps ymm15, ymm10, ymm11, 221 + vpshufd ymm14, ymm14, 0x93 + vpshufd ymm15, ymm15, 0x93 + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + vpbroadcastd ymm2, dword ptr [rsp+0x200] + vmovdqa ymm3, ymmword ptr [rsp] + vmovdqa ymm11, ymmword ptr [rsp+0x20] + vpblendd ymm3, ymm3, ymm2, 0x88 + vpblendd ymm11, ymm11, ymm2, 0x88 + vbroadcasti128 ymm2, xmmword ptr [BLAKE3_IV+rip] + vmovdqa ymm10, ymm2 + mov al, 7 +9: + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm8, ymm8, ymm12 + vmovdqa ymmword ptr [rsp+0x40], ymm4 + nop + vmovdqa ymmword ptr [rsp+0x60], ymm12 + nop + vpaddd ymm0, ymm0, ymm1 + vpaddd ymm8, ymm8, ymm9 + vpxor ymm3, ymm3, ymm0 + vpxor ymm11, ymm11, ymm8 + vbroadcasti128 ymm4, xmmword ptr [ROT16+rip] + vpshufb ymm3, ymm3, ymm4 + vpshufb ymm11, ymm11, ymm4 + vpaddd ymm2, ymm2, ymm3 + vpaddd ymm10, ymm10, ymm11 + vpxor ymm1, ymm1, ymm2 + vpxor ymm9, ymm9, ymm10 + vpsrld ymm4, ymm1, 12 + vpslld ymm1, ymm1, 20 + vpor ymm1, ymm1, ymm4 + vpsrld ymm4, ymm9, 12 + vpslld ymm9, ymm9, 20 + vpor ymm9, ymm9, ymm4 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm0, ymm0, ymm1 + vpaddd ymm8, ymm8, ymm9 + vmovdqa ymmword ptr [rsp+0x80], ymm5 + vmovdqa ymmword ptr [rsp+0xA0], ymm13 + vpxor ymm3, ymm3, ymm0 + vpxor ymm11, ymm11, ymm8 + vbroadcasti128 ymm4, xmmword ptr [ROT8+rip] + vpshufb ymm3, ymm3, ymm4 + vpshufb ymm11, ymm11, ymm4 + vpaddd ymm2, ymm2, ymm3 + vpaddd ymm10, ymm10, ymm11 + vpxor ymm1, ymm1, ymm2 + vpxor ymm9, ymm9, ymm10 + vpsrld ymm4, ymm1, 7 + vpslld ymm1, ymm1, 25 + vpor ymm1, ymm1, ymm4 + vpsrld ymm4, ymm9, 7 + vpslld ymm9, ymm9, 25 + vpor ymm9, ymm9, ymm4 + vpshufd ymm0, ymm0, 0x93 + vpshufd ymm8, ymm8, 0x93 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm11, ymm11, 0x4E + vpshufd ymm2, ymm2, 0x39 + vpshufd ymm10, ymm10, 0x39 + vpaddd ymm0, ymm0, ymm6 + vpaddd ymm8, ymm8, ymm14 + vpaddd ymm0, ymm0, ymm1 + vpaddd ymm8, ymm8, ymm9 + vpxor ymm3, ymm3, ymm0 + vpxor ymm11, ymm11, ymm8 + vbroadcasti128 ymm4, xmmword ptr [ROT16+rip] + vpshufb ymm3, ymm3, ymm4 + vpshufb ymm11, ymm11, ymm4 + vpaddd ymm2, ymm2, ymm3 + vpaddd ymm10, ymm10, ymm11 + vpxor ymm1, ymm1, ymm2 + vpxor ymm9, ymm9, ymm10 + vpsrld ymm4, ymm1, 12 + vpslld ymm1, ymm1, 20 + vpor ymm1, ymm1, ymm4 + vpsrld ymm4, ymm9, 12 + vpslld ymm9, ymm9, 20 + vpor ymm9, ymm9, ymm4 + vpaddd ymm0, ymm0, ymm7 + vpaddd ymm8, ymm8, ymm15 + vpaddd ymm0, ymm0, ymm1 + vpaddd ymm8, ymm8, ymm9 + vpxor ymm3, ymm3, ymm0 + vpxor ymm11, ymm11, ymm8 + vbroadcasti128 ymm4, xmmword ptr [ROT8+rip] + vpshufb ymm3, ymm3, ymm4 + vpshufb ymm11, ymm11, ymm4 + vpaddd ymm2, ymm2, ymm3 + vpaddd ymm10, ymm10, ymm11 + vpxor ymm1, ymm1, ymm2 + vpxor ymm9, ymm9, ymm10 + vpsrld ymm4, ymm1, 7 + vpslld ymm1, ymm1, 25 + vpor ymm1, ymm1, ymm4 + vpsrld ymm4, ymm9, 7 + vpslld ymm9, ymm9, 25 + vpor ymm9, ymm9, ymm4 + vpshufd ymm0, ymm0, 0x39 + vpshufd ymm8, ymm8, 0x39 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm11, ymm11, 0x4E + vpshufd ymm2, ymm2, 0x93 + vpshufd ymm10, ymm10, 0x93 + dec al + je 9f + vmovdqa ymm4, ymmword ptr [rsp+0x40] + vmovdqa ymm5, ymmword ptr [rsp+0x80] + vshufps ymm12, ymm4, ymm5, 214 + vpshufd ymm13, ymm4, 0x0F + vpshufd ymm4, ymm12, 0x39 + vshufps ymm12, ymm6, ymm7, 250 + vpblendd ymm13, ymm13, ymm12, 0xAA + vpunpcklqdq ymm12, ymm7, ymm5 + vpblendd ymm12, ymm12, ymm6, 0x88 + vpshufd ymm12, ymm12, 0x78 + vpunpckhdq ymm5, ymm5, ymm7 + vpunpckldq ymm6, ymm6, ymm5 + vpshufd ymm7, ymm6, 0x1E + vmovdqa ymmword ptr [rsp+0x40], ymm13 + vmovdqa ymmword ptr [rsp+0x80], ymm12 + vmovdqa ymm12, ymmword ptr [rsp+0x60] + vmovdqa ymm13, ymmword ptr [rsp+0xA0] + vshufps ymm5, ymm12, ymm13, 214 + vpshufd ymm6, ymm12, 0x0F + vpshufd ymm12, ymm5, 0x39 + vshufps ymm5, ymm14, ymm15, 250 + vpblendd ymm6, ymm6, ymm5, 0xAA + vpunpcklqdq ymm5, ymm15, ymm13 + vpblendd ymm5, ymm5, ymm14, 0x88 + vpshufd ymm5, ymm5, 0x78 + vpunpckhdq ymm13, ymm13, ymm15 + vpunpckldq ymm14, ymm14, ymm13 + vpshufd ymm15, ymm14, 0x1E + vmovdqa ymm13, ymm6 + vmovdqa ymm14, ymm5 + vmovdqa ymm5, ymmword ptr [rsp+0x40] + vmovdqa ymm6, ymmword ptr [rsp+0x80] + jmp 9b +9: + vpxor ymm0, ymm0, ymm2 + vpxor ymm1, ymm1, ymm3 + vpxor ymm8, ymm8, ymm10 + vpxor ymm9, ymm9, ymm11 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01 + vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01 + vmovdqu xmmword ptr [rbx+0x40], xmm8 + vmovdqu xmmword ptr [rbx+0x50], xmm9 + vextracti128 xmmword ptr [rbx+0x60], ymm8, 0x01 + vextracti128 xmmword ptr [rbx+0x70], ymm9, 0x01 + vmovaps xmm8, xmmword ptr [rsp+0x280] + vmovaps xmm0, xmmword ptr [rsp+0x240] + vmovaps xmm1, xmmword ptr [rsp+0x250] + vmovaps xmm2, xmmword ptr [rsp+0x260] + vmovaps xmm3, xmmword ptr [rsp+0x270] + vblendvps xmm0, xmm0, xmm1, xmm8 + vblendvps xmm2, xmm2, xmm3, xmm8 + vmovaps xmmword ptr [rsp+0x240], xmm0 + vmovaps xmmword ptr [rsp+0x260], xmm2 + add rbx, 128 + add rdi, 32 + sub rsi, 4 +3: + test rsi, 0x2 + je 3f + vbroadcasti128 ymm0, xmmword ptr [rcx] + vbroadcasti128 ymm1, xmmword ptr [rcx+0x10] + vmovd xmm13, dword ptr [rsp+0x240] + vpinsrd xmm13, xmm13, dword ptr [rsp+0x260], 1 + vpinsrd xmm13, xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vmovd xmm14, dword ptr [rsp+0x244] + vpinsrd xmm14, xmm14, dword ptr [rsp+0x264], 1 + vpinsrd xmm14, xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vinserti128 ymm13, ymm13, xmm14, 0x01 + vbroadcasti128 ymm14, xmmword ptr [ROT16+rip] + vbroadcasti128 ymm15, xmmword ptr [ROT8+rip] + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + mov dword ptr [rsp+0x200], eax + vbroadcasti128 ymm2, xmmword ptr [BLAKE3_IV+rip] + vpbroadcastd ymm8, dword ptr [rsp+0x200] + vpblendd ymm3, ymm13, ymm8, 0x88 + vmovups ymm8, ymmword ptr [r8+rdx-0x40] + vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x40], 0x01 + vmovups ymm9, ymmword ptr [r8+rdx-0x30] + vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x30], 0x01 + vshufps ymm4, ymm8, ymm9, 136 + vshufps ymm5, ymm8, ymm9, 221 + vmovups ymm8, ymmword ptr [r8+rdx-0x20] + vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x20], 0x01 + vmovups ymm9, ymmword ptr [r8+rdx-0x10] + vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x10], 0x01 + vshufps ymm6, ymm8, ymm9, 136 + vshufps ymm7, ymm8, ymm9, 221 + vpshufd ymm6, ymm6, 0x93 + vpshufd ymm7, ymm7, 0x93 + mov al, 7 +9: + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm0, ymm0, ymm1 + vpxor ymm3, ymm3, ymm0 + vpshufb ymm3, ymm3, ymm14 + vpaddd ymm2, ymm2, ymm3 + vpxor ymm1, ymm1, ymm2 + vpsrld ymm8, ymm1, 12 + vpslld ymm1, ymm1, 20 + vpor ymm1, ymm1, ymm8 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm0, ymm0, ymm1 + vpxor ymm3, ymm3, ymm0 + vpshufb ymm3, ymm3, ymm15 + vpaddd ymm2, ymm2, ymm3 + vpxor ymm1, ymm1, ymm2 + vpsrld ymm8, ymm1, 7 + vpslld ymm1, ymm1, 25 + vpor ymm1, ymm1, ymm8 + vpshufd ymm0, ymm0, 0x93 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm2, ymm2, 0x39 + vpaddd ymm0, ymm0, ymm6 + vpaddd ymm0, ymm0, ymm1 + vpxor ymm3, ymm3, ymm0 + vpshufb ymm3, ymm3, ymm14 + vpaddd ymm2, ymm2, ymm3 + vpxor ymm1, ymm1, ymm2 + vpsrld ymm8, ymm1, 12 + vpslld ymm1, ymm1, 20 + vpor ymm1, ymm1, ymm8 + vpaddd ymm0, ymm0, ymm7 + vpaddd ymm0, ymm0, ymm1 + vpxor ymm3, ymm3, ymm0 + vpshufb ymm3, ymm3, ymm15 + vpaddd ymm2, ymm2, ymm3 + vpxor ymm1, ymm1, ymm2 + vpsrld ymm8, ymm1, 7 + vpslld ymm1, ymm1, 25 + vpor ymm1, ymm1, ymm8 + vpshufd ymm0, ymm0, 0x39 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm2, ymm2, 0x93 + dec al + jz 9f + vshufps ymm8, ymm4, ymm5, 214 + vpshufd ymm9, ymm4, 0x0F + vpshufd ymm4, ymm8, 0x39 + vshufps ymm8, ymm6, ymm7, 250 + vpblendd ymm9, ymm9, ymm8, 0xAA + vpunpcklqdq ymm8, ymm7, ymm5 + vpblendd ymm8, ymm8, ymm6, 0x88 + vpshufd ymm8, ymm8, 0x78 + vpunpckhdq ymm5, ymm5, ymm7 + vpunpckldq ymm6, ymm6, ymm5 + vpshufd ymm7, ymm6, 0x1E + vmovdqa ymm5, ymm9 + vmovdqa ymm6, ymm8 + jmp 9b +9: + vpxor ymm0, ymm0, ymm2 + vpxor ymm1, ymm1, ymm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01 + vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01 + vmovaps ymm8, ymmword ptr [rsp+0x280] + vmovaps ymm0, ymmword ptr [rsp+0x240] + vmovups ymm1, ymmword ptr [rsp+0x248] + vmovaps ymm2, ymmword ptr [rsp+0x260] + vmovups ymm3, ymmword ptr [rsp+0x268] + vblendvps ymm0, ymm0, ymm1, ymm8 + vblendvps ymm2, ymm2, ymm3, ymm8 + vmovaps ymmword ptr [rsp+0x240], ymm0 + vmovaps ymmword ptr [rsp+0x260], ymm2 + add rbx, 64 + add rdi, 16 + sub rsi, 2 +3: + test rsi, 0x1 + je 4b + vmovdqu xmm0, xmmword ptr [rcx] + vmovdqu xmm1, xmmword ptr [rcx+0x10] + vmovd xmm3, dword ptr [rsp+0x240] + vpinsrd xmm3, xmm3, dword ptr [rsp+0x260], 1 + vpinsrd xmm13, xmm3, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vmovdqa xmm14, xmmword ptr [ROT16+rip] + vmovdqa xmm15, xmmword ptr [ROT8+rip] + mov r8, qword ptr [rdi] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + vmovdqa xmm2, xmmword ptr [BLAKE3_IV+rip] + vmovdqa xmm3, xmm13 + vpinsrd xmm3, xmm3, eax, 3 + vmovups xmm8, xmmword ptr [r8+rdx-0x40] + vmovups xmm9, xmmword ptr [r8+rdx-0x30] + vshufps xmm4, xmm8, xmm9, 136 + vshufps xmm5, xmm8, xmm9, 221 + vmovups xmm8, xmmword ptr [r8+rdx-0x20] + vmovups xmm9, xmmword ptr [r8+rdx-0x10] + vshufps xmm6, xmm8, xmm9, 136 + vshufps xmm7, xmm8, xmm9, 221 + vpshufd xmm6, xmm6, 0x93 + vpshufd xmm7, xmm7, 0x93 + mov al, 7 +9: + vpaddd xmm0, xmm0, xmm4 + vpaddd xmm0, xmm0, xmm1 + vpxor xmm3, xmm3, xmm0 + vpshufb xmm3, xmm3, xmm14 + vpaddd xmm2, xmm2, xmm3 + vpxor xmm1, xmm1, xmm2 + vpsrld xmm8, xmm1, 12 + vpslld xmm1, xmm1, 20 + vpor xmm1, xmm1, xmm8 + vpaddd xmm0, xmm0, xmm5 + vpaddd xmm0, xmm0, xmm1 + vpxor xmm3, xmm3, xmm0 + vpshufb xmm3, xmm3, xmm15 + vpaddd xmm2, xmm2, xmm3 + vpxor xmm1, xmm1, xmm2 + vpsrld xmm8, xmm1, 7 + vpslld xmm1, xmm1, 25 + vpor xmm1, xmm1, xmm8 + vpshufd xmm0, xmm0, 0x93 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x39 + vpaddd xmm0, xmm0, xmm6 + vpaddd xmm0, xmm0, xmm1 + vpxor xmm3, xmm3, xmm0 + vpshufb xmm3, xmm3, xmm14 + vpaddd xmm2, xmm2, xmm3 + vpxor xmm1, xmm1, xmm2 + vpsrld xmm8, xmm1, 12 + vpslld xmm1, xmm1, 20 + vpor xmm1, xmm1, xmm8 + vpaddd xmm0, xmm0, xmm7 + vpaddd xmm0, xmm0, xmm1 + vpxor xmm3, xmm3, xmm0 + vpshufb xmm3, xmm3, xmm15 + vpaddd xmm2, xmm2, xmm3 + vpxor xmm1, xmm1, xmm2 + vpsrld xmm8, xmm1, 7 + vpslld xmm1, xmm1, 25 + vpor xmm1, xmm1, xmm8 + vpshufd xmm0, xmm0, 0x39 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x93 + dec al + jz 9f + vshufps xmm8, xmm4, xmm5, 214 + vpshufd xmm9, xmm4, 0x0F + vpshufd xmm4, xmm8, 0x39 + vshufps xmm8, xmm6, xmm7, 250 + vpblendd xmm9, xmm9, xmm8, 0xAA + vpunpcklqdq xmm8, xmm7, xmm5 + vpblendd xmm8, xmm8, xmm6, 0x88 + vpshufd xmm8, xmm8, 0x78 + vpunpckhdq xmm5, xmm5, xmm7 + vpunpckldq xmm6, xmm6, xmm5 + vpshufd xmm7, xmm6, 0x1E + vmovdqa xmm5, xmm9 + vmovdqa xmm6, xmm8 + jmp 9b +9: + vpxor xmm0, xmm0, xmm2 + vpxor xmm1, xmm1, xmm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + jmp 4b + + +#ifdef __APPLE__ +.static_data +#else +.section .rodata +#endif +.p2align 6 +ADD0: + .long 0, 1, 2, 3, 4, 5, 6, 7 +ADD1: + .long 8, 8, 8, 8, 8, 8, 8, 8 +BLAKE3_IV_0: + .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667 + .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667 +BLAKE3_IV_1: + .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85 + .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85 +BLAKE3_IV_2: + .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372 + .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372 +BLAKE3_IV_3: + .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A + .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A +BLAKE3_BLOCK_LEN: + .long 0x00000040, 0x00000040, 0x00000040, 0x00000040 + .long 0x00000040, 0x00000040, 0x00000040, 0x00000040 +ROT16: + .byte 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 +ROT8: + .byte 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12 +CMP_MSB_MASK: + .long 0x80000000, 0x80000000, 0x80000000, 0x80000000 + .long 0x80000000, 0x80000000, 0x80000000, 0x80000000 +BLAKE3_IV: + .long 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A + diff --git a/src/blake3/blake3_avx512.c b/src/blake3/blake3_avx512.c new file mode 100644 index 0000000..0bdc457 --- /dev/null +++ b/src/blake3/blake3_avx512.c @@ -0,0 +1,1227 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" + +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +#define _mm_shuffle_ps2(a, b, c) \ + (_mm_castps_si128( \ + _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) + +INLINE __m128i loadu_128(const uint8_t src[16]) { + return _mm_loadu_si128((const __m128i *)src); +} + +INLINE __m256i loadu_256(const uint8_t src[32]) { + return _mm256_loadu_si256((const __m256i *)src); +} + +INLINE __m512i loadu_512(const uint8_t src[64]) { + return _mm512_loadu_si512((const __m512i *)src); +} + +INLINE void storeu_128(__m128i src, uint8_t dest[16]) { + _mm_storeu_si128((__m128i *)dest, src); +} + +INLINE void storeu_256(__m256i src, uint8_t dest[16]) { + _mm256_storeu_si256((__m256i *)dest, src); +} + +INLINE __m128i add_128(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } + +INLINE __m256i add_256(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } + +INLINE __m512i add_512(__m512i a, __m512i b) { return _mm512_add_epi32(a, b); } + +INLINE __m128i xor_128(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } + +INLINE __m256i xor_256(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } + +INLINE __m512i xor_512(__m512i a, __m512i b) { return _mm512_xor_si512(a, b); } + +INLINE __m128i set1_128(uint32_t x) { return _mm_set1_epi32((int32_t)x); } + +INLINE __m256i set1_256(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } + +INLINE __m512i set1_512(uint32_t x) { return _mm512_set1_epi32((int32_t)x); } + +INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); +} + +INLINE __m128i rot16_128(__m128i x) { return _mm_ror_epi32(x, 16); } + +INLINE __m256i rot16_256(__m256i x) { return _mm256_ror_epi32(x, 16); } + +INLINE __m512i rot16_512(__m512i x) { return _mm512_ror_epi32(x, 16); } + +INLINE __m128i rot12_128(__m128i x) { return _mm_ror_epi32(x, 12); } + +INLINE __m256i rot12_256(__m256i x) { return _mm256_ror_epi32(x, 12); } + +INLINE __m512i rot12_512(__m512i x) { return _mm512_ror_epi32(x, 12); } + +INLINE __m128i rot8_128(__m128i x) { return _mm_ror_epi32(x, 8); } + +INLINE __m256i rot8_256(__m256i x) { return _mm256_ror_epi32(x, 8); } + +INLINE __m512i rot8_512(__m512i x) { return _mm512_ror_epi32(x, 8); } + +INLINE __m128i rot7_128(__m128i x) { return _mm_ror_epi32(x, 7); } + +INLINE __m256i rot7_256(__m256i x) { return _mm256_ror_epi32(x, 7); } + +INLINE __m512i rot7_512(__m512i x) { return _mm512_ror_epi32(x, 7); } + +/* + * ---------------------------------------------------------------------------- + * compress_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = add_128(add_128(*row0, m), *row1); + *row3 = xor_128(*row3, *row0); + *row3 = rot16_128(*row3); + *row2 = add_128(*row2, *row3); + *row1 = xor_128(*row1, *row2); + *row1 = rot12_128(*row1); +} + +INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = add_128(add_128(*row0, m), *row1); + *row3 = xor_128(*row3, *row0); + *row3 = rot8_128(*row3); + *row2 = add_128(*row2, *row3); + *row1 = xor_128(*row1, *row2); + *row1 = rot7_128(*row1); +} + +// Note the optimization here of leaving row1 as the unrotated row, rather than +// row0. All the message loads below are adjusted to compensate for this. See +// discussion at https://github.com/sneves/blake2-avx2/pull/4 +INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); +} + +INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); +} + +INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + rows[0] = loadu_128((uint8_t *)&cv[0]); + rows[1] = loadu_128((uint8_t *)&cv[4]); + rows[2] = set4(B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3); + rows[3] = set4(counter_low(counter), counter_high(counter), + (uint32_t)block_len, (uint32_t)flags); + + __m128i m0 = loadu_128(&block[sizeof(__m128i) * 0]); + __m128i m1 = loadu_128(&block[sizeof(__m128i) * 1]); + __m128i m2 = loadu_128(&block[sizeof(__m128i) * 2]); + __m128i m3 = loadu_128(&block[sizeof(__m128i) * 3]); + + __m128i t0, t1, t2, t3, tt; + + // Round 1. The first round permutes the message words from the original + // input order, into the groups that get mixed in parallel. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 + t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 + t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 2. This round and all following rounds apply a fixed permutation + // to the message words from the round before. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 3 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 4 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 5 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 6 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 7 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); +} + +void blake3_compress_xof_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu_128(xor_128(rows[0], rows[2]), &out[0]); + storeu_128(xor_128(rows[1], rows[3]), &out[16]); + storeu_128(xor_128(rows[2], loadu_128((uint8_t *)&cv[0])), &out[32]); + storeu_128(xor_128(rows[3], loadu_128((uint8_t *)&cv[4])), &out[48]); +} + +void blake3_compress_in_place_avx512(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu_128(xor_128(rows[0], rows[2]), (uint8_t *)&cv[0]); + storeu_128(xor_128(rows[1], rows[3]), (uint8_t *)&cv[4]); +} + +/* + * ---------------------------------------------------------------------------- + * hash4_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn4(__m128i v[16], __m128i m[16], size_t r) { + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[15] = rot16_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot12_128(v[4]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[15] = rot8_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot7_128(v[4]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot16_128(v[15]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[4] = rot12_128(v[4]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot8_128(v[15]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + v[4] = rot7_128(v[4]); +} + +INLINE void transpose_vecs_128(__m128i vecs[4]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is + // 22/33. Note that this doesn't split the vector into two lanes, as the + // AVX2 counterparts do. + __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); + __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); + __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); + __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); + + // Interleave 64-bit lanes. + __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); + __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); + __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); + __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); + + vecs[0] = abcd_0; + vecs[1] = abcd_1; + vecs[2] = abcd_2; + vecs[3] = abcd_3; +} + +INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, + size_t block_offset, __m128i out[16]) { + out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(__m128i)]); + out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(__m128i)]); + out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(__m128i)]); + out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(__m128i)]); + out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(__m128i)]); + out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(__m128i)]); + out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(__m128i)]); + out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(__m128i)]); + out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(__m128i)]); + out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(__m128i)]); + out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(__m128i)]); + out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(__m128i)]); + out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(__m128i)]); + out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(__m128i)]); + out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(__m128i)]); + out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(__m128i)]); + for (size_t i = 0; i < 4; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs_128(&out[0]); + transpose_vecs_128(&out[4]); + transpose_vecs_128(&out[8]); + transpose_vecs_128(&out[12]); +} + +INLINE void load_counters4(uint64_t counter, bool increment_counter, + __m128i *out_lo, __m128i *out_hi) { + uint64_t mask = (increment_counter ? ~0 : 0); + __m256i mask_vec = _mm256_set1_epi64x(mask); + __m256i deltas = _mm256_setr_epi64x(0, 1, 2, 3); + deltas = _mm256_and_si256(mask_vec, deltas); + __m256i counters = + _mm256_add_epi64(_mm256_set1_epi64x((int64_t)counter), deltas); + *out_lo = _mm256_cvtepi64_epi32(counters); + *out_hi = _mm256_cvtepi64_epi32(_mm256_srli_epi64(counters, 32)); +} + +static +void blake3_hash4_avx512(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m128i h_vecs[8] = { + set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), + set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), + }; + __m128i counter_low_vec, counter_high_vec; + load_counters4(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m128i block_len_vec = set1_128(BLAKE3_BLOCK_LEN); + __m128i block_flags_vec = set1_128(block_flags); + __m128i msg_vecs[16]; + transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m128i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_128(B3_IV_0), set1_128(B3_IV_1), set1_128(B3_IV_2), set1_128(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn4(v, msg_vecs, 0); + round_fn4(v, msg_vecs, 1); + round_fn4(v, msg_vecs, 2); + round_fn4(v, msg_vecs, 3); + round_fn4(v, msg_vecs, 4); + round_fn4(v, msg_vecs, 5); + round_fn4(v, msg_vecs, 6); + h_vecs[0] = xor_128(v[0], v[8]); + h_vecs[1] = xor_128(v[1], v[9]); + h_vecs[2] = xor_128(v[2], v[10]); + h_vecs[3] = xor_128(v[3], v[11]); + h_vecs[4] = xor_128(v[4], v[12]); + h_vecs[5] = xor_128(v[5], v[13]); + h_vecs[6] = xor_128(v[6], v[14]); + h_vecs[7] = xor_128(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs_128(&h_vecs[0]); + transpose_vecs_128(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu_128(h_vecs[0], &out[0 * sizeof(__m128i)]); + storeu_128(h_vecs[4], &out[1 * sizeof(__m128i)]); + storeu_128(h_vecs[1], &out[2 * sizeof(__m128i)]); + storeu_128(h_vecs[5], &out[3 * sizeof(__m128i)]); + storeu_128(h_vecs[2], &out[4 * sizeof(__m128i)]); + storeu_128(h_vecs[6], &out[5 * sizeof(__m128i)]); + storeu_128(h_vecs[3], &out[6 * sizeof(__m128i)]); + storeu_128(h_vecs[7], &out[7 * sizeof(__m128i)]); +} + +/* + * ---------------------------------------------------------------------------- + * hash8_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn8(__m256i v[16], __m256i m[16], size_t r) { + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_256(v[0], v[4]); + v[1] = add_256(v[1], v[5]); + v[2] = add_256(v[2], v[6]); + v[3] = add_256(v[3], v[7]); + v[12] = xor_256(v[12], v[0]); + v[13] = xor_256(v[13], v[1]); + v[14] = xor_256(v[14], v[2]); + v[15] = xor_256(v[15], v[3]); + v[12] = rot16_256(v[12]); + v[13] = rot16_256(v[13]); + v[14] = rot16_256(v[14]); + v[15] = rot16_256(v[15]); + v[8] = add_256(v[8], v[12]); + v[9] = add_256(v[9], v[13]); + v[10] = add_256(v[10], v[14]); + v[11] = add_256(v[11], v[15]); + v[4] = xor_256(v[4], v[8]); + v[5] = xor_256(v[5], v[9]); + v[6] = xor_256(v[6], v[10]); + v[7] = xor_256(v[7], v[11]); + v[4] = rot12_256(v[4]); + v[5] = rot12_256(v[5]); + v[6] = rot12_256(v[6]); + v[7] = rot12_256(v[7]); + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_256(v[0], v[4]); + v[1] = add_256(v[1], v[5]); + v[2] = add_256(v[2], v[6]); + v[3] = add_256(v[3], v[7]); + v[12] = xor_256(v[12], v[0]); + v[13] = xor_256(v[13], v[1]); + v[14] = xor_256(v[14], v[2]); + v[15] = xor_256(v[15], v[3]); + v[12] = rot8_256(v[12]); + v[13] = rot8_256(v[13]); + v[14] = rot8_256(v[14]); + v[15] = rot8_256(v[15]); + v[8] = add_256(v[8], v[12]); + v[9] = add_256(v[9], v[13]); + v[10] = add_256(v[10], v[14]); + v[11] = add_256(v[11], v[15]); + v[4] = xor_256(v[4], v[8]); + v[5] = xor_256(v[5], v[9]); + v[6] = xor_256(v[6], v[10]); + v[7] = xor_256(v[7], v[11]); + v[4] = rot7_256(v[4]); + v[5] = rot7_256(v[5]); + v[6] = rot7_256(v[6]); + v[7] = rot7_256(v[7]); + + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_256(v[0], v[5]); + v[1] = add_256(v[1], v[6]); + v[2] = add_256(v[2], v[7]); + v[3] = add_256(v[3], v[4]); + v[15] = xor_256(v[15], v[0]); + v[12] = xor_256(v[12], v[1]); + v[13] = xor_256(v[13], v[2]); + v[14] = xor_256(v[14], v[3]); + v[15] = rot16_256(v[15]); + v[12] = rot16_256(v[12]); + v[13] = rot16_256(v[13]); + v[14] = rot16_256(v[14]); + v[10] = add_256(v[10], v[15]); + v[11] = add_256(v[11], v[12]); + v[8] = add_256(v[8], v[13]); + v[9] = add_256(v[9], v[14]); + v[5] = xor_256(v[5], v[10]); + v[6] = xor_256(v[6], v[11]); + v[7] = xor_256(v[7], v[8]); + v[4] = xor_256(v[4], v[9]); + v[5] = rot12_256(v[5]); + v[6] = rot12_256(v[6]); + v[7] = rot12_256(v[7]); + v[4] = rot12_256(v[4]); + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_256(v[0], v[5]); + v[1] = add_256(v[1], v[6]); + v[2] = add_256(v[2], v[7]); + v[3] = add_256(v[3], v[4]); + v[15] = xor_256(v[15], v[0]); + v[12] = xor_256(v[12], v[1]); + v[13] = xor_256(v[13], v[2]); + v[14] = xor_256(v[14], v[3]); + v[15] = rot8_256(v[15]); + v[12] = rot8_256(v[12]); + v[13] = rot8_256(v[13]); + v[14] = rot8_256(v[14]); + v[10] = add_256(v[10], v[15]); + v[11] = add_256(v[11], v[12]); + v[8] = add_256(v[8], v[13]); + v[9] = add_256(v[9], v[14]); + v[5] = xor_256(v[5], v[10]); + v[6] = xor_256(v[6], v[11]); + v[7] = xor_256(v[7], v[8]); + v[4] = xor_256(v[4], v[9]); + v[5] = rot7_256(v[5]); + v[6] = rot7_256(v[6]); + v[7] = rot7_256(v[7]); + v[4] = rot7_256(v[4]); +} + +INLINE void transpose_vecs_256(__m256i vecs[8]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high + // is 22/33/66/77. + __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); + __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); + __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); + __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); + __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); + __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); + __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); + __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); + + // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is + // 11/33. + __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); + __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); + __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); + __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); + __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); + __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); + __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); + __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); + + // Interleave 128-bit lanes. + vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); + vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); + vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); + vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); + vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); + vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); + vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); + vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); +} + +INLINE void transpose_msg_vecs8(const uint8_t *const *inputs, + size_t block_offset, __m256i out[16]) { + out[0] = loadu_256(&inputs[0][block_offset + 0 * sizeof(__m256i)]); + out[1] = loadu_256(&inputs[1][block_offset + 0 * sizeof(__m256i)]); + out[2] = loadu_256(&inputs[2][block_offset + 0 * sizeof(__m256i)]); + out[3] = loadu_256(&inputs[3][block_offset + 0 * sizeof(__m256i)]); + out[4] = loadu_256(&inputs[4][block_offset + 0 * sizeof(__m256i)]); + out[5] = loadu_256(&inputs[5][block_offset + 0 * sizeof(__m256i)]); + out[6] = loadu_256(&inputs[6][block_offset + 0 * sizeof(__m256i)]); + out[7] = loadu_256(&inputs[7][block_offset + 0 * sizeof(__m256i)]); + out[8] = loadu_256(&inputs[0][block_offset + 1 * sizeof(__m256i)]); + out[9] = loadu_256(&inputs[1][block_offset + 1 * sizeof(__m256i)]); + out[10] = loadu_256(&inputs[2][block_offset + 1 * sizeof(__m256i)]); + out[11] = loadu_256(&inputs[3][block_offset + 1 * sizeof(__m256i)]); + out[12] = loadu_256(&inputs[4][block_offset + 1 * sizeof(__m256i)]); + out[13] = loadu_256(&inputs[5][block_offset + 1 * sizeof(__m256i)]); + out[14] = loadu_256(&inputs[6][block_offset + 1 * sizeof(__m256i)]); + out[15] = loadu_256(&inputs[7][block_offset + 1 * sizeof(__m256i)]); + for (size_t i = 0; i < 8; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs_256(&out[0]); + transpose_vecs_256(&out[8]); +} + +INLINE void load_counters8(uint64_t counter, bool increment_counter, + __m256i *out_lo, __m256i *out_hi) { + uint64_t mask = (increment_counter ? ~0 : 0); + __m512i mask_vec = _mm512_set1_epi64(mask); + __m512i deltas = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + deltas = _mm512_and_si512(mask_vec, deltas); + __m512i counters = + _mm512_add_epi64(_mm512_set1_epi64((int64_t)counter), deltas); + *out_lo = _mm512_cvtepi64_epi32(counters); + *out_hi = _mm512_cvtepi64_epi32(_mm512_srli_epi64(counters, 32)); +} + +static +void blake3_hash8_avx512(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m256i h_vecs[8] = { + set1_256(key[0]), set1_256(key[1]), set1_256(key[2]), set1_256(key[3]), + set1_256(key[4]), set1_256(key[5]), set1_256(key[6]), set1_256(key[7]), + }; + __m256i counter_low_vec, counter_high_vec; + load_counters8(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m256i block_len_vec = set1_256(BLAKE3_BLOCK_LEN); + __m256i block_flags_vec = set1_256(block_flags); + __m256i msg_vecs[16]; + transpose_msg_vecs8(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m256i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_256(B3_IV_0), set1_256(B3_IV_1), set1_256(B3_IV_2), set1_256(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn8(v, msg_vecs, 0); + round_fn8(v, msg_vecs, 1); + round_fn8(v, msg_vecs, 2); + round_fn8(v, msg_vecs, 3); + round_fn8(v, msg_vecs, 4); + round_fn8(v, msg_vecs, 5); + round_fn8(v, msg_vecs, 6); + h_vecs[0] = xor_256(v[0], v[8]); + h_vecs[1] = xor_256(v[1], v[9]); + h_vecs[2] = xor_256(v[2], v[10]); + h_vecs[3] = xor_256(v[3], v[11]); + h_vecs[4] = xor_256(v[4], v[12]); + h_vecs[5] = xor_256(v[5], v[13]); + h_vecs[6] = xor_256(v[6], v[14]); + h_vecs[7] = xor_256(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs_256(h_vecs); + storeu_256(h_vecs[0], &out[0 * sizeof(__m256i)]); + storeu_256(h_vecs[1], &out[1 * sizeof(__m256i)]); + storeu_256(h_vecs[2], &out[2 * sizeof(__m256i)]); + storeu_256(h_vecs[3], &out[3 * sizeof(__m256i)]); + storeu_256(h_vecs[4], &out[4 * sizeof(__m256i)]); + storeu_256(h_vecs[5], &out[5 * sizeof(__m256i)]); + storeu_256(h_vecs[6], &out[6 * sizeof(__m256i)]); + storeu_256(h_vecs[7], &out[7 * sizeof(__m256i)]); +} + +/* + * ---------------------------------------------------------------------------- + * hash16_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn16(__m512i v[16], __m512i m[16], size_t r) { + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_512(v[0], v[4]); + v[1] = add_512(v[1], v[5]); + v[2] = add_512(v[2], v[6]); + v[3] = add_512(v[3], v[7]); + v[12] = xor_512(v[12], v[0]); + v[13] = xor_512(v[13], v[1]); + v[14] = xor_512(v[14], v[2]); + v[15] = xor_512(v[15], v[3]); + v[12] = rot16_512(v[12]); + v[13] = rot16_512(v[13]); + v[14] = rot16_512(v[14]); + v[15] = rot16_512(v[15]); + v[8] = add_512(v[8], v[12]); + v[9] = add_512(v[9], v[13]); + v[10] = add_512(v[10], v[14]); + v[11] = add_512(v[11], v[15]); + v[4] = xor_512(v[4], v[8]); + v[5] = xor_512(v[5], v[9]); + v[6] = xor_512(v[6], v[10]); + v[7] = xor_512(v[7], v[11]); + v[4] = rot12_512(v[4]); + v[5] = rot12_512(v[5]); + v[6] = rot12_512(v[6]); + v[7] = rot12_512(v[7]); + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_512(v[0], v[4]); + v[1] = add_512(v[1], v[5]); + v[2] = add_512(v[2], v[6]); + v[3] = add_512(v[3], v[7]); + v[12] = xor_512(v[12], v[0]); + v[13] = xor_512(v[13], v[1]); + v[14] = xor_512(v[14], v[2]); + v[15] = xor_512(v[15], v[3]); + v[12] = rot8_512(v[12]); + v[13] = rot8_512(v[13]); + v[14] = rot8_512(v[14]); + v[15] = rot8_512(v[15]); + v[8] = add_512(v[8], v[12]); + v[9] = add_512(v[9], v[13]); + v[10] = add_512(v[10], v[14]); + v[11] = add_512(v[11], v[15]); + v[4] = xor_512(v[4], v[8]); + v[5] = xor_512(v[5], v[9]); + v[6] = xor_512(v[6], v[10]); + v[7] = xor_512(v[7], v[11]); + v[4] = rot7_512(v[4]); + v[5] = rot7_512(v[5]); + v[6] = rot7_512(v[6]); + v[7] = rot7_512(v[7]); + + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_512(v[0], v[5]); + v[1] = add_512(v[1], v[6]); + v[2] = add_512(v[2], v[7]); + v[3] = add_512(v[3], v[4]); + v[15] = xor_512(v[15], v[0]); + v[12] = xor_512(v[12], v[1]); + v[13] = xor_512(v[13], v[2]); + v[14] = xor_512(v[14], v[3]); + v[15] = rot16_512(v[15]); + v[12] = rot16_512(v[12]); + v[13] = rot16_512(v[13]); + v[14] = rot16_512(v[14]); + v[10] = add_512(v[10], v[15]); + v[11] = add_512(v[11], v[12]); + v[8] = add_512(v[8], v[13]); + v[9] = add_512(v[9], v[14]); + v[5] = xor_512(v[5], v[10]); + v[6] = xor_512(v[6], v[11]); + v[7] = xor_512(v[7], v[8]); + v[4] = xor_512(v[4], v[9]); + v[5] = rot12_512(v[5]); + v[6] = rot12_512(v[6]); + v[7] = rot12_512(v[7]); + v[4] = rot12_512(v[4]); + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_512(v[0], v[5]); + v[1] = add_512(v[1], v[6]); + v[2] = add_512(v[2], v[7]); + v[3] = add_512(v[3], v[4]); + v[15] = xor_512(v[15], v[0]); + v[12] = xor_512(v[12], v[1]); + v[13] = xor_512(v[13], v[2]); + v[14] = xor_512(v[14], v[3]); + v[15] = rot8_512(v[15]); + v[12] = rot8_512(v[12]); + v[13] = rot8_512(v[13]); + v[14] = rot8_512(v[14]); + v[10] = add_512(v[10], v[15]); + v[11] = add_512(v[11], v[12]); + v[8] = add_512(v[8], v[13]); + v[9] = add_512(v[9], v[14]); + v[5] = xor_512(v[5], v[10]); + v[6] = xor_512(v[6], v[11]); + v[7] = xor_512(v[7], v[8]); + v[4] = xor_512(v[4], v[9]); + v[5] = rot7_512(v[5]); + v[6] = rot7_512(v[6]); + v[7] = rot7_512(v[7]); + v[4] = rot7_512(v[4]); +} + +// 0b10001000, or lanes a0/a2/b0/b2 in little-endian order +#define LO_IMM8 0x88 + +INLINE __m512i unpack_lo_128(__m512i a, __m512i b) { + return _mm512_shuffle_i32x4(a, b, LO_IMM8); +} + +// 0b11011101, or lanes a1/a3/b1/b3 in little-endian order +#define HI_IMM8 0xdd + +INLINE __m512i unpack_hi_128(__m512i a, __m512i b) { + return _mm512_shuffle_i32x4(a, b, HI_IMM8); +} + +INLINE void transpose_vecs_512(__m512i vecs[16]) { + // Interleave 32-bit lanes. The _0 unpack is lanes + // 0/0/1/1/4/4/5/5/8/8/9/9/12/12/13/13, and the _2 unpack is lanes + // 2/2/3/3/6/6/7/7/10/10/11/11/14/14/15/15. + __m512i ab_0 = _mm512_unpacklo_epi32(vecs[0], vecs[1]); + __m512i ab_2 = _mm512_unpackhi_epi32(vecs[0], vecs[1]); + __m512i cd_0 = _mm512_unpacklo_epi32(vecs[2], vecs[3]); + __m512i cd_2 = _mm512_unpackhi_epi32(vecs[2], vecs[3]); + __m512i ef_0 = _mm512_unpacklo_epi32(vecs[4], vecs[5]); + __m512i ef_2 = _mm512_unpackhi_epi32(vecs[4], vecs[5]); + __m512i gh_0 = _mm512_unpacklo_epi32(vecs[6], vecs[7]); + __m512i gh_2 = _mm512_unpackhi_epi32(vecs[6], vecs[7]); + __m512i ij_0 = _mm512_unpacklo_epi32(vecs[8], vecs[9]); + __m512i ij_2 = _mm512_unpackhi_epi32(vecs[8], vecs[9]); + __m512i kl_0 = _mm512_unpacklo_epi32(vecs[10], vecs[11]); + __m512i kl_2 = _mm512_unpackhi_epi32(vecs[10], vecs[11]); + __m512i mn_0 = _mm512_unpacklo_epi32(vecs[12], vecs[13]); + __m512i mn_2 = _mm512_unpackhi_epi32(vecs[12], vecs[13]); + __m512i op_0 = _mm512_unpacklo_epi32(vecs[14], vecs[15]); + __m512i op_2 = _mm512_unpackhi_epi32(vecs[14], vecs[15]); + + // Interleave 64-bit lanes. The _0 unpack is lanes + // 0/0/0/0/4/4/4/4/8/8/8/8/12/12/12/12, the _1 unpack is lanes + // 1/1/1/1/5/5/5/5/9/9/9/9/13/13/13/13, the _2 unpack is lanes + // 2/2/2/2/6/6/6/6/10/10/10/10/14/14/14/14, and the _3 unpack is lanes + // 3/3/3/3/7/7/7/7/11/11/11/11/15/15/15/15. + __m512i abcd_0 = _mm512_unpacklo_epi64(ab_0, cd_0); + __m512i abcd_1 = _mm512_unpackhi_epi64(ab_0, cd_0); + __m512i abcd_2 = _mm512_unpacklo_epi64(ab_2, cd_2); + __m512i abcd_3 = _mm512_unpackhi_epi64(ab_2, cd_2); + __m512i efgh_0 = _mm512_unpacklo_epi64(ef_0, gh_0); + __m512i efgh_1 = _mm512_unpackhi_epi64(ef_0, gh_0); + __m512i efgh_2 = _mm512_unpacklo_epi64(ef_2, gh_2); + __m512i efgh_3 = _mm512_unpackhi_epi64(ef_2, gh_2); + __m512i ijkl_0 = _mm512_unpacklo_epi64(ij_0, kl_0); + __m512i ijkl_1 = _mm512_unpackhi_epi64(ij_0, kl_0); + __m512i ijkl_2 = _mm512_unpacklo_epi64(ij_2, kl_2); + __m512i ijkl_3 = _mm512_unpackhi_epi64(ij_2, kl_2); + __m512i mnop_0 = _mm512_unpacklo_epi64(mn_0, op_0); + __m512i mnop_1 = _mm512_unpackhi_epi64(mn_0, op_0); + __m512i mnop_2 = _mm512_unpacklo_epi64(mn_2, op_2); + __m512i mnop_3 = _mm512_unpackhi_epi64(mn_2, op_2); + + // Interleave 128-bit lanes. The _0 unpack is + // 0/0/0/0/8/8/8/8/0/0/0/0/8/8/8/8, the _1 unpack is + // 1/1/1/1/9/9/9/9/1/1/1/1/9/9/9/9, and so on. + __m512i abcdefgh_0 = unpack_lo_128(abcd_0, efgh_0); + __m512i abcdefgh_1 = unpack_lo_128(abcd_1, efgh_1); + __m512i abcdefgh_2 = unpack_lo_128(abcd_2, efgh_2); + __m512i abcdefgh_3 = unpack_lo_128(abcd_3, efgh_3); + __m512i abcdefgh_4 = unpack_hi_128(abcd_0, efgh_0); + __m512i abcdefgh_5 = unpack_hi_128(abcd_1, efgh_1); + __m512i abcdefgh_6 = unpack_hi_128(abcd_2, efgh_2); + __m512i abcdefgh_7 = unpack_hi_128(abcd_3, efgh_3); + __m512i ijklmnop_0 = unpack_lo_128(ijkl_0, mnop_0); + __m512i ijklmnop_1 = unpack_lo_128(ijkl_1, mnop_1); + __m512i ijklmnop_2 = unpack_lo_128(ijkl_2, mnop_2); + __m512i ijklmnop_3 = unpack_lo_128(ijkl_3, mnop_3); + __m512i ijklmnop_4 = unpack_hi_128(ijkl_0, mnop_0); + __m512i ijklmnop_5 = unpack_hi_128(ijkl_1, mnop_1); + __m512i ijklmnop_6 = unpack_hi_128(ijkl_2, mnop_2); + __m512i ijklmnop_7 = unpack_hi_128(ijkl_3, mnop_3); + + // Interleave 128-bit lanes again for the final outputs. + vecs[0] = unpack_lo_128(abcdefgh_0, ijklmnop_0); + vecs[1] = unpack_lo_128(abcdefgh_1, ijklmnop_1); + vecs[2] = unpack_lo_128(abcdefgh_2, ijklmnop_2); + vecs[3] = unpack_lo_128(abcdefgh_3, ijklmnop_3); + vecs[4] = unpack_lo_128(abcdefgh_4, ijklmnop_4); + vecs[5] = unpack_lo_128(abcdefgh_5, ijklmnop_5); + vecs[6] = unpack_lo_128(abcdefgh_6, ijklmnop_6); + vecs[7] = unpack_lo_128(abcdefgh_7, ijklmnop_7); + vecs[8] = unpack_hi_128(abcdefgh_0, ijklmnop_0); + vecs[9] = unpack_hi_128(abcdefgh_1, ijklmnop_1); + vecs[10] = unpack_hi_128(abcdefgh_2, ijklmnop_2); + vecs[11] = unpack_hi_128(abcdefgh_3, ijklmnop_3); + vecs[12] = unpack_hi_128(abcdefgh_4, ijklmnop_4); + vecs[13] = unpack_hi_128(abcdefgh_5, ijklmnop_5); + vecs[14] = unpack_hi_128(abcdefgh_6, ijklmnop_6); + vecs[15] = unpack_hi_128(abcdefgh_7, ijklmnop_7); +} + +INLINE void transpose_msg_vecs16(const uint8_t *const *inputs, + size_t block_offset, __m512i out[16]) { + out[0] = loadu_512(&inputs[0][block_offset]); + out[1] = loadu_512(&inputs[1][block_offset]); + out[2] = loadu_512(&inputs[2][block_offset]); + out[3] = loadu_512(&inputs[3][block_offset]); + out[4] = loadu_512(&inputs[4][block_offset]); + out[5] = loadu_512(&inputs[5][block_offset]); + out[6] = loadu_512(&inputs[6][block_offset]); + out[7] = loadu_512(&inputs[7][block_offset]); + out[8] = loadu_512(&inputs[8][block_offset]); + out[9] = loadu_512(&inputs[9][block_offset]); + out[10] = loadu_512(&inputs[10][block_offset]); + out[11] = loadu_512(&inputs[11][block_offset]); + out[12] = loadu_512(&inputs[12][block_offset]); + out[13] = loadu_512(&inputs[13][block_offset]); + out[14] = loadu_512(&inputs[14][block_offset]); + out[15] = loadu_512(&inputs[15][block_offset]); + for (size_t i = 0; i < 16; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs_512(out); +} + +INLINE void load_counters16(uint64_t counter, bool increment_counter, + __m512i *out_lo, __m512i *out_hi) { + const __m512i mask = _mm512_set1_epi32(-(int32_t)increment_counter); + const __m512i deltas = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + const __m512i masked_deltas = _mm512_and_si512(deltas, mask); + const __m512i low_words = _mm512_add_epi32( + _mm512_set1_epi32((int32_t)counter), + masked_deltas); + // The carry bit is 1 if the high bit of the word was 1 before addition and is + // 0 after. + // NOTE: It would be a bit more natural to use _mm512_cmp_epu32_mask to + // compute the carry bits here, and originally we did, but that intrinsic is + // broken under GCC 5.4. See https://github.com/BLAKE3-team/BLAKE3/issues/271. + const __m512i carries = _mm512_srli_epi32( + _mm512_andnot_si512( + low_words, // 0 after (gets inverted by andnot) + _mm512_set1_epi32((int32_t)counter)), // and 1 before + 31); + const __m512i high_words = _mm512_add_epi32( + _mm512_set1_epi32((int32_t)(counter >> 32)), + carries); + *out_lo = low_words; + *out_hi = high_words; +} + +static +void blake3_hash16_avx512(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, + uint8_t *out) { + __m512i h_vecs[8] = { + set1_512(key[0]), set1_512(key[1]), set1_512(key[2]), set1_512(key[3]), + set1_512(key[4]), set1_512(key[5]), set1_512(key[6]), set1_512(key[7]), + }; + __m512i counter_low_vec, counter_high_vec; + load_counters16(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m512i block_len_vec = set1_512(BLAKE3_BLOCK_LEN); + __m512i block_flags_vec = set1_512(block_flags); + __m512i msg_vecs[16]; + transpose_msg_vecs16(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m512i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_512(B3_IV_0), set1_512(B3_IV_1), set1_512(B3_IV_2), set1_512(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn16(v, msg_vecs, 0); + round_fn16(v, msg_vecs, 1); + round_fn16(v, msg_vecs, 2); + round_fn16(v, msg_vecs, 3); + round_fn16(v, msg_vecs, 4); + round_fn16(v, msg_vecs, 5); + round_fn16(v, msg_vecs, 6); + h_vecs[0] = xor_512(v[0], v[8]); + h_vecs[1] = xor_512(v[1], v[9]); + h_vecs[2] = xor_512(v[2], v[10]); + h_vecs[3] = xor_512(v[3], v[11]); + h_vecs[4] = xor_512(v[4], v[12]); + h_vecs[5] = xor_512(v[5], v[13]); + h_vecs[6] = xor_512(v[6], v[14]); + h_vecs[7] = xor_512(v[7], v[15]); + + block_flags = flags; + } + + // transpose_vecs_512 operates on a 16x16 matrix of words, but we only have 8 + // state vectors. Pad the matrix with zeros. After transposition, store the + // lower half of each vector. + __m512i padded[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_512(0), set1_512(0), set1_512(0), set1_512(0), + set1_512(0), set1_512(0), set1_512(0), set1_512(0), + }; + transpose_vecs_512(padded); + _mm256_mask_storeu_epi32(&out[0 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[0])); + _mm256_mask_storeu_epi32(&out[1 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[1])); + _mm256_mask_storeu_epi32(&out[2 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[2])); + _mm256_mask_storeu_epi32(&out[3 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[3])); + _mm256_mask_storeu_epi32(&out[4 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[4])); + _mm256_mask_storeu_epi32(&out[5 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[5])); + _mm256_mask_storeu_epi32(&out[6 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[6])); + _mm256_mask_storeu_epi32(&out[7 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[7])); + _mm256_mask_storeu_epi32(&out[8 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[8])); + _mm256_mask_storeu_epi32(&out[9 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[9])); + _mm256_mask_storeu_epi32(&out[10 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[10])); + _mm256_mask_storeu_epi32(&out[11 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[11])); + _mm256_mask_storeu_epi32(&out[12 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[12])); + _mm256_mask_storeu_epi32(&out[13 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[13])); + _mm256_mask_storeu_epi32(&out[14 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[14])); + _mm256_mask_storeu_epi32(&out[15 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[15])); +} + +/* + * ---------------------------------------------------------------------------- + * hash_many_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void hash_one_avx512(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_avx512(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= 16) { + blake3_hash16_avx512(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 16; + } + inputs += 16; + num_inputs -= 16; + out = &out[16 * BLAKE3_OUT_LEN]; + } + while (num_inputs >= 8) { + blake3_hash8_avx512(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 8; + } + inputs += 8; + num_inputs -= 8; + out = &out[8 * BLAKE3_OUT_LEN]; + } + while (num_inputs >= 4) { + blake3_hash4_avx512(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 4; + } + inputs += 4; + num_inputs -= 4; + out = &out[4 * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_avx512(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/src/blake3/blake3_avx512_x86-64_unix.S b/src/blake3/blake3_avx512_x86-64_unix.S new file mode 100644 index 0000000..4286f76 --- /dev/null +++ b/src/blake3/blake3_avx512_x86-64_unix.S @@ -0,0 +1,2585 @@ +#if defined(__ELF__) && defined(__linux__) +.section .note.GNU-stack,"",%progbits +#endif + +#if defined(__ELF__) && defined(__CET__) && defined(__has_include) +#if __has_include() +#include +#endif +#endif + +#if !defined(_CET_ENDBR) +#define _CET_ENDBR +#endif + +.intel_syntax noprefix +.global _blake3_hash_many_avx512_asm +.global blake3_hash_many_avx512_asm +.global blake3_compress_in_place_avx512_asm +.global _blake3_compress_in_place_avx512_asm +.global blake3_compress_xof_avx512_asm +.global _blake3_compress_xof_avx512_asm + +#ifdef __APPLE__ +.text +#else +.section .text +#endif +.p2align 6 +_blake3_hash_many_avx512_asm: +blake3_hash_many_avx512_asm: + _CET_ENDBR + push r15 + push r14 + push r13 + push r12 + push rbx + push rbp + mov rbp, rsp + sub rsp, 144 + and rsp, 0xFFFFFFFFFFFFFFC0 + neg r9 + kmovw k1, r9d + vmovd xmm0, r8d + vpbroadcastd ymm0, xmm0 + shr r8, 32 + vmovd xmm1, r8d + vpbroadcastd ymm1, xmm1 + vmovdqa ymm4, ymm1 + vmovdqa ymm5, ymm1 + vpaddd ymm2, ymm0, ymmword ptr [ADD0+rip] + vpaddd ymm3, ymm0, ymmword ptr [ADD0+32+rip] + vpcmpltud k2, ymm2, ymm0 + vpcmpltud k3, ymm3, ymm0 + vpaddd ymm4 {k2}, ymm4, dword ptr [ADD1+rip] {1to8} + vpaddd ymm5 {k3}, ymm5, dword ptr [ADD1+rip] {1to8} + knotw k2, k1 + vmovdqa32 ymm2 {k2}, ymm0 + vmovdqa32 ymm3 {k2}, ymm0 + vmovdqa32 ymm4 {k2}, ymm1 + vmovdqa32 ymm5 {k2}, ymm1 + vmovdqa ymmword ptr [rsp], ymm2 + vmovdqa ymmword ptr [rsp+0x1*0x20], ymm3 + vmovdqa ymmword ptr [rsp+0x2*0x20], ymm4 + vmovdqa ymmword ptr [rsp+0x3*0x20], ymm5 + shl rdx, 6 + mov qword ptr [rsp+0x80], rdx + cmp rsi, 16 + jc 3f +2: + vpbroadcastd zmm0, dword ptr [rcx] + vpbroadcastd zmm1, dword ptr [rcx+0x1*0x4] + vpbroadcastd zmm2, dword ptr [rcx+0x2*0x4] + vpbroadcastd zmm3, dword ptr [rcx+0x3*0x4] + vpbroadcastd zmm4, dword ptr [rcx+0x4*0x4] + vpbroadcastd zmm5, dword ptr [rcx+0x5*0x4] + vpbroadcastd zmm6, dword ptr [rcx+0x6*0x4] + vpbroadcastd zmm7, dword ptr [rcx+0x7*0x4] + movzx eax, byte ptr [rbp+0x38] + movzx ebx, byte ptr [rbp+0x40] + or eax, ebx + xor edx, edx +.p2align 5 +9: + movzx ebx, byte ptr [rbp+0x48] + or ebx, eax + add rdx, 64 + cmp rdx, qword ptr [rsp+0x80] + cmove eax, ebx + mov dword ptr [rsp+0x88], eax + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + mov r12, qword ptr [rdi+0x40] + mov r13, qword ptr [rdi+0x48] + mov r14, qword ptr [rdi+0x50] + mov r15, qword ptr [rdi+0x58] + vmovdqu32 ymm16, ymmword ptr [rdx+r8-0x2*0x20] + vinserti64x4 zmm16, zmm16, ymmword ptr [rdx+r12-0x2*0x20], 0x01 + vmovdqu32 ymm17, ymmword ptr [rdx+r9-0x2*0x20] + vinserti64x4 zmm17, zmm17, ymmword ptr [rdx+r13-0x2*0x20], 0x01 + vpunpcklqdq zmm8, zmm16, zmm17 + vpunpckhqdq zmm9, zmm16, zmm17 + vmovdqu32 ymm18, ymmword ptr [rdx+r10-0x2*0x20] + vinserti64x4 zmm18, zmm18, ymmword ptr [rdx+r14-0x2*0x20], 0x01 + vmovdqu32 ymm19, ymmword ptr [rdx+r11-0x2*0x20] + vinserti64x4 zmm19, zmm19, ymmword ptr [rdx+r15-0x2*0x20], 0x01 + vpunpcklqdq zmm10, zmm18, zmm19 + vpunpckhqdq zmm11, zmm18, zmm19 + mov r8, qword ptr [rdi+0x20] + mov r9, qword ptr [rdi+0x28] + mov r10, qword ptr [rdi+0x30] + mov r11, qword ptr [rdi+0x38] + mov r12, qword ptr [rdi+0x60] + mov r13, qword ptr [rdi+0x68] + mov r14, qword ptr [rdi+0x70] + mov r15, qword ptr [rdi+0x78] + vmovdqu32 ymm16, ymmword ptr [rdx+r8-0x2*0x20] + vinserti64x4 zmm16, zmm16, ymmword ptr [rdx+r12-0x2*0x20], 0x01 + vmovdqu32 ymm17, ymmword ptr [rdx+r9-0x2*0x20] + vinserti64x4 zmm17, zmm17, ymmword ptr [rdx+r13-0x2*0x20], 0x01 + vpunpcklqdq zmm12, zmm16, zmm17 + vpunpckhqdq zmm13, zmm16, zmm17 + vmovdqu32 ymm18, ymmword ptr [rdx+r10-0x2*0x20] + vinserti64x4 zmm18, zmm18, ymmword ptr [rdx+r14-0x2*0x20], 0x01 + vmovdqu32 ymm19, ymmword ptr [rdx+r11-0x2*0x20] + vinserti64x4 zmm19, zmm19, ymmword ptr [rdx+r15-0x2*0x20], 0x01 + vpunpcklqdq zmm14, zmm18, zmm19 + vpunpckhqdq zmm15, zmm18, zmm19 + vmovdqa32 zmm27, zmmword ptr [INDEX0+rip] + vmovdqa32 zmm31, zmmword ptr [INDEX1+rip] + vshufps zmm16, zmm8, zmm10, 136 + vshufps zmm17, zmm12, zmm14, 136 + vmovdqa32 zmm20, zmm16 + vpermt2d zmm16, zmm27, zmm17 + vpermt2d zmm20, zmm31, zmm17 + vshufps zmm17, zmm8, zmm10, 221 + vshufps zmm30, zmm12, zmm14, 221 + vmovdqa32 zmm21, zmm17 + vpermt2d zmm17, zmm27, zmm30 + vpermt2d zmm21, zmm31, zmm30 + vshufps zmm18, zmm9, zmm11, 136 + vshufps zmm8, zmm13, zmm15, 136 + vmovdqa32 zmm22, zmm18 + vpermt2d zmm18, zmm27, zmm8 + vpermt2d zmm22, zmm31, zmm8 + vshufps zmm19, zmm9, zmm11, 221 + vshufps zmm8, zmm13, zmm15, 221 + vmovdqa32 zmm23, zmm19 + vpermt2d zmm19, zmm27, zmm8 + vpermt2d zmm23, zmm31, zmm8 + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + mov r12, qword ptr [rdi+0x40] + mov r13, qword ptr [rdi+0x48] + mov r14, qword ptr [rdi+0x50] + mov r15, qword ptr [rdi+0x58] + vmovdqu32 ymm24, ymmword ptr [r8+rdx-0x1*0x20] + vinserti64x4 zmm24, zmm24, ymmword ptr [r12+rdx-0x1*0x20], 0x01 + vmovdqu32 ymm25, ymmword ptr [r9+rdx-0x1*0x20] + vinserti64x4 zmm25, zmm25, ymmword ptr [r13+rdx-0x1*0x20], 0x01 + vpunpcklqdq zmm8, zmm24, zmm25 + vpunpckhqdq zmm9, zmm24, zmm25 + vmovdqu32 ymm24, ymmword ptr [r10+rdx-0x1*0x20] + vinserti64x4 zmm24, zmm24, ymmword ptr [r14+rdx-0x1*0x20], 0x01 + vmovdqu32 ymm25, ymmword ptr [r11+rdx-0x1*0x20] + vinserti64x4 zmm25, zmm25, ymmword ptr [r15+rdx-0x1*0x20], 0x01 + vpunpcklqdq zmm10, zmm24, zmm25 + vpunpckhqdq zmm11, zmm24, zmm25 + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r12+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r13+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r14+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + prefetcht0 [r15+rdx+0x80] + mov r8, qword ptr [rdi+0x20] + mov r9, qword ptr [rdi+0x28] + mov r10, qword ptr [rdi+0x30] + mov r11, qword ptr [rdi+0x38] + mov r12, qword ptr [rdi+0x60] + mov r13, qword ptr [rdi+0x68] + mov r14, qword ptr [rdi+0x70] + mov r15, qword ptr [rdi+0x78] + vmovdqu32 ymm24, ymmword ptr [r8+rdx-0x1*0x20] + vinserti64x4 zmm24, zmm24, ymmword ptr [r12+rdx-0x1*0x20], 0x01 + vmovdqu32 ymm25, ymmword ptr [r9+rdx-0x1*0x20] + vinserti64x4 zmm25, zmm25, ymmword ptr [r13+rdx-0x1*0x20], 0x01 + vpunpcklqdq zmm12, zmm24, zmm25 + vpunpckhqdq zmm13, zmm24, zmm25 + vmovdqu32 ymm24, ymmword ptr [r10+rdx-0x1*0x20] + vinserti64x4 zmm24, zmm24, ymmword ptr [r14+rdx-0x1*0x20], 0x01 + vmovdqu32 ymm25, ymmword ptr [r11+rdx-0x1*0x20] + vinserti64x4 zmm25, zmm25, ymmword ptr [r15+rdx-0x1*0x20], 0x01 + vpunpcklqdq zmm14, zmm24, zmm25 + vpunpckhqdq zmm15, zmm24, zmm25 + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r12+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r13+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r14+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + prefetcht0 [r15+rdx+0x80] + vshufps zmm24, zmm8, zmm10, 136 + vshufps zmm30, zmm12, zmm14, 136 + vmovdqa32 zmm28, zmm24 + vpermt2d zmm24, zmm27, zmm30 + vpermt2d zmm28, zmm31, zmm30 + vshufps zmm25, zmm8, zmm10, 221 + vshufps zmm30, zmm12, zmm14, 221 + vmovdqa32 zmm29, zmm25 + vpermt2d zmm25, zmm27, zmm30 + vpermt2d zmm29, zmm31, zmm30 + vshufps zmm26, zmm9, zmm11, 136 + vshufps zmm8, zmm13, zmm15, 136 + vmovdqa32 zmm30, zmm26 + vpermt2d zmm26, zmm27, zmm8 + vpermt2d zmm30, zmm31, zmm8 + vshufps zmm8, zmm9, zmm11, 221 + vshufps zmm10, zmm13, zmm15, 221 + vpermi2d zmm27, zmm8, zmm10 + vpermi2d zmm31, zmm8, zmm10 + vpbroadcastd zmm8, dword ptr [BLAKE3_IV_0+rip] + vpbroadcastd zmm9, dword ptr [BLAKE3_IV_1+rip] + vpbroadcastd zmm10, dword ptr [BLAKE3_IV_2+rip] + vpbroadcastd zmm11, dword ptr [BLAKE3_IV_3+rip] + vmovdqa32 zmm12, zmmword ptr [rsp] + vmovdqa32 zmm13, zmmword ptr [rsp+0x1*0x40] + vpbroadcastd zmm14, dword ptr [BLAKE3_BLOCK_LEN+rip] + vpbroadcastd zmm15, dword ptr [rsp+0x22*0x4] + vpaddd zmm0, zmm0, zmm16 + vpaddd zmm1, zmm1, zmm18 + vpaddd zmm2, zmm2, zmm20 + vpaddd zmm3, zmm3, zmm22 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm17 + vpaddd zmm1, zmm1, zmm19 + vpaddd zmm2, zmm2, zmm21 + vpaddd zmm3, zmm3, zmm23 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm24 + vpaddd zmm1, zmm1, zmm26 + vpaddd zmm2, zmm2, zmm28 + vpaddd zmm3, zmm3, zmm30 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm25 + vpaddd zmm1, zmm1, zmm27 + vpaddd zmm2, zmm2, zmm29 + vpaddd zmm3, zmm3, zmm31 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm18 + vpaddd zmm1, zmm1, zmm19 + vpaddd zmm2, zmm2, zmm23 + vpaddd zmm3, zmm3, zmm20 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm22 + vpaddd zmm1, zmm1, zmm26 + vpaddd zmm2, zmm2, zmm16 + vpaddd zmm3, zmm3, zmm29 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm17 + vpaddd zmm1, zmm1, zmm28 + vpaddd zmm2, zmm2, zmm25 + vpaddd zmm3, zmm3, zmm31 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm27 + vpaddd zmm1, zmm1, zmm21 + vpaddd zmm2, zmm2, zmm30 + vpaddd zmm3, zmm3, zmm24 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm19 + vpaddd zmm1, zmm1, zmm26 + vpaddd zmm2, zmm2, zmm29 + vpaddd zmm3, zmm3, zmm23 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm20 + vpaddd zmm1, zmm1, zmm28 + vpaddd zmm2, zmm2, zmm18 + vpaddd zmm3, zmm3, zmm30 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm22 + vpaddd zmm1, zmm1, zmm25 + vpaddd zmm2, zmm2, zmm27 + vpaddd zmm3, zmm3, zmm24 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm21 + vpaddd zmm1, zmm1, zmm16 + vpaddd zmm2, zmm2, zmm31 + vpaddd zmm3, zmm3, zmm17 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm26 + vpaddd zmm1, zmm1, zmm28 + vpaddd zmm2, zmm2, zmm30 + vpaddd zmm3, zmm3, zmm29 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm23 + vpaddd zmm1, zmm1, zmm25 + vpaddd zmm2, zmm2, zmm19 + vpaddd zmm3, zmm3, zmm31 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm20 + vpaddd zmm1, zmm1, zmm27 + vpaddd zmm2, zmm2, zmm21 + vpaddd zmm3, zmm3, zmm17 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm16 + vpaddd zmm1, zmm1, zmm18 + vpaddd zmm2, zmm2, zmm24 + vpaddd zmm3, zmm3, zmm22 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm28 + vpaddd zmm1, zmm1, zmm25 + vpaddd zmm2, zmm2, zmm31 + vpaddd zmm3, zmm3, zmm30 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm29 + vpaddd zmm1, zmm1, zmm27 + vpaddd zmm2, zmm2, zmm26 + vpaddd zmm3, zmm3, zmm24 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm23 + vpaddd zmm1, zmm1, zmm21 + vpaddd zmm2, zmm2, zmm16 + vpaddd zmm3, zmm3, zmm22 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm18 + vpaddd zmm1, zmm1, zmm19 + vpaddd zmm2, zmm2, zmm17 + vpaddd zmm3, zmm3, zmm20 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm25 + vpaddd zmm1, zmm1, zmm27 + vpaddd zmm2, zmm2, zmm24 + vpaddd zmm3, zmm3, zmm31 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm30 + vpaddd zmm1, zmm1, zmm21 + vpaddd zmm2, zmm2, zmm28 + vpaddd zmm3, zmm3, zmm17 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm29 + vpaddd zmm1, zmm1, zmm16 + vpaddd zmm2, zmm2, zmm18 + vpaddd zmm3, zmm3, zmm20 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm19 + vpaddd zmm1, zmm1, zmm26 + vpaddd zmm2, zmm2, zmm22 + vpaddd zmm3, zmm3, zmm23 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm27 + vpaddd zmm1, zmm1, zmm21 + vpaddd zmm2, zmm2, zmm17 + vpaddd zmm3, zmm3, zmm24 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm31 + vpaddd zmm1, zmm1, zmm16 + vpaddd zmm2, zmm2, zmm25 + vpaddd zmm3, zmm3, zmm22 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm30 + vpaddd zmm1, zmm1, zmm18 + vpaddd zmm2, zmm2, zmm19 + vpaddd zmm3, zmm3, zmm23 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm26 + vpaddd zmm1, zmm1, zmm28 + vpaddd zmm2, zmm2, zmm20 + vpaddd zmm3, zmm3, zmm29 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpxord zmm0, zmm0, zmm8 + vpxord zmm1, zmm1, zmm9 + vpxord zmm2, zmm2, zmm10 + vpxord zmm3, zmm3, zmm11 + vpxord zmm4, zmm4, zmm12 + vpxord zmm5, zmm5, zmm13 + vpxord zmm6, zmm6, zmm14 + vpxord zmm7, zmm7, zmm15 + movzx eax, byte ptr [rbp+0x38] + jne 9b + mov rbx, qword ptr [rbp+0x50] + vpunpckldq zmm16, zmm0, zmm1 + vpunpckhdq zmm17, zmm0, zmm1 + vpunpckldq zmm18, zmm2, zmm3 + vpunpckhdq zmm19, zmm2, zmm3 + vpunpckldq zmm20, zmm4, zmm5 + vpunpckhdq zmm21, zmm4, zmm5 + vpunpckldq zmm22, zmm6, zmm7 + vpunpckhdq zmm23, zmm6, zmm7 + vpunpcklqdq zmm0, zmm16, zmm18 + vpunpckhqdq zmm1, zmm16, zmm18 + vpunpcklqdq zmm2, zmm17, zmm19 + vpunpckhqdq zmm3, zmm17, zmm19 + vpunpcklqdq zmm4, zmm20, zmm22 + vpunpckhqdq zmm5, zmm20, zmm22 + vpunpcklqdq zmm6, zmm21, zmm23 + vpunpckhqdq zmm7, zmm21, zmm23 + vshufi32x4 zmm16, zmm0, zmm4, 0x88 + vshufi32x4 zmm17, zmm1, zmm5, 0x88 + vshufi32x4 zmm18, zmm2, zmm6, 0x88 + vshufi32x4 zmm19, zmm3, zmm7, 0x88 + vshufi32x4 zmm20, zmm0, zmm4, 0xDD + vshufi32x4 zmm21, zmm1, zmm5, 0xDD + vshufi32x4 zmm22, zmm2, zmm6, 0xDD + vshufi32x4 zmm23, zmm3, zmm7, 0xDD + vshufi32x4 zmm0, zmm16, zmm17, 0x88 + vshufi32x4 zmm1, zmm18, zmm19, 0x88 + vshufi32x4 zmm2, zmm20, zmm21, 0x88 + vshufi32x4 zmm3, zmm22, zmm23, 0x88 + vshufi32x4 zmm4, zmm16, zmm17, 0xDD + vshufi32x4 zmm5, zmm18, zmm19, 0xDD + vshufi32x4 zmm6, zmm20, zmm21, 0xDD + vshufi32x4 zmm7, zmm22, zmm23, 0xDD + vmovdqu32 zmmword ptr [rbx], zmm0 + vmovdqu32 zmmword ptr [rbx+0x1*0x40], zmm1 + vmovdqu32 zmmword ptr [rbx+0x2*0x40], zmm2 + vmovdqu32 zmmword ptr [rbx+0x3*0x40], zmm3 + vmovdqu32 zmmword ptr [rbx+0x4*0x40], zmm4 + vmovdqu32 zmmword ptr [rbx+0x5*0x40], zmm5 + vmovdqu32 zmmword ptr [rbx+0x6*0x40], zmm6 + vmovdqu32 zmmword ptr [rbx+0x7*0x40], zmm7 + vmovdqa32 zmm0, zmmword ptr [rsp] + vmovdqa32 zmm1, zmmword ptr [rsp+0x1*0x40] + vmovdqa32 zmm2, zmm0 + vpaddd zmm2{k1}, zmm0, dword ptr [ADD16+rip] {1to16} + vpcmpltud k2, zmm2, zmm0 + vpaddd zmm1 {k2}, zmm1, dword ptr [ADD1+rip] {1to16} + vmovdqa32 zmmword ptr [rsp], zmm2 + vmovdqa32 zmmword ptr [rsp+0x1*0x40], zmm1 + add rdi, 128 + add rbx, 512 + mov qword ptr [rbp+0x50], rbx + sub rsi, 16 + cmp rsi, 16 + jnc 2b + test rsi, rsi + jnz 3f +4: + vzeroupper + mov rsp, rbp + pop rbp + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + ret +.p2align 6 +3: + test esi, 0x8 + je 3f + vpbroadcastd ymm0, dword ptr [rcx] + vpbroadcastd ymm1, dword ptr [rcx+0x4] + vpbroadcastd ymm2, dword ptr [rcx+0x8] + vpbroadcastd ymm3, dword ptr [rcx+0xC] + vpbroadcastd ymm4, dword ptr [rcx+0x10] + vpbroadcastd ymm5, dword ptr [rcx+0x14] + vpbroadcastd ymm6, dword ptr [rcx+0x18] + vpbroadcastd ymm7, dword ptr [rcx+0x1C] + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + mov r12, qword ptr [rdi+0x20] + mov r13, qword ptr [rdi+0x28] + mov r14, qword ptr [rdi+0x30] + mov r15, qword ptr [rdi+0x38] + movzx eax, byte ptr [rbp+0x38] + movzx ebx, byte ptr [rbp+0x40] + or eax, ebx + xor edx, edx +2: + movzx ebx, byte ptr [rbp+0x48] + or ebx, eax + add rdx, 64 + cmp rdx, qword ptr [rsp+0x80] + cmove eax, ebx + mov dword ptr [rsp+0x88], eax + vmovups xmm8, xmmword ptr [r8+rdx-0x40] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x40], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x40] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x40], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x40] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x40], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x40] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x40], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm16, ymm12, ymm14, 136 + vshufps ymm17, ymm12, ymm14, 221 + vshufps ymm18, ymm13, ymm15, 136 + vshufps ymm19, ymm13, ymm15, 221 + vmovups xmm8, xmmword ptr [r8+rdx-0x30] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x30], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x30] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x30], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x30] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x30], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x30] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x30], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm20, ymm12, ymm14, 136 + vshufps ymm21, ymm12, ymm14, 221 + vshufps ymm22, ymm13, ymm15, 136 + vshufps ymm23, ymm13, ymm15, 221 + vmovups xmm8, xmmword ptr [r8+rdx-0x20] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x20], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x20] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x20], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x20] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x20], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x20] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x20], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm24, ymm12, ymm14, 136 + vshufps ymm25, ymm12, ymm14, 221 + vshufps ymm26, ymm13, ymm15, 136 + vshufps ymm27, ymm13, ymm15, 221 + vmovups xmm8, xmmword ptr [r8+rdx-0x10] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x10], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x10] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x10], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x10] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x10], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x10] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x10], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm28, ymm12, ymm14, 136 + vshufps ymm29, ymm12, ymm14, 221 + vshufps ymm30, ymm13, ymm15, 136 + vshufps ymm31, ymm13, ymm15, 221 + vpbroadcastd ymm8, dword ptr [BLAKE3_IV_0+rip] + vpbroadcastd ymm9, dword ptr [BLAKE3_IV_1+rip] + vpbroadcastd ymm10, dword ptr [BLAKE3_IV_2+rip] + vpbroadcastd ymm11, dword ptr [BLAKE3_IV_3+rip] + vmovdqa ymm12, ymmword ptr [rsp] + vmovdqa ymm13, ymmword ptr [rsp+0x40] + vpbroadcastd ymm14, dword ptr [BLAKE3_BLOCK_LEN+rip] + vpbroadcastd ymm15, dword ptr [rsp+0x88] + vpaddd ymm0, ymm0, ymm16 + vpaddd ymm1, ymm1, ymm18 + vpaddd ymm2, ymm2, ymm20 + vpaddd ymm3, ymm3, ymm22 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm17 + vpaddd ymm1, ymm1, ymm19 + vpaddd ymm2, ymm2, ymm21 + vpaddd ymm3, ymm3, ymm23 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm24 + vpaddd ymm1, ymm1, ymm26 + vpaddd ymm2, ymm2, ymm28 + vpaddd ymm3, ymm3, ymm30 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm25 + vpaddd ymm1, ymm1, ymm27 + vpaddd ymm2, ymm2, ymm29 + vpaddd ymm3, ymm3, ymm31 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm18 + vpaddd ymm1, ymm1, ymm19 + vpaddd ymm2, ymm2, ymm23 + vpaddd ymm3, ymm3, ymm20 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm22 + vpaddd ymm1, ymm1, ymm26 + vpaddd ymm2, ymm2, ymm16 + vpaddd ymm3, ymm3, ymm29 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm17 + vpaddd ymm1, ymm1, ymm28 + vpaddd ymm2, ymm2, ymm25 + vpaddd ymm3, ymm3, ymm31 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm27 + vpaddd ymm1, ymm1, ymm21 + vpaddd ymm2, ymm2, ymm30 + vpaddd ymm3, ymm3, ymm24 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm19 + vpaddd ymm1, ymm1, ymm26 + vpaddd ymm2, ymm2, ymm29 + vpaddd ymm3, ymm3, ymm23 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm20 + vpaddd ymm1, ymm1, ymm28 + vpaddd ymm2, ymm2, ymm18 + vpaddd ymm3, ymm3, ymm30 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm22 + vpaddd ymm1, ymm1, ymm25 + vpaddd ymm2, ymm2, ymm27 + vpaddd ymm3, ymm3, ymm24 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm21 + vpaddd ymm1, ymm1, ymm16 + vpaddd ymm2, ymm2, ymm31 + vpaddd ymm3, ymm3, ymm17 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm26 + vpaddd ymm1, ymm1, ymm28 + vpaddd ymm2, ymm2, ymm30 + vpaddd ymm3, ymm3, ymm29 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm23 + vpaddd ymm1, ymm1, ymm25 + vpaddd ymm2, ymm2, ymm19 + vpaddd ymm3, ymm3, ymm31 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm20 + vpaddd ymm1, ymm1, ymm27 + vpaddd ymm2, ymm2, ymm21 + vpaddd ymm3, ymm3, ymm17 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm16 + vpaddd ymm1, ymm1, ymm18 + vpaddd ymm2, ymm2, ymm24 + vpaddd ymm3, ymm3, ymm22 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm28 + vpaddd ymm1, ymm1, ymm25 + vpaddd ymm2, ymm2, ymm31 + vpaddd ymm3, ymm3, ymm30 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm29 + vpaddd ymm1, ymm1, ymm27 + vpaddd ymm2, ymm2, ymm26 + vpaddd ymm3, ymm3, ymm24 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm23 + vpaddd ymm1, ymm1, ymm21 + vpaddd ymm2, ymm2, ymm16 + vpaddd ymm3, ymm3, ymm22 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm18 + vpaddd ymm1, ymm1, ymm19 + vpaddd ymm2, ymm2, ymm17 + vpaddd ymm3, ymm3, ymm20 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm25 + vpaddd ymm1, ymm1, ymm27 + vpaddd ymm2, ymm2, ymm24 + vpaddd ymm3, ymm3, ymm31 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm30 + vpaddd ymm1, ymm1, ymm21 + vpaddd ymm2, ymm2, ymm28 + vpaddd ymm3, ymm3, ymm17 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm29 + vpaddd ymm1, ymm1, ymm16 + vpaddd ymm2, ymm2, ymm18 + vpaddd ymm3, ymm3, ymm20 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm19 + vpaddd ymm1, ymm1, ymm26 + vpaddd ymm2, ymm2, ymm22 + vpaddd ymm3, ymm3, ymm23 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm27 + vpaddd ymm1, ymm1, ymm21 + vpaddd ymm2, ymm2, ymm17 + vpaddd ymm3, ymm3, ymm24 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm31 + vpaddd ymm1, ymm1, ymm16 + vpaddd ymm2, ymm2, ymm25 + vpaddd ymm3, ymm3, ymm22 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm30 + vpaddd ymm1, ymm1, ymm18 + vpaddd ymm2, ymm2, ymm19 + vpaddd ymm3, ymm3, ymm23 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm26 + vpaddd ymm1, ymm1, ymm28 + vpaddd ymm2, ymm2, ymm20 + vpaddd ymm3, ymm3, ymm29 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpxor ymm0, ymm0, ymm8 + vpxor ymm1, ymm1, ymm9 + vpxor ymm2, ymm2, ymm10 + vpxor ymm3, ymm3, ymm11 + vpxor ymm4, ymm4, ymm12 + vpxor ymm5, ymm5, ymm13 + vpxor ymm6, ymm6, ymm14 + vpxor ymm7, ymm7, ymm15 + movzx eax, byte ptr [rbp+0x38] + jne 2b + mov rbx, qword ptr [rbp+0x50] + vunpcklps ymm8, ymm0, ymm1 + vunpcklps ymm9, ymm2, ymm3 + vunpckhps ymm10, ymm0, ymm1 + vunpcklps ymm11, ymm4, ymm5 + vunpcklps ymm0, ymm6, ymm7 + vshufps ymm12, ymm8, ymm9, 78 + vblendps ymm1, ymm8, ymm12, 0xCC + vshufps ymm8, ymm11, ymm0, 78 + vunpckhps ymm13, ymm2, ymm3 + vblendps ymm2, ymm11, ymm8, 0xCC + vblendps ymm3, ymm12, ymm9, 0xCC + vperm2f128 ymm12, ymm1, ymm2, 0x20 + vmovups ymmword ptr [rbx], ymm12 + vunpckhps ymm14, ymm4, ymm5 + vblendps ymm4, ymm8, ymm0, 0xCC + vunpckhps ymm15, ymm6, ymm7 + vperm2f128 ymm7, ymm3, ymm4, 0x20 + vmovups ymmword ptr [rbx+0x20], ymm7 + vshufps ymm5, ymm10, ymm13, 78 + vblendps ymm6, ymm5, ymm13, 0xCC + vshufps ymm13, ymm14, ymm15, 78 + vblendps ymm10, ymm10, ymm5, 0xCC + vblendps ymm14, ymm14, ymm13, 0xCC + vperm2f128 ymm8, ymm10, ymm14, 0x20 + vmovups ymmword ptr [rbx+0x40], ymm8 + vblendps ymm15, ymm13, ymm15, 0xCC + vperm2f128 ymm13, ymm6, ymm15, 0x20 + vmovups ymmword ptr [rbx+0x60], ymm13 + vperm2f128 ymm9, ymm1, ymm2, 0x31 + vperm2f128 ymm11, ymm3, ymm4, 0x31 + vmovups ymmword ptr [rbx+0x80], ymm9 + vperm2f128 ymm14, ymm10, ymm14, 0x31 + vperm2f128 ymm15, ymm6, ymm15, 0x31 + vmovups ymmword ptr [rbx+0xA0], ymm11 + vmovups ymmword ptr [rbx+0xC0], ymm14 + vmovups ymmword ptr [rbx+0xE0], ymm15 + vmovdqa ymm0, ymmword ptr [rsp] + vmovdqa ymm2, ymmword ptr [rsp+0x2*0x20] + vmovdqa32 ymm0 {k1}, ymmword ptr [rsp+0x1*0x20] + vmovdqa32 ymm2 {k1}, ymmword ptr [rsp+0x3*0x20] + vmovdqa ymmword ptr [rsp], ymm0 + vmovdqa ymmword ptr [rsp+0x2*0x20], ymm2 + add rbx, 256 + mov qword ptr [rbp+0x50], rbx + add rdi, 64 + sub rsi, 8 +3: + mov rbx, qword ptr [rbp+0x50] + mov r15, qword ptr [rsp+0x80] + movzx r13, byte ptr [rbp+0x38] + movzx r12, byte ptr [rbp+0x48] + test esi, 0x4 + je 3f + vbroadcasti32x4 zmm0, xmmword ptr [rcx] + vbroadcasti32x4 zmm1, xmmword ptr [rcx+0x1*0x10] + vmovdqa xmm12, xmmword ptr [rsp] + vmovdqa xmm13, xmmword ptr [rsp+0x4*0x10] + vpunpckldq xmm14, xmm12, xmm13 + vpunpckhdq xmm15, xmm12, xmm13 + vpermq ymm14, ymm14, 0xDC + vpermq ymm15, ymm15, 0xDC + vpbroadcastd zmm12, dword ptr [BLAKE3_BLOCK_LEN+rip] + vinserti64x4 zmm13, zmm14, ymm15, 0x01 + mov eax, 17476 + kmovw k2, eax + vpblendmd zmm13 {k2}, zmm13, zmm12 + vbroadcasti32x4 zmm15, xmmword ptr [BLAKE3_IV+rip] + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + mov eax, 43690 + kmovw k3, eax + mov eax, 34952 + kmovw k4, eax + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + mov dword ptr [rsp+0x88], eax + vmovdqa32 zmm2, zmm15 + vpbroadcastd zmm8, dword ptr [rsp+0x22*0x4] + vpblendmd zmm3 {k4}, zmm13, zmm8 + vmovups zmm8, zmmword ptr [r8+rdx-0x1*0x40] + vinserti32x4 zmm8, zmm8, xmmword ptr [r9+rdx-0x4*0x10], 0x01 + vinserti32x4 zmm8, zmm8, xmmword ptr [r10+rdx-0x4*0x10], 0x02 + vinserti32x4 zmm8, zmm8, xmmword ptr [r11+rdx-0x4*0x10], 0x03 + vmovups zmm9, zmmword ptr [r8+rdx-0x30] + vinserti32x4 zmm9, zmm9, xmmword ptr [r9+rdx-0x3*0x10], 0x01 + vinserti32x4 zmm9, zmm9, xmmword ptr [r10+rdx-0x3*0x10], 0x02 + vinserti32x4 zmm9, zmm9, xmmword ptr [r11+rdx-0x3*0x10], 0x03 + vshufps zmm4, zmm8, zmm9, 136 + vshufps zmm5, zmm8, zmm9, 221 + vmovups zmm8, zmmword ptr [r8+rdx-0x20] + vinserti32x4 zmm8, zmm8, xmmword ptr [r9+rdx-0x2*0x10], 0x01 + vinserti32x4 zmm8, zmm8, xmmword ptr [r10+rdx-0x2*0x10], 0x02 + vinserti32x4 zmm8, zmm8, xmmword ptr [r11+rdx-0x2*0x10], 0x03 + vmovups zmm9, zmmword ptr [r8+rdx-0x10] + vinserti32x4 zmm9, zmm9, xmmword ptr [r9+rdx-0x1*0x10], 0x01 + vinserti32x4 zmm9, zmm9, xmmword ptr [r10+rdx-0x1*0x10], 0x02 + vinserti32x4 zmm9, zmm9, xmmword ptr [r11+rdx-0x1*0x10], 0x03 + vshufps zmm6, zmm8, zmm9, 136 + vshufps zmm7, zmm8, zmm9, 221 + vpshufd zmm6, zmm6, 0x93 + vpshufd zmm7, zmm7, 0x93 + mov al, 7 +9: + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm0, zmm0, zmm1 + vpxord zmm3, zmm3, zmm0 + vprord zmm3, zmm3, 16 + vpaddd zmm2, zmm2, zmm3 + vpxord zmm1, zmm1, zmm2 + vprord zmm1, zmm1, 12 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm0, zmm0, zmm1 + vpxord zmm3, zmm3, zmm0 + vprord zmm3, zmm3, 8 + vpaddd zmm2, zmm2, zmm3 + vpxord zmm1, zmm1, zmm2 + vprord zmm1, zmm1, 7 + vpshufd zmm0, zmm0, 0x93 + vpshufd zmm3, zmm3, 0x4E + vpshufd zmm2, zmm2, 0x39 + vpaddd zmm0, zmm0, zmm6 + vpaddd zmm0, zmm0, zmm1 + vpxord zmm3, zmm3, zmm0 + vprord zmm3, zmm3, 16 + vpaddd zmm2, zmm2, zmm3 + vpxord zmm1, zmm1, zmm2 + vprord zmm1, zmm1, 12 + vpaddd zmm0, zmm0, zmm7 + vpaddd zmm0, zmm0, zmm1 + vpxord zmm3, zmm3, zmm0 + vprord zmm3, zmm3, 8 + vpaddd zmm2, zmm2, zmm3 + vpxord zmm1, zmm1, zmm2 + vprord zmm1, zmm1, 7 + vpshufd zmm0, zmm0, 0x39 + vpshufd zmm3, zmm3, 0x4E + vpshufd zmm2, zmm2, 0x93 + dec al + jz 9f + vshufps zmm8, zmm4, zmm5, 214 + vpshufd zmm9, zmm4, 0x0F + vpshufd zmm4, zmm8, 0x39 + vshufps zmm8, zmm6, zmm7, 250 + vpblendmd zmm9 {k3}, zmm9, zmm8 + vpunpcklqdq zmm8, zmm7, zmm5 + vpblendmd zmm8 {k4}, zmm8, zmm6 + vpshufd zmm8, zmm8, 0x78 + vpunpckhdq zmm5, zmm5, zmm7 + vpunpckldq zmm6, zmm6, zmm5 + vpshufd zmm7, zmm6, 0x1E + vmovdqa32 zmm5, zmm9 + vmovdqa32 zmm6, zmm8 + jmp 9b +9: + vpxord zmm0, zmm0, zmm2 + vpxord zmm1, zmm1, zmm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01 + vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01 + vextracti32x4 xmmword ptr [rbx+0x4*0x10], zmm0, 0x02 + vextracti32x4 xmmword ptr [rbx+0x5*0x10], zmm1, 0x02 + vextracti32x4 xmmword ptr [rbx+0x6*0x10], zmm0, 0x03 + vextracti32x4 xmmword ptr [rbx+0x7*0x10], zmm1, 0x03 + vmovdqa xmm0, xmmword ptr [rsp] + vmovdqa xmm2, xmmword ptr [rsp+0x40] + vmovdqa32 xmm0 {k1}, xmmword ptr [rsp+0x1*0x10] + vmovdqa32 xmm2 {k1}, xmmword ptr [rsp+0x5*0x10] + vmovdqa xmmword ptr [rsp], xmm0 + vmovdqa xmmword ptr [rsp+0x40], xmm2 + add rbx, 128 + add rdi, 32 + sub rsi, 4 +3: + test esi, 0x2 + je 3f + vbroadcasti128 ymm0, xmmword ptr [rcx] + vbroadcasti128 ymm1, xmmword ptr [rcx+0x10] + vmovd xmm13, dword ptr [rsp] + vpinsrd xmm13, xmm13, dword ptr [rsp+0x40], 1 + vpinsrd xmm13, xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vmovd xmm14, dword ptr [rsp+0x4] + vpinsrd xmm14, xmm14, dword ptr [rsp+0x44], 1 + vpinsrd xmm14, xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vinserti128 ymm13, ymm13, xmm14, 0x01 + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + mov dword ptr [rsp+0x88], eax + vbroadcasti128 ymm2, xmmword ptr [BLAKE3_IV+rip] + vpbroadcastd ymm8, dword ptr [rsp+0x88] + vpblendd ymm3, ymm13, ymm8, 0x88 + vmovups ymm8, ymmword ptr [r8+rdx-0x40] + vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x40], 0x01 + vmovups ymm9, ymmword ptr [r8+rdx-0x30] + vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x30], 0x01 + vshufps ymm4, ymm8, ymm9, 136 + vshufps ymm5, ymm8, ymm9, 221 + vmovups ymm8, ymmword ptr [r8+rdx-0x20] + vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x20], 0x01 + vmovups ymm9, ymmword ptr [r8+rdx-0x10] + vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x10], 0x01 + vshufps ymm6, ymm8, ymm9, 136 + vshufps ymm7, ymm8, ymm9, 221 + vpshufd ymm6, ymm6, 0x93 + vpshufd ymm7, ymm7, 0x93 + mov al, 7 +9: + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm0, ymm0, ymm1 + vpxord ymm3, ymm3, ymm0 + vprord ymm3, ymm3, 16 + vpaddd ymm2, ymm2, ymm3 + vpxord ymm1, ymm1, ymm2 + vprord ymm1, ymm1, 12 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm0, ymm0, ymm1 + vpxord ymm3, ymm3, ymm0 + vprord ymm3, ymm3, 8 + vpaddd ymm2, ymm2, ymm3 + vpxord ymm1, ymm1, ymm2 + vprord ymm1, ymm1, 7 + vpshufd ymm0, ymm0, 0x93 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm2, ymm2, 0x39 + vpaddd ymm0, ymm0, ymm6 + vpaddd ymm0, ymm0, ymm1 + vpxord ymm3, ymm3, ymm0 + vprord ymm3, ymm3, 16 + vpaddd ymm2, ymm2, ymm3 + vpxord ymm1, ymm1, ymm2 + vprord ymm1, ymm1, 12 + vpaddd ymm0, ymm0, ymm7 + vpaddd ymm0, ymm0, ymm1 + vpxord ymm3, ymm3, ymm0 + vprord ymm3, ymm3, 8 + vpaddd ymm2, ymm2, ymm3 + vpxord ymm1, ymm1, ymm2 + vprord ymm1, ymm1, 7 + vpshufd ymm0, ymm0, 0x39 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm2, ymm2, 0x93 + dec al + jz 9f + vshufps ymm8, ymm4, ymm5, 214 + vpshufd ymm9, ymm4, 0x0F + vpshufd ymm4, ymm8, 0x39 + vshufps ymm8, ymm6, ymm7, 250 + vpblendd ymm9, ymm9, ymm8, 0xAA + vpunpcklqdq ymm8, ymm7, ymm5 + vpblendd ymm8, ymm8, ymm6, 0x88 + vpshufd ymm8, ymm8, 0x78 + vpunpckhdq ymm5, ymm5, ymm7 + vpunpckldq ymm6, ymm6, ymm5 + vpshufd ymm7, ymm6, 0x1E + vmovdqa ymm5, ymm9 + vmovdqa ymm6, ymm8 + jmp 9b +9: + vpxor ymm0, ymm0, ymm2 + vpxor ymm1, ymm1, ymm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01 + vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01 + vmovdqa xmm0, xmmword ptr [rsp] + vmovdqa xmm2, xmmword ptr [rsp+0x4*0x10] + vmovdqu32 xmm0 {k1}, xmmword ptr [rsp+0x8] + vmovdqu32 xmm2 {k1}, xmmword ptr [rsp+0x48] + vmovdqa xmmword ptr [rsp], xmm0 + vmovdqa xmmword ptr [rsp+0x4*0x10], xmm2 + add rbx, 64 + add rdi, 16 + sub rsi, 2 +3: + test esi, 0x1 + je 4b + vmovdqu xmm0, xmmword ptr [rcx] + vmovdqu xmm1, xmmword ptr [rcx+0x10] + vmovd xmm14, dword ptr [rsp] + vpinsrd xmm14, xmm14, dword ptr [rsp+0x40], 1 + vpinsrd xmm14, xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vmovdqa xmm15, xmmword ptr [BLAKE3_IV+rip] + mov r8, qword ptr [rdi] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + vpinsrd xmm3, xmm14, eax, 3 + vmovdqa xmm2, xmm15 + vmovups xmm8, xmmword ptr [r8+rdx-0x40] + vmovups xmm9, xmmword ptr [r8+rdx-0x30] + vshufps xmm4, xmm8, xmm9, 136 + vshufps xmm5, xmm8, xmm9, 221 + vmovups xmm8, xmmword ptr [r8+rdx-0x20] + vmovups xmm9, xmmword ptr [r8+rdx-0x10] + vshufps xmm6, xmm8, xmm9, 136 + vshufps xmm7, xmm8, xmm9, 221 + vpshufd xmm6, xmm6, 0x93 + vpshufd xmm7, xmm7, 0x93 + mov al, 7 +9: + vpaddd xmm0, xmm0, xmm4 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm5 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x93 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x39 + vpaddd xmm0, xmm0, xmm6 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm7 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x39 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x93 + dec al + jz 9f + vshufps xmm8, xmm4, xmm5, 214 + vpshufd xmm9, xmm4, 0x0F + vpshufd xmm4, xmm8, 0x39 + vshufps xmm8, xmm6, xmm7, 250 + vpblendd xmm9, xmm9, xmm8, 0xAA + vpunpcklqdq xmm8, xmm7, xmm5 + vpblendd xmm8, xmm8, xmm6, 0x88 + vpshufd xmm8, xmm8, 0x78 + vpunpckhdq xmm5, xmm5, xmm7 + vpunpckldq xmm6, xmm6, xmm5 + vpshufd xmm7, xmm6, 0x1E + vmovdqa xmm5, xmm9 + vmovdqa xmm6, xmm8 + jmp 9b +9: + vpxor xmm0, xmm0, xmm2 + vpxor xmm1, xmm1, xmm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + jmp 4b +.p2align 6 +_blake3_compress_in_place_avx512_asm: +blake3_compress_in_place_avx512_asm: + _CET_ENDBR + vmovdqu xmm0, xmmword ptr [rdi] + vmovdqu xmm1, xmmword ptr [rdi+0x10] + movzx eax, r8b + movzx edx, dl + shl rax, 32 + add rdx, rax + vmovq xmm3, rcx + vmovq xmm4, rdx + vpunpcklqdq xmm3, xmm3, xmm4 + vmovaps xmm2, xmmword ptr [BLAKE3_IV+rip] + vmovups xmm8, xmmword ptr [rsi] + vmovups xmm9, xmmword ptr [rsi+0x10] + vshufps xmm4, xmm8, xmm9, 136 + vshufps xmm5, xmm8, xmm9, 221 + vmovups xmm8, xmmword ptr [rsi+0x20] + vmovups xmm9, xmmword ptr [rsi+0x30] + vshufps xmm6, xmm8, xmm9, 136 + vshufps xmm7, xmm8, xmm9, 221 + vpshufd xmm6, xmm6, 0x93 + vpshufd xmm7, xmm7, 0x93 + mov al, 7 +9: + vpaddd xmm0, xmm0, xmm4 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm5 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x93 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x39 + vpaddd xmm0, xmm0, xmm6 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm7 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x39 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x93 + dec al + jz 9f + vshufps xmm8, xmm4, xmm5, 214 + vpshufd xmm9, xmm4, 0x0F + vpshufd xmm4, xmm8, 0x39 + vshufps xmm8, xmm6, xmm7, 250 + vpblendd xmm9, xmm9, xmm8, 0xAA + vpunpcklqdq xmm8, xmm7, xmm5 + vpblendd xmm8, xmm8, xmm6, 0x88 + vpshufd xmm8, xmm8, 0x78 + vpunpckhdq xmm5, xmm5, xmm7 + vpunpckldq xmm6, xmm6, xmm5 + vpshufd xmm7, xmm6, 0x1E + vmovdqa xmm5, xmm9 + vmovdqa xmm6, xmm8 + jmp 9b +9: + vpxor xmm0, xmm0, xmm2 + vpxor xmm1, xmm1, xmm3 + vmovdqu xmmword ptr [rdi], xmm0 + vmovdqu xmmword ptr [rdi+0x10], xmm1 + ret + +.p2align 6 +_blake3_compress_xof_avx512_asm: +blake3_compress_xof_avx512_asm: + _CET_ENDBR + vmovdqu xmm0, xmmword ptr [rdi] + vmovdqu xmm1, xmmword ptr [rdi+0x10] + movzx eax, r8b + movzx edx, dl + shl rax, 32 + add rdx, rax + vmovq xmm3, rcx + vmovq xmm4, rdx + vpunpcklqdq xmm3, xmm3, xmm4 + vmovaps xmm2, xmmword ptr [BLAKE3_IV+rip] + vmovups xmm8, xmmword ptr [rsi] + vmovups xmm9, xmmword ptr [rsi+0x10] + vshufps xmm4, xmm8, xmm9, 136 + vshufps xmm5, xmm8, xmm9, 221 + vmovups xmm8, xmmword ptr [rsi+0x20] + vmovups xmm9, xmmword ptr [rsi+0x30] + vshufps xmm6, xmm8, xmm9, 136 + vshufps xmm7, xmm8, xmm9, 221 + vpshufd xmm6, xmm6, 0x93 + vpshufd xmm7, xmm7, 0x93 + mov al, 7 +9: + vpaddd xmm0, xmm0, xmm4 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm5 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x93 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x39 + vpaddd xmm0, xmm0, xmm6 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm7 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x39 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x93 + dec al + jz 9f + vshufps xmm8, xmm4, xmm5, 214 + vpshufd xmm9, xmm4, 0x0F + vpshufd xmm4, xmm8, 0x39 + vshufps xmm8, xmm6, xmm7, 250 + vpblendd xmm9, xmm9, xmm8, 0xAA + vpunpcklqdq xmm8, xmm7, xmm5 + vpblendd xmm8, xmm8, xmm6, 0x88 + vpshufd xmm8, xmm8, 0x78 + vpunpckhdq xmm5, xmm5, xmm7 + vpunpckldq xmm6, xmm6, xmm5 + vpshufd xmm7, xmm6, 0x1E + vmovdqa xmm5, xmm9 + vmovdqa xmm6, xmm8 + jmp 9b +9: + vpxor xmm0, xmm0, xmm2 + vpxor xmm1, xmm1, xmm3 + vpxor xmm2, xmm2, [rdi] + vpxor xmm3, xmm3, [rdi+0x10] + vmovdqu xmmword ptr [r9], xmm0 + vmovdqu xmmword ptr [r9+0x10], xmm1 + vmovdqu xmmword ptr [r9+0x20], xmm2 + vmovdqu xmmword ptr [r9+0x30], xmm3 + ret + +#ifdef __APPLE__ +.static_data +#else +.section .rodata +#endif +.p2align 6 +INDEX0: + .long 0, 1, 2, 3, 16, 17, 18, 19 + .long 8, 9, 10, 11, 24, 25, 26, 27 +INDEX1: + .long 4, 5, 6, 7, 20, 21, 22, 23 + .long 12, 13, 14, 15, 28, 29, 30, 31 +ADD0: + .long 0, 1, 2, 3, 4, 5, 6, 7 + .long 8, 9, 10, 11, 12, 13, 14, 15 +ADD1: .long 1 + +ADD16: .long 16 +BLAKE3_BLOCK_LEN: + .long 64 +.p2align 6 +BLAKE3_IV: +BLAKE3_IV_0: + .long 0x6A09E667 +BLAKE3_IV_1: + .long 0xBB67AE85 +BLAKE3_IV_2: + .long 0x3C6EF372 +BLAKE3_IV_3: + .long 0xA54FF53A diff --git a/src/blake3/blake3_backend.c b/src/blake3/blake3_backend.c new file mode 100644 index 0000000..72ff22e --- /dev/null +++ b/src/blake3/blake3_backend.c @@ -0,0 +1,584 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "fy-bit64.h" + +#include "blake3.h" +#include "blake3_impl.h" +#include "blake3_internal.h" + +// Declarations for implementation-specific functions. +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +/* the portable hasher (can drive the optimized ones, but not optimally) */ +extern const blake3_hasher_ops blake3_hasher_op_portable; + +#if defined(IS_X86) +void blake3_compress_in_place_sse2(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse2(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +void blake3_compress_in_place_sse2_asm(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse2_asm(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse2_asm(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +extern const blake3_hasher_ops blake3_hasher_op_sse2; + +void blake3_compress_in_place_sse41(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse41(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +void blake3_compress_in_place_sse41_asm(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse41_asm(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse41_asm(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +extern const blake3_hasher_ops blake3_hasher_op_sse41; + +void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +void blake3_hash_many_avx2_asm(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +extern const blake3_hasher_ops blake3_hasher_op_avx2; + +void blake3_compress_in_place_avx512(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +void blake3_compress_in_place_avx512_asm(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_avx512_asm(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_avx512_asm(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +extern const blake3_hasher_ops blake3_hasher_op_avx512; + +#endif + +#if defined(IS_ARM) +void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +extern const blake3_hasher_ops blake3_hasher_op_neon; +#endif + +static uint64_t supported_cpusimd_backend(void) +{ + const blake3_backend *be = &blake3_backends[B3BID_CPUSIMD]; + return be->user ? B3BF_CPUSIMD : 0; +} + +static uint64_t detected_cpusimd_backend(void) +{ + const blake3_backend *be = &blake3_backends[B3BID_CPUSIMD]; + return be->user ? B3BF_CPUSIMD : 0; +} + +static uint64_t supported_synthetic_backends(void) +{ + uint64_t backends = 0; + + backends |= supported_cpusimd_backend(); + return backends; +} + +static uint64_t detected_synthetic_backends(void) +{ + uint64_t backends = 0; + + backends |= detected_cpusimd_backend(); + return backends; +} + +static uint64_t supported_gpu_backends(void) +{ + return 0; +} + +static uint64_t detected_gpu_backends(void) +{ + return 0; +} + +#if defined(IS_X86) + +static uint64_t supported_backends_x86(void) +{ + uint64_t backends = 0; + +#if !defined(BLAKE3_NO_SSE2) + backends |= B3BF_SSE2 | B3BF_SSE2_ASM; +#endif +#if !defined(BLAKE3_NO_SSE41) + backends |= B3BF_SSE41 | B3BF_SSE41_ASM; +#endif +#if !defined(BLAKE3_NO_AVX) + backends |= B3BF_AVX2 | B3BF_AVX2_ASM; +#endif +#if !defined(BLAKE3_NO_AVX512) + backends |= B3BF_AVX512 | B3BF_AVX512_ASM; +#endif + return backends; +} + +static inline uint64_t xgetbv(void) +{ +#if defined(_MSC_VER) + return _xgetbv(0); +#else + uint32_t eax = 0, edx = 0; + __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0)); + return ((uint64_t)edx << 32) | eax; +#endif +} + +static inline void cpuid(uint32_t out[4], uint32_t id) +{ +#if defined(_MSC_VER) + __cpuid((int *)out, id); +#elif defined(__i386__) || defined(_M_IX86) + __asm__ __volatile__( + "movl %%ebx, %1\n" + "cpuid\n" + "xchgl %1, %%ebx\n" + : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id)); +#else + __asm__ __volatile__("cpuid\n" + : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id)); +#endif +} + +static inline void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) +{ +#if defined(_MSC_VER) + __cpuidex((int *)out, id, sid); +#elif defined(__i386__) || defined(_M_IX86) + __asm__ __volatile__( + "movl %%ebx, %1\n" + "cpuid\n" + "xchgl %1, %%ebx\n" + : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id), "c"(sid)); +#else + __asm__ __volatile__("cpuid\n" + : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id), "c"(sid)); +#endif +} + +static uint64_t detected_backends_x86(void) +{ + uint32_t regs[4] = {0}; + uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3]; + int max_id; + uint64_t mask = 0; + uint64_t backends = 0; + + (void)edx; + + cpuid(regs, 0); + max_id = *eax; + cpuid(regs, 1); + +#if defined(__amd64__) || defined(_M_X64) + backends |= B3BF_SSE2 | B3BF_SSE2_ASM; +#else + if (*edx & (1UL << 26)) + backends |= B3BF_SSE2 | B3BF_SSE2_ASM; +#endif + if (*ecx & (1UL << 19)) + backends |= B3BF_SSE41 | B3BF_SSE41_ASM; + + if (*ecx & (1UL << 27)) { // OSXSAVE + mask = xgetbv(); + if ((mask & 6) == 6) { // SSE and AVX states + if (max_id >= 7) { + cpuidex(regs, 7, 0); + if (*ebx & (1UL << 5)) + backends |= B3BF_AVX2 | B3BF_AVX2_ASM; + if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm + if ((*ebx & ((1UL << 31) | (1UL << 16))) == ((1UL << 31) | (1UL << 16))) + backends |= B3BF_AVX512 | B3BF_AVX512_ASM; /* AVX412VL+AVX512F */ + } + } + } + } + + return backends; +} +#endif + +#if defined(IS_ARM) +static uint64_t supported_backends_arm(void) +{ + uint64_t backends = 0; + +#if defined(IS_AARCH64) + backends |= B3BF_NEON; +#endif + + return backends; +} + +static uint64_t detected_backends_arm(void) +{ + uint64_t backends = 0; + +#if defined(IS_AARCH64) + backends |= B3BF_NEON; +#endif + + return backends; +} +#endif + +uint64_t blake3_get_supported_backends(void) +{ + uint64_t supported_backends; + + supported_backends = B3BF_PORTABLE; + + supported_backends |= supported_gpu_backends(); + +#if defined(IS_X86) + supported_backends |= supported_backends_x86(); +#endif + +#if defined(IS_ARM) + supported_backends |= supported_backends_arm(); +#endif + + supported_backends |= supported_synthetic_backends(); + + return supported_backends; +} + +uint64_t blake3_get_detected_backends(void) +{ + uint64_t detected_backends; + + detected_backends = B3BF_PORTABLE; + + detected_backends |= detected_gpu_backends(); + +#if defined(IS_X86) + detected_backends |= detected_backends_x86(); +#endif + +#if defined(IS_ARM) + detected_backends |= detected_backends_arm(); +#endif + + detected_backends |= detected_synthetic_backends(); + + return detected_backends; +} + +uint64_t blake3_get_selectable_backends(void) +{ + return blake3_get_supported_backends() | + blake3_get_detected_backends(); +} + +blake3_backend blake3_backends[B3BID_COUNT] = { + [B3BID_PORTABLE] = { + .info = { + .id = B3BID_PORTABLE, + .name = "portable", + .description = "portable C implementation", + .simd_degree = 1, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_portable, + .hash_many = blake3_hash_many_portable, + .compress_xof = blake3_compress_xof_portable, + .compress_in_place = blake3_compress_in_place_portable, + }, +#if defined(IS_X86) + +#if defined(TARGET_HAS_SSE2) && TARGET_HAS_SSE2 + [B3BID_SSE2] = { + .info = { + .id = B3BID_SSE2, + .name = "sse2", + .description = "x86 SSE2 implementation in C", + .simd_degree = 4, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_sse2, + .hash_many = blake3_hash_many_sse2, + .compress_xof = blake3_compress_xof_sse2, + .compress_in_place = blake3_compress_in_place_sse2, + }, + [B3BID_SSE2_ASM] = { + .info = { + .id = B3BID_SSE2_ASM, + .name = "sse2-asm", + .description = "x86 SSE2 implementation in assembly", + .simd_degree = 4, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_sse2, + .hash_many = blake3_hash_many_sse2_asm, + .compress_xof = blake3_compress_xof_sse2_asm, + .compress_in_place = blake3_compress_in_place_sse2_asm, + }, +#endif + +#if defined(TARGET_HAS_SSE41) && TARGET_HAS_SSE41 + [B3BID_SSE41] = { + .info = { + .id = B3BID_SSE41, + .name = "sse41", + .description = "x86 SSE41 implementation in C", + .simd_degree = 4, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_sse41, + .hash_many = blake3_hash_many_sse41, + .compress_xof = blake3_compress_xof_sse41, + .compress_in_place = blake3_compress_in_place_sse41, + }, + [B3BID_SSE41_ASM] = { + .info = { + .id = B3BID_SSE41_ASM, + .name = "sse41-asm", + .description = "x86 SSE41 implementation in assembly", + .simd_degree = 4, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_sse41, + .hash_many = blake3_hash_many_sse41_asm, + .compress_xof = blake3_compress_xof_sse41_asm, + .compress_in_place = blake3_compress_in_place_sse41_asm, + }, +#endif + +#if defined(TARGET_HAS_AVX2) && TARGET_HAS_AVX2 + [B3BID_AVX2] = { + .info = { + .id = B3BID_AVX2, + .name = "avx2", + .description = "x86 AVX2 implementation in C", + .simd_degree = 8, + .funcs = B3FF_HASH_MANY, + }, + .hasher_ops = &blake3_hasher_op_avx2, + .hash_many = blake3_hash_many_avx2, + }, + [B3BID_AVX2_ASM] = { + .info = { + .id = B3BID_AVX2_ASM, + .name = "avx2-asm", + .description = "x86 AVX2 implementation in assembly", + .simd_degree = 8, + .funcs = B3FF_HASH_MANY, + }, + .hasher_ops = &blake3_hasher_op_avx2, + .hash_many = blake3_hash_many_avx2_asm, + }, +#endif +#if defined(TARGET_HAS_AVX512) && TARGET_HAS_AVX512 + [B3BID_AVX512] = { + .info = { + .id = B3BID_AVX512, + .name = "avx512", + .description = "x86 AVX512 VL+F implementation in C", + .simd_degree = 16, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_avx512, + .hash_many = blake3_hash_many_avx512, + .compress_xof = blake3_compress_xof_avx512, + .compress_in_place = blake3_compress_in_place_avx512, + }, + [B3BID_AVX512_ASM] = { + .info = { + .id = B3BID_AVX512_ASM, + .name = "avx512-asm", + .description = "x86 AVX512 VL+F implementation in assembly", + .simd_degree = 16, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_avx512, + .hash_many = blake3_hash_many_avx512, + .compress_xof = blake3_compress_xof_avx512_asm, + .compress_in_place = blake3_compress_in_place_avx512_asm, + }, +#endif + +#endif + +#if defined(IS_ARM) + +#if defined(TARGET_HAS_NEON) && TARGET_HAS_NEON + [B3BID_NEON] = { + .info = { + .id = B3BID_NEON, + .name = "neon", + .description = "arm NEON implementation", + .simd_degree = 4, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_neon, + .hash_many = blake3_hash_many_neon, + .compress_xof = blake3_compress_xof_portable, /* no NEON for this */ + .compress_in_place = blake3_compress_in_place_portable, + }, +#endif + +#endif +}; + +const blake3_backend *blake3_backend_select_function(uint64_t selectable_backends, blake3_func_id fid) +{ + const blake3_backend *be; + unsigned int i; + + while (selectable_backends) { + i = highest_one(selectable_backends); + selectable_backends &= ~FY_BIT64(i); + be = blake3_backends + i; + if (be->info.funcs & FY_BIT64(fid)) + return be; + } + /* should never happen because portable is always set */ + assert(false); + return NULL; +} + +const blake3_backend *blake3_get_backend_by_id(blake3_backend_id id) +{ + const blake3_backend *be; + + if ((unsigned int)id >= B3BID_COUNT) + return NULL; + + be = &blake3_backends[id]; + + /* non-enabled backend? */ + if (be->info.id != id || !be->info.name) + return NULL; + + return be; +} + +const blake3_backend *blake3_get_backend_by_name(const char *name) +{ + const blake3_backend *be; + unsigned int i; + + if (!name) + return NULL; + + for (i = 0, be = blake3_backends; i < B3BID_COUNT; i++, be++) { + if (be->info.name && !strcmp(be->info.name, name)) + return be; + } + return NULL; +} + +const blake3_backend_info *blake3_get_backend_info(blake3_backend_id id) +{ + const blake3_backend_info *bei; + + if ((unsigned int)id >= B3BID_COUNT) + return NULL; + + bei = &blake3_backends[id].info; + + /* not supported */ + if (bei->id == B3BID_INVALID || !bei->name) + return NULL; + + return bei; +} diff --git a/src/blake3/blake3_be_cpusimd.c b/src/blake3/blake3_be_cpusimd.c new file mode 100644 index 0000000..8074176 --- /dev/null +++ b/src/blake3/blake3_be_cpusimd.c @@ -0,0 +1,239 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "blake3.h" +#include "blake3_impl.h" +#include "blake3_internal.h" + +#include "fy-thread.h" + +struct cpusimd_data { + unsigned int num_cpus; + unsigned int simd_cpus; + unsigned int be_simd_degree_mult; + const blake3_backend *be_best; + char *description; + struct fy_thread_pool *tp; +}; + +static void blake3_cpusimd_hash_many_thread(void *arg) +{ + blake3_hash_many_state *s = arg; + const blake3_hash_many_common_state *c; + + c = s->common; + c->hash_many(s->inputs, s->num_inputs, c->blocks, c->key, s->counter, c->increment_counter, c->flags, c->flags_start, c->flags_end, s->out); +} + +// #define CPUSIMD_CHECK + +static void blake3_hash_many_cpusimd( + const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) +{ + struct cpusimd_data *d; + const blake3_backend *be, *be_best; + unsigned int i, stepi, be_simd_degree_mult, simd_degree, count, states_count; + blake3_hash_many_state *states, *s; + blake3_hash_many_common_state common_local, *common = &common_local; + const uint8_t *const *inputsi; + uint8_t *outi; + uint64_t counteri; +#ifdef CPUSIMD_CHECK + size_t out_size; + uint8_t *out_cmp; +#endif + + (void)simd_degree; + + be = &blake3_backends[B3BID_CPUSIMD]; + d = be->user; + assert(d); + + count = (unsigned int)num_inputs; + simd_degree = be->info.simd_degree; + + be_best = d->be_best; + be_simd_degree_mult = d->be_simd_degree_mult; + +#ifdef CPUSIMD_CHECK + /* if less than the underlyind backend degree just do it there */ + out_size = BLAKE3_OUT_LEN * num_inputs; + out_cmp = alloca(out_size); + be_best->hash_many(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out_cmp); +#endif + + states_count = (count / be_simd_degree_mult) + !!(count % be_simd_degree_mult); + states = alloca(sizeof(*states) * states_count); + + /* the common */ + common->hash_many = be_best->hash_many; + common->blocks = blocks; + common->key = key; + common->increment_counter = increment_counter; + common->flags = flags; + common->flags_start = flags_start; + common->flags_end = flags_end; + + inputsi = inputs; + outi = out; + counteri = counter; + for (i = 0, s = states; i < count; i += stepi, s++) { + + stepi = (i + be_simd_degree_mult) <= count ? be_simd_degree_mult : (count - i); + + s->inputs = inputsi; + s->num_inputs = stepi; + s->counter = counteri; + s->out = outi; + s->common = common; + + if (increment_counter) + counteri += stepi; + inputsi += stepi; + outi += BLAKE3_OUT_LEN * stepi; + } + + fy_thread_arg_array_join(d->tp, blake3_cpusimd_hash_many_thread, NULL, states, sizeof(states[0]), states_count); + +#ifdef CPUSIMD_CHECK + for (i = 0; i < count; i += stepi) { + stepi = (i + be_simd_degree_mult) <= count ? be_simd_degree_mult : (count - i); + + if (memcmp(out + i * BLAKE3_OUT_LEN, out_cmp + i * BLAKE3_OUT_LEN, BLAKE3_OUT_LEN)) { + fprintf(stderr, "%s: differ at #%u\n", __func__, i); + } + } +#endif +} + +static void cpusimd_data_free(struct cpusimd_data *d) +{ + if (!d) + return; + + if (d->description) + free(d->description); + + if (d->tp) + fy_thread_pool_destroy(d->tp); + + free(d); +} + +int blake3_backend_cpusimd_setup(unsigned int num_cpus, unsigned int mult_fact) +{ + blake3_backend *be = &blake3_backends[B3BID_CPUSIMD]; + const blake3_backend *be_best; + struct cpusimd_data *d = NULL; + struct fy_thread_pool_cfg tp_cfg; + long scval; + uint64_t supported_backends, detected_backends, selectable_backends; + unsigned int num_simd_cpus; + ssize_t n; + + if (!num_cpus) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } + + if (num_cpus <= 1) + return 0; + + num_simd_cpus = round_down_to_power_of_2(num_cpus); + + if (!mult_fact) + mult_fact = 1; + + d = malloc(sizeof(*d)); + if (!d) + goto err_out; + + memset(d, 0, sizeof(*d)); + + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = 0; + tp_cfg.num_threads = num_simd_cpus; + tp_cfg.userdata = NULL; + + d->tp = fy_thread_pool_create(&tp_cfg); + if (!d->tp) + goto err_out; + + /* probe for available backends */ + supported_backends = blake3_get_supported_backends(); + detected_backends = blake3_get_detected_backends(); + selectable_backends = supported_backends & detected_backends; + + /* remove ourselves (and anything above) */ + selectable_backends &= B3BF_BIT(B3BID_CPUSIMD) - 1; + + /* select the best one for hash many */ + be_best = blake3_backend_select_function(selectable_backends, B3FID_HASH_MANY); + assert(be_best); + + d->num_cpus = num_cpus; + d->simd_cpus = num_simd_cpus; + d->be_best = be_best; + d->be_simd_degree_mult = be_best->info.simd_degree * mult_fact; + + be->hasher_ops = be_best->hasher_ops; + + d->description = NULL; + n = 0; + for (;;) { + n = snprintf(d->description, n ? (n + 1) : 0, "SIMD like acceleration using %u CPUs (using %s x %u) x %u = total x %u", + d->simd_cpus, + d->be_best->info.name, d->be_best->info.simd_degree, + mult_fact, + d->be_simd_degree_mult * d->simd_cpus); + if (d->description) + break; + + d->description = malloc(n + 1); + if (!d->description) + goto err_out; + } + + be->user = d; + + be->info.id = B3BID_CPUSIMD; + be->info.name = "cpusimd"; + be->info.simd_degree = d->simd_cpus * d->be_best->info.simd_degree * mult_fact; + be->info.description = d->description; + + be->hash_many = blake3_hash_many_cpusimd; + be->info.funcs = B3FF_HASH_MANY; + + return 0; + +err_out: + cpusimd_data_free(d); + return -1; +} + +void blake3_backend_cpusimd_cleanup(void) +{ + blake3_backend *be = &blake3_backends[B3BID_CPUSIMD]; + struct cpusimd_data *d; + + d = be->user; + if (!d) + return; + cpusimd_data_free(d); + + memset(be, 0, sizeof(*be)); +} diff --git a/src/blake3/blake3_host_state.c b/src/blake3/blake3_host_state.c new file mode 100644 index 0000000..f76b9cb --- /dev/null +++ b/src/blake3/blake3_host_state.c @@ -0,0 +1,569 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fy-bit64.h" + +#include "blake3.h" +#include "blake3_impl.h" +#include "blake3_internal.h" + +#include "fy-thread.h" + +/* 256K threshold for using alloca */ +#define BLAKE3_ALLOCA_BUFFER_SIZE (256U << 10) +#define BLAKE3_FILE_IO_BUFFER_SIZE BLAKE3_ALLOCA_BUFFER_SIZE +#define BLAKE3_MMAP_MIN_CHUNKSIZE (1U << 20) // minimum chunksize is 1MB +#define BLAKE3_MMAP_MAX_CHUNKSIZE SIZE_MAX + +static int probe_backends(blake3_host_state *hs) +{ + hs->supported_backends = blake3_get_supported_backends(); + hs->detected_backends = blake3_get_detected_backends(); + hs->selectable_backends = hs->supported_backends & hs->detected_backends; + + return 0; +} + +static int select_backends(blake3_host_state *hs) +{ + const blake3_backend *be, *be_portable; + + /* this is the fallback */ + be_portable = blake3_get_backend_by_id(B3BID_PORTABLE); + assert(be_portable); + + /* if there's a backend selected, try to force it */ + if (hs->cfg.backend && hs->cfg.backend[0] && strcmp(hs->cfg.backend, "auto")) { + be = blake3_get_backend_by_name(hs->cfg.backend); + + /* if backend exists and is selectable */ + if (be && (hs->selectable_backends & FY_BIT64(be->info.id))) { + hs->hash_many_be = (be->info.funcs & B3FF_HASH_MANY) ? be : be_portable; + hs->compress_xof_be = (be->info.funcs & B3FF_COMPRESS_XOF) ? be : be_portable; + hs->compress_in_place_be = (be->info.funcs & B3FF_COMPRESS_IN_PLACE) ? be : be_portable; + + /* and select the ops (piggyback on HASH_MANY) */ + hs->hasher_ops = (be->info.funcs & B3FF_HASH_MANY) ? be->hasher_ops : be_portable->hasher_ops; + } + } + + if (!hs->hash_many_be) { + hs->hash_many_be = blake3_backend_select_function(hs->selectable_backends, B3FID_HASH_MANY); + hs->hasher_ops = hs->hash_many_be->hasher_ops; + } + if (!hs->compress_xof_be) + hs->compress_xof_be = blake3_backend_select_function(hs->selectable_backends, B3FID_COMPRESS_XOF); + if (!hs->compress_in_place_be) + hs->compress_in_place_be = blake3_backend_select_function(hs->selectable_backends, B3FID_COMPRESS_IN_PLACE); + + /* select methods */ + hs->hash_many = hs->hash_many_be->hash_many; + hs->compress_xof = hs->compress_xof_be->compress_xof; + hs->compress_in_place = hs->compress_in_place_be->compress_in_place; + + /* select maximum simd degree */ + if (hs->simd_degree < hs->hash_many_be->info.simd_degree) + hs->simd_degree = hs->hash_many_be->info.simd_degree; + if (hs->simd_degree < hs->compress_xof_be->info.simd_degree) + hs->simd_degree = hs->compress_xof_be->info.simd_degree; + if (hs->simd_degree < hs->compress_in_place_be->info.simd_degree) + hs->simd_degree = hs->compress_in_place_be->info.simd_degree; + + return 0; +} + +static void dump_backends(blake3_host_state *hs, uint64_t backends) +{ + const blake3_backend *be; + int i; + + for (i = 0; i < B3BID_COUNT; i++) { + if (!(backends & FY_BIT64(i))) + continue; + be = blake3_get_backend_by_id(i); + assert(be); + + fprintf(stderr, "%sname: %s\n%sdescription: %s\n%ssimd_degree: %u\n%shas_hash_many: %s\n%shas_compress_xof: %s\n%shas_compress_in_place: %s\n", + " -", be->info.name, + " ", be->info.description, + " ", be->info.simd_degree, + " ", (be->info.funcs & B3FF_HASH_MANY) ? "true" : "false", + " ", (be->info.funcs & B3FF_COMPRESS_XOF) ? "true" : "false", + " ", (be->info.funcs & B3FF_COMPRESS_IN_PLACE) ? "true" : "false"); + } +} + +int blake3_host_state_setup(blake3_host_state *hs, const blake3_host_config *cfg) +{ + struct fy_thread_pool_cfg tp_cfg; + long scval; + int rc; + + (void)rc; + + assert(hs); + assert(cfg); + + memset(hs, 0, sizeof(*hs)); + hs->cfg = *cfg; + + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + + hs->num_cpus = (unsigned int)scval; + + rc = probe_backends(hs); + assert(!rc); + + rc = select_backends(hs); + assert(!rc); + + hs->mt_degree = hs->cfg.mt_degree > 0 ? hs->cfg.mt_degree : 64; /* 64K default */ + hs->tp = NULL; + + if (!hs->cfg.no_mthread) { + + if (!hs->cfg.tp) { + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = FYTPCF_STEAL_MODE; + tp_cfg.num_threads = hs->cfg.num_threads ? hs->cfg.num_threads : (hs->num_cpus * 3) / 2; + tp_cfg.userdata = NULL; + hs->tp = fy_thread_pool_create(&tp_cfg); + if (!hs->tp) + goto err_out; + } else + hs->tp = hs->cfg.tp; + } + + hs->file_io_bufsz = hs->cfg.file_io_bufsz ? hs->cfg.file_io_bufsz : BLAKE3_FILE_IO_BUFFER_SIZE; + hs->mmap_min_chunk = hs->cfg.mmap_min_chunk ? hs->cfg.mmap_min_chunk : BLAKE3_MMAP_MIN_CHUNKSIZE; + hs->mmap_max_chunk = hs->cfg.mmap_max_chunk ? hs->cfg.mmap_max_chunk : BLAKE3_MMAP_MAX_CHUNKSIZE; + + if (hs->cfg.debug) { + + fprintf(stderr, "num_cpus: %u\n", hs->num_cpus); + fprintf(stderr, "num_threads: %d\n", hs->tp ? fy_thread_pool_get_num_threads(hs->tp) : 0); + fprintf(stderr, "simd_degree: %u\n", hs->simd_degree); + fprintf(stderr, "mt_degree: %u\n", hs->mt_degree); + fprintf(stderr, "file_io_bufsz: %zu\n", hs->file_io_bufsz); + fprintf(stderr, "mmap_min_chunk: %zu\n", hs->mmap_min_chunk); + fprintf(stderr, "mmap_max_chunk: %zu\n", hs->mmap_max_chunk); + + fprintf(stderr, "supported_backends:\n"); + dump_backends(hs, hs->supported_backends); + fprintf(stderr, "detected_backends:\n"); + dump_backends(hs, hs->detected_backends); + + fprintf(stderr, "selected-backends:\n"); + fprintf(stderr, " hash_many: %s\n", hs->hash_many_be->info.name); + fprintf(stderr, " compress_xof: %s\n", hs->compress_xof_be->info.name); + fprintf(stderr, " compress_in_place: %s\n", hs->compress_in_place_be->info.name); + + } + + return 0; + +err_out: + return -1; +} + +void blake3_host_state_cleanup(blake3_host_state *hs) +{ + assert(hs); + + /* destroy the thread pool if we're the ones created it */ + if (hs->tp && !hs->cfg.tp) + fy_thread_pool_destroy(hs->tp); +} + +blake3_host_state *blake3_host_state_create(const blake3_host_config *cfg) +{ + blake3_host_state *hs; + int rc; + + hs = malloc(sizeof(*hs)); + if (!hs) + return NULL; + rc = blake3_host_state_setup(hs, cfg); + if (rc) + goto err_out; + + return hs; + +err_out: + if (hs) + free(hs); + return NULL; +} + +void blake3_host_state_destroy(blake3_host_state *hs) +{ + if (!hs) + return; + blake3_host_state_cleanup(hs); + free(hs); +} + +blake3_hasher *blake3_hasher_create(blake3_host_state *hs, const uint8_t *key, const void *context, size_t context_len) +{ + blake3_hasher *self; + + self = fy_cacheline_alloc(sizeof(*self)); + if (!self) + return NULL; + + if (!key && !context) + blake3_hasher_init(hs, self); + else if (key) + blake3_hasher_init_keyed(hs, self, key); + else if (context_len == 0) + blake3_hasher_init_derive_key(hs, self, context); + else + blake3_hasher_init_derive_key_raw(hs, self, context, context_len); + return self; +} + +void blake3_hasher_destroy(blake3_hasher *self) +{ + if (!self) + return; + fy_cacheline_free(self); +} + +#if defined(__linux__) + +static int linux_block_dev_is_rotational(dev_t dev) +{ + char *path = NULL; + int rc, fd; + uint8_t c[2]; + int rotational; + + /* by default it's the filesize */ + rotational = -1; /* not found */ + + /* read the rotational attribute of the root */ + rc = asprintf(&path, "/sys/dev/block/%u:%u/queue/rotational", major(dev), 0); + if (rc != -1) { + fd = open(path, O_RDONLY); + if (fd != -1) { + if (read(fd, c, 2) == 2) + rotational = c[0] == '1' && c[1] == '\n'; + close(fd); + } + free(path); + } + + return rotational; +} + +static ssize_t linux_file_cached_size(int fd, void *mem, size_t filesize) +{ + long pagesize = sysconf(_SC_PAGESIZE); + size_t vec_size, resident_size, i; + unsigned char *vec; + int rc; + + (void)rc; + vec_size = (filesize + pagesize - 1) / pagesize; + vec = malloc(vec_size + 1); + if (!vec) + return -1; + + rc = mincore(mem, filesize, vec); + assert(!rc); + + /* this could be parallelized */ + resident_size = 0; + for (i = 0; i < vec_size; i++) { + if (vec[i] & 1) + resident_size += pagesize; + } + + free(vec); + + if (resident_size > filesize) + resident_size = filesize; + + return (ssize_t)resident_size; +} + +static size_t blake3_mmap_file_chunksize(int fd, dev_t dev, void *mem, + size_t filesize, + size_t mmap_min_chunk, size_t mmap_max_chunk) +{ + size_t chunksize; + ssize_t cached_size; + int rotational; + + if (filesize <= mmap_min_chunk) + return filesize; + + /* by default it's the filesize */ + chunksize = filesize; + + /* starting, check if the rotational attribute exists in this dev */ + rotational = linux_block_dev_is_rotational(dev); + if (rotational == -1) { + /* attribute not found? check the non-partition */ + rotational = linux_block_dev_is_rotational(makedev(major(dev), 0)); + } + + if (rotational == 1) { + /* OK, it's rotational, but is it in cache? + * to avoid checking for the cached status of the whole file + * we just probe the MIN_CHUNKSIZE + * We will thrash in the case where the file is only cached + * for the first few bytes, but this is generally unusual. + */ + cached_size = linux_file_cached_size(fd, mem, mmap_min_chunk); + + if (cached_size <= 0) + chunksize = mmap_min_chunk; + } + + if (chunksize > mmap_max_chunk) + chunksize = mmap_max_chunk; + + return chunksize; +} + +#else + +static size_t blake3_mmap_file_chunksize(int fd, dev_t dev, void *mem, + size_t filesize, + size_t mmap_min_chunk, size_t mmap_max_chunk) +{ + if (filesize <= mmap_min_chunk) + return filesize; + + if (filesize > mmap_max_chunk) + return mmap_max_chunk; + + /* for all others, just use mmap at max */ + return filesize; +} + +#endif + +int blake3_hash_file(blake3_hasher *hasher, const char *filename, + uint8_t output[BLAKE3_OUT_LEN]) +{ + blake3_host_state *hs; + FILE *fp = NULL; + void *mem = NULL, *buf = NULL, *p; + int fd = -1, ret = -1; + size_t rdn, filesize, bufsz = 0, max_chunk, left, chunk; + struct stat sb; + dev_t dev = 0; + int rc; + + if (!hasher || !filename || !output) + return -1; + + hs = hasher->hs; + + if (hs->cfg.debug) + fprintf(stderr, "processing file %s\n", filename); + + // reset the hasher (do not initialize again) + blake3_hasher_reset(hasher); + + if (!strcmp(filename, "-")) { + fp = stdin; + } else { + fp = NULL; + + fd = open(filename, O_RDONLY); + if (fd < 0) { + if (hs->cfg.debug) + fprintf(stderr, "unable to open %s - %s\n", + filename, strerror(errno)); + goto err_out; + } + + rc = fstat(fd, &sb); + if (rc < 0) { + if (hs->cfg.debug) + fprintf(stderr, "failed to stat %s - %s\n", + filename, strerror(errno)); + goto err_out; + } + + if (!S_ISREG(sb.st_mode)) { + if (S_ISDIR(sb.st_mode)) + errno = EISDIR; + else + errno = EINVAL; + if (hs->cfg.debug) + fprintf(stderr, "not a regular file %s - %s\n", + filename, strerror(errno)); + goto err_out; + } + + filesize = (size_t)-1; + + /* try to mmap */ + if (sb.st_size > 0 && !hs->cfg.no_mmap) { + filesize = sb.st_size; + mem = mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (mem != MAP_FAILED) { + close(fd); + fd = -1; + dev = sb.st_dev; + } else + mem = NULL; + } + + /* unable to map? fallback to stream mode */ + if (!mem) { + fp = fdopen(fd, "r"); + if (!fp) { + if (hs->cfg.debug) + fprintf(stderr, "unable to fdopen %s - %s\n", + filename, strerror(errno)); + goto err_out; + } + } + } + + /* mmap case, very simple */ + if (mem) { + + /* if the file is small enough don't bother with the rest */ + max_chunk = blake3_mmap_file_chunksize(fd, dev, mem, filesize, + hs->mmap_min_chunk, hs->mmap_max_chunk); + + p = mem; + left = filesize; + while (left > 0) { + chunk = left > max_chunk ? max_chunk : left; + blake3_hasher_update(hasher, p, chunk); + p += chunk; + left -= chunk; + } + } else { + /* slow path using file reads */ + + assert(fp); + + bufsz = hs->file_io_bufsz; + + /* for less than 8MB alloca */ + if (bufsz <= BLAKE3_ALLOCA_BUFFER_SIZE) { + buf = alloca(bufsz); + } else { + buf = malloc(bufsz); + if (!buf) { + if (hs->cfg.debug) + fprintf(stderr, "Unable to allocate buffer of %zu bytes\n", + bufsz); + goto err_out; + } + } + + do { + rdn = fread(buf, 1, bufsz, fp); + if (rdn == 0) + break; + if (rdn > 0) + blake3_hasher_update(hasher, buf, rdn); + } while (rdn >= bufsz); + } + + // Finalize the hash. BLAKE3_OUT_LEN is the default output length, 32 bytes. + blake3_hasher_finalize(hasher, output, BLAKE3_OUT_LEN); + + ret = 0; + +out: + if (mem) + munmap(mem, filesize); + + if (fp && fp != stdin) + fclose(fp); + + if (fd >= 0) + close(fd); + + if (buf && (bufsz > BLAKE3_ALLOCA_BUFFER_SIZE)) + free(buf); + + return ret; + +err_out: + ret = -1; + goto out; +} + +void blake3_hash(struct blake3_hasher *hasher, + const void *mem, size_t size, uint8_t output[BLAKE3_OUT_LEN]) +{ + blake3_hasher_reset(hasher); + blake3_hasher_update(hasher, mem, size); + blake3_hasher_finalize(hasher, output, BLAKE3_OUT_LEN); +} + +const char *blake3_version(void) +{ + return BLAKE3_VERSION_STRING; +} + +void blake3_hasher_init(struct blake3_host_state *hs, struct blake3_hasher *self) +{ + hs->hasher_ops->hasher_init(hs, self); +} + +void blake3_hasher_init_keyed(struct blake3_host_state *hs, struct blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]) +{ + hs->hasher_ops->hasher_init_keyed(hs, self, key); +} + +void blake3_hasher_init_derive_key(struct blake3_host_state *hs, struct blake3_hasher *self, const char *context) +{ + hs->hasher_ops->hasher_init_derive_key(hs, self, context); +} + +void blake3_hasher_init_derive_key_raw(struct blake3_host_state *hs, struct blake3_hasher *self, const void *context, size_t context_len) +{ + hs->hasher_ops->hasher_init_derive_key_raw(hs, self, context, context_len); +} + +void blake3_hasher_update(struct blake3_hasher *self, const void *input, size_t input_len) +{ + blake3_host_state *hs = self->hs; + hs->hasher_ops->hasher_update(self, input, input_len); +} + +void blake3_hasher_finalize(const struct blake3_hasher *self, uint8_t *out, size_t out_len) +{ + blake3_host_state *hs = self->hs; + hs->hasher_ops->hasher_finalize(self, out, out_len); +} + +void blake3_hasher_finalize_seek(const struct blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len) +{ + blake3_host_state *hs = self->hs; + hs->hasher_ops->hasher_finalize_seek(self, seek, out, out_len); +} + +void blake3_hasher_reset(struct blake3_hasher *self) +{ + blake3_host_state *hs = self->hs; + hs->hasher_ops->hasher_reset(self); +} diff --git a/src/blake3/blake3_impl.h b/src/blake3/blake3_impl.h new file mode 100644 index 0000000..230dc49 --- /dev/null +++ b/src/blake3/blake3_impl.h @@ -0,0 +1,234 @@ +#ifndef BLAKE3_IMPL_H +#define BLAKE3_IMPL_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#if defined(__linux__) +#include +#endif +#endif + +#if defined(__linux__) +#include +#endif + +#include "blake3.h" + +#include "fy-endian.h" +#include "fy-blob.h" +#include "fy-bit64.h" +#include "fy-align.h" + +#if defined(HAVE_STRING_OP_OVERREAD) && HAVE_STRING_OP_OVERREAD && defined(__GNUC__) && !defined(__clang__) +#define GCC_DISABLE_WSTRING_OP_OVERREAD +#endif + +/* this is the best alignment for blake3 */ +#if defined(__x86_64__) || defined(_M_X64) +#define BLAKE3_ALIGNMENT 64 +#else +#define BLAKE3_ALIGNMENT 32 +#endif + +#define BLAKE3_ALIGN FY_ALIGNED_TO(BLAKE3_ALIGNMENT) + +// This C implementation tries to support recent versions of GCC, Clang, and +// MSVC. +#if defined(_MSC_VER) +#define INLINE static __forceinline +#else +#define INLINE static inline __attribute__((always_inline)) +#endif + +#if defined(__x86_64__) || defined(_M_X64) +#define IS_X86 +#define IS_X86_64 +#endif + +#if defined(__i386__) || defined(_M_IX86) +#define IS_X86 +#define IS_X86_32 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#define IS_AARCH64 +#define IS_ARM +#endif + +#if defined(IS_X86) +#if defined(_MSC_VER) +#include +#endif +#endif + +/* Find index of the highest set bit */ +/* x is assumed to be nonzero. */ +static inline unsigned int highest_one(uint64_t x) +{ +#if defined(__GNUC__) || defined(__clang__) + return 63 ^ (unsigned int)__builtin_clzll(x); +#elif defined(_MSC_VER) && defined(IS_X86_64) + unsigned long index; + _BitScanReverse64(&index, x); + return index; +#elif defined(_MSC_VER) && defined(IS_X86_32) + unsigned long index; + + if(x >> 32) { + _BitScanReverse(&index, (unsigned long)(x >> 32)); + index += 32; + } else + _BitScanReverse(&index, (unsigned long)x); + return index; +#else + unsigned int c = 0; + + if (x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } + if (x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } + if (x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } + if (x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } + if (x & 0x000000000000000cULL) { x >>= 2; c += 2; } + if (x & 0x0000000000000002ULL) { c += 1; } + return c; +#endif +} + +static inline unsigned int lowest_one(uint64_t x) +{ +#if defined(__GNUC__) || defined(__clang__) + return (unsigned int)__builtin_ctzll(x); +#else + unsigned int c = 0; + + if (!(x & 0x00000000ffffffffULL)) { x >>= 32; c += 32; } + if (!(x & 0x000000000000ffffULL)) { x >>= 16; c += 16; } + if (!(x & 0x00000000000000ffULL)) { x >>= 8; c += 8; } + if (!(x & 0x000000000000000fULL)) { x >>= 4; c += 4; } + if (!(x & 0x0000000000000003ULL)) { x >>= 2; c += 2; } + if (!(x & 0x0000000000000001ULL)) { c += 1; } + return c; +#endif +} + +// Count the number of 1 bits. +static inline unsigned int popcnt(uint64_t x) +{ +#if defined(__GNUC__) || defined(__clang__) + return (unsigned int)__builtin_popcountll(x); +#else + unsigned int count = 0; + while (x != 0) { + count++; + x &= x - 1; + } + return count; +#endif +} + +// Largest power of two less than or equal to x. As a special case, returns 1 +// when x is 0. +static inline uint64_t round_down_to_power_of_2(uint64_t x) +{ + return 1ULL << highest_one(x | 1); +} + +static inline uint32_t counter_low(uint64_t counter) +{ + return (uint32_t)counter; +} + +static inline uint32_t counter_high(uint64_t counter) +{ + return (uint32_t)(counter >> 32); +} + +static inline uint32_t load32(const void *src) +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + return *(const uint32_t *)src; +#else + const uint8_t *p = (const uint8_t *)src; + return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | + ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); +#endif +} + +static inline void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], uint32_t key_words[8]) +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + memcpy(key_words, key, BLAKE3_KEY_LEN); +#else + key_words[0] = load32(&key[0 * 4]); + key_words[1] = load32(&key[1 * 4]); + key_words[2] = load32(&key[2 * 4]); + key_words[3] = load32(&key[3 * 4]); + key_words[4] = load32(&key[4 * 4]); + key_words[5] = load32(&key[5 * 4]); + key_words[6] = load32(&key[6 * 4]); + key_words[7] = load32(&key[7 * 4]); +#endif +} + +static inline void store32(void *dst, uint32_t w) +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + *(uint32_t *)dst = w; +#else + uint8_t *p = (uint8_t *)dst; + + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); +#endif +} + +static inline void store_cv_words(uint8_t bytes_out[BLAKE3_OUT_LEN], uint32_t cv_words[BLAKE3_OUT_WORDS]) +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + memcpy(bytes_out, cv_words, BLAKE3_KEY_LEN); +#else + store32(&bytes_out[0 * 4], cv_words[0]); + store32(&bytes_out[1 * 4], cv_words[1]); + store32(&bytes_out[2 * 4], cv_words[2]); + store32(&bytes_out[3 * 4], cv_words[3]); + store32(&bytes_out[4 * 4], cv_words[4]); + store32(&bytes_out[5 * 4], cv_words[5]); + store32(&bytes_out[6 * 4], cv_words[6]); + store32(&bytes_out[7 * 4], cv_words[7]); +#endif +} + +/* the IV */ +#define B3_IV_0 ((uint32_t)0x6A09E667UL) +#define B3_IV_1 ((uint32_t)0xBB67AE85UL) +#define B3_IV_2 ((uint32_t)0x3C6EF372UL) +#define B3_IV_3 ((uint32_t)0xA54FF53AUL) +#define B3_IV_4 ((uint32_t)0x510E527FUL) +#define B3_IV_5 ((uint32_t)0x9B05688CUL) +#define B3_IV_6 ((uint32_t)0x1F83D9ABUL) +#define B3_IV_7 ((uint32_t)0x5BE0CD19UL) + +/* the message schedule definition */ +#define B3_MSG_SCHEDULE_DEF \ + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, \ + {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, \ + {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, \ + {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, \ + {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, \ + {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, \ + {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13} + +#endif /* BLAKE3_IMPL_H */ diff --git a/src/blake3/blake3_internal.h b/src/blake3/blake3_internal.h new file mode 100644 index 0000000..2bfa42e --- /dev/null +++ b/src/blake3/blake3_internal.h @@ -0,0 +1,158 @@ +#ifndef BLAKE3_INTERNAL_H +#define BLAKE3_INTERNAL_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "blake3.h" + +#include "blake3_impl.h" + +#define BLAKE3_MAX_DEPTH 54 + +// fwd declaration of opaque types +struct blake3_host_state; +struct blake3_hasher; +struct fy_thread_pool; + +// internal flags +enum blake3_flags { + CHUNK_START = 1 << 0, + CHUNK_END = 1 << 1, + PARENT = 1 << 2, + ROOT = 1 << 3, + KEYED_HASH = 1 << 4, + DERIVE_KEY_CONTEXT = 1 << 5, + DERIVE_KEY_MATERIAL = 1 << 6, +}; + +typedef struct blake3_chunk_state { + uint32_t cv[8] BLAKE3_ALIGN; + uint8_t buf[BLAKE3_BLOCK_LEN] BLAKE3_ALIGN; + uint64_t chunk_counter; + uint8_t buf_len; + uint8_t blocks_compressed; + uint8_t flags; +} blake3_chunk_state BLAKE3_ALIGN; + +typedef struct blake3_hasher { + struct blake3_host_state *hs; + uint32_t key[8] BLAKE3_ALIGN; + blake3_chunk_state chunk BLAKE3_ALIGN; + // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example, + // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk + // requires a 4th entry, rather than merging everything down to 1, because we + // don't know whether more input is coming. This is different from how the + // reference implementation does things. + uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN] BLAKE3_ALIGN; + uint8_t cv_stack_len; +} blake3_hasher BLAKE3_ALIGN; + +typedef void (*blake3_hash_many_f)(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out); + +typedef void (*blake3_compress_xof_f)(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64]); + +typedef void (*blake3_compress_in_place_f)(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +typedef struct blake3_backend { + const blake3_hasher_ops *hasher_ops; + blake3_backend_info info; + blake3_hash_many_f hash_many; + blake3_compress_xof_f compress_xof; + blake3_compress_in_place_f compress_in_place; + void *user; /* per backend data */ +} blake3_backend; + +// the state for the thread when doing compress subtree +typedef struct blake3_compress_subtree_state { + struct blake3_hasher *self; + // inputs + const uint8_t *input; + size_t input_len; + const uint32_t *key; + uint64_t chunk_counter; + uint8_t flags; + // outputs + uint8_t *out; + size_t n; +} blake3_compress_subtree_state; + +typedef struct blake3_hash_many_common_state { + blake3_hash_many_f hash_many; + size_t blocks; + const uint32_t *key; + bool increment_counter; + uint8_t flags, flags_start, flags_end; +} blake3_hash_many_common_state; + +typedef struct blake3_hash_many_state { + const blake3_hash_many_common_state *common; + const uint8_t *const *inputs; + size_t num_inputs; + uint64_t counter; + uint8_t *out; +} blake3_hash_many_state; + +typedef struct blake3_host_state { + blake3_host_config cfg; + unsigned int num_cpus; + uint64_t supported_backends; + uint64_t detected_backends; + uint64_t selectable_backends; + + /* backend for hash_many */ + const blake3_backend *hash_many_be; + blake3_hash_many_f hash_many; + + /* backend for compress_xof */ + const blake3_backend *compress_xof_be; + blake3_compress_xof_f compress_xof; + + /* backend for compress_in_place */ + const blake3_backend *compress_in_place_be; + blake3_compress_in_place_f compress_in_place; + + const blake3_hasher_ops *hasher_ops; + + unsigned int simd_degree; + unsigned int mt_degree; + + unsigned int num_threads; + + struct fy_thread_pool *tp; + + size_t file_io_bufsz; + size_t mmap_min_chunk; + size_t mmap_max_chunk; + +} blake3_host_state; + +int blake3_host_state_setup(blake3_host_state *hs, const blake3_host_config *cfg); +void blake3_host_state_cleanup(blake3_host_state *hs); + +extern blake3_backend blake3_backends[B3BID_COUNT]; + +const blake3_backend *blake3_backend_select_function(uint64_t selectable_backends, blake3_func_id fid); +const blake3_backend *blake3_get_backend_by_id(blake3_backend_id id); +const blake3_backend *blake3_get_backend_by_name(const char *name); +const blake3_backend_info *blake3_get_backend_info(blake3_backend_id id); + +void blake3_cpusimd_setup(unsigned int num_cpus, unsigned int mult_fact); +void blake3_cpusimd_cleanup(void); + +#endif diff --git a/src/blake3/blake3_neon.c b/src/blake3/blake3_neon.c new file mode 100644 index 0000000..e7caba3 --- /dev/null +++ b/src/blake3/blake3_neon.c @@ -0,0 +1,375 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" + +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +#ifdef __ARM_BIG_ENDIAN +#error "This implementation only supports little-endian ARM." +// It might be that all we need for big-endian support here is to get the loads +// and stores right, but step zero would be finding a way to test it in CI. +#endif + +INLINE uint32x4_t loadu_128(const uint8_t src[16]) { + // vld1q_u32 has alignment requirements. Don't use it. + uint32x4_t x; + memcpy(&x, src, 16); + return x; +} + +INLINE void storeu_128(uint32x4_t src, uint8_t dest[16]) { + // vst1q_u32 has alignment requirements. Don't use it. + memcpy(dest, &src, 16); +} + +INLINE uint32x4_t add_128(uint32x4_t a, uint32x4_t b) { + return vaddq_u32(a, b); +} + +INLINE uint32x4_t xor_128(uint32x4_t a, uint32x4_t b) { + return veorq_u32(a, b); +} + +INLINE uint32x4_t set1_128(uint32_t x) { return vld1q_dup_u32(&x); } + +INLINE uint32x4_t set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + uint32_t array[4] = {a, b, c, d}; + return vld1q_u32(array); +} + +INLINE uint32x4_t rot16_128(uint32x4_t x) { + // The straightfoward implementation would be two shifts and an or, but that's + // slower on microarchitectures we've tested. See + // https://github.com/BLAKE3-team/BLAKE3/pull/319. + // return vorrq_u32(vshrq_n_u32(x, 16), vshlq_n_u32(x, 32 - 16)); + return vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x))); +} + +INLINE uint32x4_t rot12_128(uint32x4_t x) { + // See comment in rot16_128. + // return vorrq_u32(vshrq_n_u32(x, 12), vshlq_n_u32(x, 32 - 12)); + return vsriq_n_u32(vshlq_n_u32(x, 32-12), x, 12); +} + +INLINE uint32x4_t rot8_128(uint32x4_t x) { + // See comment in rot16_128. + // return vorrq_u32(vshrq_n_u32(x, 8), vshlq_n_u32(x, 32 - 8)); +#if defined(__clang__) + return vreinterpretq_u32_u8(__builtin_shufflevector(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12)); +#elif __GNUC__ * 10000 + __GNUC_MINOR__ * 100 >=40700 + static const uint8x16_t r8 = {1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12}; + return vreinterpretq_u32_u8(__builtin_shuffle(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), r8)); +#else + return vsriq_n_u32(vshlq_n_u32(x, 32-8), x, 8); +#endif +} + +INLINE uint32x4_t rot7_128(uint32x4_t x) { + // See comment in rot16_128. + // return vorrq_u32(vshrq_n_u32(x, 7), vshlq_n_u32(x, 32 - 7)); + return vsriq_n_u32(vshlq_n_u32(x, 32-7), x, 7); +} + +// TODO: compress_neon + +// TODO: hash2_neon + +/* + * ---------------------------------------------------------------------------- + * hash4_neon + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn4(uint32x4_t v[16], uint32x4_t m[16], size_t r) { + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[15] = rot16_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot12_128(v[4]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[15] = rot8_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot7_128(v[4]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot16_128(v[15]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[4] = rot12_128(v[4]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot8_128(v[15]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + v[4] = rot7_128(v[4]); +} + +INLINE void transpose_vecs_128(uint32x4_t vecs[4]) { + // Individually transpose the four 2x2 sub-matrices in each corner. + uint32x4x2_t rows01 = vtrnq_u32(vecs[0], vecs[1]); + uint32x4x2_t rows23 = vtrnq_u32(vecs[2], vecs[3]); + + // Swap the top-right and bottom-left 2x2s (which just got transposed). + vecs[0] = + vcombine_u32(vget_low_u32(rows01.val[0]), vget_low_u32(rows23.val[0])); + vecs[1] = + vcombine_u32(vget_low_u32(rows01.val[1]), vget_low_u32(rows23.val[1])); + vecs[2] = + vcombine_u32(vget_high_u32(rows01.val[0]), vget_high_u32(rows23.val[0])); + vecs[3] = + vcombine_u32(vget_high_u32(rows01.val[1]), vget_high_u32(rows23.val[1])); +} + +INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, + size_t block_offset, uint32x4_t out[16]) { + out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(uint32x4_t)]); + out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(uint32x4_t)]); + out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(uint32x4_t)]); + out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(uint32x4_t)]); + out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(uint32x4_t)]); + out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(uint32x4_t)]); + out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(uint32x4_t)]); + out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(uint32x4_t)]); + out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(uint32x4_t)]); + out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(uint32x4_t)]); + out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(uint32x4_t)]); + out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(uint32x4_t)]); + out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(uint32x4_t)]); + out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(uint32x4_t)]); + out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(uint32x4_t)]); + out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(uint32x4_t)]); + transpose_vecs_128(&out[0]); + transpose_vecs_128(&out[4]); + transpose_vecs_128(&out[8]); + transpose_vecs_128(&out[12]); +} + +INLINE void load_counters4(uint64_t counter, bool increment_counter, + uint32x4_t *out_low, uint32x4_t *out_high) { + uint64_t mask = (increment_counter ? ~0 : 0); + *out_low = set4( + counter_low(counter + (mask & 0)), counter_low(counter + (mask & 1)), + counter_low(counter + (mask & 2)), counter_low(counter + (mask & 3))); + *out_high = set4( + counter_high(counter + (mask & 0)), counter_high(counter + (mask & 1)), + counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3))); +} + +void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + uint32x4_t h_vecs[8] = { + set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), + set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), + }; + uint32x4_t counter_low_vec, counter_high_vec; + load_counters4(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + uint32x4_t block_len_vec = set1_128(BLAKE3_BLOCK_LEN); + uint32x4_t block_flags_vec = set1_128(block_flags); + uint32x4_t msg_vecs[16]; + transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + uint32x4_t v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_128(B3_IV_0), set1_128(B3_IV_1), set1_128(B3_IV_2), set1_128(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn4(v, msg_vecs, 0); + round_fn4(v, msg_vecs, 1); + round_fn4(v, msg_vecs, 2); + round_fn4(v, msg_vecs, 3); + round_fn4(v, msg_vecs, 4); + round_fn4(v, msg_vecs, 5); + round_fn4(v, msg_vecs, 6); + h_vecs[0] = xor_128(v[0], v[8]); + h_vecs[1] = xor_128(v[1], v[9]); + h_vecs[2] = xor_128(v[2], v[10]); + h_vecs[3] = xor_128(v[3], v[11]); + h_vecs[4] = xor_128(v[4], v[12]); + h_vecs[5] = xor_128(v[5], v[13]); + h_vecs[6] = xor_128(v[6], v[14]); + h_vecs[7] = xor_128(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs_128(&h_vecs[0]); + transpose_vecs_128(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu_128(h_vecs[0], &out[0 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[4], &out[1 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[1], &out[2 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[5], &out[3 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[2], &out[4 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[6], &out[5 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[3], &out[6 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[7], &out[7 * sizeof(uint32x4_t)]); +} + +/* + * ---------------------------------------------------------------------------- + * hash_many_neon + * ---------------------------------------------------------------------------- + */ + +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +INLINE void hash_one_neon(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, uint8_t flags_end, + uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + // TODO: Implement compress_neon. However note that according to + // https://github.com/BLAKE2/BLAKE2/commit/7965d3e6e1b4193438b8d3a656787587d2579227, + // compress_neon might not be any faster than compress_portable. + blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= 4) { + blake3_hash4_neon(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 4; + } + inputs += 4; + num_inputs -= 4; + out = &out[4 * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_neon(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/src/blake3/blake3_portable.c b/src/blake3/blake3_portable.c new file mode 100644 index 0000000..bd07d07 --- /dev/null +++ b/src/blake3/blake3_portable.c @@ -0,0 +1,167 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +INLINE uint32_t rotr32(uint32_t w, uint32_t c) { + return (w >> c) | (w << (32 - c)); +} + +INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, + uint32_t x, uint32_t y) { + state[a] = state[a] + state[b] + x; + state[d] = rotr32(state[d] ^ state[a], 16); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 12); + state[a] = state[a] + state[b] + y; + state[d] = rotr32(state[d] ^ state[a], 8); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 7); +} + +INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { + // Select the message schedule based on the round. + const uint8_t *schedule = MSG_SCHEDULE[round]; + + // Mix the columns. + g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); + g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); + g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); + g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); + + // Mix the rows. + g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); + g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); + g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); + g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); +} + +INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + uint32_t block_words[16]; + block_words[0] = load32(block + 4 * 0); + block_words[1] = load32(block + 4 * 1); + block_words[2] = load32(block + 4 * 2); + block_words[3] = load32(block + 4 * 3); + block_words[4] = load32(block + 4 * 4); + block_words[5] = load32(block + 4 * 5); + block_words[6] = load32(block + 4 * 6); + block_words[7] = load32(block + 4 * 7); + block_words[8] = load32(block + 4 * 8); + block_words[9] = load32(block + 4 * 9); + block_words[10] = load32(block + 4 * 10); + block_words[11] = load32(block + 4 * 11); + block_words[12] = load32(block + 4 * 12); + block_words[13] = load32(block + 4 * 13); + block_words[14] = load32(block + 4 * 14); + block_words[15] = load32(block + 4 * 15); + + state[0] = cv[0]; + state[1] = cv[1]; + state[2] = cv[2]; + state[3] = cv[3]; + state[4] = cv[4]; + state[5] = cv[5]; + state[6] = cv[6]; + state[7] = cv[7]; + state[8] = B3_IV_0; + state[9] = B3_IV_1; + state[10] = B3_IV_2; + state[11] = B3_IV_3; + state[12] = counter_low(counter); + state[13] = counter_high(counter); + state[14] = (uint32_t)block_len; + state[15] = (uint32_t)flags; + + round_fn(state, &block_words[0], 0); + round_fn(state, &block_words[0], 1); + round_fn(state, &block_words[0], 2); + round_fn(state, &block_words[0], 3); + round_fn(state, &block_words[0], 4); + round_fn(state, &block_words[0], 5); + round_fn(state, &block_words[0], 6); +} + +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + cv[0] = state[0] ^ state[8]; + cv[1] = state[1] ^ state[9]; + cv[2] = state[2] ^ state[10]; + cv[3] = state[3] ^ state[11]; + cv[4] = state[4] ^ state[12]; + cv[5] = state[5] ^ state[13]; + cv[6] = state[6] ^ state[14]; + cv[7] = state[7] ^ state[15]; +} + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + + store32(&out[0 * 4], state[0] ^ state[8]); + store32(&out[1 * 4], state[1] ^ state[9]); + store32(&out[2 * 4], state[2] ^ state[10]); + store32(&out[3 * 4], state[3] ^ state[11]); + store32(&out[4 * 4], state[4] ^ state[12]); + store32(&out[5 * 4], state[5] ^ state[13]); + store32(&out[6 * 4], state[6] ^ state[14]); + store32(&out[7 * 4], state[7] ^ state[15]); + store32(&out[8 * 4], state[8] ^ cv[0]); + store32(&out[9 * 4], state[9] ^ cv[1]); + store32(&out[10 * 4], state[10] ^ cv[2]); + store32(&out[11 * 4], state[11] ^ cv[3]); + store32(&out[12 * 4], state[12] ^ cv[4]); + store32(&out[13 * 4], state[13] ^ cv[5]); + store32(&out[14 * 4], state[14] ^ cv[6]); + store32(&out[15 * 4], state[15] ^ cv[7]); +} + +INLINE void hash_one_portable(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + store_cv_words(out, cv); +} + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs > 0) { + hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/src/blake3/blake3_sse2.c b/src/blake3/blake3_sse2.c new file mode 100644 index 0000000..f2bb2e3 --- /dev/null +++ b/src/blake3/blake3_sse2.c @@ -0,0 +1,573 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" + +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +#define DEGREE 4 + +#define _mm_shuffle_ps2(a, b, c) \ + (_mm_castps_si128( \ + _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) + +INLINE __m128i loadu(const uint8_t src[16]) { + return _mm_loadu_si128((const __m128i *)src); +} + +INLINE void storeu(__m128i src, uint8_t dest[16]) { + _mm_storeu_si128((__m128i *)dest, src); +} + +INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } + +// Note that clang-format doesn't like the name "xor" for some reason. +INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } + +INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } + +INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); +} + +INLINE __m128i rot16(__m128i x) { + return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, 0xB1), 0xB1); +} + +INLINE __m128i rot12(__m128i x) { + return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); +} + +INLINE __m128i rot8(__m128i x) { + return xorv(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 32 - 8)); +} + +INLINE __m128i rot7(__m128i x) { + return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); +} + +INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot16(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot12(*row1); +} + +INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot8(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot7(*row1); +} + +// Note the optimization here of leaving row1 as the unrotated row, rather than +// row0. All the message loads below are adjusted to compensate for this. See +// discussion at https://github.com/sneves/blake2-avx2/pull/4 +INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); +} + +INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); +} + +INLINE __m128i blend_epi16(__m128i a, __m128i b, const int16_t imm8) { + const __m128i bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01); + __m128i mask = _mm_set1_epi16(imm8); + mask = _mm_and_si128(mask, bits); + mask = _mm_cmpeq_epi16(mask, bits); + return _mm_or_si128(_mm_and_si128(mask, b), _mm_andnot_si128(mask, a)); +} + +INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + rows[0] = loadu((uint8_t *)&cv[0]); + rows[1] = loadu((uint8_t *)&cv[4]); + rows[2] = set4(B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3); + rows[3] = set4(counter_low(counter), counter_high(counter), + (uint32_t)block_len, (uint32_t)flags); + + __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); + __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); + __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); + __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); + + __m128i t0, t1, t2, t3, tt; + + // Round 1. The first round permutes the message words from the original + // input order, into the groups that get mixed in parallel. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 + t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 + t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 2. This round and all following rounds apply a fixed permutation + // to the message words from the round before. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 3 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 4 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 5 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 6 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 7 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); +} + +void blake3_compress_in_place_sse2(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); + storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); +} + +void blake3_compress_xof_sse2(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), &out[0]); + storeu(xorv(rows[1], rows[3]), &out[16]); + storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); + storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); +} + +INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[15] = rot16(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot12(v[4]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[15] = rot8(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot7(v[4]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot16(v[15]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[4] = rot12(v[4]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot8(v[15]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + v[4] = rot7(v[4]); +} + +INLINE void transpose_vecs(__m128i vecs[DEGREE]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is + // 22/33. Note that this doesn't split the vector into two lanes, as the + // AVX2 counterparts do. + __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); + __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); + __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); + __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); + + // Interleave 64-bit lanes. + __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); + __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); + __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); + __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); + + vecs[0] = abcd_0; + vecs[1] = abcd_1; + vecs[2] = abcd_2; + vecs[3] = abcd_3; +} + +INLINE void transpose_msg_vecs(const uint8_t *const *inputs, + size_t block_offset, __m128i out[16]) { + out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); + out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); + out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); + out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); + out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); + out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); + out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); + out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); + out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); + out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); + out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); + out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); + out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); + out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); + out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); + out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); + for (size_t i = 0; i < 4; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs(&out[0]); + transpose_vecs(&out[4]); + transpose_vecs(&out[8]); + transpose_vecs(&out[12]); +} + +INLINE void load_counters(uint64_t counter, bool increment_counter, + __m128i *out_lo, __m128i *out_hi) { + const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); + const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); + const __m128i add1 = _mm_and_si128(mask, add0); + __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); + __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), + _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); + __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); + *out_lo = l; + *out_hi = h; +} + +static +void blake3_hash4_sse2(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m128i h_vecs[8] = { + set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), + set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), + }; + __m128i counter_low_vec, counter_high_vec; + load_counters(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); + __m128i block_flags_vec = set1(block_flags); + __m128i msg_vecs[16]; + transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m128i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1(B3_IV_0), set1(B3_IV_1), set1(B3_IV_2), set1(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn(v, msg_vecs, 0); + round_fn(v, msg_vecs, 1); + round_fn(v, msg_vecs, 2); + round_fn(v, msg_vecs, 3); + round_fn(v, msg_vecs, 4); + round_fn(v, msg_vecs, 5); + round_fn(v, msg_vecs, 6); + h_vecs[0] = xorv(v[0], v[8]); + h_vecs[1] = xorv(v[1], v[9]); + h_vecs[2] = xorv(v[2], v[10]); + h_vecs[3] = xorv(v[3], v[11]); + h_vecs[4] = xorv(v[4], v[12]); + h_vecs[5] = xorv(v[5], v[13]); + h_vecs[6] = xorv(v[6], v[14]); + h_vecs[7] = xorv(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs(&h_vecs[0]); + transpose_vecs(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); + storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); + storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); + storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); + storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); + storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); + storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); + storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); +} + +INLINE void hash_one_sse2(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_sse2(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= DEGREE) { + blake3_hash4_sse2(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += DEGREE; + } + inputs += DEGREE; + num_inputs -= DEGREE; + out = &out[DEGREE * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_sse2(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/src/blake3/blake3_sse2_x86-64_unix.S b/src/blake3/blake3_sse2_x86-64_unix.S new file mode 100644 index 0000000..d2d77bc --- /dev/null +++ b/src/blake3/blake3_sse2_x86-64_unix.S @@ -0,0 +1,2291 @@ +#if defined(__ELF__) && defined(__linux__) +.section .note.GNU-stack,"",%progbits +#endif + +#if defined(__ELF__) && defined(__CET__) && defined(__has_include) +#if __has_include() +#include +#endif +#endif + +#if !defined(_CET_ENDBR) +#define _CET_ENDBR +#endif + +.intel_syntax noprefix +.global blake3_hash_many_sse2_asm +.global _blake3_hash_many_sse2_asm +.global blake3_compress_in_place_sse2_asm +.global _blake3_compress_in_place_sse2_asm +.global blake3_compress_xof_sse2_asm +.global _blake3_compress_xof_sse2_asm +#ifdef __APPLE__ +.text +#else +.section .text +#endif + .p2align 6 +_blake3_hash_many_sse2_asm: +blake3_hash_many_sse2_asm: + _CET_ENDBR + push r15 + push r14 + push r13 + push r12 + push rbx + push rbp + mov rbp, rsp + sub rsp, 360 + and rsp, 0xFFFFFFFFFFFFFFC0 + neg r9d + movd xmm0, r9d + pshufd xmm0, xmm0, 0x00 + movdqa xmmword ptr [rsp+0x130], xmm0 + movdqa xmm1, xmm0 + pand xmm1, xmmword ptr [ADD0+rip] + pand xmm0, xmmword ptr [ADD1+rip] + movdqa xmmword ptr [rsp+0x150], xmm0 + movd xmm0, r8d + pshufd xmm0, xmm0, 0x00 + paddd xmm0, xmm1 + movdqa xmmword ptr [rsp+0x110], xmm0 + pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip] + pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip] + pcmpgtd xmm1, xmm0 + shr r8, 32 + movd xmm2, r8d + pshufd xmm2, xmm2, 0x00 + psubd xmm2, xmm1 + movdqa xmmword ptr [rsp+0x120], xmm2 + mov rbx, qword ptr [rbp+0x50] + mov r15, rdx + shl r15, 6 + movzx r13d, byte ptr [rbp+0x38] + movzx r12d, byte ptr [rbp+0x48] + cmp rsi, 4 + jc 3f +2: + movdqu xmm3, xmmword ptr [rcx] + pshufd xmm0, xmm3, 0x00 + pshufd xmm1, xmm3, 0x55 + pshufd xmm2, xmm3, 0xAA + pshufd xmm3, xmm3, 0xFF + movdqu xmm7, xmmword ptr [rcx+0x10] + pshufd xmm4, xmm7, 0x00 + pshufd xmm5, xmm7, 0x55 + pshufd xmm6, xmm7, 0xAA + pshufd xmm7, xmm7, 0xFF + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +9: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movdqu xmm8, xmmword ptr [r8+rdx-0x40] + movdqu xmm9, xmmword ptr [r9+rdx-0x40] + movdqu xmm10, xmmword ptr [r10+rdx-0x40] + movdqu xmm11, xmmword ptr [r11+rdx-0x40] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp], xmm8 + movdqa xmmword ptr [rsp+0x10], xmm9 + movdqa xmmword ptr [rsp+0x20], xmm12 + movdqa xmmword ptr [rsp+0x30], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x30] + movdqu xmm9, xmmword ptr [r9+rdx-0x30] + movdqu xmm10, xmmword ptr [r10+rdx-0x30] + movdqu xmm11, xmmword ptr [r11+rdx-0x30] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0x40], xmm8 + movdqa xmmword ptr [rsp+0x50], xmm9 + movdqa xmmword ptr [rsp+0x60], xmm12 + movdqa xmmword ptr [rsp+0x70], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x20] + movdqu xmm9, xmmword ptr [r9+rdx-0x20] + movdqu xmm10, xmmword ptr [r10+rdx-0x20] + movdqu xmm11, xmmword ptr [r11+rdx-0x20] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0x80], xmm8 + movdqa xmmword ptr [rsp+0x90], xmm9 + movdqa xmmword ptr [rsp+0xA0], xmm12 + movdqa xmmword ptr [rsp+0xB0], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x10] + movdqu xmm9, xmmword ptr [r9+rdx-0x10] + movdqu xmm10, xmmword ptr [r10+rdx-0x10] + movdqu xmm11, xmmword ptr [r11+rdx-0x10] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0xC0], xmm8 + movdqa xmmword ptr [rsp+0xD0], xmm9 + movdqa xmmword ptr [rsp+0xE0], xmm12 + movdqa xmmword ptr [rsp+0xF0], xmm13 + movdqa xmm9, xmmword ptr [BLAKE3_IV_1+rip] + movdqa xmm10, xmmword ptr [BLAKE3_IV_2+rip] + movdqa xmm11, xmmword ptr [BLAKE3_IV_3+rip] + movdqa xmm12, xmmword ptr [rsp+0x110] + movdqa xmm13, xmmword ptr [rsp+0x120] + movdqa xmm14, xmmword ptr [BLAKE3_BLOCK_LEN+rip] + movd xmm15, eax + pshufd xmm15, xmm15, 0x00 + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + paddd xmm0, xmmword ptr [rsp] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x40] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [BLAKE3_IV_0+rip] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x10] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x50] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x80] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0xC0] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x90] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0xD0] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x20] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x70] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x60] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x10] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x90] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xB0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0xE0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x30] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0xD0] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x40] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x20] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x60] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0xB0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x50] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0xF0] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xA0] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0xE0] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x70] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0x30] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x40] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0x50] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x80] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xC0] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0xF0] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xD0] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0xA0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x70] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x20] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x10] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x90] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0x80] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xE0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0xC0] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xD0] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0x20] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x30] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0x60] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xB0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0x10] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xF0] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0x90] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xE0] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x30] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xA0] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x40] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + pxor xmm0, xmm8 + pxor xmm1, xmm9 + pxor xmm2, xmm10 + pxor xmm3, xmm11 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + pxor xmm4, xmm12 + pxor xmm5, xmm13 + pxor xmm6, xmm14 + pxor xmm7, xmm15 + mov eax, r13d + jne 9b + movdqa xmm9, xmm0 + punpckldq xmm0, xmm1 + punpckhdq xmm9, xmm1 + movdqa xmm11, xmm2 + punpckldq xmm2, xmm3 + punpckhdq xmm11, xmm3 + movdqa xmm1, xmm0 + punpcklqdq xmm0, xmm2 + punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm9 + punpcklqdq xmm9, xmm11 + punpckhqdq xmm3, xmm11 + movdqu xmmword ptr [rbx], xmm0 + movdqu xmmword ptr [rbx+0x20], xmm1 + movdqu xmmword ptr [rbx+0x40], xmm9 + movdqu xmmword ptr [rbx+0x60], xmm3 + movdqa xmm9, xmm4 + punpckldq xmm4, xmm5 + punpckhdq xmm9, xmm5 + movdqa xmm11, xmm6 + punpckldq xmm6, xmm7 + punpckhdq xmm11, xmm7 + movdqa xmm5, xmm4 + punpcklqdq xmm4, xmm6 + punpckhqdq xmm5, xmm6 + movdqa xmm7, xmm9 + punpcklqdq xmm9, xmm11 + punpckhqdq xmm7, xmm11 + movdqu xmmword ptr [rbx+0x10], xmm4 + movdqu xmmword ptr [rbx+0x30], xmm5 + movdqu xmmword ptr [rbx+0x50], xmm9 + movdqu xmmword ptr [rbx+0x70], xmm7 + movdqa xmm1, xmmword ptr [rsp+0x110] + movdqa xmm0, xmm1 + paddd xmm1, xmmword ptr [rsp+0x150] + movdqa xmmword ptr [rsp+0x110], xmm1 + pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip] + pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip] + pcmpgtd xmm0, xmm1 + movdqa xmm1, xmmword ptr [rsp+0x120] + psubd xmm1, xmm0 + movdqa xmmword ptr [rsp+0x120], xmm1 + add rbx, 128 + add rdi, 32 + sub rsi, 4 + cmp rsi, 4 + jnc 2b + test rsi, rsi + jnz 3f +4: + mov rsp, rbp + pop rbp + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + ret +.p2align 5 +3: + test esi, 0x2 + je 3f + movups xmm0, xmmword ptr [rcx] + movups xmm1, xmmword ptr [rcx+0x10] + movaps xmm8, xmm0 + movaps xmm9, xmm1 + movd xmm13, dword ptr [rsp+0x110] + movd xmm14, dword ptr [rsp+0x120] + punpckldq xmm13, xmm14 + movaps xmmword ptr [rsp], xmm13 + movd xmm14, dword ptr [rsp+0x114] + movd xmm13, dword ptr [rsp+0x124] + punpckldq xmm14, xmm13 + movaps xmmword ptr [rsp+0x10], xmm14 + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + movaps xmm10, xmm2 + movups xmm4, xmmword ptr [r8+rdx-0x40] + movups xmm5, xmmword ptr [r8+rdx-0x30] + movaps xmm3, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm3, xmm5, 221 + movaps xmm5, xmm3 + movups xmm6, xmmword ptr [r8+rdx-0x20] + movups xmm7, xmmword ptr [r8+rdx-0x10] + movaps xmm3, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm3, xmm7, 221 + pshufd xmm7, xmm3, 0x93 + movups xmm12, xmmword ptr [r9+rdx-0x40] + movups xmm13, xmmword ptr [r9+rdx-0x30] + movaps xmm11, xmm12 + shufps xmm12, xmm13, 136 + shufps xmm11, xmm13, 221 + movaps xmm13, xmm11 + movups xmm14, xmmword ptr [r9+rdx-0x20] + movups xmm15, xmmword ptr [r9+rdx-0x10] + movaps xmm11, xmm14 + shufps xmm14, xmm15, 136 + pshufd xmm14, xmm14, 0x93 + shufps xmm11, xmm15, 221 + pshufd xmm15, xmm11, 0x93 + shl rax, 0x20 + or rax, 0x40 + movq xmm3, rax + movdqa xmmword ptr [rsp+0x20], xmm3 + movaps xmm3, xmmword ptr [rsp] + movaps xmm11, xmmword ptr [rsp+0x10] + punpcklqdq xmm3, xmmword ptr [rsp+0x20] + punpcklqdq xmm11, xmmword ptr [rsp+0x20] + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm8, xmm12 + movaps xmmword ptr [rsp+0x20], xmm4 + movaps xmmword ptr [rsp+0x30], xmm12 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + pshuflw xmm11, xmm11, 0xB1 + pshufhw xmm11, xmm11, 0xB1 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 20 + psrld xmm4, 12 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 20 + psrld xmm4, 12 + por xmm9, xmm4 + paddd xmm0, xmm5 + paddd xmm8, xmm13 + movaps xmmword ptr [rsp+0x40], xmm5 + movaps xmmword ptr [rsp+0x50], xmm13 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + movdqa xmm13, xmm3 + psrld xmm3, 8 + pslld xmm13, 24 + pxor xmm3, xmm13 + movdqa xmm13, xmm11 + psrld xmm11, 8 + pslld xmm13, 24 + pxor xmm11, xmm13 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 25 + psrld xmm4, 7 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 25 + psrld xmm4, 7 + por xmm9, xmm4 + pshufd xmm0, xmm0, 0x93 + pshufd xmm8, xmm8, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm11, xmm11, 0x4E + pshufd xmm2, xmm2, 0x39 + pshufd xmm10, xmm10, 0x39 + paddd xmm0, xmm6 + paddd xmm8, xmm14 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + pshuflw xmm11, xmm11, 0xB1 + pshufhw xmm11, xmm11, 0xB1 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 20 + psrld xmm4, 12 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 20 + psrld xmm4, 12 + por xmm9, xmm4 + paddd xmm0, xmm7 + paddd xmm8, xmm15 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + movdqa xmm13, xmm3 + psrld xmm3, 8 + pslld xmm13, 24 + pxor xmm3, xmm13 + movdqa xmm13, xmm11 + psrld xmm11, 8 + pslld xmm13, 24 + pxor xmm11, xmm13 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 25 + psrld xmm4, 7 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 25 + psrld xmm4, 7 + por xmm9, xmm4 + pshufd xmm0, xmm0, 0x39 + pshufd xmm8, xmm8, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm11, xmm11, 0x4E + pshufd xmm2, xmm2, 0x93 + pshufd xmm10, xmm10, 0x93 + dec al + je 9f + movdqa xmm12, xmmword ptr [rsp+0x20] + movdqa xmm5, xmmword ptr [rsp+0x40] + pshufd xmm13, xmm12, 0x0F + shufps xmm12, xmm5, 214 + pshufd xmm4, xmm12, 0x39 + movdqa xmm12, xmm6 + shufps xmm12, xmm7, 250 + pand xmm13, xmmword ptr [PBLENDW_0x33_MASK+rip] + pand xmm12, xmmword ptr [PBLENDW_0xCC_MASK+rip] + por xmm13, xmm12 + movdqa xmmword ptr [rsp+0x20], xmm13 + movdqa xmm12, xmm7 + punpcklqdq xmm12, xmm5 + movdqa xmm13, xmm6 + pand xmm12, xmmword ptr [PBLENDW_0x3F_MASK+rip] + pand xmm13, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm12, xmm13 + pshufd xmm12, xmm12, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmmword ptr [rsp+0x40], xmm12 + movdqa xmm5, xmmword ptr [rsp+0x30] + movdqa xmm13, xmmword ptr [rsp+0x50] + pshufd xmm6, xmm5, 0x0F + shufps xmm5, xmm13, 214 + pshufd xmm12, xmm5, 0x39 + movdqa xmm5, xmm14 + shufps xmm5, xmm15, 250 + pand xmm6, xmmword ptr [PBLENDW_0x33_MASK+rip] + pand xmm5, xmmword ptr [PBLENDW_0xCC_MASK+rip] + por xmm6, xmm5 + movdqa xmm5, xmm15 + punpcklqdq xmm5, xmm13 + movdqa xmmword ptr [rsp+0x30], xmm2 + movdqa xmm2, xmm14 + pand xmm5, xmmword ptr [PBLENDW_0x3F_MASK+rip] + pand xmm2, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm5, xmm2 + movdqa xmm2, xmmword ptr [rsp+0x30] + pshufd xmm5, xmm5, 0x78 + punpckhdq xmm13, xmm15 + punpckldq xmm14, xmm13 + pshufd xmm15, xmm14, 0x1E + movdqa xmm13, xmm6 + movdqa xmm14, xmm5 + movdqa xmm5, xmmword ptr [rsp+0x20] + movdqa xmm6, xmmword ptr [rsp+0x40] + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + pxor xmm8, xmm10 + pxor xmm9, xmm11 + mov eax, r13d + cmp rdx, r15 + jne 2b + movups xmmword ptr [rbx], xmm0 + movups xmmword ptr [rbx+0x10], xmm1 + movups xmmword ptr [rbx+0x20], xmm8 + movups xmmword ptr [rbx+0x30], xmm9 + mov eax, dword ptr [rsp+0x130] + neg eax + mov r10d, dword ptr [rsp+0x110+8*rax] + mov r11d, dword ptr [rsp+0x120+8*rax] + mov dword ptr [rsp+0x110], r10d + mov dword ptr [rsp+0x120], r11d + add rdi, 16 + add rbx, 64 + sub rsi, 2 +3: + test esi, 0x1 + je 4b + movups xmm0, xmmword ptr [rcx] + movups xmm1, xmmword ptr [rcx+0x10] + movd xmm13, dword ptr [rsp+0x110] + movd xmm14, dword ptr [rsp+0x120] + punpckldq xmm13, xmm14 + mov r8, qword ptr [rdi] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + shl rax, 32 + or rax, 64 + movq xmm12, rax + movdqa xmm3, xmm13 + punpcklqdq xmm3, xmm12 + movups xmm4, xmmword ptr [r8+rdx-0x40] + movups xmm5, xmmword ptr [r8+rdx-0x30] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [r8+rdx-0x20] + movups xmm7, xmmword ptr [r8+rdx-0x10] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pand xmm9, xmmword ptr [PBLENDW_0x33_MASK+rip] + pand xmm8, xmmword ptr [PBLENDW_0xCC_MASK+rip] + por xmm9, xmm8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm10, xmm6 + pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip] + pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm8, xmm10 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + movups xmmword ptr [rbx], xmm0 + movups xmmword ptr [rbx+0x10], xmm1 + jmp 4b + +.p2align 6 +blake3_compress_in_place_sse2_asm: +_blake3_compress_in_place_sse2_asm: + _CET_ENDBR + movups xmm0, xmmword ptr [rdi] + movups xmm1, xmmword ptr [rdi+0x10] + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + shl r8, 32 + add rdx, r8 + movq xmm3, rcx + movq xmm4, rdx + punpcklqdq xmm3, xmm4 + movups xmm4, xmmword ptr [rsi] + movups xmm5, xmmword ptr [rsi+0x10] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [rsi+0x20] + movups xmm7, xmmword ptr [rsi+0x30] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pand xmm9, xmmword ptr [PBLENDW_0x33_MASK+rip] + pand xmm8, xmmword ptr [PBLENDW_0xCC_MASK+rip] + por xmm9, xmm8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm10, xmm6 + pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip] + pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm8, xmm10 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + movups xmmword ptr [rdi], xmm0 + movups xmmword ptr [rdi+0x10], xmm1 + ret + +.p2align 6 +blake3_compress_xof_sse2_asm: +_blake3_compress_xof_sse2_asm: + _CET_ENDBR + movups xmm0, xmmword ptr [rdi] + movups xmm1, xmmword ptr [rdi+0x10] + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + movzx eax, r8b + movzx edx, dl + shl rax, 32 + add rdx, rax + movq xmm3, rcx + movq xmm4, rdx + punpcklqdq xmm3, xmm4 + movups xmm4, xmmword ptr [rsi] + movups xmm5, xmmword ptr [rsi+0x10] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [rsi+0x20] + movups xmm7, xmmword ptr [rsi+0x30] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pand xmm9, xmmword ptr [PBLENDW_0x33_MASK+rip] + pand xmm8, xmmword ptr [PBLENDW_0xCC_MASK+rip] + por xmm9, xmm8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm10, xmm6 + pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip] + pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm8, xmm10 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + movdqu xmm4, xmmword ptr [rdi] + movdqu xmm5, xmmword ptr [rdi+0x10] + pxor xmm0, xmm2 + pxor xmm1, xmm3 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + movups xmmword ptr [r9], xmm0 + movups xmmword ptr [r9+0x10], xmm1 + movups xmmword ptr [r9+0x20], xmm2 + movups xmmword ptr [r9+0x30], xmm3 + ret + + +#ifdef __APPLE__ +.static_data +#else +.section .rodata +#endif +.p2align 6 +BLAKE3_IV: + .long 0x6A09E667, 0xBB67AE85 + .long 0x3C6EF372, 0xA54FF53A +ADD0: + .long 0, 1, 2, 3 +ADD1: + .long 4, 4, 4, 4 +BLAKE3_IV_0: + .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667 +BLAKE3_IV_1: + .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85 +BLAKE3_IV_2: + .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372 +BLAKE3_IV_3: + .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A +BLAKE3_BLOCK_LEN: + .long 64, 64, 64, 64 +CMP_MSB_MASK: + .long 0x80000000, 0x80000000, 0x80000000, 0x80000000 +PBLENDW_0x33_MASK: + .long 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000 +PBLENDW_0xCC_MASK: + .long 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF +PBLENDW_0x3F_MASK: + .long 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000 +PBLENDW_0xC0_MASK: + .long 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF diff --git a/src/blake3/blake3_sse41.c b/src/blake3/blake3_sse41.c new file mode 100644 index 0000000..adadd48 --- /dev/null +++ b/src/blake3/blake3_sse41.c @@ -0,0 +1,567 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" + +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +#define DEGREE 4 + +#define _mm_shuffle_ps2(a, b, c) \ + (_mm_castps_si128( \ + _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) + +INLINE __m128i loadu(const uint8_t src[16]) { + return _mm_loadu_si128((const __m128i *)src); +} + +INLINE void storeu(__m128i src, uint8_t dest[16]) { + _mm_storeu_si128((__m128i *)dest, src); +} + +INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } + +// Note that clang-format doesn't like the name "xor" for some reason. +INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } + +INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } + +INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); +} + +INLINE __m128i rot16(__m128i x) { + return _mm_shuffle_epi8( + x, _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); +} + +INLINE __m128i rot12(__m128i x) { + return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); +} + +INLINE __m128i rot8(__m128i x) { + return _mm_shuffle_epi8( + x, _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); +} + +INLINE __m128i rot7(__m128i x) { + return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); +} + +INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot16(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot12(*row1); +} + +INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot8(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot7(*row1); +} + +// Note the optimization here of leaving row1 as the unrotated row, rather than +// row0. All the message loads below are adjusted to compensate for this. See +// discussion at https://github.com/sneves/blake2-avx2/pull/4 +INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); +} + +INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); +} + +INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + rows[0] = loadu((uint8_t *)&cv[0]); + rows[1] = loadu((uint8_t *)&cv[4]); + rows[2] = set4(B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3); + rows[3] = set4(counter_low(counter), counter_high(counter), + (uint32_t)block_len, (uint32_t)flags); + + __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); + __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); + __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); + __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); + + __m128i t0, t1, t2, t3, tt; + + // Round 1. The first round permutes the message words from the original + // input order, into the groups that get mixed in parallel. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 + t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 + t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 2. This round and all following rounds apply a fixed permutation + // to the message words from the round before. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 3 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 4 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 5 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 6 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 7 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); +} + +void blake3_compress_in_place_sse41(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); + storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); +} + +void blake3_compress_xof_sse41(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), &out[0]); + storeu(xorv(rows[1], rows[3]), &out[16]); + storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); + storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); +} + +INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[15] = rot16(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot12(v[4]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[15] = rot8(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot7(v[4]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot16(v[15]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[4] = rot12(v[4]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot8(v[15]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + v[4] = rot7(v[4]); +} + +INLINE void transpose_vecs(__m128i vecs[DEGREE]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is + // 22/33. Note that this doesn't split the vector into two lanes, as the + // AVX2 counterparts do. + __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); + __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); + __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); + __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); + + // Interleave 64-bit lanes. + __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); + __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); + __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); + __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); + + vecs[0] = abcd_0; + vecs[1] = abcd_1; + vecs[2] = abcd_2; + vecs[3] = abcd_3; +} + +INLINE void transpose_msg_vecs(const uint8_t *const *inputs, + size_t block_offset, __m128i out[16]) { + out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); + out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); + out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); + out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); + out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); + out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); + out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); + out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); + out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); + out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); + out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); + out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); + out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); + out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); + out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); + out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); + for (size_t i = 0; i < 4; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs(&out[0]); + transpose_vecs(&out[4]); + transpose_vecs(&out[8]); + transpose_vecs(&out[12]); +} + +INLINE void load_counters(uint64_t counter, bool increment_counter, + __m128i *out_lo, __m128i *out_hi) { + const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); + const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); + const __m128i add1 = _mm_and_si128(mask, add0); + __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); + __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), + _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); + __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); + *out_lo = l; + *out_hi = h; +} + +static +void blake3_hash4_sse41(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m128i h_vecs[8] = { + set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), + set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), + }; + __m128i counter_low_vec, counter_high_vec; + load_counters(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); + __m128i block_flags_vec = set1(block_flags); + __m128i msg_vecs[16]; + transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m128i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1(B3_IV_0), set1(B3_IV_1), set1(B3_IV_2), set1(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn(v, msg_vecs, 0); + round_fn(v, msg_vecs, 1); + round_fn(v, msg_vecs, 2); + round_fn(v, msg_vecs, 3); + round_fn(v, msg_vecs, 4); + round_fn(v, msg_vecs, 5); + round_fn(v, msg_vecs, 6); + h_vecs[0] = xorv(v[0], v[8]); + h_vecs[1] = xorv(v[1], v[9]); + h_vecs[2] = xorv(v[2], v[10]); + h_vecs[3] = xorv(v[3], v[11]); + h_vecs[4] = xorv(v[4], v[12]); + h_vecs[5] = xorv(v[5], v[13]); + h_vecs[6] = xorv(v[6], v[14]); + h_vecs[7] = xorv(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs(&h_vecs[0]); + transpose_vecs(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); + storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); + storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); + storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); + storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); + storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); + storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); + storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); +} + +INLINE void hash_one_sse41(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_sse41(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= DEGREE) { + blake3_hash4_sse41(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += DEGREE; + } + inputs += DEGREE; + num_inputs -= DEGREE; + out = &out[DEGREE * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_sse41(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/src/blake3/blake3_sse41_x86-64_unix.S b/src/blake3/blake3_sse41_x86-64_unix.S new file mode 100644 index 0000000..d499a9a --- /dev/null +++ b/src/blake3/blake3_sse41_x86-64_unix.S @@ -0,0 +1,2028 @@ +#if defined(__ELF__) && defined(__linux__) +.section .note.GNU-stack,"",%progbits +#endif + +#if defined(__ELF__) && defined(__CET__) && defined(__has_include) +#if __has_include() +#include +#endif +#endif + +#if !defined(_CET_ENDBR) +#define _CET_ENDBR +#endif + +.intel_syntax noprefix +.global blake3_hash_many_sse41_asm +.global _blake3_hash_many_sse41_asm +.global blake3_compress_in_place_sse41_asm +.global _blake3_compress_in_place_sse41_asm +.global blake3_compress_xof_sse41_asm +.global _blake3_compress_xof_sse41_asm +#ifdef __APPLE__ +.text +#else +.section .text +#endif + .p2align 6 +_blake3_hash_many_sse41_asm: +blake3_hash_many_sse41_asm: + _CET_ENDBR + push r15 + push r14 + push r13 + push r12 + push rbx + push rbp + mov rbp, rsp + sub rsp, 360 + and rsp, 0xFFFFFFFFFFFFFFC0 + neg r9d + movd xmm0, r9d + pshufd xmm0, xmm0, 0x00 + movdqa xmmword ptr [rsp+0x130], xmm0 + movdqa xmm1, xmm0 + pand xmm1, xmmword ptr [ADD0+rip] + pand xmm0, xmmword ptr [ADD1+rip] + movdqa xmmword ptr [rsp+0x150], xmm0 + movd xmm0, r8d + pshufd xmm0, xmm0, 0x00 + paddd xmm0, xmm1 + movdqa xmmword ptr [rsp+0x110], xmm0 + pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip] + pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip] + pcmpgtd xmm1, xmm0 + shr r8, 32 + movd xmm2, r8d + pshufd xmm2, xmm2, 0x00 + psubd xmm2, xmm1 + movdqa xmmword ptr [rsp+0x120], xmm2 + mov rbx, qword ptr [rbp+0x50] + mov r15, rdx + shl r15, 6 + movzx r13d, byte ptr [rbp+0x38] + movzx r12d, byte ptr [rbp+0x48] + cmp rsi, 4 + jc 3f +2: + movdqu xmm3, xmmword ptr [rcx] + pshufd xmm0, xmm3, 0x00 + pshufd xmm1, xmm3, 0x55 + pshufd xmm2, xmm3, 0xAA + pshufd xmm3, xmm3, 0xFF + movdqu xmm7, xmmword ptr [rcx+0x10] + pshufd xmm4, xmm7, 0x00 + pshufd xmm5, xmm7, 0x55 + pshufd xmm6, xmm7, 0xAA + pshufd xmm7, xmm7, 0xFF + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +9: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movdqu xmm8, xmmword ptr [r8+rdx-0x40] + movdqu xmm9, xmmword ptr [r9+rdx-0x40] + movdqu xmm10, xmmword ptr [r10+rdx-0x40] + movdqu xmm11, xmmword ptr [r11+rdx-0x40] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp], xmm8 + movdqa xmmword ptr [rsp+0x10], xmm9 + movdqa xmmword ptr [rsp+0x20], xmm12 + movdqa xmmword ptr [rsp+0x30], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x30] + movdqu xmm9, xmmword ptr [r9+rdx-0x30] + movdqu xmm10, xmmword ptr [r10+rdx-0x30] + movdqu xmm11, xmmword ptr [r11+rdx-0x30] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0x40], xmm8 + movdqa xmmword ptr [rsp+0x50], xmm9 + movdqa xmmword ptr [rsp+0x60], xmm12 + movdqa xmmword ptr [rsp+0x70], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x20] + movdqu xmm9, xmmword ptr [r9+rdx-0x20] + movdqu xmm10, xmmword ptr [r10+rdx-0x20] + movdqu xmm11, xmmword ptr [r11+rdx-0x20] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0x80], xmm8 + movdqa xmmword ptr [rsp+0x90], xmm9 + movdqa xmmword ptr [rsp+0xA0], xmm12 + movdqa xmmword ptr [rsp+0xB0], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x10] + movdqu xmm9, xmmword ptr [r9+rdx-0x10] + movdqu xmm10, xmmword ptr [r10+rdx-0x10] + movdqu xmm11, xmmword ptr [r11+rdx-0x10] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0xC0], xmm8 + movdqa xmmword ptr [rsp+0xD0], xmm9 + movdqa xmmword ptr [rsp+0xE0], xmm12 + movdqa xmmword ptr [rsp+0xF0], xmm13 + movdqa xmm9, xmmword ptr [BLAKE3_IV_1+rip] + movdqa xmm10, xmmword ptr [BLAKE3_IV_2+rip] + movdqa xmm11, xmmword ptr [BLAKE3_IV_3+rip] + movdqa xmm12, xmmword ptr [rsp+0x110] + movdqa xmm13, xmmword ptr [rsp+0x120] + movdqa xmm14, xmmword ptr [BLAKE3_BLOCK_LEN+rip] + movd xmm15, eax + pshufd xmm15, xmm15, 0x00 + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + paddd xmm0, xmmword ptr [rsp] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x40] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [BLAKE3_IV_0+rip] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x10] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x50] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x80] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0xC0] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x90] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0xD0] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x20] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x70] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x60] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x10] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x90] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xB0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0xE0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x30] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0xD0] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x40] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x20] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x60] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0xB0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x50] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0xF0] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xA0] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0xE0] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x70] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0x30] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x40] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0x50] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x80] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xC0] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0xF0] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xD0] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0xA0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x70] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x20] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x10] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x90] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0x80] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xE0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0xC0] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xD0] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0x20] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x30] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0x60] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xB0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0x10] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xF0] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0x90] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xE0] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x30] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xA0] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x40] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + pxor xmm0, xmm8 + pxor xmm1, xmm9 + pxor xmm2, xmm10 + pxor xmm3, xmm11 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + pxor xmm4, xmm12 + pxor xmm5, xmm13 + pxor xmm6, xmm14 + pxor xmm7, xmm15 + mov eax, r13d + jne 9b + movdqa xmm9, xmm0 + punpckldq xmm0, xmm1 + punpckhdq xmm9, xmm1 + movdqa xmm11, xmm2 + punpckldq xmm2, xmm3 + punpckhdq xmm11, xmm3 + movdqa xmm1, xmm0 + punpcklqdq xmm0, xmm2 + punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm9 + punpcklqdq xmm9, xmm11 + punpckhqdq xmm3, xmm11 + movdqu xmmword ptr [rbx], xmm0 + movdqu xmmword ptr [rbx+0x20], xmm1 + movdqu xmmword ptr [rbx+0x40], xmm9 + movdqu xmmword ptr [rbx+0x60], xmm3 + movdqa xmm9, xmm4 + punpckldq xmm4, xmm5 + punpckhdq xmm9, xmm5 + movdqa xmm11, xmm6 + punpckldq xmm6, xmm7 + punpckhdq xmm11, xmm7 + movdqa xmm5, xmm4 + punpcklqdq xmm4, xmm6 + punpckhqdq xmm5, xmm6 + movdqa xmm7, xmm9 + punpcklqdq xmm9, xmm11 + punpckhqdq xmm7, xmm11 + movdqu xmmword ptr [rbx+0x10], xmm4 + movdqu xmmword ptr [rbx+0x30], xmm5 + movdqu xmmword ptr [rbx+0x50], xmm9 + movdqu xmmword ptr [rbx+0x70], xmm7 + movdqa xmm1, xmmword ptr [rsp+0x110] + movdqa xmm0, xmm1 + paddd xmm1, xmmword ptr [rsp+0x150] + movdqa xmmword ptr [rsp+0x110], xmm1 + pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip] + pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip] + pcmpgtd xmm0, xmm1 + movdqa xmm1, xmmword ptr [rsp+0x120] + psubd xmm1, xmm0 + movdqa xmmword ptr [rsp+0x120], xmm1 + add rbx, 128 + add rdi, 32 + sub rsi, 4 + cmp rsi, 4 + jnc 2b + test rsi, rsi + jnz 3f +4: + mov rsp, rbp + pop rbp + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + ret +.p2align 5 +3: + test esi, 0x2 + je 3f + movups xmm0, xmmword ptr [rcx] + movups xmm1, xmmword ptr [rcx+0x10] + movaps xmm8, xmm0 + movaps xmm9, xmm1 + movd xmm13, dword ptr [rsp+0x110] + pinsrd xmm13, dword ptr [rsp+0x120], 1 + pinsrd xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + movaps xmmword ptr [rsp], xmm13 + movd xmm14, dword ptr [rsp+0x114] + pinsrd xmm14, dword ptr [rsp+0x124], 1 + pinsrd xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + movaps xmmword ptr [rsp+0x10], xmm14 + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + movaps xmm10, xmm2 + movups xmm4, xmmword ptr [r8+rdx-0x40] + movups xmm5, xmmword ptr [r8+rdx-0x30] + movaps xmm3, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm3, xmm5, 221 + movaps xmm5, xmm3 + movups xmm6, xmmword ptr [r8+rdx-0x20] + movups xmm7, xmmword ptr [r8+rdx-0x10] + movaps xmm3, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm3, xmm7, 221 + pshufd xmm7, xmm3, 0x93 + movups xmm12, xmmword ptr [r9+rdx-0x40] + movups xmm13, xmmword ptr [r9+rdx-0x30] + movaps xmm11, xmm12 + shufps xmm12, xmm13, 136 + shufps xmm11, xmm13, 221 + movaps xmm13, xmm11 + movups xmm14, xmmword ptr [r9+rdx-0x20] + movups xmm15, xmmword ptr [r9+rdx-0x10] + movaps xmm11, xmm14 + shufps xmm14, xmm15, 136 + pshufd xmm14, xmm14, 0x93 + shufps xmm11, xmm15, 221 + pshufd xmm15, xmm11, 0x93 + movaps xmm3, xmmword ptr [rsp] + movaps xmm11, xmmword ptr [rsp+0x10] + pinsrd xmm3, eax, 3 + pinsrd xmm11, eax, 3 + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm8, xmm12 + movaps xmmword ptr [rsp+0x20], xmm4 + movaps xmmword ptr [rsp+0x30], xmm12 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + movaps xmm12, xmmword ptr [ROT16+rip] + pshufb xmm3, xmm12 + pshufb xmm11, xmm12 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 20 + psrld xmm4, 12 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 20 + psrld xmm4, 12 + por xmm9, xmm4 + paddd xmm0, xmm5 + paddd xmm8, xmm13 + movaps xmmword ptr [rsp+0x40], xmm5 + movaps xmmword ptr [rsp+0x50], xmm13 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + movaps xmm13, xmmword ptr [ROT8+rip] + pshufb xmm3, xmm13 + pshufb xmm11, xmm13 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 25 + psrld xmm4, 7 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 25 + psrld xmm4, 7 + por xmm9, xmm4 + pshufd xmm0, xmm0, 0x93 + pshufd xmm8, xmm8, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm11, xmm11, 0x4E + pshufd xmm2, xmm2, 0x39 + pshufd xmm10, xmm10, 0x39 + paddd xmm0, xmm6 + paddd xmm8, xmm14 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + pshufb xmm3, xmm12 + pshufb xmm11, xmm12 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 20 + psrld xmm4, 12 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 20 + psrld xmm4, 12 + por xmm9, xmm4 + paddd xmm0, xmm7 + paddd xmm8, xmm15 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + pshufb xmm3, xmm13 + pshufb xmm11, xmm13 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 25 + psrld xmm4, 7 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 25 + psrld xmm4, 7 + por xmm9, xmm4 + pshufd xmm0, xmm0, 0x39 + pshufd xmm8, xmm8, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm11, xmm11, 0x4E + pshufd xmm2, xmm2, 0x93 + pshufd xmm10, xmm10, 0x93 + dec al + je 9f + movdqa xmm12, xmmword ptr [rsp+0x20] + movdqa xmm5, xmmword ptr [rsp+0x40] + pshufd xmm13, xmm12, 0x0F + shufps xmm12, xmm5, 214 + pshufd xmm4, xmm12, 0x39 + movdqa xmm12, xmm6 + shufps xmm12, xmm7, 250 + pblendw xmm13, xmm12, 0xCC + movdqa xmm12, xmm7 + punpcklqdq xmm12, xmm5 + pblendw xmm12, xmm6, 0xC0 + pshufd xmm12, xmm12, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmmword ptr [rsp+0x20], xmm13 + movdqa xmmword ptr [rsp+0x40], xmm12 + movdqa xmm5, xmmword ptr [rsp+0x30] + movdqa xmm13, xmmword ptr [rsp+0x50] + pshufd xmm6, xmm5, 0x0F + shufps xmm5, xmm13, 214 + pshufd xmm12, xmm5, 0x39 + movdqa xmm5, xmm14 + shufps xmm5, xmm15, 250 + pblendw xmm6, xmm5, 0xCC + movdqa xmm5, xmm15 + punpcklqdq xmm5, xmm13 + pblendw xmm5, xmm14, 0xC0 + pshufd xmm5, xmm5, 0x78 + punpckhdq xmm13, xmm15 + punpckldq xmm14, xmm13 + pshufd xmm15, xmm14, 0x1E + movdqa xmm13, xmm6 + movdqa xmm14, xmm5 + movdqa xmm5, xmmword ptr [rsp+0x20] + movdqa xmm6, xmmword ptr [rsp+0x40] + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + pxor xmm8, xmm10 + pxor xmm9, xmm11 + mov eax, r13d + cmp rdx, r15 + jne 2b + movups xmmword ptr [rbx], xmm0 + movups xmmword ptr [rbx+0x10], xmm1 + movups xmmword ptr [rbx+0x20], xmm8 + movups xmmword ptr [rbx+0x30], xmm9 + movdqa xmm0, xmmword ptr [rsp+0x130] + movdqa xmm1, xmmword ptr [rsp+0x110] + movdqa xmm2, xmmword ptr [rsp+0x120] + movdqu xmm3, xmmword ptr [rsp+0x118] + movdqu xmm4, xmmword ptr [rsp+0x128] + blendvps xmm1, xmm3, xmm0 + blendvps xmm2, xmm4, xmm0 + movdqa xmmword ptr [rsp+0x110], xmm1 + movdqa xmmword ptr [rsp+0x120], xmm2 + add rdi, 16 + add rbx, 64 + sub rsi, 2 +3: + test esi, 0x1 + je 4b + movups xmm0, xmmword ptr [rcx] + movups xmm1, xmmword ptr [rcx+0x10] + movd xmm13, dword ptr [rsp+0x110] + pinsrd xmm13, dword ptr [rsp+0x120], 1 + pinsrd xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + movaps xmm14, xmmword ptr [ROT8+rip] + movaps xmm15, xmmword ptr [ROT16+rip] + mov r8, qword ptr [rdi] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + movaps xmm3, xmm13 + pinsrd xmm3, eax, 3 + movups xmm4, xmmword ptr [r8+rdx-0x40] + movups xmm5, xmmword ptr [r8+rdx-0x30] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [r8+rdx-0x20] + movups xmm7, xmmword ptr [r8+rdx-0x10] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pblendw xmm9, xmm8, 0xCC + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + pblendw xmm8, xmm6, 0xC0 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + movups xmmword ptr [rbx], xmm0 + movups xmmword ptr [rbx+0x10], xmm1 + jmp 4b + +.p2align 6 +blake3_compress_in_place_sse41_asm: +_blake3_compress_in_place_sse41_asm: + _CET_ENDBR + movups xmm0, xmmword ptr [rdi] + movups xmm1, xmmword ptr [rdi+0x10] + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + shl r8, 32 + add rdx, r8 + movq xmm3, rcx + movq xmm4, rdx + punpcklqdq xmm3, xmm4 + movups xmm4, xmmword ptr [rsi] + movups xmm5, xmmword ptr [rsi+0x10] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [rsi+0x20] + movups xmm7, xmmword ptr [rsi+0x30] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + movaps xmm14, xmmword ptr [ROT8+rip] + movaps xmm15, xmmword ptr [ROT16+rip] + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pblendw xmm9, xmm8, 0xCC + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + pblendw xmm8, xmm6, 0xC0 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + movups xmmword ptr [rdi], xmm0 + movups xmmword ptr [rdi+0x10], xmm1 + ret + +.p2align 6 +blake3_compress_xof_sse41_asm: +_blake3_compress_xof_sse41_asm: + _CET_ENDBR + movups xmm0, xmmword ptr [rdi] + movups xmm1, xmmword ptr [rdi+0x10] + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + movzx eax, r8b + movzx edx, dl + shl rax, 32 + add rdx, rax + movq xmm3, rcx + movq xmm4, rdx + punpcklqdq xmm3, xmm4 + movups xmm4, xmmword ptr [rsi] + movups xmm5, xmmword ptr [rsi+0x10] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [rsi+0x20] + movups xmm7, xmmword ptr [rsi+0x30] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + movaps xmm14, xmmword ptr [ROT8+rip] + movaps xmm15, xmmword ptr [ROT16+rip] + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pblendw xmm9, xmm8, 0xCC + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + pblendw xmm8, xmm6, 0xC0 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + movdqu xmm4, xmmword ptr [rdi] + movdqu xmm5, xmmword ptr [rdi+0x10] + pxor xmm0, xmm2 + pxor xmm1, xmm3 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + movups xmmword ptr [r9], xmm0 + movups xmmword ptr [r9+0x10], xmm1 + movups xmmword ptr [r9+0x20], xmm2 + movups xmmword ptr [r9+0x30], xmm3 + ret + + +#ifdef __APPLE__ +.static_data +#else +.section .rodata +#endif +.p2align 6 +BLAKE3_IV: + .long 0x6A09E667, 0xBB67AE85 + .long 0x3C6EF372, 0xA54FF53A +ROT16: + .byte 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 +ROT8: + .byte 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12 +ADD0: + .long 0, 1, 2, 3 +ADD1: + .long 4, 4, 4, 4 +BLAKE3_IV_0: + .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667 +BLAKE3_IV_1: + .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85 +BLAKE3_IV_2: + .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372 +BLAKE3_IV_3: + .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A +BLAKE3_BLOCK_LEN: + .long 64, 64, 64, 64 +CMP_MSB_MASK: + .long 0x80000000, 0x80000000, 0x80000000, 0x80000000 diff --git a/src/blake3/fy-blake3.c b/src/blake3/fy-blake3.c new file mode 100644 index 0000000..6931abc --- /dev/null +++ b/src/blake3/fy-blake3.c @@ -0,0 +1,171 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#include "blake3.h" +#include "blake3_impl.h" + +#include + +struct fy_blake3_hasher { + uint8_t output[FY_BLAKE3_OUT_LEN] FY_CACHELINE_ALIGN; + struct fy_blake3_hasher_cfg cfg; + struct blake3_host_state *hs; + struct blake3_hasher *hasher; +}; + +struct fy_blake3_hasher *fy_blake3_hasher_create(const struct fy_blake3_hasher_cfg *cfg) +{ + struct fy_blake3_hasher *fyh = NULL; + blake3_host_config hs_cfg; + + fyh = malloc(sizeof(*fyh)); + if (!fyh) + goto err_out; + + memset(fyh, 0, sizeof(*fyh)); + + /* copy config - if it exists */ + if (cfg) + fyh->cfg = *cfg; + + memset(&hs_cfg, 0, sizeof(hs_cfg)); + hs_cfg.debug = false; + hs_cfg.backend = cfg->backend; + if (!cfg->tp) { + if (cfg->num_threads >= 0) + hs_cfg.num_threads = (unsigned int)cfg->num_threads; + else + hs_cfg.no_mthread = true; + } else + hs_cfg.tp = cfg->tp; + hs_cfg.no_mmap = cfg->no_mmap; + hs_cfg.file_io_bufsz = cfg->file_buffer; + hs_cfg.mmap_min_chunk = cfg->mmap_min_chunk; + hs_cfg.mmap_max_chunk = cfg->mmap_max_chunk; + fyh->hs = blake3_host_state_create(&hs_cfg); + if (!fyh->hs) + goto err_out; + + fyh->hasher = blake3_hasher_create(fyh->hs, fyh->cfg.key, fyh->cfg.context, fyh->cfg.context ? fyh->cfg.context_len : 0); + if (!fyh->hasher) + goto err_out; + + return fyh; + +err_out: + fy_blake3_hasher_destroy(fyh); + return NULL; +} + +void fy_blake3_hasher_destroy(struct fy_blake3_hasher *fyh) +{ + if (!fyh) + return; + if (fyh->hasher) + blake3_hasher_destroy(fyh->hasher); + if (fyh->hs) + blake3_host_state_destroy(fyh->hs); + free(fyh); +} + +void fy_blake3_hasher_update(struct fy_blake3_hasher *fyh, const void *input, size_t input_len) +{ + if (!fyh) + return; + blake3_hasher_update(fyh->hasher, input, input_len); +} + +const uint8_t *fy_blake3_hasher_finalize(struct fy_blake3_hasher *fyh) +{ + if (!fyh) + return NULL; + blake3_hasher_finalize(fyh->hasher, fyh->output, FY_BLAKE3_OUT_LEN); + return fyh->output; +} + +void fy_blake3_hasher_reset(struct fy_blake3_hasher *fyh) +{ + if (!fyh) + return; + blake3_hasher_reset(fyh->hasher); +} + +const uint8_t *fy_blake3_hash_file(struct fy_blake3_hasher *fyh, const char *filename) +{ + int rc; + + if (!fyh || !filename) + return NULL; + + rc = blake3_hash_file(fyh->hasher, filename, fyh->output); + if (rc) + return NULL; + + return fyh->output; +} + +const uint8_t *fy_blake3_hash(struct fy_blake3_hasher *fyh, const void *mem, size_t size) +{ + if (!fyh || !mem) + return NULL; + + blake3_hash(fyh->hasher, mem, size, fyh->output); + + return fyh->output; +} + +const char *fy_blake3_backend_iterate(const char **prevp) +{ + const char *backend_names[64]; /* maximum 64 backends + 1 for NULL */ + const blake3_backend_info *bei; + uint64_t backends, avail; + unsigned int i, count; + + if (!prevp) + return NULL; + + avail = blake3_get_selectable_backends() & + blake3_get_detected_backends(); + + /* first pass, fill the backend name array */ + for (i = 0, backends = avail, count = 0; backends && i < B3BID_COUNT; i++) { + if (!(backends & ((uint64_t)1 << i))) + continue; + bei = blake3_get_backend_info(i); + if (!bei) + continue; + backends &= ~((uint64_t)1 << i); + backend_names[count++] = bei->name; + } + + /* start of the iterator */ + if (!*prevp) { + if (!count) + return NULL; + i = 0; + + } else { + + /* ok, find the current spot */ + for (i = 0; i < count; i++) { + if (!strcmp(*prevp, backend_names[i])) + break; + } + + /* increase for next, last or not found? */ + if (++i >= count) { + *prevp = NULL; + return NULL; + } + } + + return *prevp = backend_names[i]; +} diff --git a/src/internal/fy-b3sum.c b/src/internal/fy-b3sum.c new file mode 100644 index 0000000..a0e21fb --- /dev/null +++ b/src/internal/fy-b3sum.c @@ -0,0 +1,545 @@ +/* + * fy-b3sum.c - blake3 utility for testing within fyaml + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define OPT_NO_MMAP 128 +#define OPT_NO_MTHREAD 129 +#define OPT_NUM_THREADS 130 +#define OPT_BUFFER_SIZE 131 +#define OPT_MT_DEGREE 132 +#define OPT_LIST_BACKENDS 133 +#define OPT_ENABLE_CPUSIMD 134 +#define OPT_CPUSIMD_NUM_CPUS 135 +#define OPT_CPUSIMD_MULT_FACT 136 +#define OPT_DERIVE_KEY 137 +#define OPT_NO_NAMES 138 +#define OPT_RAW 139 +#define OPT_QUIET 140 +#define OPT_KEYED 141 + +static struct option lopts[] = { + {"check", no_argument, 0, 'c' }, + {"derive-key", required_argument, 0, OPT_DERIVE_KEY }, + {"no-names", no_argument, 0, OPT_NO_NAMES }, + {"raw", no_argument, 0, OPT_RAW }, + {"length", required_argument, 0, 'l' }, + {"quiet", no_argument, 0, OPT_QUIET }, + {"keyed", no_argument, 0, OPT_KEYED }, + + {"num-threads", required_argument, 0, OPT_NUM_THREADS }, + {"no-mmap", no_argument, 0, OPT_NO_MMAP }, + {"no-mthread", no_argument, 0, OPT_NO_MTHREAD }, + {"buffer-size", required_argument, 0, OPT_BUFFER_SIZE }, + {"mt-degree", required_argument, 0, OPT_MT_DEGREE }, + {"backend", required_argument, 0, 'b' }, + {"enable-cpusimd", no_argument, 0, OPT_ENABLE_CPUSIMD }, + {"cpusimd-num-cpus", required_argument, 0, OPT_CPUSIMD_NUM_CPUS }, + {"cpusimd-mult-fact", required_argument, 0, OPT_CPUSIMD_MULT_FACT }, + + {"list-backends", no_argument, 0, OPT_LIST_BACKENDS }, + {"help", no_argument, 0, 'h' }, + {"debug", no_argument, 0, 'd' }, + {0, 0, 0, 0 }, +}; + +static void display_usage(FILE *fp, const char *progname) +{ + const char *s; + + s = strrchr(progname, '/'); + if (s != NULL) + progname = s + 1; + + fprintf(fp, "Usage:\n\t%s [options] [args]\n", progname); + fprintf(fp, "\noptions:\n"); + fprintf(fp, "\t--derive-key : Key derivation mode, with the given context string\n"); + fprintf(fp, "\t--no-names : Omit filenames\n"); + fprintf(fp, "\t--raw : Output result in raw bytes (single input allowed)\n"); + fprintf(fp, "\t--length , -l : Output only this amount of bytes per output (max %u)\n", BLAKE3_OUT_LEN); + fprintf(fp, "\t--check, -c : Read files with BLAKE3 checksums and check files\n"); + fprintf(fp, "\t--quiet : Do not print OK for checked files that are correct\n"); + fprintf(fp, "\t--keyed : Keyed mode with secret key read from (32 raw bytes)\n"); + fprintf(fp, "\ntuning options:\n"); + fprintf(fp, "\t--num-threads : Number of threads to use (default: number of CPUs * 3 / 2)\n"); + fprintf(fp, "\t--no-mmap : Disable file mmap\n"); + fprintf(fp, "\t--no-mthread : Disable multithreading\n"); + fprintf(fp, "\t--buffer-size : Buffer size for file I/O\n"); + fprintf(fp, "\t--mt-degree : Set the multi-thread degree (default 128)\n"); + fprintf(fp, "\t--backend , -b : Backend selection\n"); + fprintf(fp, "\t--enable-cpusimd : Enable experimental CPUSIMD support\n"); + fprintf(fp, "\t--cpusimd-num-cpus : Number of CPUs assigned to CPUSIMD\n"); + fprintf(fp, "\t--cpusimd-mult-fact : Multiplication factor for CPUSIMD\n"); + fprintf(fp, "\ninformational options:\n"); + fprintf(fp, "\t--list-backends : List available backends\n"); + fprintf(fp, "\t--debug : Enable debug messages\n"); + fprintf(fp, "\t--help, -h : Display help message\n"); + fprintf(fp, "\n"); + fprintf(fp, "\nargs:\n"); + fprintf(fp, "\t... Files to hash or checkfiles to check.\n\tIf no file given (or file is '-' hash stdin\n"); +} + +static void list_backends(const char *name) +{ + const blake3_backend_info *bei; + uint64_t backends; + bool selected; + unsigned int i; + + backends = blake3_get_selectable_backends() & + blake3_get_detected_backends(); + for (i = 0; backends && i < B3BID_COUNT; i++) { + if (!(backends & ((uint64_t)1 << i))) + continue; + bei = blake3_get_backend_info(i); + if (!bei) + continue; + backends &= ~((uint64_t)1 << i); + + if (!name) + selected = backends == 0; + else + selected = !strcmp(name, bei->name); + + printf("%c %-12s\t%s\n", + selected ? '*' : ' ', bei->name, bei->description); + } +} + +static int do_hash_file(struct blake3_hasher *hasher, const char *filename, bool no_names, bool raw, unsigned int length) +{ + static const char *hexb = "0123456789abcdef"; + uint8_t output[BLAKE3_OUT_LEN]; + size_t filename_sz, line_sz, outsz; + ssize_t wrn; + uint8_t v; + void *outp; + char *line, *s; + unsigned int i; + int rc; + + filename_sz = strlen(filename); + + rc = blake3_hash_file(hasher, filename, output); + if (rc) { + fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno)); + return -1; + } + + if (!raw) { + /* output line (optimized) */ + line_sz = (length * 2); /* the hex output */ + if (!no_names) + line_sz += 2 + filename_sz; /* 2 spaces + filename */ + line_sz++; /* '\n' */ + line = alloca(line_sz + 1); + s = line; + for (i = 0; i < length; i++) { + v = output[i]; + *s++ = hexb[v >> 4]; + *s++ = hexb[v & 15]; + } + if (!no_names) { + *s++ = ' '; + *s++ = ' '; + memcpy(s, filename, filename_sz); + s += filename_sz; + } + *s++ = '\n'; + *s = '\0'; + outp = line; + outsz = (size_t)(s - line); + } else { + outp = output; + outsz = length; + } + + wrn = fwrite(outp, 1, outsz, stdout); + if ((size_t)wrn != outsz) { + fprintf(stderr, "Unable to write to stdout! error: %s\n", strerror(errno)); + return -1; + } + + return 0; +} + +static int do_check_file(struct blake3_hasher *hasher, const char *check_filename, bool quiet) +{ + char *hash, *filename; + FILE *fp = NULL; + char linebuf[8192]; /* maximum size for a line is 8K, should be enough (PATH_MAX is 4K at linux) */ + uint8_t read_hash[BLAKE3_OUT_LEN]; + uint8_t computed_hash[BLAKE3_OUT_LEN]; + uint8_t v; + char *s; + char c; + unsigned int i, j, length; + size_t linesz; + int line, rc, exit_code; + + if (check_filename && strcmp(check_filename, "-")) { + fp = fopen(check_filename, "ra"); + if (!fp) { + fprintf(stderr, "Failed to open check file: \"%s\", error: %s\n", check_filename, strerror(errno)); + goto err_out; + } + } else { + fp = stdin; + } + + /* default error code if all is fine */ + exit_code = 0; + + line = 0; + while (fgets(linebuf, sizeof(linebuf), fp)) { + /* '\0' terminate always */ + linebuf[(sizeof(linebuf)/sizeof(linebuf[0]))-1] = '\0'; + + linesz = strlen(linebuf); + while (linesz > 0 && linebuf[linesz-1] == '\n') + linesz--; + + if (!linesz) { + fprintf(stderr, "Empty line found at file \"%s\" line #%d\n", check_filename, line); + goto err_out; + } + linebuf[linesz] = '\0'; + + length = 0; + s = linebuf; + while (isxdigit(*s)) + s++; + + length = s - linebuf; + + if (length == 0 || length > (BLAKE3_OUT_LEN * 2) || (length % 1) || !isspace(*s)) { + fprintf(stderr, "Bad line found at file \"%s\" line #%d\n", check_filename, line); + fprintf(stderr, "%s\n", linebuf); + goto err_out; + } + + *s++ = '\0'; + + while (isspace(*s)) + s++; + + length >>= 1; /* to bytes */ + hash = linebuf; + filename = s; + + for (i = 0, s = hash; i < length; i++) { + v = 0; + for (j = 0; j < 2; j++) { + v <<= 4; + c = *s++; + if (c >= '0' && c <= '9') + v |= c - '0'; + else if (c >= 'a' && c <= 'f') + v |= c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + v |= c - 'A' + 10; + else + v = 0; + } + read_hash[i] = v; + } + + rc = blake3_hash_file(hasher, filename, computed_hash); + if (rc) { + fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno)); + goto err_out; + } + + /* constant time comparison */ + v = 0; + for (i = 0; i < length; i++) + v |= (read_hash[i] ^ computed_hash[i]); + + if (v) { + printf("%s: FAILED\n", filename); + exit_code = -1; + } else if (!quiet) + printf("%s: OK\n", filename); + } + +out: + if (fp && fp != stdin) + fclose(fp); + + return exit_code; + +err_out: + exit_code = -1; + goto out; +} + +int main(int argc, char *argv[]) +{ + blake3_host_config host_cfg; + struct blake3_host_state *host_state = NULL; + struct blake3_hasher *hasher = NULL; + const char *filename; + int i, opt, lidx, rc, num_inputs, num_ok; + int exitcode = EXIT_FAILURE, opti; + size_t buffer_size = 0; + bool no_names = false, raw = false, quiet = false, keyed = false; + bool no_mmap = false, no_mthread = false, debug = false, enable_cpusimd = false; + bool do_list_backends = false, check = false, derive_key = false; + unsigned int mt_degree = 0, num_threads = 0, cpusimd_num_cpus = 0, cpusimd_mult_fact = 0, length = BLAKE3_OUT_LEN; + const char *backend = NULL, *context = NULL; + uint8_t key[BLAKE3_OUT_LEN]; + ssize_t rdn; + + while ((opt = getopt_long_only(argc, argv, "cl:b:dh", lopts, &lidx)) != -1) { + switch (opt) { + + case OPT_DERIVE_KEY: + derive_key = true; + context = optarg; + break; + case OPT_NO_NAMES: + no_names = true; + break; + case OPT_RAW: + raw = true; + break; + case 'c': + check = true; + break; + case OPT_QUIET: + quiet = true; + break; + case OPT_KEYED: + keyed = true; + break; + + case 'l': + opti = atoi(optarg);; + if (opti <= 0 || opti > BLAKE3_OUT_LEN) { + fprintf(stderr, "Error: bad length=%d (must be > 0 and <= %u)\n\n", opti, BLAKE3_OUT_LEN); + goto err_out_usage; + } + length = (unsigned int)opti; + break; + + case OPT_NO_MMAP: + no_mmap = true; + break; + case OPT_NO_MTHREAD: + no_mthread = true; + break; + case OPT_NUM_THREADS: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad num_threads=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + num_threads = (unsigned int)opti; + break; + case OPT_BUFFER_SIZE: + opti = atoi(optarg); + if (opti <= 0) { + fprintf(stderr, "Error: bad buffer-size=%d (must be > 0)\n\n", opti); + goto err_out_usage; + } + buffer_size = (unsigned int)opti; + break; + case OPT_MT_DEGREE: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad mt_degree=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + mt_degree = (unsigned int)opti; + break; + + case OPT_LIST_BACKENDS: + do_list_backends = true; + break; + + case OPT_ENABLE_CPUSIMD: + enable_cpusimd = true; + break; + case OPT_CPUSIMD_NUM_CPUS: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad cpusimd_num_cpus=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + cpusimd_num_cpus = (unsigned int)opti; + break; + case OPT_CPUSIMD_MULT_FACT: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad cpusimd_mult_fact=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + cpusimd_mult_fact = (unsigned int)opti; + break; + + case 'b': + backend = optarg; + break; + + case 'd': + debug = true; + break; + case 'h' : + display_usage(stdout, argv[0]); + goto ok_out; + default: + goto err_out_usage; + } + } + + if (enable_cpusimd) { + rc = blake3_backend_cpusimd_setup(cpusimd_num_cpus, cpusimd_mult_fact); + if (rc) { + fprintf(stderr, "Unable to enable CPUSIMD\n"); + goto err_out; + } + } + + if (do_list_backends) { + list_backends(backend); + goto ok_out; + } + + if (quiet && !check) { + fprintf(stderr, "Error: --quiet may only be used together with --check\n\n"); + goto err_out_usage; + } + + if (keyed && derive_key) { + fprintf(stderr, "Error: --keyed and --derive-key may not be used together\n\n"); + goto err_out_usage; + } + + if (check && length != BLAKE3_OUT_LEN) { + fprintf(stderr, "Error: --check and --length may not be used together\n\n"); + goto err_out_usage; + } + + if (keyed) { + rdn = fread(key, 1, BLAKE3_KEY_LEN, stdin); + if (rdn != BLAKE3_KEY_LEN) { + if (rdn >= 0 && rdn < BLAKE3_KEY_LEN) + fprintf(stderr, "Error: could not read secret key from : short key\n\n"); + else + fprintf(stderr, "Error: could not read secret key from : error %s\n\n", strerror(errno)); + goto err_out_usage; + } + rc = fgetc(stdin); + if (rc != EOF) { + fprintf(stderr, "Error: garbage trailing secret key from \n\n"); + goto err_out_usage; + } + } + + memset(&host_cfg, 0, sizeof(host_cfg)); + host_cfg.debug = debug; + host_cfg.no_mthread = no_mthread; + host_cfg.no_mmap = no_mmap; + host_cfg.num_threads = num_threads; + host_cfg.backend = backend; + host_cfg.mt_degree = mt_degree; + host_cfg.file_io_bufsz = buffer_size; + + host_state = blake3_host_state_create(&host_cfg); + if (!host_state) { + fprintf(stderr, "unable to create blake3 host state\n"); + goto err_out; + } + + hasher = blake3_hasher_create(host_state, + keyed ? key : NULL, + derive_key ? context : NULL, 0); + if (!hasher) { + fprintf(stderr, "unable to create blake3 hasher\n"); + goto err_out; + } + + num_inputs = argc - optind; + if (num_inputs <= 0) + num_inputs = 1; /* stdin mode */ + + if (raw && num_inputs > 1) { + fprintf(stderr, "Error: Raw output mode is only supported with a single input\n\n"); + goto err_out_usage; + } + + if (!length) + length = BLAKE3_OUT_LEN; + + /* we will get in the loop even when no arguments (we'll do stdin instead) */ + num_ok = 0; + i = optind; + do { + /* if no arguments, use stdin */ + filename = i < argc ? argv[i] : "-"; + + /* we can't handle '-' in keyed mode */ + if (keyed && !strcmp(filename, "-")) { + fprintf(stderr, "Cannot use in keyed mode\n"); + goto err_out_usage; + } + + if (!check) + rc = do_hash_file(hasher, filename, no_names, raw, length); + else + rc = do_check_file(hasher, filename, quiet); + if (!rc) + num_ok++; + + } while (++i < argc); + + if (num_inputs != num_ok) + goto err_out; + +ok_out: + exitcode = EXIT_SUCCESS; + +out: + if (hasher) + blake3_hasher_destroy(hasher); + + if (host_state) + blake3_host_state_destroy(host_state); + + if (enable_cpusimd) + blake3_backend_cpusimd_cleanup(); + + return exitcode; + +err_out_usage: + display_usage(stderr, argv[0]); +err_out: + exitcode = EXIT_FAILURE; + goto out; +} diff --git a/src/internal/fy-thread.c b/src/internal/fy-thread.c new file mode 100644 index 0000000..7bbac06 --- /dev/null +++ b/src/internal/fy-thread.c @@ -0,0 +1,587 @@ +/* + * fy-thread.c - thread testing internal utility + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "fy-thread.h" + +static void test_worker_thread_fn(void *arg) +{ + _Atomic(int) *p = arg; + int v, exp_v; + + /* atomically increase the counter */ + v = atomic_load(p); + for (;;) { + exp_v = v; + if (atomic_compare_exchange_strong(p, &exp_v, v + 1)) + return; + v = exp_v; + } +} + +void test_worker_threads(unsigned int num_threads) +{ + struct fy_thread_pool_cfg tp_cfg; + struct fy_thread_pool *tp; + struct fy_thread **threads, *t; + struct fy_thread_work *works; + long scval; + unsigned int i, count, num_cpus; + int rc, test_count; + + (void)rc; + + if (num_threads == 0) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } else + num_cpus = num_threads; + + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = 0; + tp_cfg.num_threads = num_cpus; + tp_cfg.userdata = NULL; + + fprintf(stderr, "calling: fy_thread_pool_create()\n"); + + tp = fy_thread_pool_create(&tp_cfg); + assert(tp); + + count = tp->num_threads; + threads = alloca(count * sizeof(*threads)); + works = alloca(count * sizeof(*works)); + + test_count = 0; + + for (i = 0; i < count; i++) { + fprintf(stderr, "calling: fy_thread_reserve(#%u)\n", i); + threads[i] = fy_thread_reserve(tp); + assert(threads[i]); + t = threads[i]; + assert(t->id == i); + } + + for (i = 0; i < count; i++) { + t = threads[i]; + fprintf(stderr, "calling: fy_thread_submit_work(#%u)\n", i); + works[i].fn = test_worker_thread_fn; + works[i].arg = &test_count; + fy_thread_submit_work(t, &works[i]); + } + + for (i = 0; i < count; i++) { + t = threads[i]; + fprintf(stderr, "calling: fy_thread_wait_work(#%u)\n", i); + fy_thread_wait_work(t); + } + + fprintf(stderr, "%s: test_count=%d\n", __func__, test_count); + if (test_count != (int)num_cpus) { + fprintf(stderr, "error: test_count=%d expected %d\n", test_count, (int)num_cpus); + abort(); + } + + for (i = 0; i < count; i++) { + t = threads[i]; + fprintf(stderr, "calling: fy_thread_pool_unreserve(#%u)\n", i); + fy_thread_unreserve(t); + } + + fprintf(stderr, "calling: fy_thread_pool_destroy()\n"); + fy_thread_pool_destroy(tp); +} + +void test_thread_join(unsigned int num_threads) +{ + struct fy_thread_pool_cfg tp_cfg; + struct fy_thread_pool *tp; + void **args; + long scval; + unsigned int count, num_cpus; + int rc, test_count; + + (void)rc; + + if (num_threads == 0) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } else + num_cpus = num_threads; + + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = 0; + tp_cfg.num_threads = num_cpus; + tp_cfg.userdata = NULL; + + fprintf(stderr, "calling: fy_thread_pool_create()\n"); + tp = fy_thread_pool_create(&tp_cfg); + assert(tp); + + count = tp->num_threads; + args = alloca(count * sizeof(*args)); + + test_count = 0; + + fy_thread_arg_join(tp, test_worker_thread_fn, NULL, &test_count, count); + + fprintf(stderr, "%s: test_count=%d\n", __func__, test_count); + assert(test_count == (int)num_cpus); + + fprintf(stderr, "calling: fy_thread_pool_destroy()\n"); + fy_thread_pool_destroy(tp); +} + +struct thread_latency_state { + struct timespec reserve; + struct timespec reserve_done; + struct timespec submit; + struct timespec execute; + struct timespec wait; + struct timespec wait_done; + struct timespec unreserve; + struct timespec unreserve_done; +}; + +long long delta_ns(struct timespec before, struct timespec after) +{ + if ((before.tv_sec == 0 && before.tv_nsec == 0) || + (after.tv_sec == 0 && after.tv_nsec == 0)) + return -1; + return (long long)((int64_t)(after.tv_sec - before.tv_sec) * (int64_t)1000000000UL + (int64_t)(after.tv_nsec - before.tv_nsec)); +} + +static void test_latency_worker_thread_fn(void *arg) +{ + struct thread_latency_state *s = arg; + + clock_gettime(CLOCK_MONOTONIC, &s->execute); +} + +void test_thread_latency(unsigned int num_threads) +{ + struct fy_thread_pool_cfg tp_cfg; + struct fy_thread_pool *tp; + struct fy_thread **threads, *t; + struct fy_thread_work *works; + long scval; + unsigned int i, count, num_cpus; + int rc; + struct thread_latency_state *states, *s; + + (void)rc; + + if (num_threads == 0) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } else + num_cpus = num_threads; + + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = 0; + tp_cfg.num_threads = num_cpus; + tp_cfg.userdata = NULL; + + tp = fy_thread_pool_create(&tp_cfg); + assert(tp); + + + count = tp->num_threads; + threads = alloca(count * sizeof(*threads)); + works = alloca(count * sizeof(*works)); + states = alloca(count * sizeof(*states)); + + memset(states, 0, count * sizeof(*states)); + + for (i = 0; i < count; i++) { + s = &states[i]; + + clock_gettime(CLOCK_MONOTONIC, &s->reserve); + + threads[i] = fy_thread_reserve(tp); + assert(threads[i]); + t = threads[i]; + assert(t->id == i); + + clock_gettime(CLOCK_MONOTONIC, &s->reserve_done); + + } + + for (i = 0; i < count; i++) { + s = &states[i]; + clock_gettime(CLOCK_MONOTONIC, &s->submit); + + t = threads[i]; + works[i].fn = test_latency_worker_thread_fn; + works[i].arg = s; + fy_thread_submit_work(t, &works[i]); + } + + for (i = 0; i < count; i++) { + s = &states[i]; + clock_gettime(CLOCK_MONOTONIC, &s->wait); + + t = threads[i]; + fy_thread_wait_work(t); + + clock_gettime(CLOCK_MONOTONIC, &s->wait_done); + } + + for (i = 0; i < count; i++) { + + s = &states[i]; + clock_gettime(CLOCK_MONOTONIC, &s->unreserve); + + t = threads[i]; + fy_thread_unreserve(t); + + clock_gettime(CLOCK_MONOTONIC, &s->unreserve_done); + } + + fy_thread_pool_destroy(tp); + + fprintf(stderr, "latency results\n"); + for (i = 0; i < count; i++) { + s = &states[i]; + + fprintf(stderr, "#%2u: reserve:%10lld submit-execute:%10lld execute-waitdone:%10lld wait:%10lld unreserve:%10lld\n", i, + delta_ns(s->reserve, s->reserve_done), + delta_ns(s->submit, s->execute), + delta_ns(s->execute, s->wait_done), + delta_ns(s->wait, s->wait_done), + delta_ns(s->unreserve, s->unreserve_done)); + } +} + +// #define STEAL_LOOP_COUNT 100000000 +#define STEAL_LOOP_COUNT 10000 +static void test_worker_thread_steal_fn(void *arg) +{ + _Atomic(int) *p = arg; + int v, exp_v; + unsigned int i; + + /* atomically increase the counter STEAL_LOOP_COUNT times */ + for (i = 0; i < STEAL_LOOP_COUNT; i++) { + v = atomic_load(p); + for (;;) { + exp_v = v; + if (atomic_compare_exchange_strong(p, &exp_v, v + 1)) + break; + v = exp_v; + } + } +} + +void test_thread_join_steal(unsigned int num_threads) +{ + struct fy_thread_pool_cfg tp_cfg; + struct fy_thread_pool *tp; + void **args; + long scval; + unsigned int count, num_cpus; + int rc, test_count; + + (void)rc; + + if (num_threads == 0) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } else + num_cpus = num_threads; + + + tp_cfg.flags = FYTPCF_STEAL_MODE; + tp_cfg.num_threads = num_cpus; + tp_cfg.userdata = NULL; + + fprintf(stderr, "calling: fy_thread_pool_create()\n"); + tp = fy_thread_pool_create(&tp_cfg); + assert(tp); + + count = tp->num_threads * 4; + args = alloca(count * sizeof(*args)); + + test_count = 0; + + fy_thread_arg_join(tp, test_worker_thread_steal_fn, NULL, &test_count, count); + + fprintf(stderr, "%s: test_count=%d\n", __func__, test_count); + if (test_count != (int)num_cpus * STEAL_LOOP_COUNT * 4) { + fprintf(stderr, "error: test_count=%d expected %d\n", test_count, (int)num_cpus * STEAL_LOOP_COUNT); + abort(); + } + + fprintf(stderr, "calling: fy_thread_pool_destroy()\n"); + fy_thread_pool_destroy(tp); +} + +struct sum_args { + struct fy_thread_pool *tp; + const uint8_t *values_start; + unsigned int count_start; + const uint8_t *values; + unsigned int count; + uint64_t sum; +}; + +static uint64_t calc_sum(const uint8_t *values, unsigned int count) +{ + unsigned int i; + uint64_t sum = 0; + + for (i = 0; i < count; i++) + sum += values[i]; + + return sum; +} + +static void test_worker_thread_sum_fn(void *arg) +{ + struct sum_args *s = arg; + struct sum_args args[2]; + uint64_t sum; + unsigned int pos; + + pos = s->values - s->values_start; + assert(pos <= s->count_start); + assert(pos + s->count <= s->count_start); + + (void)pos; + + // if (s->count <= (1 << 20) / 8) { + if (s->count <= 4096) { + // fprintf(stderr, "S<%06x-%06x>\n", pos, pos + s->count - 1); + sum = calc_sum(s->values, s->count); + } else { + memset(args, 0, sizeof(args)); + args[0].tp = args[1].tp = s->tp; + args[0].values_start = args[1].values_start = s->values_start; + args[0].count_start = args[1].count_start = s->count_start; + args[0].sum = args[1].sum = 0; + args[0].values = s->values; + args[0].count = s->count / 2; + args[1].values = s->values + args[0].count; + args[1].count = s->count - args[0].count; + + // fprintf(stderr, "M<%06x-%06x,%06x-%06x>\n", + // pos, + // pos + args[0].count - 1, + // pos + args[0].count, + // pos + args[0].count + args[1].count - 1); + + fy_thread_arg_array_join(s->tp, test_worker_thread_sum_fn, NULL, &args, sizeof(args[0]), sizeof(args)/sizeof(args[0])); + sum = args[0].sum + args[1].sum; + } + s->sum = sum; +} + +void test_thread_join_sum(unsigned int num_threads, unsigned int count, bool steal_mode, unsigned int times) +{ + struct fy_thread_pool_cfg tp_cfg; + struct fy_thread_pool *tp; + struct timespec before, after; + unsigned int i, num_cpus; + long scval; + uint8_t *values; + int rc; + uint64_t sum_single, sum_multi; + struct sum_args args[2]; + long long table_multi[times]; + long long ns; + + (void)rc; + + fprintf(stderr, "**********************************************************************\n"); + fprintf(stderr, "%s: steal_mode=%s\n", __func__, steal_mode ? "true" : "false"); + + values = malloc(count * sizeof(*values)); + assert(values); + + clock_gettime(CLOCK_MONOTONIC, &before); + for (i = 0; i < count; i++) + values[i] = (uint8_t)rand(); + + clock_gettime(CLOCK_MONOTONIC, &after); + fprintf(stderr, "%s: seeding done in %lldus\n", __func__, delta_ns(before, after) / 1000); + + clock_gettime(CLOCK_MONOTONIC, &before); + sum_single = calc_sum(values, count); + clock_gettime(CLOCK_MONOTONIC, &after); + ns = delta_ns(before, after); + fprintf(stderr, "%s: calculated sum=%"PRIu64" (single threaded) done in %lldus\n", __func__, sum_single, ns / 1000); + + if (num_threads == 0) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } else + num_cpus = num_threads; + + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = steal_mode ? FYTPCF_STEAL_MODE : 0; + tp_cfg.num_threads = num_cpus; + tp_cfg.userdata = NULL; + + tp = fy_thread_pool_create(&tp_cfg); + assert(tp); + + fprintf(stderr, "%s: calculating (multi threaded) -", __func__); + for (i = 0; i < times; i++) { + clock_gettime(CLOCK_MONOTONIC, &before); + + memset(args, 0, sizeof(args)); + args[0].tp = args[1].tp = tp; + args[0].values_start = args[1].values_start = values; + args[0].count_start = args[1].count_start = count; + args[0].sum = args[1].sum = 0; + + args[0].values = values; + args[0].count = count / 2; + args[1].values = values + args[0].count; + args[1].count = count - args[0].count; + + // fprintf(stderr, "M<%06x-%06x,%06x-%06x>\n", + // 0, + // args[0].count - 1, + // args[0].count, + // args[0].count + args[1].count - 1); + + fy_thread_arg_array_join(tp, test_worker_thread_sum_fn, NULL, &args, sizeof(args[0]), sizeof(args)/sizeof(args[0])); + + sum_multi = args[0].sum + args[1].sum; + if (sum_multi != sum_single) { + fprintf(stderr, "\nFailed sum_multi %"PRIu64" should be %"PRIu64"\n", sum_multi, sum_single); + abort(); + } + + clock_gettime(CLOCK_MONOTONIC, &after); + ns = delta_ns(before, after); + fprintf(stderr, " %lldus", ns / 1000); + fflush(stderr); + table_multi[i] = ns; + } + ns = 0; + for (i = 0; i < times; i++) + ns += table_multi[i]; + ns /= times; + fprintf(stderr, " : average %lldus\n", ns / 1000); + + fy_thread_pool_destroy(tp); + + free(values); +} + +int thread_test(unsigned int num_threads) +{ +#if 0 + test_worker_threads(num_threads); + test_thread_join(num_threads); + test_thread_latency(num_threads); + test_thread_join_steal(num_threads); +#endif + test_thread_join_sum(num_threads, 1 << 20, false, 10); /* 1M of values */ + test_thread_join_sum(num_threads, 1 << 20, true, 10); /* 1M of values */ + + return 0; +} + +#define OPT_NUM_THREADS 128 + +static struct option lopts[] = { + {"num-threads", required_argument, 0, OPT_NUM_THREADS }, + {"help", no_argument, 0, 'h' }, + {0, 0, 0, 0 }, +}; + +static void display_usage(FILE *fp, const char *progname) +{ + const char *s; + + s = strrchr(progname, '/'); + if (s != NULL) + progname = s + 1; + + fprintf(fp, "Usage:\n\t%s [options]\n", progname); + fprintf(fp, "\noptions:\n"); + fprintf(fp, "\t--num-threads : Number of threads to use (default: number of CPUs * 3 / 2)\n"); + fprintf(fp, "\t--help, -h : Display help message\n"); + fprintf(fp, "\n"); +} + +int main(int argc, char *argv[]) +{ + int opt, lidx, rc; + unsigned int num_threads = 0; + int exitcode = EXIT_FAILURE, opti; + + while ((opt = getopt_long_only(argc, argv, "h", lopts, &lidx)) != -1) { + switch (opt) { + case OPT_NUM_THREADS: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad num_threads=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + num_threads = (unsigned int)opti; + break; + case 'h' : + display_usage(stdout, argv[0]); + goto ok_out; + default: + goto err_out_usage; + } + } + + rc = thread_test(num_threads); + if (rc) { + fprintf(stderr, "Error: thread_test() failed\n"); + goto err_out; + } + +ok_out: + exitcode = EXIT_SUCCESS; + +out: + return exitcode; + +err_out_usage: + display_usage(stderr, argv[0]); +err_out: + exitcode = EXIT_FAILURE; + goto out; +} diff --git a/src/internal/libfyaml-parser.c b/src/internal/libfyaml-parser.c index 0115fc3..d24f03f 100644 --- a/src/internal/libfyaml-parser.c +++ b/src/internal/libfyaml-parser.c @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include @@ -27,6 +29,7 @@ #include "fy-parse.h" #include "fy-walk.h" +#include "fy-blob.h" #include "fy-valgrind.h" @@ -94,7 +97,7 @@ static struct option lopts[] = { #define LIBYAML_MODES "" #endif -#define MODES "parse|scan|copy|testsuite|dump|dump2|build|walk|reader|compose|iterate|comment|pathspec" LIBYAML_MODES +#define MODES "parse|scan|copy|testsuite|dump|dump2|build|walk|reader|compose|iterate|comment|pathspec|shell-split|parse-timing" LIBYAML_MODES static void display_usage(FILE *fp, char *progname) { @@ -3680,7 +3683,7 @@ int do_walk(struct fy_parser *fyp, const char *walkpath, const char *walkstart, struct fy_path_exec_cfg xcfg_local, *xcfg = &xcfg_local; char *path; unsigned int flags; - int rc, count; + int rc; flags = 0; if (sort) @@ -3716,7 +3719,6 @@ int do_walk(struct fy_parser *fyp, const char *walkpath, const char *walkstart, fypx = fy_path_exec_create(xcfg); assert(fypx); - count = 0; while ((fyd = fy_parse_load_document(fyp)) != NULL) { if (resolve) { @@ -3808,8 +3810,6 @@ next: fy_walk_result_free_rl(NULL, result); fy_parse_document_destroy(fyp, fyd); - - count++; } fy_path_exec_unref(fypx); @@ -3958,6 +3958,157 @@ int do_bad_utf8(const struct fy_parse_cfg *cfg, int argc, char *argv[]) return 0; } +int do_shell_split(int in_argc, char *in_argv[]) +{ + char buf[256], line[256 + 1]; + char *s, *e; + const char * const *argv; + int i, argc; + void *mem; + + printf("shell split; Ctrl-D to exit\n"); + buf[sizeof(buf)-1] = '\0'; + while (fgets(buf, sizeof(buf) - 1, stdin)) { + buf[sizeof(buf)-1] = '\0'; + strcpy(line, buf); + s = line; + e = s + strlen(line); + while (e > s && e[-1] == '\n') + *--e = '\0'; + + printf("input: '%s'\n", line); + + mem = fy_utf8_split_posix(line, &argc, &argv); + if (!mem) { + fprintf(stderr, "Bad input '%s'\n", line); + } else { + for (i = 0; i < argc; i++) { + fprintf(stderr, "%d: %s\n", i, argv[i]); + } + assert(argv[argc] == NULL); + free(mem); + } + } + return 0; +} + +int do_parse_timing(int argc, char *argv[], bool disable_mmap) +{ + void *blob; + size_t blob_size; + int i, c, w; + struct timespec before, after; + int64_t ns; + const uint8_t *s, *e, *ss; + size_t count; + struct fy_utf8_result res; + +#undef BEFORE +#define BEFORE() \ + do { \ + clock_gettime(CLOCK_MONOTONIC, &before); \ + } while(0) + +#undef AFTER +#define AFTER() \ + ({ \ + clock_gettime(CLOCK_MONOTONIC, &after); \ + (int64_t)(after.tv_sec - before.tv_sec) * (int64_t)1000000000UL + (int64_t)(after.tv_nsec - before.tv_nsec); \ + }) + + for (i = optind; i < argc; i++) { + + printf("file=%s\n", argv[i]); + + BEFORE(); + blob = fy_blob_read(argv[i], &blob_size); + assert(blob); + ns = AFTER(); + + printf("read %zu bytes in %"PRId64"ns\n", blob_size, ns); + + BEFORE(); + s = blob; + e = s + blob_size; + count = 0; + while (s < e) { + if (*s++ == 'e') + count++; + } + ns = AFTER(); + printf("%zu 'e' chars method 1 in %"PRId64"ns\n", count, ns); + + BEFORE(); + s = blob; + e = s + blob_size; + count = 0; + while ((ss = memchr(s, 'e', e - s)) != NULL) { + count++; + s = ss + 1; + } + ns = AFTER(); + printf("%zu 'e' chars method 2 in %"PRId64"ns\n", count, ns); + + BEFORE(); + s = blob; + e = s + blob_size; + count = 0; + while ((c = fy_utf8_get(s, e - s, &w)) >= 0) { + if (c == 'e') + count++; + s += w; + } + ns = AFTER(); + printf("%zu 'e' utf8 characters using method 1 in %"PRId64"ns\n", count, ns); + + BEFORE(); + s = blob; + e = s + blob_size; + count = 0; + while ((c = fy_utf8_get_s(s, e, &w)) >= 0) { + if (c == 'e') + count++; + s += w; + } + ns = AFTER(); + printf("%zu 'e' utf8 characters using method 2 in %"PRId64"ns\n", count, ns); + + BEFORE(); + s = blob; + e = s + blob_size - FY_UTF8_MAX_WIDTH; + count = 0; + while (s < e && (c = fy_utf8_get_s_nocheck(s, &w)) >= 0) { + if (c == 'e') + count++; + s += w; + } + while ((c = fy_utf8_get_s(s, e, &w)) >= 0) { + if (c == 'e') + count++; + s += w; + } + ns = AFTER(); + printf("%zu 'e' utf8 characters using method 3 in %"PRId64"ns\n", count, ns); + + BEFORE(); + s = blob; + e = s + blob_size; + count = 0; + while ((res = fy_utf8_get_s_res(s, e)).c >= 0) { + if (res.c == 'e') + count++; + s += res.w; + } + ns = AFTER(); + printf("%zu 'e' utf8 characters using method 4 in %"PRId64"ns\n", count, ns); + + free(blob); + + } + + return 0; +} + int apply_flags_option(const char *arg, unsigned int *flagsp, int (*modify_flags)(const char *what, unsigned int *flagsp)) { @@ -4151,7 +4302,9 @@ int main(int argc, char *argv[]) strcmp(mode, "pathspec") && strcmp(mode, "bypath") && strcmp(mode, "crash") && - strcmp(mode, "badutf8") + strcmp(mode, "badutf8") && + strcmp(mode, "shell-split") && + strcmp(mode, "parse-timing") #if defined(HAVE_LIBYAML) && HAVE_LIBYAML && strcmp(mode, "libyaml-scan") && strcmp(mode, "libyaml-parse") @@ -4401,6 +4554,18 @@ int main(int argc, char *argv[]) /* fprintf(stderr, "do_bypath() error %d\n", rc); */ goto cleanup; } + } else if (!strcmp(mode, "shell-split")) { + rc = do_shell_split(argc, argv); + if (rc < 0) { + /* fprintf(stderr, "do_shell_split() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "parse-timing")) { + rc = do_parse_timing(argc, argv, !!(cfg.flags & FYPCF_DISABLE_MMAP_OPT)); + if (rc < 0) { + /* fprintf(stderr, "do_parse_timing() error %d\n", rc); */ + goto cleanup; + } } #if defined(HAVE_LIBYAML) && HAVE_LIBYAML if (!strcmp(mode, "libyaml-diff")) { diff --git a/src/lib/fy-atom.c b/src/lib/fy-atom.c index 4086d92..e0b08cf 100644 --- a/src/lib/fy-atom.c +++ b/src/lib/fy-atom.c @@ -1345,10 +1345,13 @@ int fy_atom_iter_getc(struct fy_atom_iter *iter) return -1; /* first try the pushed ungetc */ - if (iter->unget_c != -1) { + if (iter->unget_c >= 0) { c = iter->unget_c; + /* unmatched getc/ungetc */ + if (fy_utf8_width(c) != 1) + return -1; iter->unget_c = -1; - return c & 0xff; + return c; } /* read first octet */ @@ -1361,17 +1364,17 @@ int fy_atom_iter_getc(struct fy_atom_iter *iter) int fy_atom_iter_ungetc(struct fy_atom_iter *iter, int c) { - if (!iter) + if (!iter || c >= 0x80) return -1; - if (iter->unget_c != -1) + if (iter->unget_c >= 0) return -1; - if (c == -1) { + if (c < 0) { iter->unget_c = -1; return 0; } - iter->unget_c = c & 0xff; - return c & 0xff; + iter->unget_c = c; + return c; } int fy_atom_iter_peekc(struct fy_atom_iter *iter) @@ -1395,10 +1398,10 @@ int fy_atom_iter_utf8_get(struct fy_atom_iter *iter) return -1; /* first try the pushed ungetc */ - if (iter->unget_c != -1) { + if (iter->unget_c >= 0) { c = iter->unget_c; iter->unget_c = -1; - return c & 0xff; + return c; } /* read first octet */ @@ -1431,11 +1434,11 @@ int fy_atom_iter_utf8_quoted_get(struct fy_atom_iter *iter, size_t *lenp, uint8_ return -1; /* first try the pushed ungetc */ - if (iter->unget_c != -1) { + if (iter->unget_c >= 0) { c = iter->unget_c; iter->unget_c = -1; *lenp = 0; - return c & 0xff; + return c; } /* read first octet */ @@ -1472,12 +1475,17 @@ int fy_atom_iter_utf8_quoted_get(struct fy_atom_iter *iter, size_t *lenp, uint8_ int fy_atom_iter_utf8_unget(struct fy_atom_iter *iter, int c) { - if (iter->unget_c != -1) + if (!iter) return -1; - if (c == -1) { + + if (iter->unget_c >= 0) + return -1; + if (c < 0) { iter->unget_c = -1; return 0; } + if (!fy_utf8_is_valid(c)) + return -1; iter->unget_c = c; return c; } @@ -1567,21 +1575,21 @@ bool fy_atom_is_number(struct fy_atom *atom) fy_atom_iter_start(atom, &iter); /* skip minus sign if it's there */ - c = fy_atom_iter_peekc(&iter); + c = fy_atom_iter_utf8_peek(&iter); if (c == '-') { - (void)fy_atom_iter_getc(&iter); + (void)fy_atom_iter_utf8_get(&iter); len++; } /* skip digits */ first_zero = false; dec = 0; - while ((c = fy_atom_iter_peekc(&iter)) >= 0 && isdigit(c)) { + while ((c = fy_atom_iter_utf8_peek(&iter)) >= 0 && fy_is_digit(c)) { if (dec == 0 && c == '0') first_zero = true; else if (dec == 1 && first_zero) goto err_out; /* 0[0-9] is bad */ - (void)fy_atom_iter_getc(&iter); + (void)fy_atom_iter_utf8_get(&iter); dec++; len++; } @@ -1592,14 +1600,14 @@ bool fy_atom_is_number(struct fy_atom *atom) fract = 0; /* dot? */ - c = fy_atom_iter_peekc(&iter); + c = fy_atom_iter_utf8_peek(&iter); if (c == '.') { - (void)fy_atom_iter_getc(&iter); + (void)fy_atom_iter_utf8_get(&iter); len++; /* skip decimal part */ - while ((c = fy_atom_iter_peekc(&iter)) >= 0 && isdigit(c)) { - (void)fy_atom_iter_getc(&iter); + while ((c = fy_atom_iter_utf8_peek(&iter)) >= 0 && fy_is_digit(c)) { + (void)fy_atom_iter_utf8_get(&iter); len++; fract++; } @@ -1611,21 +1619,21 @@ bool fy_atom_is_number(struct fy_atom *atom) enot = 0; /* scientific notation */ - c = fy_atom_iter_peekc(&iter); + c = fy_atom_iter_utf8_peek(&iter); if (c == 'e' || c == 'E') { - (void)fy_atom_iter_getc(&iter); + (void)fy_atom_iter_utf8_get(&iter); len++; /* skip sign if it's there */ - c = fy_atom_iter_peekc(&iter); + c = fy_atom_iter_utf8_peek(&iter); if (c == '+' || c == '-') { - (void)fy_atom_iter_getc(&iter); + (void)fy_atom_iter_utf8_get(&iter); len++; } /* skip exponent part */ - while ((c = fy_atom_iter_peekc(&iter)) >= 0 && isdigit(c)) { - (void)fy_atom_iter_getc(&iter); + while ((c = fy_atom_iter_utf8_peek(&iter)) >= 0 && fy_is_digit(c)) { + (void)fy_atom_iter_utf8_get(&iter); len++; enot++; } @@ -1634,7 +1642,7 @@ bool fy_atom_is_number(struct fy_atom *atom) goto err_out; } - c = fy_atom_iter_peekc(&iter); + c = fy_atom_iter_utf8_peek(&iter); fy_atom_iter_finish(&iter); @@ -1734,7 +1742,7 @@ fy_atom_raw_line_iter_next(struct fy_atom_raw_line_iter *iter) return NULL; while (s > iter->is) { - c = fy_utf8_get_right(iter->is, (int)(s - iter->is), &w); + c = fy_utf8_get_right(iter->is, (size_t)(s - iter->is), &w); if (c <= 0 || fy_is_lb_m(c, iter->atom->lb_mode)) break; s -= w; @@ -1748,7 +1756,7 @@ fy_atom_raw_line_iter_next(struct fy_atom_raw_line_iter *iter) /* track until the start of the content */ while (s < iter->as) { - c = fy_utf8_get(s, (int)(iter->ae - s), &w); + c = fy_utf8_get(s, (size_t)(iter->ae - s), &w); /* we should never hit that */ if (c <= 0) return NULL; @@ -1775,7 +1783,7 @@ fy_atom_raw_line_iter_next(struct fy_atom_raw_line_iter *iter) /* track until the end of the content (or lb) */ while (s < iter->ae) { - c = fy_utf8_get(s, (int)(iter->ae - s), &w); + c = fy_utf8_get(s, (size_t)(iter->ae - s), &w); /* we should never hit that */ if (c <= 0) return NULL; @@ -1803,7 +1811,7 @@ fy_atom_raw_line_iter_next(struct fy_atom_raw_line_iter *iter) /* if the stop was due to end of the atom */ if (s >= iter->ae) { while (s < iter->ie) { - c = fy_utf8_get(s, (int)(iter->ie - s), &w); + c = fy_utf8_get(s, (size_t)(iter->ie - s), &w); /* just end of input */ if (c <= 0) break; diff --git a/src/lib/fy-atom.h b/src/lib/fy-atom.h index ef874df..10ff7b7 100644 --- a/src/lib/fy-atom.h +++ b/src/lib/fy-atom.h @@ -85,6 +85,7 @@ struct fy_atom { bool valid_anchor : 1; /* atom is a valid anchor */ bool json_mode : 1; /* atom was read in json mode */ bool ends_with_eof : 1; /* atom ends at EOF of input */ + bool is_merge_key: 1; /* atom is just << */ }; }; }; @@ -116,6 +117,11 @@ static inline enum fy_lb_mode fy_atom_lb_mode(struct fy_atom *handle) return handle->lb_mode; } +static inline bool fy_atom_is_merge_key(struct fy_atom *handle) +{ + return handle && handle->is_merge_key; +} + static inline enum fy_flow_ws_mode fy_atom_flow_ws_mode(struct fy_atom *handle) { if (!handle) diff --git a/src/lib/fy-composer.c b/src/lib/fy-composer.c index 3e19017..9bdfbfb 100644 --- a/src/lib/fy-composer.c +++ b/src/lib/fy-composer.c @@ -313,6 +313,37 @@ err_out: return FYCR_ERROR; } +static void +fy_composer_halt(struct fy_composer *fyc, struct fy_path *fypp, enum fy_composer_return rc) +{ + const struct fy_composer_ops *ops; + struct fy_eventp ev_none; + struct fy_path_component *fypc; + + /* for normal stop, we don't teardown */ + if (rc == FYCR_OK_STOP) + return; + + ops = fyc->cfg.ops; + assert(ops); + + memset(&ev_none, 0, sizeof(ev_none)); + ev_none.e.type = FYET_NONE; + + /* pump FYET_NONEs to clean the stack */ + + while ((fypc = fy_path_component_list_tail(&fypp->components)) != NULL) { + + ops->process_event(fyc, fypp, &ev_none.e); + + fy_path_component_list_del(&fypp->components, fypc); + fy_path_component_free(fypc); + } + + /* and the final one to clean the root */ + ops->process_event(fyc, fypp, &ev_none.e); +} + enum fy_composer_return fy_composer_process_event(struct fy_composer *fyc, struct fy_event *fye) { @@ -331,6 +362,9 @@ fy_composer_process_event(struct fy_composer *fyc, struct fy_event *fye) rc = fy_composer_process_event_private(fyc, fye, fypp); + if (rc == FYCR_ERROR || rc == FYCR_OK_STOP) + fy_composer_halt(fyc, fypp, rc); + return rc; } diff --git a/src/lib/fy-ctype.c b/src/lib/fy-ctype.c deleted file mode 100644 index 560fec6..0000000 --- a/src/lib/fy-ctype.c +++ /dev/null @@ -1,66 +0,0 @@ -/* - * fy-ctype.c - ctype utilities - * - * Copyright (c) 2019 Pantelis Antoniou - * - * SPDX-License-Identifier: MIT - * - */ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include - -#include "fy-ctype.h" - -const char *fy_uri_esc(const char *s, size_t len, uint8_t *code, int *code_len) -{ - const char *e = s + len; - int j, k, width; - uint8_t octet; - char c; - - width = 0; - k = 0; - do { - /* check for enough space for %XX */ - if ((e - s) < 3) - return NULL; - - /* if more than one run, expect '%' */ - if (s[0] != '%') - return NULL; - - octet = 0; - for (j = 0; j < 2; j++) { - c = s[1 + j]; - octet <<= 4; - if (c >= '0' && c <= '9') - octet |= c - '0'; - else if (c >= 'a' && c <= 'f') - octet |= 10 + c - 'a'; - else - octet |= 10 + c - 'A'; - } - if (!width) { - width = fy_utf8_width_by_first_octet(octet); - if (!width) - return NULL; - k = 0; - } - if (k >= *code_len) - return NULL; - - code[k++] = octet; - - /* skip over the 3 character escape */ - s += 3; - - } while (--width > 0); - - *code_len = k; - - return s; -} - diff --git a/src/lib/fy-ctype.h b/src/lib/fy-ctype.h deleted file mode 100644 index 5e7ff0b..0000000 --- a/src/lib/fy-ctype.h +++ /dev/null @@ -1,343 +0,0 @@ -/* - * fy-ctype.h - ctype like macros header - * - * Copyright (c) 2019 Pantelis Antoniou - * - * SPDX-License-Identifier: MIT - */ -#ifndef FY_CTYPE_H -#define FY_CTYPE_H - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include -#include - -#include - -#include "fy-utf8.h" - -enum fy_lb_mode { - fylb_cr_nl, /* only \r, \n (json + >= yaml1.2 */ - fylb_cr_nl_N_L_P, /* NEL/LS/PS (yaml1.1) */ -}; - -enum fy_flow_ws_mode { - fyfws_space_tab, /* space + TAB (yaml) */ - fyfws_space, /* only space (json) */ -}; - -static inline bool fy_is_first_alpha(int c) -{ - return (c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - c == '_'; -} - -static inline bool fy_is_alpha(int c) -{ - return fy_is_first_alpha(c) || c == '-'; -} - -static inline bool fy_is_num(int c) -{ - return c >= '0' && c <= '9'; -} - -static inline bool fy_is_first_alnum(int c) -{ - return fy_is_first_alpha(c); -} - -static inline bool fy_is_alnum(int c) -{ - return fy_is_alpha(c) || fy_is_num(c); -} - -static inline bool fy_is_space(int c) -{ - return c == ' '; -} - -static inline bool fy_is_tab(int c) -{ - return c == '\t'; -} - -static inline bool fy_is_ws(int c) -{ - return fy_is_space(c) || fy_is_tab(c); -} - -static inline bool fy_is_hex(int c) -{ - return (c >= '0' && c <= '9') || - (c >= 'a' && c <= 'f') || - (c >= 'A' && c <= 'F'); -} - -static inline bool fy_is_uri(int c) -{ - return fy_is_alnum(c) || fy_utf8_strchr(";/?:@&=+$,.!~*\'()[]%", c); -} - -static inline bool fy_is_lb_r_n(int c) -{ - return c == '\r' || c == '\n'; -} - -static inline bool fy_is_lb_NEL(int c) -{ - return c == 0x85; -} - -static inline bool fy_is_lb_LS_PS(int c) -{ - return c == 0x2028 || c == 0x2029; -} - -static inline bool fy_is_unicode_lb(int c) -{ - /* note that YAML1.1 supports NEL #x85, LS #x2028 and PS #x2029 as linebreaks */ - /* YAML1.2 and higher does not */ - return fy_is_lb_NEL(c) || fy_is_lb_LS_PS(c); -} - -static inline bool fy_is_any_lb(int c) -{ - return fy_is_lb_r_n(c) || fy_is_unicode_lb(c); -} - -static inline bool fy_is_z(int c) -{ - return c <= 0; -} - -static inline bool fy_is_blank(int c) -{ - return c == ' ' || c == '\t'; -} - -#define FY_UTF8_BOM 0xfeff - -static inline bool fy_is_print(int c) -{ - return c == '\n' || c == '\r' || - (c >= 0x0020 && c <= 0x007e) || - (c >= 0x00a0 && c <= 0xd7ff) || - (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM); -} - -static inline bool fy_is_printq(int c) -{ - return c != '\t' && c != 0xa0 && !fy_is_any_lb(c) && fy_is_print(c); -} - -static inline bool fy_is_nb_char(int c) -{ - return (c >= 0x0020 && c <= 0x007e) || - (c >= 0x00a0 && c <= 0xd7ff) || - (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM); -} - -static inline bool fy_is_ns_char(int c) -{ - return fy_is_nb_char(c) && !fy_is_ws(c); -} - -static inline bool fy_is_start_indicator(int c) -{ - return !!fy_utf8_strchr(",[]{}#&*!|>'\"%%@`", c); -} - -static inline bool fy_is_indicator_before_space(int c) -{ - return !!fy_utf8_strchr("-:?`", c); -} - -static inline bool fy_is_flow_indicator(int c) -{ - return !!fy_utf8_strchr(",[]{}", c); -} - -static inline bool fy_is_path_flow_scalar_start(int c) -{ - return c == '\'' || c == '"'; -} - -static inline bool fy_is_path_flow_key_start(int c) -{ - return c == '"' || c == '\'' || c == '{' || c == '['; -} - -static inline bool fy_is_path_flow_key_end(int c) -{ - return c == '"' || c == '\'' || c == '}' || c == ']'; -} - -static inline bool fy_is_unicode_control(int c) -{ - return (c >= 0 && c <= 0x1f) || (c >= 0x80 && c <= 0x9f); -} - -static inline bool fy_is_unicode_space(int c) -{ - return c == 0x20 || c == 0xa0 || - (c >= 0x2000 && c <= 0x200a) || - c == 0x202f || c == 0x205f || c == 0x3000; -} - -static inline bool fy_is_json_unescaped(int c) -{ - return c >= 0x20 && c <= 0x110000 && c != '"' && c != '\\'; -} - -static inline bool fy_is_json_unescaped_range_only(int c) -{ - return c >= 0x20 && c <= 0x110000; -} - -static inline bool fy_is_lb_m(int c, enum fy_lb_mode lb_mode) -{ - if (fy_is_lb_r_n(c)) - return true; - return lb_mode == fylb_cr_nl_N_L_P && fy_is_unicode_lb(c); -} - -static inline bool fy_is_generic_lb_m(int c, enum fy_lb_mode lb_mode) -{ - if (fy_is_lb_r_n(c)) - return true; - return lb_mode == fylb_cr_nl_N_L_P && fy_is_lb_NEL(c); -} - -static inline bool fy_is_lbz_m(int c, enum fy_lb_mode lb_mode) -{ - return fy_is_lb_m(c, lb_mode) || fy_is_z(c); -} - -static inline bool fy_is_generic_lbz_m(int c, enum fy_lb_mode lb_mode) -{ - return fy_is_generic_lb_m(c, lb_mode) || fy_is_z(c); -} - -static inline bool fy_is_blankz_m(int c, enum fy_lb_mode lb_mode) -{ - return fy_is_ws(c) || fy_is_lbz_m(c, lb_mode); -} - -static inline bool fy_is_generic_blankz_m(int c, enum fy_lb_mode lb_mode) -{ - return fy_is_ws(c) || fy_is_generic_lbz_m(c, lb_mode); -} - -static inline bool fy_is_flow_ws_m(int c, enum fy_flow_ws_mode fws_mode) -{ - return fy_is_space(c) || (fws_mode == fyfws_space_tab && fy_is_tab(c)); -} - -#define FY_CTYPE_AT_BUILDER(_kind) \ -static inline const void * \ -fy_find_ ## _kind (const void *s, size_t len) \ -{ \ - const void *e = s + len; \ - int c, w; \ - for (; s < e && (c = fy_utf8_get(s, e - s, &w)) >= 0; s += w) { \ - assert(w); \ - if (fy_is_ ## _kind (c)) \ - return s; \ - } \ - return NULL; \ -} \ -static inline const void * \ -fy_find_non_ ## _kind (const void *s, size_t len) \ -{ \ - const void *e = s + len; \ - int c, w; \ - for (; s < e && (c = fy_utf8_get(s, e - s, &w)) >= 0; s += w) { \ - assert(w); \ - if (!(fy_is_ ## _kind (c))) \ - return s; \ - assert(w); \ - } \ - return NULL; \ -} \ -struct useless_struct_for_semicolon - -FY_CTYPE_AT_BUILDER(first_alpha); -FY_CTYPE_AT_BUILDER(alpha); -FY_CTYPE_AT_BUILDER(num); -FY_CTYPE_AT_BUILDER(first_alnum); -FY_CTYPE_AT_BUILDER(alnum); -FY_CTYPE_AT_BUILDER(space); -FY_CTYPE_AT_BUILDER(tab); -FY_CTYPE_AT_BUILDER(ws); -FY_CTYPE_AT_BUILDER(hex); -FY_CTYPE_AT_BUILDER(uri); -FY_CTYPE_AT_BUILDER(z); -FY_CTYPE_AT_BUILDER(any_lb); -FY_CTYPE_AT_BUILDER(blank); -FY_CTYPE_AT_BUILDER(print); -FY_CTYPE_AT_BUILDER(printq); -FY_CTYPE_AT_BUILDER(nb_char); -FY_CTYPE_AT_BUILDER(ns_char); -FY_CTYPE_AT_BUILDER(start_indicator); -FY_CTYPE_AT_BUILDER(indicator_before_space); -FY_CTYPE_AT_BUILDER(flow_indicator); -FY_CTYPE_AT_BUILDER(path_flow_key_start); -FY_CTYPE_AT_BUILDER(path_flow_key_end); -FY_CTYPE_AT_BUILDER(unicode_control); -FY_CTYPE_AT_BUILDER(unicode_space); -FY_CTYPE_AT_BUILDER(json_unescaped); - -/* - * Very special linebreak/ws methods - * Things get interesting due to \r\n and - * unicode linebreaks/spaces - */ - -/* skip for a _single_ linebreak */ -static inline const void *fy_skip_lb(const void *ptr, int left) -{ - int c, width; - - /* get the utf8 character at this point */ - c = fy_utf8_get(ptr, left, &width); - if (c < 0 || !fy_is_any_lb(c)) - return NULL; - - /* MS-DOS: check if next character is '\n' */ - if (c == '\r' && left > width && *(char *)ptr == '\n') - width++; - - return ptr + width; -} - -/* given a pointer to a chunk of memory, return pointer to first - * ws character after the last non-ws character, or the end - * of the chunk - */ -static inline const void *fy_last_non_ws(const void *ptr, int left) -{ - const char *s, *e; - int c; - - s = ptr; - e = s + left; - while (e > s) { - c = e[-1]; - if (c != ' ' && c != '\t') - return e; - e--; - - } - return NULL; -} - -const char *fy_uri_esc(const char *s, size_t len, uint8_t *code, int *code_len); - -#endif diff --git a/src/lib/fy-diag.c b/src/lib/fy-diag.c index 5488c08..e30003f 100644 --- a/src/lib/fy-diag.c +++ b/src/lib/fy-diag.c @@ -580,7 +580,7 @@ void fy_diag_error_atom_display(struct fy_diag *diag, enum fy_error_type type, s struct fy_raw_line l_tmp; struct fy_atom_raw_line_iter iter; int content_start_col, content_end_col, content_width; - int pass, cols, min_col, max_col, total_lines, max_line_count, max_line_col8, max_width; + int pass, cols, min_col, max_col, max_line_count, max_line_col8, max_width; int start_col, end_col; const char *color_start, *color_end, *white; bool first_line, last_line; @@ -608,7 +608,6 @@ void fy_diag_error_atom_display(struct fy_diag *diag, enum fy_error_type type, s max_col = -1; max_line_count = -1; max_line_col8 = -1; - total_lines = 0; line_shift = -1; for (pass = 0; pass < 2; pass++) { @@ -642,7 +641,7 @@ void fy_diag_error_atom_display(struct fy_diag *diag, enum fy_error_type type, s rbe = rowbuf + rowbufsz; /* if the maximum column number is less than the terminal - * width everything fits, and we're fine */ + * width everything fits, and we're fine */ if (max_line_col8 < cols) { line_shift = 0; } else { @@ -689,8 +688,6 @@ void fy_diag_error_atom_display(struct fy_diag *diag, enum fy_error_type type, s if (pass == 0) { - total_lines++; - if (min_col < 0 || content_start_col < min_col) min_col = content_start_col; if (max_col < 0 || content_end_col > max_col) @@ -1246,6 +1243,75 @@ void fy_reader_diag_report(struct fy_reader *fyr, va_end(ap); } +void fy_diag_token_vreport(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *fmt, va_list ap) +{ + struct fy_diag_report_ctx drc; + bool save_on_error; + + if (!fyt || !diag) + return; + + save_on_error = diag->on_error; + diag->on_error = false; + + memset(&drc, 0, sizeof(drc)); + drc.type = type; + drc.module = FYEM_UNKNOWN; + drc.fyt = fyt; + fy_diag_vreport(diag, &drc, fmt, ap); + + diag->on_error = save_on_error; +} + +void fy_diag_token_report(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_diag_token_vreport(diag, fyt, type, fmt, ap); + va_end(ap); +} + +void fy_diag_token_override_vreport(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *file, + int line, int column, + const char *fmt, va_list ap) +{ + struct fy_diag_report_ctx drc; + bool save_on_error; + + if (!fyt || !diag) + return; + + save_on_error = diag->on_error; + diag->on_error = false; + + memset(&drc, 0, sizeof(drc)); + drc.type = type; + drc.module = FYEM_UNKNOWN; + drc.fyt = fyt; + drc.has_override = true; + drc.override_file = file; + drc.override_line = line; + drc.override_column = column; + fy_diag_vreport(diag, &drc, fmt, ap); + + diag->on_error = save_on_error; +} + +void fy_diag_token_override_report(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *file, + int line, int column, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_diag_token_override_vreport(diag, fyt, type, file, line, column, fmt, ap); + va_end(ap); +} + void fy_diag_node_vreport(struct fy_diag *diag, struct fy_node *fyn, enum fy_error_type type, const char *fmt, va_list ap) { @@ -1355,3 +1421,59 @@ void fy_node_override_report(struct fy_node *fyn, enum fy_error_type type, fy_node_override_vreport(fyn, type, file, line, column, fmt, ap); va_end(ap); } + +void fy_parser_vlog(struct fy_parser *fyp, enum fy_error_type type, + const char *fmt, va_list ap) +{ + struct fy_diag_ctx ctx = { + .level = type, + .module = FYEM_UNKNOWN, + .source_func = "", + .source_file = "", + .source_line = 0, + .file = NULL, + .line = 0, + .column = 0, + }; + + if (!fyp || !fyp->diag) + return; + + (void)fy_vdiag(fyp->diag, &ctx, fmt, ap); +} + +void fy_parser_log(struct fy_parser *fyp, enum fy_error_type type, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_parser_vlog(fyp, type, fmt, ap); + va_end(ap); +} + +void fy_parser_vreport(struct fy_parser *fyp, enum fy_error_type type, + struct fy_token *fyt, const char *fmt, va_list ap) +{ + struct fy_diag_report_ctx fydrc_local, *fydrc = &fydrc_local; + + if (!fyp || !fyp->diag || !fyt) + return; + + memset(fydrc, 0, sizeof(*fydrc)); + fydrc->type = type; + fydrc->module = FYEM_UNKNOWN; + fydrc->fyt = fy_token_ref(fyt); + + fy_parser_diag_vreport(fyp, fydrc, fmt, ap); +} + +void fy_parser_report(struct fy_parser *fyp, enum fy_error_type type, + struct fy_token *fyt, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_parser_vreport(fyp, type, fyt, fmt, ap); + va_end(ap); +} diff --git a/src/lib/fy-diag.h b/src/lib/fy-diag.h index f19861d..62b6547 100644 --- a/src/lib/fy-diag.h +++ b/src/lib/fy-diag.h @@ -103,11 +103,14 @@ void fy_diag_report(struct fy_diag *diag, #ifdef FY_DEVMODE #define __FY_DEBUG_UNUSED__ /* nothing */ #else +#if defined(__GNUC__) && __GNUC__ >= 4 #define __FY_DEBUG_UNUSED__ __attribute__((__unused__)) +#else +#define __FY_DEBUG_UNUSED__ /* nothing */ +#endif #endif /* parser diagnostics */ - struct fy_parser; void fy_diag_cfg_from_parser_flags(struct fy_diag_cfg *cfg, enum fy_parse_cfg_flags pflags); diff --git a/src/lib/fy-doc.c b/src/lib/fy-doc.c index 7df5105..5b3a942 100644 --- a/src/lib/fy-doc.c +++ b/src/lib/fy-doc.c @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -339,9 +340,6 @@ struct fy_node *fy_node_get_nearest_child_of(struct fy_node *fyn_base, void fy_parse_document_destroy(struct fy_parser *fyp, struct fy_document *fyd) { struct fy_node *fyn; - struct fy_anchor *fya; - struct fy_anchor *fyan; - struct fy_accel_entry *xle; if (!fyd) return; @@ -352,29 +350,7 @@ void fy_parse_document_destroy(struct fy_parser *fyp, struct fy_document *fyd) fyd->root = NULL; fy_node_detach_and_free(fyn); - /* remove all anchors */ - for (fya = fy_anchor_list_head(&fyd->anchors); fya; fya = fyan) { - fyan = fy_anchor_next(&fyd->anchors, fya); - fy_anchor_list_del(&fyd->anchors, fya); - - if (fy_document_is_accelerated(fyd)) { - xle = fy_accel_entry_lookup_key_value(fyd->axl, fya->anchor, fya); - fy_accel_entry_remove(fyd->axl, xle); - - xle = fy_accel_entry_lookup_key_value(fyd->naxl, fya->fyn, fya); - fy_accel_entry_remove(fyd->naxl, xle); - } - - fy_anchor_destroy(fya); - } - - if (fy_document_is_accelerated(fyd)) { - fy_accel_cleanup(fyd->axl); - free(fyd->axl); - - fy_accel_cleanup(fyd->naxl); - free(fyd->naxl); - } + fy_document_purge_anchors(fyd); fy_document_state_unref(fyd->fyds); @@ -2933,9 +2909,40 @@ static void fy_resolve_parent_node(struct fy_document *fyd, struct fy_node *fyn, } } -typedef void (*fy_node_applyf)(struct fy_node *fyn); +void fy_document_purge_anchors(struct fy_document *fyd) +{ + struct fy_anchor *fya; + struct fy_anchor *fyan; + struct fy_accel_entry *xle; -void fy_node_apply(struct fy_node *fyn, fy_node_applyf func) + /* remove all anchors */ + for (fya = fy_anchor_list_head(&fyd->anchors); fya; fya = fyan) { + fyan = fy_anchor_next(&fyd->anchors, fya); + fy_anchor_list_del(&fyd->anchors, fya); + + if (fy_document_is_accelerated(fyd)) { + xle = fy_accel_entry_lookup_key_value(fyd->axl, fya->anchor, fya); + fy_accel_entry_remove(fyd->axl, xle); + + xle = fy_accel_entry_lookup_key_value(fyd->naxl, fya->fyn, fya); + fy_accel_entry_remove(fyd->naxl, xle); + } + + fy_anchor_destroy(fya); + } + + if (fy_document_is_accelerated(fyd)) { + fy_accel_cleanup(fyd->axl); + free(fyd->axl); + + fy_accel_cleanup(fyd->naxl); + free(fyd->naxl); + } +} + +typedef void (*fy_node_applyf)(struct fy_node *fyn, void *user); + +void fy_node_apply(struct fy_node *fyn, fy_node_applyf func, void *user) { struct fy_node *fyni; struct fy_node_pair *fynp; @@ -2943,7 +2950,7 @@ void fy_node_apply(struct fy_node *fyn, fy_node_applyf func) if (!fyn || !func) return; - (*func)(fyn); + (*func)(fyn, user); switch (fyn->type) { case FYNT_SCALAR: @@ -2952,21 +2959,21 @@ void fy_node_apply(struct fy_node *fyn, fy_node_applyf func) case FYNT_SEQUENCE: for (fyni = fy_node_list_head(&fyn->sequence); fyni; fyni = fy_node_next(&fyn->sequence, fyni)) - fy_node_apply(fyni, func); + fy_node_apply(fyni, func, user); break; case FYNT_MAPPING: for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; fynp = fy_node_pair_next(&fyn->mapping, fynp)) { - fy_node_apply(fynp->key, func); - fy_node_apply(fynp->value, func); + fy_node_apply(fynp->key, func, user); + fy_node_apply(fynp->value, func, user); } break; } } -static void clear_system_marks(struct fy_node *fyn) +static void clear_system_marks(struct fy_node *fyn, void *user) { fyn->marks &= ~FYNWF_SYSTEM_MARKS; } @@ -2974,36 +2981,66 @@ static void clear_system_marks(struct fy_node *fyn) /* clear all the system markers */ void fy_node_clear_system_marks(struct fy_node *fyn) { - fy_node_apply(fyn, clear_system_marks); + fy_node_apply(fyn, clear_system_marks, NULL); +} + +static void count_aliases(struct fy_node *fyn, void *user) +{ + int *countp = user; + + if (fyn->style == FYNS_ALIAS) + (*countp)++; +} + +int fy_node_count_aliases(struct fy_node *fyn) +{ + int count; + + count = 0; + + fy_node_apply(fyn, count_aliases, &count); + + return count; } int fy_document_resolve(struct fy_document *fyd) { - int rc; + int rc, num_aliases, num_aliases_prev; bool ret; if (!fyd) return 0; - fy_node_clear_system_marks(fyd->root); + num_aliases_prev = INT_MAX; + do { + fy_node_clear_system_marks(fyd->root); - /* for resolution to work, no reference loops should exist */ - ret = fy_check_ref_loop(fyd, fyd->root, - FYNWF_MAXDEPTH_DEFAULT | FYNWF_FOLLOW, NULL); + /* for resolution to work, no reference loops should exist */ + ret = fy_check_ref_loop(fyd, fyd->root, + FYNWF_MAXDEPTH_DEFAULT | FYNWF_FOLLOW, NULL); - fy_node_clear_system_marks(fyd->root); + fy_node_clear_system_marks(fyd->root); - if (ret) - goto err_out; + if (ret) + goto err_out; + /* now resolve any anchor nodes */ + rc = fy_resolve_anchor_node(fyd, fyd->root); + if (rc) + goto err_out_rc; - /* now resolve any anchor nodes */ - rc = fy_resolve_anchor_node(fyd, fyd->root); - if (rc) - goto err_out_rc; + /* redo parent resolution */ + fy_resolve_parent_node(fyd, fyd->root, NULL); - /* redo parent resolution */ - fy_resolve_parent_node(fyd, fyd->root, NULL); + /* count the remaining aliases */ + num_aliases = fy_node_count_aliases(fyd->root); + if (num_aliases == num_aliases_prev) + goto err_out; + + } while (num_aliases > 0); + + /* remove all anchors after resolution */ + fy_document_purge_anchors(fyd); return 0; @@ -3343,6 +3380,44 @@ enum fy_node_style fy_node_get_style(struct fy_node *fyn) return fyn ? fyn->style : FYNS_PLAIN; } +struct fy_token *fy_node_get_start_token(struct fy_node *fyn) +{ + if (!fyn) + return NULL; + + switch (fyn->type) { + case FYNT_MAPPING: + return fyn->mapping_start; + case FYNT_SEQUENCE: + return fyn->sequence_start; + case FYNT_SCALAR: + return fyn->scalar; + default: + /* should not happen, but play it safe */ + break; + } + return NULL; +} + +struct fy_token *fy_node_get_end_token(struct fy_node *fyn) +{ + if (!fyn) + return NULL; + + switch (fyn->type) { + case FYNT_MAPPING: + return fyn->mapping_end; + case FYNT_SEQUENCE: + return fyn->sequence_end; + case FYNT_SCALAR: + return fyn->scalar; + default: + /* should not happen, but play it safe */ + break; + } + return NULL; +} + bool fy_node_is_null(struct fy_node *fyn) { if (!fyn) @@ -6039,6 +6114,75 @@ err_no_parse: return NULL; } +struct fy_document * +fy_block_document_build_from_string(const struct fy_parse_cfg *cfg, + const char *str, size_t len, size_t *consumed) +{ + struct flow_reader_container frc; + struct fy_reader *fyr = NULL; + struct fy_parser fyp_data, *fyp = &fyp_data; + struct fy_parse_cfg cfg_data; + struct fy_input *fyi; + struct fy_document *fyd; + struct fy_mark mark; + int rc; + + if (!str) + return NULL; + + if (consumed) + *consumed = 0; + + if (!cfg) { + memset(&cfg_data, 0, sizeof(cfg_data)); + cfg_data.flags = FYPCF_DEFAULT_PARSE; + cfg = &cfg_data; + } + + memset(&frc, 0, sizeof(frc)); + fyr = &frc.reader; + frc.cfg = cfg; + + fy_reader_setup(fyr, &reader_ops); + + rc = fy_parse_setup(fyp, cfg); + if (rc) + goto err_no_parse; + + fyi = fy_input_from_data(str, len, NULL, false); + if (!fyi) + goto err_no_input; + + rc = fy_reader_input_open(fyr, fyi, NULL); + if (rc) + goto err_no_input_open; + + fy_parser_set_reader(fyp, fyr); + fy_parser_set_block_only_mode(fyp, true); + + fyd = fy_parse_load_document(fyp); + + fy_parse_cleanup(fyp); + + if (fyd && consumed) { + fy_reader_get_mark(fyr, &mark); + *consumed = mark.input_pos; + } + + fy_reader_cleanup(fyr); + fy_input_unref(fyi); + + return fyd; + +err_no_input_open: + fy_input_unref(fyi); +err_no_input: + fy_parse_cleanup(fyp); +err_no_parse: + fy_reader_cleanup(fyr); + return NULL; +} + int fy_node_vscanf(struct fy_node *fyn, const char *fmt, va_list ap) { size_t len; diff --git a/src/lib/fy-doc.h b/src/lib/fy-doc.h index 6c15024..d368103 100644 --- a/src/lib/fy-doc.h +++ b/src/lib/fy-doc.h @@ -124,6 +124,7 @@ struct fy_document { FY_TYPE_DECL_LIST(document); struct fy_document *fy_parse_document_create(struct fy_parser *fyp, struct fy_eventp *fyep); +void fy_document_purge_anchors(struct fy_document *fyd); struct fy_node_mapping_sort_ctx { fy_node_mapping_sort_fn key_cmp; diff --git a/src/lib/fy-docbuilder.c b/src/lib/fy-docbuilder.c index 94f6606..b1d78e8 100644 --- a/src/lib/fy-docbuilder.c +++ b/src/lib/fy-docbuilder.c @@ -538,3 +538,27 @@ fy_document_builder_load_document(struct fy_document_builder *fydb, /* get ownership of the document */ return fy_document_builder_take_document(fydb); } + +struct fy_document * +fy_document_builder_event_document(struct fy_document_builder *fydb, struct fy_eventp_list *evpl) +{ + struct fy_eventp *fyep = NULL; + int rc; + + if (!fydb || !evpl) + return NULL; + + for (fyep = fy_eventp_list_head(evpl); fyep; fyep = fy_eventp_next(evpl, fyep)) { + + if (fy_document_builder_is_document_complete(fydb)) + break; + + rc = fy_document_builder_process_event(fydb, fyep); + if (rc < 0) + return NULL; + + } + + /* get ownership of the document */ + return fy_document_builder_take_document(fydb); +} diff --git a/src/lib/fy-docbuilder.h b/src/lib/fy-docbuilder.h index aa506d6..dd2d0a0 100644 --- a/src/lib/fy-docbuilder.h +++ b/src/lib/fy-docbuilder.h @@ -92,6 +92,9 @@ struct fy_document * fy_document_builder_load_document(struct fy_document_builder *fydb, struct fy_parser *fyp); +struct fy_document * +fy_document_builder_event_document(struct fy_document_builder *fydb, struct fy_eventp_list *evpl); + struct fy_document * fy_parse_load_document_with_builder(struct fy_parser *fyp); diff --git a/src/lib/fy-docstate.c b/src/lib/fy-docstate.c index e7876e4..6c530bb 100644 --- a/src/lib/fy-docstate.c +++ b/src/lib/fy-docstate.c @@ -392,3 +392,56 @@ fy_document_state_tag_directive_iterate(struct fy_document_state *fyds, void **i return tag; } + +struct fy_tag ** +fy_document_state_tag_directives(struct fy_document_state *fyds) +{ + struct fy_tag *tags, **tagsp; + const struct fy_tag *fytag; + size_t size, len; + void *iter; + char *s; + int i; + + if (!fyds) + return NULL; + + /* first find the extents of the area we're going to need */ + i = 0; + size = 0; + iter = NULL; + while ((fytag = fy_document_state_tag_directive_iterate(fyds, &iter)) != NULL) { + size += strlen(fytag->handle) + 1 + strlen(fytag->prefix) + 1; + i++; + } + size += sizeof(struct fy_tag) * i; + size += sizeof(struct fy_tag *) * (i + 1); + + tagsp = malloc(size); + if (!tagsp) + return NULL; + memset(tagsp, 0, size); + tags = (void *)(tagsp + i + 1); + s = (void *)(tags + i); + + i = 0; + iter = NULL; + while ((fytag = fy_document_state_tag_directive_iterate(fyds, &iter)) != NULL) { + tags[i].handle = s; + len = strlen(fytag->handle); + memcpy(s, fytag->handle, len + 1); + s += len + 1; + + tags[i].prefix = s; + len = strlen(fytag->prefix); + memcpy(s, fytag->prefix, len + 1); + s += len + 1; + + tagsp[i] = &tags[i]; + + i++; + } + tagsp[i] = NULL; /* NULL terminated */ + + return tagsp; +} diff --git a/src/lib/fy-emit.c b/src/lib/fy-emit.c index 438f82d..b382876 100644 --- a/src/lib/fy-emit.c +++ b/src/lib/fy-emit.c @@ -23,6 +23,20 @@ #include "fy-parse.h" #include "fy-emit.h" +static inline struct fy_token_list *token_recycle_list(struct fy_emitter *emit, struct fy_parser *fyp) +{ + if (fyp && fyp->recycled_token_list) + return fyp->recycled_token_list; + if (emit && emit->recycled_token_list) + return emit->recycled_token_list; + return NULL; +} + +static inline void fy_emit_token_unref(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_token *fyt) +{ + fy_token_unref_rl(token_recycle_list(emit, fyp), fyt); +} + /* fwd decl */ void fy_emit_write(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int len); void fy_emit_printf(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *fmt, ...) @@ -43,7 +57,7 @@ static inline bool fy_emit_is_flow_mode(const struct fy_emitter *emit) { enum fy_emitter_cfg_flags flags = emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK); - return flags == FYECF_MODE_FLOW || flags == FYECF_MODE_FLOW_ONELINE; + return flags == FYECF_MODE_FLOW || flags == FYECF_MODE_FLOW_ONELINE || fy_emit_is_json_mode(emit); } static inline bool fy_emit_is_block_mode(const struct fy_emitter *emit) @@ -1245,8 +1259,6 @@ fy_emit_token_scalar_style(struct fy_emitter *emit, struct fy_token *fyt, int flags, int indent, enum fy_node_style style, struct fy_token *fyt_tag) { - const char *value = NULL; - size_t len = 0; bool json, flow, is_null_scalar, is_json_plain; struct fy_atom *atom; int aflags = -1; @@ -1300,17 +1312,13 @@ fy_emit_token_scalar_style(struct fy_emitter *emit, struct fy_token *fyt, if (flow && (style == FYNS_ANY || style == FYNS_LITERAL || style == FYNS_FOLDED)) { - if (fyt && !value) - value = fy_token_get_text(fyt, &len); - /* if there's a linebreak, use double quoted style */ - if (fy_find_any_lb(value, len)) { + if (aflags & FYTTAF_HAS_ANY_LB) { style = FYNS_DOUBLE_QUOTED; goto out; } - /* check if there's a non printable */ - if (!fy_find_non_print(value, len)) { + if (!(aflags & FYTTAF_HAS_NON_PRINT)) { style = FYNS_SINGLE_QUOTED; goto out; } @@ -1345,13 +1353,8 @@ fy_emit_token_scalar_style(struct fy_emitter *emit, struct fy_token *fyt, } out: - if (style == FYNS_ANY) { - if (fyt) - value = fy_token_get_text(fyt, &len); - - style = (aflags & FYTTAF_CAN_BE_PLAIN) ? - FYNS_PLAIN : FYNS_DOUBLE_QUOTED; - } + if (style == FYNS_ANY) + style = (aflags & FYTTAF_CAN_BE_PLAIN) ? FYNS_PLAIN : FYNS_DOUBLE_QUOTED; if (style == FYNS_PLAIN) { /* plains in flow mode not being able to be plains @@ -2072,6 +2075,14 @@ int fy_emit_setup(struct fy_emitter *emit, const struct fy_emitter_cfg *cfg) return 0; } +void fy_emit_save_ctx_cleanup(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) +{ + sc = &emit->s_sc; + fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_key); + fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_value); + memset(sc, 0, sizeof(*sc)); +} + void fy_emit_cleanup(struct fy_emitter *emit) { struct fy_eventp *fyep; @@ -2081,6 +2092,10 @@ void fy_emit_cleanup(struct fy_emitter *emit) if (emit->finalizer) emit->finalizer(emit); + fy_emit_save_ctx_cleanup(emit, &emit->s_sc); + while (emit->sc_stack_top > 0) + fy_emit_save_ctx_cleanup(emit, &emit->sc_stack[--emit->sc_stack_top]); + while ((fyt = fy_token_list_pop(&emit->recycled_token)) != NULL) fy_token_free(fyt); @@ -2711,63 +2726,45 @@ char *fy_emit_node_to_string(struct fy_node *fyn, enum fy_emitter_cfg_flags flag return buf; } +/* we can consume events only if the lookaheads + * are satisfied. The lookaheads depends on the + * styling mode. Some styling modes do not require + * a lookahead at all but we do it anyway. + */ static bool fy_emit_ready(struct fy_emitter *emit) { struct fy_eventp *fyep; - int need, count, level; - - /* no events in the list, not ready */ - fyep = fy_eventp_list_head(&emit->queued_events); - if (!fyep) - return false; + int need, count; - /* some events need more than one */ - switch (fyep->e.type) { - case FYET_DOCUMENT_START: - need = 1; - break; - case FYET_SEQUENCE_START: - need = 2; - break; - case FYET_MAPPING_START: - need = 3; - break; - default: - need = 0; - break; - } - - /* if we don't need any more, that's enough */ - if (!need) - return true; - - level = 0; count = 0; - for (; fyep; fyep = fy_eventp_next(&emit->queued_events, fyep)) { - count++; - - if (count > need) - return true; + need = -1; + for (fyep = fy_eventp_list_head(&emit->queued_events); fyep; + fyep = fy_eventp_next(&emit->queued_events, fyep)) { + count++; switch (fyep->e.type) { - case FYET_STREAM_START: case FYET_DOCUMENT_START: + need++; + break; + case FYET_SEQUENCE_START: - case FYET_MAPPING_START: - level++; + if (need < 0) + need = 2; break; - case FYET_STREAM_END: - case FYET_DOCUMENT_END: - case FYET_SEQUENCE_END: - case FYET_MAPPING_END: - level--; + + case FYET_MAPPING_START: + if (need < 0) + need = 2; break; + default: + need = 0; break; } - if (!level) + if (count >= need) return true; + } return false; @@ -2803,36 +2800,35 @@ fy_emit_next_event(struct fy_emitter *emit) struct fy_eventp * fy_emit_peek_next_event(struct fy_emitter *emit) { - if (!fy_emit_ready(emit)) - return NULL; - return fy_eventp_list_head(&emit->queued_events); } bool fy_emit_streaming_sequence_empty(struct fy_emitter *emit) { - enum fy_emitter_cfg_flags flags = emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK); struct fy_eventp *fyepn; - struct fy_event *fyen; - - if (flags == FYECF_MODE_BLOCK || flags == FYECF_MODE_DEJSON) - return false; fyepn = fy_emit_peek_next_event(emit); - fyen = fyepn ? &fyepn->e : NULL; - return !fyen || fyen->type == FYET_SEQUENCE_END; + /* should never happen, check on debugging mode */ + assert(fyepn); + if (!fyepn) + return false; + + return fyepn->e.type == FYET_SEQUENCE_END; } bool fy_emit_streaming_mapping_empty(struct fy_emitter *emit) { struct fy_eventp *fyepn; - struct fy_event *fyen; fyepn = fy_emit_peek_next_event(emit); - fyen = fyepn ? &fyepn->e : NULL; - return !fyen || fyen->type == FYET_MAPPING_END; + /* should never happen, check on debugging mode */ + assert(fyepn); + if (!fyepn) + return false; + + return fyepn->e.type == FYET_MAPPING_END; } static void fy_emit_goto_state(struct fy_emitter *emit, enum fy_emitter_state state) @@ -2901,7 +2897,7 @@ int fy_emit_pop_sc(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) return 0; } -static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_eventp *fyep, int flags) +static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, int flags) { struct fy_event *fye = &fyep->e; struct fy_emit_save_ctx *sc = &emit->s_sc; @@ -2945,8 +2941,11 @@ static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_eventp *fye s_flags = emit->s_flags; s_indent = emit->s_indent; - xstyle = fye->sequence_start.sequence_start->type == FYTT_BLOCK_SEQUENCE_START ? - FYNS_BLOCK : FYNS_FLOW; + if (fye->sequence_start.sequence_start) + xstyle = fye->sequence_start.sequence_start->type == FYTT_BLOCK_SEQUENCE_START ? + FYNS_BLOCK : FYNS_FLOW; + else + xstyle = FYNS_ANY; fy_emit_common_node_preamble(emit, fye->sequence_start.anchor, fye->sequence_start.tag, emit->s_flags, emit->s_indent); @@ -2961,8 +2960,6 @@ static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_eventp *fye sc->old_indent = sc->indent; sc->s_flags = s_flags; sc->s_indent = s_indent; - sc->s_flags = s_flags; - sc->s_indent = s_indent; fy_emit_sequence_prolog(emit, sc); sc->flags &= ~DDNF_MAP; @@ -2983,8 +2980,11 @@ static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_eventp *fye s_flags = emit->s_flags; s_indent = emit->s_indent; - xstyle = fye->mapping_start.mapping_start->type == FYTT_BLOCK_MAPPING_START ? - FYNS_BLOCK : FYNS_FLOW; + if (fye->mapping_start.mapping_start) + xstyle = fye->mapping_start.mapping_start->type == FYTT_BLOCK_MAPPING_START ? + FYNS_BLOCK : FYNS_FLOW; + else + xstyle = FYNS_ANY; fy_emit_common_node_preamble(emit, fye->mapping_start.anchor, fye->mapping_start.tag, emit->s_flags, emit->s_indent); @@ -2999,8 +2999,6 @@ static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_eventp *fye sc->old_indent = sc->indent; sc->s_flags = s_flags; sc->s_indent = s_indent; - sc->s_flags = s_flags; - sc->s_indent = s_indent; fy_emit_mapping_prolog(emit, sc); sc->flags &= ~DDNF_SEQ; @@ -3020,7 +3018,7 @@ static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_eventp *fye return 0; } -static int fy_emit_handle_stream_start(struct fy_emitter *emit, struct fy_eventp *fyep) +static int fy_emit_handle_stream_start(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep) { struct fy_event *fye = &fyep->e; @@ -3036,7 +3034,7 @@ static int fy_emit_handle_stream_start(struct fy_emitter *emit, struct fy_eventp return 0; } -static int fy_emit_handle_document_start(struct fy_emitter *emit, struct fy_eventp *fyep, bool first) +static int fy_emit_handle_document_start(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool first) { struct fy_event *fye = &fyep->e; struct fy_document_state *fyds; @@ -3066,7 +3064,7 @@ static int fy_emit_handle_document_start(struct fy_emitter *emit, struct fy_even return 0; } -static int fy_emit_handle_document_end(struct fy_emitter *emit, struct fy_eventp *fyep) +static int fy_emit_handle_document_end(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep) { struct fy_document_state *fyds; struct fy_event *fye = &fyep->e; @@ -3090,30 +3088,30 @@ static int fy_emit_handle_document_end(struct fy_emitter *emit, struct fy_eventp return 0; } -static int fy_emit_handle_document_content(struct fy_emitter *emit, struct fy_eventp *fyep) +static int fy_emit_handle_document_content(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep) { struct fy_event *fye = &fyep->e; int ret; /* empty document? */ if (fye->type == FYET_DOCUMENT_END) - return fy_emit_handle_document_end(emit, fyep); + return fy_emit_handle_document_end(emit, fyp, fyep); ret = fy_emit_push_state(emit, FYES_DOCUMENT_END); if (ret) return ret; - return fy_emit_streaming_node(emit, fyep, DDNF_ROOT); + return fy_emit_streaming_node(emit, fyp, fyep, DDNF_ROOT); } -static int fy_emit_handle_sequence_item(struct fy_emitter *emit, struct fy_eventp *fyep, bool first) +static int fy_emit_handle_sequence_item(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool first) { struct fy_event *fye = &fyep->e; struct fy_emit_save_ctx *sc = &emit->s_sc; struct fy_token *fyt_item = NULL; int ret; - fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_value); + fy_emit_token_unref(emit, fyp, sc->fyt_last_value); sc->fyt_last_value = NULL; switch (fye->type) { @@ -3122,14 +3120,17 @@ static int fy_emit_handle_sequence_item(struct fy_emitter *emit, struct fy_event /* emit epilog */ fy_emit_sequence_epilog(emit, sc); - /* pop state */ - ret = fy_emit_pop_sc(emit, sc); - /* pop state */ - fy_emit_goto_state(emit, fy_emit_pop_state(emit)); /* restore indent and flags */ emit->s_indent = sc->s_indent; emit->s_flags = sc->s_flags; + + /* pop state */ + ret = fy_emit_pop_sc(emit, sc); + assert(!ret); + + /* pop state */ + fy_emit_goto_state(emit, fy_emit_pop_state(emit)); return ret; case FYET_ALIAS: @@ -3164,7 +3165,7 @@ static int fy_emit_handle_sequence_item(struct fy_emitter *emit, struct fy_event fy_emit_sequence_item_prolog(emit, sc, fyt_item); - ret = fy_emit_streaming_node(emit, fyep, sc->flags); + ret = fy_emit_streaming_node(emit, fyp, fyep, sc->flags); switch (fye->type) { case FYET_ALIAS: @@ -3186,7 +3187,7 @@ static int fy_emit_handle_sequence_item(struct fy_emitter *emit, struct fy_event return ret; } -static int fy_emit_handle_mapping_key(struct fy_emitter *emit, struct fy_eventp *fyep, bool first) +static int fy_emit_handle_mapping_key(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool first) { struct fy_event *fye = &fyep->e; struct fy_emit_save_ctx *sc = &emit->s_sc; @@ -3194,9 +3195,9 @@ static int fy_emit_handle_mapping_key(struct fy_emitter *emit, struct fy_eventp int ret, aflags; bool simple_key; - fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_key); + fy_emit_token_unref(emit, fyp, sc->fyt_last_key); + fy_emit_token_unref(emit, fyp, sc->fyt_last_value); sc->fyt_last_key = NULL; - fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_value); sc->fyt_last_value = NULL; simple_key = false; @@ -3207,14 +3208,17 @@ static int fy_emit_handle_mapping_key(struct fy_emitter *emit, struct fy_eventp /* emit epilog */ fy_emit_mapping_epilog(emit, sc); - /* pop state */ - ret = fy_emit_pop_sc(emit, sc); - /* pop state */ - fy_emit_goto_state(emit, fy_emit_pop_state(emit)); /* restore indent and flags */ emit->s_indent = sc->s_indent; emit->s_flags = sc->s_flags; + + /* pop state */ + ret = fy_emit_pop_sc(emit, sc); + assert(!ret); + + /* pop state */ + fy_emit_goto_state(emit, fy_emit_pop_state(emit)); return ret; case FYET_ALIAS: @@ -3254,7 +3258,7 @@ static int fy_emit_handle_mapping_key(struct fy_emitter *emit, struct fy_eventp fy_emit_mapping_key_prolog(emit, sc, fyt_key, simple_key); - ret = fy_emit_streaming_node(emit, fyep, sc->flags); + ret = fy_emit_streaming_node(emit, fyp, fyep, sc->flags); switch (fye->type) { case FYET_ALIAS: @@ -3276,7 +3280,7 @@ static int fy_emit_handle_mapping_key(struct fy_emitter *emit, struct fy_eventp return ret; } -static int fy_emit_handle_mapping_value(struct fy_emitter *emit, struct fy_eventp *fyep, bool simple) +static int fy_emit_handle_mapping_value(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool simple) { struct fy_event *fye = &fyep->e; struct fy_emit_save_ctx *sc = &emit->s_sc; @@ -3311,7 +3315,7 @@ static int fy_emit_handle_mapping_value(struct fy_emitter *emit, struct fy_event fy_emit_mapping_value_prolog(emit, sc, fyt_value); - ret = fy_emit_streaming_node(emit, fyep, sc->flags); + ret = fy_emit_streaming_node(emit, fyp, fyep, sc->flags); switch (fye->type) { case FYET_ALIAS: @@ -3354,38 +3358,38 @@ int fy_emit_event_from_parser(struct fy_emitter *emit, struct fy_parser *fyp, st switch (emit->state) { case FYES_STREAM_START: - ret = fy_emit_handle_stream_start(emit, fyep); + ret = fy_emit_handle_stream_start(emit, fyp, fyep); break; case FYES_FIRST_DOCUMENT_START: case FYES_DOCUMENT_START: - ret = fy_emit_handle_document_start(emit, fyep, + ret = fy_emit_handle_document_start(emit, fyp, fyep, emit->state == FYES_FIRST_DOCUMENT_START); break; case FYES_DOCUMENT_CONTENT: - ret = fy_emit_handle_document_content(emit, fyep); + ret = fy_emit_handle_document_content(emit, fyp, fyep); break; case FYES_DOCUMENT_END: - ret = fy_emit_handle_document_end(emit, fyep); + ret = fy_emit_handle_document_end(emit, fyp, fyep); break; case FYES_SEQUENCE_FIRST_ITEM: case FYES_SEQUENCE_ITEM: - ret = fy_emit_handle_sequence_item(emit, fyep, + ret = fy_emit_handle_sequence_item(emit, fyp, fyep, emit->state == FYES_SEQUENCE_FIRST_ITEM); break; case FYES_MAPPING_FIRST_KEY: case FYES_MAPPING_KEY: - ret = fy_emit_handle_mapping_key(emit, fyep, + ret = fy_emit_handle_mapping_key(emit, fyp, fyep, emit->state == FYES_MAPPING_FIRST_KEY); break; case FYES_MAPPING_SIMPLE_VALUE: case FYES_MAPPING_VALUE: - ret = fy_emit_handle_mapping_value(emit, fyep, + ret = fy_emit_handle_mapping_value(emit, fyp, fyep, emit->state == FYES_MAPPING_SIMPLE_VALUE); break; diff --git a/src/lib/fy-event.c b/src/lib/fy-event.c index a359063..42beab1 100644 --- a/src/lib/fy-event.c +++ b/src/lib/fy-event.c @@ -465,7 +465,7 @@ fy_eventp_vcreate_internal(struct fy_eventp_list *recycled_list, struct fy_diag goto err_out; } - fyi = fy_input_from_data(data, len, &handle, true); + fyi = fy_input_from_malloc_data(data, len, &handle, true); if (!fyi) goto err_out; data = NULL; @@ -542,6 +542,77 @@ fy_emit_event_create(struct fy_emitter *emit, enum fy_event_type type, ...) return fye; } +int +fy_emit_vevent(struct fy_emitter *emit, enum fy_event_type type, va_list ap) +{ + struct fy_event *fye; + + if (!emit) + return -1; + + fye = fy_emit_event_vcreate(emit, type, ap); + if (!fye) + return -1; + + return fy_emit_event(emit, fye); +} + +int +fy_emit_eventf(struct fy_emitter *emit, enum fy_event_type type, ...) +{ + int rc; + va_list ap; + + va_start(ap, type); + rc = fy_emit_vevent(emit, type, ap); + va_end(ap); + + return rc; +} + +int +fy_emit_scalar_write(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *text, size_t len) +{ + return fy_emit_eventf(fye, FYET_SCALAR, style, text, len, anchor, tag); +} + +int +fy_emit_scalar_vprintf(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *fmt, va_list ap) +{ + char *buf; + size_t len; + int rc; + + rc = vasprintf(&buf, fmt, ap); + if (rc < 0) + return -1; + len = (size_t)rc; + + rc = fy_emit_scalar_write(fye, style, anchor, tag, buf, len); + free(buf); + + return rc; +} + +int +fy_emit_scalar_printf(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_emit_scalar_vprintf(fye, style, anchor, tag, fmt, ap); + va_end(ap); + + return rc; +} + struct fy_event * fy_parse_event_vcreate(struct fy_parser *fyp, enum fy_event_type type, va_list ap) { @@ -671,6 +742,36 @@ struct fy_token *fy_event_get_anchor_token(struct fy_event *fye) return NULL; } +struct fy_token *fy_event_get_and_clear_anchor_token(struct fy_event *fye) +{ + struct fy_token *fyt = NULL; + + if (!fye) + return NULL; + + switch (fye->type) { + case FYET_MAPPING_START: + fyt = fye->mapping_start.anchor; + fye->mapping_start.anchor = NULL; + break; + + case FYET_SEQUENCE_START: + fyt = fye->sequence_start.anchor; + fye->sequence_start.anchor = NULL; + break; + + case FYET_SCALAR: + fyt = fye->scalar.anchor; + fye->scalar.anchor = NULL; + break; + + default: + break; + } + + return fyt; +} + struct fy_token *fy_event_get_tag_token(struct fy_event *fye) { if (!fye) @@ -819,6 +920,89 @@ fy_event_get_node_style(struct fy_event *fye) return FYNS_ANY; } +struct fy_eventp *fy_parse_eventp_clone(struct fy_parser *fyp, struct fy_eventp *fyep_src, bool strip_anchors) +{ + struct fy_eventp *fyep = NULL; + struct fy_event *fye, *fye_src; + + if (!fyp || !fyep_src) + return NULL; + + if (fyp->recycled_eventp_list) + fyep = fy_eventp_list_pop(fyp->recycled_eventp_list); + if (!fyep) + fyep = fy_eventp_alloc(); + if (!fyep) + return NULL; + + fye_src = &fyep_src->e; + fye = &fyep->e; + + fye->type = fye_src->type; + switch (fye->type) { + case FYET_NONE: + break; + case FYET_STREAM_START: + fye->stream_start.stream_start = fy_token_ref(fye_src->stream_start.stream_start); + break; + case FYET_STREAM_END: + fye->stream_end.stream_end = fy_token_ref(fye_src->stream_end.stream_end); + break; + case FYET_DOCUMENT_START: + fye->document_start.document_start = fy_token_ref(fye_src->document_start.document_start); + fye->document_start.document_state = fy_document_state_ref(fye_src->document_start.document_state); + break; + case FYET_DOCUMENT_END: + fye->document_end.document_end = fy_token_ref(fye_src->document_end.document_end); + break; + case FYET_MAPPING_START: + if (!strip_anchors) + fye->mapping_start.anchor = fy_token_ref(fye_src->mapping_start.anchor); + else + fye->mapping_start.anchor = NULL; + fye->mapping_start.tag = fy_token_ref(fye_src->mapping_start.tag); + fye->mapping_start.mapping_start = fy_token_ref(fye_src->mapping_start.mapping_start); + break; + case FYET_MAPPING_END: + fye->mapping_end.mapping_end = fy_token_ref(fye_src->mapping_end.mapping_end); + break; + case FYET_SEQUENCE_START: + if (!strip_anchors) + fye->sequence_start.anchor = fy_token_ref(fye_src->sequence_start.anchor); + else + fye->sequence_start.anchor = NULL; + fye->sequence_start.tag = fy_token_ref(fye_src->sequence_start.tag); + fye->sequence_start.sequence_start = fy_token_ref(fye_src->sequence_start.sequence_start); + break; + case FYET_SEQUENCE_END: + fye->sequence_end.sequence_end = fy_token_ref(fye_src->sequence_end.sequence_end); + break; + case FYET_SCALAR: + if (!strip_anchors) + fye->scalar.anchor = fy_token_ref(fye_src->scalar.anchor); + else + fye->scalar.anchor = NULL; + fye->scalar.tag = fy_token_ref(fye_src->scalar.tag); + fye->scalar.value = fy_token_ref(fye_src->scalar.value); + break; + case FYET_ALIAS: + fye->alias.anchor = fy_token_ref(fye_src->alias.anchor); + break; + } + return fyep; +} + +const char *fy_event_get_anchor(struct fy_event *fye, size_t *anchor_lenp) +{ + struct fy_token *anchor; + + anchor = fy_event_get_anchor_token(fye); + if (!anchor) + return NULL; + + return fy_token_get_text(anchor, anchor_lenp); +} + const struct fy_version * fy_document_start_event_version(struct fy_event *fye) { diff --git a/src/lib/fy-event.h b/src/lib/fy-event.h index 75f20fb..90f8241 100644 --- a/src/lib/fy-event.h +++ b/src/lib/fy-event.h @@ -26,7 +26,6 @@ struct fy_eventp { struct fy_event e; }; FY_TYPE_DECL_LIST(eventp); -FY_PARSE_TYPE_DECL_ALLOC(eventp); struct fy_eventp *fy_eventp_alloc(void); void fy_eventp_free(struct fy_eventp *fyep); @@ -40,6 +39,10 @@ void fy_parse_eventp_recycle(struct fy_parser *fyp, struct fy_eventp *fyep); struct fy_eventp *fy_emit_eventp_alloc(struct fy_emitter *fye); void fy_emit_eventp_recycle(struct fy_emitter *emit, struct fy_eventp *fyep); +struct fy_eventp *fy_parse_eventp_clone(struct fy_parser *fyp, struct fy_eventp *fyep_src, bool strip_anchors); +struct fy_token *fy_event_get_and_clear_anchor_token(struct fy_event *fye); +const char *fy_event_get_anchor(struct fy_event *fye, size_t *anchor_lenp); + struct fy_document_iterator; struct fy_eventp *fy_document_iterator_eventp_alloc(struct fy_document_iterator *fydi); diff --git a/src/lib/fy-input.c b/src/lib/fy-input.c index 5d16ce6..cc29ae8 100644 --- a/src/lib/fy-input.c +++ b/src/lib/fy-input.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -259,6 +260,61 @@ void fy_input_close(struct fy_input *fyi) } } +ssize_t fy_input_estimate_queued_size(const struct fy_input *fyi) +{ + struct stat sb; + int fd, rc; + + if (!fyi || fyi->state != FYIS_QUEUED) + return 0; + + memset(&sb, 0, sizeof(sb)); + + switch (fyi->cfg.type) { + case fyit_file: + rc = stat(fyi->cfg.file.filename, &sb); + if (rc) + return -1; + break; + + case fyit_stream: + fd = fileno(fyi->cfg.stream.fp); + if (fd < 0) + return -1; + rc = fstat(fd, &sb); + if (rc) + return -1; + break; + + case fyit_memory: + return (ssize_t)fyi->cfg.memory.size; + + case fyit_alloc: + return (ssize_t)fyi->cfg.alloc.size; + + case fyit_fd: + rc = fstat(fyi->cfg.fd.fd, &sb); + if (rc) + return -1; + break; + + case fyit_callback: + default: + return SSIZE_MAX; /* cannot determine */ + } + + /* only do it for regular files */ + if ((sb.st_mode & S_IFMT) != S_IFREG) + return SSIZE_MAX; + + /* check for very impossible roll-over */ + if ((size_t)sb.st_size > (size_t)SSIZE_MAX) + return SSIZE_MAX; + + /* ok, we did find it */ + return (ssize_t)sb.st_size; +} + struct fy_diag *fy_reader_get_diag(struct fy_reader *fyr) { if (fyr && fyr->ops && fyr->ops->get_diag) @@ -611,7 +667,7 @@ const void *fy_reader_ptr_slow_path(struct fy_reader *fyr, size_t *leftp) { struct fy_input *fyi; const void *p; - int left; + size_t left; if (fyr->current_ptr) { if (leftp) diff --git a/src/lib/fy-input.h b/src/lib/fy-input.h index 6634fab..184590e 100644 --- a/src/lib/fy-input.h +++ b/src/lib/fy-input.h @@ -214,6 +214,8 @@ fy_input_unref(struct fy_input *fyi) fyi->refs--; } +ssize_t fy_input_estimate_queued_size(const struct fy_input *fyi); + struct fy_reader; enum fy_reader_mode { diff --git a/src/lib/fy-list.h b/src/lib/fy-list.h deleted file mode 100644 index 1a0b5f9..0000000 --- a/src/lib/fy-list.h +++ /dev/null @@ -1,705 +0,0 @@ -/* - * fy-list.h - slightly modified Linux kernel list.h - * - * All copyrights owned by their respective holders. - * - * SPDX-License-Identifier: GPL-2.0 - */ -#ifndef _FY_LIST_H -#define _FY_LIST_H - -#include - -/** - * container_of - cast a member of a structure out to the containing structure - * @ptr: the pointer to the member. - * @type: the type of the container struct this is embedded in. - * @member: the name of the member within the struct. - * - */ -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) - -#define LIST_POISON1 NULL -#define LIST_POISON2 NULL - -#ifndef ARCH_HAS_PREFETCH -#define ARCH_HAS_PREFETCH -static inline void prefetch(const void *x) {;} -#endif - -/* - * Simple doubly linked list implementation. - * - * Some of the internal functions ("__xxx") are useful when - * manipulating whole lists rather than single entries, as - * sometimes we already know the next/prev entries and we can - * generate better code by using them directly rather than - * using the generic single-entry routines. - */ - -struct list_head { - struct list_head *next, *prev; -}; - -#define LIST_HEAD_INIT(name) { &(name), &(name) } - -#define LIST_HEAD(name) \ - struct list_head name = LIST_HEAD_INIT(name) - -static inline void INIT_LIST_HEAD(struct list_head *list) -{ - list->next = list; - list->prev = list; -} - -/* - * Insert a new entry between two known consecutive entries. - * - * This is only for internal list manipulation where we know - * the prev/next entries already! - */ -static inline void __list_add(struct list_head *new, - struct list_head *prev, - struct list_head *next) -{ - next->prev = new; - new->next = next; - new->prev = prev; - prev->next = new; -} - -/** - * list_add - add a new entry - * @new: new entry to be added - * @head: list head to add it after - * - * Insert a new entry after the specified head. - * This is good for implementing stacks. - */ -static inline void list_add(struct list_head *new, struct list_head *head) -{ - __list_add(new, head, head->next); -} - -/** - * list_add_tail - add a new entry - * @new: new entry to be added - * @head: list head to add it before - * - * Insert a new entry before the specified head. - * This is useful for implementing queues. - */ -static inline void list_add_tail(struct list_head *new, struct list_head *head) -{ - __list_add(new, head->prev, head); -} - -/* - * Delete a list entry by making the prev/next entries - * point to each other. - * - * This is only for internal list manipulation where we know - * the prev/next entries already! - */ -static inline void __list_del(struct list_head *prev, struct list_head *next) -{ - next->prev = prev; - prev->next = next; -} - -/** - * list_del - deletes entry from list. - * @entry: the element to delete from the list. - * Note: list_empty() on entry does not return true after this, the entry is - * in an undefined state. - */ -static inline void list_del(struct list_head *entry) -{ - __list_del(entry->prev, entry->next); - entry->next = LIST_POISON1; - entry->prev = LIST_POISON2; -} - -/** - * list_replace - replace old entry by new one - * @old : the element to be replaced - * @new : the new element to insert - * - * If @old was empty, it will be overwritten. - */ -static inline void list_replace(struct list_head *old, - struct list_head *new) -{ - new->next = old->next; - new->next->prev = new; - new->prev = old->prev; - new->prev->next = new; -} - -static inline void list_replace_init(struct list_head *old, - struct list_head *new) -{ - list_replace(old, new); - INIT_LIST_HEAD(old); -} - -/** - * list_del_init - deletes entry from list and reinitialize it. - * @entry: the element to delete from the list. - */ -static inline void list_del_init(struct list_head *entry) -{ - __list_del(entry->prev, entry->next); - INIT_LIST_HEAD(entry); -} - -/** - * list_move - delete from one list and add as another's head - * @list: the entry to move - * @head: the head that will precede our entry - */ -static inline void list_move(struct list_head *list, struct list_head *head) -{ - __list_del(list->prev, list->next); - list_add(list, head); -} - -/** - * list_move_tail - delete from one list and add as another's tail - * @list: the entry to move - * @head: the head that will follow our entry - */ -static inline void list_move_tail(struct list_head *list, - struct list_head *head) -{ - __list_del(list->prev, list->next); - list_add_tail(list, head); -} - -/** - * list_is_last - tests whether @list is the last entry in list @head - * @list: the entry to test - * @head: the head of the list - */ -static inline int list_is_last(const struct list_head *list, - const struct list_head *head) -{ - return list->next == head; -} - -/** - * list_empty - tests whether a list is empty - * @head: the list to test. - */ -static inline int list_empty(const struct list_head *head) -{ - return head->next == head; -} - -/** - * list_empty_careful - tests whether a list is empty and not being modified - * @head: the list to test - * - * Description: - * tests whether a list is empty _and_ checks that no other CPU might be - * in the process of modifying either member (next or prev) - * - * NOTE: using list_empty_careful() without synchronization - * can only be safe if the only activity that can happen - * to the list entry is list_del_init(). Eg. it cannot be used - * if another CPU could re-list_add() it. - */ -static inline int list_empty_careful(const struct list_head *head) -{ - struct list_head *next = head->next; - return (next == head) && (next == head->prev); -} - -/** - * list_is_singular - tests whether a list has just one entry. - * @head: the list to test. - */ -static inline int list_is_singular(const struct list_head *head) -{ - return !list_empty(head) && (head->next == head->prev); -} - -static inline void __list_cut_position(struct list_head *list, - struct list_head *head, struct list_head *entry) -{ - struct list_head *new_first = entry->next; - list->next = head->next; - list->next->prev = list; - list->prev = entry; - entry->next = list; - head->next = new_first; - new_first->prev = head; -} - -/** - * list_cut_position - cut a list into two - * @list: a new list to add all removed entries - * @head: a list with entries - * @entry: an entry within head, could be the head itself - * and if so we won't cut the list - * - * This helper moves the initial part of @head, up to and - * including @entry, from @head to @list. You should - * pass on @entry an element you know is on @head. @list - * should be an empty list or a list you do not care about - * losing its data. - * - */ -static inline void list_cut_position(struct list_head *list, - struct list_head *head, struct list_head *entry) -{ - if (list_empty(head)) - return; - if (list_is_singular(head) && - (head->next != entry && head != entry)) - return; - if (entry == head) - INIT_LIST_HEAD(list); - else - __list_cut_position(list, head, entry); -} - -static inline void __list_splice(const struct list_head *list, - struct list_head *prev, - struct list_head *next) -{ - struct list_head *first = list->next; - struct list_head *last = list->prev; - - first->prev = prev; - prev->next = first; - - last->next = next; - next->prev = last; -} - -/** - * list_splice - join two lists, this is designed for stacks - * @list: the new list to add. - * @head: the place to add it in the first list. - */ -static inline void list_splice(const struct list_head *list, - struct list_head *head) -{ - if (!list_empty(list)) - __list_splice(list, head, head->next); -} - -/** - * list_splice_tail - join two lists, each list being a queue - * @list: the new list to add. - * @head: the place to add it in the first list. - */ -static inline void list_splice_tail(struct list_head *list, - struct list_head *head) -{ - if (!list_empty(list)) - __list_splice(list, head->prev, head); -} - -/** - * list_splice_init - join two lists and reinitialise the emptied list. - * @list: the new list to add. - * @head: the place to add it in the first list. - * - * The list at @list is reinitialised - */ -static inline void list_splice_init(struct list_head *list, - struct list_head *head) -{ - if (!list_empty(list)) { - __list_splice(list, head, head->next); - INIT_LIST_HEAD(list); - } -} - -/** - * list_splice_tail_init - join two lists and reinitialise the emptied list - * @list: the new list to add. - * @head: the place to add it in the first list. - * - * Each of the lists is a queue. - * The list at @list is reinitialised - */ -static inline void list_splice_tail_init(struct list_head *list, - struct list_head *head) -{ - if (!list_empty(list)) { - __list_splice(list, head->prev, head); - INIT_LIST_HEAD(list); - } -} - -/** - * list_entry - get the struct for this entry - * @ptr: the &struct list_head pointer. - * @type: the type of the struct this is embedded in. - * @member: the name of the list_struct within the struct. - */ -#define list_entry(ptr, type, member) \ - container_of(ptr, type, member) - -/** - * list_first_entry - get the first element from a list - * @ptr: the list head to take the element from. - * @type: the type of the struct this is embedded in. - * @member: the name of the list_struct within the struct. - * - * Note, that list is expected to be not empty. - */ -#define list_first_entry(ptr, type, member) \ - list_entry((ptr)->next, type, member) - -/** - * list_last_entry - get the last element from a list - * @ptr: the list head to take the element from. - * @type: the type of the struct this is embedded in. - * @member: the name of the list_struct within the struct. - * - * Note, that list is expected to be not empty. - */ -#define list_last_entry(ptr, type, member) \ - list_entry((ptr)->prev, type, member) - -/** - * list_for_each - iterate over a list - * @pos: the &struct list_head to use as a loop cursor. - * @head: the head for your list. - */ -#define list_for_each(pos, head) \ - for (pos = (head)->next; prefetch(pos->next), pos != (head); \ - pos = pos->next) - -/** - * __list_for_each - iterate over a list - * @pos: the &struct list_head to use as a loop cursor. - * @head: the head for your list. - * - * This variant differs from list_for_each() in that it's the - * simplest possible list iteration code, no prefetching is done. - * Use this for code that knows the list to be very short (empty - * or 1 entry) most of the time. - */ -#define __list_for_each(pos, head) \ - for (pos = (head)->next; pos != (head); pos = pos->next) - -/** - * list_for_each_prev - iterate over a list backwards - * @pos: the &struct list_head to use as a loop cursor. - * @head: the head for your list. - */ -#define list_for_each_prev(pos, head) \ - for (pos = (head)->prev; prefetch(pos->prev), pos != (head); \ - pos = pos->prev) - -/** - * list_for_each_safe - iterate over a list safe against removal of list entry - * @pos: the &struct list_head to use as a loop cursor. - * @n: another &struct list_head to use as temporary storage - * @head: the head for your list. - */ -#define list_for_each_safe(pos, n, head) \ - for (pos = (head)->next, n = pos->next; pos != (head); \ - pos = n, n = pos->next) - -/** - * list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry - * @pos: the &struct list_head to use as a loop cursor. - * @n: another &struct list_head to use as temporary storage - * @head: the head for your list. - */ -#define list_for_each_prev_safe(pos, n, head) \ - for (pos = (head)->prev, n = pos->prev; \ - prefetch(pos->prev), pos != (head); \ - pos = n, n = pos->prev) - -/** - * list_for_each_entry - iterate over list of given type - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - */ -#define list_for_each_entry(pos, head, member) \ - for (pos = list_entry((head)->next, typeof(*pos), member); \ - prefetch(pos->member.next), &pos->member != (head); \ - pos = list_entry(pos->member.next, typeof(*pos), member)) - -/** - * list_for_each_entry_reverse - iterate backwards over list of given type. - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - */ -#define list_for_each_entry_reverse(pos, head, member) \ - for (pos = list_entry((head)->prev, typeof(*pos), member); \ - prefetch(pos->member.prev), &pos->member != (head); \ - pos = list_entry(pos->member.prev, typeof(*pos), member)) - -/** - * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue() - * @pos: the type * to use as a start point - * @head: the head of the list - * @member: the name of the list_struct within the struct. - * - * Prepares a pos entry for use as a start point in list_for_each_entry_continue(). - */ -#define list_prepare_entry(pos, head, member) \ - ((pos) ? : list_entry(head, typeof(*pos), member)) - -/** - * list_for_each_entry_continue - continue iteration over list of given type - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Continue to iterate over list of given type, continuing after - * the current position. - */ -#define list_for_each_entry_continue(pos, head, member) \ - for (pos = list_entry(pos->member.next, typeof(*pos), member); \ - prefetch(pos->member.next), &pos->member != (head); \ - pos = list_entry(pos->member.next, typeof(*pos), member)) - -/** - * list_for_each_entry_continue_reverse - iterate backwards from the given point - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Start to iterate over list of given type backwards, continuing after - * the current position. - */ -#define list_for_each_entry_continue_reverse(pos, head, member) \ - for (pos = list_entry(pos->member.prev, typeof(*pos), member); \ - prefetch(pos->member.prev), &pos->member != (head); \ - pos = list_entry(pos->member.prev, typeof(*pos), member)) - -/** - * list_for_each_entry_from - iterate over list of given type from the current point - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Iterate over list of given type, continuing from current position. - */ -#define list_for_each_entry_from(pos, head, member) \ - for (; prefetch(pos->member.next), &pos->member != (head); \ - pos = list_entry(pos->member.next, typeof(*pos), member)) - -/** - * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry - * @pos: the type * to use as a loop cursor. - * @n: another type * to use as temporary storage - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - */ -#define list_for_each_entry_safe(pos, n, head, member) \ - for (pos = list_entry((head)->next, typeof(*pos), member), \ - n = list_entry(pos->member.next, typeof(*pos), member); \ - &pos->member != (head); \ - pos = n, n = list_entry(n->member.next, typeof(*n), member)) - -/** - * list_for_each_entry_safe_continue - * @pos: the type * to use as a loop cursor. - * @n: another type * to use as temporary storage - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Iterate over list of given type, continuing after current point, - * safe against removal of list entry. - */ -#define list_for_each_entry_safe_continue(pos, n, head, member) \ - for (pos = list_entry(pos->member.next, typeof(*pos), member), \ - n = list_entry(pos->member.next, typeof(*pos), member); \ - &pos->member != (head); \ - pos = n, n = list_entry(n->member.next, typeof(*n), member)) - -/** - * list_for_each_entry_safe_from - * @pos: the type * to use as a loop cursor. - * @n: another type * to use as temporary storage - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Iterate over list of given type from current point, safe against - * removal of list entry. - */ -#define list_for_each_entry_safe_from(pos, n, head, member) \ - for (n = list_entry(pos->member.next, typeof(*pos), member); \ - &pos->member != (head); \ - pos = n, n = list_entry(n->member.next, typeof(*n), member)) - -/** - * list_for_each_entry_safe_reverse - * @pos: the type * to use as a loop cursor. - * @n: another type * to use as temporary storage - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Iterate backwards over list of given type, safe against removal - * of list entry. - */ -#define list_for_each_entry_safe_reverse(pos, n, head, member) \ - for (pos = list_entry((head)->prev, typeof(*pos), member), \ - n = list_entry(pos->member.prev, typeof(*pos), member); \ - &pos->member != (head); \ - pos = n, n = list_entry(n->member.prev, typeof(*n), member)) - -/* - * Double linked lists with a single pointer list head. - * Mostly useful for hash tables where the two pointer list head is - * too wasteful. - * You lose the ability to access the tail in O(1). - */ - -struct hlist_head { - struct hlist_node *first; -}; - -struct hlist_node { - struct hlist_node *next, **pprev; -}; - -#define HLIST_HEAD_INIT { .first = NULL } -#define HLIST_HEAD(name) struct hlist_head name = { .first = NULL } -#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL) -static inline void INIT_HLIST_NODE(struct hlist_node *h) -{ - h->next = NULL; - h->pprev = NULL; -} - -static inline int hlist_unhashed(const struct hlist_node *h) -{ - return !h->pprev; -} - -static inline int hlist_empty(const struct hlist_head *h) -{ - return !h->first; -} - -static inline void __hlist_del(struct hlist_node *n) -{ - struct hlist_node *next = n->next; - struct hlist_node **pprev = n->pprev; - *pprev = next; - if (next) - next->pprev = pprev; -} - -static inline void hlist_del(struct hlist_node *n) -{ - __hlist_del(n); - n->next = LIST_POISON1; - n->pprev = LIST_POISON2; -} - -static inline void hlist_del_init(struct hlist_node *n) -{ - if (!hlist_unhashed(n)) { - __hlist_del(n); - INIT_HLIST_NODE(n); - } -} - -static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) -{ - struct hlist_node *first = h->first; - n->next = first; - if (first) - first->pprev = &n->next; - h->first = n; - n->pprev = &h->first; -} - -/* next must be != NULL */ -static inline void hlist_add_before(struct hlist_node *n, - struct hlist_node *next) -{ - n->pprev = next->pprev; - n->next = next; - next->pprev = &n->next; - *(n->pprev) = n; -} - -static inline void hlist_add_after(struct hlist_node *n, - struct hlist_node *next) -{ - next->next = n->next; - n->next = next; - next->pprev = &n->next; - - if(next->next) - next->next->pprev = &next->next; -} - -#define hlist_entry(ptr, type, member) container_of(ptr,type,member) - -#define hlist_for_each(pos, head) \ - for (pos = (head)->first; pos && ({ prefetch(pos->next); 1; }); \ - pos = pos->next) - -#define hlist_for_each_safe(pos, n, head) \ - for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \ - pos = n) - -/** - * hlist_for_each_entry - iterate over list of given type - * @tpos: the type * to use as a loop cursor. - * @pos: the &struct hlist_node to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the hlist_node within the struct. - */ -#define hlist_for_each_entry(tpos, pos, head, member) \ - for (pos = (head)->first; \ - pos && ({ prefetch(pos->next); 1;}) && \ - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = pos->next) - -/** - * hlist_for_each_entry_continue - iterate over a hlist continuing after current point - * @tpos: the type * to use as a loop cursor. - * @pos: the &struct hlist_node to use as a loop cursor. - * @member: the name of the hlist_node within the struct. - */ -#define hlist_for_each_entry_continue(tpos, pos, member) \ - for (pos = (pos)->next; \ - pos && ({ prefetch(pos->next); 1;}) && \ - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = pos->next) - -/** - * hlist_for_each_entry_from - iterate over a hlist continuing from current point - * @tpos: the type * to use as a loop cursor. - * @pos: the &struct hlist_node to use as a loop cursor. - * @member: the name of the hlist_node within the struct. - */ -#define hlist_for_each_entry_from(tpos, pos, member) \ - for (; pos && ({ prefetch(pos->next); 1;}) && \ - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = pos->next) - -/** - * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry - * @tpos: the type * to use as a loop cursor. - * @pos: the &struct hlist_node to use as a loop cursor. - * @n: another &struct hlist_node to use as temporary storage - * @head: the head for your list. - * @member: the name of the hlist_node within the struct. - */ -#define hlist_for_each_entry_safe(tpos, pos, n, head, member) \ - for (pos = (head)->first; \ - pos && ({ n = pos->next; 1; }) && \ - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = n) - -#endif diff --git a/src/lib/fy-parse.c b/src/lib/fy-parse.c index e756e37..99fed7c 100644 --- a/src/lib/fy-parse.c +++ b/src/lib/fy-parse.c @@ -135,6 +135,37 @@ err_out: return -1; } +ssize_t fy_parse_estimate_queued_input_size(struct fy_parser *fyp) +{ + struct fy_input *fyi; + ssize_t size, total; + + if (!fyp) + return 0; + + total = 0; + for (fyi = fy_input_list_head(&fyp->queued_inputs); fyi; fyi = fy_input_next(&fyp->queued_inputs, fyi)) { + if (fyi->state != FYIS_QUEUED) + continue; + + size = fy_input_estimate_queued_size(fyi); + if (size < 0) + return -1; + + /* something that's indetermined */ + if (size == SSIZE_MAX) + return SSIZE_MAX; + + total += size; + + /* roll-over check */ + if (total < 0) + return SSIZE_MAX; + } + + return total; +} + static inline void fy_token_queue_epilogue(struct fy_parser *fyp, struct fy_token *fyt) { @@ -452,6 +483,9 @@ int fy_reset_document_state(struct fy_parser *fyp) fyp->flow = FYFT_NONE; fy_parse_flow_list_recycle_all(fyp, &fyp->flow_stack); + /* and reset all aliases (TODO look for cross document aliases) */ + fy_parse_streaming_aliases_reset(fyp); + return 0; err_out: @@ -758,6 +792,7 @@ int fy_parse_setup(struct fy_parser *fyp, const struct fy_parse_cfg *cfg) fy_indent_list_init(&fyp->recycled_indent); fyp->indent = -2; fyp->indent_line = -1; + fyp->starting_indent = -1; fyp->generated_block_map = false; fyp->last_was_comma = false; @@ -782,6 +817,10 @@ int fy_parse_setup(struct fy_parser *fyp, const struct fy_parse_cfg *cfg) fyp->pending_complex_key_column = -1; fyp->last_block_mapping_key_line = -1; + fy_streaming_alias_list_init(&fyp->streaming_aliases); + fy_streaming_alias_list_init(&fyp->recycled_streaming_alias); + fy_eventp_list_init(&fyp->mks.args); + fyp->suppress_recycling = !!(fyp->cfg.flags & FYPCF_DISABLE_RECYCLING) || (getenv("FY_VALGRIND") && !getenv("FY_VALGRIND_RECYCLING")); @@ -814,6 +853,8 @@ void fy_parse_cleanup(struct fy_parser *fyp) struct fy_eventp *fyep; struct fy_token *fyt; + fy_parse_streaming_aliases_reset(fyp); + fy_input_unref(fyp->last_event_handle.fyi); fy_atom_reset(&fyp->last_event_handle); @@ -826,6 +867,7 @@ void fy_parse_cleanup(struct fy_parser *fyp) fy_parse_parse_state_log_list_recycle_all(fyp, &fyp->state_stack); fy_parse_flow_list_recycle_all(fyp, &fyp->flow_stack); + fy_parse_streaming_alias_list_recycle_all(fyp, &fyp->streaming_aliases); fy_token_unref_rl(fyp->recycled_token_list, fyp->stream_end_token); @@ -845,6 +887,7 @@ void fy_parse_cleanup(struct fy_parser *fyp) fy_parse_simple_key_vacuum(fyp); fy_parse_parse_state_log_vacuum(fyp); fy_parse_flow_vacuum(fyp); + fy_parse_streaming_alias_vacuum(fyp); /* free the recycled events */ while ((fyep = fy_eventp_list_pop(&fyp->recycled_eventp)) != NULL) { @@ -1290,6 +1333,8 @@ int fy_push_indent(struct fy_parser *fyp, int indent, bool generated_block_map, fyp->parent_indent = fyp->indent; fyp->indent = indent; fyp->indent_line = indent_line; + if (fyp->parse_block_only && fyp->starting_indent < 0) + fyp->starting_indent = indent; fyp->generated_block_map = generated_block_map; fyp_scan_debug(fyp, "push_indent %d -> %d - generated_block_map=%s\n", @@ -1685,7 +1730,7 @@ int fy_fetch_stream_end(struct fy_parser *fyp) int rc; /* only reset the stream in regular mode */ - if (!fyp->parse_flow_only) + if (!fyp->parse_flow_only && !fyp->parse_block_only) fy_reader_stream_end(fyp->reader); fy_remove_all_simple_keys(fyp); @@ -2211,9 +2256,12 @@ int fy_fetch_flow_collection_mark_start(struct fy_parser *fyp, int c) fyp_error_check(fyp, fyt, err_out, "fy_token_queue_simple() failed"); - rc = fy_save_simple_key_mark(fyp, &skm, type, NULL); - fyp_error_check(fyp, !rc, err_out_rc, - "fy_save_simple_key_mark() failed"); + /* json does not have complex keys */ + if (!fyp_json_mode(fyp)) { + rc = fy_save_simple_key_mark(fyp, &skm, type, NULL); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_save_simple_key_mark() failed"); + } /* increase flow level */ fyp->flow_level++; @@ -3358,6 +3406,9 @@ int fy_fetch_block_scalar(struct fy_parser *fyp, bool is_literal, int c) line_length += fy_utf8_width(c); } + /* keep the variables without warning */ + (void)trailing_ws; + (void)leading_ws; if (doc_start_end_detected) break; @@ -3564,6 +3615,7 @@ int fy_fetch_block_scalar(struct fy_parser *fyp, bool is_literal, int c) handle.fws_mode = fyp_fws_mode(fyp); handle.tabsize = fyp_tabsize(fyp); handle.ends_with_eof = ends_with_eof; + handle.is_merge_key = false; #ifdef ATOM_SIZE_CHECK tlength = fy_atom_format_text_length(&handle); @@ -4000,6 +4052,7 @@ int fy_reader_fetch_flow_scalar_handle(struct fy_reader *fyr, int c, int indent, handle->fws_mode = fy_reader_flow_ws_mode(fyr); handle->tabsize = fy_reader_tabsize(fyr); handle->ends_with_eof = false; /* flow scalars never end with EOF and be valid */ + handle->is_merge_key = false; /* skip over block scalar end */ fy_reader_advance_by(fyr, 1); @@ -4035,7 +4088,7 @@ int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, int indent bool has_leading_blanks; bool last_ptr; struct fy_mark mark, last_mark; - bool is_multiline, has_lb, has_ws, ends_with_eof; + bool is_multiline, has_lb, has_ws, ends_with_eof, is_merge_key; bool has_json_esc; #ifdef ATOM_SIZE_CHECK size_t tlength; @@ -4065,6 +4118,9 @@ int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, int indent flow_level == 0 || !(c == '-' && fy_utf8_strchr(",[]{}", fy_reader_peek_at(fyr, 1))), err_out, "plain scalar cannot start with '%c' followed by ,[]{} (in flow context)", c); + /* we can prime is_merge_key here */ + is_merge_key = c == '<' && fy_reader_peek_at(fyr, 1) == '<'; + fy_reader_get_mark(fyr, &mark); fy_reader_fill_atom_start(fyr, handle); @@ -4289,6 +4345,7 @@ int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, int indent handle->fws_mode = fy_reader_flow_ws_mode(fyr); handle->tabsize = fy_reader_tabsize(fyr); handle->ends_with_eof = ends_with_eof; + handle->is_merge_key = is_merge_key && length == 2; #ifdef ATOM_SIZE_CHECK tlength = fy_atom_format_text_length(handle); @@ -4626,7 +4683,7 @@ void fy_reader_skip_ws_lb(struct fy_reader *fyr) } s++; } else { - c = fy_utf8_get(s, (int)(e - s), &w); + c = fy_utf8_get(s, (size_t)(e - s), &w); if (c == FYUG_PARTIAL) { /* get the width (from the first octet */ @@ -4805,6 +4862,13 @@ int fy_fetch_tokens(struct fy_parser *fyp) "fy_scan_to_next_token() failed"); if (fyp_block_mode(fyp)) { + if (fyp->parse_block_only && fyp->starting_indent >= 0 && + fyp_column(fyp) < fyp->starting_indent) { + rc = fy_fetch_stream_end(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_stream_end() failed"); + return 0; + } rc = fy_parse_unroll_indent(fyp, fyp_column(fyp)); fyp_error_check(fyp, !rc, err_out_rc, "fy_parse_unroll_indent() failed"); @@ -5112,7 +5176,7 @@ struct fy_token *fy_scan_peek(struct fy_parser *fyp) fyp_scan_debug(fyp, "setting stream_end_produced to true"); fyp->stream_end_produced = true; - if (!fyp->parse_flow_only) { + if (!fyp->parse_flow_only && !fyp->parse_block_only) { rc = fy_reader_input_done(fyp->reader); fyp_error_check(fyp, !rc, err_out, "fy_parse_input_done() failed"); @@ -5517,6 +5581,7 @@ int fy_parse_stream_start(struct fy_parser *fyp) { fyp->indent = -2; fyp->indent_line = -1; + fyp->starting_indent = -1; fyp->generated_block_map = false; fyp->last_was_comma = false; fyp->flow = FYFT_NONE; @@ -6846,16 +6911,29 @@ err_out_rc: struct fy_event *fy_parser_parse(struct fy_parser *fyp) { - struct fy_eventp *fyep; + struct fy_eventp *fyep, *fyep2; enum fy_composer_return ret; if (!fyp) return NULL; + if ((fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT) && fy_reader_get_mode(fyp->reader) != fyrm_json) { + fyep = fy_parser_parse_resolve_prolog(fyp); + if (fyep) + return &fyep->e; + } + fyep = fy_parse_private(fyp); if (!fyep) return NULL; + if ((fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT) && fy_reader_get_mode(fyp->reader) != fyrm_json) { + fyep2 = fy_parser_parse_resolve_epilog(fyp, fyep); + if (!fyep2) + return NULL; + fyep = fyep2; + } + if (fyp->fyc) { ret = fy_composer_process_event(fyp->fyc, &fyep->e); if (ret == FYCR_ERROR) { @@ -6939,14 +7017,18 @@ static const struct fy_composer_ops parser_composer_ops = { int fy_parse_set_composer(struct fy_parser *fyp, fy_parse_composer_cb cb, void *userdata) { struct fy_composer_cfg ccfg; + bool can_change_composer; if (!fyp) return -1; + can_change_composer = fyp->state == FYPS_NONE || + fyp->state == FYPS_END || + fyp->state == FYPS_DOCUMENT_START; + /* must not be in the middle of something */ - fyp_error_check(fyp, fyp->state == FYPS_NONE || fyp->state == FYPS_END, - err_out, "cannot change composer state at state '%s'", - state_txt[fyp->state]); + fyp_error_check(fyp, can_change_composer, err_out, + "cannot change composer state at state '%s'", state_txt[fyp->state]); /* clear */ if (!cb) { @@ -7106,7 +7188,8 @@ int fy_parse_compose(struct fy_parser *fyp, fy_parse_composer_cb cb, void *userd rc_out = 0; /* reset the parser; the composer clear must always succeed */ - fy_parser_reset(fyp); + if (ret == FYCR_ERROR) + fy_parser_reset(fyp); /* clear composer */ rc = fy_parse_set_composer(fyp, NULL, NULL); @@ -7118,3 +7201,818 @@ int fy_parse_compose(struct fy_parser *fyp, fy_parse_composer_cb cb, void *userd err_out: return -1; } + +struct fy_streaming_alias * +fy_parser_streaming_alias_lookup(struct fy_parser *fyp, struct fy_token *fyt_anchor) +{ + struct fy_streaming_alias *fysa; + + /* XXX todo hashing */ + for (fysa = fy_streaming_alias_list_head(&fyp->streaming_aliases); fysa != NULL; fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa)) { + if (fy_token_cmp(fyt_anchor, fysa->anchor) == 0) + return fysa; + } + return NULL; +} + +struct fy_streaming_alias * +fy_parser_streaming_alias_lookup_pivot(struct fy_parser *fyp, struct fy_streaming_alias *fysa_pivot, struct fy_token *fyt_anchor) +{ + struct fy_streaming_alias *fysa; + + assert(fysa_pivot); + /* at first do forwards starting at the pivot */ + for (fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa_pivot); fysa != NULL; fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa)) { + if (fysa->anchor && fy_token_cmp(fyt_anchor, fysa->anchor) == 0) + return fysa; + } + /* not found? do backwards (more recent) */ + for (fysa = fy_streaming_alias_prev(&fyp->streaming_aliases, fysa_pivot); fysa != NULL; fysa = fy_streaming_alias_prev(&fyp->streaming_aliases, fysa)) { + if (fysa->anchor && fy_token_cmp(fyt_anchor, fysa->anchor) == 0) + return fysa; + } + return NULL; +} + +struct fy_streaming_alias * +fy_parse_streaming_alias_create(struct fy_parser *fyp, struct fy_token *fyt_anchor) +{ + struct fy_streaming_alias *fysa; + + fysa = fy_parse_streaming_alias_alloc(fyp); + fyp_error_check(fyp, fysa != NULL, err_out, + "fy_parse_streaming_alias_alloc() failed!"); + fysa->anchor = fyt_anchor; + fysa->collecting = false; + fysa->mapping_nest = 0; + fysa->sequence_nest = 0; + fy_eventp_list_init(&fysa->events); + + return fysa; + +err_out: + return NULL; +} + +void +fy_parse_streaming_alias_clean(struct fy_parser *fyp, struct fy_streaming_alias *fysa) +{ + struct fy_eventp *fyep; + + if (!fyp || !fysa) + return; + + fy_token_unref(fysa->anchor); + fysa->anchor = NULL; + fysa->collecting = false; + fysa->mapping_nest = 0; + fysa->sequence_nest = 0; + while ((fyep = fy_eventp_list_pop(&fysa->events)) != NULL) + fy_parse_eventp_recycle(fyp, fyep); +} + +void fy_parse_streaming_aliases_reset(struct fy_parser *fyp) +{ + struct fy_streaming_alias *fysa; + struct fy_eventp *fyep; + + if (!fyp) + return; + + while ((fysa = fy_streaming_alias_list_pop(&fyp->streaming_aliases)) != NULL) { + fy_parse_streaming_alias_clean(fyp, fysa); + fy_parse_streaming_alias_recycle(fyp, fysa); + } + if (fyp->sas.stack != NULL && fyp->sas.stack != fyp->sas.local) + free(fyp->sas.stack); + memset(&fyp->sas, 0, sizeof(fyp->sas)); + if (fyp->cts.stack != NULL && fyp->cts.stack != fyp->cts.local) + free(fyp->cts.stack); + memset(&fyp->cts, 0, sizeof(fyp->cts)); + + while ((fyep = fy_eventp_list_pop(&fyp->mks.args)) != NULL) + fy_parse_eventp_recycle(fyp, fyep); + memset(&fyp->mks, 0, sizeof(fyp->mks)); + fy_eventp_list_init(&fyp->mks.args); +} + +struct fy_streaming_alias_state * +fy_parse_streaming_alias_state_push(struct fy_parser *fyp, struct fy_streaming_alias *fysa) +{ + struct fy_streaming_alias_state *new_stack; + struct fy_streaming_alias_state *fysas; + + /* if first one, just point to the local area */ + if (fyp->sas.stack == NULL) { + fyp->sas.stack = fyp->sas.local; + fyp->sas.top = 0; + fyp->sas.alloc = ARRAY_SIZE(fyp->sas.local); + } + /* grow the stack if required */ + if (fyp->sas.top >= fyp->sas.alloc) { + assert(fyp->sas.alloc > 0); + assert(fyp->sas.stack); + new_stack = malloc(sizeof(*new_stack) * fyp->sas.alloc * 2); + fyp_error_check(fyp, new_stack != NULL, err_out, + "realloc() failed!"); + memcpy(new_stack, fyp->sas.stack, sizeof(*new_stack) * fyp->sas.alloc); + fyp->sas.alloc *= 2; + if (fyp->sas.stack != fyp->sas.local) + free(fyp->sas.stack); + fyp->sas.stack = new_stack; + } + /* finally push it */ + fysas = &fyp->sas.stack[fyp->sas.top++]; + + fysas->fysa = fysa; + fysas->next = fy_eventp_list_head(&fysa->events); + + return fysas; +err_out: + return NULL; +} + +struct fy_streaming_alias_state * +fy_parse_streaming_alias_state_pop(struct fy_parser *fyp) +{ + struct fy_streaming_alias_state *fysas; + + if (fyp->sas.top <= 0) + return NULL; + + fysas = &fyp->sas.stack[--fyp->sas.top]; + fysas->fysa = NULL; + fysas->next = NULL; + if (fyp->sas.top <= 0) + return NULL; + return &fyp->sas.stack[fyp->sas.top - 1]; +} + +struct fy_streaming_alias_state * +fy_parse_streaming_alias_state_top(struct fy_parser *fyp) +{ + if (fyp->sas.top <= 0) + return NULL; + return &fyp->sas.stack[fyp->sas.top - 1]; +} + +void +fy_parse_streaming_alias_state_next_event(struct fy_parser *fyp) +{ + struct fy_streaming_alias_state *fysas; + + fysas = fy_parse_streaming_alias_state_top(fyp); + if (!fysas) + return; + + fysas->next = fy_eventp_next(&fysas->fysa->events, fysas->next); + while (!fysas->next) { + fysas = fy_parse_streaming_alias_state_pop(fyp); + if (!fysas) + break; + } +} + +static inline int +fy_parse_streaming_alias_collection_state_set_top(struct fy_parser *fyp, int cts) +{ + int pos, off, old_cts; + + if (fyp->cts.top < 2) + return -1; + + cts &= 3; /* keep 2 bits */ + + /* get position and offset */ + pos = (fyp->cts.top - 2) / (sizeof(*fyp->cts.stack) * 8); + off = (fyp->cts.top - 2) % (sizeof(*fyp->cts.stack) * 8); + + old_cts = (int)((fyp->cts.stack[pos] >> off) & 3); + + /* clear the bits first */ + fyp->cts.stack[pos] &= ~(3 << off); + /* set the state */ + fyp->cts.stack[pos] |= (cts & 3) << off; + + return old_cts; +} + +static inline int +fy_parse_streaming_alias_collection_state_top(struct fy_parser *fyp) +{ + int pos, off; + + if (fyp->cts.top < 2) + return -1; + + /* get position and offset */ + pos = (fyp->cts.top - 2) / (sizeof(*fyp->cts.stack) * 8); + off = (fyp->cts.top - 2) % (sizeof(*fyp->cts.stack) * 8); + + return (int)((fyp->cts.stack[pos] >> off) & 3); +} + +static inline bool +fy_parse_streaming_alias_collection_state_in_map_key(struct fy_parser *fyp) +{ + int cts = fy_parse_streaming_alias_collection_state_top(fyp); + return cts == FYCTS_MAP_KEY; +} + +static inline void +fy_parse_streaming_alias_collection_state_toggle_map(struct fy_parser *fyp) +{ + int cts = fy_parse_streaming_alias_collection_state_top(fyp); + if (cts >= 0 && (cts & FYCTS_MAP)) + fy_parse_streaming_alias_collection_state_set_top(fyp, cts ^ FYCTS_MAP_TOGGLE); +} + +int +fy_parse_streaming_alias_collection_state_push(struct fy_parser *fyp, int cts) +{ + uint32_t *new_stack; + int size; + + assert(cts >= 0); + + /* if first one, just point to the local area */ + if (fyp->cts.stack == NULL) { + fyp->cts.stack = fyp->cts.local; + fyp->cts.top = 0; + fyp->cts.alloc = sizeof(fyp->cts.local) * 8; /* in bits */ + } + /* grow the stack if required */ + if (fyp->cts.top + 2 >= fyp->cts.alloc) { + assert(fyp->cts.alloc > 0); + assert(fyp->cts.stack); + size = fyp->cts.alloc / 8; /* convert to bytes */ + new_stack = malloc(size * 2); + fyp_error_check(fyp, new_stack != NULL, err_out, + "realloc() failed!"); + memcpy(new_stack, fyp->cts.stack, size); + memset((uint8_t *)new_stack + size, 0, size); + fyp->cts.alloc *= 2; + if (fyp->cts.stack != fyp->cts.local) + free(fyp->cts.stack); + fyp->cts.stack = new_stack; + } + + fyp->cts.top += 2; + fy_parse_streaming_alias_collection_state_set_top(fyp, cts); + return cts; +err_out: + return -1; +} + +int +fy_parse_streaming_alias_collection_state_pop(struct fy_parser *fyp) +{ + int ret; + + ret = fy_parse_streaming_alias_collection_state_top(fyp); + if (ret >= 0) + fyp->cts.top -= 2; + return ret; +} + +static int +fy_parser_event_resolve_hook_collect(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_eventp *fyep_clone = NULL; + struct fy_streaming_alias *fysa; + long map_add, seq_add; + + if (!fyp || !fyep) + return -1; + + map_add = seq_add = 0; + switch (fyep->e.type) { + case FYET_MAPPING_START: + map_add = 1; + break; + case FYET_MAPPING_END: + map_add = -1; + break; + case FYET_SEQUENCE_START: + seq_add = 1; + break; + case FYET_SEQUENCE_END: + seq_add = -1; + break; + default: + break; + } + + /* for all streaming aliases that are collecting... */ + for (fysa = fy_streaming_alias_list_head(&fyp->streaming_aliases); fysa != NULL; fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa)) { + if (!fysa->collecting) + continue; + + /* clone event, stripping the anchors */ + fyep_clone = fy_parse_eventp_clone(fyp, fyep, true); + fyp_error_check(fyp, fyep_clone != NULL, err_out, + "fy_parse_eventp_clone() failed!"); + + /* add to the list */ + fy_eventp_list_add_tail(&fysa->events, fyep_clone); + fyep_clone = NULL; + + /* the scalar will be collected every time */ + fysa->mapping_nest += map_add; + fysa->sequence_nest += seq_add; + + if (fysa->mapping_nest == 0 && fysa->sequence_nest == 0) + fysa->collecting = false; + } + + return 0; + +err_out: + fy_parse_eventp_recycle(fyp, fyep_clone); + fyp->stream_error = true; + return -1; +} + +struct fy_eventp *fy_parser_event_resolve_hook_alias(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_token *fyt_anchor = NULL; + struct fy_streaming_alias *fysa; + struct fy_streaming_alias_state *fysas; + + if (!fyp || !fyep) + return NULL; + + /* if it's not an alias, continue normal processing */ + if (fyep->e.type != FYET_ALIAS) + return fyep; + + /* it's an alias, get the token and dispose the event */ + fyt_anchor = fyep->e.alias.anchor; + fyep->e.alias.anchor = NULL; + fy_parse_eventp_recycle(fyp, fyep); + fyep = NULL; + + /* start generation of alias events */ + fysa = fy_parser_streaming_alias_lookup(fyp, fyt_anchor); + + /* we don't support forward aliases for now like we do in document mode ... */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt_anchor, FYEM_PARSE, + fysa, err_out, + "alias not found; note streaming mode does not support forward alias references"); + + /* if this is a reference to a collecting alias, then it's recursive */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt_anchor, FYEM_PARSE, + !fysa->collecting, err_out, + "Recursive alias reference detected"); + + fy_token_unref(fyt_anchor); + fyt_anchor = NULL; + + fysas = fy_parse_streaming_alias_state_push(fyp, fysa); + fyp_error_check(fyp, fysas, err_out, + "fy_parse_streaming_alias_push() failed!"); + assert(fysas->next); + + fyep = fy_parse_eventp_clone(fyp, fysas->next, true); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parse_eventp_clone() failed!"); + + /* advance event */ + fy_parse_streaming_alias_state_next_event(fyp); + + return fyep; + +err_out: + fy_token_unref(fyt_anchor); + fyp->stream_error = true; + return NULL; +} + +struct fy_eventp *fy_parser_event_resolve_hook_anchor_start(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_token *fyt_anchor = NULL; + struct fy_streaming_alias *fysa = NULL; + + if (!fyep) + return NULL; + + /* is there an anchor?, is so get it and clear it */ + fyt_anchor = fy_event_get_and_clear_anchor_token(&fyep->e); + if (!fyt_anchor) + return fyep; + + fysa = fy_parse_streaming_alias_create(fyp, fyt_anchor); + fyp_error_check(fyp, fysa != NULL, err_out, + "fy_parser_streaming_alias_create() failed!"); + fyt_anchor = NULL; + + /* we just mark that we're collecting + * the collection hook will pick up the first event + * including the scalar one + */ + fysa->collecting = true; + fysa->sequence_nest = 0; + fysa->mapping_nest = 0; + + /* always add to the head of the list (overrides what follows with the same name) */ + fy_streaming_alias_list_add(&fyp->streaming_aliases, fysa); + + return fyep; + +err_out: + fy_token_unref(fyt_anchor); + fy_parse_streaming_alias_clean(fyp, fysa); + fy_parse_streaming_alias_recycle(fyp, fysa); + fy_parse_eventp_recycle(fyp, fyep); + return NULL; +} + +static struct fy_document * +fy_parser_get_merge_key_argument(struct fy_parser *fyp, struct fy_document_builder *fydb, struct fy_node *fyn) +{ + struct fy_streaming_alias *fysa; + struct fy_document *fyd_new = NULL; + struct fy_document_state *fyds = NULL; + int rc; + + if (!fy_node_is_alias(fyn)) + return NULL; + + /* get the alias */ + fysa = fy_parser_streaming_alias_lookup(fyp, fyn->scalar); + FYP_TOKEN_ERROR_CHECK(fyp, fyn->scalar, FYEM_PARSE, + fysa, err_out, + "Unable to find merge key argument alias"); + + /* if this is a reference to a collecting alias, then it's recursive */ + FYP_TOKEN_ERROR_CHECK(fyp, fyn->scalar, FYEM_PARSE, + !fysa->collecting, err_out, + "merge key recursive alias reference detected"); + + fyds = fy_parser_get_document_state(fyp); + fyp_error_check(fyp, fyds, err_out, + "fy_parser_get_document_state() failed"); + + rc = fy_document_builder_set_in_document(fydb, fyds, true); + fyp_error_check(fyp, !rc, err_out, + "fy_document_builder_set_target_document() failed\n"); + + /* build a document from the events of the alias (must be the same) */ + fyd_new = fy_document_builder_event_document(fydb, &fysa->events); + fyp_error_check(fyp, fyd_new, err_out, + "fy_document_builder_event_document() failed\n"); + + FYP_TOKEN_ERROR_CHECK(fyp, fyn->scalar, FYEM_PARSE, + fy_node_is_mapping(fyd_new->root), err_out, + "alias argument does not refer to mapping"); + + return fyd_new; + +err_out: + fy_document_destroy(fyd_new); + return NULL; +} + +static struct fy_document * +fy_parser_get_merge_key_document(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_event *fye; + struct fy_document_builder_cfg cfg; + struct fy_document_builder *fydb = NULL; + struct fy_document *fyd = NULL, *fyd_arg = NULL; + struct fy_document_state *fyds; + struct fy_node *fyn = NULL, *fyn_src = NULL; + struct fy_node *new_root = NULL; + struct fy_node_pair *fynp, *fynp_exists; + void *iters, *iterm; + int rc; + + fye = &fyep->e; + + memset(&cfg, 0, sizeof(cfg)); + cfg.parse_cfg = fyp->cfg; + cfg.diag = fy_diag_ref(fyp->diag); + + fydb = fy_document_builder_create(&cfg); + fyp_error_check(fyp, fydb, err_out, + "fy_document_builder_create() failed\n"); + + fyds = fy_parser_get_document_state(fyp); + /* start with this document state */ + fyds = fy_parser_get_document_state(fyp); + rc = fy_document_builder_set_in_document(fydb, fyds, true); + fyp_error_check(fyp, !rc, err_out, + "fy_document_builder_set_in_document() failed\n"); + + fyd = fy_document_builder_load_document(fydb, fyp); + fyp_error_check(fyp, fyd, err_out, + "fy_document_builder_load_document() failed\n"); + + FYP_TOKEN_ERROR_CHECK(fyp, fy_event_get_token(fye), FYEM_PARSE, + fyd->root, err_out, + "merge key arguments must exist"); + + /* single alias, replace */ + if (fy_node_is_alias(fyd->root)) { + fyd_arg = fy_parser_get_merge_key_argument(fyp, fydb, fyd->root); + fyp_error_check(fyp, fyd_arg, err_out, + "fy_parser_get_merge_key_argument() failed\n"); + fy_node_free(fyd->root); + fyd->root = fy_node_copy(fyd, fyd_arg->root); + + fy_document_destroy(fyd_arg); + fyd_arg = NULL; + + } else if (fy_node_is_mapping(fyd->root)) { + + /* nothing, already a mapping */ + + } else if (fy_node_is_sequence(fyd->root)) { + + new_root = fy_node_create_mapping(fyd); + fyp_error_check(fyp, new_root, err_out, + "fy_node_create_mapping() failed\n"); + + iters = NULL; + while ((fyn_src = fy_node_sequence_iterate(fyd->root, &iters)) != NULL) { + + if (fy_node_is_alias(fyn_src)) { + fyd_arg = fy_parser_get_merge_key_argument(fyp, fydb, fyn_src); + fyp_error_check(fyp, fyd_arg, err_out, + "fy_parser_get_merge_key_argument() failed\n"); + + fyn = fy_node_copy(fyd, fyd_arg->root); + fyp_error_check(fyp, fyn, err_out, + "fy_node_copy() failed\n"); + fy_document_destroy(fyd_arg); + fyd_arg = NULL; + + } else { + FYP_NODE_ERROR_CHECK(fyp, fyn_src, FYEM_PARSE, + fy_node_is_mapping(fyn_src), err_out, + "merge key argument is not a mapping or an alias to such"); + fyn = fy_node_copy(fyd, fyn_src); + fyp_error_check(fyp, fyn, err_out, + "fy_node_copy() failed\n"); + } + + iterm = NULL; + while ((fynp = fy_node_mapping_iterate(fyn, &iterm)) != NULL) { + fynp_exists = fy_node_mapping_lookup_pair(new_root, fynp->key); + + /* do not override existing keys */ + if (fynp_exists) + continue; + + /* add it to the mapping */ + rc = fy_node_mapping_append(new_root, fy_node_copy(fyd, fynp->key), fy_node_copy(fyd, fynp->value)); + fyp_error_check(fyp, !rc, err_out, + "fy_node_mapping_append() failed\n"); + } + + /* and we're done with it */ + fy_node_free(fyn); + fyn = NULL; + } + fy_node_free(fyd->root); + fyd->root = new_root; + + } else { + FYP_TOKEN_ERROR(fyp, fy_event_get_token(fye), FYEM_PARSE, + "merge key argument is neither an alias or a mapping"); + goto err_out; + } + + fy_document_builder_destroy(fydb); + fydb = NULL; + + return fyd; + +err_out: + fy_node_free(new_root); + fy_document_destroy(fyd_arg); + fy_document_destroy(fyd); + fy_document_builder_destroy(fydb); + return NULL; +} + +struct fy_eventp *fy_parser_event_resolve_hook_merge_key_start(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_document *fyd = NULL; + struct fy_document_iterator *fydi = NULL; + struct fy_eventp *fyep_next = NULL; + struct fy_event *fye; + struct fy_streaming_alias *fysa; + struct fy_streaming_alias_state *fysas; + + FYP_TOKEN_ERROR_CHECK(fyp, fyep->e.scalar.value, FYEM_PARSE, + !fyp->mks.active, err_out, + "No recursive merge key processing is supported"); + + fyd = fy_parser_get_merge_key_document(fyp, fyep); + FYP_TOKEN_ERROR_CHECK(fyp, fyep->e.scalar.value, FYEM_PARSE, + fyd, err_out, + "No merge key argument document"); + + fysa = fy_parse_streaming_alias_create(fyp, NULL); + fyp_error_check(fyp, fysa, err_out, + "fy_parse_streaming_alias_create() failed"); + + fydi = fy_document_iterator_create(); + fyp_error_check(fyp, fydi, err_out, + "fy_document_iterator_create() failed"); + + fy_document_iterator_node_start(fydi, fyd->root); + + /* now append all to the (fake) streaming anchor */ + while ((fye = fy_document_iterator_body_next(fydi)) != NULL) { + fyep_next = container_of(fye, struct fy_eventp, e); + fy_eventp_list_add_tail(&fysa->events, fyep_next); + } + + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); + fyd = NULL; + + /* and add it to the start of the list */ + fy_streaming_alias_list_add(&fyp->streaming_aliases, fysa); + + /* OK, trim mapping start and end */ + fyep_next = fy_eventp_list_head(&fysa->events); + fyp_error_check(fyp, fyep_next->e.type == FYET_MAPPING_START, err_out, + "not a mapping merge key start"); + fy_eventp_list_del(&fysa->events, fyep_next); + fy_eventp_free(fyep_next); + + fyep_next = fy_eventp_list_tail(&fysa->events); + fyp_error_check(fyp, fyep_next->e.type == FYET_MAPPING_END, err_out, + "not a mapping merge key end"); + fy_eventp_list_del(&fysa->events, fyep_next); + fy_eventp_free(fyep_next); + + /* OK, push the new state */ + fysas = fy_parse_streaming_alias_state_push(fyp, fysa); + fyp_error_check(fyp, fysas, err_out, + "fy_parse_streaming_alias_push() failed!"); + assert(fysas->next); + + /* dispose of the event */ + fy_parse_eventp_recycle(fyp, fyep); + fyep = NULL; + + fyep = fy_parse_eventp_clone(fyp, fysas->next, true); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parse_eventp_clone() failed!"); + + /* advance event */ + fy_parse_streaming_alias_state_next_event(fyp); + + return fyep; + +err_out: + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); + fy_parse_eventp_recycle(fyp, fyep_next); + fy_parse_eventp_recycle(fyp, fyep); + return NULL; +} + +struct fy_eventp *fy_parser_event_resolve_hook_merge_key(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + /* will toggle the map key bit if it is in a mapping */ + fy_parse_streaming_alias_collection_state_toggle_map(fyp); + switch (fyep->e.type) { + case FYET_SCALAR: + /* if a merge key is detected do the dance */ + if (fy_parse_streaming_alias_collection_state_in_map_key(fyp) && + fy_token_scalar_style(fyep->e.scalar.value) == FYSS_PLAIN && + fy_plain_atom_streq(fy_token_atom(fyep->e.scalar.value), "<<")) + return fy_parser_event_resolve_hook_merge_key_start(fyp, fyep); + break; + case FYET_ALIAS: + /* nothing for alias */ + break; + case FYET_MAPPING_START: + fy_parse_streaming_alias_collection_state_push(fyp, FYCTS_MAP_VAL); /* it will toggle to key at the first one */ + break; + case FYET_SEQUENCE_START: + fy_parse_streaming_alias_collection_state_push(fyp, FYCTS_SEQ); + break; + case FYET_MAPPING_END: + case FYET_SEQUENCE_END: + fy_parse_streaming_alias_collection_state_pop(fyp); + break; + default: + break; + } + + return fyep; +} + +/* NOTE: order is _very_ important, do not re-arrange if you're not sure */ +struct fy_eventp *fy_parser_event_resolve_hook(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + int rc; + + /* merge key processing for version 1.1 only! */ + if (fy_reader_get_mode(fyp->reader) == fyrm_yaml_1_1) { + fyep = fy_parser_event_resolve_hook_merge_key(fyp, fyep); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parser_event_resolve_hook_collection_tracking() failed!"); + } + + /* anchor marking */ + fyep = fy_parser_event_resolve_hook_anchor_start(fyp, fyep); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parser_event_resolve_hook_anchor_start() failed!"); + + /* collect afterwards */ + rc = fy_parser_event_resolve_hook_collect(fyp, fyep); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_event_resolve_hook_collect() failed!"); + + /* finally do first entry into alias processing */ + fyep = fy_parser_event_resolve_hook_alias(fyp, fyep); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parser_event_resolve_hook_alias() failed!"); + + return fyep; + +err_out: + return NULL; +} + +struct fy_eventp *fy_parser_parse_resolve_prolog(struct fy_parser *fyp) +{ + struct fy_streaming_alias *fysa; + struct fy_streaming_alias_state *fysas; + struct fy_eventp *fyep, *fyep_src; + bool was_merge_key; + int rc; + + if (!(fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT)) + return NULL; + + fysas = fy_parse_streaming_alias_state_top(fyp); + if (!fysas) + return NULL; + + was_merge_key = false; + for (;;) { + fysa = fysas->fysa; + assert(fysa); + + assert(fysas->next != NULL); + + /* keep track of it */ + fyep_src = fysas->next; + + /* merge key entries don't have an anchor */ + was_merge_key = fysa->anchor == NULL; + + /* advance event */ + fy_parse_streaming_alias_state_next_event(fyp); + + /* not an alias, continue normally */ + if (fyep_src->e.type != FYET_ALIAS) + break; + + /* get the anchor that was defined closest to this one */ + fysa = fy_parser_streaming_alias_lookup_pivot(fyp, fysa, fyep_src->e.alias.anchor); + fyp_error_check(fyp, fysa, err_out, + "fy_parser_streaming_alias_lookup_pivot() failed!"); + + /* and push */ + fysas = fy_parse_streaming_alias_state_push(fyp, fysa); + fyp_error_check(fyp, fysas, err_out, + "fy_parse_streaming_alias_state_push() failed!"); + } + + fyep = fy_parse_eventp_clone(fyp, fyep_src, true); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parse_eventp_clone() failed!"); + + if (was_merge_key) { + rc = fy_parser_event_resolve_hook_collect(fyp, fyep_src); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_event_resolve_hook_collect() failed!"); + } + + return fyep; + +err_out: + fyp->stream_error = true; + return NULL; +} + +struct fy_eventp *fy_parser_parse_resolve_epilog(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + if (!(fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT)) + return fyep; + + fyep = fy_parser_event_resolve_hook(fyp, fyep); + if (!fyep) + fyp->stream_error = true; + + return fyep; +} diff --git a/src/lib/fy-parse.h b/src/lib/fy-parse.h index 16a5f59..d7e502d 100644 --- a/src/lib/fy-parse.h +++ b/src/lib/fy-parse.h @@ -42,6 +42,9 @@ struct fy_parser; struct fy_input; +/* for the event */ +FY_PARSE_TYPE_DECL_ALLOC(eventp); + enum fy_flow_type { FYFT_NONE, FYFT_MAP, @@ -137,6 +140,33 @@ struct fy_parse_state_log { }; FY_PARSE_TYPE_DECL(parse_state_log); +struct fy_streaming_alias { + struct list_head node; + struct fy_token *anchor; + bool collecting; + long mapping_nest; + long sequence_nest; + struct fy_eventp_list events; +}; +FY_PARSE_TYPE_DECL(streaming_alias); + +struct fy_streaming_alias_state { + struct fy_streaming_alias *fysa; + struct fy_eventp *next; +}; + +/* 2 bits per collection tracking state + * pos: * 2 + * bit 0 - 0 map, 1 seq + * bit 1 - 0 key, 1 val (if map) + */ +#define FYCTS_SCALAR 0 +#define FYCTS_SEQ 1 +#define FYCTS_MAP 2 +#define FYCTS_MAP_KEY 2 +#define FYCTS_MAP_VAL 3 +#define FYCTS_MAP_TOGGLE 1 + struct fy_parser { struct fy_parse_cfg cfg; @@ -159,6 +189,7 @@ struct fy_parser { bool bare_document_only : 1; /* no document start indicators allowed, no directives */ bool stream_has_content : 1; bool parse_flow_only : 1; /* document is in flow form, and stop parsing at the end */ + bool parse_block_only : 1; /* document is in embedded block form, and stop parsing at the end */ bool colon_follows_colon : 1; /* "foo"::bar -> "foo": :bar */ bool had_directives : 1; /* document had directives */ int flow_level; @@ -183,6 +214,7 @@ struct fy_parser { int indent; int parent_indent; int indent_line; + int starting_indent; /* simple key stack */ struct fy_simple_key_list simple_keys; /* state stack */ @@ -203,6 +235,7 @@ struct fy_parser { struct fy_simple_key_list recycled_simple_key; struct fy_parse_state_log_list recycled_parse_state_log; struct fy_flow_list recycled_flow; + struct fy_streaming_alias_list recycled_streaming_alias; struct fy_eventp_list recycled_eventp; struct fy_token_list recycled_token; @@ -226,6 +259,28 @@ struct fy_parser { /* last generated event atom */ struct fy_atom last_event_handle; + + struct fy_streaming_alias_list streaming_aliases; + /* streaming alias state */ + struct { + int alloc; + int top; + struct fy_streaming_alias_state *stack; + struct fy_streaming_alias_state local[8]; + } sas; + /* collection tracking state */ + struct { + int alloc; + int top; + uint32_t *stack; + uint32_t local[8]; /* 8 * 32 = 256 / 2 = 128 levels before needing dynamic allocation */ + } cts; + /* merge key state */ + struct { + bool active; + struct fy_eventp_list args; + struct fy_document *fyd; + } mks; }; static inline struct fy_input * @@ -536,6 +591,12 @@ fy_parser_set_flow_only_mode(struct fy_parser *fyp, bool flow_only_mode) fyp->parse_flow_only = flow_only_mode; } +static inline void +fy_parser_set_block_only_mode(struct fy_parser *fyp, bool block_only_mode) +{ + fyp->parse_block_only = block_only_mode; +} + #define fy_fill_atom_a(_fyp, _advance) \ fy_fill_atom((_fyp), (_advance), alloca(sizeof(struct fy_atom))) @@ -553,6 +614,7 @@ int fy_parse_setup(struct fy_parser *fyp, const struct fy_parse_cfg *cfg); void fy_parse_cleanup(struct fy_parser *fyp); int fy_parse_input_append(struct fy_parser *fyp, const struct fy_input_cfg *fyic); +ssize_t fy_parse_estimate_queued_input_size(struct fy_parser *fyp); struct fy_eventp *fy_parse_private(struct fy_parser *fyp); @@ -592,4 +654,12 @@ static inline int fy_document_state_version_compare(struct fy_document_state *fy int fy_parse_set_composer(struct fy_parser *fyp, fy_parse_composer_cb cb, void *userdata); +struct fy_streaming_alias * +fy_parse_streaming_alias_create(struct fy_parser *fyp, struct fy_token *fyt_anchor); +void fy_parse_streaming_alias_clean(struct fy_parser *fyp, struct fy_streaming_alias *fysa); +void fy_parse_streaming_aliases_reset(struct fy_parser *fyp); + +struct fy_eventp *fy_parser_parse_resolve_prolog(struct fy_parser *fyp); +struct fy_eventp *fy_parser_parse_resolve_epilog(struct fy_parser *fyp, struct fy_eventp *fyep); + #endif diff --git a/src/lib/fy-path.c b/src/lib/fy-path.c index f9d9f99..12fb326 100644 --- a/src/lib/fy-path.c +++ b/src/lib/fy-path.c @@ -658,3 +658,60 @@ void fy_path_component_set_sequence_user_data(struct fy_path_component *fypc, vo fypc->user_data = data; } + +void *fy_path_get_parent_user_data(struct fy_path *fypp) +{ + struct fy_path_component *parent; + + if (fy_path_in_root(fypp)) + return fy_path_get_root_user_data(fypp); + + parent = fy_path_last_not_collection_root_component(fypp); + if (fy_path_in_sequence(fypp)) + return fy_path_component_get_sequence_user_data(parent); + else + return fy_path_component_get_mapping_user_data(parent); +} + +void fy_path_set_parent_user_data(struct fy_path *fypp, void *data) +{ + struct fy_path_component *parent; + + if (fy_path_in_root(fypp)) { + fy_path_set_root_user_data(fypp, data); + } else { + parent = fy_path_last_not_collection_root_component(fypp); + if (fy_path_in_sequence(fypp)) + fy_path_component_set_sequence_user_data(parent, data); + else + fy_path_component_set_mapping_user_data(parent, data); + } +} + +void *fy_path_get_last_user_data(struct fy_path *fypp) +{ + struct fy_path_component *last; + + last = fy_path_last_component(fypp); + if (!last) + return NULL; + + if (last->type == FYPCT_SEQ) + return fy_path_component_get_sequence_user_data(last); + else + return fy_path_component_get_mapping_user_data(last); +} + +void fy_path_set_last_user_data(struct fy_path *fypp, void *data) +{ + struct fy_path_component *last; + + last = fy_path_last_component(fypp); + if (!last) + return; + + if (last->type == FYPCT_SEQ) + fy_path_component_set_sequence_user_data(last, data); + else + fy_path_component_set_mapping_user_data(last, data); +} diff --git a/src/lib/fy-token.c b/src/lib/fy-token.c index 45fe14f..bfcf27d 100644 --- a/src/lib/fy-token.c +++ b/src/lib/fy-token.c @@ -71,6 +71,10 @@ void fy_token_clean_rl(struct fy_token_list *fytl, struct fy_token *fyt) free(fyt->tag.suffix0); fyt->tag.suffix0 = NULL; } + if (fyt->tag.short0) { + free(fyt->tag.short0); + fyt->tag.short0 = NULL; + } break; case FYTT_TAG_DIRECTIVE: @@ -439,6 +443,7 @@ struct fy_token *fy_token_vcreate_rl(struct fy_token_list *fytl, enum fy_token_t fyt->tag.fyt_td = fy_token_ref(fyt_td); fyt->tag.handle0 = NULL; fyt->tag.suffix0 = NULL; + fyt->tag.short0 = NULL; break; case FYTT_VERSION_DIRECTIVE: @@ -638,11 +643,10 @@ const struct fy_mark *fy_token_end_mark(struct fy_token *fyt) int fy_token_text_analyze(struct fy_token *fyt) { - const char *s, *e; - const char *value = NULL; + struct fy_atom_iter iter; enum fy_atom_style style; - int c, w, cn, cnn, cp, col; - size_t len; + int c, cn, cnn, cp, col; + uint8_t col0si, col0ei; /* mask for --- ... at indent 0 */ int flags; if (!fyt) @@ -674,12 +678,16 @@ int fy_token_text_analyze(struct fy_token *fyt) if (!fy_atom_style_is_block(style)) flags |= FYTTAF_DIRECT_OUTPUT; - /* get value */ - value = fy_token_get_text(fyt, &len); - if (!value || len == 0) { + fy_atom_iter_start(&fyt->handle, &iter); + + col = 0; + + /* get first character */ + cn = fy_atom_iter_utf8_get(&iter); + if (cn < 0) { + /* empty? */ flags |= FYTTAF_EMPTY | FYTTAF_CAN_BE_DOUBLE_QUOTED | FYTTAF_CAN_BE_UNQUOTED_PATH_KEY; - fyt->analyze_flags = flags; - return flags; + goto out; } flags |= FYTTAF_CAN_BE_PLAIN | @@ -691,19 +699,7 @@ int fy_token_text_analyze(struct fy_token *fyt) FYTTAF_CAN_BE_PLAIN_FLOW | FYTTAF_CAN_BE_UNQUOTED_PATH_KEY; - /* start with document indicators must be quoted at indent 0 */ - if (len >= 3 && (!memcmp(value, "---", 3) || !memcmp(value, "...", 3))) - flags |= FYTTAF_QUOTE_AT_0; - - s = value; - e = value + len; - - col = 0; - - /* get first character */ - cn = fy_utf8_get(s, e - s, &w); - s += w; - col = fy_token_is_lb(fyt, cn) ? 0 : (col + 1); + col0si = col0ei = 0; /* disable folded right off the bat, it's a pain */ flags &= ~FYTTAF_CAN_BE_FOLDED; @@ -720,8 +716,8 @@ int fy_token_text_analyze(struct fy_token *fyt) flags &= ~FYTTAF_CAN_BE_PLAIN_FLOW; if ((flags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_CAN_BE_PLAIN_FLOW))) { - cnn = fy_utf8_get(s, e - s, &w); - if (fy_is_ws(cnn) && fy_is_indicator_before_space(cn)) + cnn = fy_atom_iter_utf8_peek(&iter); + if (fy_is_blankz_m(cnn, fy_token_atom_lb_mode(fyt)) && fy_is_indicator_before_space(cn)) flags &= ~(FYTTAF_CAN_BE_PLAIN | FYTTAF_CAN_BE_PLAIN_FLOW); } @@ -731,10 +727,22 @@ int fy_token_text_analyze(struct fy_token *fyt) flags &= ~FYTTAF_CAN_BE_UNQUOTED_PATH_KEY; cp = -1; - for (c = cn; c >= 0; s += w, cp = c, c = cn) { + for (c = cn; c >= 0; cp = c, c = cn) { + + if (col <= 2) { + if (cn == '-') { + col0si |= (uint8_t)1 << col; + if (col0si == 7) + flags |= FYTTAF_HAS_START_IND | FYTTAF_QUOTE_AT_0; + } else if (cn == '.') { + col0ei |= (uint8_t)1 << col; + if (col0ei == 7) + flags |= FYTTAF_HAS_END_IND | FYTTAF_QUOTE_AT_0; + } + } /* can be -1 on end */ - cn = fy_utf8_get(s, e - s, &w); + cn = fy_atom_iter_utf8_get(&iter); /* zero can't be output, only in double quoted mode */ if (c == 0) { @@ -785,12 +793,11 @@ int fy_token_text_analyze(struct fy_token *fyt) flags &= ~FYTTAF_CAN_BE_PLAIN_FLOW; /* non printable characters, turn off these styles */ - if ((flags & (FYTTAF_CAN_BE_SINGLE_QUOTED | - FYTTAF_CAN_BE_LITERAL | - FYTTAF_CAN_BE_FOLDED)) && !fy_is_print(c)) - flags &= ~(FYTTAF_CAN_BE_SINGLE_QUOTED | - FYTTAF_CAN_BE_LITERAL | + if (!fy_is_print(c)) { + flags &= ~(FYTTAF_CAN_BE_SINGLE_QUOTED | FYTTAF_CAN_BE_LITERAL | FYTTAF_CAN_BE_FOLDED); + flags |= FYTTAF_HAS_NON_PRINT; + } /* if there's an escape, it can't be direct */ if ((flags & FYTTAF_DIRECT_OUTPUT) && @@ -799,17 +806,25 @@ int fy_token_text_analyze(struct fy_token *fyt) (style == FYAS_DOUBLE_QUOTED && c == '\\'))) flags &= ~FYTTAF_DIRECT_OUTPUT; - col = fy_token_is_lb(fyt, c) ? 0 : (col + 1); + if (fy_token_is_lb(fyt, c)) { + col = 0; + col0si = col0ei = 0; + } else + col++; + + if (fy_is_any_lb(c)) + flags |= FYTTAF_HAS_ANY_LB; /* last character */ if (cn < 0) { /* if ends with whitespace or linebreak, can't be plain */ - if (fy_is_ws(cn) || fy_token_is_lb(fyt, cn)) + if (fy_is_ws(c) || fy_token_is_lb(fyt, c)) flags &= ~(FYTTAF_CAN_BE_PLAIN | FYTTAF_CAN_BE_PLAIN_FLOW); } } - +out: + fy_atom_iter_finish(&iter); fyt->analyze_flags = flags; return flags; } @@ -945,6 +960,58 @@ const char *fy_tag_token_suffix0(struct fy_token *fyt) return fyt->tag.suffix0; } +const char *fy_tag_token_short(struct fy_token *fyt, size_t *lenp) +{ + const char *handle, *suffix; + size_t handle_len, suffix_len; + char *text0; + + if (!fyt || fyt->type != FYTT_TAG) + return NULL; + + /* use the cache if it's there (and doesn't need a rebuild) */ + if (fyt->tag.short0 && !fy_token_text_needs_rebuild(fyt)) + return fyt->tag.short0; + + if (fyt->tag.short0) { + free(fyt->tag.short0); + fyt->tag.short0 = NULL; + } + + handle = fy_tag_token_handle(fyt, &handle_len); + if (!handle) + return NULL; + + suffix = fy_tag_token_suffix(fyt, &suffix_len); + if (!suffix) + return NULL; + + text0 = malloc(handle_len + suffix_len + 1); + if (!text0) + return NULL; + memcpy(text0, handle, handle_len); + memcpy(text0 + handle_len, suffix, suffix_len); + text0[handle_len + suffix_len] = '\0'; + + fyt->tag.short0 = text0; + fyt->tag.short_length = handle_len + suffix_len; + + *lenp = fyt->tag.short_length; + + return fyt->tag.short0; +} + +const char *fy_tag_token_short0(struct fy_token *fyt) +{ + const char *text; + size_t len; + + text = fy_tag_token_short(fyt, &len); + if (!text) + return NULL; + return text; +} + const struct fy_version * fy_version_directive_token_version(struct fy_token *fyt) { if (!fyt || fyt->type != FYTT_VERSION_DIRECTIVE) @@ -1524,6 +1591,7 @@ int fy_token_cmp(struct fy_token *fyt1, struct fy_token *fyt2) const char *t1, *t2; size_t l1, l2, l; int ret; + bool aoa; /* anchor or alias */ /* handles both NULL */ if (fyt1 == fyt2) @@ -1537,8 +1605,12 @@ int fy_token_cmp(struct fy_token *fyt1, struct fy_token *fyt2) if (fyt1 && !fyt2) return 1; + /* special case for comparing anchors and aliases */ + aoa = (fyt1->type == FYTT_ANCHOR || fyt1->type == FYTT_ALIAS) && + (fyt2->type == FYTT_ANCHOR || fyt2->type == FYTT_ALIAS); + /* tokens with different types can't be equal */ - if (fyt1->type != fyt2->type) + if (!aoa && fyt1->type != fyt2->type) return fyt2->type > fyt1->type ? -1 : 1; /* special case, these can't use the atom comparisons */ @@ -1702,8 +1774,11 @@ int fy_token_iter_getc(struct fy_token_iter *iter) return -1; /* first try the pushed ungetc */ - if (iter->unget_c != -1) { + if (iter->unget_c >= 0) { c = iter->unget_c; + /* unmatched getc/ungetc */ + if (fy_utf8_width(c) != 1) + return -1; iter->unget_c = -1; return c; } @@ -1722,14 +1797,18 @@ int fy_token_iter_getc(struct fy_token_iter *iter) int fy_token_iter_ungetc(struct fy_token_iter *iter, int c) { - if (iter->unget_c != -1) + if (!iter || c >= 0x80) + return -1; + + if (iter->unget_c >= 0) return -1; - if (c == -1) { + + if (c < 0) { iter->unget_c = -1; return 0; } - iter->unget_c = c & 0xff; - return c & 0xff; + iter->unget_c = c; + return c; } int fy_token_iter_peekc(struct fy_token_iter *iter) @@ -1747,8 +1826,11 @@ int fy_token_iter_utf8_get(struct fy_token_iter *iter) { int c, w, w1; + if (!iter) + return -1; + /* first try the pushed ungetc */ - if (iter->unget_c != -1) { + if (iter->unget_c >= 0) { c = iter->unget_c; iter->unget_c = -1; return c; @@ -1780,13 +1862,18 @@ int fy_token_iter_utf8_get(struct fy_token_iter *iter) int fy_token_iter_utf8_unget(struct fy_token_iter *iter, int c) { - if (iter->unget_c != -1) + if (!iter) + return -1; + + if (iter->unget_c >= 0) return -1; - if (c == -1) { + if (c < 0) { iter->unget_c = -1; return 0; } + if (!fy_utf8_is_valid(c)) + return -1; iter->unget_c = c; return c; diff --git a/src/lib/fy-token.h b/src/lib/fy-token.h index e91f577..8ab8462 100644 --- a/src/lib/fy-token.h +++ b/src/lib/fy-token.h @@ -114,6 +114,8 @@ struct fy_token { struct fy_token *fyt_td; char *handle0; /* zero terminated and allocated, only used by binding */ char *suffix0; + unsigned int short_length; + char *short0; /* zero terminated and allocated for when the short tag is requested */ struct fy_tag tag; /* prefix is now suffix */ } tag; struct { @@ -440,6 +442,10 @@ static inline bool fy_token_is_flow_ws(struct fy_token *fyt, int c) #define FYTTAF_CAN_BE_PLAIN_FLOW FY_BIT(15) #define FYTTAF_QUOTE_AT_0 FY_BIT(16) #define FYTTAF_CAN_BE_UNQUOTED_PATH_KEY FY_BIT(17) +#define FYTTAF_HAS_ANY_LB FY_BIT(18) /* any LB including unicode, not per input */ +#define FYTTAF_HAS_START_IND FY_BIT(19) /* has --- at col 0 */ +#define FYTTAF_HAS_END_IND FY_BIT(20) /* has ... at col 0 */ +#define FYTTAF_HAS_NON_PRINT FY_BIT(21) /* has any non printable utf8 */ int fy_token_text_analyze(struct fy_token *fyt); diff --git a/src/lib/fy-typelist.h b/src/lib/fy-typelist.h deleted file mode 100644 index 7fd9148..0000000 --- a/src/lib/fy-typelist.h +++ /dev/null @@ -1,163 +0,0 @@ -/* - * fy-typelist.h - typed list method builders - * - * Copyright (c) 2019 Pantelis Antoniou - * - * SPDX-License-Identifier: MIT - */ -#ifndef FY_TYPELIST_H -#define FY_TYPELIST_H - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include - -#include - -#include "fy-list.h" - -/* declare type methods */ - -#define FY_TYPE_FWD_DECL_LIST(_type) \ -/* type safe list wrapper */ \ -struct fy_ ## _type ## _list { struct list_head _lh; }; \ -\ -struct __useless_struct_to_allow_semicolon - -#define FY_TYPE_DECL_LIST(_type) \ -static inline void fy_ ## _type ## _list_init(struct fy_ ## _type ## _list *_l) \ -{ \ - INIT_LIST_HEAD(&_l->_lh); \ -} \ -static inline void fy_ ## _type ## _list_add(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ -{ \ - if (_l && _n) \ - list_add(&_n->node, &_l->_lh); \ -} \ -static inline void fy_ ## _type ## _list_add_tail(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ -{ \ - if (_l && _n) \ - list_add_tail(&_n->node, &_l->_lh); \ -} \ -static inline void fy_ ## _type ## _list_push(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ -{ \ - if (_l && _n) \ - fy_ ## _type ## _list_add(_l, _n); \ -} \ -static inline void fy_ ## _type ## _list_push_tail(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ -{ \ - if (_l && _n) \ - fy_ ## _type ## _list_add_tail(_l, _n); \ -} \ -static inline bool fy_ ## _type ## _list_empty(struct fy_ ## _type ## _list *_l) \ -{ \ - return _l ? list_empty(&_l->_lh) : true; \ -} \ -static inline bool fy_ ## _type ## _list_is_singular(struct fy_ ## _type ## _list *_l) \ -{ \ - return _l ? list_is_singular(&_l->_lh) : true; \ -} \ -static inline void fy_ ## _type ## _list_del(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ -{ \ - if (_l && _n) { \ - list_del(&_n->node); \ - INIT_LIST_HEAD(&_n->node); \ - } \ -} \ -static inline void fy_ ## _type ## _list_insert_after(struct fy_ ## _type ## _list *_l, \ - struct fy_ ## _type *_p, struct fy_ ## _type *_n) \ -{ \ - if (_l && _p && _n) \ - list_add(&_n->node, &_p->node); \ -} \ -static inline void fy_ ## _type ## _list_insert_before(struct fy_ ## _type ## _list *_l, \ - struct fy_ ## _type *_p, struct fy_ ## _type *_n) \ -{ \ - if (_l && _p && _n) \ - list_add_tail(&_n->node, &_p->node); \ -} \ -static inline struct fy_ ## _type *fy_ ## _type ## _list_head(struct fy_ ## _type ## _list *_l) \ -{ \ - return !fy_ ## _type ## _list_empty(_l) ? list_first_entry(&_l->_lh, struct fy_ ## _type, node) : NULL; \ -} \ -static inline struct fy_ ## _type *fy_ ## _type ## _list_tail(struct fy_ ## _type ## _list *_l) \ -{ \ - return !fy_ ## _type ## _list_empty(_l) ? list_last_entry(&_l->_lh, struct fy_ ## _type, node) : NULL; \ -} \ -static inline struct fy_ ## _type *fy_ ## _type ## _list_first(struct fy_ ## _type ## _list *_l) \ -{ \ - return fy_ ## _type ## _list_head(_l); \ -} \ -static inline struct fy_ ## _type *fy_ ## _type ## _list_last(struct fy_ ## _type ## _list *_l) \ -{ \ - return fy_ ## _type ## _list_tail(_l); \ -} \ -static inline struct fy_ ## _type *fy_ ## _type ## _list_pop(struct fy_ ## _type ## _list *_l) \ -{ \ - struct fy_ ## _type *_n; \ - \ - _n = fy_ ## _type ## _list_head(_l); \ - if (!_n) \ - return NULL; \ - fy_ ## _type ## _list_del(_l, _n); \ - return _n; \ -} \ -static inline struct fy_ ## _type *fy_ ## _type ## _list_pop_tail(struct fy_ ## _type ## _list *_l) \ -{ \ - struct fy_ ## _type *_n; \ - \ - _n = fy_ ## _type ## _list_tail(_l); \ - if (!_n) \ - return NULL; \ - fy_ ## _type ## _list_del(_l, _n); \ - return _n; \ -} \ -static inline struct fy_ ## _type *fy_ ## _type ## _next(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ -{ \ - if (!_n || !_l || _n->node.next == &_l->_lh) \ - return NULL; \ - return list_entry(_n->node.next, struct fy_ ## _type, node); \ -} \ -static inline struct fy_ ## _type *fy_ ## _type ## _prev(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ -{ \ - if (!_n || !_l || _n->node.prev == &_l->_lh) \ - return NULL; \ - return list_entry(_n->node.prev, struct fy_ ## _type, node); \ -} \ -static inline void fy_ ## _type ## _lists_splice( \ - struct fy_ ## _type ## _list *_l, \ - struct fy_ ## _type ## _list *_lfrom) \ -{ \ - /* check arguments for sanity and lists are not empty */ \ - if (!_l || !_lfrom || \ - fy_ ## _type ## _list_empty(_lfrom)) \ - return; \ - list_splice(&_lfrom->_lh, &_l->_lh); \ -} \ -static inline void fy_ ## _type ## _list_splice_after( \ - struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n, \ - struct fy_ ## _type ## _list *_lfrom) \ -{ \ - /* check arguments for sanity and lists are not empty */ \ - if (!_l || !_n || !_lfrom || \ - fy_ ## _type ## _list_empty(_lfrom)) \ - return; \ - list_splice(&_lfrom->_lh, &_n->node); \ -} \ -static inline void fy_ ## _type ## _list_splice_before( \ - struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n, \ - struct fy_ ## _type ## _list *_lfrom) \ -{ \ - /* check arguments for sanity and lists are not empty */ \ - if (!_l || !_n || !_lfrom || \ - fy_ ## _type ## _list_empty(_lfrom)) \ - return; \ - list_splice(&_lfrom->_lh, _n->node.prev); \ -} \ -struct __useless_struct_to_allow_semicolon - -#endif diff --git a/src/lib/fy-types.c b/src/lib/fy-types.c index fd34fab..1649e04 100644 --- a/src/lib/fy-types.c +++ b/src/lib/fy-types.c @@ -31,3 +31,4 @@ FY_PARSE_TYPE_DEFINE_SIMPLE(indent); FY_PARSE_TYPE_DEFINE_SIMPLE(simple_key); FY_PARSE_TYPE_DEFINE_SIMPLE(parse_state_log); FY_PARSE_TYPE_DEFINE_SIMPLE(flow); +FY_PARSE_TYPE_DEFINE_SIMPLE(streaming_alias); diff --git a/src/lib/fy-utf8.c b/src/lib/fy-utf8.c deleted file mode 100644 index e5103ac..0000000 --- a/src/lib/fy-utf8.c +++ /dev/null @@ -1,591 +0,0 @@ -/* - * fy-utf8.c - utf8 handling methods - * - * Copyright (c) 2019 Pantelis Antoniou - * - * SPDX-License-Identifier: MIT - */ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include - -#include - -#include "fy-utf8.h" - -const int8_t fy_utf8_width_table[32] = { - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 0, 0, 0, 0, 0, 0, 0, 0, - 2, 2, 2, 2, 3, 3, 4, 0, -}; - -int fy_utf8_get_generic(const void *ptr, int left, int *widthp) -{ - const uint8_t *p = ptr; - int i, width, value; - - if (left < 1) - return FYUG_EOF; - - /* this is the slow path */ - width = fy_utf8_width_by_first_octet(p[0]); - if (!width) - return FYUG_INV; - if (width > left) - return FYUG_PARTIAL; - - /* initial value */ - value = *p++ & (0xff >> width); - for (i = 1; i < width; i++) { - if ((*p & 0xc0) != 0x80) - return FYUG_INV; - value = (value << 6) | (*p++ & 0x3f); - } - - /* check for validity */ - if ((width == 4 && value < 0x10000) || - (width == 3 && value < 0x800) || - (width == 2 && value < 0x80) || - (value >= 0xd800 && value <= 0xdfff) || value >= 0x110000) - return FYUG_INV; - - *widthp = width; - - return value; -} - -int fy_utf8_get_right_generic(const void *ptr, int left, int *widthp) -{ - const uint8_t *s, *e; - uint8_t v; - - s = ptr; - e = s + left; - - if (left < 1) - return FYUG_EOF; - - /* single byte sequence */ - v = e[-1]; - if ((v & 0x80) == 0) { - if (widthp) - *widthp = 1; - return (int)v & 0x7f; - } - - /* the last byte must be & 0xc0 == 0x80 */ - if ((v & 0xc0) != 0x80) - return FYUG_INV; - - /* at least two byte sequence */ - if (left < 2) - return FYUG_EOF; - - v = e[-2]; - /* the first byte of the sequence (must be a two byte sequence) */ - if ((v & 0xc0) != 0x80) { - /* two byte start is 110x_xxxx */ - if ((v & 0xe0) != 0xc0) - return FYUG_INV; - return fy_utf8_get(e - 2, 2, widthp); - } - - /* at least three byte sequence */ - if (left < 3) - return FYUG_EOF; - - v = e[-3]; - /* the first byte of the sequence (must be a three byte sequence) */ - if ((v & 0xc0) != 0x80) { - /* three byte start is 1110_xxxx */ - if ((v & 0xf0) != 0xe0) - return FYUG_INV; - return fy_utf8_get(e - 3, 3, widthp); - } - - /* at least four byte sequence */ - if (left < 4) - return FYUG_EOF; - - v = e[-4]; - - /* the first byte of the sequence (must be a four byte sequence) */ - /* four byte start is 1111_0xxx */ - if ((v & 0xf8) != 0xf0) { - return FYUG_INV; - } - return fy_utf8_get(e - 4, 4, widthp); -} - -struct fy_utf8_fmt_esc_map { - const int *ch; - const int *map; -}; - -static const struct fy_utf8_fmt_esc_map esc_all = { - .ch = (const int []){ '\\', '\0', '\b', '\r', '\t', '\f', '\n', '\v', '\a', '\e', 0x85, 0xa0, 0x2028, 0x2029, -1 }, - .map = (const int []){ '\\', '0', 'b', 'r', 't', 'f', 'n', 'v', 'a', 'e', 'N', '_', 'L', 'P', 0 } -}; - -static inline int esc_map(const struct fy_utf8_fmt_esc_map *esc_map, int c) -{ - const int *ch; - int cc; - - ch = esc_map->ch; - while ((cc = *ch++) >= 0) { - if (cc == c) - return esc_map->map[(ch - esc_map->ch) - 1]; - } - return -1; -} - -static inline int fy_utf8_esc_map(int c, enum fy_utf8_escape esc) -{ - if (esc == fyue_none) - return -1; - if (esc == fyue_singlequote && c == '\'') - return '\''; - if (fy_utf8_escape_is_any_doublequote(esc) && c == '"') - return '"'; - return esc_map(&esc_all, c); -} - -int fy_utf8_format_text_length(const char *buf, size_t len, - enum fy_utf8_escape esc) -{ - int c, w, l; - const char *s, *e; - - s = buf; - e = buf + len; - l = 0; - while (s < e) { - c = fy_utf8_get(s, e - s, &w); - if (!w || c < 0) - break; - s += w; - if (fy_utf8_esc_map(c, esc)) - w = 2; - l += w; - } - return l + 1; -} - -char *fy_utf8_format_text(const char *buf, size_t len, - char *out, size_t maxsz, - enum fy_utf8_escape esc) -{ - int c, w, cc; - const char *s, *e; - char *os, *oe; - - s = buf; - e = buf + len; - os = out; - oe = out + maxsz - 1; - while (s < e) { - c = fy_utf8_get(s, e - s, &w); - if (!w || c < 0) - break; - s += w; - - if ((cc = fy_utf8_esc_map(c, esc)) > 0) { - if (os + 2 > oe) - break; - *os++ = '\\'; - *os++ = cc; - continue; - } - - if (os + w > oe) - break; - - os = fy_utf8_put_unchecked(os, c); - } - *os++ = '\0'; - return out; -} - -char *fy_utf8_format(int c, char *buf, enum fy_utf8_escape esc) -{ - int cc; - char *s; - - if (!fy_utf8_is_valid(c)) { - *buf = '\0'; - return buf; - } - - s = buf; - if ((cc = fy_utf8_esc_map(c, esc)) > 0) { - *s++ = '\\'; - *s++ = cc; - } else - s = fy_utf8_put_unchecked(s, c); - *s = '\0'; - return buf; -} - -char *fy_utf8_format_text_alloc(const char *buf, size_t len, enum fy_utf8_escape esc) -{ - int outsz; - char *out; - - outsz = fy_utf8_format_text_length(buf, len, esc); - if (outsz < 0) - return NULL; - out = malloc(outsz); - if (!out) - return NULL; - fy_utf8_format_text(buf, len, out, outsz, esc); - - return out; -} - -const void *fy_utf8_memchr_generic(const void *s, int c, size_t n) -{ - int cc, w; - const void *e; - - e = s + n; - while (s < e && (cc = fy_utf8_get(s, e - s, &w)) >= 0) { - if (c == cc) - return s; - s += w; - } - - return NULL; -} - -/* parse an escape and return utf8 value */ -int fy_utf8_parse_escape(const char **strp, size_t len, enum fy_utf8_escape esc) -{ - const char *s, *e; - char c; - int i, value, code_length, cc, w; - unsigned int hi_surrogate, lo_surrogate; - - /* why do you bother us? */ - if (esc == fyue_none) - return -1; - - if (!strp || !*strp || len < 2) - return -1; - - value = -1; - - s = *strp; - e = s + len; - - c = *s++; - - if (esc == fyue_singlequote) { - if (c != '\'') - goto out; - c = *s++; - if (c != '\'') - goto out; - - value = '\''; - goto out; - } - - /* get '\\' */ - if (c != '\\') - goto out; - - c = *s++; - - /* common YAML & JSON escapes */ - switch (c) { - case 'b': - value = '\b'; - break; - case 'f': - value = '\f'; - break; - case 'n': - value = '\n'; - break; - case 'r': - value = '\r'; - break; - case 't': - value = '\t'; - break; - case '"': - value = '"'; - break; - case '/': - value = '/'; - break; - case '\\': - value = '\\'; - break; - default: - break; - } - - if (value >= 0) - goto out; - - if (esc == fyue_doublequote || esc == fyue_doublequote_yaml_1_1) { - switch (c) { - case '0': - value = '\0'; - break; - case 'a': - value = '\a'; - break; - case '\t': - value = '\t'; - break; - case 'v': - value = '\v'; - break; - case 'e': - value = '\e'; - break; - case ' ': - value = ' '; - break; - case 'N': - value = 0x85; /* NEL */ - break; - case '_': - value = 0xa0; - break; - case 'L': - value = 0x2028; /* LS */ - break; - case 'P': /* PS 0x2029 */ - value = 0x2029; /* PS */ - break; - default: - /* weird unicode escapes */ - if ((uint8_t)c >= 0x80) { - /* in non yaml-1.1 mode we don't allow this craziness */ - if (esc == fyue_doublequote) - goto out; - - cc = fy_utf8_get(s - 1, e - (s - 1), &w); - switch (cc) { - case 0x2028: - case 0x2029: - case 0x85: - case 0xa0: - value = cc; - break; - default: - break; - } - } - break; - } - if (value >= 0) - goto out; - } - - /* finally try the unicode escapes */ - code_length = 0; - - if (esc == fyue_doublequote || esc == fyue_doublequote_yaml_1_1) { - switch (c) { - case 'x': - code_length = 2; - break; - case 'u': - code_length = 4; - break; - case 'U': - code_length = 8; - break; - default: - return -1; - } - } else if (esc == fyue_doublequote_json && c == 'u') - code_length = 4; - - if (!code_length || code_length > (e - s)) - goto out; - - value = 0; - for (i = 0; i < code_length; i++) { - c = *s++; - value <<= 4; - if (c >= '0' && c <= '9') - value |= c - '0'; - else if (c >= 'a' && c <= 'f') - value |= 10 + c - 'a'; - else if (c >= 'A' && c <= 'F') - value |= 10 + c - 'A'; - else - goto out; - } - - /* hi/lo surrogate pair */ - if (code_length == 4 && value >= 0xd800 && value <= 0xdbff && - (e - s) >= 6 && s[0] == '\\' && s[1] == 'u') { - hi_surrogate = value; - - s += 2; - - value = 0; - for (i = 0; i < code_length; i++) { - c = *s++; - value <<= 4; - if (c >= '0' && c <= '9') - value |= c - '0'; - else if (c >= 'a' && c <= 'f') - value |= 10 + c - 'a'; - else if (c >= 'A' && c <= 'F') - value |= 10 + c - 'A'; - else - return -1; - } - lo_surrogate = value; - value = 0x10000 + (hi_surrogate - 0xd800) * 0x400 + (lo_surrogate - 0xdc00); - } - -out: - *strp = s; - - return value; -} - -uint8_t fy_utf8_low_ascii_flags[0x80] = { - [0x00] = F_NON_PRINT, // NUL '\0' (null character) - [0x01] = F_NON_PRINT, // SOH (start of heading) - [0x02] = F_NON_PRINT, // STX (start of text) - [0x03] = F_NON_PRINT, // ETX (end of text) - [0x04] = F_NON_PRINT, // EOT (end of transmission) - [0x05] = F_NON_PRINT, // ENQ (enquiry) - [0x06] = F_NON_PRINT, // ACK (acknowledge) - [0x07] = F_NON_PRINT | F_QUOTE_ESC, // BEL '\a' (bell) - [0x08] = F_NON_PRINT | F_QUOTE_ESC, // BS '\b' (backspace) - [0x09] = F_NON_PRINT | F_QUOTE_ESC | F_WS, // HT '\t' (horizontal tab) - [0x0A] = F_NON_PRINT | F_QUOTE_ESC | F_LB, // LF '\n' (new line) - [0x0B] = F_NON_PRINT | F_QUOTE_ESC, // VT '\v' (vertical tab) - [0x0C] = F_NON_PRINT | F_QUOTE_ESC, // FF '\f' (form feed) - [0x0D] = F_NON_PRINT | F_QUOTE_ESC | F_LB, // CR '\r' (carriage ret) - [0x0E] = F_NON_PRINT, // SO (shift out) - [0x0F] = F_NON_PRINT, // SI (shift in) - [0x10] = F_NON_PRINT, // DLE (data link escape) - [0x11] = F_NON_PRINT, // DC1 (device control 1) - [0x12] = F_NON_PRINT, // DC2 (device control 2) - [0x13] = F_NON_PRINT, // DC3 (device control 3) - [0x14] = F_NON_PRINT, // DC4 (device control 4) - [0x15] = F_NON_PRINT, // NAK (negative ack.) - [0x16] = F_NON_PRINT, // SYN (synchronous idle) - [0x17] = F_NON_PRINT, // ETB (end of trans. blk) - [0x18] = F_NON_PRINT, // CAN (cancel) - [0x19] = F_NON_PRINT, // EM (end of medium) - [0x1A] = F_NON_PRINT, // SUB (substitute) - [0x1B] = F_NON_PRINT, // ESC (escape) - [0x1C] = F_NON_PRINT, // FS (file separator) - [0x1D] = F_NON_PRINT, // GS (group separator) - [0x1E] = F_NON_PRINT, // RS (record separator) - [0x1F] = F_NON_PRINT, // US (unit separator) - [' '] = F_WS, - ['!'] = F_PUNCT, - ['"'] = F_PUNCT, - ['#'] = F_PUNCT, - ['$'] = F_PUNCT, - ['%'] = F_PUNCT, - ['&'] = F_PUNCT, - ['\''] = F_PUNCT, - ['('] = F_PUNCT, - [')'] = F_PUNCT, - ['*'] = F_PUNCT, - ['+'] = F_PUNCT, - [','] = F_PUNCT, - ['-'] = F_PUNCT, - ['.'] = F_PUNCT, - ['/'] = F_PUNCT, - ['0'] = F_DIGIT | F_SIMPLE_SCALAR, - ['1'] = F_DIGIT | F_SIMPLE_SCALAR, - ['2'] = F_DIGIT | F_SIMPLE_SCALAR, - ['3'] = F_DIGIT | F_SIMPLE_SCALAR, - ['4'] = F_DIGIT | F_SIMPLE_SCALAR, - ['5'] = F_DIGIT | F_SIMPLE_SCALAR, - ['6'] = F_DIGIT | F_SIMPLE_SCALAR, - ['7'] = F_DIGIT | F_SIMPLE_SCALAR, - ['8'] = F_DIGIT | F_SIMPLE_SCALAR, - ['9'] = F_DIGIT | F_SIMPLE_SCALAR, - [':'] = F_PUNCT, - [';'] = F_PUNCT, - ['<'] = F_PUNCT, - ['='] = F_PUNCT, - ['>'] = F_PUNCT, - ['?'] = F_PUNCT, - ['@'] = F_PUNCT, - ['A'] = F_LETTER | F_SIMPLE_SCALAR, - ['B'] = F_LETTER | F_SIMPLE_SCALAR, - ['C'] = F_LETTER | F_SIMPLE_SCALAR, - ['D'] = F_LETTER | F_SIMPLE_SCALAR, - ['E'] = F_LETTER | F_SIMPLE_SCALAR, - ['F'] = F_LETTER | F_SIMPLE_SCALAR, - ['G'] = F_LETTER | F_SIMPLE_SCALAR, - ['H'] = F_LETTER | F_SIMPLE_SCALAR, - ['I'] = F_LETTER | F_SIMPLE_SCALAR, - ['J'] = F_LETTER | F_SIMPLE_SCALAR, - ['K'] = F_LETTER | F_SIMPLE_SCALAR, - ['L'] = F_LETTER | F_SIMPLE_SCALAR, - ['M'] = F_LETTER | F_SIMPLE_SCALAR, - ['N'] = F_LETTER | F_SIMPLE_SCALAR, - ['O'] = F_LETTER | F_SIMPLE_SCALAR, - ['P'] = F_LETTER | F_SIMPLE_SCALAR, - ['Q'] = F_LETTER | F_SIMPLE_SCALAR, - ['R'] = F_LETTER | F_SIMPLE_SCALAR, - ['S'] = F_LETTER | F_SIMPLE_SCALAR, - ['T'] = F_LETTER | F_SIMPLE_SCALAR, - ['U'] = F_LETTER | F_SIMPLE_SCALAR, - ['V'] = F_LETTER | F_SIMPLE_SCALAR, - ['W'] = F_LETTER | F_SIMPLE_SCALAR, - ['X'] = F_LETTER | F_SIMPLE_SCALAR, - ['Y'] = F_LETTER | F_SIMPLE_SCALAR, - ['Z'] = F_LETTER | F_SIMPLE_SCALAR, - ['['] = F_PUNCT, - ['\\'] = F_PUNCT, // '\\' - [']'] = F_PUNCT, - ['^'] = F_PUNCT, - ['_'] = F_PUNCT | F_SIMPLE_SCALAR, - ['`'] = F_PUNCT, - ['a'] = F_LETTER | F_SIMPLE_SCALAR, - ['b'] = F_LETTER | F_SIMPLE_SCALAR, - ['c'] = F_LETTER | F_SIMPLE_SCALAR, - ['d'] = F_LETTER | F_SIMPLE_SCALAR, - ['e'] = F_LETTER | F_SIMPLE_SCALAR, - ['f'] = F_LETTER | F_SIMPLE_SCALAR, - ['g'] = F_LETTER | F_SIMPLE_SCALAR, - ['h'] = F_LETTER | F_SIMPLE_SCALAR, - ['i'] = F_LETTER | F_SIMPLE_SCALAR, - ['j'] = F_LETTER | F_SIMPLE_SCALAR, - ['k'] = F_LETTER | F_SIMPLE_SCALAR, - ['l'] = F_LETTER | F_SIMPLE_SCALAR, - ['m'] = F_LETTER | F_SIMPLE_SCALAR, - ['n'] = F_LETTER | F_SIMPLE_SCALAR, - ['o'] = F_LETTER | F_SIMPLE_SCALAR, - ['p'] = F_LETTER | F_SIMPLE_SCALAR, - ['q'] = F_LETTER | F_SIMPLE_SCALAR, - ['r'] = F_LETTER | F_SIMPLE_SCALAR, - ['s'] = F_LETTER | F_SIMPLE_SCALAR, - ['t'] = F_LETTER | F_SIMPLE_SCALAR, - ['u'] = F_LETTER | F_SIMPLE_SCALAR, - ['v'] = F_LETTER | F_SIMPLE_SCALAR, - ['w'] = F_LETTER | F_SIMPLE_SCALAR, - ['x'] = F_LETTER | F_SIMPLE_SCALAR, - ['y'] = F_LETTER | F_SIMPLE_SCALAR, - ['z'] = F_LETTER | F_SIMPLE_SCALAR, - ['{'] = F_PUNCT, - ['|'] = F_PUNCT, - ['}'] = F_PUNCT, - ['~'] = F_PUNCT, - [0x7F] = F_NON_PRINT, // DEL -}; diff --git a/src/lib/fy-utf8.h b/src/lib/fy-utf8.h deleted file mode 100644 index b879706..0000000 --- a/src/lib/fy-utf8.h +++ /dev/null @@ -1,229 +0,0 @@ -/* - * fy-utf8.h - UTF-8 methods - * - * Copyright (c) 2019 Pantelis Antoniou - * - * SPDX-License-Identifier: MIT - */ -#ifndef FY_UTF8_H -#define FY_UTF8_H - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include - -#include - -#include "fy-utils.h" - -extern const int8_t fy_utf8_width_table[32]; - -static inline int -fy_utf8_width_by_first_octet_no_table(uint8_t c) -{ - return (c & 0x80) == 0x00 ? 1 : - (c & 0xe0) == 0xc0 ? 2 : - (c & 0xf0) == 0xe0 ? 3 : - (c & 0xf8) == 0xf0 ? 4 : 0; -} - -static inline FY_ALWAYS_INLINE int -fy_utf8_width_by_first_octet(uint8_t c) -{ - return fy_utf8_width_table[c >> 3]; -} - -/* assumes valid utf8 character */ -static inline size_t -fy_utf8_width(int c) -{ - return c < 0x80 ? 1 : - c < 0x800 ? 2 : - c < 0x10000 ? 3 : 4; -} - -static inline bool -fy_utf8_is_valid(int c) -{ - return c >= 0 && !((c >= 0xd800 && c <= 0xdfff) || c >= 0x110000); -} - -static inline bool -fy_utf8_is_printable_ascii(int c) -{ - return c >= 0x20 && c <= 0x7e; -} - -/* generic utf8 decoder (not inlined) */ -int fy_utf8_get_generic(const void *ptr, int left, int *widthp); - -/* -1 for end of input, -2 for invalid character, -3 for partial */ -#define FYUG_EOF -1 -#define FYUG_INV -2 -#define FYUG_PARTIAL -3 - -static inline int fy_utf8_get(const void *ptr, int left, int *widthp) -{ - const uint8_t *p = ptr; - - /* single byte (hot path) */ - if (left <= 0) { - *widthp = 0; - return FYUG_EOF; - } - - if (!(p[0] & 0x80)) { - *widthp = 1; - return p[0] & 0x7f; - } - return fy_utf8_get_generic(ptr, left, widthp); -} - -int fy_utf8_get_right_generic(const void *ptr, int left, int *widthp); - -static inline int fy_utf8_get_right(const void *ptr, int left, int *widthp) -{ - const uint8_t *p = ptr + left; - - /* single byte (hot path) */ - if (left > 0 && !(p[-1] & 0x80)) { - if (widthp) - *widthp = 1; - return p[-1] & 0x7f; - } - return fy_utf8_get_right_generic(ptr, left, widthp); -} - - -/* for when you _know_ that there's enough room and c is valid */ -static inline void *fy_utf8_put_unchecked(void *ptr, int c) -{ - uint8_t *s = ptr; - - assert(c >= 0); - if (c < 0x80) - *s++ = c; - else if (c < 0x800) { - *s++ = (c >> 6) | 0xc0; - *s++ = (c & 0x3f) | 0x80; - } else if (c < 0x10000) { - *s++ = (c >> 12) | 0xe0; - *s++ = ((c >> 6) & 0x3f) | 0x80; - *s++ = (c & 0x3f) | 0x80; - } else { - *s++ = (c >> 18) | 0xf0; - *s++ = ((c >> 12) & 0x3f) | 0x80; - *s++ = ((c >> 6) & 0x3f) | 0x80; - *s++ = (c & 0x3f) | 0x80; - } - return s; -} - -static inline void *fy_utf8_put(void *ptr, size_t left, int c) -{ - if (!fy_utf8_is_valid(c) || fy_utf8_width(c) > left) - return NULL; - - return fy_utf8_put_unchecked(ptr, c); -} - -/* buffer must contain at least 5 characters */ -#define FY_UTF8_FORMAT_BUFMIN 5 -enum fy_utf8_escape { - fyue_none, - fyue_singlequote, - fyue_doublequote, - fyue_doublequote_json, - fyue_doublequote_yaml_1_1, -}; - -static inline bool fy_utf8_escape_is_any_doublequote(enum fy_utf8_escape esc) -{ - return esc >= fyue_doublequote && esc <= fyue_doublequote_yaml_1_1; -} - -char *fy_utf8_format(int c, char *buf, enum fy_utf8_escape esc); - -#define fy_utf8_format_a(_c, _esc) \ - ({ \ - char *_buf = alloca(FY_UTF8_FORMAT_BUFMIN); \ - fy_utf8_format((_c), _buf, _esc); \ - }) - -int fy_utf8_format_text_length(const char *buf, size_t len, - enum fy_utf8_escape esc); -char *fy_utf8_format_text(const char *buf, size_t len, - char *out, size_t maxsz, - enum fy_utf8_escape esc); - -#define fy_utf8_format_text_a(_buf, _len, _esc) \ - ({ \ - const char *__buf = (_buf); \ - size_t __len = (_len); \ - enum fy_utf8_escape __esc = (_esc); \ - size_t _outsz = fy_utf8_format_text_length(__buf, __len, __esc); \ - char *_out = alloca(_outsz + 1); \ - fy_utf8_format_text(__buf, __len, _out, _outsz, __esc); \ - }) - -char *fy_utf8_format_text_alloc(const char *buf, size_t len, enum fy_utf8_escape esc); - -const void *fy_utf8_memchr_generic(const void *s, int c, size_t n); - -static inline const void *fy_utf8_memchr(const void *s, int c, size_t n) -{ - if (c < 0 || !n) - return NULL; - if (c < 0x80) - return memchr(s, c, n); - return fy_utf8_memchr_generic(s, c, n); -} - -static inline const void *fy_utf8_strchr(const void *s, int c) -{ - if (c < 0) - return NULL; - if (c < 0x80) - return strchr(s, c); - return fy_utf8_memchr_generic(s, c, strlen(s)); -} - -static inline int fy_utf8_count(const void *ptr, size_t len) -{ - const uint8_t *s = ptr, *e = ptr + len; - int w, count; - - count = 0; - while (s < e) { - w = fy_utf8_width_by_first_octet(*s); - - /* malformed? */ - if (!w || s + w > e) - break; - s += w; - - count++; - } - - return count; -} - -int fy_utf8_parse_escape(const char **strp, size_t len, enum fy_utf8_escape esc); - -#define F_NONE 0 -#define F_NON_PRINT FY_BIT(0) /* non printable */ -#define F_SIMPLE_SCALAR FY_BIT(1) /* part of simple scalar */ -#define F_QUOTE_ESC FY_BIT(2) /* escape form, i.e \n */ -#define F_LB FY_BIT(3) /* is a linebreak */ -#define F_WS FY_BIT(4) /* is a whitespace */ -#define F_PUNCT FY_BIT(5) /* is a punctuation mark */ -#define F_LETTER FY_BIT(6) /* is a letter a..z A..Z */ -#define F_DIGIT FY_BIT(7) /* is a digit 0..9 */ - -extern uint8_t fy_utf8_low_ascii_flags[0x80]; - -#endif diff --git a/src/lib/fy-utils.c b/src/lib/fy-utils.c deleted file mode 100644 index 8b55ccb..0000000 --- a/src/lib/fy-utils.c +++ /dev/null @@ -1,634 +0,0 @@ -/* - * fy-utils.c - Generic utilities for functionality that's missing - * from platforms. - * - * For now only used to implement memstream for Apple platforms. - * - * Copyright (c) 2019 Pantelis Antoniou - * - * SPDX-License-Identifier: MIT - */ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "fy-utf8.h" -#include "fy-ctype.h" -#include "fy-utils.h" - -#if defined(__APPLE__) && (_POSIX_C_SOURCE < 200809L) - -/* - * adapted from http://piumarta.com/software/memstream/ - * - * Under the MIT license. - */ - -/* - * ---------------------------------------------------------------------------- - * - * OPEN_MEMSTREAM(3) BSD and Linux Library Functions OPEN_MEMSTREAM(3) - * - * SYNOPSIS - * #include "memstream.h" - * - * FILE *open_memstream(char **bufp, size_t *sizep); - * - * DESCRIPTION - * The open_memstream() function opens a stream for writing to a buffer. - * The buffer is dynamically allocated (as with malloc(3)), and - * automatically grows as required. After closing the stream, the caller - * should free(3) this buffer. - * - * When the stream is closed (fclose(3)) or flushed (fflush(3)), the - * locations pointed to by bufp and sizep are updated to contain, - * respectively, a pointer to the buffer and the current size of the - * buffer. These values remain valid only as long as the caller performs - * no further output on the stream. If further output is performed, then - * the stream must again be flushed before trying to access these - * variables. - * - * A null byte is maintained at the end of the buffer. This byte is not - * included in the size value stored at sizep. - * - * The stream's file position can be changed with fseek(3) or fseeko(3). - * Moving the file position past the end of the data already written fills - * the intervening space with zeros. - * - * RETURN VALUE - * Upon successful completion open_memstream() returns a FILE pointer. - * Otherwise, NULL is returned and errno is set to indicate the error. - * - * CONFORMING TO - * POSIX.1-2008 - * - * ---------------------------------------------------------------------------- - */ - -#ifndef min -#define min(X, Y) (((X) < (Y)) ? (X) : (Y)) -#endif - -struct memstream { - size_t position; - size_t size; - size_t capacity; - char *contents; - char **ptr; - size_t *sizeloc; -}; - -static int memstream_grow(struct memstream *ms, size_t minsize) -{ - size_t newcap; - char *newcontents; - - newcap = ms->capacity * 2; - while (newcap <= minsize + 1) - newcap *= 2; - newcontents = realloc(ms->contents, newcap); - if (!newcontents) - return -1; - ms->contents = newcontents; - memset(ms->contents + ms->capacity, 0, newcap - ms->capacity); - ms->capacity = newcap; - *ms->ptr = ms->contents; - return 0; -} - -static int memstream_read(void *cookie, char *buf, int count) -{ - struct memstream *ms = cookie; - size_t n; - - n = min(ms->size - ms->position, (size_t)count); - if (n < 1) - return 0; - memcpy(buf, ms->contents, n); - ms->position += n; - return n; -} - -static int memstream_write(void *cookie, const char *buf, int count) -{ - struct memstream *ms = cookie; - - if (ms->capacity <= ms->position + (size_t)count && - memstream_grow(ms, ms->position + (size_t)count) < 0) - return -1; - memcpy(ms->contents + ms->position, buf, count); - ms->position += count; - ms->contents[ms->position] = '\0'; - if (ms->size < ms->position) - *ms->sizeloc = ms->size = ms->position; - - return count; -} - -static fpos_t memstream_seek(void *cookie, fpos_t offset, int whence) -{ - struct memstream *ms = cookie; - fpos_t pos= 0; - - switch (whence) { - case SEEK_SET: - pos = offset; - break; - case SEEK_CUR: - pos = ms->position + offset; - break; - case SEEK_END: - pos = ms->size + offset; - break; - default: - errno= EINVAL; - return -1; - } - if (pos >= (fpos_t)ms->capacity && memstream_grow(ms, pos) < 0) - return -1; - ms->position = pos; - if (ms->size < ms->position) - *ms->sizeloc = ms->size = ms->position; - return pos; -} - -static int memstream_close(void *cookie) -{ - struct memstream *ms = cookie; - - ms->size = min(ms->size, ms->position); - *ms->ptr = ms->contents; - *ms->sizeloc = ms->size; - ms->contents[ms->size]= 0; - /* ms->contents is what's returned */ - free(ms); - return 0; -} - -FILE *open_memstream(char **ptr, size_t *sizeloc) -{ - struct memstream *ms; - FILE *fp; - - if (!ptr || !sizeloc) { - errno= EINVAL; - goto err_out; - } - - ms = calloc(1, sizeof(struct memstream)); - if (!ms) - goto err_out; - - ms->position = ms->size= 0; - ms->capacity = 4096; - ms->contents = calloc(ms->capacity, 1); - if (!ms->contents) - goto err_free_ms; - ms->ptr = ptr; - ms->sizeloc = sizeloc; - fp= funopen(ms, memstream_read, memstream_write, - memstream_seek, memstream_close); - if (!fp) - goto err_free_all; - *ptr = ms->contents; - *sizeloc = ms->size; - return fp; - -err_free_all: - free(ms->contents); -err_free_ms: - free(ms); -err_out: - return NULL; -} - -#endif /* __APPLE__ && _POSIX_C_SOURCE < 200809L */ - -bool fy_tag_uri_is_valid(const char *data, size_t len) -{ - const char *s, *e; - int w, j, k, width, c; - uint8_t octet, esc_octets[4]; - - s = data; - e = s + len; - - while ((c = fy_utf8_get(s, e - s, &w)) >= 0) { - if (c != '%') { - s += w; - continue; - } - - width = 0; - k = 0; - do { - /* short URI escape */ - if ((e - s) < 3) - return false; - - if (width > 0) { - c = fy_utf8_get(s, e - s, &w); - if (c != '%') - return false; - } - - s += w; - - octet = 0; - - for (j = 0; j < 2; j++) { - c = fy_utf8_get(s, e - s, &w); - if (!fy_is_hex(c)) - return false; - s += w; - - octet <<= 4; - if (c >= '0' && c <= '9') - octet |= c - '0'; - else if (c >= 'a' && c <= 'f') - octet |= 10 + c - 'a'; - else - octet |= 10 + c - 'A'; - } - if (!width) { - width = fy_utf8_width_by_first_octet(octet); - - if (width < 1 || width > 4) - return false; - k = 0; - } - esc_octets[k++] = octet; - - } while (--width > 0); - - /* now convert to utf8 */ - c = fy_utf8_get(esc_octets, k, &w); - - if (c < 0) - return false; - } - - return true; -} - -int fy_tag_handle_length(const char *data, size_t len) -{ - const char *s, *e; - int c, w; - - s = data; - e = s + len; - - c = fy_utf8_get(s, e - s, &w); - if (c != '!') - return -1; - s += w; - - c = fy_utf8_get(s, e - s, &w); - if (fy_is_ws(c)) - return s - data; - /* if first character is !, empty handle */ - if (c == '!') { - s += w; - return s - data; - } - if (!fy_is_first_alpha(c)) - return -1; - s += w; - while (fy_is_alnum(c = fy_utf8_get(s, e - s, &w))) - s += w; - if (c == '!') - s += w; - - return s - data; -} - -int fy_tag_uri_length(const char *data, size_t len) -{ - const char *s, *e; - int c, w, cn, wn, uri_length; - - s = data; - e = s + len; - - while (fy_is_uri(c = fy_utf8_get(s, e - s, &w))) { - cn = fy_utf8_get(s + w, e - (s + w), &wn); - if ((fy_is_z(cn) || fy_is_blank(cn) || fy_is_any_lb(cn)) && fy_utf8_strchr(",}]", c)) - break; - s += w; - } - uri_length = s - data; - - if (!fy_tag_uri_is_valid(data, uri_length)) - return -1; - - return uri_length; -} - -int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info) -{ - const char *s, *e; - int total_length, handle_length, uri_length, prefix_length, suffix_length; - int c, cn, w, wn; - - s = data; - e = s + len; - - prefix_length = 0; - - /* it must start with '!' */ - c = fy_utf8_get(s, e - s, &w); - if (c != '!') - return -1; - cn = fy_utf8_get(s + w, e - (s + w), &wn); - if (cn == '<') { - prefix_length = 2; - suffix_length = 1; - } else - prefix_length = suffix_length = 0; - - if (prefix_length) { - handle_length = 0; /* set the handle to '' */ - s += prefix_length; - } else { - /* either !suffix or !handle!suffix */ - /* we scan back to back, and split handle/suffix */ - handle_length = fy_tag_handle_length(s, e - s); - if (handle_length <= 0) - return -1; - s += handle_length; - } - - uri_length = fy_tag_uri_length(s, e - s); - if (uri_length < 0) - return -1; - - /* a handle? */ - if (!prefix_length && (handle_length == 0 || data[handle_length - 1] != '!')) { - /* special case, '!', handle set to '' and suffix to '!' */ - if (handle_length == 1 && uri_length == 0) { - handle_length = 0; - uri_length = 1; - } else { - uri_length = handle_length - 1 + uri_length; - handle_length = 1; - } - } - total_length = prefix_length + handle_length + uri_length + suffix_length; - - if (total_length != (int)len) - return -1; - - info->total_length = total_length; - info->handle_length = handle_length; - info->uri_length = uri_length; - info->prefix_length = prefix_length; - info->suffix_length = suffix_length; - - return 0; -} - -/* simple terminal methods; mainly for getting size of terminal */ -int fy_term_set_raw(int fd, struct termios *oldt) -{ - struct termios newt, t; - int ret; - - /* must be a terminal */ - if (!isatty(fd)) - return -1; - - ret = tcgetattr(fd, &t); - if (ret != 0) - return ret; - - newt = t; - - cfmakeraw(&newt); - - ret = tcsetattr(fd, TCSANOW, &newt); - if (ret != 0) - return ret; - - if (oldt) - *oldt = t; - - return 0; -} - -int fy_term_restore(int fd, const struct termios *oldt) -{ - /* must be a terminal */ - if (!isatty(fd)) - return -1; - - return tcsetattr(fd, TCSANOW, oldt); -} - -ssize_t fy_term_write(int fd, const void *data, size_t count) -{ - ssize_t wrn, r; - - if (!isatty(fd)) - return -1; - r = 0; - wrn = 0; - while (count > 0) { - do { - r = write(fd, data, count); - } while (r == -1 && errno == EAGAIN); - if (r < 0) - break; - wrn += r; - data += r; - count -= r; - } - - /* return the amount written, or the last error code */ - return wrn > 0 ? wrn : r; -} - -int fy_term_safe_write(int fd, const void *data, size_t count) -{ - if (!isatty(fd)) - return -1; - - return fy_term_write(fd, data, count) == (ssize_t)count ? 0 : -1; -} - -ssize_t fy_term_read(int fd, void *data, size_t count, int timeout_us) -{ - ssize_t rdn, r; - struct timeval tv, tvto, *tvp; - fd_set rdfds; - - if (!isatty(fd)) - return -1; - - FD_ZERO(&rdfds); - - memset(&tvto, 0, sizeof(tvto)); - memset(&tv, 0, sizeof(tv)); - - if (timeout_us >= 0) { - tvto.tv_sec = timeout_us / 1000000; - tvto.tv_usec = timeout_us % 1000000; - tvp = &tv; - } else { - tvp = NULL; - } - - r = 0; - rdn = 0; - while (count > 0) { - do { - FD_SET(fd, &rdfds); - if (tvp) - *tvp = tvto; - r = select(fd + 1, &rdfds, NULL, NULL, tvp); - } while (r == -1 && errno == EAGAIN); - - /* select ends, or something weird */ - if (r <= 0 || !FD_ISSET(fd, &rdfds)) - break; - - /* now read */ - do { - r = read(fd, data, count); - } while (r == -1 && errno == EAGAIN); - if (r < 0) - break; - - rdn += r; - data += r; - count -= r; - } - - /* return the amount written, or the last error code */ - return rdn > 0 ? rdn : r; -} - -ssize_t fy_term_read_escape(int fd, void *buf, size_t count) -{ - char *p; - int r, rdn; - char c; - - /* at least 3 characters */ - if (count < 3) - return -1; - - p = buf; - rdn = 0; - - /* ESC */ - r = fy_term_read(fd, &c, 1, 100 * 1000); - if (r != 1 || c != '\x1b') - return -1; - *p++ = c; - count--; - rdn++; - - /* [ */ - r = fy_term_read(fd, &c, 1, 100 * 1000); - if (r != 1 || c != '[') - return rdn; - *p++ = c; - count--; - rdn++; - - /* read until error, out of buffer, or < 0x40 || > 0x7e */ - r = -1; - while (count > 0) { - r = fy_term_read(fd, &c, 1, 100 * 1000); - if (r != 1) - r = -1; - if (r != 1) - break; - *p++ = c; - count--; - rdn++; - - /* end of escape */ - if (c >= 0x40 && c <= 0x7e) - break; - } - - return rdn; -} - -int fy_term_query_size_raw(int fd, int *rows, int *cols) -{ - char buf[32]; - char *s, *e; - ssize_t r; - - /* must be a terminal */ - if (!isatty(fd)) - return -1; - - *rows = *cols = 0; - - /* query text area */ - r = fy_term_safe_write(fd, "\x1b[18t", 5); - if (r != 0) - return r; - - /* read a character */ - r = fy_term_read_escape(fd, buf, sizeof(buf)); - - /* return must be ESC[8;;;t */ - - if (r < 8 || r >= (int)sizeof(buf) - 2) /* minimum ESC[8;1;1t */ - return -1; - - s = buf; - e = s + r; - - /* correct response? starts with ESC[8; */ - if (s[0] != '\x1b' || s[1] != '[' || s[2] != '8' || s[3] != ';') - return -1; - s += 4; - - /* must end with t */ - if (e[-1] != 't') - return -1; - *--e = '\0'; /* remove trailing t, and zero terminate */ - - /* scan two ints separated by ; */ - r = sscanf(s, "%d;%d", rows, cols); - if (r != 2) - return -1; - - return 0; -} - -int fy_term_query_size(int fd, int *rows, int *cols) -{ - struct termios old_term; - int ret, r; - - if (!isatty(fd)) - return -1; - - r = fy_term_set_raw(fd, &old_term); - if (r != 0) - return -1; - - ret = fy_term_query_size_raw(fd, rows, cols); - - r = fy_term_restore(fd, &old_term); - if (r != 0) - return -1; - - return ret; -} diff --git a/src/lib/fy-utils.h b/src/lib/fy-utils.h deleted file mode 100644 index cf3e3c5..0000000 --- a/src/lib/fy-utils.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * fy-utils.h - internal utilities header file - * - * Copyright (c) 2019 Pantelis Antoniou - * - * SPDX-License-Identifier: MIT - */ - -#ifndef FY_UTILS_H -#define FY_UTILS_H - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include -#include - -#if defined(__APPLE__) && (_POSIX_C_SOURCE < 200809L) -FILE *open_memstream(char **ptr, size_t *sizeloc); -#endif - -int fy_tag_handle_length(const char *data, size_t len); -bool fy_tag_uri_is_valid(const char *data, size_t len); -int fy_tag_uri_length(const char *data, size_t len); - -struct fy_tag_scan_info { - int total_length; - int handle_length; - int uri_length; - int prefix_length; - int suffix_length; -}; - -int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info); - -#ifndef ARRAY_SIZE -#define ARRAY_SIZE(x) ((sizeof(x)/sizeof((x)[0]))) -#endif - -#if !defined(NDEBUG) && (defined(__GNUC__) && __GNUC__ >= 4) -#define FY_ALWAYS_INLINE __attribute__((always_inline)) -#else -#define FY_ALWAYS_INLINE /* nothing */ -#endif - -#if defined(__GNUC__) && __GNUC__ >= 4 -#define FY_UNUSED __attribute__((unused)) -#else -#define FY_UNUSED /* nothing */ -#endif - -int fy_term_set_raw(int fd, struct termios *oldt); -int fy_term_restore(int fd, const struct termios *oldt); -ssize_t fy_term_write(int fd, const void *data, size_t count); -int fy_term_safe_write(int fd, const void *data, size_t count); -ssize_t fy_term_read(int fd, void *data, size_t count, int timeout_us); -ssize_t fy_term_read_escape(int fd, void *buf, size_t count); - -/* the raw methods require the terminal to be in raw mode */ -int fy_term_query_size_raw(int fd, int *rows, int *cols); - -/* the non raw methods will set the terminal to raw and then restore */ -int fy_term_query_size(int fd, int *rows, int *cols); - -#endif diff --git a/src/thread/fy-thread.c b/src/thread/fy-thread.c new file mode 100644 index 0000000..d1f7eaa --- /dev/null +++ b/src/thread/fy-thread.c @@ -0,0 +1,1156 @@ +/* + * fy-thread.c - Lighting fast thread pool implementation + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#include +#endif + +#include "fy-diag.h" +#include "fy-utils.h" +#include "fy-thread.h" + +#undef WORK_SHUTDOWN +#define WORK_SHUTDOWN ((struct fy_thread_work *)(void *)-1) + +#ifdef FY_THREAD_DEBUG +#define TDBG(_fmt, ...) \ + do { fprintf(stderr, (_fmt) __VA_OPT__(,) __VA_ARGS__); } while(0) +#else +#define TDBG(_fmt, ...) \ + do { /* nothing */ } while(0) +#endif + +static void *fy_worker_thread_standard(void *arg); +static void *fy_worker_thread_steal(void *arg); + +static void fy_thread_work_join_standard(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn); +static void fy_thread_work_join_steal(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn); +static void fy_thread_work_join_steal_2(struct fy_thread_pool *tp, struct fy_thread_work works[2], fy_work_check_fn check_fn); + +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + +/* linux pedal to the metal implementation */ +static inline int futex(_Atomic(uint32_t) *uaddr, int futex_op, uint32_t val, const struct timespec *timeout, uint32_t *uaddr2, uint32_t val3) +{ + return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3); +} + +static inline int fwait(_Atomic(uint32_t) *futexp) +{ + long s; + uint32_t one = 1; + + while (!atomic_compare_exchange_strong(futexp, &one, 0)) { + s = futex(futexp, FUTEX_WAIT_PRIVATE, 0, NULL, NULL, 0); + if (s == -1 && errno != EAGAIN) + return -1; + } + return 0; +} + +static inline int fpost(_Atomic(uint32_t) *futexp) +{ + long s; + uint32_t zero = 0; + + if (atomic_compare_exchange_strong(futexp, &zero, 1)) { + s = futex(futexp, FUTEX_WAKE_PRIVATE, 1, NULL, NULL, 0); + if (s == -1) + return -1; + } + return 0; +} + +static inline void fy_thread_init_sync(struct fy_thread *t) +{ + /* nothing more needed for futexes */ + atomic_store(&t->submit, 0); + atomic_store(&t->done, 0); +} + +static inline struct fy_thread_work *fy_worker_wait_for_work(struct fy_thread *t) +{ + struct fy_thread_work *w; + int rc __FY_DEBUG_UNUSED__; + + while ((w = atomic_load(&t->work)) == NULL) { + rc = fwait(&t->submit); + assert(!rc); + } + return w; +} + +static inline void fy_worker_signal_work_done(struct fy_thread *t, struct fy_thread_work *work) +{ + struct fy_thread_work *exp_work; + bool ok __FY_DEBUG_UNUSED__; + int rc __FY_DEBUG_UNUSED__; + + /* note that the work won't be replaced if it's a shutdown */ + exp_work = work; + ok = atomic_compare_exchange_strong(&t->work, &exp_work, NULL); + assert(ok); + + rc = fpost(&t->done); + assert(!rc); +} + +static inline int fy_thread_submit_work_internal(struct fy_thread *t, struct fy_thread_work *work) +{ + struct fy_thread_work *exp_work; + int rc __FY_DEBUG_UNUSED__; + + /* atomically update the work */ + exp_work = NULL; + if (!atomic_compare_exchange_strong(&t->work, &exp_work, work)) + return -1; + + rc = fpost(&t->submit); + assert(!rc); + + return 0; +} + +static inline int fy_thread_wait_work_internal(struct fy_thread *t) +{ + const struct fy_thread_work *work; + + while ((work = atomic_load(&t->work)) != NULL) + fwait(&t->done); + + atomic_store(&t->done, 0); + + return 0; +} + +void fy_worker_thread_shutdown(struct fy_thread *t) +{ + int rc __FY_DEBUG_UNUSED__; + + atomic_store(&t->work, WORK_SHUTDOWN); + rc = fpost(&t->submit); + assert(!rc); + + rc = pthread_join(t->tid, NULL); + assert(!rc); + + if (!(t->tp->cfg.flags & FYTPCF_STEAL_MODE)) + atomic_store(&t->work, NULL); +} + +#else + +/* portable pthread implementation */ + +static inline void fy_thread_init_sync(struct fy_thread *t) +{ + pthread_mutex_init(&t->lock, NULL); + pthread_cond_init(&t->cond, NULL); + + pthread_mutex_init(&t->wait_lock, NULL); + pthread_cond_init(&t->wait_cond, NULL); +} + +static inline struct fy_thread_work *fy_worker_wait_for_work(struct fy_thread *t) +{ + struct fy_thread_work *work; + + pthread_mutex_lock(&t->lock); + while ((work = atomic_load(&t->work)) == NULL) + pthread_cond_wait(&t->cond, &t->lock); + pthread_mutex_unlock(&t->lock); + + return work; +} + +static inline void fy_worker_signal_work_done(struct fy_thread *t, struct fy_thread_work *work) +{ + struct fy_thread_work *exp_work; + + /* clear the work, so that the user knows we're done */ + pthread_mutex_lock(&t->wait_lock); + + /* note that the work won't be replaced if it's a shutdown */ + exp_work = work; + if (!atomic_compare_exchange_strong(&t->work, &exp_work, NULL)) + assert(exp_work == WORK_SHUTDOWN); + pthread_cond_signal(&t->wait_cond); + pthread_mutex_unlock(&t->wait_lock); +} + +static inline int fy_thread_submit_work_internal(struct fy_thread *t, struct fy_thread_work *work) +{ + struct fy_thread_work *exp_work; + int ret; + + /* atomically update the work */ + + assert(t); + assert(work); + + pthread_mutex_lock(&t->lock); + exp_work = NULL; + if (!atomic_compare_exchange_strong(&t->work, &exp_work, work)) { + assert(exp_work == WORK_SHUTDOWN); + ret = -1; + } else { + pthread_cond_signal(&t->cond); + ret = 0; + } + pthread_mutex_unlock(&t->lock); + + return ret; +} + +static inline int fy_thread_wait_work_internal(struct fy_thread *t) +{ + const struct fy_thread_work *work; + + pthread_mutex_lock(&t->wait_lock); + while ((work = atomic_load(&t->work)) != NULL) + pthread_cond_wait(&t->wait_cond, &t->wait_lock); + pthread_mutex_unlock(&t->wait_lock); + + return 0; +} + +void fy_worker_thread_shutdown(struct fy_thread *t) +{ + pthread_mutex_lock(&t->lock); + atomic_store(&t->work, WORK_SHUTDOWN); + pthread_cond_signal(&t->cond); + pthread_mutex_unlock(&t->lock); + pthread_join(t->tid, NULL); +} + +#endif + +static inline struct fy_thread *fy_thread_reserve_internal(struct fy_thread_pool *tp) +{ + struct fy_thread *t; + unsigned int slot; + _Atomic(uint64_t) *free; + uint64_t exp, v; + unsigned int i, num_threads_words; + + t = NULL; + + num_threads_words = FY_BIT64_COUNT(tp->num_threads); + for (i = 0, free = tp->freep; i < num_threads_words; i++, free++) { + v = atomic_load(free); + while (v) { + slot = FY_BIT64_LOWEST(v); + assert(v & FY_BIT64(slot)); + exp = v; /* expecting the previous value */ + v &= ~FY_BIT64(slot); /* clear this bit */ + if (atomic_compare_exchange_strong(free, &exp, v)) { + slot += i * 64; + t = tp->threads + slot; + assert(slot == t->id); + return t; + } + v = exp; + } + } + + return NULL; +} + +static inline void fy_thread_unreserve_internal(struct fy_thread *t) +{ + struct fy_thread_pool *tp; + _Atomic(uint64_t) *free; + + tp = t->tp; + free = tp->freep + (unsigned int)(t->id / 64); + atomic_fetch_or(free, FY_BIT64(t->id & 63)); +} + +static inline bool fy_thread_is_reserved_internal(struct fy_thread *t) +{ + struct fy_thread_pool *tp = t->tp; + _Atomic(uint64_t) *free; + + free = tp->freep + (unsigned int)(t->id / 64); + return !(atomic_load(free) & FY_BIT64(t->id & 63)); +} + +static inline bool fy_thread_pool_are_all_reserved_internal(struct fy_thread_pool *tp) +{ + _Atomic(uint64_t) *free; + uint64_t v, m; + unsigned int i, num_threads_words; + + num_threads_words = FY_BIT64_COUNT(tp->num_threads); + for (i = 0, free = tp->freep; i < num_threads_words - 1; i++, free++) { + v = atomic_load(free); + if (v != (uint64_t)-1) + return false; + } + v = atomic_load(free); + m = FY_BIT64(tp->num_threads & 63) - 1; + return (v & m) == m; +} + +static inline bool fy_thread_pool_is_any_reserved_internal(struct fy_thread_pool *tp) +{ + _Atomic(uint64_t) *free; + uint64_t v, m; + unsigned int i, num_threads_words; + + num_threads_words = FY_BIT64_COUNT(tp->num_threads); + for (i = 0, free = tp->freep; i < num_threads_words - 1; i++, free++) { + v = atomic_load(free); + if (!v) + return true; + } + v = atomic_load(free); + m = FY_BIT64(tp->num_threads & 63) - 1; + return !(v & m); +} + +struct fy_thread *fy_thread_reserve(struct fy_thread_pool *tp) +{ + if (!tp) + return NULL; + + /* only valid for non-work stealing thread pools */ + if (tp->cfg.flags & FYTPCF_STEAL_MODE) + return NULL; + + return fy_thread_reserve_internal(tp); +} + +void fy_thread_unreserve(struct fy_thread *t) +{ + struct fy_thread_pool *tp; + + if (!t) + return; + + tp = t->tp; + assert(tp); + + /* only valid for non-work stealing thread pools */ + if (tp->cfg.flags & FYTPCF_STEAL_MODE) + return; + + fy_thread_unreserve_internal(t); +} + +bool fy_thread_is_reserved(struct fy_thread *t) +{ + if (!t) + return false; + + return fy_thread_is_reserved_internal(t); +} + +bool fy_thread_pool_are_all_reserved(struct fy_thread_pool *tp) +{ + return fy_thread_pool_are_all_reserved_internal(tp); +} + +bool fy_thread_pool_is_any_reserved(struct fy_thread_pool *tp) +{ + return fy_thread_pool_is_any_reserved_internal(tp); +} + +int fy_thread_submit_work(struct fy_thread *t, struct fy_thread_work *work) +{ + if (!t || !work) + return -1; + + if (t->tp->cfg.flags & FYTPCF_STEAL_MODE) + return -1; + + return fy_thread_submit_work_internal(t, work); +} + +int fy_thread_wait_work(struct fy_thread *t) +{ + if (!t) + return -1; + + if (t->tp->cfg.flags & FYTPCF_STEAL_MODE) + return -1; + + return fy_thread_wait_work_internal(t); +} + +void fy_thread_pool_cleanup(struct fy_thread_pool *tp) +{ + struct fy_thread *t; + unsigned int i; + + if (!tp) + return; + + if (tp->threads) { + /* get out of steal mode */ + for (i = 0, t = tp->threads; i < tp->num_threads; i++, t++) { + fy_worker_thread_shutdown(t); + } + + fy_cacheline_free(tp->threads); + } + + memset(tp, 0, sizeof(*tp)); +} + +int fy_thread_pool_setup(struct fy_thread_pool *tp, const struct fy_thread_pool_cfg *cfg) +{ + struct fy_thread *t; + unsigned int i, num_threads, num_threads_words; + size_t size, free_offset, loot_offset, thread_bitmask_size; + void *(*start_routine)(void *); + long scval; + int rc __FY_DEBUG_UNUSED__; + + assert(tp); + + memset(tp, 0, sizeof(*tp)); + + if (!cfg) { + tp->cfg.flags = 0; + tp->cfg.num_threads = 0; + } else + tp->cfg = *cfg; + + if (!tp->cfg.num_threads) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_threads = (unsigned int)scval; + } else + num_threads = tp->cfg.num_threads; + + tp->num_threads = num_threads; + + num_threads_words = FY_BIT64_COUNT(tp->num_threads); + thread_bitmask_size = FY_BIT64_SIZE(tp->num_threads); + + /* the size of the threads, aligned to a cache line */ + size = FY_CACHELINE_SIZE_ALIGN(sizeof(*tp->threads) * tp->num_threads); + + /* the free bitmask array offset */ + free_offset = size; + size = FY_CACHELINE_SIZE_ALIGN(size + thread_bitmask_size); + + /* the loot bitmask array offset, aligned to a 64 byte cacheline */ + loot_offset = size; + size = FY_CACHELINE_SIZE_ALIGN(size + thread_bitmask_size); + + /* allocate everything in one go */ + tp->threads = fy_cacheline_alloc(size); + if (!tp->threads) + goto err_out; + + memset(tp->threads, 0, size); + + rc = pthread_key_create(&tp->key, NULL); + assert(!rc); + + tp->freep = (void *)tp->threads + free_offset; + tp->lootp = (void *)tp->threads + loot_offset; + + /* prime the thread free */ + for (i = 0; i < num_threads_words - 1; i++) + tp->freep[i] = (uint64_t)-1; + tp->freep[i] = FY_BIT64(tp->num_threads & 63) - 1; + + /* the lootp's are zero */ + + for (i = 0, t = tp->threads; i < tp->num_threads; i++, t++) { + + t->tp = tp; + t->id = i; + + fy_thread_init_sync(t); + } + + start_routine = !(tp->cfg.flags & FYTPCF_STEAL_MODE) ? + fy_worker_thread_standard : + fy_worker_thread_steal; + + for (i = 0, t = tp->threads; i < tp->num_threads; i++, t++) { + rc = pthread_create(&t->tid, NULL, start_routine, t); + if (rc) + goto err_out; + } + + return 0; + +err_out: + fy_thread_pool_cleanup(tp); + return -1; +} + +struct fy_thread_pool *fy_thread_pool_create(const struct fy_thread_pool_cfg *cfg) +{ + struct fy_thread_pool *tp; + int rc; + + tp = malloc(sizeof(*tp)); + if (!tp) + return NULL; + + rc = fy_thread_pool_setup(tp, cfg); + if (rc) { + free(tp); + return NULL; + } + + return tp; +} + +void fy_thread_pool_destroy(struct fy_thread_pool *tp) +{ + if (!tp) + return; + + fy_thread_pool_cleanup(tp); + free(tp); +} + +int fy_thread_pool_get_num_threads(struct fy_thread_pool *tp) +{ + if (!tp) + return -1; + + return (int)tp->num_threads; +} + +const struct fy_thread_pool_cfg *fy_thread_pool_get_cfg(struct fy_thread_pool *tp) +{ + if (!tp) + return NULL; + return &tp->cfg; +} + +void fy_thread_work_join(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn) +{ + if (!(tp->cfg.flags & FYTPCF_STEAL_MODE)) + fy_thread_work_join_standard(tp, works, work_count, check_fn); + else if (work_count == 2) + fy_thread_work_join_steal_2(tp, works, check_fn); + else + fy_thread_work_join_steal(tp, works, work_count, check_fn); +} + +void fy_thread_args_join(struct fy_thread_pool *tp, fy_work_exec_fn fn, fy_work_check_fn check_fn, void **args, size_t count) +{ + struct fy_thread_work *works; + size_t i; + + if (!count) + return; + + works = alloca(sizeof(*works) * count); + memset(works, 0, sizeof(*works) * count); + for (i = 0; i < count; i++) { + works[i].fn = fn; + works[i].arg = args ? args[i] : NULL; + } + + fy_thread_work_join(tp, works, count, check_fn); +} + +void fy_thread_arg_array_join(struct fy_thread_pool *tp, fy_work_exec_fn fn, fy_work_check_fn check_fn, void *args, size_t argsize, size_t count) +{ + struct fy_thread_work *works; + size_t i; + + if (!count) + return; + + works = alloca(sizeof(*works) * count); + memset(works, 0, sizeof(*works) * count); + for (i = 0; i < count; i++) { + works[i].fn = fn; + works[i].arg = args; + args += argsize; + } + + fy_thread_work_join(tp, works, count, check_fn); +} + +void fy_thread_arg_join(struct fy_thread_pool *tp, fy_work_exec_fn fn, fy_work_check_fn check_fn, void *arg, size_t count) +{ + struct fy_thread_work *works; + size_t i; + + if (!count) + return; + + works = alloca(sizeof(*works) * count); + memset(works, 0, sizeof(*works) * count); + for (i = 0; i < count; i++) { + works[i].fn = fn; + works[i].arg = arg; + } + + fy_thread_work_join(tp, works, count, check_fn); +} + +/* + * the standard (non-stealing implementation) + */ +static void *fy_worker_thread_standard(void *arg) +{ + struct fy_thread *t = arg; + struct fy_thread_pool *tp; + struct fy_thread_work *work; + + tp = t->tp; + + /* store per thread info */ + pthread_setspecific(tp->key, t); + + while ((work = fy_worker_wait_for_work(t)) != WORK_SHUTDOWN) { + work->fn(work->arg); + fy_worker_signal_work_done(t, work); + } + + return NULL; +} + +static void fy_thread_work_join_standard(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn) +{ + struct fy_thread_work **direct_work, **thread_work, *w; + struct fy_thread **threads, *t; + size_t i, direct_work_count, thread_work_count; + int rc; + + /* just a single (or no) work, or no threads? execute directly */ + if (work_count <= 1 || !tp || !tp->num_threads) { + for (i = 0, w = works; i < work_count; i++, w++) + w->fn(w->arg); + return; + } + + /* allocate the keeper of direct work */ + direct_work = alloca(work_count * sizeof(*direct_work)); + direct_work_count = 0; + + threads = alloca(work_count * sizeof(*threads)); + thread_work = alloca(work_count * sizeof(*thread_work)); + thread_work_count = 0; + + for (i = 0, w = works; i < work_count; i++, w++) { + + t = NULL; + if (!check_fn || check_fn(w->arg)) + t = fy_thread_reserve_internal(tp); + + if (t) { + threads[thread_work_count] = t; + thread_work[thread_work_count++] = w; + } else + direct_work[direct_work_count++] = w; + } + + /* if we don't have any direct_work, steal the last threaded work as direct */ + if (!direct_work_count) { + assert(thread_work_count > 0); + t = threads[thread_work_count - 1]; + w = thread_work[thread_work_count - 1]; + thread_work_count--; + + /* unreserve this */ + fy_thread_unreserve_internal(t); + direct_work[direct_work_count++] = w; + } + + /* submit the threaded work */ + for (i = 0; i < thread_work_count; i++) { + t = threads[i]; + w = thread_work[i]; + rc = fy_thread_submit_work_internal(t, w); + if (rc) { + /* unable to submit? remove work, and move to direct */ + threads[i] = NULL; + thread_work[i] = NULL; + fy_thread_unreserve_internal(t); + direct_work[direct_work_count++] = w; + } + } + + /* now perform the direct work while the threaded work is being performed in parallel */ + for (i = 0; i < direct_work_count; i++) { + w = direct_work[i]; + w->fn(w->arg); + } + + /* finally wait for all threaded work to complete */ + for (i = 0; i < thread_work_count; i++) { + t = threads[i]; + assert(t); + fy_thread_wait_work_internal(t); + fy_thread_unreserve_internal(t); + } +} + +/* + * the stealing implementation + */ +static inline struct fy_work_pool *fy_work_pool_init(struct fy_work_pool *wp, size_t work_count) +{ +#if !(defined(__linux__) && !defined(FY_THREAD_PORTABLE)) + int rc __FY_DEBUG_UNUSED__; +#endif + + if (!wp) + return NULL; + + atomic_store(&wp->work_left, work_count); +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + atomic_store(&wp->done, !work_count); +#elif defined(__APPLE__) + wp->sem = dispatch_semaphore_create(!work_count); + assert(wp->sem != NULL); + (void)rc; +#else + rc = sem_init(&wp->sem, 0, !work_count); + assert(!rc); +#endif + return wp; +} + +static inline void fy_work_pool_cleanup(struct fy_work_pool *wp) +{ +#if !(defined(__linux__) && !defined(FY_THREAD_PORTABLE)) + int rc __FY_DEBUG_UNUSED__; +#endif + + if (!wp) + return; + +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + /* nothing */ +#elif defined(__APPLE__) + /* nothing */ + (void)rc; +#else + rc = sem_destroy(&wp->sem); + assert(!rc); +#endif +} + +static inline bool fy_work_pool_signal(struct fy_work_pool *wp) +{ + size_t prev_work_left; + int rc __FY_DEBUG_UNUSED__; + + if (!wp) + return false; + + prev_work_left = atomic_fetch_sub(&wp->work_left, 1); + if (prev_work_left == 1) { +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + rc = fpost(&wp->done); + assert(!rc); +#elif defined(__APPLE__) + dispatch_semaphore_signal(wp->sem); + (void)rc; +#else + rc = sem_post(&wp->sem); + assert(!rc); +#endif + return true; + } + return false; +} + +static inline void fy_work_pool_wait(struct fy_work_pool *wp) +{ + int rc __FY_DEBUG_UNUSED__; + + if (!wp) + return; + + /* if there's any work left, wait for it */ + while (atomic_load(&wp->work_left) > 0) { +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + rc = fwait(&wp->done); +#elif defined(__APPLE__) + dispatch_semaphore_wait(wp->sem, DISPATCH_TIME_FOREVER); + (void)rc; +#else + rc = sem_wait(&wp->sem); +#endif + assert(!rc || (rc == -1 && errno == EAGAIN)); + } +} + +static inline void fy_worker_thread_steal_execute(struct fy_thread *t, struct fy_thread_work *w) +{ + struct fy_work_pool *wp; + bool signalled __FY_DEBUG_UNUSED__; + + TDBG("%s: T#%u worker executing W:%p\n", __func__, t->id, w); + wp = w->wp; + assert(wp); + w->fn(w->arg); + TDBG("%s: T#%u worker executed W:%p\n", __func__, t->id, w); + + signalled = fy_work_pool_signal(wp); + (void)signalled; + TDBG("%s: T#%u W:%p WP:%p signalled=%s\n", __func__, + t->id, w, wp, signalled ? "true" : "false"); +} + +static inline struct fy_thread_work *fy_worker_thread_steal_work(struct fy_thread_pool *tp, struct fy_thread *t_thief) +{ + struct fy_thread *t; + unsigned int slot; + _Atomic(uint64_t) *loot; + uint64_t exp, v; + unsigned int i, num_threads_words; + struct fy_thread_work *w, *w_exp; + + /* the threads that have work to steal, have the loot bit set */ + t = NULL; + + num_threads_words = FY_BIT64_COUNT(tp->num_threads); + for (i = 0, loot = tp->lootp; i < num_threads_words; i++, loot++) { + v = atomic_load(loot); + while (v) { + slot = FY_BIT64_LOWEST(v); + assert(v & FY_BIT64(slot)); + exp = v; /* expecting the previous value */ + v &= ~FY_BIT64(slot); /* clear this bit */ + + if (atomic_compare_exchange_strong(loot, &exp, v)) { + slot += i * 64; + t = tp->threads + slot; + if ((w = atomic_load(&t->next_work)) != NULL) { + w_exp = w; + if (atomic_compare_exchange_strong(&t->next_work, &w_exp, NULL)) + return w; + } + } + v = exp; + } + } + + return NULL; +} + +static void *fy_worker_thread_steal(void *arg) +{ + struct fy_thread *t = arg; + struct fy_thread_pool *tp; + struct fy_thread_work *w, *w_exp, *w_stolen, *w_last; + + tp = t->tp; + + /* store per thread info */ + pthread_setspecific(tp->key, t); + + TDBG("%s: T#%u in steal mode\n", __func__, t->id); + + while ((w = fy_worker_wait_for_work(t)) != WORK_SHUTDOWN) { + + assert(fy_thread_is_reserved_internal(t)); + + for (;;) { + fy_worker_thread_steal_execute(t, w); + w_last = w; + w_stolen = fy_worker_thread_steal_work(tp, t); + w = NULL; + if (!w_stolen) + break; + + TDBG("%s: T#%u stole W:%p\n", __func__, t->id, w_stolen); + w_exp = w_last; + if (!atomic_compare_exchange_strong(&t->work, &w_exp, w_stolen)) { + assert(w_exp != WORK_SHUTDOWN); + TDBG("%s: T#%u t->work:%p w:%p w_stolen:%p\n", __func__, t->id, atomic_load(&t->work), w, w_stolen); + abort(); + } + w = w_stolen; + } + + /* unreserve first */ + fy_thread_unreserve_internal(t); + + w_exp = w_last; + if (!atomic_compare_exchange_strong(&t->work, &w_exp, NULL)) { + assert(w_exp != WORK_SHUTDOWN); + break; + } + } + TDBG("%s: T#%u leaving steal mode\n", __func__, t->id); + + return NULL; +} + +static void fy_thread_work_join_steal(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn) +{ + struct fy_work_pool wp_local, *wp; + struct fy_thread_work *dw, *expw; + struct fy_thread *t, *tw; + bool has_loot, resolved_t; + int rc __FY_DEBUG_UNUSED__; + int tid __FY_DEBUG_UNUSED__; + + t = NULL; + resolved_t = false; + tid = -1; + (void)tid; + +#ifdef FY_THREAD_DEBUG + t = pthread_getspecific(tp->key); + tid = t ? (int)t->id : -1; + resolved_t = true; +#else + t = NULL; + tid = -1; + resolved_t = false; +#endif + + dw = NULL; + wp = NULL; + + while (work_count > 0) { + if (!dw) { + dw = works++; + work_count--; + + TDBG("%s: T#%d sdir W:%p\n", __func__, tid, dw); + + continue; + } + + has_loot = false; + if (work_count > 0 && (!check_fn || check_fn(works->arg))) { + while (work_count > 0 && (tw = fy_thread_reserve_internal(tp)) != NULL) { + + assert(!works->wp); + + if (!wp) + wp = fy_work_pool_init(&wp_local, work_count + !!dw); + + works->wp = wp; + expw = NULL; + + rc = fy_thread_submit_work_internal(tw, works); + assert(!rc); + + TDBG("%s: T#%d post W:%p to T#%u\n", __func__, tid, works, tw->id); + + works++; + work_count--; + } + + if (work_count > 0) { + + if (!resolved_t) { + t = pthread_getspecific(tp->key); + tid = t ? (int)t->id : -1; + resolved_t = true; + } + + if (t && atomic_load(&t->next_work) == NULL) { + TDBG("%s: T#%d could not post, available to steal W:%p\n", __func__, tid, works); + assert(works->wp == NULL); + + if (!wp) + wp = fy_work_pool_init(&wp_local, work_count + !!dw); + + works->wp = wp; + + expw = NULL; + if (!atomic_compare_exchange_strong(&t->next_work, &expw, works)) { + TDBG("%s: T#%d could not update next_work: W:%p\n", __func__, tid, works); + abort(); + } + + /* set the has loot bit */ + atomic_fetch_or(tp->lootp + (unsigned int)(t->id / 64), FY_BIT64(t->id & 63)); + + has_loot = true; + } + } + } + + if (dw) { + TDBG("%s: T#%d exec W:%p\n", __func__, tid, dw); + + /* execute the direct work */ + dw->fn(dw->arg); + dw = NULL; + + fy_work_pool_signal(wp); + } + + if (has_loot) { + assert(t); + expw = works; + + /* clear the has loot bit unconditionally */ + atomic_fetch_and(tp->lootp + (unsigned int)(t->id / 64), ~FY_BIT64(t->id & 63)); + + if (!atomic_compare_exchange_strong(&t->next_work, &expw, NULL)) { + TDBG("%s: T#%d had W:%p stolen, good\n", __func__, tid, works); + work_count--; + works++; + } else { + TDBG("%s: T#%d had W:%p not stolen, bad\n", __func__, tid, works); + } + } + } + + /* last out and direct work */ + if (dw) { + TDBG("%s: T#%d executing final direct W:%p\n", __func__, tid, dw); + + dw->fn(dw->arg); + dw = NULL; + + fy_work_pool_signal(wp); + } + + TDBG("%s: T#%d wait WP:%p\n", __func__, tid, wp); + + fy_work_pool_wait(wp); + fy_work_pool_cleanup(wp); + + TDBG("%s: T#%d done WP:%p\n", __func__, tid, wp); +} + +static void fy_thread_work_join_steal_2(struct fy_thread_pool *tp, struct fy_thread_work works[2], fy_work_check_fn check_fn) +{ + struct fy_work_pool wp_local, *wp; + struct fy_thread_work *expw; + struct fy_thread *t, *tw; + bool has_loot, pushed, resolved_t; + int rc __FY_DEBUG_UNUSED__; + int tid __FY_DEBUG_UNUSED__; + + t = NULL; + resolved_t = false; + tid = -1; + + (void)tid; + +#ifdef FY_THREAD_DEBUG + t = pthread_getspecific(tp->key); + tid = t ? (int)t->id : -1; + resolved_t = true; +#else + t = NULL; + tid = -1; + resolved_t = false; +#endif + + wp = NULL; + + pushed = false; + has_loot = false; + if (!check_fn || check_fn(works->arg)) { + if ((tw = fy_thread_reserve_internal(tp)) != NULL) { + + assert(!works[1].wp); + + if (!wp) + wp = fy_work_pool_init(&wp_local, 1); + + works[1].wp = wp; + expw = NULL; + + rc = fy_thread_submit_work_internal(tw, &works[1]); + assert(!rc); + + TDBG("%s: T#%d post W:%p to T#%u\n", __func__, tid, &works[1], tw->id); + + pushed = true; + + } else { + if (!resolved_t) { + t = pthread_getspecific(tp->key); + tid = t ? (int)t->id : -1; + resolved_t = true; + } + + if (t && atomic_load(&t->next_work) == NULL) { + TDBG("%s: T#%d could not post, available to steal W:%p\n", __func__, tid, &works[1]); + assert(works[1].wp == NULL); + + if (!wp) + wp = fy_work_pool_init(&wp_local, 1); + + works[1].wp = wp; + + expw = NULL; + if (!atomic_compare_exchange_strong(&t->next_work, &expw, &works[1])) { + TDBG("%s: T#%d could not update next_work: W:%p\n", __func__, tid, &works[1]); + abort(); + } + + /* set the has loot bit */ + atomic_fetch_or(tp->lootp + (unsigned int)(t->id / 64), FY_BIT64(t->id & 63)); + + has_loot = true; + } + } + } + + TDBG("%s: T#%d exec W:%p (left)\n", __func__, tid, &works[0]); + + /* execute the direct work */ + works[0].fn(works[0].arg); + + if (has_loot) { + assert(t); + expw = &works[1]; + + /* clear the has loot bit unconditionally */ + atomic_fetch_and(tp->lootp + (unsigned int)(t->id / 64), ~FY_BIT64(t->id & 63)); + + if (!atomic_compare_exchange_strong(&t->next_work, &expw, NULL)) { + TDBG("%s: T#%d had W:%p stolen\n", __func__, tid, &works[1]); + } else { + TDBG("%s: T#%d had W:%p not stolen\n", __func__, tid, &works[1]); + + TDBG("%s: T#%d exec W:%p (right)\n", __func__, tid, &works[1]); + /* execute the direct work */ + works[1].fn(works[1].arg); + fy_work_pool_signal(wp); + } + + } else if (!pushed) { + + TDBG("%s: T#%d exec W:%p (right)\n", __func__, tid, &works[1]); + /* execute the direct work */ + works[1].fn(works[1].arg); + fy_work_pool_signal(wp); + } + + TDBG("%s: T#%d wait WP:%p\n", __func__, tid, wp); + + fy_work_pool_wait(wp); + fy_work_pool_cleanup(wp); + + TDBG("%s: T#%d done WP:%p\n", __func__, tid, wp); +} diff --git a/src/thread/fy-thread.h b/src/thread/fy-thread.h new file mode 100644 index 0000000..4ce45ac --- /dev/null +++ b/src/thread/fy-thread.h @@ -0,0 +1,76 @@ +/* + * fy-thread.h - Lighting fast thread pool implementation + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_THREAD_H +#define FY_THREAD_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include + +#ifndef __APPLE__ +#include +#else +#include +#endif + +#include "fy-bit64.h" +#include "fy-align.h" + +#include + +// #define FY_THREAD_DEBUG /* define to enable debugging information to stderr */ +// #define FY_THREAD_PORTABLE /* define to use the portable implementation even on linux */ + +struct fy_work_pool { + _Atomic(size_t) work_left; +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + _Atomic(uint32_t) done; +#elif defined(__APPLE__) + dispatch_semaphore_t sem; +#else + sem_t sem; +#endif +}; + +struct fy_thread { + struct fy_thread_pool *tp; + unsigned int id; + pthread_t tid; + _Atomic(struct fy_thread_work *)work; + _Atomic(struct fy_thread_work *)next_work; +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + _Atomic(uint32_t) submit; + _Atomic(uint32_t) done; +#else + pthread_mutex_t lock; + pthread_cond_t cond; + pthread_mutex_t wait_lock; + pthread_cond_t wait_cond; +#endif +}; + +struct fy_thread_pool { + struct fy_thread_pool_cfg cfg; + unsigned int num_threads; + struct fy_thread *threads; + _Atomic(uint64_t) *freep; + _Atomic(uint64_t) *lootp; + pthread_key_t key; +}; + +/* those are internal only */ +int fy_thread_pool_setup(struct fy_thread_pool *tp, const struct fy_thread_pool_cfg *cfg); +void fy_thread_pool_cleanup(struct fy_thread_pool *tp); + +#endif diff --git a/src/tool/fy-tool.c b/src/tool/fy-tool.c index eed9d47..52d0d56 100644 --- a/src/tool/fy-tool.c +++ b/src/tool/fy-tool.c @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -66,6 +67,7 @@ #define OPT_PARSE_DUMP 1007 #define OPT_YAML_VERSION_DUMP 1008 #define OPT_COMPOSE 1009 +#define OPT_B3SUM 1010 #define OPT_STRIP_LABELS 2000 #define OPT_STRIP_TAGS 2001 @@ -98,6 +100,20 @@ #define OPT_YAML_1_2 4001 #define OPT_YAML_1_3 4002 +/* b3sum options */ +#define OPT_CHECK 5000 +#define OPT_DERIVE_KEY 5001 +#define OPT_NO_NAMES 5002 +#define OPT_RAW 5003 +#define OPT_KEYED 5005 +#define OPT_LENGTH 5006 +#define OPT_LIST_BACKENDS 5007 +#define OPT_BACKEND 5008 +#define OPT_NUM_THREADS 5009 +#define OPT_FILE_BUFFER 5010 +#define OPT_MMAP_MIN_CHUNK 5011 +#define OPT_MMAP_MAX_CHUNK 5012 + static struct option lopts[] = { {"include", required_argument, 0, 'I' }, {"debug-level", required_argument, 0, 'd' }, @@ -123,6 +139,7 @@ static struct option lopts[] = { {"compose", no_argument, 0, OPT_COMPOSE }, {"dump-path", no_argument, 0, OPT_DUMP_PATH }, {"yaml-version-dump", no_argument, 0, OPT_YAML_VERSION_DUMP }, + {"b3sum", no_argument, 0, OPT_B3SUM }, {"strip-labels", no_argument, 0, OPT_STRIP_LABELS }, {"strip-tags", no_argument, 0, OPT_STRIP_TAGS }, {"strip-doc", no_argument, 0, OPT_STRIP_DOC }, @@ -153,6 +170,20 @@ static struct option lopts[] = { {"to", required_argument, 0, 'T' }, {"from", required_argument, 0, 'F' }, {"quiet", no_argument, 0, 'q' }, + + {"check", no_argument, 0, OPT_CHECK }, + {"derive-key", required_argument, 0, OPT_DERIVE_KEY }, + {"no-names", no_argument, 0, OPT_NO_NAMES }, + {"raw", no_argument, 0, OPT_RAW }, + {"length", required_argument, 0, OPT_LENGTH }, + {"keyed", no_argument, 0, OPT_KEYED }, + {"list-backends", no_argument, 0, OPT_LIST_BACKENDS }, + {"backend", required_argument, 0, OPT_BACKEND }, + {"num-threads", required_argument, 0, OPT_NUM_THREADS }, + {"file-buffer", required_argument, 0, OPT_FILE_BUFFER }, + {"mmap-min-chunk", required_argument, 0, OPT_MMAP_MIN_CHUNK }, + {"mmap-max-chunk", required_argument, 0, OPT_MMAP_MAX_CHUNK }, + {"help", no_argument, 0, 'h' }, {"version", no_argument, 0, 'v' }, {0, 0, 0, 0 }, @@ -377,7 +408,24 @@ static void display_usage(FILE *fp, char *progname, int tool_mode) fprintf(fp, "\t{\n\t \"foo\": \"bar\"\n\t}\n"); break; case OPT_YAML_VERSION_DUMP: - fprintf(fp, "\tDisplay information about the YAML versions libfyaml supports)\n"); + fprintf(fp, "\tDisplay information about the YAML versions libfyaml supports\n"); + fprintf(fp, "\n"); + break; + + case OPT_B3SUM: + fprintf(fp, "\tBLAKE3 hash b3sum utility\n"); + fprintf(fp, "\t--derive-key : Key derivation mode, with the given context string\n"); + fprintf(fp, "\t--no-names : Omit filenames\n"); + fprintf(fp, "\t--raw : Output result in raw bytes (single input allowed)\n"); + fprintf(fp, "\t--length : Output only this amount of bytes per output (max %u)\n", FY_BLAKE3_OUT_LEN); + fprintf(fp, "\t--check : Read files with BLAKE3 checksums and check files\n"); + fprintf(fp, "\t--keyed : Keyed mode with secret key read from (32 raw bytes)\n"); + fprintf(fp, "\t--backend : Select a BLAKE3 backend instead of the default\n"); + fprintf(fp, "\t--list-backends : Print out a list of available backends\n"); + fprintf(fp, "\t--num-threads : Number of threads, -1 disable, 0 let system decide, >= 1 explicit\n"); + fprintf(fp, "\t--file-buffer : Size of file I/O buffer (non-mmap case), 0 let system decide\n"); + fprintf(fp, "\t--mmap-min-chunk : Size of minimum mmap chunk, 0 let system decide\n"); + fprintf(fp, "\t--mmap-max-chunk : Size of maximum mmap chunk, 0 let system decide\n"); fprintf(fp, "\n"); break; } @@ -1538,6 +1586,299 @@ err_out: return FYCR_ERROR; } +struct b3sum_config { + bool no_names : 1, + raw : 1, + keyed : 1, + check : 1, + derive_key : 1, + quiet : 1, + list_backends : 1, + no_mmap : 1; + size_t file_buffer; + size_t mmap_min_chunk; + size_t mmap_max_chunk; + unsigned int length; + const char *context; + size_t context_len; + const char *backend; + unsigned int num_threads; +}; + +struct b3sum_config default_b3sum_cfg = { + .length = FY_BLAKE3_OUT_LEN, +}; + +static int do_b3sum_hash_file(struct fy_blake3_hasher *hasher, const char *filename, bool no_names, bool raw, unsigned int length) +{ + static const char *hexb = "0123456789abcdef"; + const uint8_t *output; + size_t filename_sz, line_sz, outsz; + ssize_t wrn; + uint8_t v; + const void *outp; + char *line, *s; + unsigned int i; + + filename_sz = strlen(filename); + + output = fy_blake3_hash_file(hasher, filename); + if (!output) { + fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno)); + return -1; + } + + if (!raw) { + /* output line (optimized) */ + line_sz = (length * 2); /* the hex output */ + if (!no_names) + line_sz += 2 + filename_sz; /* 2 spaces + filename */ + line_sz++; /* '\n' */ + line = alloca(line_sz + 1); + s = line; + for (i = 0; i < length; i++) { + v = output[i]; + *s++ = hexb[v >> 4]; + *s++ = hexb[v & 15]; + } + if (!no_names) { + *s++ = ' '; + *s++ = ' '; + memcpy(s, filename, filename_sz); + s += filename_sz; + } + *s++ = '\n'; + *s = '\0'; + outp = line; + outsz = (size_t)(s - line); + } else { + outp = output; + outsz = length; + } + + wrn = fwrite(outp, 1, outsz, stdout); + if ((size_t)wrn != outsz) { + fprintf(stderr, "Unable to write to stdout! error: %s\n", strerror(errno)); + return -1; + } + + return 0; +} + +static int do_b3sum_check_file(struct fy_blake3_hasher *hasher, const char *check_filename, bool quiet) +{ + char *hash, *filename; + FILE *fp = NULL; + char linebuf[8192]; /* maximum size for a line is 8K, should be enough (PATH_MAX is 4K at linux) */ + uint8_t read_hash[FY_BLAKE3_OUT_LEN], v; + const uint8_t *computed_hash; + char *s; + char c; + unsigned int i, j, length; + size_t linesz; + int line, exit_code; + + if (check_filename && strcmp(check_filename, "-")) { + fp = fopen(check_filename, "ra"); + if (!fp) { + fprintf(stderr, "Failed to open check file: \"%s\", error: %s\n", check_filename, strerror(errno)); + goto err_out; + } + } else { + fp = stdin; + } + + /* default error code if all is fine */ + exit_code = 0; + + line = 0; + while (fgets(linebuf, sizeof(linebuf), fp)) { + /* '\0' terminate always */ + linebuf[(sizeof(linebuf)/sizeof(linebuf[0]))-1] = '\0'; + + linesz = strlen(linebuf); + while (linesz > 0 && linebuf[linesz-1] == '\n') + linesz--; + + if (!linesz) { + fprintf(stderr, "Empty line found at file \"%s\" line #%d\n", check_filename, line); + goto err_out; + } + linebuf[linesz] = '\0'; + + length = 0; + s = linebuf; + while (isxdigit(*s)) + s++; + + length = s - linebuf; + + if (length == 0 || length > (FY_BLAKE3_OUT_LEN * 2) || (length % 1) || !isspace(*s)) { + fprintf(stderr, "Bad line found at file \"%s\" line #%d\n", check_filename, line); + fprintf(stderr, "%s\n", linebuf); + goto err_out; + } + + *s++ = '\0'; + + while (isspace(*s)) + s++; + + length >>= 1; /* to bytes */ + hash = linebuf; + filename = s; + + for (i = 0, s = hash; i < length; i++) { + v = 0; + for (j = 0; j < 2; j++) { + v <<= 4; + c = *s++; + if (c >= '0' && c <= '9') + v |= c - '0'; + else if (c >= 'a' && c <= 'f') + v |= c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + v |= c - 'A' + 10; + else + v = 0; + } + read_hash[i] = v; + } + + computed_hash = fy_blake3_hash_file(hasher, filename); + if (!computed_hash) { + fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno)); + goto err_out; + } + + /* constant time comparison */ + v = 0; + for (i = 0; i < length; i++) + v |= (read_hash[i] ^ computed_hash[i]); + + if (v) { + printf("%s: FAILED\n", filename); + exit_code = -1; + } else if (!quiet) + printf("%s: OK\n", filename); + } + +out: + if (fp && fp != stdin) + fclose(fp); + + return exit_code; + +err_out: + exit_code = -1; + goto out; +} + +static int +do_b3sum(int argc, char *argv[], int optind, const struct b3sum_config *cfg) +{ + struct fy_blake3_hasher_cfg hcfg; + struct fy_blake3_hasher *hasher; + uint8_t key[FY_BLAKE3_OUT_LEN]; + const char *filename; + int rc, num_inputs, num_ok, i; + size_t rdn; + const char *backend, *prev; + + if (cfg->list_backends) { + prev = NULL; + while ((backend = fy_blake3_backend_iterate(&prev)) != NULL) + printf("%s\n", backend); + return 0; + } + + if (cfg->quiet && !cfg->check) { + fprintf(stderr, "Error: --quiet may only be used together with --check\n\n"); + return 1; + } + + if (cfg->keyed && cfg->derive_key) { + fprintf(stderr, "Error: --keyed and --derive-key may not be used together\n\n"); + return 1; + } + + if (cfg->check && cfg->length != FY_BLAKE3_OUT_LEN) { + fprintf(stderr, "Error: --check and --length may not be used together\n\n"); + return 1; + } + + if (cfg->keyed) { + rdn = fread(key, 1, FY_BLAKE3_KEY_LEN, stdin); + if (rdn != FY_BLAKE3_KEY_LEN) { + if (rdn >= 0 && rdn < FY_BLAKE3_KEY_LEN) + fprintf(stderr, "Error: could not read secret key from : short key\n\n"); + else + fprintf(stderr, "Error: could not read secret key from : error %s\n\n", strerror(errno)); + return 1; + } + rc = fgetc(stdin); + if (rc != EOF) { + fprintf(stderr, "Error: garbage trailing secret key from \n\n"); + return -1; + } + } + + num_inputs = argc - optind; + if (num_inputs <= 0) + num_inputs = 1; /* stdin mode */ + + if (cfg->raw && num_inputs > 1) { + fprintf(stderr, "Error: Raw output mode is only supported with a single input\n\n"); + return 1; + } + + /* we can't handle '-' in keyed mode */ + if (cfg->keyed) { + for (i = optind; i < argc; i++) { + if (!strcmp(argv[i], "-")) { + fprintf(stderr, "Cannot use in keyed mode\n"); + return 1; + } + } + } + + memset(&hcfg, 0, sizeof(hcfg)); + hcfg.key = cfg->keyed ? key : NULL; + hcfg.context = cfg->derive_key ? cfg->context : NULL; + hcfg.context_len = cfg->derive_key ? cfg->context_len : 0; + hcfg.backend = cfg->backend; + hcfg.no_mmap = cfg->no_mmap; + hcfg.file_buffer = cfg->file_buffer; + hcfg.mmap_min_chunk = cfg->mmap_min_chunk; + hcfg.mmap_max_chunk = cfg->mmap_max_chunk; + hcfg.num_threads = cfg->num_threads; + hasher = fy_blake3_hasher_create(&hcfg); + if (!hasher) { + fprintf(stderr, "unable to create blake3 hasher\n"); + return -1; + } + + /* we will get in the loop even when no arguments (we'll do stdin instead) */ + num_ok = 0; + i = optind; + do { + /* if no arguments, use stdin */ + filename = i < argc ? argv[i] : "-"; + + if (!cfg->check) + rc = do_b3sum_hash_file(hasher, filename, cfg->no_names, cfg->raw, cfg->length); + else + rc = do_b3sum_check_file(hasher, filename, cfg->quiet); + if (!rc) + num_ok++; + + } while (++i < argc); + + fy_blake3_hasher_destroy(hasher); + + return num_inputs == num_ok ? 0 : -1; +} + int main(int argc, char *argv[]) { struct fy_parse_cfg cfg = { @@ -1555,7 +1896,7 @@ int main(int argc, char *argv[]) struct fy_emitter_cfg emit_cfg; struct fy_parser *fyp = NULL; struct fy_emitter *fye = NULL; - int rc, exitcode = EXIT_FAILURE, opt, lidx, count, i, j, step = 1; + int rc, exitcode = EXIT_FAILURE, opt, lidx, i, j, step = 1; enum fy_error_module errmod; unsigned int errmod_mask; bool show; @@ -1599,6 +1940,9 @@ int main(int argc, char *argv[]) struct composer_data cd; bool dump_path = DUMP_PATH_DEFAULT; const char *input_arg; + /* b3sum */ + int opti; + struct b3sum_config b3cfg = default_b3sum_cfg; fy_valgrind_check(&argc, &argv); @@ -1629,6 +1973,8 @@ int main(int argc, char *argv[]) tool_mode = OPT_COMPOSE; else if (!strcmp(progname, "fy-yaml-version-dump")) tool_mode = OPT_YAML_VERSION_DUMP; + else if (!strcmp(progname, "fy-b3sum")) + tool_mode = OPT_B3SUM; else tool_mode = OPT_TOOL; @@ -1668,8 +2014,7 @@ int main(int argc, char *argv[]) indent = atoi(optarg); if (indent < 0 || indent > FYECF_INDENT_MASK) { fprintf(stderr, "bad indent option %s\n", optarg); - display_usage(stderr, progname, tool_mode); - return EXIT_FAILURE; + goto err_out_usage; } break; @@ -1677,16 +2022,14 @@ int main(int argc, char *argv[]) width = atoi(optarg); if (width < 0 || width > FYECF_WIDTH_MASK) { fprintf(stderr, "bad width option %s\n", optarg); - display_usage(stderr, progname, tool_mode); - return EXIT_FAILURE; + goto err_out_usage; } break; case 'd': dcfg.level = fy_string_to_error_type(optarg); if (dcfg.level == FYET_MAX) { fprintf(stderr, "bad debug level option %s\n", optarg); - display_usage(stderr, progname, tool_mode); - return EXIT_FAILURE; + goto err_out_usage; } break; case OPT_DISABLE_DIAG: @@ -1697,8 +2040,7 @@ int main(int argc, char *argv[]) errmod = fy_string_to_error_module(optarg); if (errmod == FYEM_MAX) { fprintf(stderr, "bad error module option %s\n", optarg); - display_usage(stderr, progname, tool_mode); - return EXIT_FAILURE; + goto err_out_usage; } errmod_mask = FY_BIT(errmod); } @@ -1722,8 +2064,7 @@ int main(int argc, char *argv[]) } else { fprintf(stderr, "bad %s option %s\n", show ? "show" : "hide", optarg); - display_usage(stderr, progname, tool_mode); - return EXIT_FAILURE; + goto err_out_usage; } break; @@ -1751,16 +2092,14 @@ int main(int argc, char *argv[]) du.colorize = false; } else { fprintf(stderr, "bad color option %s\n", optarg); - display_usage(stderr, progname, tool_mode); - return EXIT_FAILURE; + goto err_out_usage; } break; case 'm': rc = apply_mode_flags(optarg, &emit_flags); if (rc) { fprintf(stderr, "bad mode option %s\n", optarg); - display_usage(stderr, progname, tool_mode); - return EXIT_FAILURE; + goto err_out_usage; } break; case 'V': @@ -1774,6 +2113,7 @@ int main(int argc, char *argv[]) dcfg.output_fn = no_diag_output_fn; dcfg.fp = NULL; dcfg.colorize = false; + b3cfg.quiet = true; break; case 'f': file = optarg; @@ -1797,6 +2137,7 @@ int main(int argc, char *argv[]) case OPT_PARSE_DUMP: case OPT_COMPOSE: case OPT_YAML_VERSION_DUMP: + case OPT_B3SUM: tool_mode = opt; break; case OPT_STRIP_LABELS: @@ -1824,8 +2165,7 @@ int main(int argc, char *argv[]) cfg.flags |= FYPCF_JSON_FORCE; else { fprintf(stderr, "bad json option %s\n", optarg); - display_usage(stderr, progname, tool_mode); - return EXIT_FAILURE; + goto err_out_usage; } break; case OPT_DISABLE_ACCEL: @@ -1839,6 +2179,7 @@ int main(int argc, char *argv[]) break; case OPT_DISABLE_MMAP: cfg.flags |= FYPCF_DISABLE_MMAP_OPT; + b3cfg.no_mmap = true; break; case OPT_DUMP_PATHEXPR: dump_pathexpr = true; @@ -1888,6 +2229,74 @@ int main(int argc, char *argv[]) case OPT_TSV_FORMAT: tsv_format = true; break; + + case OPT_DERIVE_KEY: + b3cfg.derive_key = true; + b3cfg.context = optarg; + b3cfg.context_len = strlen(optarg); + break; + case OPT_NO_NAMES: + b3cfg.no_names = true; + break; + case OPT_RAW: + b3cfg.raw = true; + break; + case OPT_CHECK: + b3cfg.check = true; + break; + case OPT_KEYED: + b3cfg.keyed = true; + break; + + case OPT_LENGTH: + opti = atoi(optarg); + if (opti <= 0 || opti > FY_BLAKE3_OUT_LEN) { + fprintf(stderr, "Error: bad length=%d (must be > 0 and <= %u)\n\n", opti, FY_BLAKE3_OUT_LEN); + goto err_out_usage; + } + b3cfg.length = (unsigned int)opti; + break; + + case OPT_LIST_BACKENDS: + b3cfg.list_backends = true; + break; + + case OPT_BACKEND: + b3cfg.backend = optarg; + break; + + case OPT_NUM_THREADS: + b3cfg.num_threads = atoi(optarg); + break; + + case OPT_FILE_BUFFER: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad file-buffer=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + b3cfg.file_buffer = (size_t)opti; + break; + + case OPT_MMAP_MIN_CHUNK: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad mmap-min-chunk=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + b3cfg.mmap_min_chunk = (size_t)opti; + break; + + case OPT_MMAP_MAX_CHUNK: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad mmap-max-chunk=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + b3cfg.mmap_max_chunk = (size_t)opti; + break; + + case 'h' : default: if (opt != 'h') @@ -1900,6 +2309,17 @@ int main(int argc, char *argv[]) } } + if (tool_mode == OPT_B3SUM) { + + rc = do_b3sum(argc, argv, optind, &b3cfg); + if (rc == 1) { + /* display usage */ + goto err_out_usage; + } + exitcode = !rc ? EXIT_SUCCESS : EXIT_FAILURE; + goto cleanup; + } + if (tool_mode == OPT_YAML_VERSION_DUMP) { const struct fy_version *vers; void *iter; @@ -2064,7 +2484,6 @@ int main(int argc, char *argv[]) break; case OPT_DUMP: - count = 0; for (i = optind; ; i++) { if (optind < argc) { @@ -2093,19 +2512,17 @@ int main(int argc, char *argv[]) if (rc) goto cleanup; - count++; } } else { while ((fyev = fy_parser_parse(fyp)) != NULL) { if (!null_output) { - rc = fy_emit_event(fye, fyev); + rc = fy_emit_event_from_parser(fye, fyp, fyev); if (rc) goto cleanup; } else { fy_parser_event_free(fyp, fyev); } } - count++; } if (fy_parser_get_stream_error(fyp)) @@ -2131,7 +2548,6 @@ int main(int argc, char *argv[]) goto cleanup; } - count = 0; while ((fyd = fy_parse_load_document(fyp)) != NULL) { for (i = optind, j = 0; i < argc; i += step, j++) { @@ -2163,7 +2579,6 @@ int main(int argc, char *argv[]) if (rc) goto cleanup; - count++; } fy_parse_document_destroy(fyp, fyd); @@ -2322,7 +2737,6 @@ int main(int argc, char *argv[]) } else stdin_input = false; - count = 0; for (;;) { if (!stdin_input) { @@ -2392,7 +2806,6 @@ int main(int argc, char *argv[]) goto cleanup; } - count = 0; for (i = optind; i < argc; i++) { rc = set_parser_input(fyp, argv[i], false); if (rc) { @@ -2411,7 +2824,6 @@ int main(int argc, char *argv[]) fy_parser_event_free(fyp, fyev); } } - count++; if (fy_parser_get_stream_error(fyp)) goto cleanup; @@ -2431,15 +2843,12 @@ int main(int argc, char *argv[]) cd.single_document = false; cd.verbose = dump_path; - count = 0; for (i = optind; i < argc; i++) { rc = set_parser_input(fyp, argv[i], false); if (rc) { fprintf(stderr, "failed to set parser input to '%s' for dump\n", argv[i]); goto cleanup; } - - count++; } /* ignore errors for now */ @@ -2493,4 +2902,9 @@ cleanup: } return exitcode; + +err_out_usage: + exitcode = EXIT_FAILURE; + display_usage(stderr, progname, tool_mode); + goto cleanup; } diff --git a/src/util/fy-align.h b/src/util/fy-align.h new file mode 100644 index 0000000..1e7e180 --- /dev/null +++ b/src/util/fy-align.h @@ -0,0 +1,95 @@ +/* + * fy-align.h - various align related stuff + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_ALIGN_H +#define FY_ALIGN_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +#define FY_ALIGNED_TO(x) __attribute__ ((aligned(x))) +#elif defined(_MSC_VER) +#define FY_ALIGNED_TO(x) __declspec(align(x)) +#else +#define FY_ALIGNED_TO(x) /* nothing */ +#endif + +#define FY_ALIGN(_align, _x) (((_x) + ((_align) - 1)) & ~((_align) - 1)) + +/* cachelines are universally 64 bytes */ +#define FY_CACHELINE_SIZE 64 +#define FY_CACHELINE_SIZE_ALIGN(_x) FY_ALIGN(FY_CACHELINE_SIZE, _x) +#define FY_CACHELINE_ALIGN FY_ALIGNED_TO(FY_CACHELINE_SIZE) + +/* provide posix_memalign for platforms that don't have it */ +#ifdef _WIN32 +static inline int posix_memalign(void **ptr, size_t align, size_t size) +{ + void *p; + + /* must be a power of two */ + if ((size & (size -1)) != 0) { + *ptr = NULL; + return EINVAL; + } + + p = _aligned_malloc(size, align); + if (!p) { + *ptr = NULL; + return ENOMEM; + } + return p; +} + +static inline void posix_memalign_free(void *p) +{ + _aligned_free(p); +} + +#else +/* normal implementations just use free */ +static inline void posix_memalign_free(void *p) +{ + free(p); +} +#endif + +static inline void *fy_align_alloc(size_t align, size_t size) +{ + void *p; + int rc; + + size = FY_ALIGN(align, size); + rc = posix_memalign(&p, align, size); + if (rc) + return NULL; + return p; +} + +static inline void fy_align_free(void *p) +{ + if (p) + posix_memalign_free(p); +} + +static inline void *fy_cacheline_alloc(size_t size) +{ + return fy_align_alloc(FY_CACHELINE_SIZE, size); +} + +static inline void fy_cacheline_free(void *p) +{ + fy_align_free(p); +} + +#endif diff --git a/src/util/fy-bit64.h b/src/util/fy-bit64.h new file mode 100644 index 0000000..b01fc6d --- /dev/null +++ b/src/util/fy-bit64.h @@ -0,0 +1,86 @@ +/* + * fy-bit64.h - various bit64 methods + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_BIT64_H +#define FY_BIT64_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#define FY_BIT64(x) ((uint64_t)1 << (x)) +#define FY_BIT64_COUNT(_x) (((_x) + 63) / 64) +#define FY_BIT64_SIZE(_x) ((size_t)((((_x) + 63) / 64) * sizeof(uint64_t))) + +#if (defined(__GNUC__) && __GNUC__ >= 4) || defined(__clang__) + +#define FY_BIT64_LOWEST(_x) ((unsigned int)__builtin_ctzll((unsigned long long)(_x))) +#define FY_BIT64_HIGHEST(_x) ((unsigned int)__builtin_clzll((unsigned long long)(_x))) +#define FY_BIT64_POPCNT(_x) ((unsigned int)__builtin_popcountl((unsigned long long)(_x))) + +static inline unsigned int fy_bit64_lowest(uint64_t x) +{ + return FY_BIT64_LOWEST(x); +} + +static inline unsigned int fy_bit64_highest(uint64_t x) +{ + return FY_BIT64_HIGHEST(x); +} + +static inline unsigned int FY_bit64_popcnt(uint64_t x) +{ + return FY_BIT64_POPCNT(x); +} + +#else /* portable implementation */ + +static inline unsigned int FY_BIT64_LOWEST(uint64_t x) +{ + unsigned int c = 0; + + if (!(x & 0x00000000ffffffffULL)) { x >>= 32; c += 32; } + if (!(x & 0x000000000000ffffULL)) { x >>= 16; c += 16; } + if (!(x & 0x00000000000000ffULL)) { x >>= 8; c += 8; } + if (!(x & 0x000000000000000fULL)) { x >>= 4; c += 4; } + if (!(x & 0x0000000000000003ULL)) { x >>= 2; c += 2; } + if (!(x & 0x0000000000000001ULL)) { c += 1; } + return c; +} + +static inline unsigned int FY_BIT64_HIGHEST(uint64_t x) +{ + unsigned int c = 0; + + if (x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } + if (x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } + if (x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } + if (x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } + if (x & 0x000000000000000cULL) { x >>= 2; c += 2; } + if (x & 0x0000000000000002ULL) { c += 1; } + return c; +} + +static inline unsigned int FY_BIT64_POPCNT(uint64_t x) +{ + unsigned int count; + + for (count = 0; x; x &= x - 1) + count++; + return count; +} + +#define FY_BIT64_LOWEST(_x) fy_bit64_lowest(_x) +#define FY_BIT64_HIGEST(_x) fy_bit64_highest(_x) +#define FY_BIT64_POPCNT(_x) fy_bit64_popcnt(_x) + +#endif + +#endif diff --git a/src/util/fy-blob.c b/src/util/fy-blob.c new file mode 100644 index 0000000..152afdd --- /dev/null +++ b/src/util/fy-blob.c @@ -0,0 +1,119 @@ +/* + * fy-blob.c - binary blob handling + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include + +#include "fy-blob.h" + +/* define optimized methods for probing */ +BR_DEFINE_WX(br_probe_w, 8, false, false); +BR_DEFINE_WX(br_probe_w, 16, false, false); +BR_DEFINE_WX(br_probe_w, 32, false, false); +BR_DEFINE_WX(br_probe_w, 64, false, false); + +/* define optimized methods for writing */ +BR_DEFINE_WX(br_native_w, 8, false, true); +BR_DEFINE_WX(br_native_w, 16, false, true); +BR_DEFINE_WX(br_native_w, 32, false, true); +BR_DEFINE_WX(br_native_w, 64, false, true); + +BR_DEFINE_WX(br_bswap_w, 8, true, true); +BR_DEFINE_WX(br_bswap_w, 16, true, true); +BR_DEFINE_WX(br_bswap_w, 32, true, true); +BR_DEFINE_WX(br_bswap_w, 64, true, true); + +void *fy_blob_read(const char *file, size_t *sizep) +{ + struct stat sb; + void *blob = NULL; + uint8_t *p; + size_t size, left; + ssize_t rdn; + int fd = -1, rc; + + if (!file || !sizep) + goto err_out; + + fd = open(file, O_RDONLY); + if (fd < 0) + goto err_out; + + rc = fstat(fd, &sb); + if (rc < 0) + goto err_out; + + size = sb.st_size; + blob = malloc(size); + if (!blob) + goto err_out; + + p = blob; + left = size; + while (left > 0) { + do { + rdn = read(fd, p, left); + } while (rdn == -1 && errno == EAGAIN); + if (rdn < 0) + goto err_out; + p += rdn; + left -= rdn; + } + close(fd); + + *sizep = size; + return blob; + +err_out: + if (blob) + free(blob); + if (fd >= 0) + close(fd); + return NULL; +} + +int fy_blob_write(const char *file, const void *blob, size_t size) +{ + int fd = -1; + const uint8_t *p; + size_t left; + ssize_t wrn; + + if (!file || !blob) + return -1; + + fd = open(file, O_WRONLY|O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IRGRP | S_IROTH); + if (fd < 0) + goto err_out; + + p = blob; + left = size; + while (left > 0) { + do { + wrn = write(fd, p, left); + } while (wrn == -1 && errno == EAGAIN); + if (wrn < 0) + goto err_out; + p += wrn; + left -= wrn; + } + close(fd); + + return 0; + +err_out: + if (fd >= 0) + close(fd); + return -1; +} diff --git a/src/util/fy-blob.h b/src/util/fy-blob.h new file mode 100644 index 0000000..301455d --- /dev/null +++ b/src/util/fy-blob.h @@ -0,0 +1,455 @@ +/* + * fy-blob.h - binary blob handling support + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_BLOB_H +#define FY_BLOB_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#ifdef HAVE_BYTESWAP_H +#include +#else +#ifdef HAVE___BUILTIN_BSWAP16 +#define bswap_16(value) __builtin_bswap16(value) +#else +#define bswap_16(value) ((((value)&0xff) << 8) | ((value) >> 8)) +#endif + +#ifdef HAVE___BUILTIN_BSWAP32 +#define bswap_32(value) __builtin_bswap32(value) +#else +#define bswap_32(value) \ + (((uint32_t)bswap_16((uint16_t)((value)&0xffff)) << 16) | \ + (uint32_t)bswap_16((uint16_t)((value) >> 16))) +#endif + +#ifdef HAVE___BUILTIN_BSWAP64 +#define bswap_64(value) __builtin_bswap64(value) +#else +#define bswap_64(value) \ + (((uint64_t)bswap_32((uint32_t)((value)&0xffffffff)) << 32) | \ + (uint64_t)bswap_32((uint32_t)((value) >> 32))) +#endif +#endif + +#include +#include + +#include "fy-endian.h" + +/* special unaligned types for pointer accesses */ + +#if defined(__GNUC__) && __GNUC__ >= 4 +typedef union { uint64_t v; } __attribute__((__packed__)) br_64; +typedef union { uint32_t v; } __attribute__((__packed__)) br_32; +typedef union { uint16_t v; } __attribute__((__packed__)) br_16; +typedef union { uint8_t v; } br_8; +#elif defined(_MSC_VER) +#pragma pack(push, 1) +typedef union { uint64_t v; } br_64; +typedef union { uint32_t v; } br_32; +typedef union { uint16_t v; } br_16; +typedef union { uint8_t v; } br_8; +#pragma pack(pop) +#else +#error Unsupported compiler +#endif + +enum blob_id_size { + BID_U8, + BID_U16, + BID_U32, + BID_U64, +}; + +static inline enum blob_id_size +blob_count_to_id_size(unsigned long long count) +{ + if (count < (1LLU << 8)) + return BID_U8; + if (count < (1LLU << 16)) + return BID_U16; + if (count < (1LLU << 32)) + return BID_U32; + return BID_U64; +} + +static inline unsigned int +blob_id_size_to_byte_size(enum blob_id_size size) +{ + return 1U << size; +} + +static inline unsigned int +blob_id_size_to_bit_size(enum blob_id_size size) +{ + return blob_id_size_to_byte_size(size) * 8; +} + +enum blob_endian_type { + BET_NATIVE_ENDIAN, + BET_LITTLE_ENDIAN, + BET_BIG_ENDIAN +}; + +struct blob_region { + union { + uint8_t *wstart; + const uint8_t *rstart; + }; + size_t size; + enum blob_endian_type endian; + bool bswap; + size_t curr; +}; + +#define BR_DEFINE_WX(_pfx, _bits, _bswap, _write) \ +size_t _pfx ## _bits (struct blob_region *br, uint ## _bits ## _t v) \ +{ \ + size_t pos; \ +\ + assert(br->curr < br->size); \ + if (_write) { \ + if (_bswap) \ + v = bswap_ ## _bits (v); \ + ((br_ ## _bits *)(br->wstart + br->curr))->v = v; \ + } \ + pos = br->curr; \ + br->curr += sizeof(v); \ + return pos; \ +} \ +struct __useless_struct_to_allow_semicolon + +#define BR_DEFINE_RX(_pfx, _bits, _bswap) \ +uint ## _bits ## _t _pfx ## _bits (struct blob_region *br) \ +{ \ + uint ## _bits ## _t v; \ +\ + assert(br->curr + sizeof(v) <= br->size); \ + assert(br->rstart); \ + v = ((const br_ ## _bits *)(br->rstart + br->curr))->v; \ + if (_bswap) \ + v = bswap_ ## _bits (v); \ + br->curr += sizeof(v); \ + return v; \ +} \ +struct __useless_struct_to_allow_semicolon + +static inline void br_setup_common(struct blob_region *br, size_t size, enum blob_endian_type endian) +{ + memset(br, 0, sizeof(*br)); + br->size = size; + br->endian = endian; + + switch (br->endian) { + case BET_NATIVE_ENDIAN: + /* native endian never needs swap */ + br->bswap = false; + break; + + case BET_LITTLE_ENDIAN: +#if __BYTE_ORDER == __LITTLE_ENDIAN + br->bswap = false; +#else + br->bswap = true; +#endif + break; + + case BET_BIG_ENDIAN: +#if __BYTE_ORDER == __LITTLE_ENDIAN + br->bswap = true; +#else + br->bswap = false; +#endif + break; + } + + br->curr = 0; +} + +static inline void br_wsetup(struct blob_region *br, void *data, size_t size, enum blob_endian_type endian) +{ + /* write with null data means probe */ + br_setup_common(br, size, endian); + br->wstart = data; +} + +static inline void br_rsetup(struct blob_region *br, const void *data, size_t size, enum blob_endian_type endian) +{ + /* read makes no sense without data */ + assert(data); + br_setup_common(br, size, endian); + br->rstart = data; +} + +static inline void br_reset(struct blob_region *br) +{ + br->curr = 0; +} + +static inline size_t br_curr(struct blob_region *br) +{ + return br->curr; +} + +static inline size_t br_write(struct blob_region *br, const void *data, size_t size) +{ + size_t pos; + + assert(br->curr + size <= br->size); + if (br->wstart) + memcpy(br->wstart + br->curr, data, size); + pos = br->curr; + br->curr += size; + return pos; +} + +static inline size_t br_w0(struct blob_region *br, size_t size) +{ + size_t pos; + + assert(br->curr + size <= br->size); + if (br->wstart) + memset(br->wstart + br->curr, 0, size); + pos = br->curr; + br->curr += size; + return pos; +} + +static inline size_t br_wskip(struct blob_region *br, size_t size) +{ + return br_w0(br, size); +} + +static inline size_t br_wskip_to(struct blob_region *br, size_t offset) +{ + assert(br->curr <= offset); + return br_w0(br, offset - br->curr); +} + +static inline BR_DEFINE_WX(br_w, 8, br->bswap, br->wstart != NULL); +static inline BR_DEFINE_WX(br_w, 16, br->bswap, br->wstart != NULL); +static inline BR_DEFINE_WX(br_w, 32, br->bswap, br->wstart != NULL); +static inline BR_DEFINE_WX(br_w, 64, br->bswap, br->wstart != NULL); + +typedef size_t (*br_wid_func)(struct blob_region *br, int id); + +static inline size_t br_wid8(struct blob_region *br, int id) +{ + assert((unsigned int)id < ((unsigned int)1 << 8)); + return br_w8(br, (uint8_t)id); +} + +static inline size_t br_wid16(struct blob_region *br, int id) +{ + assert((unsigned int)id < ((unsigned int)1 << 16)); + return br_w16(br, (uint16_t)id); +} + +static inline size_t br_wid32(struct blob_region *br, int id) +{ + assert((unsigned long)id < ((unsigned long)1 << 32)); + return br_w32(br, (uint32_t)id); +} + +static inline size_t br_wid64(struct blob_region *br, int id) +{ + assert(id >= 0); + return br_w64(br, (uint64_t)id); +} + +static inline br_wid_func br_wid_get_func(enum blob_id_size id_size) +{ + switch (id_size) { + case BID_U8: + return br_wid8; + + case BID_U16: + return br_wid16; + + case BID_U32: + return br_wid32; + + case BID_U64: + return br_wid64; + } + return NULL; +} + +static inline size_t br_wstr(struct blob_region *br, const char *str, size_t len) +{ + if (len == (size_t)-1) + len = strlen(str); + return br_write(br, str, len + 1); +} + +static inline size_t br_wX(struct blob_region *br, enum blob_id_size x_size, uint64_t x) +{ + switch (x_size) { + case BID_U8: + assert(x < ((uint64_t)1 << 8)); + return br_w8(br, (uint8_t)x); + + case BID_U16: + assert(x < ((uint64_t)1 << 16)); + return br_w16(br, (uint16_t)x); + + case BID_U32: + assert(x < ((uint64_t)1 << 32)); + return br_w32(br, (uint32_t)x); + + case BID_U64: + return br_w64(br, x); + } + return (size_t)-1; +} + +static inline size_t br_wid(struct blob_region *br, enum blob_id_size id_size, int id) +{ + assert(id >= 0); + switch (id_size) { + case BID_U8: + return br_wid8(br, id); + + case BID_U16: + return br_wid16(br, id); + + case BID_U32: + return br_wid32(br, id); + + case BID_U64: + return br_wid64(br, id); + } + return br->curr; +} + +static inline size_t br_read(struct blob_region *br, void *data, size_t size) +{ + size_t pos; + + assert(br->curr + size <= br->size); + assert(br->rstart); + memcpy(data, br->rstart + br->curr, size); + pos = br->curr; + br->curr += size; + return pos; +} + +static inline void br_rskip(struct blob_region *br, size_t size) +{ + assert(br->curr + size <= br->size); + br->curr += size; +} + +static inline void br_rskip_to(struct blob_region *br, size_t offset) +{ + assert(br->curr <= offset); + assert(offset <= br->size); + br->curr = offset; +} + +static inline BR_DEFINE_RX(br_r, 8, br->bswap); +static inline BR_DEFINE_RX(br_r, 16, br->bswap); +static inline BR_DEFINE_RX(br_r, 32, br->bswap); +static inline BR_DEFINE_RX(br_r, 64, br->bswap); + +typedef int (*br_rid_func)(struct blob_region *br); + +static inline int br_rid8(struct blob_region *br) +{ + return (int)br_r8(br); +} + +static inline int br_rid16(struct blob_region *br) +{ + return (int)br_r16(br); +} + +static inline int br_rid32(struct blob_region *br) +{ + return (int)br_r32(br); +} + +static inline int br_rid64(struct blob_region *br) +{ + return (int)br_r64(br); +} + +static inline br_rid_func br_rid_get_func(enum blob_id_size id_size) +{ + switch (id_size) { + case BID_U8: + return br_rid8; + + case BID_U16: + return br_rid16; + + case BID_U32: + return br_rid32; + + case BID_U64: + return br_rid64; + } + return NULL; +} + +static inline uint64_t br_rX(struct blob_region *br, enum blob_id_size x_size) +{ + switch (x_size) { + case BID_U8: + return br_r8(br); + + case BID_U16: + return br_r16(br); + + case BID_U32: + return br_r32(br); + + case BID_U64: + return br_r64(br); + } + return (uint64_t)-1; +} + +static inline int br_rid(struct blob_region *br, enum blob_id_size id_size) +{ + int id; + + switch (id_size) { + case BID_U8: + id = br_rid8(br); + break; + + case BID_U16: + id = br_rid16(br); + break; + + case BID_U32: + id = br_rid32(br); + break; + + case BID_U64: + id = br_rid64(br); + break; + + default: + id = -1; + break; + } + assert(id >= 0); + return id; +} + +void *fy_blob_read(const char *file, size_t *sizep); +int fy_blob_write(const char *file, const void *blob, size_t size); + +#endif diff --git a/src/util/fy-ctype.c b/src/util/fy-ctype.c new file mode 100644 index 0000000..1c9463f --- /dev/null +++ b/src/util/fy-ctype.c @@ -0,0 +1,67 @@ +/* + * fy-ctype.c - ctype utilities + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + * + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include "fy-ctype.h" + +const char *fy_uri_esc(const char *s, size_t len, uint8_t *code, int *code_len) +{ + const char *e = s + len; + int j, k, width; + uint8_t octet; + char c; + + width = 0; + k = 0; + do { + /* check for enough space for %XX */ + if ((e - s) < 3) + return NULL; + + /* if more than one run, expect '%' */ + if (s[0] != '%') + return NULL; + + octet = 0; + for (j = 0; j < 2; j++) { + c = s[1 + j]; + octet <<= 4; + if (c >= '0' && c <= '9') + octet |= c - '0'; + else if (c >= 'a' && c <= 'f') + octet |= 10 + c - 'a'; + else + octet |= 10 + c - 'A'; + } + if (!width) { + width = fy_utf8_width_by_first_octet(octet); + if (!width) + return NULL; + k = 0; + } + if (k >= *code_len) + return NULL; + + code[k++] = octet; + + /* skip over the 3 character escape */ + s += 3; + + } while (--width > 0); + + *code_len = k; + + return s; +} + diff --git a/src/util/fy-ctype.h b/src/util/fy-ctype.h new file mode 100644 index 0000000..06b5ecd --- /dev/null +++ b/src/util/fy-ctype.h @@ -0,0 +1,353 @@ +/* + * fy-ctype.h - ctype like macros header + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_CTYPE_H +#define FY_CTYPE_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "fy-utf8.h" + +enum fy_lb_mode { + fylb_cr_nl, /* only \r, \n (json + >= yaml1.2 */ + fylb_cr_nl_N_L_P, /* NEL/LS/PS (yaml1.1) */ +}; + +enum fy_flow_ws_mode { + fyfws_space_tab, /* space + TAB (yaml) */ + fyfws_space, /* only space (json) */ +}; + +static inline bool fy_is_first_alpha(int c) +{ + return (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + c == '_'; +} + +static inline bool fy_is_alpha(int c) +{ + return fy_is_first_alpha(c) || c == '-'; +} + +static inline bool fy_is_num(int c) +{ + return c >= '0' && c <= '9'; +} + +static inline bool fy_is_digit(int c) +{ + return c >= '0' && c <= '9'; +} + +static inline bool fy_is_xdigit(int c) +{ + return fy_is_digit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + +static inline bool fy_is_first_alnum(int c) +{ + return fy_is_first_alpha(c); +} + +static inline bool fy_is_alnum(int c) +{ + return fy_is_alpha(c) || fy_is_num(c); +} + +static inline bool fy_is_space(int c) +{ + return c == ' '; +} + +static inline bool fy_is_tab(int c) +{ + return c == '\t'; +} + +static inline bool fy_is_ws(int c) +{ + return fy_is_space(c) || fy_is_tab(c); +} + +static inline bool fy_is_hex(int c) +{ + return (c >= '0' && c <= '9') || + (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F'); +} + +static inline bool fy_is_uri(int c) +{ + return fy_is_alnum(c) || fy_utf8_strchr(";/?:@&=+$,.!~*\'()[]%", c); +} + +static inline bool fy_is_lb_r_n(int c) +{ + return c == '\r' || c == '\n'; +} + +static inline bool fy_is_lb_NEL(int c) +{ + return c == 0x85; +} + +static inline bool fy_is_lb_LS_PS(int c) +{ + return c == 0x2028 || c == 0x2029; +} + +static inline bool fy_is_unicode_lb(int c) +{ + /* note that YAML1.1 supports NEL #x85, LS #x2028 and PS #x2029 as linebreaks */ + /* YAML1.2 and higher does not */ + return fy_is_lb_NEL(c) || fy_is_lb_LS_PS(c); +} + +static inline bool fy_is_any_lb(int c) +{ + return fy_is_lb_r_n(c) || fy_is_unicode_lb(c); +} + +static inline bool fy_is_z(int c) +{ + return c <= 0; +} + +static inline bool fy_is_blank(int c) +{ + return c == ' ' || c == '\t'; +} + +#define FY_UTF8_BOM 0xfeff + +static inline bool fy_is_print(int c) +{ + return c == '\n' || c == '\r' || + (c >= 0x0020 && c <= 0x007e) || + (c >= 0x00a0 && c <= 0xd7ff) || + (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM); +} + +static inline bool fy_is_printq(int c) +{ + return c != '\t' && c != 0xa0 && !fy_is_any_lb(c) && fy_is_print(c); +} + +static inline bool fy_is_nb_char(int c) +{ + return (c >= 0x0020 && c <= 0x007e) || + (c >= 0x00a0 && c <= 0xd7ff) || + (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM); +} + +static inline bool fy_is_ns_char(int c) +{ + return fy_is_nb_char(c) && !fy_is_ws(c); +} + +static inline bool fy_is_start_indicator(int c) +{ + return !!fy_utf8_strchr(",[]{}#&*!|>'\"%%@`", c); +} + +static inline bool fy_is_indicator_before_space(int c) +{ + return !!fy_utf8_strchr("-:?`", c); +} + +static inline bool fy_is_flow_indicator(int c) +{ + return !!fy_utf8_strchr(",[]{}", c); +} + +static inline bool fy_is_path_flow_scalar_start(int c) +{ + return c == '\'' || c == '"'; +} + +static inline bool fy_is_path_flow_key_start(int c) +{ + return c == '"' || c == '\'' || c == '{' || c == '['; +} + +static inline bool fy_is_path_flow_key_end(int c) +{ + return c == '"' || c == '\'' || c == '}' || c == ']'; +} + +static inline bool fy_is_unicode_control(int c) +{ + return (c >= 0 && c <= 0x1f) || (c >= 0x80 && c <= 0x9f); +} + +static inline bool fy_is_unicode_space(int c) +{ + return c == 0x20 || c == 0xa0 || + (c >= 0x2000 && c <= 0x200a) || + c == 0x202f || c == 0x205f || c == 0x3000; +} + +static inline bool fy_is_json_unescaped(int c) +{ + return c >= 0x20 && c <= 0x110000 && c != '"' && c != '\\'; +} + +static inline bool fy_is_json_unescaped_range_only(int c) +{ + return c >= 0x20 && c <= 0x110000; +} + +static inline bool fy_is_lb_m(int c, enum fy_lb_mode lb_mode) +{ + if (fy_is_lb_r_n(c)) + return true; + return lb_mode == fylb_cr_nl_N_L_P && fy_is_unicode_lb(c); +} + +static inline bool fy_is_generic_lb_m(int c, enum fy_lb_mode lb_mode) +{ + if (fy_is_lb_r_n(c)) + return true; + return lb_mode == fylb_cr_nl_N_L_P && fy_is_lb_NEL(c); +} + +static inline bool fy_is_lbz_m(int c, enum fy_lb_mode lb_mode) +{ + return fy_is_lb_m(c, lb_mode) || fy_is_z(c); +} + +static inline bool fy_is_generic_lbz_m(int c, enum fy_lb_mode lb_mode) +{ + return fy_is_generic_lb_m(c, lb_mode) || fy_is_z(c); +} + +static inline bool fy_is_blankz_m(int c, enum fy_lb_mode lb_mode) +{ + return fy_is_ws(c) || fy_is_lbz_m(c, lb_mode); +} + +static inline bool fy_is_generic_blankz_m(int c, enum fy_lb_mode lb_mode) +{ + return fy_is_ws(c) || fy_is_generic_lbz_m(c, lb_mode); +} + +static inline bool fy_is_flow_ws_m(int c, enum fy_flow_ws_mode fws_mode) +{ + return fy_is_space(c) || (fws_mode == fyfws_space_tab && fy_is_tab(c)); +} + +#define FY_CTYPE_AT_BUILDER(_kind) \ +static inline const void * \ +fy_find_ ## _kind (const void *s, size_t len) \ +{ \ + const void *e = s + len; \ + int c, w; \ + for (; s < e && (c = fy_utf8_get(s, e - s, &w)) >= 0; s += w) { \ + assert(w); \ + if (fy_is_ ## _kind (c)) \ + return s; \ + } \ + return NULL; \ +} \ +static inline const void * \ +fy_find_non_ ## _kind (const void *s, size_t len) \ +{ \ + const void *e = s + len; \ + int c, w; \ + for (; s < e && (c = fy_utf8_get(s, e - s, &w)) >= 0; s += w) { \ + assert(w); \ + if (!(fy_is_ ## _kind (c))) \ + return s; \ + assert(w); \ + } \ + return NULL; \ +} \ +struct useless_struct_for_semicolon + +FY_CTYPE_AT_BUILDER(first_alpha); +FY_CTYPE_AT_BUILDER(alpha); +FY_CTYPE_AT_BUILDER(num); +FY_CTYPE_AT_BUILDER(digit); +FY_CTYPE_AT_BUILDER(xdigit); +FY_CTYPE_AT_BUILDER(first_alnum); +FY_CTYPE_AT_BUILDER(alnum); +FY_CTYPE_AT_BUILDER(space); +FY_CTYPE_AT_BUILDER(tab); +FY_CTYPE_AT_BUILDER(ws); +FY_CTYPE_AT_BUILDER(hex); +FY_CTYPE_AT_BUILDER(uri); +FY_CTYPE_AT_BUILDER(z); +FY_CTYPE_AT_BUILDER(any_lb); +FY_CTYPE_AT_BUILDER(blank); +FY_CTYPE_AT_BUILDER(print); +FY_CTYPE_AT_BUILDER(printq); +FY_CTYPE_AT_BUILDER(nb_char); +FY_CTYPE_AT_BUILDER(ns_char); +FY_CTYPE_AT_BUILDER(start_indicator); +FY_CTYPE_AT_BUILDER(indicator_before_space); +FY_CTYPE_AT_BUILDER(flow_indicator); +FY_CTYPE_AT_BUILDER(path_flow_key_start); +FY_CTYPE_AT_BUILDER(path_flow_key_end); +FY_CTYPE_AT_BUILDER(unicode_control); +FY_CTYPE_AT_BUILDER(unicode_space); +FY_CTYPE_AT_BUILDER(json_unescaped); + +/* + * Very special linebreak/ws methods + * Things get interesting due to \r\n and + * unicode linebreaks/spaces + */ + +/* skip for a _single_ linebreak */ +static inline const void *fy_skip_lb(const void *ptr, size_t left) +{ + int c, width; + + /* get the utf8 character at this point */ + c = fy_utf8_get(ptr, left, &width); + if (c < 0 || !fy_is_any_lb(c)) + return NULL; + + /* MS-DOS: check if next character is '\n' */ + if (c == '\r' && left > (size_t)width && *(char *)ptr == '\n') + width++; + + return ptr + width; +} + +/* given a pointer to a chunk of memory, return pointer to first + * ws character after the last non-ws character, or the end + * of the chunk + */ +static inline const void *fy_last_non_ws(const void *ptr, size_t left) +{ + const char *s, *e; + int c; + + s = ptr; + e = s + left; + while (e > s) { + c = e[-1]; + if (c != ' ' && c != '\t') + return e; + e--; + + } + return NULL; +} + +const char *fy_uri_esc(const char *s, size_t len, uint8_t *code, int *code_len); + +#endif diff --git a/src/util/fy-endian.h b/src/util/fy-endian.h new file mode 100644 index 0000000..a3be6b8 --- /dev/null +++ b/src/util/fy-endian.h @@ -0,0 +1,50 @@ +/* + * fy-endian.h - simple system endian header wrapper + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ENDIAN_H +#define FY_ENDIAN_H + +#if defined(__linux__) || defined(__CYGWIN__) || defined(__OpenBSD__) +# include +#elif defined(__APPLE__) +# include +# include +#elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) +# include +#elif defined(_MSC_VER) +# include +# ifdef __GNUC__ +# include +# endif +#else +# error unsupported platform +#endif + +#if !defined(__BYTE_ORDER) && defined(BYTE_ORDER) +#define __BYTE_ORDER BYTE_ORDER +#endif + +#if !defined(__BIG_ENDIAN) && defined(BIG_ENDIAN) +#define __BIG_ENDIAN BIG_ENDIAN +#endif + +#if !defined(__LITTLE_ENDIAN) && defined(LITTLE_ENDIAN) +#define __LITTLE_ENDIAN LITTLE_ENDIAN +#endif + +#if !defined(__BYTE_ORDER) || !defined(__BIG_ENDIAN) || !defined(__LITTLE_ENDIAN) +# error Platform does not define endian macros +#endif + +/* no-one cares about PDP endian anymore */ + +/* make the macros work for 8 bit too */ +#ifndef bswap_8 +#define bswap_8(x) (x) +#endif + +#endif diff --git a/src/util/fy-id.h b/src/util/fy-id.h new file mode 100644 index 0000000..95c120e --- /dev/null +++ b/src/util/fy-id.h @@ -0,0 +1,239 @@ +/* + * fy-id.h - small ID allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ID_H +#define FY_ID_H + +#include +#include +#include +#include + +#include + +typedef unsigned int fy_id_bits; + +#define FY_ID_BITS_SZ ((size_t)(sizeof(fy_id_bits))) +#define FY_ID_BITS_BITS (FY_ID_BITS_SZ * 8) /* yes, every character is 8 bits in 2023 */ +#define FY_ID_BITS_MASK ((size_t)(FY_ID_BITS_BITS - 1)) + +#define FY_ID_BITS_ARRAY_COUNT_BITS(_bits) \ + (((_bits) + FY_ID_BITS_MASK) & ~FY_ID_BITS_MASK) + +#define FY_ID_BITS_ARRAY_COUNT(_bits) \ + (FY_ID_BITS_ARRAY_COUNT_BITS(_bits) / FY_ID_BITS_BITS) + +#define FY_ID_OFFSET(_id) ((_id) / FY_ID_BITS_BITS) +#define FY_ID_BIT_MASK(_id) ((fy_id_bits)1 << ((_id) & FY_ID_BITS_MASK)) + +#if defined(__GNUC__) && __GNUC__ >= 4 +static inline int fy_id_ffs(const fy_id_bits id_bit) +{ + if (!id_bit) + return -1; + return __builtin_ffs(id_bit) - 1; +} + +#else +static inline int fy_id_ffs(const fy_id_bits id_bit) +{ + return ffs((int)id_bits) - 1; +} +#endif + +#if defined(__GNUC__) && __GNUC__ >= 4 +static inline int fy_id_popcount(const fy_id_bits id_bit) +{ + return __builtin_popcount(id_bit); +} + +#else +static inline int fy_id_popcount(fy_id_bits id_bit) +{ + int count, pos; + + count = 0; + while ((pos = fy_id_ffs(id_bit)) >= 0) { + count++; + id_bit &= ~((fy_id_bits)1 << pos); + } + return count; +} +#endif + +static inline void fy_id_reset(fy_id_bits *bits, const size_t count) +{ + memset(bits, 0, sizeof(*bits) * count); +} + +static inline int fy_id_alloc(fy_id_bits *bits, const size_t count) +{ + size_t i; + int pos; + + for (i = 0; i < count; i++, bits++) { + pos = fy_id_ffs(~*bits); + if (pos >= 0) { + *bits |= FY_ID_BIT_MASK(pos); + return (i * FY_ID_BITS_BITS) + pos; + } + } + return -1; +} + +static inline int fy_id_alloc_fixed(fy_id_bits *bits, const size_t count, const int id) +{ + size_t offset; + fy_id_bits m; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return -1; + bits += offset; + m = FY_ID_BIT_MASK(id); + if (*bits & m) + return -1; + *bits |= m; + return id; +} + +static inline bool fy_id_is_used(fy_id_bits *bits, const size_t count, const int id) +{ + size_t offset; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return false; + return !!(bits[offset] & FY_ID_BIT_MASK(id)); +} + +static inline bool fy_id_is_used_no_check(fy_id_bits *bits, const size_t count, const int id) +{ + size_t offset; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return false; + return !!(bits[offset] & FY_ID_BIT_MASK(id)); +} + +static inline bool fy_id_is_free(fy_id_bits *bits, const size_t count, const int id) +{ + size_t offset; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return false; + return !(bits[offset] & FY_ID_BIT_MASK(id)); +} + +static inline void fy_id_free(fy_id_bits *bits, const size_t count, int id) +{ + size_t offset; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return; + bits += offset; + *bits &= ~FY_ID_BIT_MASK(id); /* no need to check, if bit is free, it is still free */ +} + +static inline void fy_id_set_used(fy_id_bits *bits, const size_t count, const int id) +{ + assert(FY_ID_OFFSET(id) < count); + bits[FY_ID_OFFSET(id)] |= FY_ID_BIT_MASK(id); +} + +static inline void fy_id_set_free(fy_id_bits *bits, const size_t count, const int id) +{ + assert(FY_ID_OFFSET(id) < count); + bits[FY_ID_OFFSET(id)] &= ~FY_ID_BIT_MASK(id); +} + +static inline size_t fy_id_count_used(fy_id_bits *bits, const size_t count) +{ + size_t i, popcount; + + popcount = 0; + for (i = 0; i < count; i++, bits++) + popcount += fy_id_popcount(*bits); + return popcount; +} + +static inline size_t fy_id_count_free(fy_id_bits *bits, const size_t count) +{ + size_t i, popcount; + + popcount = 0; + for (i = 0; i < count; i++, bits++) + popcount += fy_id_popcount(~*bits); + return popcount; +} + +struct fy_id_iter { + const fy_id_bits *s; + fy_id_bits m; +}; + +static inline void fy_id_iter_begin(const fy_id_bits *bits, const size_t count, struct fy_id_iter *iterp) +{ + iterp->s = bits; + iterp->m = ~(fy_id_bits)0; +} + +static inline int fy_id_iter_next(const fy_id_bits *bits, const size_t count, struct fy_id_iter *iterp) +{ + const fy_id_bits *s, *e; + fy_id_bits m; + int pos; + + s = iterp->s; + if (!s) + return -1; + + m = iterp->m; + e = bits + count; + + if (s >= e) + return -1; + + /* scan until we find a set bit or hit the end */ + pos = -1; + while (s < e && (pos = fy_id_ffs(*s & m)) < 0) { + s++; + m = ~(fy_id_bits)0; + } + + /* not found */ + if (pos < 0) { + iterp->s = NULL; + iterp->m = 0; + return -1; + } + + /* remove this bit from the mask */ + m &= ~((fy_id_bits)1 << pos); + + /* advance by the number of fy_id_bits we scanned */ + pos += (int)(s - bits) * FY_ID_BITS_BITS; + + /* if we run out of mask, advance */ + if (!m) { + s++; + m = ~(fy_id_bits)0; + } + iterp->s = s; + iterp->m = m; + return pos; +} + +static inline void fy_id_iter_end(fy_id_bits *bits, const size_t count, const struct fy_id_iter *iterp) +{ + /* nothing */ +} + +#endif diff --git a/src/util/fy-list.h b/src/util/fy-list.h new file mode 100644 index 0000000..1a0b5f9 --- /dev/null +++ b/src/util/fy-list.h @@ -0,0 +1,705 @@ +/* + * fy-list.h - slightly modified Linux kernel list.h + * + * All copyrights owned by their respective holders. + * + * SPDX-License-Identifier: GPL-2.0 + */ +#ifndef _FY_LIST_H +#define _FY_LIST_H + +#include + +/** + * container_of - cast a member of a structure out to the containing structure + * @ptr: the pointer to the member. + * @type: the type of the container struct this is embedded in. + * @member: the name of the member within the struct. + * + */ +#define container_of(ptr, type, member) ({ \ + const typeof( ((type *)0)->member ) *__mptr = (ptr); \ + (type *)( (char *)__mptr - offsetof(type,member) );}) + +#define LIST_POISON1 NULL +#define LIST_POISON2 NULL + +#ifndef ARCH_HAS_PREFETCH +#define ARCH_HAS_PREFETCH +static inline void prefetch(const void *x) {;} +#endif + +/* + * Simple doubly linked list implementation. + * + * Some of the internal functions ("__xxx") are useful when + * manipulating whole lists rather than single entries, as + * sometimes we already know the next/prev entries and we can + * generate better code by using them directly rather than + * using the generic single-entry routines. + */ + +struct list_head { + struct list_head *next, *prev; +}; + +#define LIST_HEAD_INIT(name) { &(name), &(name) } + +#define LIST_HEAD(name) \ + struct list_head name = LIST_HEAD_INIT(name) + +static inline void INIT_LIST_HEAD(struct list_head *list) +{ + list->next = list; + list->prev = list; +} + +/* + * Insert a new entry between two known consecutive entries. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __list_add(struct list_head *new, + struct list_head *prev, + struct list_head *next) +{ + next->prev = new; + new->next = next; + new->prev = prev; + prev->next = new; +} + +/** + * list_add - add a new entry + * @new: new entry to be added + * @head: list head to add it after + * + * Insert a new entry after the specified head. + * This is good for implementing stacks. + */ +static inline void list_add(struct list_head *new, struct list_head *head) +{ + __list_add(new, head, head->next); +} + +/** + * list_add_tail - add a new entry + * @new: new entry to be added + * @head: list head to add it before + * + * Insert a new entry before the specified head. + * This is useful for implementing queues. + */ +static inline void list_add_tail(struct list_head *new, struct list_head *head) +{ + __list_add(new, head->prev, head); +} + +/* + * Delete a list entry by making the prev/next entries + * point to each other. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __list_del(struct list_head *prev, struct list_head *next) +{ + next->prev = prev; + prev->next = next; +} + +/** + * list_del - deletes entry from list. + * @entry: the element to delete from the list. + * Note: list_empty() on entry does not return true after this, the entry is + * in an undefined state. + */ +static inline void list_del(struct list_head *entry) +{ + __list_del(entry->prev, entry->next); + entry->next = LIST_POISON1; + entry->prev = LIST_POISON2; +} + +/** + * list_replace - replace old entry by new one + * @old : the element to be replaced + * @new : the new element to insert + * + * If @old was empty, it will be overwritten. + */ +static inline void list_replace(struct list_head *old, + struct list_head *new) +{ + new->next = old->next; + new->next->prev = new; + new->prev = old->prev; + new->prev->next = new; +} + +static inline void list_replace_init(struct list_head *old, + struct list_head *new) +{ + list_replace(old, new); + INIT_LIST_HEAD(old); +} + +/** + * list_del_init - deletes entry from list and reinitialize it. + * @entry: the element to delete from the list. + */ +static inline void list_del_init(struct list_head *entry) +{ + __list_del(entry->prev, entry->next); + INIT_LIST_HEAD(entry); +} + +/** + * list_move - delete from one list and add as another's head + * @list: the entry to move + * @head: the head that will precede our entry + */ +static inline void list_move(struct list_head *list, struct list_head *head) +{ + __list_del(list->prev, list->next); + list_add(list, head); +} + +/** + * list_move_tail - delete from one list and add as another's tail + * @list: the entry to move + * @head: the head that will follow our entry + */ +static inline void list_move_tail(struct list_head *list, + struct list_head *head) +{ + __list_del(list->prev, list->next); + list_add_tail(list, head); +} + +/** + * list_is_last - tests whether @list is the last entry in list @head + * @list: the entry to test + * @head: the head of the list + */ +static inline int list_is_last(const struct list_head *list, + const struct list_head *head) +{ + return list->next == head; +} + +/** + * list_empty - tests whether a list is empty + * @head: the list to test. + */ +static inline int list_empty(const struct list_head *head) +{ + return head->next == head; +} + +/** + * list_empty_careful - tests whether a list is empty and not being modified + * @head: the list to test + * + * Description: + * tests whether a list is empty _and_ checks that no other CPU might be + * in the process of modifying either member (next or prev) + * + * NOTE: using list_empty_careful() without synchronization + * can only be safe if the only activity that can happen + * to the list entry is list_del_init(). Eg. it cannot be used + * if another CPU could re-list_add() it. + */ +static inline int list_empty_careful(const struct list_head *head) +{ + struct list_head *next = head->next; + return (next == head) && (next == head->prev); +} + +/** + * list_is_singular - tests whether a list has just one entry. + * @head: the list to test. + */ +static inline int list_is_singular(const struct list_head *head) +{ + return !list_empty(head) && (head->next == head->prev); +} + +static inline void __list_cut_position(struct list_head *list, + struct list_head *head, struct list_head *entry) +{ + struct list_head *new_first = entry->next; + list->next = head->next; + list->next->prev = list; + list->prev = entry; + entry->next = list; + head->next = new_first; + new_first->prev = head; +} + +/** + * list_cut_position - cut a list into two + * @list: a new list to add all removed entries + * @head: a list with entries + * @entry: an entry within head, could be the head itself + * and if so we won't cut the list + * + * This helper moves the initial part of @head, up to and + * including @entry, from @head to @list. You should + * pass on @entry an element you know is on @head. @list + * should be an empty list or a list you do not care about + * losing its data. + * + */ +static inline void list_cut_position(struct list_head *list, + struct list_head *head, struct list_head *entry) +{ + if (list_empty(head)) + return; + if (list_is_singular(head) && + (head->next != entry && head != entry)) + return; + if (entry == head) + INIT_LIST_HEAD(list); + else + __list_cut_position(list, head, entry); +} + +static inline void __list_splice(const struct list_head *list, + struct list_head *prev, + struct list_head *next) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + + first->prev = prev; + prev->next = first; + + last->next = next; + next->prev = last; +} + +/** + * list_splice - join two lists, this is designed for stacks + * @list: the new list to add. + * @head: the place to add it in the first list. + */ +static inline void list_splice(const struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) + __list_splice(list, head, head->next); +} + +/** + * list_splice_tail - join two lists, each list being a queue + * @list: the new list to add. + * @head: the place to add it in the first list. + */ +static inline void list_splice_tail(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) + __list_splice(list, head->prev, head); +} + +/** + * list_splice_init - join two lists and reinitialise the emptied list. + * @list: the new list to add. + * @head: the place to add it in the first list. + * + * The list at @list is reinitialised + */ +static inline void list_splice_init(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) { + __list_splice(list, head, head->next); + INIT_LIST_HEAD(list); + } +} + +/** + * list_splice_tail_init - join two lists and reinitialise the emptied list + * @list: the new list to add. + * @head: the place to add it in the first list. + * + * Each of the lists is a queue. + * The list at @list is reinitialised + */ +static inline void list_splice_tail_init(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) { + __list_splice(list, head->prev, head); + INIT_LIST_HEAD(list); + } +} + +/** + * list_entry - get the struct for this entry + * @ptr: the &struct list_head pointer. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_struct within the struct. + */ +#define list_entry(ptr, type, member) \ + container_of(ptr, type, member) + +/** + * list_first_entry - get the first element from a list + * @ptr: the list head to take the element from. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_struct within the struct. + * + * Note, that list is expected to be not empty. + */ +#define list_first_entry(ptr, type, member) \ + list_entry((ptr)->next, type, member) + +/** + * list_last_entry - get the last element from a list + * @ptr: the list head to take the element from. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_struct within the struct. + * + * Note, that list is expected to be not empty. + */ +#define list_last_entry(ptr, type, member) \ + list_entry((ptr)->prev, type, member) + +/** + * list_for_each - iterate over a list + * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + */ +#define list_for_each(pos, head) \ + for (pos = (head)->next; prefetch(pos->next), pos != (head); \ + pos = pos->next) + +/** + * __list_for_each - iterate over a list + * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + * + * This variant differs from list_for_each() in that it's the + * simplest possible list iteration code, no prefetching is done. + * Use this for code that knows the list to be very short (empty + * or 1 entry) most of the time. + */ +#define __list_for_each(pos, head) \ + for (pos = (head)->next; pos != (head); pos = pos->next) + +/** + * list_for_each_prev - iterate over a list backwards + * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + */ +#define list_for_each_prev(pos, head) \ + for (pos = (head)->prev; prefetch(pos->prev), pos != (head); \ + pos = pos->prev) + +/** + * list_for_each_safe - iterate over a list safe against removal of list entry + * @pos: the &struct list_head to use as a loop cursor. + * @n: another &struct list_head to use as temporary storage + * @head: the head for your list. + */ +#define list_for_each_safe(pos, n, head) \ + for (pos = (head)->next, n = pos->next; pos != (head); \ + pos = n, n = pos->next) + +/** + * list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry + * @pos: the &struct list_head to use as a loop cursor. + * @n: another &struct list_head to use as temporary storage + * @head: the head for your list. + */ +#define list_for_each_prev_safe(pos, n, head) \ + for (pos = (head)->prev, n = pos->prev; \ + prefetch(pos->prev), pos != (head); \ + pos = n, n = pos->prev) + +/** + * list_for_each_entry - iterate over list of given type + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry(pos, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member); \ + prefetch(pos->member.next), &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +/** + * list_for_each_entry_reverse - iterate backwards over list of given type. + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry_reverse(pos, head, member) \ + for (pos = list_entry((head)->prev, typeof(*pos), member); \ + prefetch(pos->member.prev), &pos->member != (head); \ + pos = list_entry(pos->member.prev, typeof(*pos), member)) + +/** + * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue() + * @pos: the type * to use as a start point + * @head: the head of the list + * @member: the name of the list_struct within the struct. + * + * Prepares a pos entry for use as a start point in list_for_each_entry_continue(). + */ +#define list_prepare_entry(pos, head, member) \ + ((pos) ? : list_entry(head, typeof(*pos), member)) + +/** + * list_for_each_entry_continue - continue iteration over list of given type + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Continue to iterate over list of given type, continuing after + * the current position. + */ +#define list_for_each_entry_continue(pos, head, member) \ + for (pos = list_entry(pos->member.next, typeof(*pos), member); \ + prefetch(pos->member.next), &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +/** + * list_for_each_entry_continue_reverse - iterate backwards from the given point + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Start to iterate over list of given type backwards, continuing after + * the current position. + */ +#define list_for_each_entry_continue_reverse(pos, head, member) \ + for (pos = list_entry(pos->member.prev, typeof(*pos), member); \ + prefetch(pos->member.prev), &pos->member != (head); \ + pos = list_entry(pos->member.prev, typeof(*pos), member)) + +/** + * list_for_each_entry_from - iterate over list of given type from the current point + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate over list of given type, continuing from current position. + */ +#define list_for_each_entry_from(pos, head, member) \ + for (; prefetch(pos->member.next), &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +/** + * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry_safe(pos, n, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member), \ + n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + +/** + * list_for_each_entry_safe_continue + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate over list of given type, continuing after current point, + * safe against removal of list entry. + */ +#define list_for_each_entry_safe_continue(pos, n, head, member) \ + for (pos = list_entry(pos->member.next, typeof(*pos), member), \ + n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + +/** + * list_for_each_entry_safe_from + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate over list of given type from current point, safe against + * removal of list entry. + */ +#define list_for_each_entry_safe_from(pos, n, head, member) \ + for (n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + +/** + * list_for_each_entry_safe_reverse + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate backwards over list of given type, safe against removal + * of list entry. + */ +#define list_for_each_entry_safe_reverse(pos, n, head, member) \ + for (pos = list_entry((head)->prev, typeof(*pos), member), \ + n = list_entry(pos->member.prev, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.prev, typeof(*n), member)) + +/* + * Double linked lists with a single pointer list head. + * Mostly useful for hash tables where the two pointer list head is + * too wasteful. + * You lose the ability to access the tail in O(1). + */ + +struct hlist_head { + struct hlist_node *first; +}; + +struct hlist_node { + struct hlist_node *next, **pprev; +}; + +#define HLIST_HEAD_INIT { .first = NULL } +#define HLIST_HEAD(name) struct hlist_head name = { .first = NULL } +#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL) +static inline void INIT_HLIST_NODE(struct hlist_node *h) +{ + h->next = NULL; + h->pprev = NULL; +} + +static inline int hlist_unhashed(const struct hlist_node *h) +{ + return !h->pprev; +} + +static inline int hlist_empty(const struct hlist_head *h) +{ + return !h->first; +} + +static inline void __hlist_del(struct hlist_node *n) +{ + struct hlist_node *next = n->next; + struct hlist_node **pprev = n->pprev; + *pprev = next; + if (next) + next->pprev = pprev; +} + +static inline void hlist_del(struct hlist_node *n) +{ + __hlist_del(n); + n->next = LIST_POISON1; + n->pprev = LIST_POISON2; +} + +static inline void hlist_del_init(struct hlist_node *n) +{ + if (!hlist_unhashed(n)) { + __hlist_del(n); + INIT_HLIST_NODE(n); + } +} + +static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) +{ + struct hlist_node *first = h->first; + n->next = first; + if (first) + first->pprev = &n->next; + h->first = n; + n->pprev = &h->first; +} + +/* next must be != NULL */ +static inline void hlist_add_before(struct hlist_node *n, + struct hlist_node *next) +{ + n->pprev = next->pprev; + n->next = next; + next->pprev = &n->next; + *(n->pprev) = n; +} + +static inline void hlist_add_after(struct hlist_node *n, + struct hlist_node *next) +{ + next->next = n->next; + n->next = next; + next->pprev = &n->next; + + if(next->next) + next->next->pprev = &next->next; +} + +#define hlist_entry(ptr, type, member) container_of(ptr,type,member) + +#define hlist_for_each(pos, head) \ + for (pos = (head)->first; pos && ({ prefetch(pos->next); 1; }); \ + pos = pos->next) + +#define hlist_for_each_safe(pos, n, head) \ + for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \ + pos = n) + +/** + * hlist_for_each_entry - iterate over list of given type + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the hlist_node within the struct. + */ +#define hlist_for_each_entry(tpos, pos, head, member) \ + for (pos = (head)->first; \ + pos && ({ prefetch(pos->next); 1;}) && \ + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ + pos = pos->next) + +/** + * hlist_for_each_entry_continue - iterate over a hlist continuing after current point + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. + * @member: the name of the hlist_node within the struct. + */ +#define hlist_for_each_entry_continue(tpos, pos, member) \ + for (pos = (pos)->next; \ + pos && ({ prefetch(pos->next); 1;}) && \ + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ + pos = pos->next) + +/** + * hlist_for_each_entry_from - iterate over a hlist continuing from current point + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. + * @member: the name of the hlist_node within the struct. + */ +#define hlist_for_each_entry_from(tpos, pos, member) \ + for (; pos && ({ prefetch(pos->next); 1;}) && \ + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ + pos = pos->next) + +/** + * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. + * @n: another &struct hlist_node to use as temporary storage + * @head: the head for your list. + * @member: the name of the hlist_node within the struct. + */ +#define hlist_for_each_entry_safe(tpos, pos, n, head, member) \ + for (pos = (head)->first; \ + pos && ({ n = pos->next; 1; }) && \ + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ + pos = n) + +#endif diff --git a/src/util/fy-typelist.h b/src/util/fy-typelist.h new file mode 100644 index 0000000..9739e73 --- /dev/null +++ b/src/util/fy-typelist.h @@ -0,0 +1,161 @@ +/* + * fy-typelist.h - typed list method builders + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_TYPELIST_H +#define FY_TYPELIST_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#include "fy-list.h" + +/* declare type methods */ + +#define FY_TYPE_FWD_DECL_LIST(_type) \ +/* type safe list wrapper */ \ +struct fy_ ## _type ## _list { struct list_head _lh; }; \ +\ +struct __useless_struct_to_allow_semicolon + +#define FY_TYPE_DECL_LIST(_type) \ +static inline void fy_ ## _type ## _list_init(struct fy_ ## _type ## _list *_l) \ +{ \ + INIT_LIST_HEAD(&_l->_lh); \ +} \ +static inline void fy_ ## _type ## _list_add(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (_l && _n) \ + list_add(&_n->node, &_l->_lh); \ +} \ +static inline void fy_ ## _type ## _list_add_tail(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (_l && _n) \ + list_add_tail(&_n->node, &_l->_lh); \ +} \ +static inline void fy_ ## _type ## _list_push(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (_l && _n) \ + fy_ ## _type ## _list_add(_l, _n); \ +} \ +static inline void fy_ ## _type ## _list_push_tail(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (_l && _n) \ + fy_ ## _type ## _list_add_tail(_l, _n); \ +} \ +static inline bool fy_ ## _type ## _list_empty(struct fy_ ## _type ## _list *_l) \ +{ \ + return _l ? list_empty(&_l->_lh) : true; \ +} \ +static inline bool fy_ ## _type ## _list_is_singular(struct fy_ ## _type ## _list *_l) \ +{ \ + return _l ? list_is_singular(&_l->_lh) : true; \ +} \ +static inline void fy_ ## _type ## _list_del(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (_l && _n) { \ + list_del(&_n->node); \ + INIT_LIST_HEAD(&_n->node); \ + } \ +} \ +static inline void fy_ ## _type ## _list_insert_after(struct fy_ ## _type ## _list *_l, \ + struct fy_ ## _type *_p, struct fy_ ## _type *_n) \ +{ \ + if (_l && _p && _n) \ + list_add(&_n->node, &_p->node); \ +} \ +static inline void fy_ ## _type ## _list_insert_before(struct fy_ ## _type ## _list *_l, \ + struct fy_ ## _type *_p, struct fy_ ## _type *_n) \ +{ \ + if (_l && _p && _n) \ + list_add_tail(&_n->node, &_p->node); \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_head(struct fy_ ## _type ## _list *_l) \ +{ \ + return !fy_ ## _type ## _list_empty(_l) ? list_first_entry(&_l->_lh, struct fy_ ## _type, node) : NULL; \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_tail(struct fy_ ## _type ## _list *_l) \ +{ \ + return !fy_ ## _type ## _list_empty(_l) ? list_last_entry(&_l->_lh, struct fy_ ## _type, node) : NULL; \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_first(struct fy_ ## _type ## _list *_l) \ +{ \ + return fy_ ## _type ## _list_head(_l); \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_last(struct fy_ ## _type ## _list *_l) \ +{ \ + return fy_ ## _type ## _list_tail(_l); \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_pop(struct fy_ ## _type ## _list *_l) \ +{ \ + struct fy_ ## _type *_n; \ + \ + _n = fy_ ## _type ## _list_head(_l); \ + if (!_n) \ + return NULL; \ + fy_ ## _type ## _list_del(_l, _n); \ + return _n; \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_pop_tail(struct fy_ ## _type ## _list *_l) \ +{ \ + struct fy_ ## _type *_n; \ + \ + _n = fy_ ## _type ## _list_tail(_l); \ + if (!_n) \ + return NULL; \ + fy_ ## _type ## _list_del(_l, _n); \ + return _n; \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _next(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (!_n || !_l || _n->node.next == &_l->_lh) \ + return NULL; \ + return list_entry(_n->node.next, struct fy_ ## _type, node); \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _prev(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (!_n || !_l || _n->node.prev == &_l->_lh) \ + return NULL; \ + return list_entry(_n->node.prev, struct fy_ ## _type, node); \ +} \ +static inline void fy_ ## _type ## _lists_splice( \ + struct fy_ ## _type ## _list *_l, \ + struct fy_ ## _type ## _list *_lfrom) \ +{ \ + /* check arguments for sanity and lists are not empty */ \ + if (!_l || !_lfrom || \ + fy_ ## _type ## _list_empty(_lfrom)) \ + return; \ + list_splice(&_lfrom->_lh, &_l->_lh); \ +} \ +static inline void fy_ ## _type ## _list_splice_after( \ + struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n, \ + struct fy_ ## _type ## _list *_lfrom) \ +{ \ + /* check arguments for sanity and lists are not empty */ \ + if (!_l || !_n || !_lfrom || \ + fy_ ## _type ## _list_empty(_lfrom)) \ + return; \ + list_splice(&_lfrom->_lh, &_n->node); \ +} \ +static inline void fy_ ## _type ## _list_splice_before( \ + struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n, \ + struct fy_ ## _type ## _list *_lfrom) \ +{ \ + /* check arguments for sanity and lists are not empty */ \ + if (!_l || !_n || !_lfrom || \ + fy_ ## _type ## _list_empty(_lfrom)) \ + return; \ + list_splice(&_lfrom->_lh, _n->node.prev); \ +} \ +struct __useless_struct_to_allow_semicolon + +#endif diff --git a/src/util/fy-utf8.c b/src/util/fy-utf8.c new file mode 100644 index 0000000..45c071c --- /dev/null +++ b/src/util/fy-utf8.c @@ -0,0 +1,1062 @@ +/* + * fy-utf8.c - utf8 handling methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#include "fy-utf8.h" + +/* to avoid dragging in libfyaml.h */ +#ifndef FY_BIT +#define FY_BIT(x) (1U << (x)) +#endif + +const int8_t fy_utf8_width_table[32] = { + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, + 2, 2, 2, 2, 3, 3, 4, 0, +}; + +int fy_utf8_get_generic(const void *ptr, size_t left, int *widthp) +{ + const uint8_t *p = ptr; + int i, width, value; + + if (left < 1) + return FYUG_EOF; + + /* this is the slow path */ + width = fy_utf8_width_by_first_octet(p[0]); + if (!width) + return FYUG_INV; + if ((size_t)width > left) + return FYUG_PARTIAL; + + /* initial value */ + value = *p++ & (0xff >> width); + for (i = 1; i < width; i++) { + if ((*p & 0xc0) != 0x80) + return FYUG_INV; + value = (value << 6) | (*p++ & 0x3f); + } + + /* check for validity */ + if ((width == 4 && value < 0x10000) || + (width == 3 && value < 0x800) || + (width == 2 && value < 0x80) || + (value >= 0xd800 && value <= 0xdfff) || value >= 0x110000) + return FYUG_INV; + + *widthp = width; + + return value; +} + +int fy_utf8_get_right_generic(const void *ptr, size_t left, int *widthp) +{ + const uint8_t *s, *e; + uint8_t v; + + s = ptr; + e = s + left; + + if (left < 1) + return FYUG_EOF; + + /* single byte sequence */ + v = e[-1]; + if ((v & 0x80) == 0) { + if (widthp) + *widthp = 1; + return (int)v & 0x7f; + } + + /* the last byte must be & 0xc0 == 0x80 */ + if ((v & 0xc0) != 0x80) + return FYUG_INV; + + /* at least two byte sequence */ + if (left < 2) + return FYUG_EOF; + + v = e[-2]; + /* the first byte of the sequence (must be a two byte sequence) */ + if ((v & 0xc0) != 0x80) { + /* two byte start is 110x_xxxx */ + if ((v & 0xe0) != 0xc0) + return FYUG_INV; + return fy_utf8_get(e - 2, 2, widthp); + } + + /* at least three byte sequence */ + if (left < 3) + return FYUG_EOF; + + v = e[-3]; + /* the first byte of the sequence (must be a three byte sequence) */ + if ((v & 0xc0) != 0x80) { + /* three byte start is 1110_xxxx */ + if ((v & 0xf0) != 0xe0) + return FYUG_INV; + return fy_utf8_get(e - 3, 3, widthp); + } + + /* at least four byte sequence */ + if (left < 4) + return FYUG_EOF; + + v = e[-4]; + + /* the first byte of the sequence (must be a four byte sequence) */ + /* four byte start is 1111_0xxx */ + if ((v & 0xf8) != 0xf0) { + return FYUG_INV; + } + return fy_utf8_get(e - 4, 4, widthp); +} + +struct fy_utf8_fmt_esc_map { + const int *ch; + const int *map; +}; + +static const struct fy_utf8_fmt_esc_map esc_all = { + .ch = (const int []){ '\\', '\0', '\b', '\r', '\t', '\f', '\n', '\v', '\a', '\e', 0x85, 0xa0, 0x2028, 0x2029, -1 }, + .map = (const int []){ '\\', '0', 'b', 'r', 't', 'f', 'n', 'v', 'a', 'e', 'N', '_', 'L', 'P', 0 } +}; + +static inline int esc_map(const struct fy_utf8_fmt_esc_map *esc_map, int c) +{ + const int *ch; + int cc; + + ch = esc_map->ch; + while ((cc = *ch++) >= 0) { + if (cc == c) + return esc_map->map[(ch - esc_map->ch) - 1]; + } + return -1; +} + +static inline int fy_utf8_esc_map(int c, enum fy_utf8_escape esc) +{ + if (esc == fyue_none) + return -1; + if (esc == fyue_singlequote && c == '\'') + return '\''; + if (fy_utf8_escape_is_any_doublequote(esc) && c == '"') + return '"'; + return esc_map(&esc_all, c); +} + +size_t fy_utf8_format_text_length(const char *buf, size_t len, + enum fy_utf8_escape esc) +{ + int c, w; + ssize_t l; + const char *s, *e; + + s = buf; + e = buf + len; + l = 0; + while (s < e) { + c = fy_utf8_get(s, e - s, &w); + if (!w || c < 0) + break; + s += w; + if (fy_utf8_esc_map(c, esc)) + w = 2; + l += w; + } + return l + 1; +} + +char *fy_utf8_format_text(const char *buf, size_t len, + char *out, size_t maxsz, + enum fy_utf8_escape esc) +{ + int c, w, cc; + const char *s, *e; + char *os, *oe; + + s = buf; + e = buf + len; + os = out; + oe = out + maxsz - 1; + while (s < e) { + c = fy_utf8_get(s, e - s, &w); + if (!w || c < 0) + break; + s += w; + + if ((cc = fy_utf8_esc_map(c, esc)) > 0) { + if (os + 2 > oe) + break; + *os++ = '\\'; + *os++ = cc; + continue; + } + + if (os + w > oe) + break; + + os = fy_utf8_put_unchecked(os, c); + } + *os++ = '\0'; + return out; +} + +char *fy_utf8_format(int c, char *buf, enum fy_utf8_escape esc) +{ + int cc; + char *s; + + if (!fy_utf8_is_valid(c)) { + *buf = '\0'; + return buf; + } + + s = buf; + if ((cc = fy_utf8_esc_map(c, esc)) > 0) { + *s++ = '\\'; + *s++ = cc; + } else + s = fy_utf8_put_unchecked(s, c); + *s = '\0'; + return buf; +} + +char *fy_utf8_format_text_alloc(const char *buf, size_t len, enum fy_utf8_escape esc) +{ + size_t outsz; + char *out; + + outsz = fy_utf8_format_text_length(buf, len, esc); + if ((ssize_t)outsz < 0) + return NULL; + out = malloc(outsz); + if (!out) + return NULL; + fy_utf8_format_text(buf, len, out, outsz, esc); + + return out; +} + +const void *fy_utf8_memchr_generic(const void *s, int c, size_t n) +{ + int cc, w; + const void *e; + + e = s + n; + while (s < e && (cc = fy_utf8_get(s, e - s, &w)) >= 0) { + if (c == cc) + return s; + s += w; + } + + return NULL; +} + +/* parse an escape and return utf8 value */ +int fy_utf8_parse_escape(const char **strp, size_t len, enum fy_utf8_escape esc) +{ + const char *s, *e; + char c; + int i, value, code_length, cc, w; + unsigned int hi_surrogate, lo_surrogate; + + /* why do you bother us? */ + if (esc == fyue_none) + return -1; + + if (!strp || !*strp || len < 2) + return -1; + + value = -1; + + s = *strp; + e = s + len; + + c = *s++; + + if (esc == fyue_singlequote) { + if (c != '\'') + goto out; + c = *s++; + if (c != '\'') + goto out; + + value = '\''; + goto out; + } + + /* get '\\' */ + if (c != '\\') + goto out; + + c = *s++; + + /* common YAML & JSON escapes */ + switch (c) { + case 'b': + value = '\b'; + break; + case 'f': + value = '\f'; + break; + case 'n': + value = '\n'; + break; + case 'r': + value = '\r'; + break; + case 't': + value = '\t'; + break; + case '"': + value = '"'; + break; + case '/': + value = '/'; + break; + case '\\': + value = '\\'; + break; + default: + break; + } + + if (value >= 0) + goto out; + + if (esc == fyue_doublequote || esc == fyue_doublequote_yaml_1_1) { + switch (c) { + case '0': + value = '\0'; + break; + case 'a': + value = '\a'; + break; + case '\t': + value = '\t'; + break; + case 'v': + value = '\v'; + break; + case 'e': + value = '\e'; + break; + case ' ': + value = ' '; + break; + case 'N': + value = 0x85; /* NEL */ + break; + case '_': + value = 0xa0; + break; + case 'L': + value = 0x2028; /* LS */ + break; + case 'P': /* PS 0x2029 */ + value = 0x2029; /* PS */ + break; + default: + /* weird unicode escapes */ + if ((uint8_t)c >= 0x80) { + /* in non yaml-1.1 mode we don't allow this craziness */ + if (esc == fyue_doublequote) + goto out; + + cc = fy_utf8_get(s - 1, e - (s - 1), &w); + switch (cc) { + case 0x2028: + case 0x2029: + case 0x85: + case 0xa0: + value = cc; + break; + default: + break; + } + } + break; + } + if (value >= 0) + goto out; + } + + /* finally try the unicode escapes */ + code_length = 0; + + if (esc == fyue_doublequote || esc == fyue_doublequote_yaml_1_1) { + switch (c) { + case 'x': + code_length = 2; + break; + case 'u': + code_length = 4; + break; + case 'U': + code_length = 8; + break; + default: + return -1; + } + } else if (esc == fyue_doublequote_json && c == 'u') + code_length = 4; + + if (!code_length || code_length > (e - s)) + goto out; + + value = 0; + for (i = 0; i < code_length; i++) { + c = *s++; + value <<= 4; + if (c >= '0' && c <= '9') + value |= c - '0'; + else if (c >= 'a' && c <= 'f') + value |= 10 + c - 'a'; + else if (c >= 'A' && c <= 'F') + value |= 10 + c - 'A'; + else + goto out; + } + + /* hi/lo surrogate pair */ + if (code_length == 4 && value >= 0xd800 && value <= 0xdbff && + (e - s) >= 6 && s[0] == '\\' && s[1] == 'u') { + hi_surrogate = value; + + s += 2; + + value = 0; + for (i = 0; i < code_length; i++) { + c = *s++; + value <<= 4; + if (c >= '0' && c <= '9') + value |= c - '0'; + else if (c >= 'a' && c <= 'f') + value |= 10 + c - 'a'; + else if (c >= 'A' && c <= 'F') + value |= 10 + c - 'A'; + else + return -1; + } + lo_surrogate = value; + value = 0x10000 + (hi_surrogate - 0xd800) * 0x400 + (lo_surrogate - 0xdc00); + } + +out: + *strp = s; + + return value; +} + +uint8_t fy_utf8_low_ascii_flags[0x80] = { + [0x00] = F_NON_PRINT, // NUL '\0' (null character) + [0x01] = F_NON_PRINT, // SOH (start of heading) + [0x02] = F_NON_PRINT, // STX (start of text) + [0x03] = F_NON_PRINT, // ETX (end of text) + [0x04] = F_NON_PRINT, // EOT (end of transmission) + [0x05] = F_NON_PRINT, // ENQ (enquiry) + [0x06] = F_NON_PRINT, // ACK (acknowledge) + [0x07] = F_NON_PRINT | F_QUOTE_ESC, // BEL '\a' (bell) + [0x08] = F_NON_PRINT | F_QUOTE_ESC, // BS '\b' (backspace) + [0x09] = F_NON_PRINT | F_QUOTE_ESC | F_WS, // HT '\t' (horizontal tab) + [0x0A] = F_NON_PRINT | F_QUOTE_ESC | F_LB, // LF '\n' (new line) + [0x0B] = F_NON_PRINT | F_QUOTE_ESC, // VT '\v' (vertical tab) + [0x0C] = F_NON_PRINT | F_QUOTE_ESC, // FF '\f' (form feed) + [0x0D] = F_NON_PRINT | F_QUOTE_ESC | F_LB, // CR '\r' (carriage ret) + [0x0E] = F_NON_PRINT, // SO (shift out) + [0x0F] = F_NON_PRINT, // SI (shift in) + [0x10] = F_NON_PRINT, // DLE (data link escape) + [0x11] = F_NON_PRINT, // DC1 (device control 1) + [0x12] = F_NON_PRINT, // DC2 (device control 2) + [0x13] = F_NON_PRINT, // DC3 (device control 3) + [0x14] = F_NON_PRINT, // DC4 (device control 4) + [0x15] = F_NON_PRINT, // NAK (negative ack.) + [0x16] = F_NON_PRINT, // SYN (synchronous idle) + [0x17] = F_NON_PRINT, // ETB (end of trans. blk) + [0x18] = F_NON_PRINT, // CAN (cancel) + [0x19] = F_NON_PRINT, // EM (end of medium) + [0x1A] = F_NON_PRINT, // SUB (substitute) + [0x1B] = F_NON_PRINT, // ESC (escape) + [0x1C] = F_NON_PRINT, // FS (file separator) + [0x1D] = F_NON_PRINT, // GS (group separator) + [0x1E] = F_NON_PRINT, // RS (record separator) + [0x1F] = F_NON_PRINT, // US (unit separator) + [' '] = F_WS, + ['!'] = F_PUNCT, + ['"'] = F_PUNCT, + ['#'] = F_PUNCT, + ['$'] = F_PUNCT, + ['%'] = F_PUNCT, + ['&'] = F_PUNCT, + ['\''] = F_PUNCT, + ['('] = F_PUNCT, + [')'] = F_PUNCT, + ['*'] = F_PUNCT, + ['+'] = F_PUNCT, + [','] = F_PUNCT, + ['-'] = F_PUNCT, + ['.'] = F_PUNCT, + ['/'] = F_PUNCT, + ['0'] = F_DIGIT | F_SIMPLE_SCALAR, + ['1'] = F_DIGIT | F_SIMPLE_SCALAR, + ['2'] = F_DIGIT | F_SIMPLE_SCALAR, + ['3'] = F_DIGIT | F_SIMPLE_SCALAR, + ['4'] = F_DIGIT | F_SIMPLE_SCALAR, + ['5'] = F_DIGIT | F_SIMPLE_SCALAR, + ['6'] = F_DIGIT | F_SIMPLE_SCALAR, + ['7'] = F_DIGIT | F_SIMPLE_SCALAR, + ['8'] = F_DIGIT | F_SIMPLE_SCALAR, + ['9'] = F_DIGIT | F_SIMPLE_SCALAR, + [':'] = F_PUNCT, + [';'] = F_PUNCT, + ['<'] = F_PUNCT, + ['='] = F_PUNCT, + ['>'] = F_PUNCT, + ['?'] = F_PUNCT, + ['@'] = F_PUNCT, + ['A'] = F_LETTER | F_SIMPLE_SCALAR, + ['B'] = F_LETTER | F_SIMPLE_SCALAR, + ['C'] = F_LETTER | F_SIMPLE_SCALAR, + ['D'] = F_LETTER | F_SIMPLE_SCALAR, + ['E'] = F_LETTER | F_SIMPLE_SCALAR, + ['F'] = F_LETTER | F_SIMPLE_SCALAR, + ['G'] = F_LETTER | F_SIMPLE_SCALAR, + ['H'] = F_LETTER | F_SIMPLE_SCALAR, + ['I'] = F_LETTER | F_SIMPLE_SCALAR, + ['J'] = F_LETTER | F_SIMPLE_SCALAR, + ['K'] = F_LETTER | F_SIMPLE_SCALAR, + ['L'] = F_LETTER | F_SIMPLE_SCALAR, + ['M'] = F_LETTER | F_SIMPLE_SCALAR, + ['N'] = F_LETTER | F_SIMPLE_SCALAR, + ['O'] = F_LETTER | F_SIMPLE_SCALAR, + ['P'] = F_LETTER | F_SIMPLE_SCALAR, + ['Q'] = F_LETTER | F_SIMPLE_SCALAR, + ['R'] = F_LETTER | F_SIMPLE_SCALAR, + ['S'] = F_LETTER | F_SIMPLE_SCALAR, + ['T'] = F_LETTER | F_SIMPLE_SCALAR, + ['U'] = F_LETTER | F_SIMPLE_SCALAR, + ['V'] = F_LETTER | F_SIMPLE_SCALAR, + ['W'] = F_LETTER | F_SIMPLE_SCALAR, + ['X'] = F_LETTER | F_SIMPLE_SCALAR, + ['Y'] = F_LETTER | F_SIMPLE_SCALAR, + ['Z'] = F_LETTER | F_SIMPLE_SCALAR, + ['['] = F_PUNCT, + ['\\'] = F_PUNCT, // '\\' + [']'] = F_PUNCT, + ['^'] = F_PUNCT, + ['_'] = F_PUNCT | F_SIMPLE_SCALAR, + ['`'] = F_PUNCT, + ['a'] = F_LETTER | F_SIMPLE_SCALAR, + ['b'] = F_LETTER | F_SIMPLE_SCALAR, + ['c'] = F_LETTER | F_SIMPLE_SCALAR, + ['d'] = F_LETTER | F_SIMPLE_SCALAR, + ['e'] = F_LETTER | F_SIMPLE_SCALAR, + ['f'] = F_LETTER | F_SIMPLE_SCALAR, + ['g'] = F_LETTER | F_SIMPLE_SCALAR, + ['h'] = F_LETTER | F_SIMPLE_SCALAR, + ['i'] = F_LETTER | F_SIMPLE_SCALAR, + ['j'] = F_LETTER | F_SIMPLE_SCALAR, + ['k'] = F_LETTER | F_SIMPLE_SCALAR, + ['l'] = F_LETTER | F_SIMPLE_SCALAR, + ['m'] = F_LETTER | F_SIMPLE_SCALAR, + ['n'] = F_LETTER | F_SIMPLE_SCALAR, + ['o'] = F_LETTER | F_SIMPLE_SCALAR, + ['p'] = F_LETTER | F_SIMPLE_SCALAR, + ['q'] = F_LETTER | F_SIMPLE_SCALAR, + ['r'] = F_LETTER | F_SIMPLE_SCALAR, + ['s'] = F_LETTER | F_SIMPLE_SCALAR, + ['t'] = F_LETTER | F_SIMPLE_SCALAR, + ['u'] = F_LETTER | F_SIMPLE_SCALAR, + ['v'] = F_LETTER | F_SIMPLE_SCALAR, + ['w'] = F_LETTER | F_SIMPLE_SCALAR, + ['x'] = F_LETTER | F_SIMPLE_SCALAR, + ['y'] = F_LETTER | F_SIMPLE_SCALAR, + ['z'] = F_LETTER | F_SIMPLE_SCALAR, + ['{'] = F_PUNCT, + ['|'] = F_PUNCT, + ['}'] = F_PUNCT, + ['~'] = F_PUNCT, + [0x7F] = F_NON_PRINT, // DEL +}; + +void *fy_utf8_split_posix(const char *str, int *argcp, const char * const *argvp[]) +{ + enum split_state { + SS_WS, /* at whitespace */ + SS_WS_BS, /* backslash at whitespace */ + SS_UQ, /* at unquoted */ + SS_UQ_BS, /* backslash at unquoted */ + SS_SQ, /* at single quote */ + SS_SQ_BS, /* backslash at single quote */ + SS_DQ, /* at double quote */ + SS_DQ_BS, /* backslash at double quote */ + SS_DQ_BS_OCT1, /* in \nnn octal escape digit 1 */ + SS_DQ_BS_OCT2, /* in \nnn octal escape digit 2 */ + SS_DQ_BS_HEX0, /* in \xNN hex escape digit 0 */ + SS_DQ_BS_HEX1, /* in \xNN hex escape digit 1 */ + SS_DQ_BS_HEX2, /* in \xNN hex escape digit 2 */ + SS_DQ_BS_C, /* in \cN control character escape */ + SS_WS_BS_ERR_EOL, /* EOL while waiting for char after \ */ + SS_UQ_BS_ERR_EOL, /* EOL while waiting for char after \ */ + SS_SQ_BS_ERR_EOL, /* EOL while waiting for char after \ */ + SS_DQ_BS_ERR_EOL, /* EOL while waiting for char after \ */ + SS_DQ_BS_ERR_BAD, /* bad escape */ + SS_SQ_NC_ERR, /* no closing single quote */ + SS_DQ_NC_ERR, /* no closing double quote */ + SS_DQ_BS_ERR_OCT_BAD, /* bad octal escape */ + SS_DQ_BS_ERR_HEX_BAD, /* bad hex escape */ + SS_DQ_BS_ERR_C_BAD, /* bad control char */ + SS_DONE /* final state */ + }; + const char *s, *e; + enum split_state ss; + int c, w, val, tmp, outc, i; + size_t arg_alloc = strlen(str) + 1, tmplen; + char *arg = NULL, *args, *arge, *tmparg; + int argv_alloc = 64, argv_count = 0; + char **argv = NULL, **tmpargv; + void *mem; + + if (!str || !argcp || !argvp) + return NULL; + + /* the temporary is guaranteed to fit one split */ + arg = alloca(arg_alloc); + args = arg; + arge = arg + arg_alloc - 1; + + argv = alloca(sizeof(*argv) * argv_alloc); + + e = str + strlen(str); + +#undef OUTC +#define OUTC(_c) \ + do { \ + assert(args < arge); \ + args = fy_utf8_put(args, arge - args, (_c)); \ + assert(args); \ + } while(0) + +#undef NEW_ARGV +#define NEW_ARGV() \ + do { \ + if (args > arg) { \ + if (argv_count + 1 >= argv_alloc) { \ + argv_alloc *= 2; \ + tmpargv = alloca((sizeof(*tmpargv) * argv_alloc)); \ + memcpy(tmpargv, argv, argv_count * sizeof(*tmpargv)); \ + } \ + tmplen = args - arg; \ + tmparg = alloca(tmplen + 1); \ + memcpy(tmparg, arg, tmplen); \ + tmparg[tmplen] = '\0'; \ + argv[argv_count] = tmparg; \ + argv[++argv_count] = NULL; \ + } \ + args = arg; \ + } while(0) + +#define GOTO(_ss) \ + do { \ + ss = (_ss); \ + } while(0) + + + ss = SS_WS; + s = str; + val = 0; + while (!(ss >= SS_WS_BS_ERR_EOL && ss <= SS_DONE)) { + c = fy_utf8_get(s, e - s, &w); + switch (ss) { + case SS_WS: + if (c < 0) { + NEW_ARGV(); + GOTO(SS_DONE); + break; + } + switch (c) { + case ' ': + case '\t': + break; /* no change */ + case '\\': + GOTO(SS_WS_BS); + break; + case '"': + GOTO(SS_DQ); /* start double quoted */ + NEW_ARGV(); + break; + case '\'': + GOTO(SS_SQ); /* start single quoted */ + NEW_ARGV(); + break; + default: + GOTO(SS_UQ); /* start unquoted */ + NEW_ARGV(); + OUTC(c); + break; + } + break; + + case SS_WS_BS: + if (c < 0) { + GOTO(SS_WS_BS_ERR_EOL); + break; + } + if (c == '\n') { /* backslash newline, continuation */ + GOTO(SS_WS); + break; + } + GOTO(SS_UQ); + NEW_ARGV(); + OUTC(c); + break; + + case SS_UQ: + if (c <= 0) { + NEW_ARGV(); + GOTO(SS_DONE); + break; + } + + switch (c) { + case ' ': + case '\t': + GOTO(SS_WS); + NEW_ARGV(); + break; /* no change */ + case '\\': + GOTO(SS_UQ_BS); + break; + case '"': + GOTO(SS_DQ); /* double quoted */ + break; + case '\'': + GOTO(SS_SQ); /* single quoted */ + break; + default: + OUTC(c); + break; + } + break; + + case SS_UQ_BS: + if (c < 0) { + GOTO(SS_UQ_BS_ERR_EOL); + break; + } + if (c == '\n') { /* backslash new line only */ + GOTO(SS_UQ); + break; + } + GOTO(SS_UQ); + OUTC(c); + break; + + case SS_SQ: + if (c < 0) { + GOTO(SS_SQ_NC_ERR); /* no closing ' */ + break; + } + switch (c) { + case '\'': /* end of quote, back to unquoted */ + GOTO(SS_UQ); + break; + case '\\': + OUTC(c); + GOTO(SS_SQ_BS); /* backslash */ + break; + default: + OUTC(c); + break; + } + break; + + case SS_SQ_BS: + if (c < 0) { + GOTO(SS_SQ_BS_ERR_EOL); + break; + } + if (c == '\n') { /* backslash new line only */ + GOTO(SS_SQ); /* back to single quoted */ + break; + } + GOTO(SS_UQ); /* always back to */ + OUTC(c); + break; + + case SS_DQ: + if (c < 0) { + GOTO(SS_DQ_NC_ERR); /* no closing ' */ + break; + } + switch (c) { + case '"': /* end of quote, back to unquoted */ + GOTO(SS_UQ); + break; + case '\\': + GOTO(SS_DQ_BS); /* backslash */ + break; + default: + OUTC(c); + break; + } + break; + + case SS_DQ_BS: + if (c < 0) { + GOTO(SS_DQ_BS_ERR_EOL); + break; + } + outc = -1; + switch (c) { + case 'a': + outc = '\a'; + break; + case 'b': + outc = '\b'; + break; + case 'e': + outc = '\x1b'; /* escape */ + break; + case 'n': + outc = '\n'; + break; + case 'r': + outc = '\r'; + break; + case 't': + outc = '\t'; + break; + case 'v': + outc = '\v'; + break; + case '\\': + outc = '\\'; + break; + case '\'': + outc = '\''; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + val = c - '0'; + GOTO(SS_DQ_BS_OCT1); + break; + case 'x': + val = 0; + GOTO(SS_DQ_BS_HEX0); + break; + case 'c': + val = 0; + GOTO(SS_DQ_BS_C); + break; + default: + GOTO(SS_DQ_BS_ERR_BAD); /* unknown escape */ + break; + } + if (outc > 0) { + OUTC(outc); + GOTO(SS_DQ); + } + break; + + case SS_DQ_BS_OCT1: + case SS_DQ_BS_OCT2: + if (c < 0) { + OUTC(val); + NEW_ARGV(); + GOTO(SS_DONE); + break; + } + if (!(c >= '0' && c <= '7')) { + OUTC(val); + GOTO(SS_DQ); + w = 0; /* redo, same */ + break; + } + val *= 8; + val += c - '0'; + if (ss == SS_DQ_BS_OCT2) { + OUTC(val); + GOTO(SS_DQ); + } else + GOTO(SS_DQ_BS_OCT2); + break; + + case SS_DQ_BS_HEX0: + case SS_DQ_BS_HEX1: + case SS_DQ_BS_HEX2: + tmp = -1; + if (c >= '0' && c <= '9') + tmp = c - '0'; + else if (c >= 'a' && c <= 'f') + tmp = 10 + c - 'a'; + else if (c >= 'A' && c <= 'F') + tmp = 10 + c - 'A'; + + switch (ss) { + case SS_DQ_BS_HEX0: + if (tmp < 0) { + fprintf(stderr, "tmp=%d c=%c\n", tmp, c); + GOTO(SS_DQ_BS_ERR_HEX_BAD); + break; + } + val = tmp; + GOTO(SS_DQ_BS_HEX1); + break; + + case SS_DQ_BS_HEX1: + case SS_DQ_BS_HEX2: + if (tmp < 0) { + GOTO(SS_DQ); + OUTC(val); + w = 0; /* redo */ + break; + } + val *= 16; + val += tmp; + if (ss == SS_DQ_BS_HEX1) + GOTO(SS_DQ_BS_HEX2); + else { + OUTC(val); + GOTO(SS_DQ); + } + break; + + default: + abort(); + } + break; + + case SS_DQ_BS_C: + outc = -1; + if (c >= 'a' && c <= 'z') + outc = c - 'a' + 1; + else if (c >= 'A' && c <= 'Z') + outc = c - 'Z' + 1; + if (outc > 0x20) { + outc = -1; + GOTO(SS_DQ_BS_ERR_C_BAD); + } else + GOTO(SS_DQ); + if (outc > 0) + OUTC(outc); + break; + default: + abort(); + break; + } + s += w; + } + + /* someething went wrong */ + if (ss != SS_DONE) + return NULL; + + tmplen = (argv_count + 1) * sizeof(*argv); + for (i = 0; i < argv_count; i++) + tmplen += strlen(argv[i]) + 1; + + mem = malloc(tmplen); + if (!mem) + return NULL; + + tmpargv = mem; + tmparg = mem + (argv_count + 1) * sizeof(*tmpargv); + for (i = 0; i < argv_count; i++) { + tmpargv[i] = tmparg; + strcpy(tmparg, argv[i]); + tmparg += strlen(argv[i]) + 1; + } + tmpargv[i] = NULL; + + *argcp = i; + *argvp = mem; + + return mem; +} + +int fy_utf8_get_generic_s(const void *ptr, const void *ptr_end, int *widthp) +{ + const uint8_t *p = ptr; + int i, width, value; + + if (ptr >= ptr_end) + return FYUG_EOF; + + /* this is the slow path */ + width = fy_utf8_width_by_first_octet(p[0]); + if (!width) + return FYUG_INV; + if (ptr + width > ptr_end) + return FYUG_PARTIAL; + + /* initial value */ + value = *p++ & (0xff >> width); + for (i = 1; i < width; i++) { + if ((*p & 0xc0) != 0x80) + return FYUG_INV; + value = (value << 6) | (*p++ & 0x3f); + } + + /* check for validity */ + if ((width == 4 && value < 0x10000) || + (width == 3 && value < 0x800) || + (width == 2 && value < 0x80) || + (value >= 0xd800 && value <= 0xdfff) || value >= 0x110000) + return FYUG_INV; + + *widthp = width; + + return value; +} + +int fy_utf8_get_generic_s_nocheck(const void *ptr, int *widthp) +{ + const uint8_t *p = ptr; + int i, width, value; + + /* this is the slow path */ + width = fy_utf8_width_by_first_octet(p[0]); + if (!width) + return FYUG_INV; + + /* initial value */ + value = *p++ & (0xff >> width); + for (i = 1; i < width; i++) { + if ((*p & 0xc0) != 0x80) + return FYUG_INV; + value = (value << 6) | (*p++ & 0x3f); + } + + /* check for validity */ + if ((width == 4 && value < 0x10000) || + (width == 3 && value < 0x800) || + (width == 2 && value < 0x80) || + (value >= 0xd800 && value <= 0xdfff) || value >= 0x110000) + return FYUG_INV; + + *widthp = width; + + return value; +} diff --git a/src/util/fy-utf8.h b/src/util/fy-utf8.h new file mode 100644 index 0000000..a877c95 --- /dev/null +++ b/src/util/fy-utf8.h @@ -0,0 +1,289 @@ +/* + * fy-utf8.h - UTF-8 methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_UTF8_H +#define FY_UTF8_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "fy-utils.h" + +#define FY_UTF8_MIN_WIDTH 1 +#define FY_UTF8_MAX_WIDTH 4 + +extern const int8_t fy_utf8_width_table[32]; + +static inline int +fy_utf8_width_by_first_octet_no_table(uint8_t c) +{ + return (c & 0x80) == 0x00 ? 1 : + (c & 0xe0) == 0xc0 ? 2 : + (c & 0xf0) == 0xe0 ? 3 : + (c & 0xf8) == 0xf0 ? 4 : 0; +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_width_by_first_octet(uint8_t c) +{ + return fy_utf8_width_table[c >> 3]; +} + +/* assumes valid utf8 character */ +static inline size_t +fy_utf8_width(int c) +{ + return c < 0x80 ? 1 : + c < 0x800 ? 2 : + c < 0x10000 ? 3 : 4; +} + +static inline bool +fy_utf8_is_valid(int c) +{ + return c >= 0 && !((c >= 0xd800 && c <= 0xdfff) || c >= 0x110000); +} + +static inline bool +fy_utf8_is_printable_ascii(int c) +{ + return c >= 0x20 && c <= 0x7e; +} + +/* generic utf8 decoder (not inlined) */ +int fy_utf8_get_generic(const void *ptr, size_t left, int *widthp); + +/* -1 for end of input, -2 for invalid character, -3 for partial */ +#define FYUG_EOF -1 +#define FYUG_INV -2 +#define FYUG_PARTIAL -3 + +static inline int fy_utf8_get(const void *ptr, size_t left, int *widthp) +{ + const uint8_t *p = ptr; + + /* single byte (hot path) */ + if (left <= 0) { + *widthp = 0; + return FYUG_EOF; + } + + if (!(p[0] & 0x80)) { + *widthp = 1; + return p[0] & 0x7f; + } + return fy_utf8_get_generic(ptr, left, widthp); +} + +int fy_utf8_get_right_generic(const void *ptr, size_t left, int *widthp); + +static inline int fy_utf8_get_right(const void *ptr, size_t left, int *widthp) +{ + const uint8_t *p = ptr + left; + + /* single byte (hot path) */ + if (left > 0 && !(p[-1] & 0x80)) { + if (widthp) + *widthp = 1; + return p[-1] & 0x7f; + } + return fy_utf8_get_right_generic(ptr, left, widthp); +} + + +/* for when you _know_ that there's enough room and c is valid */ +static inline void *fy_utf8_put_unchecked(void *ptr, int c) +{ + uint8_t *s = ptr; + + assert(c >= 0); + if (c < 0x80) + *s++ = c; + else if (c < 0x800) { + *s++ = (c >> 6) | 0xc0; + *s++ = (c & 0x3f) | 0x80; + } else if (c < 0x10000) { + *s++ = (c >> 12) | 0xe0; + *s++ = ((c >> 6) & 0x3f) | 0x80; + *s++ = (c & 0x3f) | 0x80; + } else { + *s++ = (c >> 18) | 0xf0; + *s++ = ((c >> 12) & 0x3f) | 0x80; + *s++ = ((c >> 6) & 0x3f) | 0x80; + *s++ = (c & 0x3f) | 0x80; + } + return s; +} + +static inline void *fy_utf8_put(void *ptr, size_t left, int c) +{ + if (!fy_utf8_is_valid(c) || fy_utf8_width(c) > left) + return NULL; + + return fy_utf8_put_unchecked(ptr, c); +} + +/* buffer must contain at least 5 characters */ +#define FY_UTF8_FORMAT_BUFMIN 5 +enum fy_utf8_escape { + fyue_none, + fyue_singlequote, + fyue_doublequote, + fyue_doublequote_json, + fyue_doublequote_yaml_1_1, +}; + +static inline bool fy_utf8_escape_is_any_doublequote(enum fy_utf8_escape esc) +{ + return esc >= fyue_doublequote && esc <= fyue_doublequote_yaml_1_1; +} + +char *fy_utf8_format(int c, char *buf, enum fy_utf8_escape esc); + +#define fy_utf8_format_a(_c, _esc) \ + ({ \ + char *_buf = alloca(FY_UTF8_FORMAT_BUFMIN); \ + fy_utf8_format((_c), _buf, _esc); \ + }) + +size_t fy_utf8_format_text_length(const char *buf, size_t len, + enum fy_utf8_escape esc); +char *fy_utf8_format_text(const char *buf, size_t len, + char *out, size_t maxsz, + enum fy_utf8_escape esc); + +#define fy_utf8_format_text_a(_buf, _len, _esc) \ + ({ \ + const char *__buf = (_buf); \ + size_t __len = (_len); \ + enum fy_utf8_escape __esc = (_esc); \ + size_t _outsz = fy_utf8_format_text_length(__buf, __len, __esc); \ + char *_out = alloca(_outsz + 1); \ + fy_utf8_format_text(__buf, __len, _out, _outsz, __esc); \ + }) + +char *fy_utf8_format_text_alloc(const char *buf, size_t len, enum fy_utf8_escape esc); + +const void *fy_utf8_memchr_generic(const void *s, int c, size_t n); + +static inline const void *fy_utf8_memchr(const void *s, int c, size_t n) +{ + if (c < 0 || !n) + return NULL; + if (c < 0x80) + return memchr(s, c, n); + return fy_utf8_memchr_generic(s, c, n); +} + +static inline const void *fy_utf8_strchr(const void *s, int c) +{ + if (c < 0) + return NULL; + if (c < 0x80) + return strchr(s, c); + return fy_utf8_memchr_generic(s, c, strlen(s)); +} + +static inline int fy_utf8_count(const void *ptr, size_t len) +{ + const uint8_t *s = ptr, *e = ptr + len; + int w, count; + + count = 0; + while (s < e) { + w = fy_utf8_width_by_first_octet(*s); + + /* malformed? */ + if (!w || s + w > e) + break; + s += w; + + count++; + } + + return count; +} + +int fy_utf8_parse_escape(const char **strp, size_t len, enum fy_utf8_escape esc); + +#define F_NONE 0 +#define F_NON_PRINT FY_BIT(0) /* non printable */ +#define F_SIMPLE_SCALAR FY_BIT(1) /* part of simple scalar */ +#define F_QUOTE_ESC FY_BIT(2) /* escape form, i.e \n */ +#define F_LB FY_BIT(3) /* is a linebreak */ +#define F_WS FY_BIT(4) /* is a whitespace */ +#define F_PUNCT FY_BIT(5) /* is a punctuation mark */ +#define F_LETTER FY_BIT(6) /* is a letter a..z A..Z */ +#define F_DIGIT FY_BIT(7) /* is a digit 0..9 */ + +extern uint8_t fy_utf8_low_ascii_flags[0x80]; + +void *fy_utf8_split_posix(const char *str, int *argcp, const char * const *argvp[]); + +int fy_utf8_get_generic_s(const void *ptr, const void *ptr_end, int *widthp); +int fy_utf8_get_generic_s_nocheck(const void *ptr, int *widthp); + +static inline int fy_utf8_get_s(const void *ptr, const void *ptr_end, int *widthp) +{ + const uint8_t *p = ptr; + + /* single byte (hot path) */ + if (ptr >= ptr_end) { + *widthp = 0; + return FYUG_EOF; + } + + if (!(p[0] & 0x80)) { + *widthp = 1; + return p[0]; + } + return fy_utf8_get_generic_s(ptr, ptr_end, widthp); +} + +static inline int fy_utf8_get_s_nocheck(const void *ptr, int *widthp) +{ + const uint8_t *p = ptr; + + if (!(p[0] & 0x80)) { + *widthp = 1; + return p[0]; + } + return fy_utf8_get_generic_s_nocheck(ptr, widthp); +} + +/* for most 64 bit arches this will fit in a single register */ +struct fy_utf8_result { + int c; + int w; +}; + +/* probably the most performant version */ +static inline struct fy_utf8_result fy_utf8_get_s_res(const void *ptr, const void *ptr_end) +{ + const uint8_t *p = ptr; + int c, width; + + /* single byte (hot path) */ + if (ptr >= ptr_end) + return (struct fy_utf8_result){ FYUG_EOF, 0 }; + + if (!(p[0] & 0x80)) + return (struct fy_utf8_result){ p[0], 1 }; + + c = fy_utf8_get_generic_s(ptr, ptr_end, &width); + if (c < 0) + return (struct fy_utf8_result){ c, 0 }; + return (struct fy_utf8_result){ c, width }; +} + +#endif diff --git a/src/util/fy-utils.c b/src/util/fy-utils.c new file mode 100644 index 0000000..6b45f15 --- /dev/null +++ b/src/util/fy-utils.c @@ -0,0 +1,877 @@ +/* + * fy-utils.c - Generic utilities for functionality that's missing + * from platforms. + * + * For now only used to implement memstream for Apple platforms. + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fy-utf8.h" +#include "fy-ctype.h" +#include "fy-utils.h" + +#if defined(__APPLE__) && (_POSIX_C_SOURCE < 200809L) + +/* + * adapted from http://piumarta.com/software/memstream/ + * + * Under the MIT license. + */ + +/* + * ---------------------------------------------------------------------------- + * + * OPEN_MEMSTREAM(3) BSD and Linux Library Functions OPEN_MEMSTREAM(3) + * + * SYNOPSIS + * #include "memstream.h" + * + * FILE *open_memstream(char **bufp, size_t *sizep); + * + * DESCRIPTION + * The open_memstream() function opens a stream for writing to a buffer. + * The buffer is dynamically allocated (as with malloc(3)), and + * automatically grows as required. After closing the stream, the caller + * should free(3) this buffer. + * + * When the stream is closed (fclose(3)) or flushed (fflush(3)), the + * locations pointed to by bufp and sizep are updated to contain, + * respectively, a pointer to the buffer and the current size of the + * buffer. These values remain valid only as long as the caller performs + * no further output on the stream. If further output is performed, then + * the stream must again be flushed before trying to access these + * variables. + * + * A null byte is maintained at the end of the buffer. This byte is not + * included in the size value stored at sizep. + * + * The stream's file position can be changed with fseek(3) or fseeko(3). + * Moving the file position past the end of the data already written fills + * the intervening space with zeros. + * + * RETURN VALUE + * Upon successful completion open_memstream() returns a FILE pointer. + * Otherwise, NULL is returned and errno is set to indicate the error. + * + * CONFORMING TO + * POSIX.1-2008 + * + * ---------------------------------------------------------------------------- + */ + +#ifndef min +#define min(X, Y) (((X) < (Y)) ? (X) : (Y)) +#endif + +struct memstream { + size_t position; + size_t size; + size_t capacity; + char *contents; + char **ptr; + size_t *sizeloc; +}; + +static int memstream_grow(struct memstream *ms, size_t minsize) +{ + size_t newcap; + char *newcontents; + + newcap = ms->capacity * 2; + while (newcap <= minsize + 1) + newcap *= 2; + newcontents = realloc(ms->contents, newcap); + if (!newcontents) + return -1; + ms->contents = newcontents; + memset(ms->contents + ms->capacity, 0, newcap - ms->capacity); + ms->capacity = newcap; + *ms->ptr = ms->contents; + return 0; +} + +static int memstream_read(void *cookie, char *buf, int count) +{ + struct memstream *ms = cookie; + size_t n; + + n = min(ms->size - ms->position, (size_t)count); + if (n < 1) + return 0; + memcpy(buf, ms->contents, n); + ms->position += n; + return n; +} + +static int memstream_write(void *cookie, const char *buf, int count) +{ + struct memstream *ms = cookie; + + if (ms->capacity <= ms->position + (size_t)count && + memstream_grow(ms, ms->position + (size_t)count) < 0) + return -1; + memcpy(ms->contents + ms->position, buf, count); + ms->position += count; + ms->contents[ms->position] = '\0'; + if (ms->size < ms->position) + *ms->sizeloc = ms->size = ms->position; + + return count; +} + +static fpos_t memstream_seek(void *cookie, fpos_t offset, int whence) +{ + struct memstream *ms = cookie; + fpos_t pos= 0; + + switch (whence) { + case SEEK_SET: + pos = offset; + break; + case SEEK_CUR: + pos = ms->position + offset; + break; + case SEEK_END: + pos = ms->size + offset; + break; + default: + errno= EINVAL; + return -1; + } + if (pos >= (fpos_t)ms->capacity && memstream_grow(ms, pos) < 0) + return -1; + ms->position = pos; + if (ms->size < ms->position) + *ms->sizeloc = ms->size = ms->position; + return pos; +} + +static int memstream_close(void *cookie) +{ + struct memstream *ms = cookie; + + ms->size = min(ms->size, ms->position); + *ms->ptr = ms->contents; + *ms->sizeloc = ms->size; + ms->contents[ms->size]= 0; + /* ms->contents is what's returned */ + free(ms); + return 0; +} + +FILE *open_memstream(char **ptr, size_t *sizeloc) +{ + struct memstream *ms; + FILE *fp; + + if (!ptr || !sizeloc) { + errno= EINVAL; + goto err_out; + } + + ms = calloc(1, sizeof(struct memstream)); + if (!ms) + goto err_out; + + ms->position = ms->size= 0; + ms->capacity = 4096; + ms->contents = calloc(ms->capacity, 1); + if (!ms->contents) + goto err_free_ms; + ms->ptr = ptr; + ms->sizeloc = sizeloc; + fp= funopen(ms, memstream_read, memstream_write, + memstream_seek, memstream_close); + if (!fp) + goto err_free_all; + *ptr = ms->contents; + *sizeloc = ms->size; + return fp; + +err_free_all: + free(ms->contents); +err_free_ms: + free(ms); +err_out: + return NULL; +} + +#endif /* __APPLE__ && _POSIX_C_SOURCE < 200809L */ + +bool fy_tag_uri_is_valid(const char *data, size_t len) +{ + const char *s, *e; + int w, j, k, width, c; + uint8_t octet, esc_octets[4]; + + s = data; + e = s + len; + + while ((c = fy_utf8_get(s, e - s, &w)) >= 0) { + if (c != '%') { + s += w; + continue; + } + + width = 0; + k = 0; + do { + /* short URI escape */ + if ((e - s) < 3) + return false; + + if (width > 0) { + c = fy_utf8_get(s, e - s, &w); + if (c != '%') + return false; + } + + s += w; + + octet = 0; + + for (j = 0; j < 2; j++) { + c = fy_utf8_get(s, e - s, &w); + if (!fy_is_hex(c)) + return false; + s += w; + + octet <<= 4; + if (c >= '0' && c <= '9') + octet |= c - '0'; + else if (c >= 'a' && c <= 'f') + octet |= 10 + c - 'a'; + else + octet |= 10 + c - 'A'; + } + if (!width) { + width = fy_utf8_width_by_first_octet(octet); + + if (width < 1 || width > 4) + return false; + k = 0; + } + esc_octets[k++] = octet; + + } while (--width > 0); + + /* now convert to utf8 */ + c = fy_utf8_get(esc_octets, k, &w); + + if (c < 0) + return false; + } + + return true; +} + +int fy_tag_handle_length(const char *data, size_t len) +{ + const char *s, *e; + int c, w; + + s = data; + e = s + len; + + c = fy_utf8_get(s, e - s, &w); + if (c != '!') + return -1; + s += w; + + c = fy_utf8_get(s, e - s, &w); + if (fy_is_ws(c)) + return s - data; + /* if first character is !, empty handle */ + if (c == '!') { + s += w; + return s - data; + } + if (!fy_is_first_alpha(c)) + return -1; + s += w; + while (fy_is_alnum(c = fy_utf8_get(s, e - s, &w))) + s += w; + if (c == '!') + s += w; + + return s - data; +} + +int fy_tag_uri_length(const char *data, size_t len) +{ + const char *s, *e; + int c, w, cn, wn, uri_length; + + s = data; + e = s + len; + + while (fy_is_uri(c = fy_utf8_get(s, e - s, &w))) { + cn = fy_utf8_get(s + w, e - (s + w), &wn); + if ((fy_is_z(cn) || fy_is_blank(cn) || fy_is_any_lb(cn)) && fy_utf8_strchr(",}]", c)) + break; + s += w; + } + uri_length = s - data; + + if (!fy_tag_uri_is_valid(data, uri_length)) + return -1; + + return uri_length; +} + +int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info) +{ + const char *s, *e; + int total_length, handle_length, uri_length, prefix_length, suffix_length; + int c, cn, w, wn; + + s = data; + e = s + len; + + prefix_length = 0; + + /* it must start with '!' */ + c = fy_utf8_get(s, e - s, &w); + if (c != '!') + return -1; + cn = fy_utf8_get(s + w, e - (s + w), &wn); + if (cn == '<') { + prefix_length = 2; + suffix_length = 1; + } else + prefix_length = suffix_length = 0; + + if (prefix_length) { + handle_length = 0; /* set the handle to '' */ + s += prefix_length; + } else { + /* either !suffix or !handle!suffix */ + /* we scan back to back, and split handle/suffix */ + handle_length = fy_tag_handle_length(s, e - s); + if (handle_length <= 0) + return -1; + s += handle_length; + } + + uri_length = fy_tag_uri_length(s, e - s); + if (uri_length < 0) + return -1; + + /* a handle? */ + if (!prefix_length && (handle_length == 0 || data[handle_length - 1] != '!')) { + /* special case, '!', handle set to '' and suffix to '!' */ + if (handle_length == 1 && uri_length == 0) { + handle_length = 0; + uri_length = 1; + } else { + uri_length = handle_length - 1 + uri_length; + handle_length = 1; + } + } + total_length = prefix_length + handle_length + uri_length + suffix_length; + + if (total_length != (int)len) + return -1; + + info->total_length = total_length; + info->handle_length = handle_length; + info->uri_length = uri_length; + info->prefix_length = prefix_length; + info->suffix_length = suffix_length; + + return 0; +} + +/* simple terminal methods; mainly for getting size of terminal */ +int fy_term_set_raw(int fd, struct termios *oldt) +{ + struct termios newt, t; + int ret; + + /* must be a terminal */ + if (!isatty(fd)) + return -1; + + ret = tcgetattr(fd, &t); + if (ret != 0) + return ret; + + newt = t; + + cfmakeraw(&newt); + + ret = tcsetattr(fd, TCSANOW, &newt); + if (ret != 0) + return ret; + + if (oldt) + *oldt = t; + + return 0; +} + +int fy_term_restore(int fd, const struct termios *oldt) +{ + /* must be a terminal */ + if (!isatty(fd)) + return -1; + + return tcsetattr(fd, TCSANOW, oldt); +} + +ssize_t fy_term_write(int fd, const void *data, size_t count) +{ + ssize_t wrn, r; + + if (!isatty(fd)) + return -1; + r = 0; + wrn = 0; + while (count > 0) { + do { + r = write(fd, data, count); + } while (r == -1 && errno == EAGAIN); + if (r < 0) + break; + wrn += r; + data += r; + count -= r; + } + + /* return the amount written, or the last error code */ + return wrn > 0 ? wrn : r; +} + +int fy_term_safe_write(int fd, const void *data, size_t count) +{ + if (!isatty(fd)) + return -1; + + return fy_term_write(fd, data, count) == (ssize_t)count ? 0 : -1; +} + +ssize_t fy_term_read(int fd, void *data, size_t count, int timeout_us) +{ + ssize_t rdn, r; + struct timeval tv, tvto, *tvp; + fd_set rdfds; + + if (!isatty(fd)) + return -1; + + FD_ZERO(&rdfds); + + memset(&tvto, 0, sizeof(tvto)); + memset(&tv, 0, sizeof(tv)); + + if (timeout_us >= 0) { + tvto.tv_sec = timeout_us / 1000000; + tvto.tv_usec = timeout_us % 1000000; + tvp = &tv; + } else { + tvp = NULL; + } + + r = 0; + rdn = 0; + while (count > 0) { + do { + FD_SET(fd, &rdfds); + if (tvp) + *tvp = tvto; + r = select(fd + 1, &rdfds, NULL, NULL, tvp); + } while (r == -1 && errno == EAGAIN); + + /* select ends, or something weird */ + if (r <= 0 || !FD_ISSET(fd, &rdfds)) + break; + + /* now read */ + do { + r = read(fd, data, count); + } while (r == -1 && errno == EAGAIN); + if (r < 0) + break; + + rdn += r; + data += r; + count -= r; + } + + /* return the amount written, or the last error code */ + return rdn > 0 ? rdn : r; +} + +ssize_t fy_term_read_escape(int fd, void *buf, size_t count) +{ + char *p; + int r, rdn; + char c; + + /* at least 3 characters */ + if (count < 3) + return -1; + + p = buf; + rdn = 0; + + /* ESC */ + r = fy_term_read(fd, &c, 1, 100 * 1000); + if (r != 1 || c != '\x1b') + return -1; + *p++ = c; + count--; + rdn++; + + /* [ */ + r = fy_term_read(fd, &c, 1, 100 * 1000); + if (r != 1 || c != '[') + return rdn; + *p++ = c; + count--; + rdn++; + + /* read until error, out of buffer, or < 0x40 || > 0x7e */ + r = -1; + while (count > 0) { + r = fy_term_read(fd, &c, 1, 100 * 1000); + if (r != 1) + r = -1; + if (r != 1) + break; + *p++ = c; + count--; + rdn++; + + /* end of escape */ + if (c >= 0x40 && c <= 0x7e) + break; + } + + return rdn; +} + +int fy_term_query_size_raw(int fd, int *rows, int *cols) +{ + char buf[32]; + char *s, *e; + ssize_t r; + + /* must be a terminal */ + if (!isatty(fd)) + return -1; + + *rows = *cols = 0; + + /* query text area */ + r = fy_term_safe_write(fd, "\x1b[18t", 5); + if (r != 0) + return r; + + /* read a character */ + r = fy_term_read_escape(fd, buf, sizeof(buf)); + + /* return must be ESC[8;;;t */ + + if (r < 8 || r >= (int)sizeof(buf) - 2) /* minimum ESC[8;1;1t */ + return -1; + + s = buf; + e = s + r; + + /* correct response? starts with ESC[8; */ + if (s[0] != '\x1b' || s[1] != '[' || s[2] != '8' || s[3] != ';') + return -1; + s += 4; + + /* must end with t */ + if (e[-1] != 't') + return -1; + *--e = '\0'; /* remove trailing t, and zero terminate */ + + /* scan two ints separated by ; */ + r = sscanf(s, "%d;%d", rows, cols); + if (r != 2) + return -1; + + return 0; +} + +int fy_term_query_size(int fd, int *rows, int *cols) +{ + struct termios old_term; + int ret, r; + + if (!isatty(fd)) + return -1; + + r = fy_term_set_raw(fd, &old_term); + if (r != 0) + return -1; + + ret = fy_term_query_size_raw(fd, rows, cols); + + r = fy_term_restore(fd, &old_term); + if (r != 0) + return -1; + + return ret; +} + +int fy_comment_iter_begin(const char *comment, size_t size, struct fy_comment_iter *iter) +{ + if (!comment || !iter) + return -1; + + memset(iter, 0, sizeof(*iter)); + iter->start = comment; + iter->size = size == (size_t)-1 ? strlen(comment) : size; + iter->end = iter->start + iter->size; + + if (!iter->size) + return -1; + + iter->next = iter->start; + iter->line = 0; + + return 0; +} + +const char *fy_comment_iter_next_line(struct fy_comment_iter *iter, size_t *lenp) +{ + const char *s, *e, *le, *t; + + if (!iter || !lenp || !iter->start) + return NULL; + + /* no more */ + if (iter->next >= iter->end) + return NULL; +again: + *lenp = 0; + + s = iter->next; + e = iter->end; + + /* skip whitespace */ + while (s < e && isblank(*s)) + s++; + + if (s >= e) + return NULL; + + /* find end of line */ + le = memchr(s, '\n', e - s); + if (!le) { + le = e; + iter->next = e; + } else { + iter->next = le + 1; + } + + /* final? check if ends in *\/ */ + if (le >= e && (le - s) > 2 && le[-2] == '*' && le[-1] == '/') + le -= 2; + + /* backtrack while there's space at the end of line */ + while (le > s && isblank(le[-1])) + le--; + + /* check if the whole line is punctuation so it's formatting */ + t = s; + while (t < le && ispunct(*t)) + t++; + + /* everything is punctuation? */ + if (t > s && t >= le) { + if (((le - s) == 2 && s[0] == '/' && s[1] == '/') || /* '//' -> empty line */ + ((le - s) == 1 && s[0] == '*')) { /* '*' -> empty line */ + iter->line++; + return ""; + } + /* for anything else, just try again */ + goto again; + } + + /* something is there, skip over '// ' or '* ' */ + if ((le - s) > 3 && s[0] == '/' && s[1] == '/' && isblank(s[2])) + s += 3; + else if ((le - s) > 2 && s[0] == '*' && isblank(s[1])) + s += 2; + else if (iter->line == 0 && (le - s) > 3 && s[0] == '/' && s[1] == '*' && isblank(s[2])) + s += 3; + + iter->line++; + *lenp = le - s; + return s; +} + +void fy_comment_iter_end(struct fy_comment_iter *iter) +{ + /* nothing */ +} + +char *fy_get_cooked_comment(const char *raw_comment, size_t size) +{ + struct fy_comment_iter iter; + FILE *fp; + char *buf; + const char *line; + size_t len, line_len; + int ret; + + if (!raw_comment) + return NULL; + + fp = open_memstream(&buf, &len); + if (!fp) + return NULL; + + ret = 0; + fy_comment_iter_begin(raw_comment, size, &iter); + while ((line = fy_comment_iter_next_line(&iter, &line_len)) != NULL) { + ret = fprintf(fp, "%.*s\n", (int)line_len, line); + if (ret < 0) + break; + } + fy_comment_iter_end(&iter); + + fclose(fp); + + if (ret < 0) { + if (buf) + free(buf); + return NULL; + } + + /* must be freed */ + return buf; +} + +int fy_keyword_iter_begin(const char *text, size_t size, const char *keyword, struct fy_keyword_iter *iter) +{ + if (!text || !size || !keyword || !iter) + return -1; + + memset(iter, 0, sizeof(*iter)); + iter->keyword = keyword; + iter->keyword_len = strlen(keyword); + iter->start = text; + iter->size = size == (size_t)-1 ? strlen(text) : size; + iter->end = iter->start + iter->size; + iter->pc = '\n'; + + if (!iter->size) + return -1; + + iter->next = iter->start; + + return 0; +} + +const char *fy_keyword_iter_next(struct fy_keyword_iter *iter) +{ + const char *keyword; + size_t keyword_len; + const char *s, *e; + int c, w, pc, mode; + + if (!iter || !iter->start) + return NULL; + + /* no more */ + if (iter->next >= iter->end) + return NULL; + + s = iter->next; + e = iter->end; + + keyword = iter->keyword; + keyword_len = iter->keyword_len; + + mode = 0; + pc = iter->pc; + while ((c = fy_utf8_get_s(s, e, &w)) >= 0) { + + /* simple state machine to handle quoting */ + switch (mode) { + case 0: /* unquoted */ + if (c == keyword[0] && (fy_is_any_lb(pc) || fy_is_ws(pc)) && + (size_t)(e - s) > (keyword_len + 1) && !memcmp(s, keyword, keyword_len) && + (fy_is_ws(s[keyword_len]) || fy_is_any_lb(s[keyword_len]))) { + + iter->next = s; + iter->pc = pc; + return s; + } + + if (c == '\'') + mode = 1; + else if (c == '"') + mode = 3; + break; + case 1: /* single quote */ + if (c == '\\') + mode = 2; /* escaped single quote? */ + else if (c == '\'') + mode = 0; /* back to unquoted */ + break; + case 2: /* single quote backslash */ + mode = 1; /* back to single quote mode always */ + break; + case 3: /* double quote */ + if (c == '\\') + mode = 4; /* escaped quote? */ + else if (c == '"') + mode = 0; /* back to unquoted */ + break; + case 4: + mode = 3; /* back to double quote mode always */ + break; + } + s += w; + pc = c; + } + + return NULL; +} + +void fy_keyword_iter_advance(struct fy_keyword_iter *iter, size_t advance) +{ + int w; + const char *prev; + + if (!iter || !iter->next) + return; + + prev = iter->next; + + iter->next += advance; + if (iter->next >= iter->end) + iter->next = iter->end; + + iter->pc = fy_utf8_get_right(prev, (size_t)(iter->next - prev), &w); + if (iter->pc < 0) + iter->pc = '\n'; +} + +void fy_keyword_iter_end(struct fy_keyword_iter *iter) +{ + /* nothing */ +} diff --git a/src/util/fy-utils.h b/src/util/fy-utils.h new file mode 100644 index 0000000..20758b2 --- /dev/null +++ b/src/util/fy-utils.h @@ -0,0 +1,109 @@ +/* + * fy-utils.h - internal utilities header file + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_UTILS_H +#define FY_UTILS_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#if defined(__linux__) +#include +#endif + +#if defined(__APPLE__) && (_POSIX_C_SOURCE < 200809L) +FILE *open_memstream(char **ptr, size_t *sizeloc); +#endif + +int fy_tag_handle_length(const char *data, size_t len); +bool fy_tag_uri_is_valid(const char *data, size_t len); +int fy_tag_uri_length(const char *data, size_t len); + +struct fy_tag_scan_info { + int total_length; + int handle_length; + int uri_length; + int prefix_length; + int suffix_length; +}; + +int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info); + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) ((sizeof(x)/sizeof((x)[0]))) +#endif + +#if !defined(NDEBUG) && (defined(__GNUC__) && __GNUC__ >= 4) +#define FY_ALWAYS_INLINE __attribute__((always_inline)) +#else +#define FY_ALWAYS_INLINE /* nothing */ +#endif + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define FY_UNUSED __attribute__((unused)) +#else +#define FY_UNUSED /* nothing */ +#endif + +int fy_term_set_raw(int fd, struct termios *oldt); +int fy_term_restore(int fd, const struct termios *oldt); +ssize_t fy_term_write(int fd, const void *data, size_t count); +int fy_term_safe_write(int fd, const void *data, size_t count); +ssize_t fy_term_read(int fd, void *data, size_t count, int timeout_us); +ssize_t fy_term_read_escape(int fd, void *buf, size_t count); + +/* the raw methods require the terminal to be in raw mode */ +int fy_term_query_size_raw(int fd, int *rows, int *cols); + +/* the non raw methods will set the terminal to raw and then restore */ +int fy_term_query_size(int fd, int *rows, int *cols); + +struct fy_comment_iter { + const char *start; + size_t size; + const char *end; + const char *next; + int line; +}; + +int fy_comment_iter_begin(const char *comment, size_t size, struct fy_comment_iter *iter); +const char *fy_comment_iter_next_line(struct fy_comment_iter *iter, size_t *lenp); +void fy_comment_iter_end(struct fy_comment_iter *iter); + +char *fy_get_cooked_comment(const char *raw_comment, size_t size); + +struct fy_keyword_iter { + const char *keyword; + size_t keyword_len; + const char *start; + size_t size; + const char *end; + const char *next; + int pc; +}; + +int fy_keyword_iter_begin(const char *text, size_t size, const char *keyword, struct fy_keyword_iter *iter); +const char *fy_keyword_iter_next(struct fy_keyword_iter *iter); +void fy_keyword_iter_advance(struct fy_keyword_iter *iter, size_t advance); +void fy_keyword_iter_end(struct fy_keyword_iter *iter); + +#if !defined(S_ISREG) && defined(S_IFMT) && defined(S_IFREG) +#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) +#endif + +#if !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR) +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +#endif + +#endif diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..610a062 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +*.timestamp diff --git a/test/Makefile.am b/test/Makefile.am index f7add5b..f70ddc1 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -29,7 +29,8 @@ if HAVE_COMPATIBLE_CHECK check_PROGRAMS = libfyaml-test libfyaml_test_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/valgrind/ \ - -I$(top_srcdir)/src/lib/ + -I$(top_srcdir)/src/lib/ \ + -I$(top_srcdir)/src/util libfyaml_test_LDADD = $(AM_LDADD) $(CHECK_LIBS) $(top_builddir)/src/libfyaml.la libfyaml_test_CFLAGS = $(AM_CFLAGS) $(CHECK_CFLAGS) libfyaml_test_LDFLAGS = $(AM_LDFLAGS) $(CHECK_LDFLAGS) @@ -43,7 +44,9 @@ libfyaml_test_SOURCES = \ # the private tests require static compilation # if we're compiling shared only, disable the private parts if HAVE_STATIC -libfyaml_test_SOURCES += libfyaml-test-private.c +libfyaml_test_SOURCES += \ + libfyaml-test-private.c \ + libfyaml-test-private-id.c libfyaml_test_LDFLAGS += -static endif @@ -68,24 +71,31 @@ TESTS += jsontestsuite.test # normal YAML testsuite using document event stream TESTS += testsuite-evstream.test -testsuite.test: test-suite-data json-test-suite-data +# resolution equivalence +TESTS += testsuite-resolution.test -test-suite-data: - @GIT@ clone "@TESTSUITEURL@" -- $@ && \ - cd $@ && \ - @GIT@ checkout --detach @TESTSUITECHECKOUT@ +testsuite.test: test-suite-data.checkout.timestamp +if HAVE_JQ +testsuite-json.test: test-suite-data.checkout.timestamp +endif +testsuite-evstream.test: test-suite-data.checkout.timestamp +testsuite-resolution.test: test-suite-data.checkout.timestamp -jsontestsuite.test: json-test-suite-data +test-suite-data.checkout.timestamp: + @GIT@ clone -q "@TESTSUITEURL@" -- test-suite-data && \ + @GIT@ -C test-suite-data checkout -q --detach @TESTSUITECHECKOUT@ && \ + touch test-suite-data.checkout.timestamp -json-test-suite-data: - @GIT@ clone "@JSONTESTSUITEURL@" -- $@ && \ - cd $@ && \ - @GIT@ checkout --detach @JSONTESTSUITECHECKOUT@ +jsontestsuite.test: json-test-suite-data.checkout.timestamp -check-local: test-suite-data json-test-suite-data +json-test-suite-data.checkout.timestamp: + @GIT@ clone -q "@JSONTESTSUITEURL@" -- json-test-suite-data && \ + @GIT@ -C json-test-suite-data checkout -q --detach @JSONTESTSUITECHECKOUT@ && \ + touch json-test-suite-data.checkout.timestamp distclean-local: - @rm -rf test-suite-data json-test-suite-data + @rm -rf test-suite-data test-suite-data.checkout.timestamp + @rm -rf json-test-suite-data json-test-suite-data.checkout.timestamp endif endif diff --git a/test/libfyaml-test-core.c b/test/libfyaml-test-core.c index d4cd39f..8d3e4e6 100644 --- a/test/libfyaml-test-core.c +++ b/test/libfyaml-test-core.c @@ -1922,6 +1922,80 @@ START_TEST(scanf_check) } END_TEST +START_TEST(token_test) { + struct fy_document *fyd; + struct fy_node *fyn_sequence, *fyn_mapping, *fyn_scalar; + struct fy_token *fyn_sequence_start, *fyn_sequence_end, + *fyn_mapping_start, *fyn_mapping_end, *fyn_scalar_token; + struct fy_parse_cfg cfg = { + .flags = FYPCF_PARSE_COMMENTS + }; + + /* build document */ + + fyd = fy_document_build_from_string(&cfg, "- name: key\n" + " value: value\n", FY_NT); + + + /* Root (sequence) */ + fyn_sequence = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_sequence, NULL); + + fyn_sequence_start = fy_node_get_start_token(fyn_sequence); + ck_assert_ptr_ne(fyn_sequence_start, NULL); + + ck_assert_ptr_ne(fy_token_start_mark(fyn_sequence_start), NULL); + + ck_assert_int_eq(fy_token_start_mark(fyn_sequence_start)->line, 0); + ck_assert_int_eq(fy_token_start_mark(fyn_sequence_start)->column, 0); + + fyn_sequence_end = fy_node_get_end_token(fyn_sequence); + ck_assert_ptr_ne(fyn_sequence_end, NULL); + + ck_assert_int_eq(fy_token_end_mark(fyn_sequence_end)->line, 2); + ck_assert_int_eq(fy_token_end_mark(fyn_sequence_end)->column, 0); + + /* Mapping (sequence item) */ + fyn_mapping = fy_node_sequence_get_by_index(fyn_sequence, 0); + ck_assert_ptr_ne(fyn_mapping, NULL); + + fyn_mapping_start = fy_node_get_start_token(fyn_mapping); + ck_assert_ptr_ne(fyn_mapping_start, NULL); + + ck_assert_ptr_ne(fy_token_start_mark(fyn_mapping_start), NULL); + + ck_assert_int_eq(fy_token_start_mark(fyn_mapping_start)->line, 0); + ck_assert_int_eq(fy_token_start_mark(fyn_mapping_start)->column, 2); + + fyn_mapping_end = fy_node_get_end_token(fyn_mapping); + ck_assert_ptr_ne(fyn_mapping_end, NULL); + + ck_assert_ptr_ne(fy_token_start_mark(fyn_mapping_end), NULL); + + ck_assert_int_eq(fy_token_start_mark(fyn_mapping_end)->line, 2); + ck_assert_int_eq(fy_token_start_mark(fyn_mapping_end)->column, 0); + + /* Scalar (key) */ + fyn_scalar = fy_node_pair_key(fy_node_mapping_get_by_index(fyn_mapping, 0)); + ck_assert_ptr_ne(fyn_scalar, NULL); + + fyn_scalar_token = fy_node_get_start_token(fyn_scalar); + ck_assert_ptr_ne(fyn_scalar_token, NULL); + ck_assert_ptr_eq(fyn_scalar_token, fy_node_get_end_token(fyn_scalar)); + + ck_assert_ptr_ne(fy_token_start_mark(fyn_scalar_token), NULL); + ck_assert_int_eq(fy_token_start_mark(fyn_scalar_token)->line, 0); + ck_assert_int_eq(fy_token_start_mark(fyn_scalar_token)->column, 2); + + ck_assert_ptr_ne(fy_token_end_mark(fyn_scalar_token), NULL); + ck_assert_int_eq(fy_token_end_mark(fyn_scalar_token)->line, 0); + ck_assert_int_eq(fy_token_end_mark(fyn_scalar_token)->column, 6); + + /* cleanup */ + fy_document_destroy(fyd); +} +END_TEST + TCase *libfyaml_case_core(void) { TCase *tc; @@ -1991,5 +2065,7 @@ TCase *libfyaml_case_core(void) tcase_add_test(tc, scanf_check); + tcase_add_test(tc, token_test); + return tc; } diff --git a/test/libfyaml-test-private-id.c b/test/libfyaml-test-private-id.c new file mode 100644 index 0000000..a016211 --- /dev/null +++ b/test/libfyaml-test-private-id.c @@ -0,0 +1,359 @@ +/* + * libfyaml-test-private-id.c - libfyaml id allocation and handling + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "fy-utils.h" +#include "fy-id.h" + +/* check ffs works */ +START_TEST(id_ffs) +{ + static const struct { + fy_id_bits v; + int r; + } ffs_check[] = { + { .v = 0, .r = -1 }, + { .v = ~(fy_id_bits)0, .r = 0 }, + { .v = ((fy_id_bits)1 << 0), .r = 0 }, + { .v = ((fy_id_bits)1 << (FY_ID_BITS_BITS - 1)), .r = (FY_ID_BITS_BITS - 1) }, + { .v = ((fy_id_bits)1 << 0) | ((fy_id_bits)1 << (FY_ID_BITS_BITS - 1)), .r = 0 }, + { .v = ((fy_id_bits)1 << (FY_ID_BITS_BITS / 2)), .r = (FY_ID_BITS_BITS / 2), }, + }; + unsigned int i; + int r; + + for (i = 0; i < ARRAY_SIZE(ffs_check); i++) { + r = fy_id_ffs(ffs_check[i].v); + ck_assert_int_eq(r, ffs_check[i].r); + } +} +END_TEST + +/* a random bit number for the following tests, not a power of 2 */ +#define BA_BITS 67 + +#define BA_DECL \ + size_t ba_count = FY_ID_BITS_ARRAY_COUNT(BA_BITS); \ + fy_id_bits *ba = alloca(ba_count * sizeof(*ba)) + +/* verify that reset clears everything */ +START_TEST(id_reset) +{ + BA_DECL; + unsigned int i; + + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_count; i++) + ck_assert_int_eq(ba[i], 0); +} +END_TEST + +/* verify that allocation to the full works, and then fails */ +START_TEST(id_alloc_full) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i; + int id, expected_id; + + fy_id_reset(ba, ba_count); + + /* allocate all, verify that we get numbers in sequence */ + expected_id = 0; + for (i = 0; i < ba_bits_actual; i++) { + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, expected_id); + expected_id++; + } + + /* full, it must fail now */ + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, -1); +} +END_TEST + +/* verify that allocation when almost full works, and then fails */ +START_TEST(id_alloc_almost_full) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i; + int id, expected_id; + + /* fill the array */ + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_bits_actual; i++) + fy_id_set_used(ba, ba_count, i); + + /* allocate all, verify that we get numbers in sequence */ + expected_id = 0; + for (i = 0; i < ba_bits_actual; i++) { + /* free one, and allocate, it must succeed at the exact spot */ + fy_id_free(ba, ba_count, i); + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, expected_id); + expected_id++; + + /* now it must fail */ + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, -1); + } +} +END_TEST + +/* verify that allocation of even bits works */ +START_TEST(id_alloc_even) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i, j; + int id, expected_id; + + /* fill the array */ + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_bits_actual; i++) + fy_id_set_used(ba, ba_count, i); + + /* free the even ids */ + j = ba_bits_actual / 2; + for (i = 0; i < j; i++) + fy_id_free(ba, ba_count, i * 2); + + /* allocate all, verify that we get numbers in sequence */ + expected_id = 0; + for (i = 0; i < j; i++) { + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, expected_id); + expected_id += 2; + } +} +END_TEST + +/* verify that allocation of odd bits works */ +START_TEST(id_alloc_odd) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i, j; + int id, expected_id; + + /* fill the array */ + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_bits_actual; i++) + fy_id_set_used(ba, ba_count, i); + + /* free the even ids */ + j = ba_bits_actual / 2; + for (i = 0; i < j; i++) + fy_id_free(ba, ba_count, i * 2 + 1); + + /* allocate all, verify that we get numbers in sequence */ + expected_id = 1; + for (i = 0; i < j; i++) { + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, expected_id); + expected_id += 2; + } +} +END_TEST + +/* verify that a specific allocation sequence works */ +START_TEST(id_alloc_seq) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i; + int id, expected_id; + int check_ids[] = { + 0, + ba_bits_actual / 2 - 1, + ba_bits_actual / 2, + ba_bits_actual / 2 + 1, + ba_bits_actual - 1 + }; + + /* fill the array */ + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_bits_actual; i++) + fy_id_set_used(ba, ba_count, i); + + /* free those specific bits */ + for (i = 0; i < ARRAY_SIZE(check_ids); i++) + fy_id_free(ba, ba_count, check_ids[i]); + + /* now allocate in sequence */ + expected_id = 0; + for (i = 0; i < ARRAY_SIZE(check_ids); i++) { + expected_id = check_ids[i]; + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, expected_id); + } +} +END_TEST + +/* verify that a iterator works for a single bit in array */ +START_TEST(id_iter_single) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i; + int id, expected_id, found_id; + struct fy_id_iter iter; + + for (i = 0; i < ba_bits_actual; i++) { + fy_id_reset(ba, ba_count); + fy_id_set_used(ba, ba_count, i); + + expected_id = i; + found_id = -1; + fy_id_iter_begin(ba, ba_count, &iter); + while ((id = fy_id_iter_next(ba, ba_count, &iter)) >= 0) { + /* must find a single one */ + ck_assert_int_lt(found_id, 0); + found_id = id; + ck_assert_int_eq(id, expected_id); + } + fy_id_iter_end(ba, ba_count, &iter); + } + +} +END_TEST + +/* verify that a iterator works for a full array */ +START_TEST(id_iter_full) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i; + int id, expected_id; + struct fy_id_iter iter; + + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_bits_actual; i++) + fy_id_set_used(ba, ba_count, i); + + expected_id = 0; + fy_id_iter_begin(ba, ba_count, &iter); + while ((id = fy_id_iter_next(ba, ba_count, &iter)) >= 0) { + ck_assert_int_eq(id, expected_id); + expected_id++; + } + + /* we must have run through the whole array */ + ck_assert_int_eq(expected_id, ba_bits_actual); + + fy_id_iter_end(ba, ba_count, &iter); +} +END_TEST + +/* verify that a iterator works for sequences up to 3 bits */ +START_TEST(id_iter_seq) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + int idtab[][3] = { + { 0, ba_bits_actual - 1, -1 }, + { 0, 1, -1 }, + { ba_bits_actual - 2, ba_bits_actual - 1, -1 }, + { 0, FY_ID_BITS_BITS - 1, -1 }, + { 0, FY_ID_BITS_BITS, -1 }, + { FY_ID_BITS_BITS - 1, FY_ID_BITS_BITS, -1 }, + { FY_ID_BITS_BITS, FY_ID_BITS_BITS + 1, -1 }, + { 0, 1, 2 }, + { 0, ba_bits_actual - 2, ba_bits_actual - 1 }, + { ba_bits_actual - 3, ba_bits_actual - 2, ba_bits_actual - 1 }, + }; + unsigned int i; + int id, expected_id; + int p0, p1, p2; + struct fy_id_iter iter; + + for (i = 0; i < ARRAY_SIZE(idtab); i++) { + + p0 = idtab[i][0]; + p1 = idtab[i][1]; + p2 = idtab[i][2]; + + fy_id_reset(ba, ba_count); + if (p0 >= 0) + fy_id_set_used(ba, ba_count, p0); + if (p1 >= 0) + fy_id_set_used(ba, ba_count, p1); + if (p2 >= 0) + fy_id_set_used(ba, ba_count, p2); + + fy_id_iter_begin(ba, ba_count, &iter); + + if (p0 >= 0) { + id = fy_id_iter_next(ba, ba_count, &iter); + expected_id = p0; + ck_assert_int_eq(id, expected_id); + fy_id_set_free(ba, ba_count, expected_id); + } + + if (p1 >= 0) { + id = fy_id_iter_next(ba, ba_count, &iter); + expected_id = p1; + ck_assert_int_eq(id, expected_id); + fy_id_set_free(ba, ba_count, expected_id); + } + + if (p2 >= 0) { + id = fy_id_iter_next(ba, ba_count, &iter); + expected_id = p2; + ck_assert_int_eq(id, expected_id); + fy_id_set_free(ba, ba_count, expected_id); + } + fy_id_iter_end(ba, ba_count, &iter); + + /* and now it should be empty */ + fy_id_iter_begin(ba, ba_count, &iter); + id = fy_id_iter_next(ba, ba_count, &iter); + ck_assert_int_eq(id, -1); + fy_id_iter_end(ba, ba_count, &iter); + } +} +END_TEST + +TCase *libfyaml_case_private_id(void) +{ + TCase *tc; + + tc = tcase_create("private-id"); + + tcase_add_test(tc, id_ffs); + + tcase_add_test(tc, id_reset); + tcase_add_test(tc, id_alloc_full); + tcase_add_test(tc, id_alloc_almost_full); + tcase_add_test(tc, id_alloc_even); + tcase_add_test(tc, id_alloc_odd); + tcase_add_test(tc, id_alloc_seq); + + tcase_add_test(tc, id_iter_single); + tcase_add_test(tc, id_iter_full); + tcase_add_test(tc, id_iter_seq); + + return tc; +} diff --git a/test/libfyaml-test.c b/test/libfyaml-test.c index b5986c5..0f45c42 100644 --- a/test/libfyaml-test.c +++ b/test/libfyaml-test.c @@ -39,6 +39,7 @@ static void display_usage(FILE *fp, char *progname) #if defined(HAVE_STATIC) && HAVE_STATIC extern TCase *libfyaml_case_private(void); +extern TCase *libfyaml_case_private_id(void); #endif extern TCase *libfyaml_case_core(void); extern TCase *libfyaml_case_meta(void); @@ -52,6 +53,7 @@ Suite *libfyaml_suite(void) #if defined(HAVE_STATIC) && HAVE_STATIC suite_add_tcase(s, libfyaml_case_private()); + suite_add_tcase(s, libfyaml_case_private_id()); #endif suite_add_tcase(s, libfyaml_case_core()); suite_add_tcase(s, libfyaml_case_meta()); diff --git a/test/testsuite-resolution.test b/test/testsuite-resolution.test new file mode 100755 index 0000000..2aaf07d --- /dev/null +++ b/test/testsuite-resolution.test @@ -0,0 +1,99 @@ +#!/bin/bash + +count=0 +for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do + # there must be a test there + if [ ! -e "$tst/===" ]; then + continue + fi + # skip tests that are expected to fail + if [ -e "$tst/error" ]; then + continue + fi + # there must be an alias in the test + grep -q '\*[A-Za-z]' "$tst/in.yaml" + if [ $? -ne 0 ]; then + continue + fi + count=`expr $count + 1` + done +done + +# output plan +echo 1..$count + +# X38W has duplicate keys after resolution +skiplist="2JQS X38W" +xfaillist="" + +i=0 +for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + + for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do + + # there must be a test there + if [ ! -e "$tst/===" ]; then + continue + fi + + # skip tests that are expected to fail + if [ -e "$tst/error" ]; then + continue + fi + + # there must be an alias in the test + grep -q '\*[A-Za-z]' "$tst/in.yaml" + if [ $? -ne 0 ]; then + continue + fi + + i=`expr $i + 1` + + # strip trailing / + t=${tst%/} + # remove test-suite-data/ + test_subtest_id=`echo $t | cut -d/ -f2-` + test_id=`echo $test_subtest_id | cut -d/ -f1` + subtest_id=`echo $test_subtest_id | cut -s -d/ -f2` + + desctxt=`cat 2>/dev/null "$tst/==="` + + t1=`mktemp` + t2=`mktemp` + + directive="" + for skip in $skiplist; do + if [ "$test_subtest_id" == "$skip" ]; then + directive=" # skip: duplicate keys in testcase; cannot load as document" + break + fi + done + + res="ok" + if [ "x$directive" == "x" ]; then + res="not ok" + + ${TOP_BUILDDIR}/src/fy-tool --dump --resolve "$tst/in.yaml" | ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t1" + ${TOP_BUILDDIR}/src/fy-tool --dump --resolve --streaming "$tst/in.yaml" | ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t2" + + diff -u "$t1" "$t2" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + + rm -f "$t1" "$t2" + + for xfail in $xfaillist; do + if [ "$test_subtest_id" == "$xfail" ]; then + directive=" # TODO: known failure." + break + fi + done + fi + + echo "$res $i $test_subtest_id - $desctxt$directive" + done +done