sphinx/
src/fy-tool
src/libfyaml-parser
+src/fy-thread
+src/fy-b3sum
+src/*.a
stamp-h1
tags
TAGS
include(GNUInstallDirs)
set(LIBHDRS
- src/valgrind/fy-valgrind.h
- src/xxhash/xxhash.h
src/lib/fy-accel.h
src/lib/fy-atom.h
src/lib/fy-composer.h
- src/lib/fy-ctype.h
src/lib/fy-diag.h
src/lib/fy-doc.h
- src/lib/fy-docbuilder.c
+ src/lib/fy-docbuilder.h
src/lib/fy-docstate.h
src/lib/fy-dump.h
- src/lib/fy-emit-accum.h
- src/lib/fy-emit.h
- src/lib/fy-event.h
+ src/lib/fy-emit.h src/lib/fy-emit-accum.h
src/lib/fy-input.h
- src/lib/fy-list.h
src/lib/fy-parse.h
src/lib/fy-path.h
src/lib/fy-token.h
- src/lib/fy-typelist.h
src/lib/fy-types.h
- src/lib/fy-utf8.h
- src/lib/fy-utils.h
src/lib/fy-walk.h
+ src/util/fy-blob.h
+ src/util/fy-ctype.h
+ src/util/fy-endian.h
+ src/util/fy-id.h
+ src/util/fy-list.h
+ src/util/fy-typelist.h
+ src/util/fy-utf8.h
+ src/util/fy-utils.h
+ src/xxhash/xxhash.h
)
set(LIBSRCS
src/lib/fy-accel.c
src/lib/fy-atom.c
src/lib/fy-composer.c
- src/lib/fy-ctype.c
src/lib/fy-diag.c
src/lib/fy-doc.c
+ src/lib/fy-docbuilder.c
src/lib/fy-docstate.c
src/lib/fy-dump.c
src/lib/fy-emit.c
src/lib/fy-path.c
src/lib/fy-token.c
src/lib/fy-types.c
- src/lib/fy-utf8.c
- src/lib/fy-utils.c
src/lib/fy-walk.c
+ src/util/fy-blob.c
+ src/util/fy-ctype.c
+ src/util/fy-utf8.c
+ src/util/fy-utils.c
src/xxhash/xxhash.c
)
set(LIBHDRSPUB
${LIBHDRSPUB}
)
+include(CheckIncludeFile)
+check_include_file("byteswap.h" HAVE_BYTESWAP_H)
+include(CheckFunctionExists)
+check_function_exists(__builtin_bswap16 HAVE___BUILTIN_BSWAP16)
+check_function_exists(__builtin_bswap32 HAVE___BUILTIN_BSWAP32)
+check_function_exists(__builtin_bswap64 HAVE___BUILTIN_BSWAP64)
+
add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
set_target_properties(${PROJECT_NAME} PROPERTIES FOLDER "Libraries")
$<INSTALL_INTERFACE:include>
PRIVATE
${${PROJECT_NAME}_SOURCE_DIR}/src/lib
- ${${PROJECT_NAME}_SOURCE_DIR}/src/lib/internal
+ ${${PROJECT_NAME}_SOURCE_DIR}/src/util
${${PROJECT_NAME}_SOURCE_DIR}/src/xxhash
)
-AC_PREREQ(2.61)
+AC_PREREQ(2.69)
AC_INIT([libfyaml],
m4_esyscmd([build-aux/git-version-gen .tarball-version]),
[pantelis.antoniou@konsulko.com])
+AC_CONFIG_AUX_DIR([build-aux])
+
+AC_PATH_PROG([M4], [m4 gm4], [no])
+if test "x$M4" = xno ; then
+ AC_MSG_ERROR([m4 missing])
+fi
+
+AC_SUBST(ACLOCAL_AMFLAGS, "-I m4")
+
+AC_CANONICAL_TARGET
+
AC_CONFIG_SRCDIR([src/lib/fy-parse.c])
-AC_CONFIG_HEADER([config.h])
+AC_CONFIG_HEADERS([config.h])
AC_CONFIG_MACRO_DIR([m4])
-AC_CONFIG_AUX_DIR([build-aux])
-AM_INIT_AUTOMAKE([foreign 1.8.5 -Wall subdir-objects ])
+AM_INIT_AUTOMAKE([foreign 1.8.5 subdir-objects -Wno-portability])
m4_pattern_allow([^(AM_EXTRA_RECURSIVE_TARGETS|AM_PROG_AR)$])dnl
AC_PROG_MKDIR_P
AC_PROG_CC
-AC_PROG_CC_C99
AM_PROG_CC_C_O
+AM_PROG_AS
AC_PROG_GCC_TRADITIONAL
AC_USE_SYSTEM_EXTENSIONS
AC_PROG_AWK
AC_PROG_LN_S
-AX_PTHREAD
+AX_PTHREAD([], [AC_MSG_ERROR("Missing required pthread support")])
-AC_PATH_PROG([M4], [m4 gm4], [no])
-if test "x$M4" = xno ; then
- AC_MSG_ERROR([m4 missing])
+# in some cases PTHREAD_LIBS is empty - force -lpthread */
+if test "x$PTHREAD_LIBS" = "x"; then
+ PTHREAD_LIBS="-lpthread"
fi
-AC_SUBST(ACLOCAL_AMFLAGS, "-I m4")
+CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
+LIBS="$LIBS $PTHREAD_LIBS"
# pkg-config
PKG_PROG_PKG_CONFIG
AC_LTDL_ENABLE_INSTALL
-AC_LIBLTDL_INSTALLABLE
-AC_LIBTOOL_DLOPEN
-AC_LIBTOOL_WIN32_DLL
-AC_PROG_LIBTOOL
+LT_INIT([dlopen win32-dll])
AC_SUBST(LTDLINCL)
AC_SUBST(LIBLTDL)
AC_CONFIG_SUBDIRS(libltdl)
-AC_HEADER_STDC
AC_C_CONST
AC_C_BIGENDIAN
AC_TYPE_PID_T
AC_CHECK_TYPES(ssize_t, , [AC_DEFINE([ssize_t], [signed long],
[Define ssize_t if it is not done by the standard libs.])])
AC_TYPE_OFF_T
-AC_TYPE_SIGNAL
AC_TYPE_UID_T
AC_CHECK_DECLS(environ)
+AC_CHECK_HEADERS([byteswap.h])
+AC_CHECK_FUNCS([__builtin_bswap16 __builtin_bswap32 __builtin_bswap64])
+
dnl for old autoconf version AX_APPEND_COMPILE_FLAGS does not work
m4_version_prereq(2.64,
[AX_APPEND_COMPILE_FLAGS([-Wall -Wsign-compare -Wno-stringop-overflow -fvisibility=hidden],
AC_DEFINE_UNQUOTED([HAVE_STATIC], [$HAVE_STATIC], [Define to 1 if static linking is available])
AM_CONDITIONAL([HAVE_STATIC], [ test x$HAVE_STATIC = x1 ])
-dnl for old autoconf version AX_APPEND_COMPILE_FLAGS does not work
-m4_version_prereq(2.64,
- [AX_APPEND_COMPILE_FLAGS([-Wall -Wsign-compare -Wno-stringop-overflow -fvisibility=hidden],
- [CFLAGS], [-pedantic -Werror])],
- [CFLAGS="$CFLAGS -Wall -Wsign-compare"])
+dnl check whether warning flags are supported
+AX_CHECK_COMPILE_FLAG([-Wstringop-overread], , , [-Werror])
+AS_IF([test "x$ax_cv_check_cflags__Werror__wstringop_overread" = x"yes"],
+ [HAVE_WSTRINGOP_OVERREAD=1], [HAVE_WSTRINGOP_OVERREAD=0])
+AC_DEFINE_UNQUOTED([HAVE_WSTRINGOP_OVERREAD], [$HAVE_WSTRINGOP_OVERREAD], [Define to 1 if -Wstringop-overread is supported])
+
+dnl Per target optimizations
+AC_ARG_ENABLE([portable-target],
+ AS_HELP_STRING([--enable-portable-target],
+ [Enable portable mode (disable per-target optimizations)]))
+HAVE_PORTABLE_TARGET=0
+if test "x$enable_portable_target" == "xyes"; then
+ HAVE_PORTABLE_TARGET=1
+fi
+AC_SUBST(HAVE_PORTABLE_TARGET)
+AC_DEFINE_UNQUOTED([HAVE_PORTABLE_TARGET], [$HAVE_PORTABLE_TARGET], [Define to 1 if PORTABLE_TARGET is enabled])
+AM_CONDITIONAL([HAVE_PORTABLE_TARGET], [ test x$HAVE_PORTABLE_TARGET == x1 ])
+
+AM_CONDITIONAL([TARGET_CPU_X86], [ test x$target_cpu = xx86 ])
+AM_CONDITIONAL([TARGET_CPU_X86_64], [ test x$target_cpu = xx86_64 ])
+AM_CONDITIONAL([TARGET_CPU_ANY_X86], [ test x$target_cpu = xx86 -o x$target_cpu = xx86_64 ])
+AM_CONDITIONAL([TARGET_CPU_ARM], [ test x$target_cpu = xarm ])
+AM_CONDITIONAL([TARGET_CPU_ARM64], [ test x$target_cpu = xaarch64 ])
+AM_CONDITIONAL([TARGET_CPU_ANY_ARM], [ test x$target_cpu = xarm -o x$target_cpu = xaarch64 ])
+
+AM_COND_IF([TARGET_CPU_ANY_X86], [m4_version_prereq(2.64, [
+ AX_CHECK_COMPILE_FLAG([-msse2], , , [-Werror])
+ AX_CHECK_COMPILE_FLAG([-msse4.1], , , [-Werror])
+ AX_CHECK_COMPILE_FLAG([-mavx2], , , [-Werror])
+ AX_CHECK_COMPILE_FLAG([-mavx512f -mavx512vl], , , [-Werror])
+ ], [true])
+])
+
+AM_COND_IF([TARGET_CPU_ARM], [m4_version_prereq(2.64, [
+ AX_CHECK_COMPILE_FLAG([-mfpu=neon], , , [-Werror])
+ ], [true])
+])
+
+AM_CONDITIONAL([TARGET_HAS_SSE2], [ test "x$ax_cv_check_cflags__Werror__msse2" = x"yes" -a "x$enable_portable_target" != "xyes" ])
+AM_CONDITIONAL([TARGET_HAS_SSE41], [ test "x$ax_cv_check_cflags__Werror__msse4_1" = x"yes" -a "x$enable_portable_target" != "xyes" ])
+AM_CONDITIONAL([TARGET_HAS_AVX2], [ test "x$ax_cv_check_cflags__Werror__mavx2" = x"yes" -a "x$enable_portable_target" != "xyes" ])
+AM_CONDITIONAL([TARGET_HAS_AVX512], [ test "x$ax_cv_check_cflags__Werror__mavx512f__mavx512vl" = x"yes" -a "x$enable_portable_target" != "xyes" ])
+
+AM_CONDITIONAL([TARGET_HAS_NEON], [ test \( x$target_cpu = xaarch64 -o "x$ax_cv_check_cflags__Werror__mcpu_neon" = x"yes" \) -a "x$enable_portable_target" != "xyes" ])
+
+AM_COND_IF([TARGET_HAS_SSE2], AC_DEFINE([TARGET_HAS_SSE2], [1], [SSE2 target support]))
+AM_COND_IF([TARGET_HAS_SSE41], AC_DEFINE([TARGET_HAS_SSE41], [1], [SSE41 target support]))
+AM_COND_IF([TARGET_HAS_AVX2], AC_DEFINE([TARGET_HAS_AVX2], [1], [AVX2 target support]))
+AM_COND_IF([TARGET_HAS_AVX512], AC_DEFINE([TARGET_HAS_AVX512], [1], [AVX512 target support]))
+
+AM_COND_IF([TARGET_HAS_NEON], AC_DEFINE([TARGET_HAS_NEON], [1], [NEON target support]))
dnl ASAN enable switch
AC_ARG_ENABLE([asan],
[ test x$HAVE_ASAN = x1 ])
AC_DEFINE_UNQUOTED([HAVE_ASAN], [$HAVE_ASAN], [Define to 1 if ASAN is enabled])
-# include -lm in the link
-AC_SEARCH_LIBS([llrintf], [m], [], [AC_MSG_ERROR([unable to find the llrintf() function])])
-
# check if there's a qsort_r available (musl does not have it)
AC_CHECK_FUNC([qsort_r],
HAVE_QSORT_R=1,
AM_CONDITIONAL([HAVE_JQ], [ test x$HAVE_JQ = x1 ])
AC_SUBST(JQ, "$JQ")
+# check for libclang
+AC_ARG_WITH([libclang],
+ dnl enable internet available when checking
+AC_ARG_ENABLE([network],
+ AS_HELP_STRING([--disable-network],
+ [Disable tests requiring network access]))
+
+ [AS_HELP_STRING([--with-libclang=[<version>]],
+ [Enable support for libclang @<:@default=auto@:>@])],
+ [with_libclang="$withval"],
+ [with_libclang=auto])
+
+HAVE_LIBCLANG=0
+LIBCLANG_CFLAGS=""
+LIBCLANG_CPPFLAGS=""
+LIBCLANG_LDFLAGS=""
+LIBCLANG_LIBS=""
+dnl TODO think about cross compilation
+if test "x$with_libclang" != "xno"; then
+ if test "x$with_libclang" == "xauto"; then
+ # try to auto detect (llvm 15 down to 8)
+ check="llvm-config llvm-config-15 llvm-config-14 llvm-config-13 llvm-config-12 llvm-config-11 llvm-config-10"
+ else
+ if test "x$with_libclang" == "xlatest"; then
+ check="llvm-config"
+ else
+ check="llvm-config-$with_libclang"
+ fi
+ fi
+ orig_CFLAGS="$CFLAGS"
+ orig_CPPFLAGS="$CPPFLAGS"
+ orig_LDFLAGS="$LDFLAGS"
+ orig_LIBS="$LIBS"
+ for cfg in $check ; do
+ # no caching
+ AC_PATH_PROG([LLVM_CONFIG], [$cfg])
+ if test "x$LLVM_CONFIG" != "x" ; then
+ LIBCLANG_CFLAGS=`$LLVM_CONFIG --cflags`
+ LIBCLANG_CPPFLAGS=`$LLVM_CONFIG --cppflags`
+ LIBCLANG_LDFLAGS=`$LLVM_CONFIG --ldflags`
+ LIBCLANG_LIBS="-lclang"
+ CPPFLAGS="$CPPFLAGS $LIBCLANG_CPPFLAGS"
+ CFLAGS="$CFLAGS $LIBCLANG_CFLAGS"
+ LDFLAGS="$LDFLAGS $LIBCLANG_LDFLAGS"
+ LIBS="$LIBS $LIBCLANG_LIBS"
+ AC_CHECK_LIB([clang], [clang_createIndex],
+ [HAVE_LIBCLANG_LIB=1], [HAVE_LIBCLANG_LIB=0])
+ AC_CHECK_HEADER([clang-c/Index.h],
+ [HAVE_LIBCLANG_HDR=1], [HAVE_LIBCLANG_HDR=0])
+ # if we found it, break out
+ if test "x$HAVE_LIBCLANG_LIB" = "x1" -a "x$HAVE_LIBCLANG_HDR" = "x1" ; then
+ HAVE_LIBCLANG=1
+ break
+ fi
+ $as_unset ac_cv_lib_clang_clang_createIndex $as_unset ac_cv_header_clang_c_Index_h \
+ LIBCLANG_CFLAGS LIBCLANG_CPPFLAGS LIBCLANG_LIBS HAVE_LIBCLANG_LIB HAVE_LIBCLANG_HDR
+ fi
+ CFLAGS=$orig_CFLAGS
+ LDFLAGS=$orig_LDFLAGS
+ LIBS=$orig_LIBS
+ if test "x$HAVE_LIBCLANG" = "x1"; then
+ break
+ fi
+ $as_unset ac_cv_path_LLVM_CONFIG LLVM_CONFIG
+ done
+ CFLAGS=$orig_CFLAGS
+ CPPFLAGS=$orig_CPPFLAGS
+ LDFLAGS=$orig_LDFLAGS
+ LIBS=$orig_LIBS
+fi
+
+dnl it is safe to update CFLAGS, CPPFLAGS and LDFLAGS from clang
+dnl we will only link against libclang when we have to
+
+AC_SUBST(HAVE_LIBCLANG)
+AC_SUBST(LIBCLANG_CFLAGS)
+AC_SUBST(LIBCLANG_CPPFLAGS)
+AC_SUBST(LIBCLANG_LDFLAGS)
+AC_SUBST(LIBCLANG_LIBS)
+AC_DEFINE_UNQUOTED([HAVE_LIBCLANG], [$HAVE_LIBCLANG], [Define to 1 if libclang is available])
+AM_CONDITIONAL([HAVE_LIBCLANG], [ test x$HAVE_LIBCLANG = x1 ])
+
+dnl AS_IF([test "x$with_libclang" != xno],
+dnl [AC_CHECK_LIB([libclang], [main],
+dnl [AC_SUBST([LIBREADLINE], ["-lreadline -lncurses"])
+dnl AC_DEFINE([HAVE_LIBREADLINE], [1],
+dnl [Define if you have libreadline])
+dnl ],
+dnl [if test "x$with_readline" != xcheck; then
+dnl AC_MSG_FAILURE(
+dnl [--with-readline was given, but test for readline failed])
+dnl fi
+dnl ], -lncurses)])
+
+dnl trim cflags, cppflags, ldflags
+CFLAGS=`echo $CFLAGS | sed -e 's/^[ ]*//'`
+CPPFLAGS=`echo $CPPFLAGS | sed -e 's/^[ ]*//'`
+LDFLAGS=`echo $LDFLAGS | sed -e 's/^[ ]*//'`
+
# Shave by default on supported autoconf versions
dnl m4_version_prereq(2.64, SHAVE_INIT([build-aux], [enable]), [true])
m4_ifdef([SHAVE_INIT], [SHAVE_INIT([build-aux], [enable])])
LIBTOOL_VERSION: ${LIBTOOL_VERSION}
prefix: ${prefix}
C compiler: ${CC}
- CFLAGS: ${CFLAGS}
Linker: ${LD}
+ CFLAGS: ${CFLAGS}
LDFLAGS: ${LDFLAGS}
LIBS: ${LIBS}
HAVE_ASAN: ${HAVE_ASAN}
HAVE_NETWORK: ${HAVE_NETWORK}
HAVE_DEVMODE: ${HAVE_DEVMODE}
HAVE_SPHINX: ${HAVE_SPHINX}
- GIT: $GIT
- DOCKER: $DOCKER
+ GIT: ${GIT}
+ DOCKER: ${DOCKER}
+ HAVE_LIBCLANG: ${HAVE_LIBCLANG}
+ LIBCLANG_LIBS: ${LIBCLANG_LIBS}
TESTSUITEURL: $TESTSUITEURL
TESTSUITECHECKOUT: $TESTSUITECHECKOUT
JSONTESTSUITEURL: $JSONTESTSUITEURL
#if defined(__GNUC__) && __GNUC__ >= 4
#define FY_EXPORT __attribute__ ((visibility ("default")))
#define FY_DEPRECATED __attribute__ ((deprecated))
+#define FY_FORMAT(_t, _x, _y) __attribute__ ((format(_t, _x, _y)))
#else
#define FY_EXPORT /* nothing */
#define FY_DEPRECATED /* nothing */
+#define FY_FORMAT(_t, x, y) /* nothing */
#endif
/* make a copy of an allocated string and return it on stack
fy_parser_get_stream_error(struct fy_parser *fyp)
FY_EXPORT;
+/**
+ * fy_parser_vlog() - Log using the parsers diagnostics printf style (va_arg)
+ *
+ * Output a log on the parser diagnostic output
+ *
+ * @fyp: The parser
+ * @type: The error type
+ * @fmt: The printf format string
+ * @ap: The argument list
+ */
+void
+fy_parser_vlog(struct fy_parser *fyp, enum fy_error_type type, const char *fmt, va_list ap)
+ FY_EXPORT;
+
+/**
+ * fy_parser_log() - Log using the parsers diagnostics printf style
+ *
+ * Output a report on the parser's diagnostics
+ *
+ * @fyp: The parser
+ * @type: The error type
+ * @fmt: The printf format string
+ * @...: The extra arguments
+ */
+void
+fy_parser_log(struct fy_parser *fyp, enum fy_error_type type, const char *fmt, ...)
+ FY_FORMAT(printf, 3, 4)
+ FY_EXPORT;
+
+#ifndef NDEBUG
+
+#define fy_parser_debug(_fyp, _fmt, ...) \
+ fy_parser_log((_fyp), FYET_DEBUG, (_fmt) , ## __VA_ARGS__)
+#else
+
+#define fy_parser_debug(_fyp, _fmt, ...) \
+ do { } while(0)
+
+#endif
+
+#define fy_parser_info(_fyp, _fmt, ...) \
+ fy_parser_log((_fyp), FYET_INFO, (_fmt) , ## __VA_ARGS__)
+#define fy_parser_notice(_fyp, _fmt, ...) \
+ fy_parser_log((_fyp), FYET_NOTICE, (_fmt) , ## __VA_ARGS__)
+#define fy_parser_warning(_fyp, _fmt, ...) \
+ fy_parser_log((_fyp), FYET_WARNING, (_fmt) , ## __VA_ARGS__)
+#define fy_parser_error(_fyp, _fmt, ...) \
+ fy_parser_log((_fyp), FYET_ERROR, (_fmt) , ## __VA_ARGS__)
+
+/**
+ * fy_parser_vreport() - Report using the parsers diagnostics printf style and mark token (va_arg)
+ *
+ * Output a report on the parser and indicate the token's position
+ *
+ * @fyp: The parser
+ * @type: The error type
+ * @fyt: The token
+ * @fmt: The printf format string
+ * @ap: The argument list
+ */
+void
+fy_parser_vreport(struct fy_parser *fyp, enum fy_error_type type, struct fy_token *fyt,
+ const char *fmt, va_list ap)
+ FY_EXPORT;
+
+/**
+ * fy_parser_report() - Report using the parsers diagnostics printf style and mark token
+ *
+ * Output a report on the parser and indicate the token's position
+ *
+ * @fyp: The parser
+ * @type: The error type
+ * @fyt: The token
+ * @fmt: The printf format string
+ * @..: The extra arguments
+ */
+void
+fy_parser_report(struct fy_parser *fyp, enum fy_error_type type, struct fy_token *fyt,
+ const char *fmt, ...)
+ FY_FORMAT(printf, 4, 5)
+ FY_EXPORT;
+
+#ifndef NDEBUG
+
+#define fy_parser_report_debug(_fyp, _fyt, _fmt, ...) \
+ fy_parser_report((_fyp), FYET_DEBUG, (_fyt), (_fmt) , ## __VA_ARGS__)
+#else
+
+#define fy_parser_report_debug(_fyp, _fyt, _fmt, ...) \
+ do { } while(0)
+
+#endif
+
+#define fy_parser_report_info(_fyp, _fyt, _fmt, ...) \
+ fy_parser_report((_fyp), FYET_INFO, (_fyt), (_fmt) , ## __VA_ARGS__)
+#define fy_parser_report_notice(_fyp, _fyt, _fmt, ...) \
+ fy_parser_report((_fyp), FYET_NOTICE, (_fyt), (_fmt) , ## __VA_ARGS__)
+#define fy_parser_report_warning(_fyp, _fyt, _fmt, ...) \
+ fy_parser_report((_fyp), FYET_WARNING, (_fyt), (_fmt) , ## __VA_ARGS__)
+#define fy_parser_report_error(_fyp, _fyt, _fmt, ...) \
+ fy_parser_report((_fyp), FYET_ERROR, (_fyt), (_fmt) , ## __VA_ARGS__)
+
/**
* fy_token_scalar_style() - Get the style of a scalar token
*
* fy_token_scalar_is_null() - Test whether the scalar is null (content)
*
* @fyt: The scalar token to check for NULLity.
- *
+ *
* Note that this is different than null of the YAML type system.
* It is null as in null content. It is also different than an
* empty scalar.
fy_emit_event(struct fy_emitter *emit, struct fy_event *fye)
FY_EXPORT;
+/**
+ * fy_emit_vevent() - Queue (and possibly emit) an event using varargs
+ *
+ * Queue and output using the emitter. The format is identical to
+ * the one used in fy_emit_event_vcreate().
+ *
+ * @emit: The emitter to use
+ * @type: The event type to create
+ * @ap: The variable argument list pointer.
+ *
+ * Returns:
+ * 0 on success, -1 on error
+ */
+int
+fy_emit_vevent(struct fy_emitter *emit, enum fy_event_type type, va_list ap)
+ FY_EXPORT;
+
+/**
+ * fy_emit_eventf() - Queue (and possibly emit) an event using variable args
+ *
+ * Queue and output using the emitter. The format is identical to
+ * the one used in fy_emit_event_create().
+ *
+ * @emit: The emitter to use
+ * @type: The event type to create
+ * @...: The optional extra arguments
+ *
+ * Returns:
+ * 0 on success, -1 on error
+ */
+int
+fy_emit_eventf(struct fy_emitter *emit, enum fy_event_type type, ...)
+ FY_EXPORT;
+
+/**
+ * fy_emit_scalar_write() - Emit a scalar with write semantics
+ *
+ * Queue and output using the emitter a scalar using a standard
+ * write interface.
+ *
+ * @emit: The emitter to use
+ * @style: The scalar style to use
+ * @anchor: The anchor or NULL
+ * @tag: The tag or NULL
+ * @buf: Pointer to the buffer
+ * @count: The number of bytes to write
+ *
+ * Returns:
+ * 0 on success, -1 on error
+ */
+int
+fy_emit_scalar_write(struct fy_emitter *fye, enum fy_scalar_style style,
+ const char *anchor, const char *tag,
+ const char *buf, size_t count)
+ FY_EXPORT;
+
+/**
+ * fy_emit_scalar_vprintf() - Emit a scalar with vprintf semantics
+ *
+ * Queue and output using the emitter a scalar using a standard
+ * vprintf interface.
+ *
+ * @emit: The emitter to use
+ * @style: The scalar style to use
+ * @anchor: The anchor or NULL
+ * @tag: The tag or NULL
+ * @fmt: The printf like format string
+ * @ap: The argument list
+ *
+ * Returns:
+ * 0 on success, -1 on error
+ */
+int
+fy_emit_scalar_vprintf(struct fy_emitter *fye, enum fy_scalar_style style,
+ const char *anchor, const char *tag,
+ const char *fmt, va_list ap)
+ FY_EXPORT;
+
+/**
+ * fy_emit_scalar_printf() - Emit a scalar with printf semantics
+ *
+ * Queue and output using the emitter a scalar using a standard
+ * printf interface.
+ *
+ * @emit: The emitter to use
+ * @style: The scalar style to use
+ * @anchor: The anchor or NULL
+ * @tag: The tag or NULL
+ * @fmt: The printf like format string
+ * @...: The extra arguments
+ *
+ * Returns:
+ * 0 on success, -1 on error
+ */
+int
+fy_emit_scalar_printf(struct fy_emitter *fye, enum fy_scalar_style style,
+ const char *anchor, const char *tag,
+ const char *fmt, ...)
+ FY_FORMAT(printf, 5, 6)
+ FY_EXPORT;
+
/**
* fy_emit_event_from_parser() - Queue (and possibly emit) an event
* generated by the parser.
*/
struct fy_document *
fy_document_buildf(const struct fy_parse_cfg *cfg, const char *fmt, ...)
- __attribute__((format(printf, 2, 3)))
+ FY_FORMAT(printf, 2, 3)
FY_EXPORT;
/**
const char *str, size_t len, size_t *consumed)
FY_EXPORT;
+/**
+ * fy_block_document_build_from_string() - Create a document using the provided YAML source.
+ *
+ * Create a document parsing the provided string as a YAML source.
+ *
+ * The document is a block document, and it terminates when indentation
+ * appears to do so.
+ *
+ * Example of block documents:
+ *
+ * this-is-yaml
+ * foo: bar <- starts here
+ * baz:
+ * - 1
+ * - 2
+ * this-is-yaml-no-more
+ *
+ * @cfg: The parse configuration to use or NULL for the default.
+ * @str: The YAML source to use.
+ * @len: The length of the string (or -1 if '\0' terminated)
+ * @consumed: A pointer to the consumed amount
+ *
+ * Returns:
+ * The created document, or NULL on error.
+ */
+struct fy_document *
+fy_block_document_build_from_string(const struct fy_parse_cfg *cfg,
+ const char *str, size_t len, size_t *consumed)
+ FY_EXPORT;
+
/**
* fy_document_root() - Return the root node of the document
*
fy_node_get_style(struct fy_node *fyn)
FY_EXPORT;
+/**
+ * fy_node_get_start_token() - Get the node start token
+ *
+ * Retrieve the node start token.
+ *
+ * For scalars, this is the same as the end token.
+ *
+ * @fyn: The node
+ *
+ * Returns:
+ * The node start token
+ */
+struct fy_token * fy_node_get_start_token(struct fy_node *fyn)
+ FY_EXPORT;
+
+/**
+ * fy_node_get_end_token() - Get the node end token
+ *
+ * Retrieve the node end token.
+ *
+ * For scalars, this is the same as the start token.
+ *
+ * @fyn: The node
+ *
+ * Returns:
+ * The node end token
+ */
+ struct fy_token * fy_node_get_end_token(struct fy_node *fyn)
+ FY_EXPORT;
+
/**
* fy_node_is_scalar() - Check whether the node is a scalar
*
*/
struct fy_node *
fy_node_buildf(struct fy_document *fyd, const char *fmt, ...)
- __attribute__((format(printf, 2, 3)))
+ FY_FORMAT(printf, 2, 3)
FY_EXPORT;
/**
*/
struct fy_node *
fy_node_create_scalarf(struct fy_document *fyd, const char *fmt, ...)
- FY_EXPORT
- __attribute__((format(printf, 2, 3)));
+ FY_FORMAT(printf, 2, 3)
+ FY_EXPORT;
/**
* fy_node_create_sequence() - Create an empty sequence node.
*/
int
fy_node_scanf(struct fy_node *fyn, const char *fmt, ...)
- __attribute__((format(scanf, 2, 3)))
+ FY_FORMAT(scanf, 2, 3)
FY_EXPORT;
/**
*/
int
fy_document_scanf(struct fy_document *fyd, const char *fmt, ...)
- __attribute__((format(scanf, 2, 3)))
+ FY_FORMAT(scanf, 2, 3)
FY_EXPORT;
/**
*/
int
fy_node_set_anchorf(struct fy_node *fyn, const char *fmt, ...)
- FY_EXPORT __attribute__((format(printf, 2, 3)));
+ FY_FORMAT(printf, 2, 3)
+ FY_EXPORT;
/**
* fy_node_remove_anchor() - Remove an anchor
void
fy_node_report(struct fy_node *fyn, enum fy_error_type type,
const char *fmt, ...)
- __attribute__((format(printf, 3, 4)))
+ FY_FORMAT(printf, 3, 4)
FY_EXPORT;
/**
fy_node_override_report(struct fy_node *fyn, enum fy_error_type type,
const char *file, int line, int column,
const char *fmt, ...)
- __attribute__((format(printf, 6, 7)))
+ FY_FORMAT(printf, 6, 7)
FY_EXPORT;
typedef void (*fy_diag_output_fn)(struct fy_diag *diag, void *user,
*/
int
fy_diag_printf(struct fy_diag *diag, const char *fmt, ...)
- FY_EXPORT
- __attribute__((format(printf, 2, 3)));
+ FY_FORMAT(printf, 2, 3)
+ FY_EXPORT;
/**
* struct fy_diag_ctx - The diagnostics context
int
fy_diagf(struct fy_diag *diag, const struct fy_diag_ctx *fydc,
const char *fmt, ...)
- FY_EXPORT
- __attribute__((format(printf, 3, 4)));
+ FY_FORMAT(printf, 3, 4)
+ FY_EXPORT;
#define fy_diag_diag(_diag, _level, _fmt, ...) \
({ \
#define fy_error(_diag, _fmt, ...) \
fy_diag_diag((_diag), FYET_ERROR, (_fmt) , ## __VA_ARGS__)
+/**
+ * fy_diag_token_vreport() - Report about a token vprintf style using
+ * the given diagnostic object
+ *
+ * Output a report about the given token via the specific
+ * error type, and using the reporting configuration of the token's
+ * document.
+ *
+ * @diag: The diag object
+ * @fyt: The token
+ * @type: The error type
+ * @fmt: The printf format string
+ * @ap: The argument list
+ */
+void
+fy_diag_token_vreport(struct fy_diag *diag, struct fy_token *fyt,
+ enum fy_error_type type, const char *fmt, va_list ap)
+ FY_EXPORT;
+
+/**
+ * fy_diag_token_report() - Report about a token printf style using
+ * the given diagnostic object
+ *
+ * Output a report about the given token via the specific
+ * error type, and using the reporting configuration of the token's
+ * document.
+ *
+ * @diag: The diag object
+ * @fyt: The token
+ * @type: The error type
+ * @fmt: The printf format string
+ * @...: The extra arguments.
+ */
+void
+fy_diag_token_report(struct fy_diag *diag, struct fy_token *fye,
+ enum fy_error_type type, const char *fmt, ...)
+ FY_FORMAT(printf, 4, 5)
+ FY_EXPORT;
+
+/**
+ * fy_diag_token_override_vreport() - Report about a token vprintf style,
+ * overriding file, line and column info using
+ * the given diagnostic object
+ *
+ * Output a report about the given token via the specific
+ * error type, and using the reporting configuration of the token's
+ * document. This method will use the overrides provided in order
+ * to massage the reporting information.
+ * If @file is NULL, no file location will be reported.
+ * If either @line or @column is negative no location will be reported.
+ *
+ * @diag: The diag object
+ * @fyt: The token
+ * @type: The error type
+ * @file: The file override
+ * @line: The line override
+ * @column: The column override
+ * @fmt: The printf format string
+ * @ap: The argument list
+ */
+void
+fy_diag_token_override_vreport(struct fy_diag *diag, struct fy_token *fyt,
+ enum fy_error_type type, const char *file,
+ int line, int column, const char *fmt, va_list ap)
+ FY_EXPORT;
+
+/**
+ * fy_diag_token_override_report() - Report about a token printf style,
+ * overriding file, line and column info using
+ * the given diagnostic object
+ *
+ * Output a report about the given token via the specific
+ * error type, and using the reporting configuration of the token's
+ * document. This method will use the overrides provided in order
+ * to massage the reporting information.
+ * If @file is NULL, no file location will be reported.
+ * If either @line or @column is negative no location will be reported.
+ *
+ * @diag: The diag object
+ * @fyt: The token
+ * @type: The error type
+ * @file: The file override
+ * @line: The line override
+ * @column: The column override
+ * @fmt: The printf format string
+ * @...: The extra arguments.
+ */
+void
+fy_diag_token_override_report(struct fy_diag *diag, struct fy_token *fyt,
+ enum fy_error_type type, const char *file,
+ int line, int column, const char *fmt, ...)
+ FY_FORMAT(printf, 7, 8)
+ FY_EXPORT;
+
/**
* fy_diag_node_vreport() - Report about a node vprintf style using
* the given diagnostic object
void
fy_diag_node_report(struct fy_diag *diag, struct fy_node *fyn,
enum fy_error_type type, const char *fmt, ...)
- __attribute__((format(printf, 4, 5)))
+ FY_FORMAT(printf, 4, 5)
FY_EXPORT;
/**
fy_diag_node_override_report(struct fy_diag *diag, struct fy_node *fyn,
enum fy_error_type type, const char *file,
int line, int column, const char *fmt, ...)
- __attribute__((format(printf, 7, 8)))
+ FY_FORMAT(printf, 7, 8)
FY_EXPORT;
/**
fy_tag_token_suffix(struct fy_token *fyt, size_t *lenp)
FY_EXPORT;
+/**
+ * fy_tag_token_short() - Get the short tag of the tag token
+ *
+ * Retrieve the short tag contents. The short tag is the same
+ * one that will need to be emitted.
+ * Will fail if token is not a tag token, or if a memory error happens.
+ *
+ * @fyt: The tag token out of which the short pointer
+ * will be returned.
+ * @lenp: Pointer to a variable that will hold the returned length
+ *
+ * Returns:
+ * A pointer to the text representation of the short tag, while
+ * @lenp will be assigned the character length of said representation.
+ * NULL in case of an error.
+ */
+const char *
+fy_tag_token_short(struct fy_token *fyt, size_t *lenp)
+ FY_EXPORT;
+
/**
* fy_tag_token_handle0() - Get the handle contained in the
* tag token as zero terminated string
fy_tag_token_handle0(struct fy_token *fyt)
FY_EXPORT;
+/**
+ * fy_tag_token_short0() - Get the short tag of the tag token as zero
+ * terminated string.
+ *
+ * Retrieve the short tag contents. The short tag is the same
+ * one that will need to be emitted.
+ * Will fail if token is not a tag token, or if a memory error happens.
+ *
+ * @fyt: The tag token out of which the short pointer will be returned.
+ *
+ * Returns:
+ * A pointer to the null terminated text representation of the short tag.
+ * NULL in case of an error.
+ */
+const char *
+fy_tag_token_short0(struct fy_token *fyt)
+ FY_EXPORT;
+
/**
* fy_tag_token_suffix0() - Get the suffix contained in the
* tag token as zero terminated string
fy_document_state_tag_directive_iterate(struct fy_document_state *fyds, void **prevp)
FY_EXPORT;
+/**
+ * fy_document_state_tag_directives() - Get all the tag directives in a
+ * malloc'ed array
+ *
+ * Return all the tag directives in a dynamically allocated area.
+ * Must be free()'d when not in use.
+ *
+ * @fyds: The document state
+ *
+ * Returns:
+ * An array of fy_tag pointer structures, terminated with a NULL pointer
+ * NULL on error
+ */
+struct fy_tag **
+fy_document_state_tag_directives(struct fy_document_state *fyds)
+ FY_EXPORT;
+
/**
* fy_document_state_tag_is_default() - Test whether the given tag is a default one
*
*
* @emit: The emitter
* @type: The event type to create
+ * @...: The optional extra arguments
*
* Returns:
* The created event or NULL in case of an error
fy_path_component_set_sequence_user_data(struct fy_path_component *fypc, void *data)
FY_EXPORT;
+/**
+ * fy_path_get_parent_user_data() - Return the userdata of the parent collection
+ *
+ * @fypp: The path
+ *
+ * Returns:
+ * The user data associated with the parent collection of the path, or NULL if no path
+ */
+void *
+fy_path_get_parent_user_data(struct fy_path *path)
+ FY_EXPORT;
+
+/**
+ * fy_path_set_parent_user_data() - Set the user data associated with the parent collection
+ *
+ * Note, no error condition if not a path
+ *
+ * @fypp: The path
+ * @data: The data to set as parent collection data
+ */
+void
+fy_path_set_parent_user_data(struct fy_path *path, void *data)
+ FY_EXPORT;
+
+/**
+ * fy_path_get_last_user_data() - Return the userdata of the last collection
+ *
+ * @fypp: The path
+ *
+ * Returns:
+ * The user data associated with the last collection of the path, or NULL if no path
+ */
+void *
+fy_path_get_last_user_data(struct fy_path *path)
+ FY_EXPORT;
+
+/**
+ * fy_path_set_last_user_data() - Set the user data associated with the last collection
+ *
+ * Note, no error condition if not a path
+ *
+ * @fypp: The path
+ * @data: The data to set as last collection data
+ */
+void
+fy_path_set_last_user_data(struct fy_path *path, void *data)
+ FY_EXPORT;
+
/**
* fy_path_last_component() - Get the very last component of a path
*
fy_document_iterator_get_error(struct fy_document_iterator *fydi)
FY_EXPORT;
+/*
+ * The libfyaml's push-pull thread implementation
+ *
+ */
+
+/* opaque types for the user */
+struct fy_thread_pool;
+struct fy_thread;
+struct fy_work_pool;
+
+/**
+ * typedef fy_work_exec_fn - Work exec function
+ *
+ * The callback executed on work submission
+ *
+ * @arg: The argument to the method
+ *
+ */
+typedef void (*fy_work_exec_fn)(void *arg);
+
+/**
+ * typedef fy_work_check_fn - Work check function
+ *
+ * Work checker function to decide if it's worth to
+ * offload to a thread.
+ *
+ * @arg: The argument to the method
+ *
+ * Returns:
+ * true if it should offload to thread, false otherwise
+ *
+ */
+typedef bool (*fy_work_check_fn)(const void *arg);
+
+/**
+ * struct fy_thread_work - Work submitted to a thread for execution
+ *
+ * @fn: The execution function for this work
+ * @arg: The argument to the fn
+ * @wp: Used internally, must be set to NULL on entry
+ *
+ * This is the structure describing the work submitted
+ * to a thread for execution.
+ */
+struct fy_thread_work {
+ fy_work_exec_fn fn;
+ void *arg;
+ struct fy_work_pool *wp;
+};
+
+/*
+ * enum fy_thread_pool_cfg_flags - Thread pool configuration flags
+ *
+ * These flags control the operation of the thread pool.
+ * For now only the steal mode flag is defined.
+ *
+ * @FYTPCF_STEAL_MODE: Enable steal mode for the thread pool
+ */
+enum fy_thread_pool_cfg_flags {
+ FYTPCF_STEAL_MODE = FY_BIT(0),
+};
+
+/**
+ * struct fy_thread_pool_cfg - thread pool configuration structure.
+ *
+ * Argument to the fy_thread_pool_create() method.
+ *
+ * @flags: Thread pool configuration flags
+ * @num_threads: Number of threads, if 0 == online CPUs
+ * @userdata: A userdata pointer
+ */
+struct fy_thread_pool_cfg {
+ enum fy_thread_pool_cfg_flags flags;
+ unsigned int num_threads;
+ void *userdata;
+};
+
+/**
+ * fy_thread_pool_create() - Create a thread pool
+ *
+ * Creates a thread pool with its configuration @cfg
+ * The thread pool may be destroyed by a corresponding call to
+ * fy_thread_pool_destroy().
+ *
+ * @cfg: The configuration for the thread pool
+ *
+ * Returns:
+ * A pointer to the thread pool or NULL in case of an error.
+ */
+struct fy_thread_pool *
+fy_thread_pool_create(const struct fy_thread_pool_cfg *cfg)
+ FY_EXPORT;
+
+/**
+ * fy_thread_pool_destroy() - Destroy the given thread pool
+ *
+ * Destroy a thread pool created earlier via fy_thread_pool_create().
+ * Note that this function will block until all threads
+ * of the pool are destroyed.
+ *
+ * @tp: The thread pool to destroy
+ */
+void
+fy_thread_pool_destroy(struct fy_thread_pool *tp)
+ FY_EXPORT;
+
+/**
+ * fy_thread_pool_get_num_threads() - Get the number of threads
+ *
+ * Returns the actual number of created threads.
+ *
+ * @tp: The thread pool
+ *
+ * Returns:
+ * > 0 for the number of actual threads created,
+ * -1 on error
+ */
+int
+fy_thread_pool_get_num_threads(struct fy_thread_pool *tp)
+ FY_EXPORT;
+
+/**
+ * fy_thread_pool_get_cfg() - Get the configuration of a thread pool
+ *
+ * @tp: The thread pool
+ *
+ * Returns:
+ * The configuration of the thread pool
+ */
+const struct fy_thread_pool_cfg *
+fy_thread_pool_get_cfg(struct fy_thread_pool *tp)
+ FY_EXPORT;
+
+/*
+ * fy_thread_reserve() - Reserve a thread from the pool.
+ *
+ * Reserve a thread from the pool and return it.
+ * Note this is only valid for a non-work stealing thread pool.
+ * You release the thread again via a call to fy_thread_unreserve.
+ *
+ * @tp: The thread pool
+ *
+ * Returns:
+ * A reserved thread if not NULL, NULL if no threads are available.
+ */
+struct fy_thread *
+fy_thread_reserve(struct fy_thread_pool *tp)
+ FY_EXPORT;
+
+/*
+ * fy_thread_unreserve() - Unreserve a previously reserved thread
+ *
+ * Unreserve a thread previously reserved via a call to fy_thread_reserve()
+ * Note this is only valid for a non-work stealing thread pool.
+ *
+ * @t: The thread
+ */
+void
+fy_thread_unreserve(struct fy_thread *t)
+ FY_EXPORT;
+
+/*
+ * fy_thread_submit_work() - Submit work for execution
+ *
+ * Submit work for execution. If successful the thread
+ * will start executing the work in parallel with the
+ * calling thread. You can wait for the thread to
+ * terminate via a call to fy_thread_wait_work().
+ * The thread must have been reserved earlier via fy_thread_reserve()
+ *
+ * Note this is only valid for a non-work stealing thread pool.
+ *
+ * @t: The thread
+ * @work: The work
+ *
+ * Returns:
+ * 0 if work has been submitted, -1 otherwise.
+ */
+int
+fy_thread_submit_work(struct fy_thread *t, struct fy_thread_work *work)
+ FY_EXPORT;
+
+/*
+ * fy_thread_wait_work() - Wait for completion of submitted work
+ *
+ * Wait until submitted work to the thread has finished.
+ * Note this is only valid for a non-work stealing thread pool.
+ *
+ * @t: The thread
+ *
+ * Returns:
+ * 0 if work finished, -1 on error.
+ */
+int
+fy_thread_wait_work(struct fy_thread *t)
+ FY_EXPORT;
+
+/*
+ * fy_thread_work_join() - Submit works for execution and wait
+ *
+ * Submit works for possible parallel execution. If no offloading
+ * is possible at the time execute in the current context.
+ * It is possible to use in both stealing and non-stealing mode
+ * with the difference being that stealing mode is about 30% faster.
+ *
+ * @tp: The thread pool
+ * @works: Pointer to an array of works sized @work_count
+ * @work_count: The size of the @works array
+ * @check_fn: Pointer to a check function, or NULL for no checks
+ */
+void
+fy_thread_work_join(struct fy_thread_pool *tp,
+ struct fy_thread_work *works, size_t work_count,
+ fy_work_check_fn check_fn)
+ FY_EXPORT;
+
+/*
+ * fy_thread_args_join() - Execute function in parallel using arguments as pointers
+ *
+ * Execute @fn possibly in parallel using the threads in the thread pool.
+ * The arguments of the function are provided by the args array.
+ *
+ * @tp: The thread pool
+ * @fn: The function to execute in parallel
+ * @check_fn: Pointer to a check function, or NULL for no checks
+ * @args: An args array sized @count of argument pointers
+ * @count: The count of the args array items
+ */
+void
+fy_thread_args_join(struct fy_thread_pool *tp,
+ fy_work_exec_fn, fy_work_check_fn check_fn,
+ void **args, size_t count)
+ FY_EXPORT;
+
+/*
+ * fy_thread_arg_array_join() - Execute function in parallel using argument array
+ *
+ * Execute @fn possibly in parallel using the threads in the thread pool.
+ * The arguments of the function are provided by the args array.
+ *
+ * @tp: The thread pool
+ * @fn: The function to execute in parallel
+ * @check_fn: Pointer to a check function, or NULL for no checks
+ * @args: An args array of @argsize items
+ * @argsize: The size of each argument array item
+ * @count: The count of the args array items
+ */
+void
+fy_thread_arg_array_join(struct fy_thread_pool *tp,
+ fy_work_exec_fn fn, fy_work_check_fn check_fn,
+ void *args, size_t argsize, size_t count)
+ FY_EXPORT;
+
+/*
+ * fy_thread_arg_array_join() - Execute function in parallel with the same argument
+ *
+ * Execute @fn possibly in parallel using the threads in the thread pool.
+ * The argument of the functions is the same.
+ *
+ * @tp: The thread pool
+ * @fn: The function to execute in parallel
+ * @arg: The common argument
+ * @count: The count of executions
+ */
+void
+fy_thread_arg_join(struct fy_thread_pool *tp,
+ fy_work_exec_fn fn, fy_work_check_fn check_fn,
+ void *arg, size_t count)
+ FY_EXPORT;
+
+/*
+ * Minimal exposing of internal BLAKE3 implementation
+ *
+ */
+
+/* BLAKE3 key length */
+#define FY_BLAKE3_KEY_LEN 32
+/* BLAKE3 output length */
+#define FY_BLAKE3_OUT_LEN 32
+
+/* opaque BLAKE3 hasher type for the user*/
+struct fy_blake3_hasher;
+
+/**
+ * fy_blake3_backend_iterate() - Iterate over the supported BLAKE3 backends
+ *
+ * This method iterates over the supported BLAKE3 backends.
+ * The start of the iteration is signalled by a NULL in \*prevp.
+ *
+ * The default backend is always the last in sequence, so for
+ * example if the order is [ "portable", "sse2", NULL ] the
+ * default is "sse2".
+ *
+ * @prevp: The previous backend pointer, or NULL at start
+ *
+ * Returns:
+ * The next backend or NULL at the end.
+ */
+const char *
+fy_blake3_backend_iterate(const char **prevp)
+ FY_EXPORT;
+
+/**
+ * struct fy_blake3_hasher_cfg - BLAKE3 hasher configuration
+ *
+ * Argument to the fy_blake3_hasher_create() method which
+ * is the fyaml's user facing BLAKE3 API.
+ * It is very minimal, on purpose, since it's meant to be
+ * exposing a full blown BLAKE3 API.
+ *
+ * @backend: NULL for default, or a specific backend name
+ * @file_buffer: Use this amount of buffer for buffering, zero for default
+ * @mmap_min_chunk: Minimum chunk size for mmap case
+ * @mmap_max_chunk: Maximum chunk size for mmap case
+ * @no_mmap: Disable mmap for file access
+ * @key: pointer to a FY_BLAKE3_KEY_LEN area when in keyed mode.
+ * NULL otherwise.
+ * @context: pointer to a context when in key derivation mode.
+ * NULL otherwise.
+ * @context_len: The size of the context when in key derivation mode.
+ * 0 otherwise.
+ * @tp: The thread pool to use, if NULL, create a private one
+ * @num_threads: Number of threads to use
+ * - 0 means default: NUM_CPUS * 3 / 2
+ * - > 0 specific number of threads
+ * - -1 disable threading entirely
+ */
+struct fy_blake3_hasher_cfg {
+ const char *backend;
+ size_t file_buffer;
+ size_t mmap_min_chunk;
+ size_t mmap_max_chunk;
+ bool no_mmap;
+ const uint8_t *key;
+ const void *context;
+ size_t context_len;
+ struct fy_thread_pool *tp;
+ int num_threads;
+};
+
+/**
+ * fy_blake3_hasher_create() - Create a BLAKE3 hasher object.
+ *
+ * Creates a BLAKE3 hasher with its configuration @cfg
+ * The hasher may be destroyed by a corresponding call to
+ * fy_blake3_hasher_destroy().
+ *
+ * @cfg: The configuration for the BLAKE3 hasher
+ *
+ * Returns:
+ * A pointer to the BLAKE3 hasher or NULL in case of an error.
+ */
+struct fy_blake3_hasher *
+fy_blake3_hasher_create(const struct fy_blake3_hasher_cfg *cfg)
+ FY_EXPORT;
+
+/**
+ * fy_blake3_hasher_destroy() - Destroy the given BLAKE3 hasher
+ *
+ * Destroy a BLAKE3 hasher created earlier via fy_blake3_hasher_create().
+ *
+ * @fyh: The BLAKE3 hasher to destroy
+ */
+void
+fy_blake3_hasher_destroy(struct fy_blake3_hasher *fyh)
+ FY_EXPORT;
+
+/**
+ * fy_blake3_hasher_update() - Update the BLAKE3 hasher state with the given input
+ *
+ * Updates the BLAKE3 hasher state by hashing the given input.
+ *
+ * @fyh: The BLAKE3 hasher
+ * @input: Pointer to the input
+ * @input_len: Size of the input in bytes
+ */
+void
+fy_blake3_hasher_update(struct fy_blake3_hasher *fyh, const void *input, size_t input_len)
+ FY_EXPORT;
+
+/**
+ * fy_blake3_hasher_finalize() - Finalize the hash and get output
+ *
+ * Finalizes the BLAKE3 hasher and returns the output
+ *
+ * @fyh: The BLAKE3 hasher
+ *
+ * Returns:
+ * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL
+ * in case of an error.
+ */
+const uint8_t *
+fy_blake3_hasher_finalize(struct fy_blake3_hasher *fyh)
+ FY_EXPORT;
+
+/**
+ * fy_blake3_hasher_reset() - Resets the hasher
+ *
+ * Resets the hasher for re-use
+ *
+ * @fyh: The BLAKE3 hasher
+ */
+void
+fy_blake3_hasher_reset(struct fy_blake3_hasher *fyh)
+ FY_EXPORT;
+
+/**
+ * fy_blake3_hash() - BLAKE3 hash a memory area
+ *
+ * Hash a memory area and return the BLAKE3 output.
+ *
+ * @fyh: The BLAKE3 hasher
+ * @mem: Pointer to the memory to use
+ * @size: The size of the memory in bytes
+ *
+ * Returns:
+ * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL
+ * in case of an error.
+ */
+const uint8_t *
+fy_blake3_hash(struct fy_blake3_hasher *fyh, const void *mem, size_t size)
+ FY_EXPORT;
+
+/**
+ * fy_blake3_hash_file() - BLAKE3 hash a file.
+ *
+ * Hash the given file (possibly using mmap)
+ *
+ * @fyh: The BLAKE3 hasher
+ * @filename: The filename
+ *
+ * Returns:
+ * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL
+ * in case of an error.
+ */
+const uint8_t *
+fy_blake3_hash_file(struct fy_blake3_hasher *fyh, const char *filename)
+ FY_EXPORT;
+
#ifdef __cplusplus
}
#endif
Name: libfyaml
Description: Fancy YAML 1.3 parser library
Version: @PACKAGE_VERSION@
-Libs: -L${libdir} @ASAN_LIBS@ -lfyaml @PTHREAD_LIBS@
+Libs: -L${libdir} @ASAN_LIBS@ @PTHREAD_LIBS@ -lfyaml
Cflags: -I${includedir} @ASAN_CFLAGS@ @PTHREAD_CFLAGS@
+bin_PROGRAMS =
+noinst_PROGRAMS =
+noinst_LIBRARIES =
+lib_LTLIBRARIES =
+noinst_LTLIBRARIES =
+
AM_CPPFLAGS = \
-I$(top_srcdir)/src \
-I$(top_srcdir)/include
AM_CFLAGS =
-lib_LTLIBRARIES = libfyaml.la
+lib_LTLIBRARIES += libfyaml.la
libfyaml_la_SOURCES = \
lib/fy-parse.c lib/fy-parse.h \
- lib/fy-utf8.c lib/fy-utf8.h \
lib/fy-types.c lib/fy-types.h \
- lib/fy-list.h \
- lib/fy-typelist.h \
lib/fy-diag.c lib/fy-diag.h \
lib/fy-dump.c lib/fy-dump.h \
lib/fy-atom.c lib/fy-atom.h \
- lib/fy-ctype.c lib/fy-ctype.h \
lib/fy-token.c lib/fy-token.h \
lib/fy-input.c lib/fy-input.h \
lib/fy-docstate.c lib/fy-docstate.h \
lib/fy-doc.c lib/fy-doc.h \
lib/fy-docbuilder.c lib/fy-docbuilder.h \
lib/fy-emit.c lib/fy-emit.h lib/fy-emit-accum.h \
- lib/fy-utils.c lib/fy-utils.h \
lib/fy-event.h lib/fy-event.c \
- xxhash/xxhash.c xxhash/xxhash.h \
lib/fy-accel.c lib/fy-accel.h \
lib/fy-walk.c lib/fy-walk.h \
lib/fy-path.c lib/fy-path.h \
- lib/fy-composer.c lib/fy-composer.h
+ lib/fy-composer.c lib/fy-composer.h \
+ xxhash/xxhash.c xxhash/xxhash.h \
+ util/fy-list.h \
+ util/fy-typelist.h \
+ util/fy-ctype.c util/fy-ctype.h \
+ util/fy-utf8.c util/fy-utf8.h \
+ util/fy-utils.c util/fy-utils.h \
+ util/fy-endian.h \
+ util/fy-blob.c util/fy-blob.h \
+ util/fy-id.h \
+ util/fy-align.h \
+ util/fy-bit64.h \
+ thread/fy-thread.h thread/fy-thread.c
libfyaml_la_CPPFLAGS = $(AM_CPPFLAGS) \
- -I$(top_srcdir)/src/lib \
- -I$(top_srcdir)/src/xxhash
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/xxhash \
+ -I$(top_srcdir)/src/util \
+ -I$(top_srcdir)/src/thread
libfyaml_la_CFLAGS = $(AM_CFLAGS)
-libfyaml_la_LDFLAGS = $(AM_LDFLAGS) $(AM_LIBLDFLAGS) \
+libfyaml_la_LDFLAGS = -no-undefined $(AM_LDFLAGS) $(AM_LIBLDFLAGS) \
-version $(LIBTOOL_VERSION)
+libfyaml_la_LIBADD =
-bin_PROGRAMS =
-noinst_PROGRAMS =
+# blake3 section
+libfyaml_la_CPPFLAGS += -I$(top_srcdir)/src/blake3
+libfyaml_la_SOURCES += blake3/blake3.h \
+ blake3/blake3_impl.h \
+ blake3/blake3_internal.h \
+ blake3/blake3_host_state.c \
+ blake3/blake3_backend.c \
+ blake3/blake3_be_cpusimd.c \
+ blake3/fy-blake3.c
+
+LIBB3_COMMON_CPPFLAGS=$(AM_CPPFLAGS) -I$(top_srcdir)/src/util -I$(top_srcdir)/src/thread
+
+# the portable implementation
+noinst_LTLIBRARIES += libb3portable.la
+libb3portable_la_SOURCES = blake3/blake3_portable.c blake3/blake3.c
+libb3portable_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -DHASHER_SUFFIX=portable -DSIMD_DEGREE=1
+libb3portable_la_CFLAGS = $(AM_CPPFLAGS)
+libfyaml_la_LIBADD += libb3portable.la
+
+# note that each target lib has blake3 compiled again but with a suffix
+if TARGET_HAS_SSE2
+noinst_LTLIBRARIES += libb3sse2.la
+libb3sse2_la_SOURCES = blake3/blake3_sse2.c blake3/blake3_sse2_x86-64_unix.S blake3/blake3.c
+libb3sse2_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -msse2 -DHASHER_SUFFIX=sse2 -DSIMD_DEGREE=4
+libb3sse2_la_CFLAGS = $(AM_CPPFLAGS) -msse2
+libfyaml_la_LIBADD += libb3sse2.la
+endif
+
+if TARGET_HAS_SSE41
+noinst_LTLIBRARIES += libb3sse41.la
+libb3sse41_la_SOURCES = blake3/blake3_sse41.c blake3/blake3_sse41_x86-64_unix.S blake3/blake3.c
+libb3sse41_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -msse4.1 -DHASHER_SUFFIX=sse41 -DSIMD_DEGREE=4
+libb3sse41_la_CFLAGS = $(AM_CPPFLAGS) -msse4.1
+libfyaml_la_LIBADD += libb3sse41.la
+endif
+
+if TARGET_HAS_AVX2
+noinst_LTLIBRARIES += libb3avx2.la
+libb3avx2_la_SOURCES = blake3/blake3_avx2.c blake3/blake3_avx2_x86-64_unix.S blake3/blake3.c
+libb3avx2_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -mavx2 -DHASHER_SUFFIX=avx2 -DSIMD_DEGREE=8
+libb3avx2_la_CFLAGS = $(AM_CPPFLAGS) -mavx2
+libfyaml_la_LIBADD += libb3avx2.la
+endif
+
+if TARGET_HAS_AVX512
+noinst_LTLIBRARIES += libb3avx512.la
+libb3avx512_la_SOURCES = blake3/blake3_avx512.c blake3/blake3_avx512_x86-64_unix.S blake3/blake3.c
+libb3avx512_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -mavx512f -mavx512vl -DHASHER_SUFFIX=avx512 -DSIMD_DEGREE=16
+libb3avx512_la_CFLAGS = $(AM_CPPFLAGS) -mavx512f -mavx512vl
+libfyaml_la_LIBADD += libb3avx512.la
+endif
+
+if TARGET_HAS_NEON
+noinst_LTLIBRARIES += libb3neon.la
+libb3neon_la_SOURCES = blake3/blake3_neon.c blake3/blake3.c
+libb3neon_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -DHASHER_SUFFIX=neon -DSIMD_DEGREE=4
+libb3neon_la_CFLAGS = $(AM_CPPFLAGS)
+if TARGET_CPU_ARM
+libb3neon_la_CPPFLAGS += -mfpu=neon
+libb3neon_la_CFLAGS += -mfpu=neon
+endif
+libfyaml_la_LIBADD += libb3neon.la
+endif
# libfyaml-parser needs both LIBYAML and static
if HAVE_LIBYAML
libfyaml_parser_CPPFLAGS = $(AM_CPPFLAGS) \
-I$(top_srcdir)/src/valgrind \
-I$(top_srcdir)/src/lib \
- -I$(top_srcdir)/src/xxhash
+ -I$(top_srcdir)/src/xxhash \
+ -I$(top_srcdir)/src/util \
+ -I$(top_srcdir)/src/thread
libfyaml_parser_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la
libfyaml_parser_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS)
endif
endif
+# fy-thread
+if HAVE_STATIC
+
+noinst_PROGRAMS += fy-thread
+
+fy_thread_SOURCES = \
+ internal/fy-thread.c \
+ valgrind/fy-valgrind.h
+
+fy_thread_CPPFLAGS = $(AM_CPPFLAGS) \
+ -I$(top_srcdir)/src/valgrind \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/xxhash \
+ -I$(top_srcdir)/src/util \
+ -I$(top_srcdir)/src/thread
+fy_thread_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la
+fy_thread_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS)
+
+fy_thread_LDFLAGS = $(AM_LDFLAGS) -static
+endif
+
+# fy-b3sum
+if HAVE_STATIC
+
+noinst_PROGRAMS += fy-b3sum
+
+fy_b3sum_SOURCES = \
+ internal/fy-b3sum.c \
+ valgrind/fy-valgrind.h
+
+fy_b3sum_CPPFLAGS = $(AM_CPPFLAGS) \
+ -I$(top_srcdir)/src/valgrind \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/xxhash \
+ -I$(top_srcdir)/src/util \
+ -I$(top_srcdir)/src/thread \
+ -I$(top_srcdir)/src/blake3
+fy_b3sum_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la
+fy_b3sum_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS)
+
+fy_b3sum_LDFLAGS = $(AM_LDFLAGS) -static
+endif
+
bin_PROGRAMS += fy-tool
fy_tool_SOURCES = \
--- /dev/null
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <assert.h>
+#include <stdbool.h>
+#include <string.h>
+#include <stdio.h>
+#include <alloca.h>
+#include <pthread.h>
+#include <stdlib.h>
+
+#include "blake3.h"
+#include "blake3_impl.h"
+#include "blake3_internal.h"
+
+#include "fy-thread.h"
+
+// GCC gets confused sometimes
+#ifdef GCC_DISABLE_WSTRING_OP_OVERREAD
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wstringop-overread"
+#endif
+
+#undef HASHER_OP
+#ifdef HASHER_SUFFIX
+// two levels of indirection required
+#define __HASHER_OP(_name, _sfx) _name ##_ ## _sfx
+#define _HASHER_OP(_name, _sfx) __HASHER_OP(_name, _sfx)
+#define HASHER_OP(_name) _HASHER_OP(_name, HASHER_SUFFIX)
+#else
+#define HASHER_OP(_name) _name
+#endif
+
+#ifndef SIMD_DEGREE
+#define SIMD_DEGREE (hs->simd_degree)
+#endif
+
+#ifndef SIMD_DEGREE_OR_2
+#define SIMD_DEGREE_OR_2 (SIMD_DEGREE >= 2 ? SIMD_DEGREE : 2)
+#endif
+
+// keep the same naming
+#define MAX_SIMD_DEGREE SIMD_DEGREE
+#define MAX_SIMD_DEGREE_OR_2 SIMD_DEGREE_OR_2
+#define blake3_simd_degree() SIMD_DEGREE
+
+// rename to keep the same
+#define blake3_compress_in_place hasher->hs->compress_in_place
+#define blake3_compress_xof hasher->hs->compress_xof
+#define blake3_hash_many self->hs->hash_many
+
+INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8],
+ uint8_t flags) {
+ memcpy(self->cv, key, BLAKE3_KEY_LEN);
+ self->chunk_counter = 0;
+ self->buf_len = 0;
+ self->blocks_compressed = 0;
+ self->flags = flags;
+}
+
+INLINE void chunk_state_reset(blake3_chunk_state *self, const uint32_t key[8],
+ uint64_t chunk_counter) {
+ memcpy(self->cv, key, BLAKE3_KEY_LEN);
+ self->chunk_counter = chunk_counter;
+ self->blocks_compressed = 0;
+ self->buf_len = 0;
+}
+
+INLINE size_t chunk_state_len(const blake3_chunk_state *self) {
+ return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) +
+ ((size_t)self->buf_len);
+}
+
+INLINE size_t chunk_state_fill_buf(blake3_chunk_state *self,
+ const uint8_t *input, size_t input_len) {
+ size_t take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len);
+ if (take > input_len) {
+ take = input_len;
+ }
+ uint8_t *dest = self->buf + ((size_t)self->buf_len);
+ memcpy(dest, input, take);
+ self->buf_len += (uint8_t)take;
+ return take;
+}
+
+INLINE uint8_t chunk_state_maybe_start_flag(const blake3_chunk_state *self) {
+ if (self->blocks_compressed == 0) {
+ return CHUNK_START;
+ } else {
+ return 0;
+ }
+}
+
+typedef struct {
+ uint32_t input_cv[8] BLAKE3_ALIGN;
+ uint8_t block[BLAKE3_BLOCK_LEN] BLAKE3_ALIGN;
+ uint64_t counter;
+ uint8_t block_len;
+ uint8_t flags;
+} output_t;
+
+INLINE output_t make_output(const uint32_t input_cv[BLAKE3_OUT_WORDS],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags) {
+ output_t ret;
+ memcpy(ret.input_cv, input_cv, BLAKE3_OUT_LEN);
+ memcpy(ret.block, block, block_len);
+ ret.block_len = block_len;
+ ret.counter = counter;
+ ret.flags = flags;
+ return ret;
+}
+
+// Chaining values within a given chunk (specifically the compress_in_place
+// interface) are represented as words. This avoids unnecessary bytes<->words
+// conversion overhead in the portable implementation. However, the hash_many
+// interface handles both user input and parent node blocks, so it accepts
+// bytes. For that reason, chaining values in the CV stack are represented as
+// bytes.
+INLINE void output_chaining_value(const blake3_hasher *hasher, output_t *self, uint8_t cv[BLAKE3_OUT_LEN]) {
+ uint32_t cv_words[BLAKE3_OUT_WORDS] BLAKE3_ALIGN;
+ memcpy(cv_words, self->input_cv, BLAKE3_OUT_LEN);
+ memset(self->block + self->block_len, 0, BLAKE3_BLOCK_LEN - self->block_len);
+ blake3_compress_in_place(cv_words, self->block, self->block_len,
+ self->counter, self->flags);
+ store_cv_words(cv, cv_words);
+}
+
+INLINE void output_root_bytes(const blake3_hasher *hasher, output_t *self, uint64_t seek, uint8_t *out,
+ size_t out_len) {
+ uint64_t output_block_counter = seek / 64;
+ size_t offset_within_block = seek % 64;
+ uint8_t wide_buf[64] BLAKE3_ALIGN;
+ while (out_len > 0) {
+ memset(self->block + self->block_len, 0, BLAKE3_BLOCK_LEN - self->block_len);
+ blake3_compress_xof(self->input_cv, self->block, self->block_len,
+ output_block_counter, self->flags | ROOT, wide_buf);
+ size_t available_bytes = 64 - offset_within_block;
+ size_t memcpy_len;
+ if (out_len > available_bytes) {
+ memcpy_len = available_bytes;
+ } else {
+ memcpy_len = out_len;
+ }
+ memcpy(out, wide_buf + offset_within_block, memcpy_len);
+ out += memcpy_len;
+ out_len -= memcpy_len;
+ output_block_counter += 1;
+ offset_within_block = 0;
+ }
+}
+
+INLINE void chunk_state_update(blake3_hasher *hasher, blake3_chunk_state *self, const uint8_t *input,
+ size_t input_len) {
+ if (self->buf_len > 0) {
+ size_t take = chunk_state_fill_buf(self, input, input_len);
+ input += take;
+ input_len -= take;
+ if (input_len > 0) {
+ blake3_compress_in_place(
+ self->cv, self->buf, BLAKE3_BLOCK_LEN, self->chunk_counter,
+ self->flags | chunk_state_maybe_start_flag(self));
+ self->blocks_compressed += 1;
+ self->buf_len = 0;
+ }
+ }
+
+ while (input_len > BLAKE3_BLOCK_LEN) {
+ blake3_compress_in_place(self->cv, input, BLAKE3_BLOCK_LEN,
+ self->chunk_counter,
+ self->flags | chunk_state_maybe_start_flag(self));
+ self->blocks_compressed += 1;
+ input += BLAKE3_BLOCK_LEN;
+ input_len -= BLAKE3_BLOCK_LEN;
+ }
+
+ size_t take = chunk_state_fill_buf(self, input, input_len);
+ input += take;
+ input_len -= take;
+}
+
+INLINE output_t chunk_state_output(const blake3_chunk_state *self) {
+ uint8_t block_flags =
+ self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END;
+ return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter,
+ block_flags);
+}
+
+INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN],
+ const uint32_t key[8], uint8_t flags) {
+ return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT);
+}
+
+// Given some input larger than one chunk, return the number of bytes that
+// should go in the left subtree. This is the largest power-of-2 number of
+// chunks that leaves at least 1 byte for the right subtree.
+INLINE size_t left_len(size_t content_len) {
+ // Subtract 1 to reserve at least one byte for the right side. content_len
+ // should always be greater than BLAKE3_CHUNK_LEN.
+ size_t full_chunks = (content_len - 1) / BLAKE3_CHUNK_LEN;
+ return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN;
+}
+
+// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE chunks at the same time
+// on a single thread. Write out the chunk chaining values and return the
+// number of chunks hashed. These chunks are never the root and never empty;
+// those cases use a different codepath.
+INLINE size_t compress_chunks_parallel(blake3_hasher *self,
+ const uint8_t *input, size_t input_len,
+ const uint32_t key[8],
+ uint64_t chunk_counter, uint8_t flags,
+ uint8_t *out) {
+#if defined(BLAKE3_TESTING)
+ assert(0 < input_len);
+ assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN);
+#endif
+
+ const uint8_t *chunks_array[MAX_SIMD_DEGREE];
+ size_t input_position = 0;
+ size_t chunks_array_len = 0;
+ while (input_len - input_position >= BLAKE3_CHUNK_LEN) {
+ chunks_array[chunks_array_len] = &input[input_position];
+ input_position += BLAKE3_CHUNK_LEN;
+ chunks_array_len += 1;
+ }
+ size_t i = chunks_array_len;
+ while (i < MAX_SIMD_DEGREE)
+ chunks_array[i++] = NULL;
+
+ blake3_hash_many(chunks_array, chunks_array_len,
+ BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN, key, chunk_counter,
+ true, flags, CHUNK_START, CHUNK_END, out);
+
+ // Hash the remaining partial chunk, if there is one. Note that the empty
+ // chunk (meaning the empty message) is a different codepath.
+ if (input_len > input_position) {
+ uint64_t counter = chunk_counter + (uint64_t)chunks_array_len;
+ blake3_chunk_state chunk_state;
+ chunk_state_init(&chunk_state, key, flags);
+ chunk_state.chunk_counter = counter;
+ chunk_state_update(self, &chunk_state, &input[input_position],
+ input_len - input_position);
+ output_t output = chunk_state_output(&chunk_state);
+ output_chaining_value(self, &output, &out[chunks_array_len * BLAKE3_OUT_LEN]);
+ return chunks_array_len + 1;
+ } else {
+ return chunks_array_len;
+ }
+}
+
+// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE parents at the same time
+// on a single thread. Write out the parent chaining values and return the
+// number of parents hashed. (If there's an odd input chaining value left over,
+// return it as an additional output.) These parents are never the root and
+// never empty; those cases use a different codepath.
+INLINE size_t compress_parents_parallel(blake3_hasher *self,
+ const uint8_t *child_chaining_values,
+ size_t num_chaining_values,
+ const uint32_t key[8], uint8_t flags,
+ uint8_t *out) {
+#if defined(BLAKE3_TESTING)
+ assert(2 <= num_chaining_values);
+ assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2);
+#endif
+
+ const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2];
+ size_t parents_array_len = 0;
+ while (num_chaining_values - (2 * parents_array_len) >= 2) {
+ parents_array[parents_array_len] =
+ &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN];
+ parents_array_len += 1;
+ }
+
+ blake3_hash_many(parents_array, parents_array_len, 1, key,
+ 0, // Parents always use counter 0.
+ false, flags | PARENT,
+ 0, // Parents have no start flags.
+ 0, // Parents have no end flags.
+ out);
+
+ // If there's an odd child left over, it becomes an output.
+ if (num_chaining_values > 2 * parents_array_len) {
+ memcpy(&out[parents_array_len * BLAKE3_OUT_LEN],
+ &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN],
+ BLAKE3_OUT_LEN);
+ return parents_array_len + 1;
+ } else {
+ return parents_array_len;
+ }
+}
+
+static size_t blake3_compress_subtree_wide(blake3_hasher *self,
+ const uint8_t *input,
+ size_t input_len,
+ const uint32_t key[8],
+ uint64_t chunk_counter,
+ uint8_t flags, uint8_t *out);
+
+static bool blake3_compress_subtree_wide_work_check(const void *arg) {
+ const blake3_compress_subtree_state *s = arg;
+
+ /* only off-load to thread if we have enough input */
+ return s->input_len >= s->self->hs->mt_degree * BLAKE3_CHUNK_LEN;
+}
+
+static void blake3_compress_subtree_wide_thread(void *arg) {
+ blake3_compress_subtree_state *s = arg;
+
+ s->n = blake3_compress_subtree_wide(s->self,
+ s->input, s->input_len,
+ s->key,
+ s->chunk_counter,
+ s->flags,
+ s->out);
+}
+
+// The wide helper function returns (writes out) an array of chaining values
+// and returns the length of that array. The number of chaining values returned
+// is the dynamically detected SIMD degree, at most MAX_SIMD_DEGREE. Or fewer,
+// if the input is shorter than that many chunks. The reason for maintaining a
+// wide array of chaining values going back up the tree, is to allow the
+// implementation to hash as many parents in parallel as possible.
+//
+// As a special case when the SIMD degree is 1, this function will still return
+// at least 2 outputs. This guarantees that this function doesn't perform the
+// root compression. (If it did, it would use the wrong flags, and also we
+// wouldn't be able to implement extendable output.) Note that this function is
+// not used when the whole input is only 1 chunk long; that's a different
+// codepath.
+//
+// Why not just have the caller split the input on the first update(), instead
+// of implementing this special rule? Because we don't want to limit SIMD or
+// multi-threading parallelism for that update().
+static size_t blake3_compress_subtree_wide(blake3_hasher *self, const uint8_t *input,
+ size_t input_len,
+ const uint32_t key[8],
+ uint64_t chunk_counter,
+ uint8_t flags, uint8_t *out) {
+ // Note that the single chunk case does *not* bump the SIMD degree up to 2
+ // when it is 1. If this implementation adds multi-threading in the future,
+ // this gives us the option of multi-threading even the 2-chunk case, which
+ // can help performance on smaller platforms.
+ if (input_len <= blake3_simd_degree() * BLAKE3_CHUNK_LEN) {
+ return compress_chunks_parallel(self, input, input_len, key, chunk_counter, flags,
+ out);
+ }
+
+ // With more than simd_degree chunks, we need to recurse. Start by dividing
+ // the input into left and right subtrees. (Note that this is only optimal
+ // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree
+ // of 3 or something, we'll need a more complicated strategy.)
+ size_t left_input_len = left_len(input_len);
+ size_t right_input_len = input_len - left_input_len;
+ const uint8_t *right_input = &input[left_input_len];
+ uint64_t right_chunk_counter =
+ chunk_counter + (uint64_t)(left_input_len / BLAKE3_CHUNK_LEN);
+
+ // Make space for the child outputs. Here we use MAX_SIMD_DEGREE_OR_2 to
+ // account for the special case of returning 2 outputs when the SIMD degree
+ // is 1.
+ uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN];
+ size_t degree = blake3_simd_degree();
+ if (left_input_len > BLAKE3_CHUNK_LEN && degree == 1) {
+ // The special case: We always use a degree of at least two, to make
+ // sure there are two outputs. Except, as noted above, at the chunk
+ // level, where we allow degree=1. (Note that the 1-chunk-input case is
+ // a different codepath.)
+ degree = 2;
+ }
+ uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN];
+
+ // Recurse
+ size_t left_n, right_n;
+ if (self->hs->cfg.no_mthread) {
+ left_n = blake3_compress_subtree_wide(self, input, left_input_len, key,
+ chunk_counter, flags, cv_array);
+ right_n = blake3_compress_subtree_wide(self, right_input, right_input_len, key,
+ right_chunk_counter, flags, right_cvs);
+ } else {
+
+ // this is the multi-threaded implementation
+ blake3_compress_subtree_state states[2];
+
+ /* common */
+ states[0].self = states[1].self = self;
+ states[0].key = states[1].key = key;
+ states[0].flags = states[1].flags = flags;
+ states[0].n = states[1].n = 0;
+
+ /* left */
+ states[0].input = input;
+ states[0].input_len = left_input_len;
+ states[0].chunk_counter = chunk_counter;
+ states[0].out = cv_array;
+
+ /* right */
+ states[1].input = right_input;
+ states[1].input_len = right_input_len;
+ states[1].chunk_counter = right_chunk_counter;
+ states[1].out = right_cvs;
+
+ fy_thread_arg_array_join(self->hs->tp,
+ blake3_compress_subtree_wide_thread,
+ blake3_compress_subtree_wide_work_check,
+ states, sizeof(states[0]), 2);
+
+ left_n = states[0].n;
+ right_n = states[1].n;
+ }
+
+ // The special case again. If SIMD_DEGREE=1, then we'll have left_n=1 and
+ // right_n=1. Rather than compressing them into a single output, return
+ // them directly, to make sure we always have at least two outputs.
+ if (left_n == 1) {
+ memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN);
+ return 2;
+ }
+
+ // Otherwise, do one layer of parent node compression.
+ size_t num_chaining_values = left_n + right_n;
+ return compress_parents_parallel(self, cv_array, num_chaining_values, key, flags,
+ out);
+}
+
+// Hash a subtree with compress_subtree_wide(), and then condense the resulting
+// list of chaining values down to a single parent node. Don't compress that
+// last parent node, however. Instead, return its message bytes (the
+// concatenated chaining values of its children). This is necessary when the
+// first call to update() supplies a complete subtree, because the topmost
+// parent node of that subtree could end up being the root. It's also necessary
+// for extended output in the general case.
+//
+// As with compress_subtree_wide(), this function is not used on inputs of 1
+// chunk or less. That's a different codepath.
+INLINE void compress_subtree_to_parent_node(blake3_hasher *self,
+ const uint8_t *input, size_t input_len, const uint32_t key[8],
+ uint64_t chunk_counter, uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN]) {
+#if defined(BLAKE3_TESTING)
+ assert(input_len > BLAKE3_CHUNK_LEN);
+#endif
+
+ uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN];
+ size_t num_cvs = blake3_compress_subtree_wide(self, input, input_len, key,
+ chunk_counter, flags, cv_array);
+ assert(num_cvs <= MAX_SIMD_DEGREE_OR_2);
+
+ // If MAX_SIMD_DEGREE is greater than 2 and there's enough input,
+ // compress_subtree_wide() returns more than 2 chaining values. Condense
+ // them into 2 by forming parent nodes repeatedly.
+ uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2];
+ // The second half of this loop condition is always true, and we just
+ // asserted it above. But GCC can't tell that it's always true, and if NDEBUG
+ // is set on platforms where MAX_SIMD_DEGREE_OR_2 == 2, GCC emits spurious
+ // warnings here. GCC 8.5 is particularly sensitive, so if you're changing
+ // this code, test it against that version.
+ while (num_cvs > 2 && num_cvs <= MAX_SIMD_DEGREE_OR_2) {
+ num_cvs =
+ compress_parents_parallel(self, cv_array, num_cvs, key, flags, out_array);
+ memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN);
+ }
+ memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN);
+}
+
+INLINE void hasher_init_base(blake3_host_state *hs, blake3_hasher *self, const uint32_t key[8],
+ uint8_t flags) {
+ memset(self, 0, sizeof(*self));
+ self->hs = hs;
+ memcpy(self->key, key, BLAKE3_KEY_LEN);
+ chunk_state_init(&self->chunk, key, flags);
+ self->cv_stack_len = 0;
+}
+
+static const uint32_t IV[8] BLAKE3_ALIGN = {
+ B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3,
+ B3_IV_4, B3_IV_5, B3_IV_6, B3_IV_7,
+};
+
+void HASHER_OP(blake3_hasher_init) (blake3_host_state *hs, blake3_hasher *self) { hasher_init_base(hs, self, IV, 0); }
+
+void HASHER_OP(blake3_hasher_init_keyed) (blake3_host_state *hs, blake3_hasher *self,
+ const uint8_t key[BLAKE3_KEY_LEN]) {
+ uint32_t key_words[8] BLAKE3_ALIGN;
+ load_key_words(key, key_words);
+ hasher_init_base(hs, self, key_words, KEYED_HASH);
+}
+
+void HASHER_OP(blake3_hasher_init_derive_key_raw) (blake3_host_state *hs, blake3_hasher *self, const void *context,
+ size_t context_len) {
+ blake3_hasher context_hasher;
+ hasher_init_base(hs, &context_hasher, IV, DERIVE_KEY_CONTEXT);
+ blake3_hasher_update(&context_hasher, context, context_len);
+ uint8_t context_key[BLAKE3_KEY_LEN];
+ blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN);
+ uint32_t context_key_words[8];
+ load_key_words(context_key, context_key_words);
+ hasher_init_base(hs, self, context_key_words, DERIVE_KEY_MATERIAL);
+}
+
+void HASHER_OP(blake3_hasher_init_derive_key) (blake3_host_state *hs, blake3_hasher *self, const char *context) {
+ blake3_hasher_init_derive_key_raw(hs, self, context, strlen(context));
+}
+
+// As described in hasher_push_cv() below, we do "lazy merging", delaying
+// merges until right before the next CV is about to be added. This is
+// different from the reference implementation. Another difference is that we
+// aren't always merging 1 chunk at a time. Instead, each CV might represent
+// any power-of-two number of chunks, as long as the smaller-above-larger stack
+// order is maintained. Instead of the "count the trailing 0-bits" algorithm
+// described in the spec, we use a "count the total number of 1-bits" variant
+// that doesn't require us to retain the subtree size of the CV on top of the
+// stack. The principle is the same: each CV that should remain in the stack is
+// represented by a 1-bit in the total number of chunks (or bytes) so far.
+INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) {
+ size_t post_merge_stack_len = (size_t)popcnt(total_len);
+ while (self->cv_stack_len > post_merge_stack_len) {
+ uint8_t *parent_node =
+ &self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN];
+ output_t output = parent_output(parent_node, self->key, self->chunk.flags);
+ output_chaining_value(self, &output, parent_node);
+ self->cv_stack_len -= 1;
+ }
+}
+
+// In reference_impl.rs, we merge the new CV with existing CVs from the stack
+// before pushing it. We can do that because we know more input is coming, so
+// we know none of the merges are root.
+//
+// This setting is different. We want to feed as much input as possible to
+// compress_subtree_wide(), without setting aside anything for the chunk_state.
+// If the user gives us 64 KiB, we want to parallelize over all 64 KiB at once
+// as a single subtree, if at all possible.
+//
+// This leads to two problems:
+// 1) This 64 KiB input might be the only call that ever gets made to update.
+// In this case, the root node of the 64 KiB subtree would be the root node
+// of the whole tree, and it would need to be ROOT finalized. We can't
+// compress it until we know.
+// 2) This 64 KiB input might complete a larger tree, whose root node is
+// similarly going to be the the root of the whole tree. For example, maybe
+// we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the
+// node at the root of the 256 KiB subtree until we know how to finalize it.
+//
+// The second problem is solved with "lazy merging". That is, when we're about
+// to add a CV to the stack, we don't merge it with anything first, as the
+// reference impl does. Instead we do merges using the *previous* CV that was
+// added, which is sitting on top of the stack, and we put the new CV
+// (unmerged) on top of the stack afterwards. This guarantees that we never
+// merge the root node until finalize().
+//
+// Solving the first problem requires an additional tool,
+// compress_subtree_to_parent_node(). That function always returns the top
+// *two* chaining values of the subtree it's compressing. We then do lazy
+// merging with each of them separately, so that the second CV will always
+// remain unmerged. (That also helps us support extendable output when we're
+// hashing an input all-at-once.)
+INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN],
+ uint64_t chunk_counter) {
+ hasher_merge_cv_stack(self, chunk_counter);
+ memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv,
+ BLAKE3_OUT_LEN);
+ self->cv_stack_len += 1;
+}
+
+void HASHER_OP(blake3_hasher_update) (blake3_hasher *self, const void *input,
+ size_t input_len) {
+ // Explicitly checking for zero avoids causing UB by passing a null pointer
+ // to memcpy. This comes up in practice with things like:
+ // std::vector<uint8_t> v;
+ // blake3_hasher_update(&hasher, v.data(), v.size());
+ if (input_len == 0) {
+ return;
+ }
+
+ const uint8_t *input_bytes = (const uint8_t *)input;
+
+ // If we have some partial chunk bytes in the internal chunk_state, we need
+ // to finish that chunk first.
+ if (chunk_state_len(&self->chunk) > 0) {
+ size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk);
+ if (take > input_len) {
+ take = input_len;
+ }
+ chunk_state_update(self, &self->chunk, input_bytes, take);
+ input_bytes += take;
+ input_len -= take;
+ // If we've filled the current chunk and there's more coming, finalize this
+ // chunk and proceed. In this case we know it's not the root.
+ if (input_len > 0) {
+ output_t output = chunk_state_output(&self->chunk);
+ uint8_t chunk_cv[32] BLAKE3_ALIGN;
+ output_chaining_value(self, &output, chunk_cv);
+ hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter);
+ chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1);
+ } else {
+ return;
+ }
+ }
+
+ // Now the chunk_state is clear, and we have more input. If there's more than
+ // a single chunk (so, definitely not the root chunk), hash the largest whole
+ // subtree we can, with the full benefits of SIMD (and maybe in the future,
+ // multi-threading) parallelism. Two restrictions:
+ // - The subtree has to be a power-of-2 number of chunks. Only subtrees along
+ // the right edge can be incomplete, and we don't know where the right edge
+ // is going to be until we get to finalize().
+ // - The subtree must evenly divide the total number of chunks up until this
+ // point (if total is not 0). If the current incomplete subtree is only
+ // waiting for 1 more chunk, we can't hash a subtree of 4 chunks. We have
+ // to complete the current subtree first.
+ // Because we might need to break up the input to form powers of 2, or to
+ // evenly divide what we already have, this part runs in a loop.
+ while (input_len > BLAKE3_CHUNK_LEN) {
+ size_t subtree_len = round_down_to_power_of_2(input_len);
+ uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN;
+ // Shrink the subtree_len until it evenly divides the count so far. We know
+ // that subtree_len itself is a power of 2, so we can use a bitmasking
+ // trick instead of an actual remainder operation. (Note that if the caller
+ // consistently passes power-of-2 inputs of the same size, as is hopefully
+ // typical, this loop condition will always fail, and subtree_len will
+ // always be the full length of the input.)
+ //
+ // An aside: We don't have to shrink subtree_len quite this much. For
+ // example, if count_so_far is 1, we could pass 2 chunks to
+ // compress_subtree_to_parent_node. Since we'll get 2 CVs back, we'll still
+ // get the right answer in the end, and we might get to use 2-way SIMD
+ // parallelism. The problem with this optimization, is that it gets us
+ // stuck always hashing 2 chunks. The total number of chunks will remain
+ // odd, and we'll never graduate to higher degrees of parallelism. See
+ // https://github.com/BLAKE3-team/BLAKE3/issues/69.
+ while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) {
+ subtree_len /= 2;
+ }
+ // The shrunken subtree_len might now be 1 chunk long. If so, hash that one
+ // chunk by itself. Otherwise, compress the subtree into a pair of CVs.
+ uint64_t subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN;
+ if (subtree_len <= BLAKE3_CHUNK_LEN) {
+ blake3_chunk_state chunk_state;
+ chunk_state_init(&chunk_state, self->key, self->chunk.flags);
+ chunk_state.chunk_counter = self->chunk.chunk_counter;
+ chunk_state_update(self, &chunk_state, input_bytes, subtree_len);
+ output_t output = chunk_state_output(&chunk_state);
+ uint8_t cv[BLAKE3_OUT_LEN] BLAKE3_ALIGN;
+ output_chaining_value(self, &output, cv);
+ hasher_push_cv(self, cv, chunk_state.chunk_counter);
+ } else {
+ // This is the high-performance happy path, though getting here depends
+ // on the caller giving us a long enough input.
+ uint8_t cv_pair[2 * BLAKE3_OUT_LEN] BLAKE3_ALIGN;
+ compress_subtree_to_parent_node(self, input_bytes, subtree_len, self->key,
+ self->chunk.chunk_counter,
+ self->chunk.flags, cv_pair);
+ hasher_push_cv(self, cv_pair, self->chunk.chunk_counter);
+ hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN],
+ self->chunk.chunk_counter + (subtree_chunks / 2));
+ }
+ self->chunk.chunk_counter += subtree_chunks;
+ input_bytes += subtree_len;
+ input_len -= subtree_len;
+ }
+
+ // If there's any remaining input less than a full chunk, add it to the chunk
+ // state. In that case, also do a final merge loop to make sure the subtree
+ // stack doesn't contain any unmerged pairs. The remaining input means we
+ // know these merges are non-root. This merge loop isn't strictly necessary
+ // here, because hasher_push_chunk_cv already does its own merge loop, but it
+ // simplifies blake3_hasher_finalize below.
+ if (input_len > 0) {
+ chunk_state_update(self, &self->chunk, input_bytes, input_len);
+ hasher_merge_cv_stack(self, self->chunk.chunk_counter);
+ }
+}
+
+void HASHER_OP(blake3_hasher_finalize_seek) (const blake3_hasher *self, uint64_t seek,
+ uint8_t *out, size_t out_len) {
+ // Explicitly checking for zero avoids causing UB by passing a null pointer
+ // to memcpy. This comes up in practice with things like:
+ // std::vector<uint8_t> v;
+ // blake3_hasher_finalize(&hasher, v.data(), v.size());
+ if (out_len == 0) {
+ return;
+ }
+
+ // If the subtree stack is empty, then the current chunk is the root.
+ if (self->cv_stack_len == 0) {
+ output_t output = chunk_state_output(&self->chunk);
+ output_root_bytes(self, &output, seek, out, out_len);
+ return;
+ }
+ // If there are any bytes in the chunk state, finalize that chunk and do a
+ // roll-up merge between that chunk hash and every subtree in the stack. In
+ // this case, the extra merge loop at the end of blake3_hasher_update
+ // guarantees that none of the subtrees in the stack need to be merged with
+ // each other first. Otherwise, if there are no bytes in the chunk state,
+ // then the top of the stack is a chunk hash, and we start the merge from
+ // that.
+ output_t output;
+ size_t cvs_remaining;
+ if (chunk_state_len(&self->chunk) > 0) {
+ cvs_remaining = self->cv_stack_len;
+ output = chunk_state_output(&self->chunk);
+ } else {
+ // There are always at least 2 CVs in the stack in this case.
+ cvs_remaining = self->cv_stack_len - 2;
+ output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key,
+ self->chunk.flags);
+ }
+ while (cvs_remaining > 0) {
+ cvs_remaining -= 1;
+ uint8_t parent_block[BLAKE3_BLOCK_LEN] BLAKE3_ALIGN;
+ memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32);
+ output_chaining_value(self, &output, &parent_block[32]);
+ output = parent_output(parent_block, self->key, self->chunk.flags);
+ }
+ output_root_bytes(self, &output, seek, out, out_len);
+}
+
+void HASHER_OP(blake3_hasher_finalize) (const blake3_hasher *self, uint8_t *out,
+ size_t out_len) {
+ HASHER_OP(blake3_hasher_finalize_seek) (self, 0, out, out_len);
+}
+
+void HASHER_OP(blake3_hasher_reset) (blake3_hasher *self) {
+ chunk_state_reset(&self->chunk, self->key, 0);
+ self->cv_stack_len = 0;
+}
+#ifdef GCC_DISABLE_WSTRING_OP_OVERREAD
+#pragma GCC diagnostic pop
+#endif
+
+const blake3_hasher_ops HASHER_OP(blake3_hasher_op) = {
+ .hasher_init = HASHER_OP(blake3_hasher_init),
+ .hasher_init_keyed = HASHER_OP(blake3_hasher_init_keyed),
+ .hasher_init_derive_key = HASHER_OP(blake3_hasher_init_derive_key),
+ .hasher_init_derive_key_raw = HASHER_OP(blake3_hasher_init_derive_key_raw),
+ .hasher_update = HASHER_OP(blake3_hasher_update),
+ .hasher_finalize = HASHER_OP(blake3_hasher_finalize),
+ .hasher_finalize_seek = HASHER_OP(blake3_hasher_finalize_seek),
+ .hasher_reset = HASHER_OP(blake3_hasher_reset),
+};
--- /dev/null
+#ifndef BLAKE3_H
+#define BLAKE3_H
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+#include <pthread.h>
+#include <stdalign.h>
+
+#define BLAKE3_VERSION_STRING "1.4.1"
+
+#define BLAKE3_KEY_LEN 32
+#define BLAKE3_KEY_WORDS (BLAKE3_KEY_LEN / 4) // words are 4 bytes
+#define BLAKE3_OUT_LEN 32
+#define BLAKE3_OUT_WORDS (BLAKE3_OUT_LEN / 4) // words are 4 bytes
+#define BLAKE3_BLOCK_LEN 64
+#define BLAKE3_CHUNK_LEN 1024
+
+struct blake_host_state;
+struct blake_hasher;
+
+const char *blake3_version(void);
+
+/* NOTE: maximum supported backends is 64
+ * and backends are ordered by reverse order
+ * of preference.
+ *
+ * PORTABLE is always available and is the fall-back.
+ */
+
+typedef enum {
+
+ /* just the portable implementation */
+ B3BID_PORTABLE,
+
+ /* x86 */
+ B3BID_SSE2,
+ B3BID_SSE2_ASM,
+ B3BID_SSE41,
+ B3BID_SSE41_ASM,
+ B3BID_AVX2,
+ B3BID_AVX2_ASM,
+ B3BID_AVX512,
+ B3BID_AVX512_ASM,
+
+ /* arm */
+ B3BID_NEON,
+
+ /* gpu */
+ B3BID_VULKAN,
+ B3BID_OPENCL,
+ B3BID_CUDA,
+
+ /* CPU simd (experimental) */
+ B3BID_CPUSIMD,
+
+} blake3_backend_id;
+
+#define B3BID_COUNT (B3BID_CPUSIMD+1)
+#define B3BID_INVALID ((blake3_backend_id)-1)
+
+#define B3BF_BIT(_x) ((uint64_t)1 << (_x))
+
+#define B3BF_PORTABLE B3BF_BIT(B3BID_PORTABLE)
+
+#define B3BF_SSE2 B3BF_BIT(B3BID_SSE2)
+#define B3BF_SSE2_ASM B3BF_BIT(B3BID_SSE2_ASM)
+#define B3BF_SSE41 B3BF_BIT(B3BID_SSE41)
+#define B3BF_SSE41_ASM B3BF_BIT(B3BID_SSE41_ASM)
+#define B3BF_AVX2 B3BF_BIT(B3BID_AVX2)
+#define B3BF_AVX2_ASM B3BF_BIT(B3BID_AVX2_ASM)
+#define B3BF_AVX512 B3BF_BIT(B3BID_AVX512)
+#define B3BF_AVX512_ASM B3BF_BIT(B3BID_AVX512_ASM)
+
+#define B3BF_NEON B3BF_BIT(B3BID_NEON)
+
+#define B3BF_VULKAN B3BF_BIT(B3BID_VULKAN)
+#define B3BF_OPENCL B3BF_BIT(B3BID_OPENCL)
+#define B3BF_CUDA B3BF_BIT(B3BID_CUDA)
+
+#define B3BF_CPUSIMD B3BF_BIT(B3BID_CPUSIMD)
+
+uint64_t blake3_get_supported_backends(void);
+uint64_t blake3_get_detected_backends(void);
+uint64_t blake3_get_selectable_backends(void);
+
+typedef enum {
+ B3FID_HASH_MANY,
+ B3FID_COMPRESS_XOF,
+ B3FID_COMPRESS_IN_PLACE,
+} blake3_func_id;
+
+#define B3FID_COUNT (B3FID_COMPRESS_IN_PLACE+1)
+#define B3FID_INVALID ((blake3_func_id)-1)
+
+#define B3FF_BIT(_x) ((uint64_t)1 << (_x))
+
+#define B3FF_HASH_MANY B3FF_BIT(B3FID_HASH_MANY)
+#define B3FF_COMPRESS_XOF B3FF_BIT(B3FID_COMPRESS_XOF)
+#define B3FF_COMPRESS_IN_PLACE B3FF_BIT(B3FID_COMPRESS_IN_PLACE)
+
+typedef struct blake3_backend_info {
+ blake3_backend_id id;
+ const char *name;
+ const char *description;
+ unsigned int simd_degree;
+ uint64_t funcs;
+} blake3_backend_info;
+
+const blake3_backend_info *blake3_get_backend_info(blake3_backend_id id);
+
+typedef struct blake3_host_config {
+ bool debug;
+ bool no_mthread;
+ bool no_mmap;
+ unsigned int num_threads;
+ unsigned int mt_degree; // number of chunks to be worth spinning up a thread
+ const char *backend; // backend selection string, or NULL for default
+ size_t file_io_bufsz; // buffer size when doing file I/O
+ size_t mmap_min_chunk; // minimum chunk size for mmap
+ size_t mmap_max_chunk; // maximum chunk size for mmap
+ struct fy_thread_pool *tp; // use this thread pool instead of spinning one up
+} blake3_host_config;
+
+struct blake3_host_state *blake3_host_state_create(const blake3_host_config *cfg);
+void blake3_host_state_destroy(struct blake3_host_state *hs);
+
+struct blake3_hasher *blake3_hasher_create(struct blake3_host_state *hs, const uint8_t *key, const void *context, size_t context_len);
+void blake3_hasher_destroy(struct blake3_hasher *self);
+
+/* simple optimized method for file hashing */
+int blake3_hash_file(struct blake3_hasher *hasher, const char *filename, uint8_t output[BLAKE3_OUT_LEN]);
+void blake3_hash(struct blake3_hasher *hasher, const void *mem, size_t size, uint8_t output[BLAKE3_OUT_LEN]);
+
+/* experimental CPUSIMD enable */
+int blake3_backend_cpusimd_setup(unsigned int num_cpus, unsigned int mult_fact);
+void blake3_backend_cpusimd_cleanup(void);
+
+void blake3_hasher_init(struct blake3_host_state *hs, struct blake3_hasher *self);
+void blake3_hasher_init_keyed(struct blake3_host_state *hs, struct blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]);
+void blake3_hasher_init_derive_key(struct blake3_host_state *hs, struct blake3_hasher *self, const char *context);
+void blake3_hasher_init_derive_key_raw(struct blake3_host_state *hs, struct blake3_hasher *self, const void *context, size_t context_len);
+
+void blake3_hasher_update(struct blake3_hasher *self, const void *input, size_t input_len);
+void blake3_hasher_finalize(const struct blake3_hasher *self, uint8_t *out, size_t out_len);
+void blake3_hasher_finalize_seek(const struct blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len);
+void blake3_hasher_reset(struct blake3_hasher *self);
+
+typedef struct blake3_hasher_ops {
+ void (*hasher_init)(struct blake3_host_state *hs, struct blake3_hasher *self);
+ void (*hasher_init_keyed)(struct blake3_host_state *hs, struct blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]);
+ void (*hasher_init_derive_key)(struct blake3_host_state *hs, struct blake3_hasher *self, const char *context);
+ void (*hasher_init_derive_key_raw)(struct blake3_host_state *hs, struct blake3_hasher *self, const void *context, size_t context_len);
+
+ void (*hasher_update)(struct blake3_hasher *self, const void *input, size_t input_len);
+ void (*hasher_finalize)(const struct blake3_hasher *self, uint8_t *out, size_t out_len);
+ void (*hasher_finalize_seek)(const struct blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len);
+ void (*hasher_reset)(struct blake3_hasher *self);
+} blake3_hasher_ops;
+
+#endif /* BLAKE3_H */
--- /dev/null
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "blake3_impl.h"
+
+#include <immintrin.h>
+
+/* define it here instead of the header */
+static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF };
+
+#define DEGREE 8
+
+INLINE __m256i loadu(const uint8_t src[32]) {
+ return _mm256_loadu_si256((const __m256i *)src);
+}
+
+INLINE void storeu(__m256i src, uint8_t dest[16]) {
+ _mm256_storeu_si256((__m256i *)dest, src);
+}
+
+INLINE __m256i addv(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); }
+
+// Note that clang-format doesn't like the name "xor" for some reason.
+INLINE __m256i xorv(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); }
+
+INLINE __m256i set1(uint32_t x) { return _mm256_set1_epi32((int32_t)x); }
+
+INLINE __m256i rot16(__m256i x) {
+ return _mm256_shuffle_epi8(
+ x, _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2,
+ 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2));
+}
+
+INLINE __m256i rot12(__m256i x) {
+ return _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 32 - 12));
+}
+
+INLINE __m256i rot8(__m256i x) {
+ return _mm256_shuffle_epi8(
+ x, _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1,
+ 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1));
+}
+
+INLINE __m256i rot7(__m256i x) {
+ return _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 32 - 7));
+}
+
+INLINE void round_fn(__m256i v[16], __m256i m[16], size_t r) {
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
+ v[0] = addv(v[0], v[4]);
+ v[1] = addv(v[1], v[5]);
+ v[2] = addv(v[2], v[6]);
+ v[3] = addv(v[3], v[7]);
+ v[12] = xorv(v[12], v[0]);
+ v[13] = xorv(v[13], v[1]);
+ v[14] = xorv(v[14], v[2]);
+ v[15] = xorv(v[15], v[3]);
+ v[12] = rot16(v[12]);
+ v[13] = rot16(v[13]);
+ v[14] = rot16(v[14]);
+ v[15] = rot16(v[15]);
+ v[8] = addv(v[8], v[12]);
+ v[9] = addv(v[9], v[13]);
+ v[10] = addv(v[10], v[14]);
+ v[11] = addv(v[11], v[15]);
+ v[4] = xorv(v[4], v[8]);
+ v[5] = xorv(v[5], v[9]);
+ v[6] = xorv(v[6], v[10]);
+ v[7] = xorv(v[7], v[11]);
+ v[4] = rot12(v[4]);
+ v[5] = rot12(v[5]);
+ v[6] = rot12(v[6]);
+ v[7] = rot12(v[7]);
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
+ v[0] = addv(v[0], v[4]);
+ v[1] = addv(v[1], v[5]);
+ v[2] = addv(v[2], v[6]);
+ v[3] = addv(v[3], v[7]);
+ v[12] = xorv(v[12], v[0]);
+ v[13] = xorv(v[13], v[1]);
+ v[14] = xorv(v[14], v[2]);
+ v[15] = xorv(v[15], v[3]);
+ v[12] = rot8(v[12]);
+ v[13] = rot8(v[13]);
+ v[14] = rot8(v[14]);
+ v[15] = rot8(v[15]);
+ v[8] = addv(v[8], v[12]);
+ v[9] = addv(v[9], v[13]);
+ v[10] = addv(v[10], v[14]);
+ v[11] = addv(v[11], v[15]);
+ v[4] = xorv(v[4], v[8]);
+ v[5] = xorv(v[5], v[9]);
+ v[6] = xorv(v[6], v[10]);
+ v[7] = xorv(v[7], v[11]);
+ v[4] = rot7(v[4]);
+ v[5] = rot7(v[5]);
+ v[6] = rot7(v[6]);
+ v[7] = rot7(v[7]);
+
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
+ v[0] = addv(v[0], v[5]);
+ v[1] = addv(v[1], v[6]);
+ v[2] = addv(v[2], v[7]);
+ v[3] = addv(v[3], v[4]);
+ v[15] = xorv(v[15], v[0]);
+ v[12] = xorv(v[12], v[1]);
+ v[13] = xorv(v[13], v[2]);
+ v[14] = xorv(v[14], v[3]);
+ v[15] = rot16(v[15]);
+ v[12] = rot16(v[12]);
+ v[13] = rot16(v[13]);
+ v[14] = rot16(v[14]);
+ v[10] = addv(v[10], v[15]);
+ v[11] = addv(v[11], v[12]);
+ v[8] = addv(v[8], v[13]);
+ v[9] = addv(v[9], v[14]);
+ v[5] = xorv(v[5], v[10]);
+ v[6] = xorv(v[6], v[11]);
+ v[7] = xorv(v[7], v[8]);
+ v[4] = xorv(v[4], v[9]);
+ v[5] = rot12(v[5]);
+ v[6] = rot12(v[6]);
+ v[7] = rot12(v[7]);
+ v[4] = rot12(v[4]);
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
+ v[0] = addv(v[0], v[5]);
+ v[1] = addv(v[1], v[6]);
+ v[2] = addv(v[2], v[7]);
+ v[3] = addv(v[3], v[4]);
+ v[15] = xorv(v[15], v[0]);
+ v[12] = xorv(v[12], v[1]);
+ v[13] = xorv(v[13], v[2]);
+ v[14] = xorv(v[14], v[3]);
+ v[15] = rot8(v[15]);
+ v[12] = rot8(v[12]);
+ v[13] = rot8(v[13]);
+ v[14] = rot8(v[14]);
+ v[10] = addv(v[10], v[15]);
+ v[11] = addv(v[11], v[12]);
+ v[8] = addv(v[8], v[13]);
+ v[9] = addv(v[9], v[14]);
+ v[5] = xorv(v[5], v[10]);
+ v[6] = xorv(v[6], v[11]);
+ v[7] = xorv(v[7], v[8]);
+ v[4] = xorv(v[4], v[9]);
+ v[5] = rot7(v[5]);
+ v[6] = rot7(v[6]);
+ v[7] = rot7(v[7]);
+ v[4] = rot7(v[4]);
+}
+
+INLINE void transpose_vecs(__m256i vecs[DEGREE]) {
+ // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high
+ // is 22/33/66/77.
+ __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]);
+ __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]);
+ __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]);
+ __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]);
+ __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]);
+ __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]);
+ __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]);
+ __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]);
+
+ // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is
+ // 11/33.
+ __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145);
+ __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145);
+ __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367);
+ __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367);
+ __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145);
+ __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145);
+ __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367);
+ __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367);
+
+ // Interleave 128-bit lanes.
+ vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20);
+ vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20);
+ vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20);
+ vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20);
+ vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31);
+ vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31);
+ vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31);
+ vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31);
+}
+
+INLINE void transpose_msg_vecs(const uint8_t *const *inputs,
+ size_t block_offset, __m256i out[16]) {
+ out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m256i)]);
+ out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m256i)]);
+ out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m256i)]);
+ out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m256i)]);
+ out[4] = loadu(&inputs[4][block_offset + 0 * sizeof(__m256i)]);
+ out[5] = loadu(&inputs[5][block_offset + 0 * sizeof(__m256i)]);
+ out[6] = loadu(&inputs[6][block_offset + 0 * sizeof(__m256i)]);
+ out[7] = loadu(&inputs[7][block_offset + 0 * sizeof(__m256i)]);
+ out[8] = loadu(&inputs[0][block_offset + 1 * sizeof(__m256i)]);
+ out[9] = loadu(&inputs[1][block_offset + 1 * sizeof(__m256i)]);
+ out[10] = loadu(&inputs[2][block_offset + 1 * sizeof(__m256i)]);
+ out[11] = loadu(&inputs[3][block_offset + 1 * sizeof(__m256i)]);
+ out[12] = loadu(&inputs[4][block_offset + 1 * sizeof(__m256i)]);
+ out[13] = loadu(&inputs[5][block_offset + 1 * sizeof(__m256i)]);
+ out[14] = loadu(&inputs[6][block_offset + 1 * sizeof(__m256i)]);
+ out[15] = loadu(&inputs[7][block_offset + 1 * sizeof(__m256i)]);
+ for (size_t i = 0; i < 8; ++i) {
+ _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
+ }
+ transpose_vecs(&out[0]);
+ transpose_vecs(&out[8]);
+}
+
+INLINE void load_counters(uint64_t counter, bool increment_counter,
+ __m256i *out_lo, __m256i *out_hi) {
+ const __m256i mask = _mm256_set1_epi32(-(int32_t)increment_counter);
+ const __m256i add0 = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
+ const __m256i add1 = _mm256_and_si256(mask, add0);
+ __m256i l = _mm256_add_epi32(_mm256_set1_epi32((int32_t)counter), add1);
+ __m256i carry = _mm256_cmpgt_epi32(_mm256_xor_si256(add1, _mm256_set1_epi32(0x80000000)),
+ _mm256_xor_si256( l, _mm256_set1_epi32(0x80000000)));
+ __m256i h = _mm256_sub_epi32(_mm256_set1_epi32((int32_t)(counter >> 32)), carry);
+ *out_lo = l;
+ *out_hi = h;
+}
+
+static
+void blake3_hash8_avx2(const uint8_t *const *inputs, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ bool increment_counter, uint8_t flags,
+ uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
+ __m256i h_vecs[8] = {
+ set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]),
+ set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]),
+ };
+ __m256i counter_low_vec, counter_high_vec;
+ load_counters(counter, increment_counter, &counter_low_vec,
+ &counter_high_vec);
+ uint8_t block_flags = flags | flags_start;
+
+ for (size_t block = 0; block < blocks; block++) {
+ if (block + 1 == blocks) {
+ block_flags |= flags_end;
+ }
+ __m256i block_len_vec = set1(BLAKE3_BLOCK_LEN);
+ __m256i block_flags_vec = set1(block_flags);
+ __m256i msg_vecs[16];
+ transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
+
+ __m256i v[16] = {
+ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
+ h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
+ set1(B3_IV_0), set1(B3_IV_1), set1(B3_IV_2), set1(B3_IV_3),
+ counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
+ };
+ round_fn(v, msg_vecs, 0);
+ round_fn(v, msg_vecs, 1);
+ round_fn(v, msg_vecs, 2);
+ round_fn(v, msg_vecs, 3);
+ round_fn(v, msg_vecs, 4);
+ round_fn(v, msg_vecs, 5);
+ round_fn(v, msg_vecs, 6);
+ h_vecs[0] = xorv(v[0], v[8]);
+ h_vecs[1] = xorv(v[1], v[9]);
+ h_vecs[2] = xorv(v[2], v[10]);
+ h_vecs[3] = xorv(v[3], v[11]);
+ h_vecs[4] = xorv(v[4], v[12]);
+ h_vecs[5] = xorv(v[5], v[13]);
+ h_vecs[6] = xorv(v[6], v[14]);
+ h_vecs[7] = xorv(v[7], v[15]);
+
+ block_flags = flags;
+ }
+
+ transpose_vecs(h_vecs);
+ storeu(h_vecs[0], &out[0 * sizeof(__m256i)]);
+ storeu(h_vecs[1], &out[1 * sizeof(__m256i)]);
+ storeu(h_vecs[2], &out[2 * sizeof(__m256i)]);
+ storeu(h_vecs[3], &out[3 * sizeof(__m256i)]);
+ storeu(h_vecs[4], &out[4 * sizeof(__m256i)]);
+ storeu(h_vecs[5], &out[5 * sizeof(__m256i)]);
+ storeu(h_vecs[6], &out[6 * sizeof(__m256i)]);
+ storeu(h_vecs[7], &out[7 * sizeof(__m256i)]);
+}
+
+#if !defined(BLAKE3_NO_SSE41)
+void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+#else
+void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+#endif
+
+void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out) {
+ while (num_inputs >= DEGREE) {
+ blake3_hash8_avx2(inputs, blocks, key, counter, increment_counter, flags,
+ flags_start, flags_end, out);
+ if (increment_counter) {
+ counter += DEGREE;
+ }
+ inputs += DEGREE;
+ num_inputs -= DEGREE;
+ out = &out[DEGREE * BLAKE3_OUT_LEN];
+ }
+#if !defined(BLAKE3_NO_SSE41)
+ blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter,
+ increment_counter, flags, flags_start, flags_end, out);
+#else
+ blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter,
+ increment_counter, flags, flags_start, flags_end,
+ out);
+#endif
+}
--- /dev/null
+#if defined(__ELF__) && defined(__linux__)
+.section .note.GNU-stack,"",%progbits
+#endif
+
+#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
+#if __has_include(<cet.h>)
+#include <cet.h>
+#endif
+#endif
+
+#if !defined(_CET_ENDBR)
+#define _CET_ENDBR
+#endif
+
+.intel_syntax noprefix
+.global _blake3_hash_many_avx2_asm
+.global blake3_hash_many_avx2_asm
+#ifdef __APPLE__
+.text
+#else
+.section .text
+#endif
+ .p2align 6
+_blake3_hash_many_avx2_asm:
+blake3_hash_many_avx2_asm:
+ _CET_ENDBR
+ push r15
+ push r14
+ push r13
+ push r12
+ push rbx
+ push rbp
+ mov rbp, rsp
+ sub rsp, 680
+ and rsp, 0xFFFFFFFFFFFFFFC0
+ neg r9d
+ vmovd xmm0, r9d
+ vpbroadcastd ymm0, xmm0
+ vmovdqa ymmword ptr [rsp+0x280], ymm0
+ vpand ymm1, ymm0, ymmword ptr [ADD0+rip]
+ vpand ymm2, ymm0, ymmword ptr [ADD1+rip]
+ vmovdqa ymmword ptr [rsp+0x220], ymm2
+ vmovd xmm2, r8d
+ vpbroadcastd ymm2, xmm2
+ vpaddd ymm2, ymm2, ymm1
+ vmovdqa ymmword ptr [rsp+0x240], ymm2
+ vpxor ymm1, ymm1, ymmword ptr [CMP_MSB_MASK+rip]
+ vpxor ymm2, ymm2, ymmword ptr [CMP_MSB_MASK+rip]
+ vpcmpgtd ymm2, ymm1, ymm2
+ shr r8, 32
+ vmovd xmm3, r8d
+ vpbroadcastd ymm3, xmm3
+ vpsubd ymm3, ymm3, ymm2
+ vmovdqa ymmword ptr [rsp+0x260], ymm3
+ shl rdx, 6
+ mov qword ptr [rsp+0x2A0], rdx
+ cmp rsi, 8
+ jc 3f
+2:
+ vpbroadcastd ymm0, dword ptr [rcx]
+ vpbroadcastd ymm1, dword ptr [rcx+0x4]
+ vpbroadcastd ymm2, dword ptr [rcx+0x8]
+ vpbroadcastd ymm3, dword ptr [rcx+0xC]
+ vpbroadcastd ymm4, dword ptr [rcx+0x10]
+ vpbroadcastd ymm5, dword ptr [rcx+0x14]
+ vpbroadcastd ymm6, dword ptr [rcx+0x18]
+ vpbroadcastd ymm7, dword ptr [rcx+0x1C]
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ mov r10, qword ptr [rdi+0x10]
+ mov r11, qword ptr [rdi+0x18]
+ mov r12, qword ptr [rdi+0x20]
+ mov r13, qword ptr [rdi+0x28]
+ mov r14, qword ptr [rdi+0x30]
+ mov r15, qword ptr [rdi+0x38]
+ movzx eax, byte ptr [rbp+0x38]
+ movzx ebx, byte ptr [rbp+0x40]
+ or eax, ebx
+ xor edx, edx
+.p2align 5
+9:
+ movzx ebx, byte ptr [rbp+0x48]
+ or ebx, eax
+ add rdx, 64
+ cmp rdx, qword ptr [rsp+0x2A0]
+ cmove eax, ebx
+ mov dword ptr [rsp+0x200], eax
+ vmovups xmm8, xmmword ptr [r8+rdx-0x40]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x40], 0x01
+ vmovups xmm9, xmmword ptr [r9+rdx-0x40]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x40], 0x01
+ vunpcklpd ymm12, ymm8, ymm9
+ vunpckhpd ymm13, ymm8, ymm9
+ vmovups xmm10, xmmword ptr [r10+rdx-0x40]
+ vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x40], 0x01
+ vmovups xmm11, xmmword ptr [r11+rdx-0x40]
+ vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x40], 0x01
+ vunpcklpd ymm14, ymm10, ymm11
+ vunpckhpd ymm15, ymm10, ymm11
+ vshufps ymm8, ymm12, ymm14, 136
+ vmovaps ymmword ptr [rsp], ymm8
+ vshufps ymm9, ymm12, ymm14, 221
+ vmovaps ymmword ptr [rsp+0x20], ymm9
+ vshufps ymm10, ymm13, ymm15, 136
+ vmovaps ymmword ptr [rsp+0x40], ymm10
+ vshufps ymm11, ymm13, ymm15, 221
+ vmovaps ymmword ptr [rsp+0x60], ymm11
+ vmovups xmm8, xmmword ptr [r8+rdx-0x30]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x30], 0x01
+ vmovups xmm9, xmmword ptr [r9+rdx-0x30]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x30], 0x01
+ vunpcklpd ymm12, ymm8, ymm9
+ vunpckhpd ymm13, ymm8, ymm9
+ vmovups xmm10, xmmword ptr [r10+rdx-0x30]
+ vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x30], 0x01
+ vmovups xmm11, xmmword ptr [r11+rdx-0x30]
+ vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x30], 0x01
+ vunpcklpd ymm14, ymm10, ymm11
+ vunpckhpd ymm15, ymm10, ymm11
+ vshufps ymm8, ymm12, ymm14, 136
+ vmovaps ymmword ptr [rsp+0x80], ymm8
+ vshufps ymm9, ymm12, ymm14, 221
+ vmovaps ymmword ptr [rsp+0xA0], ymm9
+ vshufps ymm10, ymm13, ymm15, 136
+ vmovaps ymmword ptr [rsp+0xC0], ymm10
+ vshufps ymm11, ymm13, ymm15, 221
+ vmovaps ymmword ptr [rsp+0xE0], ymm11
+ vmovups xmm8, xmmword ptr [r8+rdx-0x20]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x20], 0x01
+ vmovups xmm9, xmmword ptr [r9+rdx-0x20]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x20], 0x01
+ vunpcklpd ymm12, ymm8, ymm9
+ vunpckhpd ymm13, ymm8, ymm9
+ vmovups xmm10, xmmword ptr [r10+rdx-0x20]
+ vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x20], 0x01
+ vmovups xmm11, xmmword ptr [r11+rdx-0x20]
+ vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x20], 0x01
+ vunpcklpd ymm14, ymm10, ymm11
+ vunpckhpd ymm15, ymm10, ymm11
+ vshufps ymm8, ymm12, ymm14, 136
+ vmovaps ymmword ptr [rsp+0x100], ymm8
+ vshufps ymm9, ymm12, ymm14, 221
+ vmovaps ymmword ptr [rsp+0x120], ymm9
+ vshufps ymm10, ymm13, ymm15, 136
+ vmovaps ymmword ptr [rsp+0x140], ymm10
+ vshufps ymm11, ymm13, ymm15, 221
+ vmovaps ymmword ptr [rsp+0x160], ymm11
+ vmovups xmm8, xmmword ptr [r8+rdx-0x10]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x10], 0x01
+ vmovups xmm9, xmmword ptr [r9+rdx-0x10]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x10], 0x01
+ vunpcklpd ymm12, ymm8, ymm9
+ vunpckhpd ymm13, ymm8, ymm9
+ vmovups xmm10, xmmword ptr [r10+rdx-0x10]
+ vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x10], 0x01
+ vmovups xmm11, xmmword ptr [r11+rdx-0x10]
+ vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x10], 0x01
+ vunpcklpd ymm14, ymm10, ymm11
+ vunpckhpd ymm15, ymm10, ymm11
+ vshufps ymm8, ymm12, ymm14, 136
+ vmovaps ymmword ptr [rsp+0x180], ymm8
+ vshufps ymm9, ymm12, ymm14, 221
+ vmovaps ymmword ptr [rsp+0x1A0], ymm9
+ vshufps ymm10, ymm13, ymm15, 136
+ vmovaps ymmword ptr [rsp+0x1C0], ymm10
+ vshufps ymm11, ymm13, ymm15, 221
+ vmovaps ymmword ptr [rsp+0x1E0], ymm11
+ vpbroadcastd ymm15, dword ptr [rsp+0x200]
+ prefetcht0 [r8+rdx+0x80]
+ prefetcht0 [r12+rdx+0x80]
+ prefetcht0 [r9+rdx+0x80]
+ prefetcht0 [r13+rdx+0x80]
+ prefetcht0 [r10+rdx+0x80]
+ prefetcht0 [r14+rdx+0x80]
+ prefetcht0 [r11+rdx+0x80]
+ prefetcht0 [r15+rdx+0x80]
+ vpaddd ymm0, ymm0, ymmword ptr [rsp]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x40]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x80]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm0, ymmword ptr [rsp+0x240]
+ vpxor ymm13, ymm1, ymmword ptr [rsp+0x260]
+ vpxor ymm14, ymm2, ymmword ptr [BLAKE3_BLOCK_LEN+rip]
+ vpxor ymm15, ymm3, ymm15
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [BLAKE3_IV_0+rip]
+ vpaddd ymm9, ymm13, ymmword ptr [BLAKE3_IV_1+rip]
+ vpaddd ymm10, ymm14, ymmword ptr [BLAKE3_IV_2+rip]
+ vpaddd ymm11, ymm15, ymmword ptr [BLAKE3_IV_3+rip]
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x20]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x60]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0xA0]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x100]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x180]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1C0]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x120]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1A0]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x40]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x60]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0xE0]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x80]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0xC0]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1A0]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x20]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x120]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x160]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1C0]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x60]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1A0]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x80]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x40]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1C0]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0xC0]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x120]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x160]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0xA0]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1E0]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x20]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x140]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1C0]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1A0]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0xE0]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x120]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x60]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x80]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0xA0]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x20]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x40]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x100]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x180]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x120]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1E0]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1C0]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1A0]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x140]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0xE0]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x40]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x60]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x20]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x80]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x120]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x100]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1C0]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x180]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x20]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1A0]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x40]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x80]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x60]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0xC0]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x160]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x20]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1E0]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x120]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0]
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxor ymm12, ymm12, ymm0
+ vpxor ymm13, ymm13, ymm1
+ vpxor ymm14, ymm14, ymm2
+ vpxor ymm15, ymm15, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpshufb ymm15, ymm15, ymm8
+ vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxor ymm4, ymm4, ymm8
+ vpxor ymm5, ymm5, ymm9
+ vpxor ymm6, ymm6, ymm10
+ vpxor ymm7, ymm7, ymm11
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1C0]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x40]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x60]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT16+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vmovdqa ymmword ptr [rsp+0x200], ymm8
+ vpsrld ymm8, ymm5, 12
+ vpslld ymm5, ymm5, 20
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 12
+ vpslld ymm6, ymm6, 20
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 12
+ vpslld ymm7, ymm7, 20
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 12
+ vpslld ymm4, ymm4, 20
+ vpor ymm4, ymm4, ymm8
+ vpaddd ymm0, ymm0, ymmword ptr [rsp+0x140]
+ vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180]
+ vpaddd ymm2, ymm2, ymmword ptr [rsp+0x80]
+ vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1A0]
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxor ymm15, ymm15, ymm0
+ vpxor ymm12, ymm12, ymm1
+ vpxor ymm13, ymm13, ymm2
+ vpxor ymm14, ymm14, ymm3
+ vbroadcasti128 ymm8, xmmword ptr [ROT8+rip]
+ vpshufb ymm15, ymm15, ymm8
+ vpshufb ymm12, ymm12, ymm8
+ vpshufb ymm13, ymm13, ymm8
+ vpshufb ymm14, ymm14, ymm8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200]
+ vpaddd ymm9, ymm9, ymm14
+ vpxor ymm5, ymm5, ymm10
+ vpxor ymm6, ymm6, ymm11
+ vpxor ymm7, ymm7, ymm8
+ vpxor ymm4, ymm4, ymm9
+ vpxor ymm0, ymm0, ymm8
+ vpxor ymm1, ymm1, ymm9
+ vpxor ymm2, ymm2, ymm10
+ vpxor ymm3, ymm3, ymm11
+ vpsrld ymm8, ymm5, 7
+ vpslld ymm5, ymm5, 25
+ vpor ymm5, ymm5, ymm8
+ vpsrld ymm8, ymm6, 7
+ vpslld ymm6, ymm6, 25
+ vpor ymm6, ymm6, ymm8
+ vpsrld ymm8, ymm7, 7
+ vpslld ymm7, ymm7, 25
+ vpor ymm7, ymm7, ymm8
+ vpsrld ymm8, ymm4, 7
+ vpslld ymm4, ymm4, 25
+ vpor ymm4, ymm4, ymm8
+ vpxor ymm4, ymm4, ymm12
+ vpxor ymm5, ymm5, ymm13
+ vpxor ymm6, ymm6, ymm14
+ vpxor ymm7, ymm7, ymm15
+ movzx eax, byte ptr [rbp+0x38]
+ jne 9b
+ mov rbx, qword ptr [rbp+0x50]
+ vunpcklps ymm8, ymm0, ymm1
+ vunpcklps ymm9, ymm2, ymm3
+ vunpckhps ymm10, ymm0, ymm1
+ vunpcklps ymm11, ymm4, ymm5
+ vunpcklps ymm0, ymm6, ymm7
+ vshufps ymm12, ymm8, ymm9, 78
+ vblendps ymm1, ymm8, ymm12, 0xCC
+ vshufps ymm8, ymm11, ymm0, 78
+ vunpckhps ymm13, ymm2, ymm3
+ vblendps ymm2, ymm11, ymm8, 0xCC
+ vblendps ymm3, ymm12, ymm9, 0xCC
+ vperm2f128 ymm12, ymm1, ymm2, 0x20
+ vmovups ymmword ptr [rbx], ymm12
+ vunpckhps ymm14, ymm4, ymm5
+ vblendps ymm4, ymm8, ymm0, 0xCC
+ vunpckhps ymm15, ymm6, ymm7
+ vperm2f128 ymm7, ymm3, ymm4, 0x20
+ vmovups ymmword ptr [rbx+0x20], ymm7
+ vshufps ymm5, ymm10, ymm13, 78
+ vblendps ymm6, ymm5, ymm13, 0xCC
+ vshufps ymm13, ymm14, ymm15, 78
+ vblendps ymm10, ymm10, ymm5, 0xCC
+ vblendps ymm14, ymm14, ymm13, 0xCC
+ vperm2f128 ymm8, ymm10, ymm14, 0x20
+ vmovups ymmword ptr [rbx+0x40], ymm8
+ vblendps ymm15, ymm13, ymm15, 0xCC
+ vperm2f128 ymm13, ymm6, ymm15, 0x20
+ vmovups ymmword ptr [rbx+0x60], ymm13
+ vperm2f128 ymm9, ymm1, ymm2, 0x31
+ vperm2f128 ymm11, ymm3, ymm4, 0x31
+ vmovups ymmword ptr [rbx+0x80], ymm9
+ vperm2f128 ymm14, ymm10, ymm14, 0x31
+ vperm2f128 ymm15, ymm6, ymm15, 0x31
+ vmovups ymmword ptr [rbx+0xA0], ymm11
+ vmovups ymmword ptr [rbx+0xC0], ymm14
+ vmovups ymmword ptr [rbx+0xE0], ymm15
+ vmovdqa ymm0, ymmword ptr [rsp+0x220]
+ vpaddd ymm1, ymm0, ymmword ptr [rsp+0x240]
+ vmovdqa ymmword ptr [rsp+0x240], ymm1
+ vpxor ymm0, ymm0, ymmword ptr [CMP_MSB_MASK+rip]
+ vpxor ymm2, ymm1, ymmword ptr [CMP_MSB_MASK+rip]
+ vpcmpgtd ymm2, ymm0, ymm2
+ vmovdqa ymm0, ymmword ptr [rsp+0x260]
+ vpsubd ymm2, ymm0, ymm2
+ vmovdqa ymmword ptr [rsp+0x260], ymm2
+ add rdi, 64
+ add rbx, 256
+ mov qword ptr [rbp+0x50], rbx
+ sub rsi, 8
+ cmp rsi, 8
+ jnc 2b
+ test rsi, rsi
+ jnz 3f
+4:
+ vzeroupper
+ mov rsp, rbp
+ pop rbp
+ pop rbx
+ pop r12
+ pop r13
+ pop r14
+ pop r15
+ ret
+.p2align 5
+3:
+ mov rbx, qword ptr [rbp+0x50]
+ mov r15, qword ptr [rsp+0x2A0]
+ movzx r13d, byte ptr [rbp+0x38]
+ movzx r12d, byte ptr [rbp+0x48]
+ test rsi, 0x4
+ je 3f
+ vbroadcasti128 ymm0, xmmword ptr [rcx]
+ vbroadcasti128 ymm1, xmmword ptr [rcx+0x10]
+ vmovdqa ymm8, ymm0
+ vmovdqa ymm9, ymm1
+ vbroadcasti128 ymm12, xmmword ptr [rsp+0x240]
+ vbroadcasti128 ymm13, xmmword ptr [rsp+0x260]
+ vpunpckldq ymm14, ymm12, ymm13
+ vpunpckhdq ymm15, ymm12, ymm13
+ vpermq ymm14, ymm14, 0x50
+ vpermq ymm15, ymm15, 0x50
+ vbroadcasti128 ymm12, xmmword ptr [BLAKE3_BLOCK_LEN+rip]
+ vpblendd ymm14, ymm14, ymm12, 0x44
+ vpblendd ymm15, ymm15, ymm12, 0x44
+ vmovdqa ymmword ptr [rsp], ymm14
+ vmovdqa ymmword ptr [rsp+0x20], ymm15
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ mov r10, qword ptr [rdi+0x10]
+ mov r11, qword ptr [rdi+0x18]
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+.p2align 5
+2:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ mov dword ptr [rsp+0x200], eax
+ vmovups ymm2, ymmword ptr [r8+rdx-0x40]
+ vinsertf128 ymm2, ymm2, xmmword ptr [r9+rdx-0x40], 0x01
+ vmovups ymm3, ymmword ptr [r8+rdx-0x30]
+ vinsertf128 ymm3, ymm3, xmmword ptr [r9+rdx-0x30], 0x01
+ vshufps ymm4, ymm2, ymm3, 136
+ vshufps ymm5, ymm2, ymm3, 221
+ vmovups ymm2, ymmword ptr [r8+rdx-0x20]
+ vinsertf128 ymm2, ymm2, xmmword ptr [r9+rdx-0x20], 0x01
+ vmovups ymm3, ymmword ptr [r8+rdx-0x10]
+ vinsertf128 ymm3, ymm3, xmmword ptr [r9+rdx-0x10], 0x01
+ vshufps ymm6, ymm2, ymm3, 136
+ vshufps ymm7, ymm2, ymm3, 221
+ vpshufd ymm6, ymm6, 0x93
+ vpshufd ymm7, ymm7, 0x93
+ vmovups ymm10, ymmword ptr [r10+rdx-0x40]
+ vinsertf128 ymm10, ymm10, xmmword ptr [r11+rdx-0x40], 0x01
+ vmovups ymm11, ymmword ptr [r10+rdx-0x30]
+ vinsertf128 ymm11, ymm11, xmmword ptr [r11+rdx-0x30], 0x01
+ vshufps ymm12, ymm10, ymm11, 136
+ vshufps ymm13, ymm10, ymm11, 221
+ vmovups ymm10, ymmword ptr [r10+rdx-0x20]
+ vinsertf128 ymm10, ymm10, xmmword ptr [r11+rdx-0x20], 0x01
+ vmovups ymm11, ymmword ptr [r10+rdx-0x10]
+ vinsertf128 ymm11, ymm11, xmmword ptr [r11+rdx-0x10], 0x01
+ vshufps ymm14, ymm10, ymm11, 136
+ vshufps ymm15, ymm10, ymm11, 221
+ vpshufd ymm14, ymm14, 0x93
+ vpshufd ymm15, ymm15, 0x93
+ prefetcht0 [r8+rdx+0x80]
+ prefetcht0 [r9+rdx+0x80]
+ prefetcht0 [r10+rdx+0x80]
+ prefetcht0 [r11+rdx+0x80]
+ vpbroadcastd ymm2, dword ptr [rsp+0x200]
+ vmovdqa ymm3, ymmword ptr [rsp]
+ vmovdqa ymm11, ymmword ptr [rsp+0x20]
+ vpblendd ymm3, ymm3, ymm2, 0x88
+ vpblendd ymm11, ymm11, ymm2, 0x88
+ vbroadcasti128 ymm2, xmmword ptr [BLAKE3_IV+rip]
+ vmovdqa ymm10, ymm2
+ mov al, 7
+9:
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm8, ymm8, ymm12
+ vmovdqa ymmword ptr [rsp+0x40], ymm4
+ nop
+ vmovdqa ymmword ptr [rsp+0x60], ymm12
+ nop
+ vpaddd ymm0, ymm0, ymm1
+ vpaddd ymm8, ymm8, ymm9
+ vpxor ymm3, ymm3, ymm0
+ vpxor ymm11, ymm11, ymm8
+ vbroadcasti128 ymm4, xmmword ptr [ROT16+rip]
+ vpshufb ymm3, ymm3, ymm4
+ vpshufb ymm11, ymm11, ymm4
+ vpaddd ymm2, ymm2, ymm3
+ vpaddd ymm10, ymm10, ymm11
+ vpxor ymm1, ymm1, ymm2
+ vpxor ymm9, ymm9, ymm10
+ vpsrld ymm4, ymm1, 12
+ vpslld ymm1, ymm1, 20
+ vpor ymm1, ymm1, ymm4
+ vpsrld ymm4, ymm9, 12
+ vpslld ymm9, ymm9, 20
+ vpor ymm9, ymm9, ymm4
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm0, ymm0, ymm1
+ vpaddd ymm8, ymm8, ymm9
+ vmovdqa ymmword ptr [rsp+0x80], ymm5
+ vmovdqa ymmword ptr [rsp+0xA0], ymm13
+ vpxor ymm3, ymm3, ymm0
+ vpxor ymm11, ymm11, ymm8
+ vbroadcasti128 ymm4, xmmword ptr [ROT8+rip]
+ vpshufb ymm3, ymm3, ymm4
+ vpshufb ymm11, ymm11, ymm4
+ vpaddd ymm2, ymm2, ymm3
+ vpaddd ymm10, ymm10, ymm11
+ vpxor ymm1, ymm1, ymm2
+ vpxor ymm9, ymm9, ymm10
+ vpsrld ymm4, ymm1, 7
+ vpslld ymm1, ymm1, 25
+ vpor ymm1, ymm1, ymm4
+ vpsrld ymm4, ymm9, 7
+ vpslld ymm9, ymm9, 25
+ vpor ymm9, ymm9, ymm4
+ vpshufd ymm0, ymm0, 0x93
+ vpshufd ymm8, ymm8, 0x93
+ vpshufd ymm3, ymm3, 0x4E
+ vpshufd ymm11, ymm11, 0x4E
+ vpshufd ymm2, ymm2, 0x39
+ vpshufd ymm10, ymm10, 0x39
+ vpaddd ymm0, ymm0, ymm6
+ vpaddd ymm8, ymm8, ymm14
+ vpaddd ymm0, ymm0, ymm1
+ vpaddd ymm8, ymm8, ymm9
+ vpxor ymm3, ymm3, ymm0
+ vpxor ymm11, ymm11, ymm8
+ vbroadcasti128 ymm4, xmmword ptr [ROT16+rip]
+ vpshufb ymm3, ymm3, ymm4
+ vpshufb ymm11, ymm11, ymm4
+ vpaddd ymm2, ymm2, ymm3
+ vpaddd ymm10, ymm10, ymm11
+ vpxor ymm1, ymm1, ymm2
+ vpxor ymm9, ymm9, ymm10
+ vpsrld ymm4, ymm1, 12
+ vpslld ymm1, ymm1, 20
+ vpor ymm1, ymm1, ymm4
+ vpsrld ymm4, ymm9, 12
+ vpslld ymm9, ymm9, 20
+ vpor ymm9, ymm9, ymm4
+ vpaddd ymm0, ymm0, ymm7
+ vpaddd ymm8, ymm8, ymm15
+ vpaddd ymm0, ymm0, ymm1
+ vpaddd ymm8, ymm8, ymm9
+ vpxor ymm3, ymm3, ymm0
+ vpxor ymm11, ymm11, ymm8
+ vbroadcasti128 ymm4, xmmword ptr [ROT8+rip]
+ vpshufb ymm3, ymm3, ymm4
+ vpshufb ymm11, ymm11, ymm4
+ vpaddd ymm2, ymm2, ymm3
+ vpaddd ymm10, ymm10, ymm11
+ vpxor ymm1, ymm1, ymm2
+ vpxor ymm9, ymm9, ymm10
+ vpsrld ymm4, ymm1, 7
+ vpslld ymm1, ymm1, 25
+ vpor ymm1, ymm1, ymm4
+ vpsrld ymm4, ymm9, 7
+ vpslld ymm9, ymm9, 25
+ vpor ymm9, ymm9, ymm4
+ vpshufd ymm0, ymm0, 0x39
+ vpshufd ymm8, ymm8, 0x39
+ vpshufd ymm3, ymm3, 0x4E
+ vpshufd ymm11, ymm11, 0x4E
+ vpshufd ymm2, ymm2, 0x93
+ vpshufd ymm10, ymm10, 0x93
+ dec al
+ je 9f
+ vmovdqa ymm4, ymmword ptr [rsp+0x40]
+ vmovdqa ymm5, ymmword ptr [rsp+0x80]
+ vshufps ymm12, ymm4, ymm5, 214
+ vpshufd ymm13, ymm4, 0x0F
+ vpshufd ymm4, ymm12, 0x39
+ vshufps ymm12, ymm6, ymm7, 250
+ vpblendd ymm13, ymm13, ymm12, 0xAA
+ vpunpcklqdq ymm12, ymm7, ymm5
+ vpblendd ymm12, ymm12, ymm6, 0x88
+ vpshufd ymm12, ymm12, 0x78
+ vpunpckhdq ymm5, ymm5, ymm7
+ vpunpckldq ymm6, ymm6, ymm5
+ vpshufd ymm7, ymm6, 0x1E
+ vmovdqa ymmword ptr [rsp+0x40], ymm13
+ vmovdqa ymmword ptr [rsp+0x80], ymm12
+ vmovdqa ymm12, ymmword ptr [rsp+0x60]
+ vmovdqa ymm13, ymmword ptr [rsp+0xA0]
+ vshufps ymm5, ymm12, ymm13, 214
+ vpshufd ymm6, ymm12, 0x0F
+ vpshufd ymm12, ymm5, 0x39
+ vshufps ymm5, ymm14, ymm15, 250
+ vpblendd ymm6, ymm6, ymm5, 0xAA
+ vpunpcklqdq ymm5, ymm15, ymm13
+ vpblendd ymm5, ymm5, ymm14, 0x88
+ vpshufd ymm5, ymm5, 0x78
+ vpunpckhdq ymm13, ymm13, ymm15
+ vpunpckldq ymm14, ymm14, ymm13
+ vpshufd ymm15, ymm14, 0x1E
+ vmovdqa ymm13, ymm6
+ vmovdqa ymm14, ymm5
+ vmovdqa ymm5, ymmword ptr [rsp+0x40]
+ vmovdqa ymm6, ymmword ptr [rsp+0x80]
+ jmp 9b
+9:
+ vpxor ymm0, ymm0, ymm2
+ vpxor ymm1, ymm1, ymm3
+ vpxor ymm8, ymm8, ymm10
+ vpxor ymm9, ymm9, ymm11
+ mov eax, r13d
+ cmp rdx, r15
+ jne 2b
+ vmovdqu xmmword ptr [rbx], xmm0
+ vmovdqu xmmword ptr [rbx+0x10], xmm1
+ vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01
+ vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01
+ vmovdqu xmmword ptr [rbx+0x40], xmm8
+ vmovdqu xmmword ptr [rbx+0x50], xmm9
+ vextracti128 xmmword ptr [rbx+0x60], ymm8, 0x01
+ vextracti128 xmmword ptr [rbx+0x70], ymm9, 0x01
+ vmovaps xmm8, xmmword ptr [rsp+0x280]
+ vmovaps xmm0, xmmword ptr [rsp+0x240]
+ vmovaps xmm1, xmmword ptr [rsp+0x250]
+ vmovaps xmm2, xmmword ptr [rsp+0x260]
+ vmovaps xmm3, xmmword ptr [rsp+0x270]
+ vblendvps xmm0, xmm0, xmm1, xmm8
+ vblendvps xmm2, xmm2, xmm3, xmm8
+ vmovaps xmmword ptr [rsp+0x240], xmm0
+ vmovaps xmmword ptr [rsp+0x260], xmm2
+ add rbx, 128
+ add rdi, 32
+ sub rsi, 4
+3:
+ test rsi, 0x2
+ je 3f
+ vbroadcasti128 ymm0, xmmword ptr [rcx]
+ vbroadcasti128 ymm1, xmmword ptr [rcx+0x10]
+ vmovd xmm13, dword ptr [rsp+0x240]
+ vpinsrd xmm13, xmm13, dword ptr [rsp+0x260], 1
+ vpinsrd xmm13, xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2
+ vmovd xmm14, dword ptr [rsp+0x244]
+ vpinsrd xmm14, xmm14, dword ptr [rsp+0x264], 1
+ vpinsrd xmm14, xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2
+ vinserti128 ymm13, ymm13, xmm14, 0x01
+ vbroadcasti128 ymm14, xmmword ptr [ROT16+rip]
+ vbroadcasti128 ymm15, xmmword ptr [ROT8+rip]
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+.p2align 5
+2:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ mov dword ptr [rsp+0x200], eax
+ vbroadcasti128 ymm2, xmmword ptr [BLAKE3_IV+rip]
+ vpbroadcastd ymm8, dword ptr [rsp+0x200]
+ vpblendd ymm3, ymm13, ymm8, 0x88
+ vmovups ymm8, ymmword ptr [r8+rdx-0x40]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x40], 0x01
+ vmovups ymm9, ymmword ptr [r8+rdx-0x30]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x30], 0x01
+ vshufps ymm4, ymm8, ymm9, 136
+ vshufps ymm5, ymm8, ymm9, 221
+ vmovups ymm8, ymmword ptr [r8+rdx-0x20]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x20], 0x01
+ vmovups ymm9, ymmword ptr [r8+rdx-0x10]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x10], 0x01
+ vshufps ymm6, ymm8, ymm9, 136
+ vshufps ymm7, ymm8, ymm9, 221
+ vpshufd ymm6, ymm6, 0x93
+ vpshufd ymm7, ymm7, 0x93
+ mov al, 7
+9:
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm0, ymm0, ymm1
+ vpxor ymm3, ymm3, ymm0
+ vpshufb ymm3, ymm3, ymm14
+ vpaddd ymm2, ymm2, ymm3
+ vpxor ymm1, ymm1, ymm2
+ vpsrld ymm8, ymm1, 12
+ vpslld ymm1, ymm1, 20
+ vpor ymm1, ymm1, ymm8
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm0, ymm0, ymm1
+ vpxor ymm3, ymm3, ymm0
+ vpshufb ymm3, ymm3, ymm15
+ vpaddd ymm2, ymm2, ymm3
+ vpxor ymm1, ymm1, ymm2
+ vpsrld ymm8, ymm1, 7
+ vpslld ymm1, ymm1, 25
+ vpor ymm1, ymm1, ymm8
+ vpshufd ymm0, ymm0, 0x93
+ vpshufd ymm3, ymm3, 0x4E
+ vpshufd ymm2, ymm2, 0x39
+ vpaddd ymm0, ymm0, ymm6
+ vpaddd ymm0, ymm0, ymm1
+ vpxor ymm3, ymm3, ymm0
+ vpshufb ymm3, ymm3, ymm14
+ vpaddd ymm2, ymm2, ymm3
+ vpxor ymm1, ymm1, ymm2
+ vpsrld ymm8, ymm1, 12
+ vpslld ymm1, ymm1, 20
+ vpor ymm1, ymm1, ymm8
+ vpaddd ymm0, ymm0, ymm7
+ vpaddd ymm0, ymm0, ymm1
+ vpxor ymm3, ymm3, ymm0
+ vpshufb ymm3, ymm3, ymm15
+ vpaddd ymm2, ymm2, ymm3
+ vpxor ymm1, ymm1, ymm2
+ vpsrld ymm8, ymm1, 7
+ vpslld ymm1, ymm1, 25
+ vpor ymm1, ymm1, ymm8
+ vpshufd ymm0, ymm0, 0x39
+ vpshufd ymm3, ymm3, 0x4E
+ vpshufd ymm2, ymm2, 0x93
+ dec al
+ jz 9f
+ vshufps ymm8, ymm4, ymm5, 214
+ vpshufd ymm9, ymm4, 0x0F
+ vpshufd ymm4, ymm8, 0x39
+ vshufps ymm8, ymm6, ymm7, 250
+ vpblendd ymm9, ymm9, ymm8, 0xAA
+ vpunpcklqdq ymm8, ymm7, ymm5
+ vpblendd ymm8, ymm8, ymm6, 0x88
+ vpshufd ymm8, ymm8, 0x78
+ vpunpckhdq ymm5, ymm5, ymm7
+ vpunpckldq ymm6, ymm6, ymm5
+ vpshufd ymm7, ymm6, 0x1E
+ vmovdqa ymm5, ymm9
+ vmovdqa ymm6, ymm8
+ jmp 9b
+9:
+ vpxor ymm0, ymm0, ymm2
+ vpxor ymm1, ymm1, ymm3
+ mov eax, r13d
+ cmp rdx, r15
+ jne 2b
+ vmovdqu xmmword ptr [rbx], xmm0
+ vmovdqu xmmword ptr [rbx+0x10], xmm1
+ vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01
+ vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01
+ vmovaps ymm8, ymmword ptr [rsp+0x280]
+ vmovaps ymm0, ymmword ptr [rsp+0x240]
+ vmovups ymm1, ymmword ptr [rsp+0x248]
+ vmovaps ymm2, ymmword ptr [rsp+0x260]
+ vmovups ymm3, ymmword ptr [rsp+0x268]
+ vblendvps ymm0, ymm0, ymm1, ymm8
+ vblendvps ymm2, ymm2, ymm3, ymm8
+ vmovaps ymmword ptr [rsp+0x240], ymm0
+ vmovaps ymmword ptr [rsp+0x260], ymm2
+ add rbx, 64
+ add rdi, 16
+ sub rsi, 2
+3:
+ test rsi, 0x1
+ je 4b
+ vmovdqu xmm0, xmmword ptr [rcx]
+ vmovdqu xmm1, xmmword ptr [rcx+0x10]
+ vmovd xmm3, dword ptr [rsp+0x240]
+ vpinsrd xmm3, xmm3, dword ptr [rsp+0x260], 1
+ vpinsrd xmm13, xmm3, dword ptr [BLAKE3_BLOCK_LEN+rip], 2
+ vmovdqa xmm14, xmmword ptr [ROT16+rip]
+ vmovdqa xmm15, xmmword ptr [ROT8+rip]
+ mov r8, qword ptr [rdi]
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+.p2align 5
+2:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ vmovdqa xmm2, xmmword ptr [BLAKE3_IV+rip]
+ vmovdqa xmm3, xmm13
+ vpinsrd xmm3, xmm3, eax, 3
+ vmovups xmm8, xmmword ptr [r8+rdx-0x40]
+ vmovups xmm9, xmmword ptr [r8+rdx-0x30]
+ vshufps xmm4, xmm8, xmm9, 136
+ vshufps xmm5, xmm8, xmm9, 221
+ vmovups xmm8, xmmword ptr [r8+rdx-0x20]
+ vmovups xmm9, xmmword ptr [r8+rdx-0x10]
+ vshufps xmm6, xmm8, xmm9, 136
+ vshufps xmm7, xmm8, xmm9, 221
+ vpshufd xmm6, xmm6, 0x93
+ vpshufd xmm7, xmm7, 0x93
+ mov al, 7
+9:
+ vpaddd xmm0, xmm0, xmm4
+ vpaddd xmm0, xmm0, xmm1
+ vpxor xmm3, xmm3, xmm0
+ vpshufb xmm3, xmm3, xmm14
+ vpaddd xmm2, xmm2, xmm3
+ vpxor xmm1, xmm1, xmm2
+ vpsrld xmm8, xmm1, 12
+ vpslld xmm1, xmm1, 20
+ vpor xmm1, xmm1, xmm8
+ vpaddd xmm0, xmm0, xmm5
+ vpaddd xmm0, xmm0, xmm1
+ vpxor xmm3, xmm3, xmm0
+ vpshufb xmm3, xmm3, xmm15
+ vpaddd xmm2, xmm2, xmm3
+ vpxor xmm1, xmm1, xmm2
+ vpsrld xmm8, xmm1, 7
+ vpslld xmm1, xmm1, 25
+ vpor xmm1, xmm1, xmm8
+ vpshufd xmm0, xmm0, 0x93
+ vpshufd xmm3, xmm3, 0x4E
+ vpshufd xmm2, xmm2, 0x39
+ vpaddd xmm0, xmm0, xmm6
+ vpaddd xmm0, xmm0, xmm1
+ vpxor xmm3, xmm3, xmm0
+ vpshufb xmm3, xmm3, xmm14
+ vpaddd xmm2, xmm2, xmm3
+ vpxor xmm1, xmm1, xmm2
+ vpsrld xmm8, xmm1, 12
+ vpslld xmm1, xmm1, 20
+ vpor xmm1, xmm1, xmm8
+ vpaddd xmm0, xmm0, xmm7
+ vpaddd xmm0, xmm0, xmm1
+ vpxor xmm3, xmm3, xmm0
+ vpshufb xmm3, xmm3, xmm15
+ vpaddd xmm2, xmm2, xmm3
+ vpxor xmm1, xmm1, xmm2
+ vpsrld xmm8, xmm1, 7
+ vpslld xmm1, xmm1, 25
+ vpor xmm1, xmm1, xmm8
+ vpshufd xmm0, xmm0, 0x39
+ vpshufd xmm3, xmm3, 0x4E
+ vpshufd xmm2, xmm2, 0x93
+ dec al
+ jz 9f
+ vshufps xmm8, xmm4, xmm5, 214
+ vpshufd xmm9, xmm4, 0x0F
+ vpshufd xmm4, xmm8, 0x39
+ vshufps xmm8, xmm6, xmm7, 250
+ vpblendd xmm9, xmm9, xmm8, 0xAA
+ vpunpcklqdq xmm8, xmm7, xmm5
+ vpblendd xmm8, xmm8, xmm6, 0x88
+ vpshufd xmm8, xmm8, 0x78
+ vpunpckhdq xmm5, xmm5, xmm7
+ vpunpckldq xmm6, xmm6, xmm5
+ vpshufd xmm7, xmm6, 0x1E
+ vmovdqa xmm5, xmm9
+ vmovdqa xmm6, xmm8
+ jmp 9b
+9:
+ vpxor xmm0, xmm0, xmm2
+ vpxor xmm1, xmm1, xmm3
+ mov eax, r13d
+ cmp rdx, r15
+ jne 2b
+ vmovdqu xmmword ptr [rbx], xmm0
+ vmovdqu xmmword ptr [rbx+0x10], xmm1
+ jmp 4b
+
+
+#ifdef __APPLE__
+.static_data
+#else
+.section .rodata
+#endif
+.p2align 6
+ADD0:
+ .long 0, 1, 2, 3, 4, 5, 6, 7
+ADD1:
+ .long 8, 8, 8, 8, 8, 8, 8, 8
+BLAKE3_IV_0:
+ .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667
+ .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667
+BLAKE3_IV_1:
+ .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85
+ .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85
+BLAKE3_IV_2:
+ .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372
+ .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372
+BLAKE3_IV_3:
+ .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A
+ .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A
+BLAKE3_BLOCK_LEN:
+ .long 0x00000040, 0x00000040, 0x00000040, 0x00000040
+ .long 0x00000040, 0x00000040, 0x00000040, 0x00000040
+ROT16:
+ .byte 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13
+ROT8:
+ .byte 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12
+CMP_MSB_MASK:
+ .long 0x80000000, 0x80000000, 0x80000000, 0x80000000
+ .long 0x80000000, 0x80000000, 0x80000000, 0x80000000
+BLAKE3_IV:
+ .long 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A
+
--- /dev/null
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "blake3_impl.h"
+
+#include <immintrin.h>
+
+/* define it here instead of the header */
+static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF };
+
+#define _mm_shuffle_ps2(a, b, c) \
+ (_mm_castps_si128( \
+ _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c))))
+
+INLINE __m128i loadu_128(const uint8_t src[16]) {
+ return _mm_loadu_si128((const __m128i *)src);
+}
+
+INLINE __m256i loadu_256(const uint8_t src[32]) {
+ return _mm256_loadu_si256((const __m256i *)src);
+}
+
+INLINE __m512i loadu_512(const uint8_t src[64]) {
+ return _mm512_loadu_si512((const __m512i *)src);
+}
+
+INLINE void storeu_128(__m128i src, uint8_t dest[16]) {
+ _mm_storeu_si128((__m128i *)dest, src);
+}
+
+INLINE void storeu_256(__m256i src, uint8_t dest[16]) {
+ _mm256_storeu_si256((__m256i *)dest, src);
+}
+
+INLINE __m128i add_128(__m128i a, __m128i b) { return _mm_add_epi32(a, b); }
+
+INLINE __m256i add_256(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); }
+
+INLINE __m512i add_512(__m512i a, __m512i b) { return _mm512_add_epi32(a, b); }
+
+INLINE __m128i xor_128(__m128i a, __m128i b) { return _mm_xor_si128(a, b); }
+
+INLINE __m256i xor_256(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); }
+
+INLINE __m512i xor_512(__m512i a, __m512i b) { return _mm512_xor_si512(a, b); }
+
+INLINE __m128i set1_128(uint32_t x) { return _mm_set1_epi32((int32_t)x); }
+
+INLINE __m256i set1_256(uint32_t x) { return _mm256_set1_epi32((int32_t)x); }
+
+INLINE __m512i set1_512(uint32_t x) { return _mm512_set1_epi32((int32_t)x); }
+
+INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
+ return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d);
+}
+
+INLINE __m128i rot16_128(__m128i x) { return _mm_ror_epi32(x, 16); }
+
+INLINE __m256i rot16_256(__m256i x) { return _mm256_ror_epi32(x, 16); }
+
+INLINE __m512i rot16_512(__m512i x) { return _mm512_ror_epi32(x, 16); }
+
+INLINE __m128i rot12_128(__m128i x) { return _mm_ror_epi32(x, 12); }
+
+INLINE __m256i rot12_256(__m256i x) { return _mm256_ror_epi32(x, 12); }
+
+INLINE __m512i rot12_512(__m512i x) { return _mm512_ror_epi32(x, 12); }
+
+INLINE __m128i rot8_128(__m128i x) { return _mm_ror_epi32(x, 8); }
+
+INLINE __m256i rot8_256(__m256i x) { return _mm256_ror_epi32(x, 8); }
+
+INLINE __m512i rot8_512(__m512i x) { return _mm512_ror_epi32(x, 8); }
+
+INLINE __m128i rot7_128(__m128i x) { return _mm_ror_epi32(x, 7); }
+
+INLINE __m256i rot7_256(__m256i x) { return _mm256_ror_epi32(x, 7); }
+
+INLINE __m512i rot7_512(__m512i x) { return _mm512_ror_epi32(x, 7); }
+
+/*
+ * ----------------------------------------------------------------------------
+ * compress_avx512
+ * ----------------------------------------------------------------------------
+ */
+
+INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
+ __m128i m) {
+ *row0 = add_128(add_128(*row0, m), *row1);
+ *row3 = xor_128(*row3, *row0);
+ *row3 = rot16_128(*row3);
+ *row2 = add_128(*row2, *row3);
+ *row1 = xor_128(*row1, *row2);
+ *row1 = rot12_128(*row1);
+}
+
+INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
+ __m128i m) {
+ *row0 = add_128(add_128(*row0, m), *row1);
+ *row3 = xor_128(*row3, *row0);
+ *row3 = rot8_128(*row3);
+ *row2 = add_128(*row2, *row3);
+ *row1 = xor_128(*row1, *row2);
+ *row1 = rot7_128(*row1);
+}
+
+// Note the optimization here of leaving row1 as the unrotated row, rather than
+// row0. All the message loads below are adjusted to compensate for this. See
+// discussion at https://github.com/sneves/blake2-avx2/pull/4
+INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) {
+ *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
+ *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
+ *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
+}
+
+INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) {
+ *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
+ *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
+ *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
+}
+
+INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter, uint8_t flags) {
+ rows[0] = loadu_128((uint8_t *)&cv[0]);
+ rows[1] = loadu_128((uint8_t *)&cv[4]);
+ rows[2] = set4(B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3);
+ rows[3] = set4(counter_low(counter), counter_high(counter),
+ (uint32_t)block_len, (uint32_t)flags);
+
+ __m128i m0 = loadu_128(&block[sizeof(__m128i) * 0]);
+ __m128i m1 = loadu_128(&block[sizeof(__m128i) * 1]);
+ __m128i m2 = loadu_128(&block[sizeof(__m128i) * 2]);
+ __m128i m3 = loadu_128(&block[sizeof(__m128i) * 3]);
+
+ __m128i t0, t1, t2, t3, tt;
+
+ // Round 1. The first round permutes the message words from the original
+ // input order, into the groups that get mixed in parallel.
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8
+ t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9
+ t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 2. This round and all following rounds apply a fixed permutation
+ // to the message words from the round before.
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 3
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 4
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 5
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 6
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 7
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+}
+
+void blake3_compress_xof_avx512(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags, uint8_t out[64]) {
+ __m128i rows[4];
+ compress_pre(rows, cv, block, block_len, counter, flags);
+ storeu_128(xor_128(rows[0], rows[2]), &out[0]);
+ storeu_128(xor_128(rows[1], rows[3]), &out[16]);
+ storeu_128(xor_128(rows[2], loadu_128((uint8_t *)&cv[0])), &out[32]);
+ storeu_128(xor_128(rows[3], loadu_128((uint8_t *)&cv[4])), &out[48]);
+}
+
+void blake3_compress_in_place_avx512(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags) {
+ __m128i rows[4];
+ compress_pre(rows, cv, block, block_len, counter, flags);
+ storeu_128(xor_128(rows[0], rows[2]), (uint8_t *)&cv[0]);
+ storeu_128(xor_128(rows[1], rows[3]), (uint8_t *)&cv[4]);
+}
+
+/*
+ * ----------------------------------------------------------------------------
+ * hash4_avx512
+ * ----------------------------------------------------------------------------
+ */
+
+INLINE void round_fn4(__m128i v[16], __m128i m[16], size_t r) {
+ v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
+ v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
+ v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
+ v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
+ v[0] = add_128(v[0], v[4]);
+ v[1] = add_128(v[1], v[5]);
+ v[2] = add_128(v[2], v[6]);
+ v[3] = add_128(v[3], v[7]);
+ v[12] = xor_128(v[12], v[0]);
+ v[13] = xor_128(v[13], v[1]);
+ v[14] = xor_128(v[14], v[2]);
+ v[15] = xor_128(v[15], v[3]);
+ v[12] = rot16_128(v[12]);
+ v[13] = rot16_128(v[13]);
+ v[14] = rot16_128(v[14]);
+ v[15] = rot16_128(v[15]);
+ v[8] = add_128(v[8], v[12]);
+ v[9] = add_128(v[9], v[13]);
+ v[10] = add_128(v[10], v[14]);
+ v[11] = add_128(v[11], v[15]);
+ v[4] = xor_128(v[4], v[8]);
+ v[5] = xor_128(v[5], v[9]);
+ v[6] = xor_128(v[6], v[10]);
+ v[7] = xor_128(v[7], v[11]);
+ v[4] = rot12_128(v[4]);
+ v[5] = rot12_128(v[5]);
+ v[6] = rot12_128(v[6]);
+ v[7] = rot12_128(v[7]);
+ v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
+ v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
+ v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
+ v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
+ v[0] = add_128(v[0], v[4]);
+ v[1] = add_128(v[1], v[5]);
+ v[2] = add_128(v[2], v[6]);
+ v[3] = add_128(v[3], v[7]);
+ v[12] = xor_128(v[12], v[0]);
+ v[13] = xor_128(v[13], v[1]);
+ v[14] = xor_128(v[14], v[2]);
+ v[15] = xor_128(v[15], v[3]);
+ v[12] = rot8_128(v[12]);
+ v[13] = rot8_128(v[13]);
+ v[14] = rot8_128(v[14]);
+ v[15] = rot8_128(v[15]);
+ v[8] = add_128(v[8], v[12]);
+ v[9] = add_128(v[9], v[13]);
+ v[10] = add_128(v[10], v[14]);
+ v[11] = add_128(v[11], v[15]);
+ v[4] = xor_128(v[4], v[8]);
+ v[5] = xor_128(v[5], v[9]);
+ v[6] = xor_128(v[6], v[10]);
+ v[7] = xor_128(v[7], v[11]);
+ v[4] = rot7_128(v[4]);
+ v[5] = rot7_128(v[5]);
+ v[6] = rot7_128(v[6]);
+ v[7] = rot7_128(v[7]);
+
+ v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
+ v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
+ v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
+ v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
+ v[0] = add_128(v[0], v[5]);
+ v[1] = add_128(v[1], v[6]);
+ v[2] = add_128(v[2], v[7]);
+ v[3] = add_128(v[3], v[4]);
+ v[15] = xor_128(v[15], v[0]);
+ v[12] = xor_128(v[12], v[1]);
+ v[13] = xor_128(v[13], v[2]);
+ v[14] = xor_128(v[14], v[3]);
+ v[15] = rot16_128(v[15]);
+ v[12] = rot16_128(v[12]);
+ v[13] = rot16_128(v[13]);
+ v[14] = rot16_128(v[14]);
+ v[10] = add_128(v[10], v[15]);
+ v[11] = add_128(v[11], v[12]);
+ v[8] = add_128(v[8], v[13]);
+ v[9] = add_128(v[9], v[14]);
+ v[5] = xor_128(v[5], v[10]);
+ v[6] = xor_128(v[6], v[11]);
+ v[7] = xor_128(v[7], v[8]);
+ v[4] = xor_128(v[4], v[9]);
+ v[5] = rot12_128(v[5]);
+ v[6] = rot12_128(v[6]);
+ v[7] = rot12_128(v[7]);
+ v[4] = rot12_128(v[4]);
+ v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
+ v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
+ v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
+ v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
+ v[0] = add_128(v[0], v[5]);
+ v[1] = add_128(v[1], v[6]);
+ v[2] = add_128(v[2], v[7]);
+ v[3] = add_128(v[3], v[4]);
+ v[15] = xor_128(v[15], v[0]);
+ v[12] = xor_128(v[12], v[1]);
+ v[13] = xor_128(v[13], v[2]);
+ v[14] = xor_128(v[14], v[3]);
+ v[15] = rot8_128(v[15]);
+ v[12] = rot8_128(v[12]);
+ v[13] = rot8_128(v[13]);
+ v[14] = rot8_128(v[14]);
+ v[10] = add_128(v[10], v[15]);
+ v[11] = add_128(v[11], v[12]);
+ v[8] = add_128(v[8], v[13]);
+ v[9] = add_128(v[9], v[14]);
+ v[5] = xor_128(v[5], v[10]);
+ v[6] = xor_128(v[6], v[11]);
+ v[7] = xor_128(v[7], v[8]);
+ v[4] = xor_128(v[4], v[9]);
+ v[5] = rot7_128(v[5]);
+ v[6] = rot7_128(v[6]);
+ v[7] = rot7_128(v[7]);
+ v[4] = rot7_128(v[4]);
+}
+
+INLINE void transpose_vecs_128(__m128i vecs[4]) {
+ // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is
+ // 22/33. Note that this doesn't split the vector into two lanes, as the
+ // AVX2 counterparts do.
+ __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
+ __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
+ __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
+ __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
+
+ // Interleave 64-bit lanes.
+ __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01);
+ __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01);
+ __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23);
+ __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23);
+
+ vecs[0] = abcd_0;
+ vecs[1] = abcd_1;
+ vecs[2] = abcd_2;
+ vecs[3] = abcd_3;
+}
+
+INLINE void transpose_msg_vecs4(const uint8_t *const *inputs,
+ size_t block_offset, __m128i out[16]) {
+ out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(__m128i)]);
+ out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(__m128i)]);
+ out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(__m128i)]);
+ out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(__m128i)]);
+ out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(__m128i)]);
+ out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(__m128i)]);
+ out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(__m128i)]);
+ out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(__m128i)]);
+ out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(__m128i)]);
+ out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(__m128i)]);
+ out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(__m128i)]);
+ out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(__m128i)]);
+ out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(__m128i)]);
+ out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(__m128i)]);
+ out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(__m128i)]);
+ out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(__m128i)]);
+ for (size_t i = 0; i < 4; ++i) {
+ _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
+ }
+ transpose_vecs_128(&out[0]);
+ transpose_vecs_128(&out[4]);
+ transpose_vecs_128(&out[8]);
+ transpose_vecs_128(&out[12]);
+}
+
+INLINE void load_counters4(uint64_t counter, bool increment_counter,
+ __m128i *out_lo, __m128i *out_hi) {
+ uint64_t mask = (increment_counter ? ~0 : 0);
+ __m256i mask_vec = _mm256_set1_epi64x(mask);
+ __m256i deltas = _mm256_setr_epi64x(0, 1, 2, 3);
+ deltas = _mm256_and_si256(mask_vec, deltas);
+ __m256i counters =
+ _mm256_add_epi64(_mm256_set1_epi64x((int64_t)counter), deltas);
+ *out_lo = _mm256_cvtepi64_epi32(counters);
+ *out_hi = _mm256_cvtepi64_epi32(_mm256_srli_epi64(counters, 32));
+}
+
+static
+void blake3_hash4_avx512(const uint8_t *const *inputs, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ bool increment_counter, uint8_t flags,
+ uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
+ __m128i h_vecs[8] = {
+ set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]),
+ set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]),
+ };
+ __m128i counter_low_vec, counter_high_vec;
+ load_counters4(counter, increment_counter, &counter_low_vec,
+ &counter_high_vec);
+ uint8_t block_flags = flags | flags_start;
+
+ for (size_t block = 0; block < blocks; block++) {
+ if (block + 1 == blocks) {
+ block_flags |= flags_end;
+ }
+ __m128i block_len_vec = set1_128(BLAKE3_BLOCK_LEN);
+ __m128i block_flags_vec = set1_128(block_flags);
+ __m128i msg_vecs[16];
+ transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
+
+ __m128i v[16] = {
+ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
+ h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
+ set1_128(B3_IV_0), set1_128(B3_IV_1), set1_128(B3_IV_2), set1_128(B3_IV_3),
+ counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
+ };
+ round_fn4(v, msg_vecs, 0);
+ round_fn4(v, msg_vecs, 1);
+ round_fn4(v, msg_vecs, 2);
+ round_fn4(v, msg_vecs, 3);
+ round_fn4(v, msg_vecs, 4);
+ round_fn4(v, msg_vecs, 5);
+ round_fn4(v, msg_vecs, 6);
+ h_vecs[0] = xor_128(v[0], v[8]);
+ h_vecs[1] = xor_128(v[1], v[9]);
+ h_vecs[2] = xor_128(v[2], v[10]);
+ h_vecs[3] = xor_128(v[3], v[11]);
+ h_vecs[4] = xor_128(v[4], v[12]);
+ h_vecs[5] = xor_128(v[5], v[13]);
+ h_vecs[6] = xor_128(v[6], v[14]);
+ h_vecs[7] = xor_128(v[7], v[15]);
+
+ block_flags = flags;
+ }
+
+ transpose_vecs_128(&h_vecs[0]);
+ transpose_vecs_128(&h_vecs[4]);
+ // The first four vecs now contain the first half of each output, and the
+ // second four vecs contain the second half of each output.
+ storeu_128(h_vecs[0], &out[0 * sizeof(__m128i)]);
+ storeu_128(h_vecs[4], &out[1 * sizeof(__m128i)]);
+ storeu_128(h_vecs[1], &out[2 * sizeof(__m128i)]);
+ storeu_128(h_vecs[5], &out[3 * sizeof(__m128i)]);
+ storeu_128(h_vecs[2], &out[4 * sizeof(__m128i)]);
+ storeu_128(h_vecs[6], &out[5 * sizeof(__m128i)]);
+ storeu_128(h_vecs[3], &out[6 * sizeof(__m128i)]);
+ storeu_128(h_vecs[7], &out[7 * sizeof(__m128i)]);
+}
+
+/*
+ * ----------------------------------------------------------------------------
+ * hash8_avx512
+ * ----------------------------------------------------------------------------
+ */
+
+INLINE void round_fn8(__m256i v[16], __m256i m[16], size_t r) {
+ v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
+ v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
+ v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
+ v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
+ v[0] = add_256(v[0], v[4]);
+ v[1] = add_256(v[1], v[5]);
+ v[2] = add_256(v[2], v[6]);
+ v[3] = add_256(v[3], v[7]);
+ v[12] = xor_256(v[12], v[0]);
+ v[13] = xor_256(v[13], v[1]);
+ v[14] = xor_256(v[14], v[2]);
+ v[15] = xor_256(v[15], v[3]);
+ v[12] = rot16_256(v[12]);
+ v[13] = rot16_256(v[13]);
+ v[14] = rot16_256(v[14]);
+ v[15] = rot16_256(v[15]);
+ v[8] = add_256(v[8], v[12]);
+ v[9] = add_256(v[9], v[13]);
+ v[10] = add_256(v[10], v[14]);
+ v[11] = add_256(v[11], v[15]);
+ v[4] = xor_256(v[4], v[8]);
+ v[5] = xor_256(v[5], v[9]);
+ v[6] = xor_256(v[6], v[10]);
+ v[7] = xor_256(v[7], v[11]);
+ v[4] = rot12_256(v[4]);
+ v[5] = rot12_256(v[5]);
+ v[6] = rot12_256(v[6]);
+ v[7] = rot12_256(v[7]);
+ v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
+ v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
+ v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
+ v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
+ v[0] = add_256(v[0], v[4]);
+ v[1] = add_256(v[1], v[5]);
+ v[2] = add_256(v[2], v[6]);
+ v[3] = add_256(v[3], v[7]);
+ v[12] = xor_256(v[12], v[0]);
+ v[13] = xor_256(v[13], v[1]);
+ v[14] = xor_256(v[14], v[2]);
+ v[15] = xor_256(v[15], v[3]);
+ v[12] = rot8_256(v[12]);
+ v[13] = rot8_256(v[13]);
+ v[14] = rot8_256(v[14]);
+ v[15] = rot8_256(v[15]);
+ v[8] = add_256(v[8], v[12]);
+ v[9] = add_256(v[9], v[13]);
+ v[10] = add_256(v[10], v[14]);
+ v[11] = add_256(v[11], v[15]);
+ v[4] = xor_256(v[4], v[8]);
+ v[5] = xor_256(v[5], v[9]);
+ v[6] = xor_256(v[6], v[10]);
+ v[7] = xor_256(v[7], v[11]);
+ v[4] = rot7_256(v[4]);
+ v[5] = rot7_256(v[5]);
+ v[6] = rot7_256(v[6]);
+ v[7] = rot7_256(v[7]);
+
+ v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
+ v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
+ v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
+ v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
+ v[0] = add_256(v[0], v[5]);
+ v[1] = add_256(v[1], v[6]);
+ v[2] = add_256(v[2], v[7]);
+ v[3] = add_256(v[3], v[4]);
+ v[15] = xor_256(v[15], v[0]);
+ v[12] = xor_256(v[12], v[1]);
+ v[13] = xor_256(v[13], v[2]);
+ v[14] = xor_256(v[14], v[3]);
+ v[15] = rot16_256(v[15]);
+ v[12] = rot16_256(v[12]);
+ v[13] = rot16_256(v[13]);
+ v[14] = rot16_256(v[14]);
+ v[10] = add_256(v[10], v[15]);
+ v[11] = add_256(v[11], v[12]);
+ v[8] = add_256(v[8], v[13]);
+ v[9] = add_256(v[9], v[14]);
+ v[5] = xor_256(v[5], v[10]);
+ v[6] = xor_256(v[6], v[11]);
+ v[7] = xor_256(v[7], v[8]);
+ v[4] = xor_256(v[4], v[9]);
+ v[5] = rot12_256(v[5]);
+ v[6] = rot12_256(v[6]);
+ v[7] = rot12_256(v[7]);
+ v[4] = rot12_256(v[4]);
+ v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
+ v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
+ v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
+ v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
+ v[0] = add_256(v[0], v[5]);
+ v[1] = add_256(v[1], v[6]);
+ v[2] = add_256(v[2], v[7]);
+ v[3] = add_256(v[3], v[4]);
+ v[15] = xor_256(v[15], v[0]);
+ v[12] = xor_256(v[12], v[1]);
+ v[13] = xor_256(v[13], v[2]);
+ v[14] = xor_256(v[14], v[3]);
+ v[15] = rot8_256(v[15]);
+ v[12] = rot8_256(v[12]);
+ v[13] = rot8_256(v[13]);
+ v[14] = rot8_256(v[14]);
+ v[10] = add_256(v[10], v[15]);
+ v[11] = add_256(v[11], v[12]);
+ v[8] = add_256(v[8], v[13]);
+ v[9] = add_256(v[9], v[14]);
+ v[5] = xor_256(v[5], v[10]);
+ v[6] = xor_256(v[6], v[11]);
+ v[7] = xor_256(v[7], v[8]);
+ v[4] = xor_256(v[4], v[9]);
+ v[5] = rot7_256(v[5]);
+ v[6] = rot7_256(v[6]);
+ v[7] = rot7_256(v[7]);
+ v[4] = rot7_256(v[4]);
+}
+
+INLINE void transpose_vecs_256(__m256i vecs[8]) {
+ // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high
+ // is 22/33/66/77.
+ __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]);
+ __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]);
+ __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]);
+ __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]);
+ __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]);
+ __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]);
+ __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]);
+ __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]);
+
+ // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is
+ // 11/33.
+ __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145);
+ __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145);
+ __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367);
+ __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367);
+ __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145);
+ __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145);
+ __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367);
+ __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367);
+
+ // Interleave 128-bit lanes.
+ vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20);
+ vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20);
+ vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20);
+ vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20);
+ vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31);
+ vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31);
+ vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31);
+ vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31);
+}
+
+INLINE void transpose_msg_vecs8(const uint8_t *const *inputs,
+ size_t block_offset, __m256i out[16]) {
+ out[0] = loadu_256(&inputs[0][block_offset + 0 * sizeof(__m256i)]);
+ out[1] = loadu_256(&inputs[1][block_offset + 0 * sizeof(__m256i)]);
+ out[2] = loadu_256(&inputs[2][block_offset + 0 * sizeof(__m256i)]);
+ out[3] = loadu_256(&inputs[3][block_offset + 0 * sizeof(__m256i)]);
+ out[4] = loadu_256(&inputs[4][block_offset + 0 * sizeof(__m256i)]);
+ out[5] = loadu_256(&inputs[5][block_offset + 0 * sizeof(__m256i)]);
+ out[6] = loadu_256(&inputs[6][block_offset + 0 * sizeof(__m256i)]);
+ out[7] = loadu_256(&inputs[7][block_offset + 0 * sizeof(__m256i)]);
+ out[8] = loadu_256(&inputs[0][block_offset + 1 * sizeof(__m256i)]);
+ out[9] = loadu_256(&inputs[1][block_offset + 1 * sizeof(__m256i)]);
+ out[10] = loadu_256(&inputs[2][block_offset + 1 * sizeof(__m256i)]);
+ out[11] = loadu_256(&inputs[3][block_offset + 1 * sizeof(__m256i)]);
+ out[12] = loadu_256(&inputs[4][block_offset + 1 * sizeof(__m256i)]);
+ out[13] = loadu_256(&inputs[5][block_offset + 1 * sizeof(__m256i)]);
+ out[14] = loadu_256(&inputs[6][block_offset + 1 * sizeof(__m256i)]);
+ out[15] = loadu_256(&inputs[7][block_offset + 1 * sizeof(__m256i)]);
+ for (size_t i = 0; i < 8; ++i) {
+ _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
+ }
+ transpose_vecs_256(&out[0]);
+ transpose_vecs_256(&out[8]);
+}
+
+INLINE void load_counters8(uint64_t counter, bool increment_counter,
+ __m256i *out_lo, __m256i *out_hi) {
+ uint64_t mask = (increment_counter ? ~0 : 0);
+ __m512i mask_vec = _mm512_set1_epi64(mask);
+ __m512i deltas = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7);
+ deltas = _mm512_and_si512(mask_vec, deltas);
+ __m512i counters =
+ _mm512_add_epi64(_mm512_set1_epi64((int64_t)counter), deltas);
+ *out_lo = _mm512_cvtepi64_epi32(counters);
+ *out_hi = _mm512_cvtepi64_epi32(_mm512_srli_epi64(counters, 32));
+}
+
+static
+void blake3_hash8_avx512(const uint8_t *const *inputs, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ bool increment_counter, uint8_t flags,
+ uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
+ __m256i h_vecs[8] = {
+ set1_256(key[0]), set1_256(key[1]), set1_256(key[2]), set1_256(key[3]),
+ set1_256(key[4]), set1_256(key[5]), set1_256(key[6]), set1_256(key[7]),
+ };
+ __m256i counter_low_vec, counter_high_vec;
+ load_counters8(counter, increment_counter, &counter_low_vec,
+ &counter_high_vec);
+ uint8_t block_flags = flags | flags_start;
+
+ for (size_t block = 0; block < blocks; block++) {
+ if (block + 1 == blocks) {
+ block_flags |= flags_end;
+ }
+ __m256i block_len_vec = set1_256(BLAKE3_BLOCK_LEN);
+ __m256i block_flags_vec = set1_256(block_flags);
+ __m256i msg_vecs[16];
+ transpose_msg_vecs8(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
+
+ __m256i v[16] = {
+ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
+ h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
+ set1_256(B3_IV_0), set1_256(B3_IV_1), set1_256(B3_IV_2), set1_256(B3_IV_3),
+ counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
+ };
+ round_fn8(v, msg_vecs, 0);
+ round_fn8(v, msg_vecs, 1);
+ round_fn8(v, msg_vecs, 2);
+ round_fn8(v, msg_vecs, 3);
+ round_fn8(v, msg_vecs, 4);
+ round_fn8(v, msg_vecs, 5);
+ round_fn8(v, msg_vecs, 6);
+ h_vecs[0] = xor_256(v[0], v[8]);
+ h_vecs[1] = xor_256(v[1], v[9]);
+ h_vecs[2] = xor_256(v[2], v[10]);
+ h_vecs[3] = xor_256(v[3], v[11]);
+ h_vecs[4] = xor_256(v[4], v[12]);
+ h_vecs[5] = xor_256(v[5], v[13]);
+ h_vecs[6] = xor_256(v[6], v[14]);
+ h_vecs[7] = xor_256(v[7], v[15]);
+
+ block_flags = flags;
+ }
+
+ transpose_vecs_256(h_vecs);
+ storeu_256(h_vecs[0], &out[0 * sizeof(__m256i)]);
+ storeu_256(h_vecs[1], &out[1 * sizeof(__m256i)]);
+ storeu_256(h_vecs[2], &out[2 * sizeof(__m256i)]);
+ storeu_256(h_vecs[3], &out[3 * sizeof(__m256i)]);
+ storeu_256(h_vecs[4], &out[4 * sizeof(__m256i)]);
+ storeu_256(h_vecs[5], &out[5 * sizeof(__m256i)]);
+ storeu_256(h_vecs[6], &out[6 * sizeof(__m256i)]);
+ storeu_256(h_vecs[7], &out[7 * sizeof(__m256i)]);
+}
+
+/*
+ * ----------------------------------------------------------------------------
+ * hash16_avx512
+ * ----------------------------------------------------------------------------
+ */
+
+INLINE void round_fn16(__m512i v[16], __m512i m[16], size_t r) {
+ v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
+ v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
+ v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
+ v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
+ v[0] = add_512(v[0], v[4]);
+ v[1] = add_512(v[1], v[5]);
+ v[2] = add_512(v[2], v[6]);
+ v[3] = add_512(v[3], v[7]);
+ v[12] = xor_512(v[12], v[0]);
+ v[13] = xor_512(v[13], v[1]);
+ v[14] = xor_512(v[14], v[2]);
+ v[15] = xor_512(v[15], v[3]);
+ v[12] = rot16_512(v[12]);
+ v[13] = rot16_512(v[13]);
+ v[14] = rot16_512(v[14]);
+ v[15] = rot16_512(v[15]);
+ v[8] = add_512(v[8], v[12]);
+ v[9] = add_512(v[9], v[13]);
+ v[10] = add_512(v[10], v[14]);
+ v[11] = add_512(v[11], v[15]);
+ v[4] = xor_512(v[4], v[8]);
+ v[5] = xor_512(v[5], v[9]);
+ v[6] = xor_512(v[6], v[10]);
+ v[7] = xor_512(v[7], v[11]);
+ v[4] = rot12_512(v[4]);
+ v[5] = rot12_512(v[5]);
+ v[6] = rot12_512(v[6]);
+ v[7] = rot12_512(v[7]);
+ v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
+ v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
+ v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
+ v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
+ v[0] = add_512(v[0], v[4]);
+ v[1] = add_512(v[1], v[5]);
+ v[2] = add_512(v[2], v[6]);
+ v[3] = add_512(v[3], v[7]);
+ v[12] = xor_512(v[12], v[0]);
+ v[13] = xor_512(v[13], v[1]);
+ v[14] = xor_512(v[14], v[2]);
+ v[15] = xor_512(v[15], v[3]);
+ v[12] = rot8_512(v[12]);
+ v[13] = rot8_512(v[13]);
+ v[14] = rot8_512(v[14]);
+ v[15] = rot8_512(v[15]);
+ v[8] = add_512(v[8], v[12]);
+ v[9] = add_512(v[9], v[13]);
+ v[10] = add_512(v[10], v[14]);
+ v[11] = add_512(v[11], v[15]);
+ v[4] = xor_512(v[4], v[8]);
+ v[5] = xor_512(v[5], v[9]);
+ v[6] = xor_512(v[6], v[10]);
+ v[7] = xor_512(v[7], v[11]);
+ v[4] = rot7_512(v[4]);
+ v[5] = rot7_512(v[5]);
+ v[6] = rot7_512(v[6]);
+ v[7] = rot7_512(v[7]);
+
+ v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
+ v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
+ v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
+ v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
+ v[0] = add_512(v[0], v[5]);
+ v[1] = add_512(v[1], v[6]);
+ v[2] = add_512(v[2], v[7]);
+ v[3] = add_512(v[3], v[4]);
+ v[15] = xor_512(v[15], v[0]);
+ v[12] = xor_512(v[12], v[1]);
+ v[13] = xor_512(v[13], v[2]);
+ v[14] = xor_512(v[14], v[3]);
+ v[15] = rot16_512(v[15]);
+ v[12] = rot16_512(v[12]);
+ v[13] = rot16_512(v[13]);
+ v[14] = rot16_512(v[14]);
+ v[10] = add_512(v[10], v[15]);
+ v[11] = add_512(v[11], v[12]);
+ v[8] = add_512(v[8], v[13]);
+ v[9] = add_512(v[9], v[14]);
+ v[5] = xor_512(v[5], v[10]);
+ v[6] = xor_512(v[6], v[11]);
+ v[7] = xor_512(v[7], v[8]);
+ v[4] = xor_512(v[4], v[9]);
+ v[5] = rot12_512(v[5]);
+ v[6] = rot12_512(v[6]);
+ v[7] = rot12_512(v[7]);
+ v[4] = rot12_512(v[4]);
+ v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
+ v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
+ v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
+ v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
+ v[0] = add_512(v[0], v[5]);
+ v[1] = add_512(v[1], v[6]);
+ v[2] = add_512(v[2], v[7]);
+ v[3] = add_512(v[3], v[4]);
+ v[15] = xor_512(v[15], v[0]);
+ v[12] = xor_512(v[12], v[1]);
+ v[13] = xor_512(v[13], v[2]);
+ v[14] = xor_512(v[14], v[3]);
+ v[15] = rot8_512(v[15]);
+ v[12] = rot8_512(v[12]);
+ v[13] = rot8_512(v[13]);
+ v[14] = rot8_512(v[14]);
+ v[10] = add_512(v[10], v[15]);
+ v[11] = add_512(v[11], v[12]);
+ v[8] = add_512(v[8], v[13]);
+ v[9] = add_512(v[9], v[14]);
+ v[5] = xor_512(v[5], v[10]);
+ v[6] = xor_512(v[6], v[11]);
+ v[7] = xor_512(v[7], v[8]);
+ v[4] = xor_512(v[4], v[9]);
+ v[5] = rot7_512(v[5]);
+ v[6] = rot7_512(v[6]);
+ v[7] = rot7_512(v[7]);
+ v[4] = rot7_512(v[4]);
+}
+
+// 0b10001000, or lanes a0/a2/b0/b2 in little-endian order
+#define LO_IMM8 0x88
+
+INLINE __m512i unpack_lo_128(__m512i a, __m512i b) {
+ return _mm512_shuffle_i32x4(a, b, LO_IMM8);
+}
+
+// 0b11011101, or lanes a1/a3/b1/b3 in little-endian order
+#define HI_IMM8 0xdd
+
+INLINE __m512i unpack_hi_128(__m512i a, __m512i b) {
+ return _mm512_shuffle_i32x4(a, b, HI_IMM8);
+}
+
+INLINE void transpose_vecs_512(__m512i vecs[16]) {
+ // Interleave 32-bit lanes. The _0 unpack is lanes
+ // 0/0/1/1/4/4/5/5/8/8/9/9/12/12/13/13, and the _2 unpack is lanes
+ // 2/2/3/3/6/6/7/7/10/10/11/11/14/14/15/15.
+ __m512i ab_0 = _mm512_unpacklo_epi32(vecs[0], vecs[1]);
+ __m512i ab_2 = _mm512_unpackhi_epi32(vecs[0], vecs[1]);
+ __m512i cd_0 = _mm512_unpacklo_epi32(vecs[2], vecs[3]);
+ __m512i cd_2 = _mm512_unpackhi_epi32(vecs[2], vecs[3]);
+ __m512i ef_0 = _mm512_unpacklo_epi32(vecs[4], vecs[5]);
+ __m512i ef_2 = _mm512_unpackhi_epi32(vecs[4], vecs[5]);
+ __m512i gh_0 = _mm512_unpacklo_epi32(vecs[6], vecs[7]);
+ __m512i gh_2 = _mm512_unpackhi_epi32(vecs[6], vecs[7]);
+ __m512i ij_0 = _mm512_unpacklo_epi32(vecs[8], vecs[9]);
+ __m512i ij_2 = _mm512_unpackhi_epi32(vecs[8], vecs[9]);
+ __m512i kl_0 = _mm512_unpacklo_epi32(vecs[10], vecs[11]);
+ __m512i kl_2 = _mm512_unpackhi_epi32(vecs[10], vecs[11]);
+ __m512i mn_0 = _mm512_unpacklo_epi32(vecs[12], vecs[13]);
+ __m512i mn_2 = _mm512_unpackhi_epi32(vecs[12], vecs[13]);
+ __m512i op_0 = _mm512_unpacklo_epi32(vecs[14], vecs[15]);
+ __m512i op_2 = _mm512_unpackhi_epi32(vecs[14], vecs[15]);
+
+ // Interleave 64-bit lanes. The _0 unpack is lanes
+ // 0/0/0/0/4/4/4/4/8/8/8/8/12/12/12/12, the _1 unpack is lanes
+ // 1/1/1/1/5/5/5/5/9/9/9/9/13/13/13/13, the _2 unpack is lanes
+ // 2/2/2/2/6/6/6/6/10/10/10/10/14/14/14/14, and the _3 unpack is lanes
+ // 3/3/3/3/7/7/7/7/11/11/11/11/15/15/15/15.
+ __m512i abcd_0 = _mm512_unpacklo_epi64(ab_0, cd_0);
+ __m512i abcd_1 = _mm512_unpackhi_epi64(ab_0, cd_0);
+ __m512i abcd_2 = _mm512_unpacklo_epi64(ab_2, cd_2);
+ __m512i abcd_3 = _mm512_unpackhi_epi64(ab_2, cd_2);
+ __m512i efgh_0 = _mm512_unpacklo_epi64(ef_0, gh_0);
+ __m512i efgh_1 = _mm512_unpackhi_epi64(ef_0, gh_0);
+ __m512i efgh_2 = _mm512_unpacklo_epi64(ef_2, gh_2);
+ __m512i efgh_3 = _mm512_unpackhi_epi64(ef_2, gh_2);
+ __m512i ijkl_0 = _mm512_unpacklo_epi64(ij_0, kl_0);
+ __m512i ijkl_1 = _mm512_unpackhi_epi64(ij_0, kl_0);
+ __m512i ijkl_2 = _mm512_unpacklo_epi64(ij_2, kl_2);
+ __m512i ijkl_3 = _mm512_unpackhi_epi64(ij_2, kl_2);
+ __m512i mnop_0 = _mm512_unpacklo_epi64(mn_0, op_0);
+ __m512i mnop_1 = _mm512_unpackhi_epi64(mn_0, op_0);
+ __m512i mnop_2 = _mm512_unpacklo_epi64(mn_2, op_2);
+ __m512i mnop_3 = _mm512_unpackhi_epi64(mn_2, op_2);
+
+ // Interleave 128-bit lanes. The _0 unpack is
+ // 0/0/0/0/8/8/8/8/0/0/0/0/8/8/8/8, the _1 unpack is
+ // 1/1/1/1/9/9/9/9/1/1/1/1/9/9/9/9, and so on.
+ __m512i abcdefgh_0 = unpack_lo_128(abcd_0, efgh_0);
+ __m512i abcdefgh_1 = unpack_lo_128(abcd_1, efgh_1);
+ __m512i abcdefgh_2 = unpack_lo_128(abcd_2, efgh_2);
+ __m512i abcdefgh_3 = unpack_lo_128(abcd_3, efgh_3);
+ __m512i abcdefgh_4 = unpack_hi_128(abcd_0, efgh_0);
+ __m512i abcdefgh_5 = unpack_hi_128(abcd_1, efgh_1);
+ __m512i abcdefgh_6 = unpack_hi_128(abcd_2, efgh_2);
+ __m512i abcdefgh_7 = unpack_hi_128(abcd_3, efgh_3);
+ __m512i ijklmnop_0 = unpack_lo_128(ijkl_0, mnop_0);
+ __m512i ijklmnop_1 = unpack_lo_128(ijkl_1, mnop_1);
+ __m512i ijklmnop_2 = unpack_lo_128(ijkl_2, mnop_2);
+ __m512i ijklmnop_3 = unpack_lo_128(ijkl_3, mnop_3);
+ __m512i ijklmnop_4 = unpack_hi_128(ijkl_0, mnop_0);
+ __m512i ijklmnop_5 = unpack_hi_128(ijkl_1, mnop_1);
+ __m512i ijklmnop_6 = unpack_hi_128(ijkl_2, mnop_2);
+ __m512i ijklmnop_7 = unpack_hi_128(ijkl_3, mnop_3);
+
+ // Interleave 128-bit lanes again for the final outputs.
+ vecs[0] = unpack_lo_128(abcdefgh_0, ijklmnop_0);
+ vecs[1] = unpack_lo_128(abcdefgh_1, ijklmnop_1);
+ vecs[2] = unpack_lo_128(abcdefgh_2, ijklmnop_2);
+ vecs[3] = unpack_lo_128(abcdefgh_3, ijklmnop_3);
+ vecs[4] = unpack_lo_128(abcdefgh_4, ijklmnop_4);
+ vecs[5] = unpack_lo_128(abcdefgh_5, ijklmnop_5);
+ vecs[6] = unpack_lo_128(abcdefgh_6, ijklmnop_6);
+ vecs[7] = unpack_lo_128(abcdefgh_7, ijklmnop_7);
+ vecs[8] = unpack_hi_128(abcdefgh_0, ijklmnop_0);
+ vecs[9] = unpack_hi_128(abcdefgh_1, ijklmnop_1);
+ vecs[10] = unpack_hi_128(abcdefgh_2, ijklmnop_2);
+ vecs[11] = unpack_hi_128(abcdefgh_3, ijklmnop_3);
+ vecs[12] = unpack_hi_128(abcdefgh_4, ijklmnop_4);
+ vecs[13] = unpack_hi_128(abcdefgh_5, ijklmnop_5);
+ vecs[14] = unpack_hi_128(abcdefgh_6, ijklmnop_6);
+ vecs[15] = unpack_hi_128(abcdefgh_7, ijklmnop_7);
+}
+
+INLINE void transpose_msg_vecs16(const uint8_t *const *inputs,
+ size_t block_offset, __m512i out[16]) {
+ out[0] = loadu_512(&inputs[0][block_offset]);
+ out[1] = loadu_512(&inputs[1][block_offset]);
+ out[2] = loadu_512(&inputs[2][block_offset]);
+ out[3] = loadu_512(&inputs[3][block_offset]);
+ out[4] = loadu_512(&inputs[4][block_offset]);
+ out[5] = loadu_512(&inputs[5][block_offset]);
+ out[6] = loadu_512(&inputs[6][block_offset]);
+ out[7] = loadu_512(&inputs[7][block_offset]);
+ out[8] = loadu_512(&inputs[8][block_offset]);
+ out[9] = loadu_512(&inputs[9][block_offset]);
+ out[10] = loadu_512(&inputs[10][block_offset]);
+ out[11] = loadu_512(&inputs[11][block_offset]);
+ out[12] = loadu_512(&inputs[12][block_offset]);
+ out[13] = loadu_512(&inputs[13][block_offset]);
+ out[14] = loadu_512(&inputs[14][block_offset]);
+ out[15] = loadu_512(&inputs[15][block_offset]);
+ for (size_t i = 0; i < 16; ++i) {
+ _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
+ }
+ transpose_vecs_512(out);
+}
+
+INLINE void load_counters16(uint64_t counter, bool increment_counter,
+ __m512i *out_lo, __m512i *out_hi) {
+ const __m512i mask = _mm512_set1_epi32(-(int32_t)increment_counter);
+ const __m512i deltas = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
+ const __m512i masked_deltas = _mm512_and_si512(deltas, mask);
+ const __m512i low_words = _mm512_add_epi32(
+ _mm512_set1_epi32((int32_t)counter),
+ masked_deltas);
+ // The carry bit is 1 if the high bit of the word was 1 before addition and is
+ // 0 after.
+ // NOTE: It would be a bit more natural to use _mm512_cmp_epu32_mask to
+ // compute the carry bits here, and originally we did, but that intrinsic is
+ // broken under GCC 5.4. See https://github.com/BLAKE3-team/BLAKE3/issues/271.
+ const __m512i carries = _mm512_srli_epi32(
+ _mm512_andnot_si512(
+ low_words, // 0 after (gets inverted by andnot)
+ _mm512_set1_epi32((int32_t)counter)), // and 1 before
+ 31);
+ const __m512i high_words = _mm512_add_epi32(
+ _mm512_set1_epi32((int32_t)(counter >> 32)),
+ carries);
+ *out_lo = low_words;
+ *out_hi = high_words;
+}
+
+static
+void blake3_hash16_avx512(const uint8_t *const *inputs, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ bool increment_counter, uint8_t flags,
+ uint8_t flags_start, uint8_t flags_end,
+ uint8_t *out) {
+ __m512i h_vecs[8] = {
+ set1_512(key[0]), set1_512(key[1]), set1_512(key[2]), set1_512(key[3]),
+ set1_512(key[4]), set1_512(key[5]), set1_512(key[6]), set1_512(key[7]),
+ };
+ __m512i counter_low_vec, counter_high_vec;
+ load_counters16(counter, increment_counter, &counter_low_vec,
+ &counter_high_vec);
+ uint8_t block_flags = flags | flags_start;
+
+ for (size_t block = 0; block < blocks; block++) {
+ if (block + 1 == blocks) {
+ block_flags |= flags_end;
+ }
+ __m512i block_len_vec = set1_512(BLAKE3_BLOCK_LEN);
+ __m512i block_flags_vec = set1_512(block_flags);
+ __m512i msg_vecs[16];
+ transpose_msg_vecs16(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
+
+ __m512i v[16] = {
+ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
+ h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
+ set1_512(B3_IV_0), set1_512(B3_IV_1), set1_512(B3_IV_2), set1_512(B3_IV_3),
+ counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
+ };
+ round_fn16(v, msg_vecs, 0);
+ round_fn16(v, msg_vecs, 1);
+ round_fn16(v, msg_vecs, 2);
+ round_fn16(v, msg_vecs, 3);
+ round_fn16(v, msg_vecs, 4);
+ round_fn16(v, msg_vecs, 5);
+ round_fn16(v, msg_vecs, 6);
+ h_vecs[0] = xor_512(v[0], v[8]);
+ h_vecs[1] = xor_512(v[1], v[9]);
+ h_vecs[2] = xor_512(v[2], v[10]);
+ h_vecs[3] = xor_512(v[3], v[11]);
+ h_vecs[4] = xor_512(v[4], v[12]);
+ h_vecs[5] = xor_512(v[5], v[13]);
+ h_vecs[6] = xor_512(v[6], v[14]);
+ h_vecs[7] = xor_512(v[7], v[15]);
+
+ block_flags = flags;
+ }
+
+ // transpose_vecs_512 operates on a 16x16 matrix of words, but we only have 8
+ // state vectors. Pad the matrix with zeros. After transposition, store the
+ // lower half of each vector.
+ __m512i padded[16] = {
+ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
+ h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
+ set1_512(0), set1_512(0), set1_512(0), set1_512(0),
+ set1_512(0), set1_512(0), set1_512(0), set1_512(0),
+ };
+ transpose_vecs_512(padded);
+ _mm256_mask_storeu_epi32(&out[0 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[0]));
+ _mm256_mask_storeu_epi32(&out[1 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[1]));
+ _mm256_mask_storeu_epi32(&out[2 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[2]));
+ _mm256_mask_storeu_epi32(&out[3 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[3]));
+ _mm256_mask_storeu_epi32(&out[4 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[4]));
+ _mm256_mask_storeu_epi32(&out[5 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[5]));
+ _mm256_mask_storeu_epi32(&out[6 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[6]));
+ _mm256_mask_storeu_epi32(&out[7 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[7]));
+ _mm256_mask_storeu_epi32(&out[8 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[8]));
+ _mm256_mask_storeu_epi32(&out[9 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[9]));
+ _mm256_mask_storeu_epi32(&out[10 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[10]));
+ _mm256_mask_storeu_epi32(&out[11 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[11]));
+ _mm256_mask_storeu_epi32(&out[12 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[12]));
+ _mm256_mask_storeu_epi32(&out[13 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[13]));
+ _mm256_mask_storeu_epi32(&out[14 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[14]));
+ _mm256_mask_storeu_epi32(&out[15 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[15]));
+}
+
+/*
+ * ----------------------------------------------------------------------------
+ * hash_many_avx512
+ * ----------------------------------------------------------------------------
+ */
+
+INLINE void hash_one_avx512(const uint8_t *input, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) {
+ uint32_t cv[8];
+ memcpy(cv, key, BLAKE3_KEY_LEN);
+ uint8_t block_flags = flags | flags_start;
+ while (blocks > 0) {
+ if (blocks == 1) {
+ block_flags |= flags_end;
+ }
+ blake3_compress_in_place_avx512(cv, input, BLAKE3_BLOCK_LEN, counter,
+ block_flags);
+ input = &input[BLAKE3_BLOCK_LEN];
+ blocks -= 1;
+ block_flags = flags;
+ }
+ memcpy(out, cv, BLAKE3_OUT_LEN);
+}
+
+void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out) {
+ while (num_inputs >= 16) {
+ blake3_hash16_avx512(inputs, blocks, key, counter, increment_counter, flags,
+ flags_start, flags_end, out);
+ if (increment_counter) {
+ counter += 16;
+ }
+ inputs += 16;
+ num_inputs -= 16;
+ out = &out[16 * BLAKE3_OUT_LEN];
+ }
+ while (num_inputs >= 8) {
+ blake3_hash8_avx512(inputs, blocks, key, counter, increment_counter, flags,
+ flags_start, flags_end, out);
+ if (increment_counter) {
+ counter += 8;
+ }
+ inputs += 8;
+ num_inputs -= 8;
+ out = &out[8 * BLAKE3_OUT_LEN];
+ }
+ while (num_inputs >= 4) {
+ blake3_hash4_avx512(inputs, blocks, key, counter, increment_counter, flags,
+ flags_start, flags_end, out);
+ if (increment_counter) {
+ counter += 4;
+ }
+ inputs += 4;
+ num_inputs -= 4;
+ out = &out[4 * BLAKE3_OUT_LEN];
+ }
+ while (num_inputs > 0) {
+ hash_one_avx512(inputs[0], blocks, key, counter, flags, flags_start,
+ flags_end, out);
+ if (increment_counter) {
+ counter += 1;
+ }
+ inputs += 1;
+ num_inputs -= 1;
+ out = &out[BLAKE3_OUT_LEN];
+ }
+}
--- /dev/null
+#if defined(__ELF__) && defined(__linux__)
+.section .note.GNU-stack,"",%progbits
+#endif
+
+#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
+#if __has_include(<cet.h>)
+#include <cet.h>
+#endif
+#endif
+
+#if !defined(_CET_ENDBR)
+#define _CET_ENDBR
+#endif
+
+.intel_syntax noprefix
+.global _blake3_hash_many_avx512_asm
+.global blake3_hash_many_avx512_asm
+.global blake3_compress_in_place_avx512_asm
+.global _blake3_compress_in_place_avx512_asm
+.global blake3_compress_xof_avx512_asm
+.global _blake3_compress_xof_avx512_asm
+
+#ifdef __APPLE__
+.text
+#else
+.section .text
+#endif
+.p2align 6
+_blake3_hash_many_avx512_asm:
+blake3_hash_many_avx512_asm:
+ _CET_ENDBR
+ push r15
+ push r14
+ push r13
+ push r12
+ push rbx
+ push rbp
+ mov rbp, rsp
+ sub rsp, 144
+ and rsp, 0xFFFFFFFFFFFFFFC0
+ neg r9
+ kmovw k1, r9d
+ vmovd xmm0, r8d
+ vpbroadcastd ymm0, xmm0
+ shr r8, 32
+ vmovd xmm1, r8d
+ vpbroadcastd ymm1, xmm1
+ vmovdqa ymm4, ymm1
+ vmovdqa ymm5, ymm1
+ vpaddd ymm2, ymm0, ymmword ptr [ADD0+rip]
+ vpaddd ymm3, ymm0, ymmword ptr [ADD0+32+rip]
+ vpcmpltud k2, ymm2, ymm0
+ vpcmpltud k3, ymm3, ymm0
+ vpaddd ymm4 {k2}, ymm4, dword ptr [ADD1+rip] {1to8}
+ vpaddd ymm5 {k3}, ymm5, dword ptr [ADD1+rip] {1to8}
+ knotw k2, k1
+ vmovdqa32 ymm2 {k2}, ymm0
+ vmovdqa32 ymm3 {k2}, ymm0
+ vmovdqa32 ymm4 {k2}, ymm1
+ vmovdqa32 ymm5 {k2}, ymm1
+ vmovdqa ymmword ptr [rsp], ymm2
+ vmovdqa ymmword ptr [rsp+0x1*0x20], ymm3
+ vmovdqa ymmword ptr [rsp+0x2*0x20], ymm4
+ vmovdqa ymmword ptr [rsp+0x3*0x20], ymm5
+ shl rdx, 6
+ mov qword ptr [rsp+0x80], rdx
+ cmp rsi, 16
+ jc 3f
+2:
+ vpbroadcastd zmm0, dword ptr [rcx]
+ vpbroadcastd zmm1, dword ptr [rcx+0x1*0x4]
+ vpbroadcastd zmm2, dword ptr [rcx+0x2*0x4]
+ vpbroadcastd zmm3, dword ptr [rcx+0x3*0x4]
+ vpbroadcastd zmm4, dword ptr [rcx+0x4*0x4]
+ vpbroadcastd zmm5, dword ptr [rcx+0x5*0x4]
+ vpbroadcastd zmm6, dword ptr [rcx+0x6*0x4]
+ vpbroadcastd zmm7, dword ptr [rcx+0x7*0x4]
+ movzx eax, byte ptr [rbp+0x38]
+ movzx ebx, byte ptr [rbp+0x40]
+ or eax, ebx
+ xor edx, edx
+.p2align 5
+9:
+ movzx ebx, byte ptr [rbp+0x48]
+ or ebx, eax
+ add rdx, 64
+ cmp rdx, qword ptr [rsp+0x80]
+ cmove eax, ebx
+ mov dword ptr [rsp+0x88], eax
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ mov r10, qword ptr [rdi+0x10]
+ mov r11, qword ptr [rdi+0x18]
+ mov r12, qword ptr [rdi+0x40]
+ mov r13, qword ptr [rdi+0x48]
+ mov r14, qword ptr [rdi+0x50]
+ mov r15, qword ptr [rdi+0x58]
+ vmovdqu32 ymm16, ymmword ptr [rdx+r8-0x2*0x20]
+ vinserti64x4 zmm16, zmm16, ymmword ptr [rdx+r12-0x2*0x20], 0x01
+ vmovdqu32 ymm17, ymmword ptr [rdx+r9-0x2*0x20]
+ vinserti64x4 zmm17, zmm17, ymmword ptr [rdx+r13-0x2*0x20], 0x01
+ vpunpcklqdq zmm8, zmm16, zmm17
+ vpunpckhqdq zmm9, zmm16, zmm17
+ vmovdqu32 ymm18, ymmword ptr [rdx+r10-0x2*0x20]
+ vinserti64x4 zmm18, zmm18, ymmword ptr [rdx+r14-0x2*0x20], 0x01
+ vmovdqu32 ymm19, ymmword ptr [rdx+r11-0x2*0x20]
+ vinserti64x4 zmm19, zmm19, ymmword ptr [rdx+r15-0x2*0x20], 0x01
+ vpunpcklqdq zmm10, zmm18, zmm19
+ vpunpckhqdq zmm11, zmm18, zmm19
+ mov r8, qword ptr [rdi+0x20]
+ mov r9, qword ptr [rdi+0x28]
+ mov r10, qword ptr [rdi+0x30]
+ mov r11, qword ptr [rdi+0x38]
+ mov r12, qword ptr [rdi+0x60]
+ mov r13, qword ptr [rdi+0x68]
+ mov r14, qword ptr [rdi+0x70]
+ mov r15, qword ptr [rdi+0x78]
+ vmovdqu32 ymm16, ymmword ptr [rdx+r8-0x2*0x20]
+ vinserti64x4 zmm16, zmm16, ymmword ptr [rdx+r12-0x2*0x20], 0x01
+ vmovdqu32 ymm17, ymmword ptr [rdx+r9-0x2*0x20]
+ vinserti64x4 zmm17, zmm17, ymmword ptr [rdx+r13-0x2*0x20], 0x01
+ vpunpcklqdq zmm12, zmm16, zmm17
+ vpunpckhqdq zmm13, zmm16, zmm17
+ vmovdqu32 ymm18, ymmword ptr [rdx+r10-0x2*0x20]
+ vinserti64x4 zmm18, zmm18, ymmword ptr [rdx+r14-0x2*0x20], 0x01
+ vmovdqu32 ymm19, ymmword ptr [rdx+r11-0x2*0x20]
+ vinserti64x4 zmm19, zmm19, ymmword ptr [rdx+r15-0x2*0x20], 0x01
+ vpunpcklqdq zmm14, zmm18, zmm19
+ vpunpckhqdq zmm15, zmm18, zmm19
+ vmovdqa32 zmm27, zmmword ptr [INDEX0+rip]
+ vmovdqa32 zmm31, zmmword ptr [INDEX1+rip]
+ vshufps zmm16, zmm8, zmm10, 136
+ vshufps zmm17, zmm12, zmm14, 136
+ vmovdqa32 zmm20, zmm16
+ vpermt2d zmm16, zmm27, zmm17
+ vpermt2d zmm20, zmm31, zmm17
+ vshufps zmm17, zmm8, zmm10, 221
+ vshufps zmm30, zmm12, zmm14, 221
+ vmovdqa32 zmm21, zmm17
+ vpermt2d zmm17, zmm27, zmm30
+ vpermt2d zmm21, zmm31, zmm30
+ vshufps zmm18, zmm9, zmm11, 136
+ vshufps zmm8, zmm13, zmm15, 136
+ vmovdqa32 zmm22, zmm18
+ vpermt2d zmm18, zmm27, zmm8
+ vpermt2d zmm22, zmm31, zmm8
+ vshufps zmm19, zmm9, zmm11, 221
+ vshufps zmm8, zmm13, zmm15, 221
+ vmovdqa32 zmm23, zmm19
+ vpermt2d zmm19, zmm27, zmm8
+ vpermt2d zmm23, zmm31, zmm8
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ mov r10, qword ptr [rdi+0x10]
+ mov r11, qword ptr [rdi+0x18]
+ mov r12, qword ptr [rdi+0x40]
+ mov r13, qword ptr [rdi+0x48]
+ mov r14, qword ptr [rdi+0x50]
+ mov r15, qword ptr [rdi+0x58]
+ vmovdqu32 ymm24, ymmword ptr [r8+rdx-0x1*0x20]
+ vinserti64x4 zmm24, zmm24, ymmword ptr [r12+rdx-0x1*0x20], 0x01
+ vmovdqu32 ymm25, ymmword ptr [r9+rdx-0x1*0x20]
+ vinserti64x4 zmm25, zmm25, ymmword ptr [r13+rdx-0x1*0x20], 0x01
+ vpunpcklqdq zmm8, zmm24, zmm25
+ vpunpckhqdq zmm9, zmm24, zmm25
+ vmovdqu32 ymm24, ymmword ptr [r10+rdx-0x1*0x20]
+ vinserti64x4 zmm24, zmm24, ymmword ptr [r14+rdx-0x1*0x20], 0x01
+ vmovdqu32 ymm25, ymmword ptr [r11+rdx-0x1*0x20]
+ vinserti64x4 zmm25, zmm25, ymmword ptr [r15+rdx-0x1*0x20], 0x01
+ vpunpcklqdq zmm10, zmm24, zmm25
+ vpunpckhqdq zmm11, zmm24, zmm25
+ prefetcht0 [r8+rdx+0x80]
+ prefetcht0 [r12+rdx+0x80]
+ prefetcht0 [r9+rdx+0x80]
+ prefetcht0 [r13+rdx+0x80]
+ prefetcht0 [r10+rdx+0x80]
+ prefetcht0 [r14+rdx+0x80]
+ prefetcht0 [r11+rdx+0x80]
+ prefetcht0 [r15+rdx+0x80]
+ mov r8, qword ptr [rdi+0x20]
+ mov r9, qword ptr [rdi+0x28]
+ mov r10, qword ptr [rdi+0x30]
+ mov r11, qword ptr [rdi+0x38]
+ mov r12, qword ptr [rdi+0x60]
+ mov r13, qword ptr [rdi+0x68]
+ mov r14, qword ptr [rdi+0x70]
+ mov r15, qword ptr [rdi+0x78]
+ vmovdqu32 ymm24, ymmword ptr [r8+rdx-0x1*0x20]
+ vinserti64x4 zmm24, zmm24, ymmword ptr [r12+rdx-0x1*0x20], 0x01
+ vmovdqu32 ymm25, ymmword ptr [r9+rdx-0x1*0x20]
+ vinserti64x4 zmm25, zmm25, ymmword ptr [r13+rdx-0x1*0x20], 0x01
+ vpunpcklqdq zmm12, zmm24, zmm25
+ vpunpckhqdq zmm13, zmm24, zmm25
+ vmovdqu32 ymm24, ymmword ptr [r10+rdx-0x1*0x20]
+ vinserti64x4 zmm24, zmm24, ymmword ptr [r14+rdx-0x1*0x20], 0x01
+ vmovdqu32 ymm25, ymmword ptr [r11+rdx-0x1*0x20]
+ vinserti64x4 zmm25, zmm25, ymmword ptr [r15+rdx-0x1*0x20], 0x01
+ vpunpcklqdq zmm14, zmm24, zmm25
+ vpunpckhqdq zmm15, zmm24, zmm25
+ prefetcht0 [r8+rdx+0x80]
+ prefetcht0 [r12+rdx+0x80]
+ prefetcht0 [r9+rdx+0x80]
+ prefetcht0 [r13+rdx+0x80]
+ prefetcht0 [r10+rdx+0x80]
+ prefetcht0 [r14+rdx+0x80]
+ prefetcht0 [r11+rdx+0x80]
+ prefetcht0 [r15+rdx+0x80]
+ vshufps zmm24, zmm8, zmm10, 136
+ vshufps zmm30, zmm12, zmm14, 136
+ vmovdqa32 zmm28, zmm24
+ vpermt2d zmm24, zmm27, zmm30
+ vpermt2d zmm28, zmm31, zmm30
+ vshufps zmm25, zmm8, zmm10, 221
+ vshufps zmm30, zmm12, zmm14, 221
+ vmovdqa32 zmm29, zmm25
+ vpermt2d zmm25, zmm27, zmm30
+ vpermt2d zmm29, zmm31, zmm30
+ vshufps zmm26, zmm9, zmm11, 136
+ vshufps zmm8, zmm13, zmm15, 136
+ vmovdqa32 zmm30, zmm26
+ vpermt2d zmm26, zmm27, zmm8
+ vpermt2d zmm30, zmm31, zmm8
+ vshufps zmm8, zmm9, zmm11, 221
+ vshufps zmm10, zmm13, zmm15, 221
+ vpermi2d zmm27, zmm8, zmm10
+ vpermi2d zmm31, zmm8, zmm10
+ vpbroadcastd zmm8, dword ptr [BLAKE3_IV_0+rip]
+ vpbroadcastd zmm9, dword ptr [BLAKE3_IV_1+rip]
+ vpbroadcastd zmm10, dword ptr [BLAKE3_IV_2+rip]
+ vpbroadcastd zmm11, dword ptr [BLAKE3_IV_3+rip]
+ vmovdqa32 zmm12, zmmword ptr [rsp]
+ vmovdqa32 zmm13, zmmword ptr [rsp+0x1*0x40]
+ vpbroadcastd zmm14, dword ptr [BLAKE3_BLOCK_LEN+rip]
+ vpbroadcastd zmm15, dword ptr [rsp+0x22*0x4]
+ vpaddd zmm0, zmm0, zmm16
+ vpaddd zmm1, zmm1, zmm18
+ vpaddd zmm2, zmm2, zmm20
+ vpaddd zmm3, zmm3, zmm22
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vprord zmm15, zmm15, 16
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 12
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vpaddd zmm0, zmm0, zmm17
+ vpaddd zmm1, zmm1, zmm19
+ vpaddd zmm2, zmm2, zmm21
+ vpaddd zmm3, zmm3, zmm23
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vprord zmm15, zmm15, 8
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 7
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vpaddd zmm0, zmm0, zmm24
+ vpaddd zmm1, zmm1, zmm26
+ vpaddd zmm2, zmm2, zmm28
+ vpaddd zmm3, zmm3, zmm30
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 16
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vprord zmm4, zmm4, 12
+ vpaddd zmm0, zmm0, zmm25
+ vpaddd zmm1, zmm1, zmm27
+ vpaddd zmm2, zmm2, zmm29
+ vpaddd zmm3, zmm3, zmm31
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 8
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vprord zmm4, zmm4, 7
+ vpaddd zmm0, zmm0, zmm18
+ vpaddd zmm1, zmm1, zmm19
+ vpaddd zmm2, zmm2, zmm23
+ vpaddd zmm3, zmm3, zmm20
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vprord zmm15, zmm15, 16
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 12
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vpaddd zmm0, zmm0, zmm22
+ vpaddd zmm1, zmm1, zmm26
+ vpaddd zmm2, zmm2, zmm16
+ vpaddd zmm3, zmm3, zmm29
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vprord zmm15, zmm15, 8
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 7
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vpaddd zmm0, zmm0, zmm17
+ vpaddd zmm1, zmm1, zmm28
+ vpaddd zmm2, zmm2, zmm25
+ vpaddd zmm3, zmm3, zmm31
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 16
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vprord zmm4, zmm4, 12
+ vpaddd zmm0, zmm0, zmm27
+ vpaddd zmm1, zmm1, zmm21
+ vpaddd zmm2, zmm2, zmm30
+ vpaddd zmm3, zmm3, zmm24
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 8
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vprord zmm4, zmm4, 7
+ vpaddd zmm0, zmm0, zmm19
+ vpaddd zmm1, zmm1, zmm26
+ vpaddd zmm2, zmm2, zmm29
+ vpaddd zmm3, zmm3, zmm23
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vprord zmm15, zmm15, 16
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 12
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vpaddd zmm0, zmm0, zmm20
+ vpaddd zmm1, zmm1, zmm28
+ vpaddd zmm2, zmm2, zmm18
+ vpaddd zmm3, zmm3, zmm30
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vprord zmm15, zmm15, 8
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 7
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vpaddd zmm0, zmm0, zmm22
+ vpaddd zmm1, zmm1, zmm25
+ vpaddd zmm2, zmm2, zmm27
+ vpaddd zmm3, zmm3, zmm24
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 16
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vprord zmm4, zmm4, 12
+ vpaddd zmm0, zmm0, zmm21
+ vpaddd zmm1, zmm1, zmm16
+ vpaddd zmm2, zmm2, zmm31
+ vpaddd zmm3, zmm3, zmm17
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 8
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vprord zmm4, zmm4, 7
+ vpaddd zmm0, zmm0, zmm26
+ vpaddd zmm1, zmm1, zmm28
+ vpaddd zmm2, zmm2, zmm30
+ vpaddd zmm3, zmm3, zmm29
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vprord zmm15, zmm15, 16
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 12
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vpaddd zmm0, zmm0, zmm23
+ vpaddd zmm1, zmm1, zmm25
+ vpaddd zmm2, zmm2, zmm19
+ vpaddd zmm3, zmm3, zmm31
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vprord zmm15, zmm15, 8
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 7
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vpaddd zmm0, zmm0, zmm20
+ vpaddd zmm1, zmm1, zmm27
+ vpaddd zmm2, zmm2, zmm21
+ vpaddd zmm3, zmm3, zmm17
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 16
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vprord zmm4, zmm4, 12
+ vpaddd zmm0, zmm0, zmm16
+ vpaddd zmm1, zmm1, zmm18
+ vpaddd zmm2, zmm2, zmm24
+ vpaddd zmm3, zmm3, zmm22
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 8
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vprord zmm4, zmm4, 7
+ vpaddd zmm0, zmm0, zmm28
+ vpaddd zmm1, zmm1, zmm25
+ vpaddd zmm2, zmm2, zmm31
+ vpaddd zmm3, zmm3, zmm30
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vprord zmm15, zmm15, 16
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 12
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vpaddd zmm0, zmm0, zmm29
+ vpaddd zmm1, zmm1, zmm27
+ vpaddd zmm2, zmm2, zmm26
+ vpaddd zmm3, zmm3, zmm24
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vprord zmm15, zmm15, 8
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 7
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vpaddd zmm0, zmm0, zmm23
+ vpaddd zmm1, zmm1, zmm21
+ vpaddd zmm2, zmm2, zmm16
+ vpaddd zmm3, zmm3, zmm22
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 16
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vprord zmm4, zmm4, 12
+ vpaddd zmm0, zmm0, zmm18
+ vpaddd zmm1, zmm1, zmm19
+ vpaddd zmm2, zmm2, zmm17
+ vpaddd zmm3, zmm3, zmm20
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 8
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vprord zmm4, zmm4, 7
+ vpaddd zmm0, zmm0, zmm25
+ vpaddd zmm1, zmm1, zmm27
+ vpaddd zmm2, zmm2, zmm24
+ vpaddd zmm3, zmm3, zmm31
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vprord zmm15, zmm15, 16
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 12
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vpaddd zmm0, zmm0, zmm30
+ vpaddd zmm1, zmm1, zmm21
+ vpaddd zmm2, zmm2, zmm28
+ vpaddd zmm3, zmm3, zmm17
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vprord zmm15, zmm15, 8
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 7
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vpaddd zmm0, zmm0, zmm29
+ vpaddd zmm1, zmm1, zmm16
+ vpaddd zmm2, zmm2, zmm18
+ vpaddd zmm3, zmm3, zmm20
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 16
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vprord zmm4, zmm4, 12
+ vpaddd zmm0, zmm0, zmm19
+ vpaddd zmm1, zmm1, zmm26
+ vpaddd zmm2, zmm2, zmm22
+ vpaddd zmm3, zmm3, zmm23
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 8
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vprord zmm4, zmm4, 7
+ vpaddd zmm0, zmm0, zmm27
+ vpaddd zmm1, zmm1, zmm21
+ vpaddd zmm2, zmm2, zmm17
+ vpaddd zmm3, zmm3, zmm24
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vprord zmm15, zmm15, 16
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 12
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vpaddd zmm0, zmm0, zmm31
+ vpaddd zmm1, zmm1, zmm16
+ vpaddd zmm2, zmm2, zmm25
+ vpaddd zmm3, zmm3, zmm22
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm1, zmm1, zmm5
+ vpaddd zmm2, zmm2, zmm6
+ vpaddd zmm3, zmm3, zmm7
+ vpxord zmm12, zmm12, zmm0
+ vpxord zmm13, zmm13, zmm1
+ vpxord zmm14, zmm14, zmm2
+ vpxord zmm15, zmm15, zmm3
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vprord zmm15, zmm15, 8
+ vpaddd zmm8, zmm8, zmm12
+ vpaddd zmm9, zmm9, zmm13
+ vpaddd zmm10, zmm10, zmm14
+ vpaddd zmm11, zmm11, zmm15
+ vpxord zmm4, zmm4, zmm8
+ vpxord zmm5, zmm5, zmm9
+ vpxord zmm6, zmm6, zmm10
+ vpxord zmm7, zmm7, zmm11
+ vprord zmm4, zmm4, 7
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vpaddd zmm0, zmm0, zmm30
+ vpaddd zmm1, zmm1, zmm18
+ vpaddd zmm2, zmm2, zmm19
+ vpaddd zmm3, zmm3, zmm23
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 16
+ vprord zmm12, zmm12, 16
+ vprord zmm13, zmm13, 16
+ vprord zmm14, zmm14, 16
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 12
+ vprord zmm6, zmm6, 12
+ vprord zmm7, zmm7, 12
+ vprord zmm4, zmm4, 12
+ vpaddd zmm0, zmm0, zmm26
+ vpaddd zmm1, zmm1, zmm28
+ vpaddd zmm2, zmm2, zmm20
+ vpaddd zmm3, zmm3, zmm29
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm1, zmm1, zmm6
+ vpaddd zmm2, zmm2, zmm7
+ vpaddd zmm3, zmm3, zmm4
+ vpxord zmm15, zmm15, zmm0
+ vpxord zmm12, zmm12, zmm1
+ vpxord zmm13, zmm13, zmm2
+ vpxord zmm14, zmm14, zmm3
+ vprord zmm15, zmm15, 8
+ vprord zmm12, zmm12, 8
+ vprord zmm13, zmm13, 8
+ vprord zmm14, zmm14, 8
+ vpaddd zmm10, zmm10, zmm15
+ vpaddd zmm11, zmm11, zmm12
+ vpaddd zmm8, zmm8, zmm13
+ vpaddd zmm9, zmm9, zmm14
+ vpxord zmm5, zmm5, zmm10
+ vpxord zmm6, zmm6, zmm11
+ vpxord zmm7, zmm7, zmm8
+ vpxord zmm4, zmm4, zmm9
+ vprord zmm5, zmm5, 7
+ vprord zmm6, zmm6, 7
+ vprord zmm7, zmm7, 7
+ vprord zmm4, zmm4, 7
+ vpxord zmm0, zmm0, zmm8
+ vpxord zmm1, zmm1, zmm9
+ vpxord zmm2, zmm2, zmm10
+ vpxord zmm3, zmm3, zmm11
+ vpxord zmm4, zmm4, zmm12
+ vpxord zmm5, zmm5, zmm13
+ vpxord zmm6, zmm6, zmm14
+ vpxord zmm7, zmm7, zmm15
+ movzx eax, byte ptr [rbp+0x38]
+ jne 9b
+ mov rbx, qword ptr [rbp+0x50]
+ vpunpckldq zmm16, zmm0, zmm1
+ vpunpckhdq zmm17, zmm0, zmm1
+ vpunpckldq zmm18, zmm2, zmm3
+ vpunpckhdq zmm19, zmm2, zmm3
+ vpunpckldq zmm20, zmm4, zmm5
+ vpunpckhdq zmm21, zmm4, zmm5
+ vpunpckldq zmm22, zmm6, zmm7
+ vpunpckhdq zmm23, zmm6, zmm7
+ vpunpcklqdq zmm0, zmm16, zmm18
+ vpunpckhqdq zmm1, zmm16, zmm18
+ vpunpcklqdq zmm2, zmm17, zmm19
+ vpunpckhqdq zmm3, zmm17, zmm19
+ vpunpcklqdq zmm4, zmm20, zmm22
+ vpunpckhqdq zmm5, zmm20, zmm22
+ vpunpcklqdq zmm6, zmm21, zmm23
+ vpunpckhqdq zmm7, zmm21, zmm23
+ vshufi32x4 zmm16, zmm0, zmm4, 0x88
+ vshufi32x4 zmm17, zmm1, zmm5, 0x88
+ vshufi32x4 zmm18, zmm2, zmm6, 0x88
+ vshufi32x4 zmm19, zmm3, zmm7, 0x88
+ vshufi32x4 zmm20, zmm0, zmm4, 0xDD
+ vshufi32x4 zmm21, zmm1, zmm5, 0xDD
+ vshufi32x4 zmm22, zmm2, zmm6, 0xDD
+ vshufi32x4 zmm23, zmm3, zmm7, 0xDD
+ vshufi32x4 zmm0, zmm16, zmm17, 0x88
+ vshufi32x4 zmm1, zmm18, zmm19, 0x88
+ vshufi32x4 zmm2, zmm20, zmm21, 0x88
+ vshufi32x4 zmm3, zmm22, zmm23, 0x88
+ vshufi32x4 zmm4, zmm16, zmm17, 0xDD
+ vshufi32x4 zmm5, zmm18, zmm19, 0xDD
+ vshufi32x4 zmm6, zmm20, zmm21, 0xDD
+ vshufi32x4 zmm7, zmm22, zmm23, 0xDD
+ vmovdqu32 zmmword ptr [rbx], zmm0
+ vmovdqu32 zmmword ptr [rbx+0x1*0x40], zmm1
+ vmovdqu32 zmmword ptr [rbx+0x2*0x40], zmm2
+ vmovdqu32 zmmword ptr [rbx+0x3*0x40], zmm3
+ vmovdqu32 zmmword ptr [rbx+0x4*0x40], zmm4
+ vmovdqu32 zmmword ptr [rbx+0x5*0x40], zmm5
+ vmovdqu32 zmmword ptr [rbx+0x6*0x40], zmm6
+ vmovdqu32 zmmword ptr [rbx+0x7*0x40], zmm7
+ vmovdqa32 zmm0, zmmword ptr [rsp]
+ vmovdqa32 zmm1, zmmword ptr [rsp+0x1*0x40]
+ vmovdqa32 zmm2, zmm0
+ vpaddd zmm2{k1}, zmm0, dword ptr [ADD16+rip] {1to16}
+ vpcmpltud k2, zmm2, zmm0
+ vpaddd zmm1 {k2}, zmm1, dword ptr [ADD1+rip] {1to16}
+ vmovdqa32 zmmword ptr [rsp], zmm2
+ vmovdqa32 zmmword ptr [rsp+0x1*0x40], zmm1
+ add rdi, 128
+ add rbx, 512
+ mov qword ptr [rbp+0x50], rbx
+ sub rsi, 16
+ cmp rsi, 16
+ jnc 2b
+ test rsi, rsi
+ jnz 3f
+4:
+ vzeroupper
+ mov rsp, rbp
+ pop rbp
+ pop rbx
+ pop r12
+ pop r13
+ pop r14
+ pop r15
+ ret
+.p2align 6
+3:
+ test esi, 0x8
+ je 3f
+ vpbroadcastd ymm0, dword ptr [rcx]
+ vpbroadcastd ymm1, dword ptr [rcx+0x4]
+ vpbroadcastd ymm2, dword ptr [rcx+0x8]
+ vpbroadcastd ymm3, dword ptr [rcx+0xC]
+ vpbroadcastd ymm4, dword ptr [rcx+0x10]
+ vpbroadcastd ymm5, dword ptr [rcx+0x14]
+ vpbroadcastd ymm6, dword ptr [rcx+0x18]
+ vpbroadcastd ymm7, dword ptr [rcx+0x1C]
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ mov r10, qword ptr [rdi+0x10]
+ mov r11, qword ptr [rdi+0x18]
+ mov r12, qword ptr [rdi+0x20]
+ mov r13, qword ptr [rdi+0x28]
+ mov r14, qword ptr [rdi+0x30]
+ mov r15, qword ptr [rdi+0x38]
+ movzx eax, byte ptr [rbp+0x38]
+ movzx ebx, byte ptr [rbp+0x40]
+ or eax, ebx
+ xor edx, edx
+2:
+ movzx ebx, byte ptr [rbp+0x48]
+ or ebx, eax
+ add rdx, 64
+ cmp rdx, qword ptr [rsp+0x80]
+ cmove eax, ebx
+ mov dword ptr [rsp+0x88], eax
+ vmovups xmm8, xmmword ptr [r8+rdx-0x40]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x40], 0x01
+ vmovups xmm9, xmmword ptr [r9+rdx-0x40]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x40], 0x01
+ vunpcklpd ymm12, ymm8, ymm9
+ vunpckhpd ymm13, ymm8, ymm9
+ vmovups xmm10, xmmword ptr [r10+rdx-0x40]
+ vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x40], 0x01
+ vmovups xmm11, xmmword ptr [r11+rdx-0x40]
+ vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x40], 0x01
+ vunpcklpd ymm14, ymm10, ymm11
+ vunpckhpd ymm15, ymm10, ymm11
+ vshufps ymm16, ymm12, ymm14, 136
+ vshufps ymm17, ymm12, ymm14, 221
+ vshufps ymm18, ymm13, ymm15, 136
+ vshufps ymm19, ymm13, ymm15, 221
+ vmovups xmm8, xmmword ptr [r8+rdx-0x30]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x30], 0x01
+ vmovups xmm9, xmmword ptr [r9+rdx-0x30]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x30], 0x01
+ vunpcklpd ymm12, ymm8, ymm9
+ vunpckhpd ymm13, ymm8, ymm9
+ vmovups xmm10, xmmword ptr [r10+rdx-0x30]
+ vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x30], 0x01
+ vmovups xmm11, xmmword ptr [r11+rdx-0x30]
+ vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x30], 0x01
+ vunpcklpd ymm14, ymm10, ymm11
+ vunpckhpd ymm15, ymm10, ymm11
+ vshufps ymm20, ymm12, ymm14, 136
+ vshufps ymm21, ymm12, ymm14, 221
+ vshufps ymm22, ymm13, ymm15, 136
+ vshufps ymm23, ymm13, ymm15, 221
+ vmovups xmm8, xmmword ptr [r8+rdx-0x20]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x20], 0x01
+ vmovups xmm9, xmmword ptr [r9+rdx-0x20]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x20], 0x01
+ vunpcklpd ymm12, ymm8, ymm9
+ vunpckhpd ymm13, ymm8, ymm9
+ vmovups xmm10, xmmword ptr [r10+rdx-0x20]
+ vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x20], 0x01
+ vmovups xmm11, xmmword ptr [r11+rdx-0x20]
+ vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x20], 0x01
+ vunpcklpd ymm14, ymm10, ymm11
+ vunpckhpd ymm15, ymm10, ymm11
+ vshufps ymm24, ymm12, ymm14, 136
+ vshufps ymm25, ymm12, ymm14, 221
+ vshufps ymm26, ymm13, ymm15, 136
+ vshufps ymm27, ymm13, ymm15, 221
+ vmovups xmm8, xmmword ptr [r8+rdx-0x10]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x10], 0x01
+ vmovups xmm9, xmmword ptr [r9+rdx-0x10]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x10], 0x01
+ vunpcklpd ymm12, ymm8, ymm9
+ vunpckhpd ymm13, ymm8, ymm9
+ vmovups xmm10, xmmword ptr [r10+rdx-0x10]
+ vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x10], 0x01
+ vmovups xmm11, xmmword ptr [r11+rdx-0x10]
+ vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x10], 0x01
+ vunpcklpd ymm14, ymm10, ymm11
+ vunpckhpd ymm15, ymm10, ymm11
+ vshufps ymm28, ymm12, ymm14, 136
+ vshufps ymm29, ymm12, ymm14, 221
+ vshufps ymm30, ymm13, ymm15, 136
+ vshufps ymm31, ymm13, ymm15, 221
+ vpbroadcastd ymm8, dword ptr [BLAKE3_IV_0+rip]
+ vpbroadcastd ymm9, dword ptr [BLAKE3_IV_1+rip]
+ vpbroadcastd ymm10, dword ptr [BLAKE3_IV_2+rip]
+ vpbroadcastd ymm11, dword ptr [BLAKE3_IV_3+rip]
+ vmovdqa ymm12, ymmword ptr [rsp]
+ vmovdqa ymm13, ymmword ptr [rsp+0x40]
+ vpbroadcastd ymm14, dword ptr [BLAKE3_BLOCK_LEN+rip]
+ vpbroadcastd ymm15, dword ptr [rsp+0x88]
+ vpaddd ymm0, ymm0, ymm16
+ vpaddd ymm1, ymm1, ymm18
+ vpaddd ymm2, ymm2, ymm20
+ vpaddd ymm3, ymm3, ymm22
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vprord ymm15, ymm15, 16
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 12
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vpaddd ymm0, ymm0, ymm17
+ vpaddd ymm1, ymm1, ymm19
+ vpaddd ymm2, ymm2, ymm21
+ vpaddd ymm3, ymm3, ymm23
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vprord ymm15, ymm15, 8
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 7
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vpaddd ymm0, ymm0, ymm24
+ vpaddd ymm1, ymm1, ymm26
+ vpaddd ymm2, ymm2, ymm28
+ vpaddd ymm3, ymm3, ymm30
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 16
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vprord ymm4, ymm4, 12
+ vpaddd ymm0, ymm0, ymm25
+ vpaddd ymm1, ymm1, ymm27
+ vpaddd ymm2, ymm2, ymm29
+ vpaddd ymm3, ymm3, ymm31
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 8
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vprord ymm4, ymm4, 7
+ vpaddd ymm0, ymm0, ymm18
+ vpaddd ymm1, ymm1, ymm19
+ vpaddd ymm2, ymm2, ymm23
+ vpaddd ymm3, ymm3, ymm20
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vprord ymm15, ymm15, 16
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 12
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vpaddd ymm0, ymm0, ymm22
+ vpaddd ymm1, ymm1, ymm26
+ vpaddd ymm2, ymm2, ymm16
+ vpaddd ymm3, ymm3, ymm29
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vprord ymm15, ymm15, 8
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 7
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vpaddd ymm0, ymm0, ymm17
+ vpaddd ymm1, ymm1, ymm28
+ vpaddd ymm2, ymm2, ymm25
+ vpaddd ymm3, ymm3, ymm31
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 16
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vprord ymm4, ymm4, 12
+ vpaddd ymm0, ymm0, ymm27
+ vpaddd ymm1, ymm1, ymm21
+ vpaddd ymm2, ymm2, ymm30
+ vpaddd ymm3, ymm3, ymm24
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 8
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vprord ymm4, ymm4, 7
+ vpaddd ymm0, ymm0, ymm19
+ vpaddd ymm1, ymm1, ymm26
+ vpaddd ymm2, ymm2, ymm29
+ vpaddd ymm3, ymm3, ymm23
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vprord ymm15, ymm15, 16
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 12
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vpaddd ymm0, ymm0, ymm20
+ vpaddd ymm1, ymm1, ymm28
+ vpaddd ymm2, ymm2, ymm18
+ vpaddd ymm3, ymm3, ymm30
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vprord ymm15, ymm15, 8
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 7
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vpaddd ymm0, ymm0, ymm22
+ vpaddd ymm1, ymm1, ymm25
+ vpaddd ymm2, ymm2, ymm27
+ vpaddd ymm3, ymm3, ymm24
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 16
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vprord ymm4, ymm4, 12
+ vpaddd ymm0, ymm0, ymm21
+ vpaddd ymm1, ymm1, ymm16
+ vpaddd ymm2, ymm2, ymm31
+ vpaddd ymm3, ymm3, ymm17
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 8
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vprord ymm4, ymm4, 7
+ vpaddd ymm0, ymm0, ymm26
+ vpaddd ymm1, ymm1, ymm28
+ vpaddd ymm2, ymm2, ymm30
+ vpaddd ymm3, ymm3, ymm29
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vprord ymm15, ymm15, 16
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 12
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vpaddd ymm0, ymm0, ymm23
+ vpaddd ymm1, ymm1, ymm25
+ vpaddd ymm2, ymm2, ymm19
+ vpaddd ymm3, ymm3, ymm31
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vprord ymm15, ymm15, 8
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 7
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vpaddd ymm0, ymm0, ymm20
+ vpaddd ymm1, ymm1, ymm27
+ vpaddd ymm2, ymm2, ymm21
+ vpaddd ymm3, ymm3, ymm17
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 16
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vprord ymm4, ymm4, 12
+ vpaddd ymm0, ymm0, ymm16
+ vpaddd ymm1, ymm1, ymm18
+ vpaddd ymm2, ymm2, ymm24
+ vpaddd ymm3, ymm3, ymm22
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 8
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vprord ymm4, ymm4, 7
+ vpaddd ymm0, ymm0, ymm28
+ vpaddd ymm1, ymm1, ymm25
+ vpaddd ymm2, ymm2, ymm31
+ vpaddd ymm3, ymm3, ymm30
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vprord ymm15, ymm15, 16
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 12
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vpaddd ymm0, ymm0, ymm29
+ vpaddd ymm1, ymm1, ymm27
+ vpaddd ymm2, ymm2, ymm26
+ vpaddd ymm3, ymm3, ymm24
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vprord ymm15, ymm15, 8
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 7
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vpaddd ymm0, ymm0, ymm23
+ vpaddd ymm1, ymm1, ymm21
+ vpaddd ymm2, ymm2, ymm16
+ vpaddd ymm3, ymm3, ymm22
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 16
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vprord ymm4, ymm4, 12
+ vpaddd ymm0, ymm0, ymm18
+ vpaddd ymm1, ymm1, ymm19
+ vpaddd ymm2, ymm2, ymm17
+ vpaddd ymm3, ymm3, ymm20
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 8
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vprord ymm4, ymm4, 7
+ vpaddd ymm0, ymm0, ymm25
+ vpaddd ymm1, ymm1, ymm27
+ vpaddd ymm2, ymm2, ymm24
+ vpaddd ymm3, ymm3, ymm31
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vprord ymm15, ymm15, 16
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 12
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vpaddd ymm0, ymm0, ymm30
+ vpaddd ymm1, ymm1, ymm21
+ vpaddd ymm2, ymm2, ymm28
+ vpaddd ymm3, ymm3, ymm17
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vprord ymm15, ymm15, 8
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 7
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vpaddd ymm0, ymm0, ymm29
+ vpaddd ymm1, ymm1, ymm16
+ vpaddd ymm2, ymm2, ymm18
+ vpaddd ymm3, ymm3, ymm20
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 16
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vprord ymm4, ymm4, 12
+ vpaddd ymm0, ymm0, ymm19
+ vpaddd ymm1, ymm1, ymm26
+ vpaddd ymm2, ymm2, ymm22
+ vpaddd ymm3, ymm3, ymm23
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 8
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vprord ymm4, ymm4, 7
+ vpaddd ymm0, ymm0, ymm27
+ vpaddd ymm1, ymm1, ymm21
+ vpaddd ymm2, ymm2, ymm17
+ vpaddd ymm3, ymm3, ymm24
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vprord ymm15, ymm15, 16
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 12
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vpaddd ymm0, ymm0, ymm31
+ vpaddd ymm1, ymm1, ymm16
+ vpaddd ymm2, ymm2, ymm25
+ vpaddd ymm3, ymm3, ymm22
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm1, ymm1, ymm5
+ vpaddd ymm2, ymm2, ymm6
+ vpaddd ymm3, ymm3, ymm7
+ vpxord ymm12, ymm12, ymm0
+ vpxord ymm13, ymm13, ymm1
+ vpxord ymm14, ymm14, ymm2
+ vpxord ymm15, ymm15, ymm3
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vprord ymm15, ymm15, 8
+ vpaddd ymm8, ymm8, ymm12
+ vpaddd ymm9, ymm9, ymm13
+ vpaddd ymm10, ymm10, ymm14
+ vpaddd ymm11, ymm11, ymm15
+ vpxord ymm4, ymm4, ymm8
+ vpxord ymm5, ymm5, ymm9
+ vpxord ymm6, ymm6, ymm10
+ vpxord ymm7, ymm7, ymm11
+ vprord ymm4, ymm4, 7
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vpaddd ymm0, ymm0, ymm30
+ vpaddd ymm1, ymm1, ymm18
+ vpaddd ymm2, ymm2, ymm19
+ vpaddd ymm3, ymm3, ymm23
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 16
+ vprord ymm12, ymm12, 16
+ vprord ymm13, ymm13, 16
+ vprord ymm14, ymm14, 16
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 12
+ vprord ymm6, ymm6, 12
+ vprord ymm7, ymm7, 12
+ vprord ymm4, ymm4, 12
+ vpaddd ymm0, ymm0, ymm26
+ vpaddd ymm1, ymm1, ymm28
+ vpaddd ymm2, ymm2, ymm20
+ vpaddd ymm3, ymm3, ymm29
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm1, ymm1, ymm6
+ vpaddd ymm2, ymm2, ymm7
+ vpaddd ymm3, ymm3, ymm4
+ vpxord ymm15, ymm15, ymm0
+ vpxord ymm12, ymm12, ymm1
+ vpxord ymm13, ymm13, ymm2
+ vpxord ymm14, ymm14, ymm3
+ vprord ymm15, ymm15, 8
+ vprord ymm12, ymm12, 8
+ vprord ymm13, ymm13, 8
+ vprord ymm14, ymm14, 8
+ vpaddd ymm10, ymm10, ymm15
+ vpaddd ymm11, ymm11, ymm12
+ vpaddd ymm8, ymm8, ymm13
+ vpaddd ymm9, ymm9, ymm14
+ vpxord ymm5, ymm5, ymm10
+ vpxord ymm6, ymm6, ymm11
+ vpxord ymm7, ymm7, ymm8
+ vpxord ymm4, ymm4, ymm9
+ vprord ymm5, ymm5, 7
+ vprord ymm6, ymm6, 7
+ vprord ymm7, ymm7, 7
+ vprord ymm4, ymm4, 7
+ vpxor ymm0, ymm0, ymm8
+ vpxor ymm1, ymm1, ymm9
+ vpxor ymm2, ymm2, ymm10
+ vpxor ymm3, ymm3, ymm11
+ vpxor ymm4, ymm4, ymm12
+ vpxor ymm5, ymm5, ymm13
+ vpxor ymm6, ymm6, ymm14
+ vpxor ymm7, ymm7, ymm15
+ movzx eax, byte ptr [rbp+0x38]
+ jne 2b
+ mov rbx, qword ptr [rbp+0x50]
+ vunpcklps ymm8, ymm0, ymm1
+ vunpcklps ymm9, ymm2, ymm3
+ vunpckhps ymm10, ymm0, ymm1
+ vunpcklps ymm11, ymm4, ymm5
+ vunpcklps ymm0, ymm6, ymm7
+ vshufps ymm12, ymm8, ymm9, 78
+ vblendps ymm1, ymm8, ymm12, 0xCC
+ vshufps ymm8, ymm11, ymm0, 78
+ vunpckhps ymm13, ymm2, ymm3
+ vblendps ymm2, ymm11, ymm8, 0xCC
+ vblendps ymm3, ymm12, ymm9, 0xCC
+ vperm2f128 ymm12, ymm1, ymm2, 0x20
+ vmovups ymmword ptr [rbx], ymm12
+ vunpckhps ymm14, ymm4, ymm5
+ vblendps ymm4, ymm8, ymm0, 0xCC
+ vunpckhps ymm15, ymm6, ymm7
+ vperm2f128 ymm7, ymm3, ymm4, 0x20
+ vmovups ymmword ptr [rbx+0x20], ymm7
+ vshufps ymm5, ymm10, ymm13, 78
+ vblendps ymm6, ymm5, ymm13, 0xCC
+ vshufps ymm13, ymm14, ymm15, 78
+ vblendps ymm10, ymm10, ymm5, 0xCC
+ vblendps ymm14, ymm14, ymm13, 0xCC
+ vperm2f128 ymm8, ymm10, ymm14, 0x20
+ vmovups ymmword ptr [rbx+0x40], ymm8
+ vblendps ymm15, ymm13, ymm15, 0xCC
+ vperm2f128 ymm13, ymm6, ymm15, 0x20
+ vmovups ymmword ptr [rbx+0x60], ymm13
+ vperm2f128 ymm9, ymm1, ymm2, 0x31
+ vperm2f128 ymm11, ymm3, ymm4, 0x31
+ vmovups ymmword ptr [rbx+0x80], ymm9
+ vperm2f128 ymm14, ymm10, ymm14, 0x31
+ vperm2f128 ymm15, ymm6, ymm15, 0x31
+ vmovups ymmword ptr [rbx+0xA0], ymm11
+ vmovups ymmword ptr [rbx+0xC0], ymm14
+ vmovups ymmword ptr [rbx+0xE0], ymm15
+ vmovdqa ymm0, ymmword ptr [rsp]
+ vmovdqa ymm2, ymmword ptr [rsp+0x2*0x20]
+ vmovdqa32 ymm0 {k1}, ymmword ptr [rsp+0x1*0x20]
+ vmovdqa32 ymm2 {k1}, ymmword ptr [rsp+0x3*0x20]
+ vmovdqa ymmword ptr [rsp], ymm0
+ vmovdqa ymmword ptr [rsp+0x2*0x20], ymm2
+ add rbx, 256
+ mov qword ptr [rbp+0x50], rbx
+ add rdi, 64
+ sub rsi, 8
+3:
+ mov rbx, qword ptr [rbp+0x50]
+ mov r15, qword ptr [rsp+0x80]
+ movzx r13, byte ptr [rbp+0x38]
+ movzx r12, byte ptr [rbp+0x48]
+ test esi, 0x4
+ je 3f
+ vbroadcasti32x4 zmm0, xmmword ptr [rcx]
+ vbroadcasti32x4 zmm1, xmmword ptr [rcx+0x1*0x10]
+ vmovdqa xmm12, xmmword ptr [rsp]
+ vmovdqa xmm13, xmmword ptr [rsp+0x4*0x10]
+ vpunpckldq xmm14, xmm12, xmm13
+ vpunpckhdq xmm15, xmm12, xmm13
+ vpermq ymm14, ymm14, 0xDC
+ vpermq ymm15, ymm15, 0xDC
+ vpbroadcastd zmm12, dword ptr [BLAKE3_BLOCK_LEN+rip]
+ vinserti64x4 zmm13, zmm14, ymm15, 0x01
+ mov eax, 17476
+ kmovw k2, eax
+ vpblendmd zmm13 {k2}, zmm13, zmm12
+ vbroadcasti32x4 zmm15, xmmword ptr [BLAKE3_IV+rip]
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ mov r10, qword ptr [rdi+0x10]
+ mov r11, qword ptr [rdi+0x18]
+ mov eax, 43690
+ kmovw k3, eax
+ mov eax, 34952
+ kmovw k4, eax
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+.p2align 5
+2:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ mov dword ptr [rsp+0x88], eax
+ vmovdqa32 zmm2, zmm15
+ vpbroadcastd zmm8, dword ptr [rsp+0x22*0x4]
+ vpblendmd zmm3 {k4}, zmm13, zmm8
+ vmovups zmm8, zmmword ptr [r8+rdx-0x1*0x40]
+ vinserti32x4 zmm8, zmm8, xmmword ptr [r9+rdx-0x4*0x10], 0x01
+ vinserti32x4 zmm8, zmm8, xmmword ptr [r10+rdx-0x4*0x10], 0x02
+ vinserti32x4 zmm8, zmm8, xmmword ptr [r11+rdx-0x4*0x10], 0x03
+ vmovups zmm9, zmmword ptr [r8+rdx-0x30]
+ vinserti32x4 zmm9, zmm9, xmmword ptr [r9+rdx-0x3*0x10], 0x01
+ vinserti32x4 zmm9, zmm9, xmmword ptr [r10+rdx-0x3*0x10], 0x02
+ vinserti32x4 zmm9, zmm9, xmmword ptr [r11+rdx-0x3*0x10], 0x03
+ vshufps zmm4, zmm8, zmm9, 136
+ vshufps zmm5, zmm8, zmm9, 221
+ vmovups zmm8, zmmword ptr [r8+rdx-0x20]
+ vinserti32x4 zmm8, zmm8, xmmword ptr [r9+rdx-0x2*0x10], 0x01
+ vinserti32x4 zmm8, zmm8, xmmword ptr [r10+rdx-0x2*0x10], 0x02
+ vinserti32x4 zmm8, zmm8, xmmword ptr [r11+rdx-0x2*0x10], 0x03
+ vmovups zmm9, zmmword ptr [r8+rdx-0x10]
+ vinserti32x4 zmm9, zmm9, xmmword ptr [r9+rdx-0x1*0x10], 0x01
+ vinserti32x4 zmm9, zmm9, xmmword ptr [r10+rdx-0x1*0x10], 0x02
+ vinserti32x4 zmm9, zmm9, xmmword ptr [r11+rdx-0x1*0x10], 0x03
+ vshufps zmm6, zmm8, zmm9, 136
+ vshufps zmm7, zmm8, zmm9, 221
+ vpshufd zmm6, zmm6, 0x93
+ vpshufd zmm7, zmm7, 0x93
+ mov al, 7
+9:
+ vpaddd zmm0, zmm0, zmm4
+ vpaddd zmm0, zmm0, zmm1
+ vpxord zmm3, zmm3, zmm0
+ vprord zmm3, zmm3, 16
+ vpaddd zmm2, zmm2, zmm3
+ vpxord zmm1, zmm1, zmm2
+ vprord zmm1, zmm1, 12
+ vpaddd zmm0, zmm0, zmm5
+ vpaddd zmm0, zmm0, zmm1
+ vpxord zmm3, zmm3, zmm0
+ vprord zmm3, zmm3, 8
+ vpaddd zmm2, zmm2, zmm3
+ vpxord zmm1, zmm1, zmm2
+ vprord zmm1, zmm1, 7
+ vpshufd zmm0, zmm0, 0x93
+ vpshufd zmm3, zmm3, 0x4E
+ vpshufd zmm2, zmm2, 0x39
+ vpaddd zmm0, zmm0, zmm6
+ vpaddd zmm0, zmm0, zmm1
+ vpxord zmm3, zmm3, zmm0
+ vprord zmm3, zmm3, 16
+ vpaddd zmm2, zmm2, zmm3
+ vpxord zmm1, zmm1, zmm2
+ vprord zmm1, zmm1, 12
+ vpaddd zmm0, zmm0, zmm7
+ vpaddd zmm0, zmm0, zmm1
+ vpxord zmm3, zmm3, zmm0
+ vprord zmm3, zmm3, 8
+ vpaddd zmm2, zmm2, zmm3
+ vpxord zmm1, zmm1, zmm2
+ vprord zmm1, zmm1, 7
+ vpshufd zmm0, zmm0, 0x39
+ vpshufd zmm3, zmm3, 0x4E
+ vpshufd zmm2, zmm2, 0x93
+ dec al
+ jz 9f
+ vshufps zmm8, zmm4, zmm5, 214
+ vpshufd zmm9, zmm4, 0x0F
+ vpshufd zmm4, zmm8, 0x39
+ vshufps zmm8, zmm6, zmm7, 250
+ vpblendmd zmm9 {k3}, zmm9, zmm8
+ vpunpcklqdq zmm8, zmm7, zmm5
+ vpblendmd zmm8 {k4}, zmm8, zmm6
+ vpshufd zmm8, zmm8, 0x78
+ vpunpckhdq zmm5, zmm5, zmm7
+ vpunpckldq zmm6, zmm6, zmm5
+ vpshufd zmm7, zmm6, 0x1E
+ vmovdqa32 zmm5, zmm9
+ vmovdqa32 zmm6, zmm8
+ jmp 9b
+9:
+ vpxord zmm0, zmm0, zmm2
+ vpxord zmm1, zmm1, zmm3
+ mov eax, r13d
+ cmp rdx, r15
+ jne 2b
+ vmovdqu xmmword ptr [rbx], xmm0
+ vmovdqu xmmword ptr [rbx+0x10], xmm1
+ vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01
+ vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01
+ vextracti32x4 xmmword ptr [rbx+0x4*0x10], zmm0, 0x02
+ vextracti32x4 xmmword ptr [rbx+0x5*0x10], zmm1, 0x02
+ vextracti32x4 xmmword ptr [rbx+0x6*0x10], zmm0, 0x03
+ vextracti32x4 xmmword ptr [rbx+0x7*0x10], zmm1, 0x03
+ vmovdqa xmm0, xmmword ptr [rsp]
+ vmovdqa xmm2, xmmword ptr [rsp+0x40]
+ vmovdqa32 xmm0 {k1}, xmmword ptr [rsp+0x1*0x10]
+ vmovdqa32 xmm2 {k1}, xmmword ptr [rsp+0x5*0x10]
+ vmovdqa xmmword ptr [rsp], xmm0
+ vmovdqa xmmword ptr [rsp+0x40], xmm2
+ add rbx, 128
+ add rdi, 32
+ sub rsi, 4
+3:
+ test esi, 0x2
+ je 3f
+ vbroadcasti128 ymm0, xmmword ptr [rcx]
+ vbroadcasti128 ymm1, xmmword ptr [rcx+0x10]
+ vmovd xmm13, dword ptr [rsp]
+ vpinsrd xmm13, xmm13, dword ptr [rsp+0x40], 1
+ vpinsrd xmm13, xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2
+ vmovd xmm14, dword ptr [rsp+0x4]
+ vpinsrd xmm14, xmm14, dword ptr [rsp+0x44], 1
+ vpinsrd xmm14, xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2
+ vinserti128 ymm13, ymm13, xmm14, 0x01
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+.p2align 5
+2:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ mov dword ptr [rsp+0x88], eax
+ vbroadcasti128 ymm2, xmmword ptr [BLAKE3_IV+rip]
+ vpbroadcastd ymm8, dword ptr [rsp+0x88]
+ vpblendd ymm3, ymm13, ymm8, 0x88
+ vmovups ymm8, ymmword ptr [r8+rdx-0x40]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x40], 0x01
+ vmovups ymm9, ymmword ptr [r8+rdx-0x30]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x30], 0x01
+ vshufps ymm4, ymm8, ymm9, 136
+ vshufps ymm5, ymm8, ymm9, 221
+ vmovups ymm8, ymmword ptr [r8+rdx-0x20]
+ vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x20], 0x01
+ vmovups ymm9, ymmword ptr [r8+rdx-0x10]
+ vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x10], 0x01
+ vshufps ymm6, ymm8, ymm9, 136
+ vshufps ymm7, ymm8, ymm9, 221
+ vpshufd ymm6, ymm6, 0x93
+ vpshufd ymm7, ymm7, 0x93
+ mov al, 7
+9:
+ vpaddd ymm0, ymm0, ymm4
+ vpaddd ymm0, ymm0, ymm1
+ vpxord ymm3, ymm3, ymm0
+ vprord ymm3, ymm3, 16
+ vpaddd ymm2, ymm2, ymm3
+ vpxord ymm1, ymm1, ymm2
+ vprord ymm1, ymm1, 12
+ vpaddd ymm0, ymm0, ymm5
+ vpaddd ymm0, ymm0, ymm1
+ vpxord ymm3, ymm3, ymm0
+ vprord ymm3, ymm3, 8
+ vpaddd ymm2, ymm2, ymm3
+ vpxord ymm1, ymm1, ymm2
+ vprord ymm1, ymm1, 7
+ vpshufd ymm0, ymm0, 0x93
+ vpshufd ymm3, ymm3, 0x4E
+ vpshufd ymm2, ymm2, 0x39
+ vpaddd ymm0, ymm0, ymm6
+ vpaddd ymm0, ymm0, ymm1
+ vpxord ymm3, ymm3, ymm0
+ vprord ymm3, ymm3, 16
+ vpaddd ymm2, ymm2, ymm3
+ vpxord ymm1, ymm1, ymm2
+ vprord ymm1, ymm1, 12
+ vpaddd ymm0, ymm0, ymm7
+ vpaddd ymm0, ymm0, ymm1
+ vpxord ymm3, ymm3, ymm0
+ vprord ymm3, ymm3, 8
+ vpaddd ymm2, ymm2, ymm3
+ vpxord ymm1, ymm1, ymm2
+ vprord ymm1, ymm1, 7
+ vpshufd ymm0, ymm0, 0x39
+ vpshufd ymm3, ymm3, 0x4E
+ vpshufd ymm2, ymm2, 0x93
+ dec al
+ jz 9f
+ vshufps ymm8, ymm4, ymm5, 214
+ vpshufd ymm9, ymm4, 0x0F
+ vpshufd ymm4, ymm8, 0x39
+ vshufps ymm8, ymm6, ymm7, 250
+ vpblendd ymm9, ymm9, ymm8, 0xAA
+ vpunpcklqdq ymm8, ymm7, ymm5
+ vpblendd ymm8, ymm8, ymm6, 0x88
+ vpshufd ymm8, ymm8, 0x78
+ vpunpckhdq ymm5, ymm5, ymm7
+ vpunpckldq ymm6, ymm6, ymm5
+ vpshufd ymm7, ymm6, 0x1E
+ vmovdqa ymm5, ymm9
+ vmovdqa ymm6, ymm8
+ jmp 9b
+9:
+ vpxor ymm0, ymm0, ymm2
+ vpxor ymm1, ymm1, ymm3
+ mov eax, r13d
+ cmp rdx, r15
+ jne 2b
+ vmovdqu xmmword ptr [rbx], xmm0
+ vmovdqu xmmword ptr [rbx+0x10], xmm1
+ vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01
+ vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01
+ vmovdqa xmm0, xmmword ptr [rsp]
+ vmovdqa xmm2, xmmword ptr [rsp+0x4*0x10]
+ vmovdqu32 xmm0 {k1}, xmmword ptr [rsp+0x8]
+ vmovdqu32 xmm2 {k1}, xmmword ptr [rsp+0x48]
+ vmovdqa xmmword ptr [rsp], xmm0
+ vmovdqa xmmword ptr [rsp+0x4*0x10], xmm2
+ add rbx, 64
+ add rdi, 16
+ sub rsi, 2
+3:
+ test esi, 0x1
+ je 4b
+ vmovdqu xmm0, xmmword ptr [rcx]
+ vmovdqu xmm1, xmmword ptr [rcx+0x10]
+ vmovd xmm14, dword ptr [rsp]
+ vpinsrd xmm14, xmm14, dword ptr [rsp+0x40], 1
+ vpinsrd xmm14, xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2
+ vmovdqa xmm15, xmmword ptr [BLAKE3_IV+rip]
+ mov r8, qword ptr [rdi]
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+.p2align 5
+2:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ vpinsrd xmm3, xmm14, eax, 3
+ vmovdqa xmm2, xmm15
+ vmovups xmm8, xmmword ptr [r8+rdx-0x40]
+ vmovups xmm9, xmmword ptr [r8+rdx-0x30]
+ vshufps xmm4, xmm8, xmm9, 136
+ vshufps xmm5, xmm8, xmm9, 221
+ vmovups xmm8, xmmword ptr [r8+rdx-0x20]
+ vmovups xmm9, xmmword ptr [r8+rdx-0x10]
+ vshufps xmm6, xmm8, xmm9, 136
+ vshufps xmm7, xmm8, xmm9, 221
+ vpshufd xmm6, xmm6, 0x93
+ vpshufd xmm7, xmm7, 0x93
+ mov al, 7
+9:
+ vpaddd xmm0, xmm0, xmm4
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 16
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 12
+ vpaddd xmm0, xmm0, xmm5
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 8
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 7
+ vpshufd xmm0, xmm0, 0x93
+ vpshufd xmm3, xmm3, 0x4E
+ vpshufd xmm2, xmm2, 0x39
+ vpaddd xmm0, xmm0, xmm6
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 16
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 12
+ vpaddd xmm0, xmm0, xmm7
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 8
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 7
+ vpshufd xmm0, xmm0, 0x39
+ vpshufd xmm3, xmm3, 0x4E
+ vpshufd xmm2, xmm2, 0x93
+ dec al
+ jz 9f
+ vshufps xmm8, xmm4, xmm5, 214
+ vpshufd xmm9, xmm4, 0x0F
+ vpshufd xmm4, xmm8, 0x39
+ vshufps xmm8, xmm6, xmm7, 250
+ vpblendd xmm9, xmm9, xmm8, 0xAA
+ vpunpcklqdq xmm8, xmm7, xmm5
+ vpblendd xmm8, xmm8, xmm6, 0x88
+ vpshufd xmm8, xmm8, 0x78
+ vpunpckhdq xmm5, xmm5, xmm7
+ vpunpckldq xmm6, xmm6, xmm5
+ vpshufd xmm7, xmm6, 0x1E
+ vmovdqa xmm5, xmm9
+ vmovdqa xmm6, xmm8
+ jmp 9b
+9:
+ vpxor xmm0, xmm0, xmm2
+ vpxor xmm1, xmm1, xmm3
+ mov eax, r13d
+ cmp rdx, r15
+ jne 2b
+ vmovdqu xmmword ptr [rbx], xmm0
+ vmovdqu xmmword ptr [rbx+0x10], xmm1
+ jmp 4b
+.p2align 6
+_blake3_compress_in_place_avx512_asm:
+blake3_compress_in_place_avx512_asm:
+ _CET_ENDBR
+ vmovdqu xmm0, xmmword ptr [rdi]
+ vmovdqu xmm1, xmmword ptr [rdi+0x10]
+ movzx eax, r8b
+ movzx edx, dl
+ shl rax, 32
+ add rdx, rax
+ vmovq xmm3, rcx
+ vmovq xmm4, rdx
+ vpunpcklqdq xmm3, xmm3, xmm4
+ vmovaps xmm2, xmmword ptr [BLAKE3_IV+rip]
+ vmovups xmm8, xmmword ptr [rsi]
+ vmovups xmm9, xmmword ptr [rsi+0x10]
+ vshufps xmm4, xmm8, xmm9, 136
+ vshufps xmm5, xmm8, xmm9, 221
+ vmovups xmm8, xmmword ptr [rsi+0x20]
+ vmovups xmm9, xmmword ptr [rsi+0x30]
+ vshufps xmm6, xmm8, xmm9, 136
+ vshufps xmm7, xmm8, xmm9, 221
+ vpshufd xmm6, xmm6, 0x93
+ vpshufd xmm7, xmm7, 0x93
+ mov al, 7
+9:
+ vpaddd xmm0, xmm0, xmm4
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 16
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 12
+ vpaddd xmm0, xmm0, xmm5
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 8
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 7
+ vpshufd xmm0, xmm0, 0x93
+ vpshufd xmm3, xmm3, 0x4E
+ vpshufd xmm2, xmm2, 0x39
+ vpaddd xmm0, xmm0, xmm6
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 16
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 12
+ vpaddd xmm0, xmm0, xmm7
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 8
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 7
+ vpshufd xmm0, xmm0, 0x39
+ vpshufd xmm3, xmm3, 0x4E
+ vpshufd xmm2, xmm2, 0x93
+ dec al
+ jz 9f
+ vshufps xmm8, xmm4, xmm5, 214
+ vpshufd xmm9, xmm4, 0x0F
+ vpshufd xmm4, xmm8, 0x39
+ vshufps xmm8, xmm6, xmm7, 250
+ vpblendd xmm9, xmm9, xmm8, 0xAA
+ vpunpcklqdq xmm8, xmm7, xmm5
+ vpblendd xmm8, xmm8, xmm6, 0x88
+ vpshufd xmm8, xmm8, 0x78
+ vpunpckhdq xmm5, xmm5, xmm7
+ vpunpckldq xmm6, xmm6, xmm5
+ vpshufd xmm7, xmm6, 0x1E
+ vmovdqa xmm5, xmm9
+ vmovdqa xmm6, xmm8
+ jmp 9b
+9:
+ vpxor xmm0, xmm0, xmm2
+ vpxor xmm1, xmm1, xmm3
+ vmovdqu xmmword ptr [rdi], xmm0
+ vmovdqu xmmword ptr [rdi+0x10], xmm1
+ ret
+
+.p2align 6
+_blake3_compress_xof_avx512_asm:
+blake3_compress_xof_avx512_asm:
+ _CET_ENDBR
+ vmovdqu xmm0, xmmword ptr [rdi]
+ vmovdqu xmm1, xmmword ptr [rdi+0x10]
+ movzx eax, r8b
+ movzx edx, dl
+ shl rax, 32
+ add rdx, rax
+ vmovq xmm3, rcx
+ vmovq xmm4, rdx
+ vpunpcklqdq xmm3, xmm3, xmm4
+ vmovaps xmm2, xmmword ptr [BLAKE3_IV+rip]
+ vmovups xmm8, xmmword ptr [rsi]
+ vmovups xmm9, xmmword ptr [rsi+0x10]
+ vshufps xmm4, xmm8, xmm9, 136
+ vshufps xmm5, xmm8, xmm9, 221
+ vmovups xmm8, xmmword ptr [rsi+0x20]
+ vmovups xmm9, xmmword ptr [rsi+0x30]
+ vshufps xmm6, xmm8, xmm9, 136
+ vshufps xmm7, xmm8, xmm9, 221
+ vpshufd xmm6, xmm6, 0x93
+ vpshufd xmm7, xmm7, 0x93
+ mov al, 7
+9:
+ vpaddd xmm0, xmm0, xmm4
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 16
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 12
+ vpaddd xmm0, xmm0, xmm5
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 8
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 7
+ vpshufd xmm0, xmm0, 0x93
+ vpshufd xmm3, xmm3, 0x4E
+ vpshufd xmm2, xmm2, 0x39
+ vpaddd xmm0, xmm0, xmm6
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 16
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 12
+ vpaddd xmm0, xmm0, xmm7
+ vpaddd xmm0, xmm0, xmm1
+ vpxord xmm3, xmm3, xmm0
+ vprord xmm3, xmm3, 8
+ vpaddd xmm2, xmm2, xmm3
+ vpxord xmm1, xmm1, xmm2
+ vprord xmm1, xmm1, 7
+ vpshufd xmm0, xmm0, 0x39
+ vpshufd xmm3, xmm3, 0x4E
+ vpshufd xmm2, xmm2, 0x93
+ dec al
+ jz 9f
+ vshufps xmm8, xmm4, xmm5, 214
+ vpshufd xmm9, xmm4, 0x0F
+ vpshufd xmm4, xmm8, 0x39
+ vshufps xmm8, xmm6, xmm7, 250
+ vpblendd xmm9, xmm9, xmm8, 0xAA
+ vpunpcklqdq xmm8, xmm7, xmm5
+ vpblendd xmm8, xmm8, xmm6, 0x88
+ vpshufd xmm8, xmm8, 0x78
+ vpunpckhdq xmm5, xmm5, xmm7
+ vpunpckldq xmm6, xmm6, xmm5
+ vpshufd xmm7, xmm6, 0x1E
+ vmovdqa xmm5, xmm9
+ vmovdqa xmm6, xmm8
+ jmp 9b
+9:
+ vpxor xmm0, xmm0, xmm2
+ vpxor xmm1, xmm1, xmm3
+ vpxor xmm2, xmm2, [rdi]
+ vpxor xmm3, xmm3, [rdi+0x10]
+ vmovdqu xmmword ptr [r9], xmm0
+ vmovdqu xmmword ptr [r9+0x10], xmm1
+ vmovdqu xmmword ptr [r9+0x20], xmm2
+ vmovdqu xmmword ptr [r9+0x30], xmm3
+ ret
+
+#ifdef __APPLE__
+.static_data
+#else
+.section .rodata
+#endif
+.p2align 6
+INDEX0:
+ .long 0, 1, 2, 3, 16, 17, 18, 19
+ .long 8, 9, 10, 11, 24, 25, 26, 27
+INDEX1:
+ .long 4, 5, 6, 7, 20, 21, 22, 23
+ .long 12, 13, 14, 15, 28, 29, 30, 31
+ADD0:
+ .long 0, 1, 2, 3, 4, 5, 6, 7
+ .long 8, 9, 10, 11, 12, 13, 14, 15
+ADD1: .long 1
+
+ADD16: .long 16
+BLAKE3_BLOCK_LEN:
+ .long 64
+.p2align 6
+BLAKE3_IV:
+BLAKE3_IV_0:
+ .long 0x6A09E667
+BLAKE3_IV_1:
+ .long 0xBB67AE85
+BLAKE3_IV_2:
+ .long 0x3C6EF372
+BLAKE3_IV_3:
+ .long 0xA54FF53A
--- /dev/null
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <assert.h>
+#include <stdbool.h>
+#include <string.h>
+#include <stdio.h>
+#include <alloca.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "fy-bit64.h"
+
+#include "blake3.h"
+#include "blake3_impl.h"
+#include "blake3_internal.h"
+
+// Declarations for implementation-specific functions.
+void blake3_compress_in_place_portable(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags);
+
+void blake3_compress_xof_portable(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags, uint8_t out[64]);
+
+void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+
+/* the portable hasher (can drive the optimized ones, but not optimally) */
+extern const blake3_hasher_ops blake3_hasher_op_portable;
+
+#if defined(IS_X86)
+void blake3_compress_in_place_sse2(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags);
+void blake3_compress_xof_sse2(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags, uint8_t out[64]);
+void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+void blake3_compress_in_place_sse2_asm(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags);
+void blake3_compress_xof_sse2_asm(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags, uint8_t out[64]);
+void blake3_hash_many_sse2_asm(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+
+extern const blake3_hasher_ops blake3_hasher_op_sse2;
+
+void blake3_compress_in_place_sse41(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags);
+void blake3_compress_xof_sse41(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags, uint8_t out[64]);
+void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+void blake3_compress_in_place_sse41_asm(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags);
+void blake3_compress_xof_sse41_asm(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags, uint8_t out[64]);
+void blake3_hash_many_sse41_asm(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+
+extern const blake3_hasher_ops blake3_hasher_op_sse41;
+
+void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+void blake3_hash_many_avx2_asm(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+
+extern const blake3_hasher_ops blake3_hasher_op_avx2;
+
+void blake3_compress_in_place_avx512(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags);
+void blake3_compress_xof_avx512(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags, uint8_t out[64]);
+void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+void blake3_compress_in_place_avx512_asm(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags);
+void blake3_compress_xof_avx512_asm(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags, uint8_t out[64]);
+void blake3_hash_many_avx512_asm(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+
+extern const blake3_hasher_ops blake3_hasher_op_avx512;
+
+#endif
+
+#if defined(IS_ARM)
+void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out);
+
+extern const blake3_hasher_ops blake3_hasher_op_neon;
+#endif
+
+static uint64_t supported_cpusimd_backend(void)
+{
+ const blake3_backend *be = &blake3_backends[B3BID_CPUSIMD];
+ return be->user ? B3BF_CPUSIMD : 0;
+}
+
+static uint64_t detected_cpusimd_backend(void)
+{
+ const blake3_backend *be = &blake3_backends[B3BID_CPUSIMD];
+ return be->user ? B3BF_CPUSIMD : 0;
+}
+
+static uint64_t supported_synthetic_backends(void)
+{
+ uint64_t backends = 0;
+
+ backends |= supported_cpusimd_backend();
+ return backends;
+}
+
+static uint64_t detected_synthetic_backends(void)
+{
+ uint64_t backends = 0;
+
+ backends |= detected_cpusimd_backend();
+ return backends;
+}
+
+static uint64_t supported_gpu_backends(void)
+{
+ return 0;
+}
+
+static uint64_t detected_gpu_backends(void)
+{
+ return 0;
+}
+
+#if defined(IS_X86)
+
+static uint64_t supported_backends_x86(void)
+{
+ uint64_t backends = 0;
+
+#if !defined(BLAKE3_NO_SSE2)
+ backends |= B3BF_SSE2 | B3BF_SSE2_ASM;
+#endif
+#if !defined(BLAKE3_NO_SSE41)
+ backends |= B3BF_SSE41 | B3BF_SSE41_ASM;
+#endif
+#if !defined(BLAKE3_NO_AVX)
+ backends |= B3BF_AVX2 | B3BF_AVX2_ASM;
+#endif
+#if !defined(BLAKE3_NO_AVX512)
+ backends |= B3BF_AVX512 | B3BF_AVX512_ASM;
+#endif
+ return backends;
+}
+
+static inline uint64_t xgetbv(void)
+{
+#if defined(_MSC_VER)
+ return _xgetbv(0);
+#else
+ uint32_t eax = 0, edx = 0;
+ __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0));
+ return ((uint64_t)edx << 32) | eax;
+#endif
+}
+
+static inline void cpuid(uint32_t out[4], uint32_t id)
+{
+#if defined(_MSC_VER)
+ __cpuid((int *)out, id);
+#elif defined(__i386__) || defined(_M_IX86)
+ __asm__ __volatile__(
+ "movl %%ebx, %1\n"
+ "cpuid\n"
+ "xchgl %1, %%ebx\n"
+ : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3])
+ : "a"(id));
+#else
+ __asm__ __volatile__("cpuid\n"
+ : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3])
+ : "a"(id));
+#endif
+}
+
+static inline void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid)
+{
+#if defined(_MSC_VER)
+ __cpuidex((int *)out, id, sid);
+#elif defined(__i386__) || defined(_M_IX86)
+ __asm__ __volatile__(
+ "movl %%ebx, %1\n"
+ "cpuid\n"
+ "xchgl %1, %%ebx\n"
+ : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3])
+ : "a"(id), "c"(sid));
+#else
+ __asm__ __volatile__("cpuid\n"
+ : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3])
+ : "a"(id), "c"(sid));
+#endif
+}
+
+static uint64_t detected_backends_x86(void)
+{
+ uint32_t regs[4] = {0};
+ uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3];
+ int max_id;
+ uint64_t mask = 0;
+ uint64_t backends = 0;
+
+ (void)edx;
+
+ cpuid(regs, 0);
+ max_id = *eax;
+ cpuid(regs, 1);
+
+#if defined(__amd64__) || defined(_M_X64)
+ backends |= B3BF_SSE2 | B3BF_SSE2_ASM;
+#else
+ if (*edx & (1UL << 26))
+ backends |= B3BF_SSE2 | B3BF_SSE2_ASM;
+#endif
+ if (*ecx & (1UL << 19))
+ backends |= B3BF_SSE41 | B3BF_SSE41_ASM;
+
+ if (*ecx & (1UL << 27)) { // OSXSAVE
+ mask = xgetbv();
+ if ((mask & 6) == 6) { // SSE and AVX states
+ if (max_id >= 7) {
+ cpuidex(regs, 7, 0);
+ if (*ebx & (1UL << 5))
+ backends |= B3BF_AVX2 | B3BF_AVX2_ASM;
+ if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm
+ if ((*ebx & ((1UL << 31) | (1UL << 16))) == ((1UL << 31) | (1UL << 16)))
+ backends |= B3BF_AVX512 | B3BF_AVX512_ASM; /* AVX412VL+AVX512F */
+ }
+ }
+ }
+ }
+
+ return backends;
+}
+#endif
+
+#if defined(IS_ARM)
+static uint64_t supported_backends_arm(void)
+{
+ uint64_t backends = 0;
+
+#if defined(IS_AARCH64)
+ backends |= B3BF_NEON;
+#endif
+
+ return backends;
+}
+
+static uint64_t detected_backends_arm(void)
+{
+ uint64_t backends = 0;
+
+#if defined(IS_AARCH64)
+ backends |= B3BF_NEON;
+#endif
+
+ return backends;
+}
+#endif
+
+uint64_t blake3_get_supported_backends(void)
+{
+ uint64_t supported_backends;
+
+ supported_backends = B3BF_PORTABLE;
+
+ supported_backends |= supported_gpu_backends();
+
+#if defined(IS_X86)
+ supported_backends |= supported_backends_x86();
+#endif
+
+#if defined(IS_ARM)
+ supported_backends |= supported_backends_arm();
+#endif
+
+ supported_backends |= supported_synthetic_backends();
+
+ return supported_backends;
+}
+
+uint64_t blake3_get_detected_backends(void)
+{
+ uint64_t detected_backends;
+
+ detected_backends = B3BF_PORTABLE;
+
+ detected_backends |= detected_gpu_backends();
+
+#if defined(IS_X86)
+ detected_backends |= detected_backends_x86();
+#endif
+
+#if defined(IS_ARM)
+ detected_backends |= detected_backends_arm();
+#endif
+
+ detected_backends |= detected_synthetic_backends();
+
+ return detected_backends;
+}
+
+uint64_t blake3_get_selectable_backends(void)
+{
+ return blake3_get_supported_backends() |
+ blake3_get_detected_backends();
+}
+
+blake3_backend blake3_backends[B3BID_COUNT] = {
+ [B3BID_PORTABLE] = {
+ .info = {
+ .id = B3BID_PORTABLE,
+ .name = "portable",
+ .description = "portable C implementation",
+ .simd_degree = 1,
+ .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE,
+ },
+ .hasher_ops = &blake3_hasher_op_portable,
+ .hash_many = blake3_hash_many_portable,
+ .compress_xof = blake3_compress_xof_portable,
+ .compress_in_place = blake3_compress_in_place_portable,
+ },
+#if defined(IS_X86)
+
+#if defined(TARGET_HAS_SSE2) && TARGET_HAS_SSE2
+ [B3BID_SSE2] = {
+ .info = {
+ .id = B3BID_SSE2,
+ .name = "sse2",
+ .description = "x86 SSE2 implementation in C",
+ .simd_degree = 4,
+ .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE,
+ },
+ .hasher_ops = &blake3_hasher_op_sse2,
+ .hash_many = blake3_hash_many_sse2,
+ .compress_xof = blake3_compress_xof_sse2,
+ .compress_in_place = blake3_compress_in_place_sse2,
+ },
+ [B3BID_SSE2_ASM] = {
+ .info = {
+ .id = B3BID_SSE2_ASM,
+ .name = "sse2-asm",
+ .description = "x86 SSE2 implementation in assembly",
+ .simd_degree = 4,
+ .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE,
+ },
+ .hasher_ops = &blake3_hasher_op_sse2,
+ .hash_many = blake3_hash_many_sse2_asm,
+ .compress_xof = blake3_compress_xof_sse2_asm,
+ .compress_in_place = blake3_compress_in_place_sse2_asm,
+ },
+#endif
+
+#if defined(TARGET_HAS_SSE41) && TARGET_HAS_SSE41
+ [B3BID_SSE41] = {
+ .info = {
+ .id = B3BID_SSE41,
+ .name = "sse41",
+ .description = "x86 SSE41 implementation in C",
+ .simd_degree = 4,
+ .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE,
+ },
+ .hasher_ops = &blake3_hasher_op_sse41,
+ .hash_many = blake3_hash_many_sse41,
+ .compress_xof = blake3_compress_xof_sse41,
+ .compress_in_place = blake3_compress_in_place_sse41,
+ },
+ [B3BID_SSE41_ASM] = {
+ .info = {
+ .id = B3BID_SSE41_ASM,
+ .name = "sse41-asm",
+ .description = "x86 SSE41 implementation in assembly",
+ .simd_degree = 4,
+ .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE,
+ },
+ .hasher_ops = &blake3_hasher_op_sse41,
+ .hash_many = blake3_hash_many_sse41_asm,
+ .compress_xof = blake3_compress_xof_sse41_asm,
+ .compress_in_place = blake3_compress_in_place_sse41_asm,
+ },
+#endif
+
+#if defined(TARGET_HAS_AVX2) && TARGET_HAS_AVX2
+ [B3BID_AVX2] = {
+ .info = {
+ .id = B3BID_AVX2,
+ .name = "avx2",
+ .description = "x86 AVX2 implementation in C",
+ .simd_degree = 8,
+ .funcs = B3FF_HASH_MANY,
+ },
+ .hasher_ops = &blake3_hasher_op_avx2,
+ .hash_many = blake3_hash_many_avx2,
+ },
+ [B3BID_AVX2_ASM] = {
+ .info = {
+ .id = B3BID_AVX2_ASM,
+ .name = "avx2-asm",
+ .description = "x86 AVX2 implementation in assembly",
+ .simd_degree = 8,
+ .funcs = B3FF_HASH_MANY,
+ },
+ .hasher_ops = &blake3_hasher_op_avx2,
+ .hash_many = blake3_hash_many_avx2_asm,
+ },
+#endif
+#if defined(TARGET_HAS_AVX512) && TARGET_HAS_AVX512
+ [B3BID_AVX512] = {
+ .info = {
+ .id = B3BID_AVX512,
+ .name = "avx512",
+ .description = "x86 AVX512 VL+F implementation in C",
+ .simd_degree = 16,
+ .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE,
+ },
+ .hasher_ops = &blake3_hasher_op_avx512,
+ .hash_many = blake3_hash_many_avx512,
+ .compress_xof = blake3_compress_xof_avx512,
+ .compress_in_place = blake3_compress_in_place_avx512,
+ },
+ [B3BID_AVX512_ASM] = {
+ .info = {
+ .id = B3BID_AVX512_ASM,
+ .name = "avx512-asm",
+ .description = "x86 AVX512 VL+F implementation in assembly",
+ .simd_degree = 16,
+ .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE,
+ },
+ .hasher_ops = &blake3_hasher_op_avx512,
+ .hash_many = blake3_hash_many_avx512,
+ .compress_xof = blake3_compress_xof_avx512_asm,
+ .compress_in_place = blake3_compress_in_place_avx512_asm,
+ },
+#endif
+
+#endif
+
+#if defined(IS_ARM)
+
+#if defined(TARGET_HAS_NEON) && TARGET_HAS_NEON
+ [B3BID_NEON] = {
+ .info = {
+ .id = B3BID_NEON,
+ .name = "neon",
+ .description = "arm NEON implementation",
+ .simd_degree = 4,
+ .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE,
+ },
+ .hasher_ops = &blake3_hasher_op_neon,
+ .hash_many = blake3_hash_many_neon,
+ .compress_xof = blake3_compress_xof_portable, /* no NEON for this */
+ .compress_in_place = blake3_compress_in_place_portable,
+ },
+#endif
+
+#endif
+};
+
+const blake3_backend *blake3_backend_select_function(uint64_t selectable_backends, blake3_func_id fid)
+{
+ const blake3_backend *be;
+ unsigned int i;
+
+ while (selectable_backends) {
+ i = highest_one(selectable_backends);
+ selectable_backends &= ~FY_BIT64(i);
+ be = blake3_backends + i;
+ if (be->info.funcs & FY_BIT64(fid))
+ return be;
+ }
+ /* should never happen because portable is always set */
+ assert(false);
+ return NULL;
+}
+
+const blake3_backend *blake3_get_backend_by_id(blake3_backend_id id)
+{
+ const blake3_backend *be;
+
+ if ((unsigned int)id >= B3BID_COUNT)
+ return NULL;
+
+ be = &blake3_backends[id];
+
+ /* non-enabled backend? */
+ if (be->info.id != id || !be->info.name)
+ return NULL;
+
+ return be;
+}
+
+const blake3_backend *blake3_get_backend_by_name(const char *name)
+{
+ const blake3_backend *be;
+ unsigned int i;
+
+ if (!name)
+ return NULL;
+
+ for (i = 0, be = blake3_backends; i < B3BID_COUNT; i++, be++) {
+ if (be->info.name && !strcmp(be->info.name, name))
+ return be;
+ }
+ return NULL;
+}
+
+const blake3_backend_info *blake3_get_backend_info(blake3_backend_id id)
+{
+ const blake3_backend_info *bei;
+
+ if ((unsigned int)id >= B3BID_COUNT)
+ return NULL;
+
+ bei = &blake3_backends[id].info;
+
+ /* not supported */
+ if (bei->id == B3BID_INVALID || !bei->name)
+ return NULL;
+
+ return bei;
+}
--- /dev/null
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <assert.h>
+#include <stdbool.h>
+#include <string.h>
+#include <stdio.h>
+#include <alloca.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "blake3.h"
+#include "blake3_impl.h"
+#include "blake3_internal.h"
+
+#include "fy-thread.h"
+
+struct cpusimd_data {
+ unsigned int num_cpus;
+ unsigned int simd_cpus;
+ unsigned int be_simd_degree_mult;
+ const blake3_backend *be_best;
+ char *description;
+ struct fy_thread_pool *tp;
+};
+
+static void blake3_cpusimd_hash_many_thread(void *arg)
+{
+ blake3_hash_many_state *s = arg;
+ const blake3_hash_many_common_state *c;
+
+ c = s->common;
+ c->hash_many(s->inputs, s->num_inputs, c->blocks, c->key, s->counter, c->increment_counter, c->flags, c->flags_start, c->flags_end, s->out);
+}
+
+// #define CPUSIMD_CHECK
+
+static void blake3_hash_many_cpusimd(
+ const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out)
+{
+ struct cpusimd_data *d;
+ const blake3_backend *be, *be_best;
+ unsigned int i, stepi, be_simd_degree_mult, simd_degree, count, states_count;
+ blake3_hash_many_state *states, *s;
+ blake3_hash_many_common_state common_local, *common = &common_local;
+ const uint8_t *const *inputsi;
+ uint8_t *outi;
+ uint64_t counteri;
+#ifdef CPUSIMD_CHECK
+ size_t out_size;
+ uint8_t *out_cmp;
+#endif
+
+ (void)simd_degree;
+
+ be = &blake3_backends[B3BID_CPUSIMD];
+ d = be->user;
+ assert(d);
+
+ count = (unsigned int)num_inputs;
+ simd_degree = be->info.simd_degree;
+
+ be_best = d->be_best;
+ be_simd_degree_mult = d->be_simd_degree_mult;
+
+#ifdef CPUSIMD_CHECK
+ /* if less than the underlyind backend degree just do it there */
+ out_size = BLAKE3_OUT_LEN * num_inputs;
+ out_cmp = alloca(out_size);
+ be_best->hash_many(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out_cmp);
+#endif
+
+ states_count = (count / be_simd_degree_mult) + !!(count % be_simd_degree_mult);
+ states = alloca(sizeof(*states) * states_count);
+
+ /* the common */
+ common->hash_many = be_best->hash_many;
+ common->blocks = blocks;
+ common->key = key;
+ common->increment_counter = increment_counter;
+ common->flags = flags;
+ common->flags_start = flags_start;
+ common->flags_end = flags_end;
+
+ inputsi = inputs;
+ outi = out;
+ counteri = counter;
+ for (i = 0, s = states; i < count; i += stepi, s++) {
+
+ stepi = (i + be_simd_degree_mult) <= count ? be_simd_degree_mult : (count - i);
+
+ s->inputs = inputsi;
+ s->num_inputs = stepi;
+ s->counter = counteri;
+ s->out = outi;
+ s->common = common;
+
+ if (increment_counter)
+ counteri += stepi;
+ inputsi += stepi;
+ outi += BLAKE3_OUT_LEN * stepi;
+ }
+
+ fy_thread_arg_array_join(d->tp, blake3_cpusimd_hash_many_thread, NULL, states, sizeof(states[0]), states_count);
+
+#ifdef CPUSIMD_CHECK
+ for (i = 0; i < count; i += stepi) {
+ stepi = (i + be_simd_degree_mult) <= count ? be_simd_degree_mult : (count - i);
+
+ if (memcmp(out + i * BLAKE3_OUT_LEN, out_cmp + i * BLAKE3_OUT_LEN, BLAKE3_OUT_LEN)) {
+ fprintf(stderr, "%s: differ at #%u\n", __func__, i);
+ }
+ }
+#endif
+}
+
+static void cpusimd_data_free(struct cpusimd_data *d)
+{
+ if (!d)
+ return;
+
+ if (d->description)
+ free(d->description);
+
+ if (d->tp)
+ fy_thread_pool_destroy(d->tp);
+
+ free(d);
+}
+
+int blake3_backend_cpusimd_setup(unsigned int num_cpus, unsigned int mult_fact)
+{
+ blake3_backend *be = &blake3_backends[B3BID_CPUSIMD];
+ const blake3_backend *be_best;
+ struct cpusimd_data *d = NULL;
+ struct fy_thread_pool_cfg tp_cfg;
+ long scval;
+ uint64_t supported_backends, detected_backends, selectable_backends;
+ unsigned int num_simd_cpus;
+ ssize_t n;
+
+ if (!num_cpus) {
+ scval = sysconf(_SC_NPROCESSORS_ONLN);
+ assert(scval > 0);
+ num_cpus = (unsigned int)scval;
+ }
+
+ if (num_cpus <= 1)
+ return 0;
+
+ num_simd_cpus = round_down_to_power_of_2(num_cpus);
+
+ if (!mult_fact)
+ mult_fact = 1;
+
+ d = malloc(sizeof(*d));
+ if (!d)
+ goto err_out;
+
+ memset(d, 0, sizeof(*d));
+
+ memset(&tp_cfg, 0, sizeof(tp_cfg));
+ tp_cfg.flags = 0;
+ tp_cfg.num_threads = num_simd_cpus;
+ tp_cfg.userdata = NULL;
+
+ d->tp = fy_thread_pool_create(&tp_cfg);
+ if (!d->tp)
+ goto err_out;
+
+ /* probe for available backends */
+ supported_backends = blake3_get_supported_backends();
+ detected_backends = blake3_get_detected_backends();
+ selectable_backends = supported_backends & detected_backends;
+
+ /* remove ourselves (and anything above) */
+ selectable_backends &= B3BF_BIT(B3BID_CPUSIMD) - 1;
+
+ /* select the best one for hash many */
+ be_best = blake3_backend_select_function(selectable_backends, B3FID_HASH_MANY);
+ assert(be_best);
+
+ d->num_cpus = num_cpus;
+ d->simd_cpus = num_simd_cpus;
+ d->be_best = be_best;
+ d->be_simd_degree_mult = be_best->info.simd_degree * mult_fact;
+
+ be->hasher_ops = be_best->hasher_ops;
+
+ d->description = NULL;
+ n = 0;
+ for (;;) {
+ n = snprintf(d->description, n ? (n + 1) : 0, "SIMD like acceleration using %u CPUs (using %s x %u) x %u = total x %u",
+ d->simd_cpus,
+ d->be_best->info.name, d->be_best->info.simd_degree,
+ mult_fact,
+ d->be_simd_degree_mult * d->simd_cpus);
+ if (d->description)
+ break;
+
+ d->description = malloc(n + 1);
+ if (!d->description)
+ goto err_out;
+ }
+
+ be->user = d;
+
+ be->info.id = B3BID_CPUSIMD;
+ be->info.name = "cpusimd";
+ be->info.simd_degree = d->simd_cpus * d->be_best->info.simd_degree * mult_fact;
+ be->info.description = d->description;
+
+ be->hash_many = blake3_hash_many_cpusimd;
+ be->info.funcs = B3FF_HASH_MANY;
+
+ return 0;
+
+err_out:
+ cpusimd_data_free(d);
+ return -1;
+}
+
+void blake3_backend_cpusimd_cleanup(void)
+{
+ blake3_backend *be = &blake3_backends[B3BID_CPUSIMD];
+ struct cpusimd_data *d;
+
+ d = be->user;
+ if (!d)
+ return;
+ cpusimd_data_free(d);
+
+ memset(be, 0, sizeof(*be));
+}
--- /dev/null
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <assert.h>
+#include <stdbool.h>
+#include <string.h>
+#include <stdio.h>
+#include <alloca.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+
+#include "fy-bit64.h"
+
+#include "blake3.h"
+#include "blake3_impl.h"
+#include "blake3_internal.h"
+
+#include "fy-thread.h"
+
+/* 256K threshold for using alloca */
+#define BLAKE3_ALLOCA_BUFFER_SIZE (256U << 10)
+#define BLAKE3_FILE_IO_BUFFER_SIZE BLAKE3_ALLOCA_BUFFER_SIZE
+#define BLAKE3_MMAP_MIN_CHUNKSIZE (1U << 20) // minimum chunksize is 1MB
+#define BLAKE3_MMAP_MAX_CHUNKSIZE SIZE_MAX
+
+static int probe_backends(blake3_host_state *hs)
+{
+ hs->supported_backends = blake3_get_supported_backends();
+ hs->detected_backends = blake3_get_detected_backends();
+ hs->selectable_backends = hs->supported_backends & hs->detected_backends;
+
+ return 0;
+}
+
+static int select_backends(blake3_host_state *hs)
+{
+ const blake3_backend *be, *be_portable;
+
+ /* this is the fallback */
+ be_portable = blake3_get_backend_by_id(B3BID_PORTABLE);
+ assert(be_portable);
+
+ /* if there's a backend selected, try to force it */
+ if (hs->cfg.backend && hs->cfg.backend[0] && strcmp(hs->cfg.backend, "auto")) {
+ be = blake3_get_backend_by_name(hs->cfg.backend);
+
+ /* if backend exists and is selectable */
+ if (be && (hs->selectable_backends & FY_BIT64(be->info.id))) {
+ hs->hash_many_be = (be->info.funcs & B3FF_HASH_MANY) ? be : be_portable;
+ hs->compress_xof_be = (be->info.funcs & B3FF_COMPRESS_XOF) ? be : be_portable;
+ hs->compress_in_place_be = (be->info.funcs & B3FF_COMPRESS_IN_PLACE) ? be : be_portable;
+
+ /* and select the ops (piggyback on HASH_MANY) */
+ hs->hasher_ops = (be->info.funcs & B3FF_HASH_MANY) ? be->hasher_ops : be_portable->hasher_ops;
+ }
+ }
+
+ if (!hs->hash_many_be) {
+ hs->hash_many_be = blake3_backend_select_function(hs->selectable_backends, B3FID_HASH_MANY);
+ hs->hasher_ops = hs->hash_many_be->hasher_ops;
+ }
+ if (!hs->compress_xof_be)
+ hs->compress_xof_be = blake3_backend_select_function(hs->selectable_backends, B3FID_COMPRESS_XOF);
+ if (!hs->compress_in_place_be)
+ hs->compress_in_place_be = blake3_backend_select_function(hs->selectable_backends, B3FID_COMPRESS_IN_PLACE);
+
+ /* select methods */
+ hs->hash_many = hs->hash_many_be->hash_many;
+ hs->compress_xof = hs->compress_xof_be->compress_xof;
+ hs->compress_in_place = hs->compress_in_place_be->compress_in_place;
+
+ /* select maximum simd degree */
+ if (hs->simd_degree < hs->hash_many_be->info.simd_degree)
+ hs->simd_degree = hs->hash_many_be->info.simd_degree;
+ if (hs->simd_degree < hs->compress_xof_be->info.simd_degree)
+ hs->simd_degree = hs->compress_xof_be->info.simd_degree;
+ if (hs->simd_degree < hs->compress_in_place_be->info.simd_degree)
+ hs->simd_degree = hs->compress_in_place_be->info.simd_degree;
+
+ return 0;
+}
+
+static void dump_backends(blake3_host_state *hs, uint64_t backends)
+{
+ const blake3_backend *be;
+ int i;
+
+ for (i = 0; i < B3BID_COUNT; i++) {
+ if (!(backends & FY_BIT64(i)))
+ continue;
+ be = blake3_get_backend_by_id(i);
+ assert(be);
+
+ fprintf(stderr, "%sname: %s\n%sdescription: %s\n%ssimd_degree: %u\n%shas_hash_many: %s\n%shas_compress_xof: %s\n%shas_compress_in_place: %s\n",
+ " -", be->info.name,
+ " ", be->info.description,
+ " ", be->info.simd_degree,
+ " ", (be->info.funcs & B3FF_HASH_MANY) ? "true" : "false",
+ " ", (be->info.funcs & B3FF_COMPRESS_XOF) ? "true" : "false",
+ " ", (be->info.funcs & B3FF_COMPRESS_IN_PLACE) ? "true" : "false");
+ }
+}
+
+int blake3_host_state_setup(blake3_host_state *hs, const blake3_host_config *cfg)
+{
+ struct fy_thread_pool_cfg tp_cfg;
+ long scval;
+ int rc;
+
+ (void)rc;
+
+ assert(hs);
+ assert(cfg);
+
+ memset(hs, 0, sizeof(*hs));
+ hs->cfg = *cfg;
+
+ scval = sysconf(_SC_NPROCESSORS_ONLN);
+ assert(scval > 0);
+
+ hs->num_cpus = (unsigned int)scval;
+
+ rc = probe_backends(hs);
+ assert(!rc);
+
+ rc = select_backends(hs);
+ assert(!rc);
+
+ hs->mt_degree = hs->cfg.mt_degree > 0 ? hs->cfg.mt_degree : 64; /* 64K default */
+ hs->tp = NULL;
+
+ if (!hs->cfg.no_mthread) {
+
+ if (!hs->cfg.tp) {
+ memset(&tp_cfg, 0, sizeof(tp_cfg));
+ tp_cfg.flags = FYTPCF_STEAL_MODE;
+ tp_cfg.num_threads = hs->cfg.num_threads ? hs->cfg.num_threads : (hs->num_cpus * 3) / 2;
+ tp_cfg.userdata = NULL;
+ hs->tp = fy_thread_pool_create(&tp_cfg);
+ if (!hs->tp)
+ goto err_out;
+ } else
+ hs->tp = hs->cfg.tp;
+ }
+
+ hs->file_io_bufsz = hs->cfg.file_io_bufsz ? hs->cfg.file_io_bufsz : BLAKE3_FILE_IO_BUFFER_SIZE;
+ hs->mmap_min_chunk = hs->cfg.mmap_min_chunk ? hs->cfg.mmap_min_chunk : BLAKE3_MMAP_MIN_CHUNKSIZE;
+ hs->mmap_max_chunk = hs->cfg.mmap_max_chunk ? hs->cfg.mmap_max_chunk : BLAKE3_MMAP_MAX_CHUNKSIZE;
+
+ if (hs->cfg.debug) {
+
+ fprintf(stderr, "num_cpus: %u\n", hs->num_cpus);
+ fprintf(stderr, "num_threads: %d\n", hs->tp ? fy_thread_pool_get_num_threads(hs->tp) : 0);
+ fprintf(stderr, "simd_degree: %u\n", hs->simd_degree);
+ fprintf(stderr, "mt_degree: %u\n", hs->mt_degree);
+ fprintf(stderr, "file_io_bufsz: %zu\n", hs->file_io_bufsz);
+ fprintf(stderr, "mmap_min_chunk: %zu\n", hs->mmap_min_chunk);
+ fprintf(stderr, "mmap_max_chunk: %zu\n", hs->mmap_max_chunk);
+
+ fprintf(stderr, "supported_backends:\n");
+ dump_backends(hs, hs->supported_backends);
+ fprintf(stderr, "detected_backends:\n");
+ dump_backends(hs, hs->detected_backends);
+
+ fprintf(stderr, "selected-backends:\n");
+ fprintf(stderr, " hash_many: %s\n", hs->hash_many_be->info.name);
+ fprintf(stderr, " compress_xof: %s\n", hs->compress_xof_be->info.name);
+ fprintf(stderr, " compress_in_place: %s\n", hs->compress_in_place_be->info.name);
+
+ }
+
+ return 0;
+
+err_out:
+ return -1;
+}
+
+void blake3_host_state_cleanup(blake3_host_state *hs)
+{
+ assert(hs);
+
+ /* destroy the thread pool if we're the ones created it */
+ if (hs->tp && !hs->cfg.tp)
+ fy_thread_pool_destroy(hs->tp);
+}
+
+blake3_host_state *blake3_host_state_create(const blake3_host_config *cfg)
+{
+ blake3_host_state *hs;
+ int rc;
+
+ hs = malloc(sizeof(*hs));
+ if (!hs)
+ return NULL;
+ rc = blake3_host_state_setup(hs, cfg);
+ if (rc)
+ goto err_out;
+
+ return hs;
+
+err_out:
+ if (hs)
+ free(hs);
+ return NULL;
+}
+
+void blake3_host_state_destroy(blake3_host_state *hs)
+{
+ if (!hs)
+ return;
+ blake3_host_state_cleanup(hs);
+ free(hs);
+}
+
+blake3_hasher *blake3_hasher_create(blake3_host_state *hs, const uint8_t *key, const void *context, size_t context_len)
+{
+ blake3_hasher *self;
+
+ self = fy_cacheline_alloc(sizeof(*self));
+ if (!self)
+ return NULL;
+
+ if (!key && !context)
+ blake3_hasher_init(hs, self);
+ else if (key)
+ blake3_hasher_init_keyed(hs, self, key);
+ else if (context_len == 0)
+ blake3_hasher_init_derive_key(hs, self, context);
+ else
+ blake3_hasher_init_derive_key_raw(hs, self, context, context_len);
+ return self;
+}
+
+void blake3_hasher_destroy(blake3_hasher *self)
+{
+ if (!self)
+ return;
+ fy_cacheline_free(self);
+}
+
+#if defined(__linux__)
+
+static int linux_block_dev_is_rotational(dev_t dev)
+{
+ char *path = NULL;
+ int rc, fd;
+ uint8_t c[2];
+ int rotational;
+
+ /* by default it's the filesize */
+ rotational = -1; /* not found */
+
+ /* read the rotational attribute of the root */
+ rc = asprintf(&path, "/sys/dev/block/%u:%u/queue/rotational", major(dev), 0);
+ if (rc != -1) {
+ fd = open(path, O_RDONLY);
+ if (fd != -1) {
+ if (read(fd, c, 2) == 2)
+ rotational = c[0] == '1' && c[1] == '\n';
+ close(fd);
+ }
+ free(path);
+ }
+
+ return rotational;
+}
+
+static ssize_t linux_file_cached_size(int fd, void *mem, size_t filesize)
+{
+ long pagesize = sysconf(_SC_PAGESIZE);
+ size_t vec_size, resident_size, i;
+ unsigned char *vec;
+ int rc;
+
+ (void)rc;
+ vec_size = (filesize + pagesize - 1) / pagesize;
+ vec = malloc(vec_size + 1);
+ if (!vec)
+ return -1;
+
+ rc = mincore(mem, filesize, vec);
+ assert(!rc);
+
+ /* this could be parallelized */
+ resident_size = 0;
+ for (i = 0; i < vec_size; i++) {
+ if (vec[i] & 1)
+ resident_size += pagesize;
+ }
+
+ free(vec);
+
+ if (resident_size > filesize)
+ resident_size = filesize;
+
+ return (ssize_t)resident_size;
+}
+
+static size_t blake3_mmap_file_chunksize(int fd, dev_t dev, void *mem,
+ size_t filesize,
+ size_t mmap_min_chunk, size_t mmap_max_chunk)
+{
+ size_t chunksize;
+ ssize_t cached_size;
+ int rotational;
+
+ if (filesize <= mmap_min_chunk)
+ return filesize;
+
+ /* by default it's the filesize */
+ chunksize = filesize;
+
+ /* starting, check if the rotational attribute exists in this dev */
+ rotational = linux_block_dev_is_rotational(dev);
+ if (rotational == -1) {
+ /* attribute not found? check the non-partition */
+ rotational = linux_block_dev_is_rotational(makedev(major(dev), 0));
+ }
+
+ if (rotational == 1) {
+ /* OK, it's rotational, but is it in cache?
+ * to avoid checking for the cached status of the whole file
+ * we just probe the MIN_CHUNKSIZE
+ * We will thrash in the case where the file is only cached
+ * for the first few bytes, but this is generally unusual.
+ */
+ cached_size = linux_file_cached_size(fd, mem, mmap_min_chunk);
+
+ if (cached_size <= 0)
+ chunksize = mmap_min_chunk;
+ }
+
+ if (chunksize > mmap_max_chunk)
+ chunksize = mmap_max_chunk;
+
+ return chunksize;
+}
+
+#else
+
+static size_t blake3_mmap_file_chunksize(int fd, dev_t dev, void *mem,
+ size_t filesize,
+ size_t mmap_min_chunk, size_t mmap_max_chunk)
+{
+ if (filesize <= mmap_min_chunk)
+ return filesize;
+
+ if (filesize > mmap_max_chunk)
+ return mmap_max_chunk;
+
+ /* for all others, just use mmap at max */
+ return filesize;
+}
+
+#endif
+
+int blake3_hash_file(blake3_hasher *hasher, const char *filename,
+ uint8_t output[BLAKE3_OUT_LEN])
+{
+ blake3_host_state *hs;
+ FILE *fp = NULL;
+ void *mem = NULL, *buf = NULL, *p;
+ int fd = -1, ret = -1;
+ size_t rdn, filesize, bufsz = 0, max_chunk, left, chunk;
+ struct stat sb;
+ dev_t dev = 0;
+ int rc;
+
+ if (!hasher || !filename || !output)
+ return -1;
+
+ hs = hasher->hs;
+
+ if (hs->cfg.debug)
+ fprintf(stderr, "processing file %s\n", filename);
+
+ // reset the hasher (do not initialize again)
+ blake3_hasher_reset(hasher);
+
+ if (!strcmp(filename, "-")) {
+ fp = stdin;
+ } else {
+ fp = NULL;
+
+ fd = open(filename, O_RDONLY);
+ if (fd < 0) {
+ if (hs->cfg.debug)
+ fprintf(stderr, "unable to open %s - %s\n",
+ filename, strerror(errno));
+ goto err_out;
+ }
+
+ rc = fstat(fd, &sb);
+ if (rc < 0) {
+ if (hs->cfg.debug)
+ fprintf(stderr, "failed to stat %s - %s\n",
+ filename, strerror(errno));
+ goto err_out;
+ }
+
+ if (!S_ISREG(sb.st_mode)) {
+ if (S_ISDIR(sb.st_mode))
+ errno = EISDIR;
+ else
+ errno = EINVAL;
+ if (hs->cfg.debug)
+ fprintf(stderr, "not a regular file %s - %s\n",
+ filename, strerror(errno));
+ goto err_out;
+ }
+
+ filesize = (size_t)-1;
+
+ /* try to mmap */
+ if (sb.st_size > 0 && !hs->cfg.no_mmap) {
+ filesize = sb.st_size;
+ mem = mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+ if (mem != MAP_FAILED) {
+ close(fd);
+ fd = -1;
+ dev = sb.st_dev;
+ } else
+ mem = NULL;
+ }
+
+ /* unable to map? fallback to stream mode */
+ if (!mem) {
+ fp = fdopen(fd, "r");
+ if (!fp) {
+ if (hs->cfg.debug)
+ fprintf(stderr, "unable to fdopen %s - %s\n",
+ filename, strerror(errno));
+ goto err_out;
+ }
+ }
+ }
+
+ /* mmap case, very simple */
+ if (mem) {
+
+ /* if the file is small enough don't bother with the rest */
+ max_chunk = blake3_mmap_file_chunksize(fd, dev, mem, filesize,
+ hs->mmap_min_chunk, hs->mmap_max_chunk);
+
+ p = mem;
+ left = filesize;
+ while (left > 0) {
+ chunk = left > max_chunk ? max_chunk : left;
+ blake3_hasher_update(hasher, p, chunk);
+ p += chunk;
+ left -= chunk;
+ }
+ } else {
+ /* slow path using file reads */
+
+ assert(fp);
+
+ bufsz = hs->file_io_bufsz;
+
+ /* for less than 8MB alloca */
+ if (bufsz <= BLAKE3_ALLOCA_BUFFER_SIZE) {
+ buf = alloca(bufsz);
+ } else {
+ buf = malloc(bufsz);
+ if (!buf) {
+ if (hs->cfg.debug)
+ fprintf(stderr, "Unable to allocate buffer of %zu bytes\n",
+ bufsz);
+ goto err_out;
+ }
+ }
+
+ do {
+ rdn = fread(buf, 1, bufsz, fp);
+ if (rdn == 0)
+ break;
+ if (rdn > 0)
+ blake3_hasher_update(hasher, buf, rdn);
+ } while (rdn >= bufsz);
+ }
+
+ // Finalize the hash. BLAKE3_OUT_LEN is the default output length, 32 bytes.
+ blake3_hasher_finalize(hasher, output, BLAKE3_OUT_LEN);
+
+ ret = 0;
+
+out:
+ if (mem)
+ munmap(mem, filesize);
+
+ if (fp && fp != stdin)
+ fclose(fp);
+
+ if (fd >= 0)
+ close(fd);
+
+ if (buf && (bufsz > BLAKE3_ALLOCA_BUFFER_SIZE))
+ free(buf);
+
+ return ret;
+
+err_out:
+ ret = -1;
+ goto out;
+}
+
+void blake3_hash(struct blake3_hasher *hasher,
+ const void *mem, size_t size, uint8_t output[BLAKE3_OUT_LEN])
+{
+ blake3_hasher_reset(hasher);
+ blake3_hasher_update(hasher, mem, size);
+ blake3_hasher_finalize(hasher, output, BLAKE3_OUT_LEN);
+}
+
+const char *blake3_version(void)
+{
+ return BLAKE3_VERSION_STRING;
+}
+
+void blake3_hasher_init(struct blake3_host_state *hs, struct blake3_hasher *self)
+{
+ hs->hasher_ops->hasher_init(hs, self);
+}
+
+void blake3_hasher_init_keyed(struct blake3_host_state *hs, struct blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN])
+{
+ hs->hasher_ops->hasher_init_keyed(hs, self, key);
+}
+
+void blake3_hasher_init_derive_key(struct blake3_host_state *hs, struct blake3_hasher *self, const char *context)
+{
+ hs->hasher_ops->hasher_init_derive_key(hs, self, context);
+}
+
+void blake3_hasher_init_derive_key_raw(struct blake3_host_state *hs, struct blake3_hasher *self, const void *context, size_t context_len)
+{
+ hs->hasher_ops->hasher_init_derive_key_raw(hs, self, context, context_len);
+}
+
+void blake3_hasher_update(struct blake3_hasher *self, const void *input, size_t input_len)
+{
+ blake3_host_state *hs = self->hs;
+ hs->hasher_ops->hasher_update(self, input, input_len);
+}
+
+void blake3_hasher_finalize(const struct blake3_hasher *self, uint8_t *out, size_t out_len)
+{
+ blake3_host_state *hs = self->hs;
+ hs->hasher_ops->hasher_finalize(self, out, out_len);
+}
+
+void blake3_hasher_finalize_seek(const struct blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len)
+{
+ blake3_host_state *hs = self->hs;
+ hs->hasher_ops->hasher_finalize_seek(self, seek, out, out_len);
+}
+
+void blake3_hasher_reset(struct blake3_hasher *self)
+{
+ blake3_host_state *hs = self->hs;
+ hs->hasher_ops->hasher_reset(self);
+}
--- /dev/null
+#ifndef BLAKE3_IMPL_H
+#define BLAKE3_IMPL_H
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdatomic.h>
+#include <stdalign.h>
+
+#ifndef _WIN32
+#include <stdlib.h>
+#if defined(__linux__)
+#include <malloc.h>
+#endif
+#endif
+
+#if defined(__linux__)
+#include <sys/sysmacros.h>
+#endif
+
+#include "blake3.h"
+
+#include "fy-endian.h"
+#include "fy-blob.h"
+#include "fy-bit64.h"
+#include "fy-align.h"
+
+#if defined(HAVE_STRING_OP_OVERREAD) && HAVE_STRING_OP_OVERREAD && defined(__GNUC__) && !defined(__clang__)
+#define GCC_DISABLE_WSTRING_OP_OVERREAD
+#endif
+
+/* this is the best alignment for blake3 */
+#if defined(__x86_64__) || defined(_M_X64)
+#define BLAKE3_ALIGNMENT 64
+#else
+#define BLAKE3_ALIGNMENT 32
+#endif
+
+#define BLAKE3_ALIGN FY_ALIGNED_TO(BLAKE3_ALIGNMENT)
+
+// This C implementation tries to support recent versions of GCC, Clang, and
+// MSVC.
+#if defined(_MSC_VER)
+#define INLINE static __forceinline
+#else
+#define INLINE static inline __attribute__((always_inline))
+#endif
+
+#if defined(__x86_64__) || defined(_M_X64)
+#define IS_X86
+#define IS_X86_64
+#endif
+
+#if defined(__i386__) || defined(_M_IX86)
+#define IS_X86
+#define IS_X86_32
+#endif
+
+#if defined(__aarch64__) || defined(_M_ARM64)
+#define IS_AARCH64
+#define IS_ARM
+#endif
+
+#if defined(IS_X86)
+#if defined(_MSC_VER)
+#include <intrin.h>
+#endif
+#endif
+
+/* Find index of the highest set bit */
+/* x is assumed to be nonzero. */
+static inline unsigned int highest_one(uint64_t x)
+{
+#if defined(__GNUC__) || defined(__clang__)
+ return 63 ^ (unsigned int)__builtin_clzll(x);
+#elif defined(_MSC_VER) && defined(IS_X86_64)
+ unsigned long index;
+ _BitScanReverse64(&index, x);
+ return index;
+#elif defined(_MSC_VER) && defined(IS_X86_32)
+ unsigned long index;
+
+ if(x >> 32) {
+ _BitScanReverse(&index, (unsigned long)(x >> 32));
+ index += 32;
+ } else
+ _BitScanReverse(&index, (unsigned long)x);
+ return index;
+#else
+ unsigned int c = 0;
+
+ if (x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }
+ if (x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }
+ if (x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }
+ if (x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }
+ if (x & 0x000000000000000cULL) { x >>= 2; c += 2; }
+ if (x & 0x0000000000000002ULL) { c += 1; }
+ return c;
+#endif
+}
+
+static inline unsigned int lowest_one(uint64_t x)
+{
+#if defined(__GNUC__) || defined(__clang__)
+ return (unsigned int)__builtin_ctzll(x);
+#else
+ unsigned int c = 0;
+
+ if (!(x & 0x00000000ffffffffULL)) { x >>= 32; c += 32; }
+ if (!(x & 0x000000000000ffffULL)) { x >>= 16; c += 16; }
+ if (!(x & 0x00000000000000ffULL)) { x >>= 8; c += 8; }
+ if (!(x & 0x000000000000000fULL)) { x >>= 4; c += 4; }
+ if (!(x & 0x0000000000000003ULL)) { x >>= 2; c += 2; }
+ if (!(x & 0x0000000000000001ULL)) { c += 1; }
+ return c;
+#endif
+}
+
+// Count the number of 1 bits.
+static inline unsigned int popcnt(uint64_t x)
+{
+#if defined(__GNUC__) || defined(__clang__)
+ return (unsigned int)__builtin_popcountll(x);
+#else
+ unsigned int count = 0;
+ while (x != 0) {
+ count++;
+ x &= x - 1;
+ }
+ return count;
+#endif
+}
+
+// Largest power of two less than or equal to x. As a special case, returns 1
+// when x is 0.
+static inline uint64_t round_down_to_power_of_2(uint64_t x)
+{
+ return 1ULL << highest_one(x | 1);
+}
+
+static inline uint32_t counter_low(uint64_t counter)
+{
+ return (uint32_t)counter;
+}
+
+static inline uint32_t counter_high(uint64_t counter)
+{
+ return (uint32_t)(counter >> 32);
+}
+
+static inline uint32_t load32(const void *src)
+{
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+ return *(const uint32_t *)src;
+#else
+ const uint8_t *p = (const uint8_t *)src;
+ return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |
+ ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);
+#endif
+}
+
+static inline void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], uint32_t key_words[8])
+{
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+ memcpy(key_words, key, BLAKE3_KEY_LEN);
+#else
+ key_words[0] = load32(&key[0 * 4]);
+ key_words[1] = load32(&key[1 * 4]);
+ key_words[2] = load32(&key[2 * 4]);
+ key_words[3] = load32(&key[3 * 4]);
+ key_words[4] = load32(&key[4 * 4]);
+ key_words[5] = load32(&key[5 * 4]);
+ key_words[6] = load32(&key[6 * 4]);
+ key_words[7] = load32(&key[7 * 4]);
+#endif
+}
+
+static inline void store32(void *dst, uint32_t w)
+{
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+ *(uint32_t *)dst = w;
+#else
+ uint8_t *p = (uint8_t *)dst;
+
+ p[0] = (uint8_t)(w >> 0);
+ p[1] = (uint8_t)(w >> 8);
+ p[2] = (uint8_t)(w >> 16);
+ p[3] = (uint8_t)(w >> 24);
+#endif
+}
+
+static inline void store_cv_words(uint8_t bytes_out[BLAKE3_OUT_LEN], uint32_t cv_words[BLAKE3_OUT_WORDS])
+{
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+ memcpy(bytes_out, cv_words, BLAKE3_KEY_LEN);
+#else
+ store32(&bytes_out[0 * 4], cv_words[0]);
+ store32(&bytes_out[1 * 4], cv_words[1]);
+ store32(&bytes_out[2 * 4], cv_words[2]);
+ store32(&bytes_out[3 * 4], cv_words[3]);
+ store32(&bytes_out[4 * 4], cv_words[4]);
+ store32(&bytes_out[5 * 4], cv_words[5]);
+ store32(&bytes_out[6 * 4], cv_words[6]);
+ store32(&bytes_out[7 * 4], cv_words[7]);
+#endif
+}
+
+/* the IV */
+#define B3_IV_0 ((uint32_t)0x6A09E667UL)
+#define B3_IV_1 ((uint32_t)0xBB67AE85UL)
+#define B3_IV_2 ((uint32_t)0x3C6EF372UL)
+#define B3_IV_3 ((uint32_t)0xA54FF53AUL)
+#define B3_IV_4 ((uint32_t)0x510E527FUL)
+#define B3_IV_5 ((uint32_t)0x9B05688CUL)
+#define B3_IV_6 ((uint32_t)0x1F83D9ABUL)
+#define B3_IV_7 ((uint32_t)0x5BE0CD19UL)
+
+/* the message schedule definition */
+#define B3_MSG_SCHEDULE_DEF \
+ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, \
+ {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, \
+ {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, \
+ {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, \
+ {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, \
+ {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, \
+ {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}
+
+#endif /* BLAKE3_IMPL_H */
--- /dev/null
+#ifndef BLAKE3_INTERNAL_H
+#define BLAKE3_INTERNAL_H
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+#include <pthread.h>
+#include <stdatomic.h>
+
+#include "blake3.h"
+
+#include "blake3_impl.h"
+
+#define BLAKE3_MAX_DEPTH 54
+
+// fwd declaration of opaque types
+struct blake3_host_state;
+struct blake3_hasher;
+struct fy_thread_pool;
+
+// internal flags
+enum blake3_flags {
+ CHUNK_START = 1 << 0,
+ CHUNK_END = 1 << 1,
+ PARENT = 1 << 2,
+ ROOT = 1 << 3,
+ KEYED_HASH = 1 << 4,
+ DERIVE_KEY_CONTEXT = 1 << 5,
+ DERIVE_KEY_MATERIAL = 1 << 6,
+};
+
+typedef struct blake3_chunk_state {
+ uint32_t cv[8] BLAKE3_ALIGN;
+ uint8_t buf[BLAKE3_BLOCK_LEN] BLAKE3_ALIGN;
+ uint64_t chunk_counter;
+ uint8_t buf_len;
+ uint8_t blocks_compressed;
+ uint8_t flags;
+} blake3_chunk_state BLAKE3_ALIGN;
+
+typedef struct blake3_hasher {
+ struct blake3_host_state *hs;
+ uint32_t key[8] BLAKE3_ALIGN;
+ blake3_chunk_state chunk BLAKE3_ALIGN;
+ // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example,
+ // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk
+ // requires a 4th entry, rather than merging everything down to 1, because we
+ // don't know whether more input is coming. This is different from how the
+ // reference implementation does things.
+ uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN] BLAKE3_ALIGN;
+ uint8_t cv_stack_len;
+} blake3_hasher BLAKE3_ALIGN;
+
+typedef void (*blake3_hash_many_f)(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8], uint64_t counter,
+ bool increment_counter, uint8_t flags,
+ uint8_t flags_start, uint8_t flags_end, uint8_t *out);
+
+typedef void (*blake3_compress_xof_f)(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter, uint8_t flags,
+ uint8_t out[64]);
+
+typedef void (*blake3_compress_in_place_f)(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags);
+
+typedef struct blake3_backend {
+ const blake3_hasher_ops *hasher_ops;
+ blake3_backend_info info;
+ blake3_hash_many_f hash_many;
+ blake3_compress_xof_f compress_xof;
+ blake3_compress_in_place_f compress_in_place;
+ void *user; /* per backend data */
+} blake3_backend;
+
+// the state for the thread when doing compress subtree
+typedef struct blake3_compress_subtree_state {
+ struct blake3_hasher *self;
+ // inputs
+ const uint8_t *input;
+ size_t input_len;
+ const uint32_t *key;
+ uint64_t chunk_counter;
+ uint8_t flags;
+ // outputs
+ uint8_t *out;
+ size_t n;
+} blake3_compress_subtree_state;
+
+typedef struct blake3_hash_many_common_state {
+ blake3_hash_many_f hash_many;
+ size_t blocks;
+ const uint32_t *key;
+ bool increment_counter;
+ uint8_t flags, flags_start, flags_end;
+} blake3_hash_many_common_state;
+
+typedef struct blake3_hash_many_state {
+ const blake3_hash_many_common_state *common;
+ const uint8_t *const *inputs;
+ size_t num_inputs;
+ uint64_t counter;
+ uint8_t *out;
+} blake3_hash_many_state;
+
+typedef struct blake3_host_state {
+ blake3_host_config cfg;
+ unsigned int num_cpus;
+ uint64_t supported_backends;
+ uint64_t detected_backends;
+ uint64_t selectable_backends;
+
+ /* backend for hash_many */
+ const blake3_backend *hash_many_be;
+ blake3_hash_many_f hash_many;
+
+ /* backend for compress_xof */
+ const blake3_backend *compress_xof_be;
+ blake3_compress_xof_f compress_xof;
+
+ /* backend for compress_in_place */
+ const blake3_backend *compress_in_place_be;
+ blake3_compress_in_place_f compress_in_place;
+
+ const blake3_hasher_ops *hasher_ops;
+
+ unsigned int simd_degree;
+ unsigned int mt_degree;
+
+ unsigned int num_threads;
+
+ struct fy_thread_pool *tp;
+
+ size_t file_io_bufsz;
+ size_t mmap_min_chunk;
+ size_t mmap_max_chunk;
+
+} blake3_host_state;
+
+int blake3_host_state_setup(blake3_host_state *hs, const blake3_host_config *cfg);
+void blake3_host_state_cleanup(blake3_host_state *hs);
+
+extern blake3_backend blake3_backends[B3BID_COUNT];
+
+const blake3_backend *blake3_backend_select_function(uint64_t selectable_backends, blake3_func_id fid);
+const blake3_backend *blake3_get_backend_by_id(blake3_backend_id id);
+const blake3_backend *blake3_get_backend_by_name(const char *name);
+const blake3_backend_info *blake3_get_backend_info(blake3_backend_id id);
+
+void blake3_cpusimd_setup(unsigned int num_cpus, unsigned int mult_fact);
+void blake3_cpusimd_cleanup(void);
+
+#endif
--- /dev/null
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "blake3_impl.h"
+
+#include <arm_neon.h>
+
+/* define it here instead of the header */
+static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF };
+
+#ifdef __ARM_BIG_ENDIAN
+#error "This implementation only supports little-endian ARM."
+// It might be that all we need for big-endian support here is to get the loads
+// and stores right, but step zero would be finding a way to test it in CI.
+#endif
+
+INLINE uint32x4_t loadu_128(const uint8_t src[16]) {
+ // vld1q_u32 has alignment requirements. Don't use it.
+ uint32x4_t x;
+ memcpy(&x, src, 16);
+ return x;
+}
+
+INLINE void storeu_128(uint32x4_t src, uint8_t dest[16]) {
+ // vst1q_u32 has alignment requirements. Don't use it.
+ memcpy(dest, &src, 16);
+}
+
+INLINE uint32x4_t add_128(uint32x4_t a, uint32x4_t b) {
+ return vaddq_u32(a, b);
+}
+
+INLINE uint32x4_t xor_128(uint32x4_t a, uint32x4_t b) {
+ return veorq_u32(a, b);
+}
+
+INLINE uint32x4_t set1_128(uint32_t x) { return vld1q_dup_u32(&x); }
+
+INLINE uint32x4_t set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
+ uint32_t array[4] = {a, b, c, d};
+ return vld1q_u32(array);
+}
+
+INLINE uint32x4_t rot16_128(uint32x4_t x) {
+ // The straightfoward implementation would be two shifts and an or, but that's
+ // slower on microarchitectures we've tested. See
+ // https://github.com/BLAKE3-team/BLAKE3/pull/319.
+ // return vorrq_u32(vshrq_n_u32(x, 16), vshlq_n_u32(x, 32 - 16));
+ return vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x)));
+}
+
+INLINE uint32x4_t rot12_128(uint32x4_t x) {
+ // See comment in rot16_128.
+ // return vorrq_u32(vshrq_n_u32(x, 12), vshlq_n_u32(x, 32 - 12));
+ return vsriq_n_u32(vshlq_n_u32(x, 32-12), x, 12);
+}
+
+INLINE uint32x4_t rot8_128(uint32x4_t x) {
+ // See comment in rot16_128.
+ // return vorrq_u32(vshrq_n_u32(x, 8), vshlq_n_u32(x, 32 - 8));
+#if defined(__clang__)
+ return vreinterpretq_u32_u8(__builtin_shufflevector(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12));
+#elif __GNUC__ * 10000 + __GNUC_MINOR__ * 100 >=40700
+ static const uint8x16_t r8 = {1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12};
+ return vreinterpretq_u32_u8(__builtin_shuffle(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), r8));
+#else
+ return vsriq_n_u32(vshlq_n_u32(x, 32-8), x, 8);
+#endif
+}
+
+INLINE uint32x4_t rot7_128(uint32x4_t x) {
+ // See comment in rot16_128.
+ // return vorrq_u32(vshrq_n_u32(x, 7), vshlq_n_u32(x, 32 - 7));
+ return vsriq_n_u32(vshlq_n_u32(x, 32-7), x, 7);
+}
+
+// TODO: compress_neon
+
+// TODO: hash2_neon
+
+/*
+ * ----------------------------------------------------------------------------
+ * hash4_neon
+ * ----------------------------------------------------------------------------
+ */
+
+INLINE void round_fn4(uint32x4_t v[16], uint32x4_t m[16], size_t r) {
+ v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
+ v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
+ v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
+ v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
+ v[0] = add_128(v[0], v[4]);
+ v[1] = add_128(v[1], v[5]);
+ v[2] = add_128(v[2], v[6]);
+ v[3] = add_128(v[3], v[7]);
+ v[12] = xor_128(v[12], v[0]);
+ v[13] = xor_128(v[13], v[1]);
+ v[14] = xor_128(v[14], v[2]);
+ v[15] = xor_128(v[15], v[3]);
+ v[12] = rot16_128(v[12]);
+ v[13] = rot16_128(v[13]);
+ v[14] = rot16_128(v[14]);
+ v[15] = rot16_128(v[15]);
+ v[8] = add_128(v[8], v[12]);
+ v[9] = add_128(v[9], v[13]);
+ v[10] = add_128(v[10], v[14]);
+ v[11] = add_128(v[11], v[15]);
+ v[4] = xor_128(v[4], v[8]);
+ v[5] = xor_128(v[5], v[9]);
+ v[6] = xor_128(v[6], v[10]);
+ v[7] = xor_128(v[7], v[11]);
+ v[4] = rot12_128(v[4]);
+ v[5] = rot12_128(v[5]);
+ v[6] = rot12_128(v[6]);
+ v[7] = rot12_128(v[7]);
+ v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
+ v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
+ v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
+ v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
+ v[0] = add_128(v[0], v[4]);
+ v[1] = add_128(v[1], v[5]);
+ v[2] = add_128(v[2], v[6]);
+ v[3] = add_128(v[3], v[7]);
+ v[12] = xor_128(v[12], v[0]);
+ v[13] = xor_128(v[13], v[1]);
+ v[14] = xor_128(v[14], v[2]);
+ v[15] = xor_128(v[15], v[3]);
+ v[12] = rot8_128(v[12]);
+ v[13] = rot8_128(v[13]);
+ v[14] = rot8_128(v[14]);
+ v[15] = rot8_128(v[15]);
+ v[8] = add_128(v[8], v[12]);
+ v[9] = add_128(v[9], v[13]);
+ v[10] = add_128(v[10], v[14]);
+ v[11] = add_128(v[11], v[15]);
+ v[4] = xor_128(v[4], v[8]);
+ v[5] = xor_128(v[5], v[9]);
+ v[6] = xor_128(v[6], v[10]);
+ v[7] = xor_128(v[7], v[11]);
+ v[4] = rot7_128(v[4]);
+ v[5] = rot7_128(v[5]);
+ v[6] = rot7_128(v[6]);
+ v[7] = rot7_128(v[7]);
+
+ v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
+ v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
+ v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
+ v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
+ v[0] = add_128(v[0], v[5]);
+ v[1] = add_128(v[1], v[6]);
+ v[2] = add_128(v[2], v[7]);
+ v[3] = add_128(v[3], v[4]);
+ v[15] = xor_128(v[15], v[0]);
+ v[12] = xor_128(v[12], v[1]);
+ v[13] = xor_128(v[13], v[2]);
+ v[14] = xor_128(v[14], v[3]);
+ v[15] = rot16_128(v[15]);
+ v[12] = rot16_128(v[12]);
+ v[13] = rot16_128(v[13]);
+ v[14] = rot16_128(v[14]);
+ v[10] = add_128(v[10], v[15]);
+ v[11] = add_128(v[11], v[12]);
+ v[8] = add_128(v[8], v[13]);
+ v[9] = add_128(v[9], v[14]);
+ v[5] = xor_128(v[5], v[10]);
+ v[6] = xor_128(v[6], v[11]);
+ v[7] = xor_128(v[7], v[8]);
+ v[4] = xor_128(v[4], v[9]);
+ v[5] = rot12_128(v[5]);
+ v[6] = rot12_128(v[6]);
+ v[7] = rot12_128(v[7]);
+ v[4] = rot12_128(v[4]);
+ v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
+ v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
+ v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
+ v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
+ v[0] = add_128(v[0], v[5]);
+ v[1] = add_128(v[1], v[6]);
+ v[2] = add_128(v[2], v[7]);
+ v[3] = add_128(v[3], v[4]);
+ v[15] = xor_128(v[15], v[0]);
+ v[12] = xor_128(v[12], v[1]);
+ v[13] = xor_128(v[13], v[2]);
+ v[14] = xor_128(v[14], v[3]);
+ v[15] = rot8_128(v[15]);
+ v[12] = rot8_128(v[12]);
+ v[13] = rot8_128(v[13]);
+ v[14] = rot8_128(v[14]);
+ v[10] = add_128(v[10], v[15]);
+ v[11] = add_128(v[11], v[12]);
+ v[8] = add_128(v[8], v[13]);
+ v[9] = add_128(v[9], v[14]);
+ v[5] = xor_128(v[5], v[10]);
+ v[6] = xor_128(v[6], v[11]);
+ v[7] = xor_128(v[7], v[8]);
+ v[4] = xor_128(v[4], v[9]);
+ v[5] = rot7_128(v[5]);
+ v[6] = rot7_128(v[6]);
+ v[7] = rot7_128(v[7]);
+ v[4] = rot7_128(v[4]);
+}
+
+INLINE void transpose_vecs_128(uint32x4_t vecs[4]) {
+ // Individually transpose the four 2x2 sub-matrices in each corner.
+ uint32x4x2_t rows01 = vtrnq_u32(vecs[0], vecs[1]);
+ uint32x4x2_t rows23 = vtrnq_u32(vecs[2], vecs[3]);
+
+ // Swap the top-right and bottom-left 2x2s (which just got transposed).
+ vecs[0] =
+ vcombine_u32(vget_low_u32(rows01.val[0]), vget_low_u32(rows23.val[0]));
+ vecs[1] =
+ vcombine_u32(vget_low_u32(rows01.val[1]), vget_low_u32(rows23.val[1]));
+ vecs[2] =
+ vcombine_u32(vget_high_u32(rows01.val[0]), vget_high_u32(rows23.val[0]));
+ vecs[3] =
+ vcombine_u32(vget_high_u32(rows01.val[1]), vget_high_u32(rows23.val[1]));
+}
+
+INLINE void transpose_msg_vecs4(const uint8_t *const *inputs,
+ size_t block_offset, uint32x4_t out[16]) {
+ out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(uint32x4_t)]);
+ out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(uint32x4_t)]);
+ out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(uint32x4_t)]);
+ out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(uint32x4_t)]);
+ out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(uint32x4_t)]);
+ out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(uint32x4_t)]);
+ out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(uint32x4_t)]);
+ out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(uint32x4_t)]);
+ out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(uint32x4_t)]);
+ out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(uint32x4_t)]);
+ out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(uint32x4_t)]);
+ out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(uint32x4_t)]);
+ out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(uint32x4_t)]);
+ out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(uint32x4_t)]);
+ out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(uint32x4_t)]);
+ out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(uint32x4_t)]);
+ transpose_vecs_128(&out[0]);
+ transpose_vecs_128(&out[4]);
+ transpose_vecs_128(&out[8]);
+ transpose_vecs_128(&out[12]);
+}
+
+INLINE void load_counters4(uint64_t counter, bool increment_counter,
+ uint32x4_t *out_low, uint32x4_t *out_high) {
+ uint64_t mask = (increment_counter ? ~0 : 0);
+ *out_low = set4(
+ counter_low(counter + (mask & 0)), counter_low(counter + (mask & 1)),
+ counter_low(counter + (mask & 2)), counter_low(counter + (mask & 3)));
+ *out_high = set4(
+ counter_high(counter + (mask & 0)), counter_high(counter + (mask & 1)),
+ counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3)));
+}
+
+void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ bool increment_counter, uint8_t flags,
+ uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
+ uint32x4_t h_vecs[8] = {
+ set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]),
+ set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]),
+ };
+ uint32x4_t counter_low_vec, counter_high_vec;
+ load_counters4(counter, increment_counter, &counter_low_vec,
+ &counter_high_vec);
+ uint8_t block_flags = flags | flags_start;
+
+ for (size_t block = 0; block < blocks; block++) {
+ if (block + 1 == blocks) {
+ block_flags |= flags_end;
+ }
+ uint32x4_t block_len_vec = set1_128(BLAKE3_BLOCK_LEN);
+ uint32x4_t block_flags_vec = set1_128(block_flags);
+ uint32x4_t msg_vecs[16];
+ transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
+
+ uint32x4_t v[16] = {
+ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
+ h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
+ set1_128(B3_IV_0), set1_128(B3_IV_1), set1_128(B3_IV_2), set1_128(B3_IV_3),
+ counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
+ };
+ round_fn4(v, msg_vecs, 0);
+ round_fn4(v, msg_vecs, 1);
+ round_fn4(v, msg_vecs, 2);
+ round_fn4(v, msg_vecs, 3);
+ round_fn4(v, msg_vecs, 4);
+ round_fn4(v, msg_vecs, 5);
+ round_fn4(v, msg_vecs, 6);
+ h_vecs[0] = xor_128(v[0], v[8]);
+ h_vecs[1] = xor_128(v[1], v[9]);
+ h_vecs[2] = xor_128(v[2], v[10]);
+ h_vecs[3] = xor_128(v[3], v[11]);
+ h_vecs[4] = xor_128(v[4], v[12]);
+ h_vecs[5] = xor_128(v[5], v[13]);
+ h_vecs[6] = xor_128(v[6], v[14]);
+ h_vecs[7] = xor_128(v[7], v[15]);
+
+ block_flags = flags;
+ }
+
+ transpose_vecs_128(&h_vecs[0]);
+ transpose_vecs_128(&h_vecs[4]);
+ // The first four vecs now contain the first half of each output, and the
+ // second four vecs contain the second half of each output.
+ storeu_128(h_vecs[0], &out[0 * sizeof(uint32x4_t)]);
+ storeu_128(h_vecs[4], &out[1 * sizeof(uint32x4_t)]);
+ storeu_128(h_vecs[1], &out[2 * sizeof(uint32x4_t)]);
+ storeu_128(h_vecs[5], &out[3 * sizeof(uint32x4_t)]);
+ storeu_128(h_vecs[2], &out[4 * sizeof(uint32x4_t)]);
+ storeu_128(h_vecs[6], &out[5 * sizeof(uint32x4_t)]);
+ storeu_128(h_vecs[3], &out[6 * sizeof(uint32x4_t)]);
+ storeu_128(h_vecs[7], &out[7 * sizeof(uint32x4_t)]);
+}
+
+/*
+ * ----------------------------------------------------------------------------
+ * hash_many_neon
+ * ----------------------------------------------------------------------------
+ */
+
+void blake3_compress_in_place_portable(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags);
+
+INLINE void hash_one_neon(const uint8_t *input, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ uint8_t flags, uint8_t flags_start, uint8_t flags_end,
+ uint8_t out[BLAKE3_OUT_LEN]) {
+ uint32_t cv[8];
+ memcpy(cv, key, BLAKE3_KEY_LEN);
+ uint8_t block_flags = flags | flags_start;
+ while (blocks > 0) {
+ if (blocks == 1) {
+ block_flags |= flags_end;
+ }
+ // TODO: Implement compress_neon. However note that according to
+ // https://github.com/BLAKE2/BLAKE2/commit/7965d3e6e1b4193438b8d3a656787587d2579227,
+ // compress_neon might not be any faster than compress_portable.
+ blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter,
+ block_flags);
+ input = &input[BLAKE3_BLOCK_LEN];
+ blocks -= 1;
+ block_flags = flags;
+ }
+ memcpy(out, cv, BLAKE3_OUT_LEN);
+}
+
+void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out) {
+ while (num_inputs >= 4) {
+ blake3_hash4_neon(inputs, blocks, key, counter, increment_counter, flags,
+ flags_start, flags_end, out);
+ if (increment_counter) {
+ counter += 4;
+ }
+ inputs += 4;
+ num_inputs -= 4;
+ out = &out[4 * BLAKE3_OUT_LEN];
+ }
+ while (num_inputs > 0) {
+ hash_one_neon(inputs[0], blocks, key, counter, flags, flags_start,
+ flags_end, out);
+ if (increment_counter) {
+ counter += 1;
+ }
+ inputs += 1;
+ num_inputs -= 1;
+ out = &out[BLAKE3_OUT_LEN];
+ }
+}
--- /dev/null
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "blake3_impl.h"
+#include <string.h>
+
+/* define it here instead of the header */
+static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF };
+
+INLINE uint32_t rotr32(uint32_t w, uint32_t c) {
+ return (w >> c) | (w << (32 - c));
+}
+
+INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d,
+ uint32_t x, uint32_t y) {
+ state[a] = state[a] + state[b] + x;
+ state[d] = rotr32(state[d] ^ state[a], 16);
+ state[c] = state[c] + state[d];
+ state[b] = rotr32(state[b] ^ state[c], 12);
+ state[a] = state[a] + state[b] + y;
+ state[d] = rotr32(state[d] ^ state[a], 8);
+ state[c] = state[c] + state[d];
+ state[b] = rotr32(state[b] ^ state[c], 7);
+}
+
+INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) {
+ // Select the message schedule based on the round.
+ const uint8_t *schedule = MSG_SCHEDULE[round];
+
+ // Mix the columns.
+ g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]);
+ g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]);
+ g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]);
+ g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]);
+
+ // Mix the rows.
+ g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]);
+ g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]);
+ g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]);
+ g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]);
+}
+
+INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter, uint8_t flags) {
+ uint32_t block_words[16];
+ block_words[0] = load32(block + 4 * 0);
+ block_words[1] = load32(block + 4 * 1);
+ block_words[2] = load32(block + 4 * 2);
+ block_words[3] = load32(block + 4 * 3);
+ block_words[4] = load32(block + 4 * 4);
+ block_words[5] = load32(block + 4 * 5);
+ block_words[6] = load32(block + 4 * 6);
+ block_words[7] = load32(block + 4 * 7);
+ block_words[8] = load32(block + 4 * 8);
+ block_words[9] = load32(block + 4 * 9);
+ block_words[10] = load32(block + 4 * 10);
+ block_words[11] = load32(block + 4 * 11);
+ block_words[12] = load32(block + 4 * 12);
+ block_words[13] = load32(block + 4 * 13);
+ block_words[14] = load32(block + 4 * 14);
+ block_words[15] = load32(block + 4 * 15);
+
+ state[0] = cv[0];
+ state[1] = cv[1];
+ state[2] = cv[2];
+ state[3] = cv[3];
+ state[4] = cv[4];
+ state[5] = cv[5];
+ state[6] = cv[6];
+ state[7] = cv[7];
+ state[8] = B3_IV_0;
+ state[9] = B3_IV_1;
+ state[10] = B3_IV_2;
+ state[11] = B3_IV_3;
+ state[12] = counter_low(counter);
+ state[13] = counter_high(counter);
+ state[14] = (uint32_t)block_len;
+ state[15] = (uint32_t)flags;
+
+ round_fn(state, &block_words[0], 0);
+ round_fn(state, &block_words[0], 1);
+ round_fn(state, &block_words[0], 2);
+ round_fn(state, &block_words[0], 3);
+ round_fn(state, &block_words[0], 4);
+ round_fn(state, &block_words[0], 5);
+ round_fn(state, &block_words[0], 6);
+}
+
+void blake3_compress_in_place_portable(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags) {
+ uint32_t state[16];
+ compress_pre(state, cv, block, block_len, counter, flags);
+ cv[0] = state[0] ^ state[8];
+ cv[1] = state[1] ^ state[9];
+ cv[2] = state[2] ^ state[10];
+ cv[3] = state[3] ^ state[11];
+ cv[4] = state[4] ^ state[12];
+ cv[5] = state[5] ^ state[13];
+ cv[6] = state[6] ^ state[14];
+ cv[7] = state[7] ^ state[15];
+}
+
+void blake3_compress_xof_portable(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags, uint8_t out[64]) {
+ uint32_t state[16];
+ compress_pre(state, cv, block, block_len, counter, flags);
+
+ store32(&out[0 * 4], state[0] ^ state[8]);
+ store32(&out[1 * 4], state[1] ^ state[9]);
+ store32(&out[2 * 4], state[2] ^ state[10]);
+ store32(&out[3 * 4], state[3] ^ state[11]);
+ store32(&out[4 * 4], state[4] ^ state[12]);
+ store32(&out[5 * 4], state[5] ^ state[13]);
+ store32(&out[6 * 4], state[6] ^ state[14]);
+ store32(&out[7 * 4], state[7] ^ state[15]);
+ store32(&out[8 * 4], state[8] ^ cv[0]);
+ store32(&out[9 * 4], state[9] ^ cv[1]);
+ store32(&out[10 * 4], state[10] ^ cv[2]);
+ store32(&out[11 * 4], state[11] ^ cv[3]);
+ store32(&out[12 * 4], state[12] ^ cv[4]);
+ store32(&out[13 * 4], state[13] ^ cv[5]);
+ store32(&out[14 * 4], state[14] ^ cv[6]);
+ store32(&out[15 * 4], state[15] ^ cv[7]);
+}
+
+INLINE void hash_one_portable(const uint8_t *input, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) {
+ uint32_t cv[8];
+ memcpy(cv, key, BLAKE3_KEY_LEN);
+ uint8_t block_flags = flags | flags_start;
+ while (blocks > 0) {
+ if (blocks == 1) {
+ block_flags |= flags_end;
+ }
+ blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter,
+ block_flags);
+ input = &input[BLAKE3_BLOCK_LEN];
+ blocks -= 1;
+ block_flags = flags;
+ }
+ store_cv_words(out, cv);
+}
+
+void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out) {
+ while (num_inputs > 0) {
+ hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start,
+ flags_end, out);
+ if (increment_counter) {
+ counter += 1;
+ }
+ inputs += 1;
+ num_inputs -= 1;
+ out = &out[BLAKE3_OUT_LEN];
+ }
+}
--- /dev/null
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "blake3_impl.h"
+
+#include <immintrin.h>
+
+/* define it here instead of the header */
+static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF };
+
+#define DEGREE 4
+
+#define _mm_shuffle_ps2(a, b, c) \
+ (_mm_castps_si128( \
+ _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c))))
+
+INLINE __m128i loadu(const uint8_t src[16]) {
+ return _mm_loadu_si128((const __m128i *)src);
+}
+
+INLINE void storeu(__m128i src, uint8_t dest[16]) {
+ _mm_storeu_si128((__m128i *)dest, src);
+}
+
+INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); }
+
+// Note that clang-format doesn't like the name "xor" for some reason.
+INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); }
+
+INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); }
+
+INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
+ return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d);
+}
+
+INLINE __m128i rot16(__m128i x) {
+ return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, 0xB1), 0xB1);
+}
+
+INLINE __m128i rot12(__m128i x) {
+ return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12));
+}
+
+INLINE __m128i rot8(__m128i x) {
+ return xorv(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 32 - 8));
+}
+
+INLINE __m128i rot7(__m128i x) {
+ return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7));
+}
+
+INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
+ __m128i m) {
+ *row0 = addv(addv(*row0, m), *row1);
+ *row3 = xorv(*row3, *row0);
+ *row3 = rot16(*row3);
+ *row2 = addv(*row2, *row3);
+ *row1 = xorv(*row1, *row2);
+ *row1 = rot12(*row1);
+}
+
+INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
+ __m128i m) {
+ *row0 = addv(addv(*row0, m), *row1);
+ *row3 = xorv(*row3, *row0);
+ *row3 = rot8(*row3);
+ *row2 = addv(*row2, *row3);
+ *row1 = xorv(*row1, *row2);
+ *row1 = rot7(*row1);
+}
+
+// Note the optimization here of leaving row1 as the unrotated row, rather than
+// row0. All the message loads below are adjusted to compensate for this. See
+// discussion at https://github.com/sneves/blake2-avx2/pull/4
+INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) {
+ *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
+ *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
+ *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
+}
+
+INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) {
+ *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
+ *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
+ *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
+}
+
+INLINE __m128i blend_epi16(__m128i a, __m128i b, const int16_t imm8) {
+ const __m128i bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01);
+ __m128i mask = _mm_set1_epi16(imm8);
+ mask = _mm_and_si128(mask, bits);
+ mask = _mm_cmpeq_epi16(mask, bits);
+ return _mm_or_si128(_mm_and_si128(mask, b), _mm_andnot_si128(mask, a));
+}
+
+INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter, uint8_t flags) {
+ rows[0] = loadu((uint8_t *)&cv[0]);
+ rows[1] = loadu((uint8_t *)&cv[4]);
+ rows[2] = set4(B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3);
+ rows[3] = set4(counter_low(counter), counter_high(counter),
+ (uint32_t)block_len, (uint32_t)flags);
+
+ __m128i m0 = loadu(&block[sizeof(__m128i) * 0]);
+ __m128i m1 = loadu(&block[sizeof(__m128i) * 1]);
+ __m128i m2 = loadu(&block[sizeof(__m128i) * 2]);
+ __m128i m3 = loadu(&block[sizeof(__m128i) * 3]);
+
+ __m128i t0, t1, t2, t3, tt;
+
+ // Round 1. The first round permutes the message words from the original
+ // input order, into the groups that get mixed in parallel.
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8
+ t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9
+ t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 2. This round and all following rounds apply a fixed permutation
+ // to the message words from the round before.
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 3
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 4
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 5
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 6
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 7
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+}
+
+void blake3_compress_in_place_sse2(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags) {
+ __m128i rows[4];
+ compress_pre(rows, cv, block, block_len, counter, flags);
+ storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]);
+ storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]);
+}
+
+void blake3_compress_xof_sse2(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags, uint8_t out[64]) {
+ __m128i rows[4];
+ compress_pre(rows, cv, block, block_len, counter, flags);
+ storeu(xorv(rows[0], rows[2]), &out[0]);
+ storeu(xorv(rows[1], rows[3]), &out[16]);
+ storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]);
+ storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]);
+}
+
+INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) {
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
+ v[0] = addv(v[0], v[4]);
+ v[1] = addv(v[1], v[5]);
+ v[2] = addv(v[2], v[6]);
+ v[3] = addv(v[3], v[7]);
+ v[12] = xorv(v[12], v[0]);
+ v[13] = xorv(v[13], v[1]);
+ v[14] = xorv(v[14], v[2]);
+ v[15] = xorv(v[15], v[3]);
+ v[12] = rot16(v[12]);
+ v[13] = rot16(v[13]);
+ v[14] = rot16(v[14]);
+ v[15] = rot16(v[15]);
+ v[8] = addv(v[8], v[12]);
+ v[9] = addv(v[9], v[13]);
+ v[10] = addv(v[10], v[14]);
+ v[11] = addv(v[11], v[15]);
+ v[4] = xorv(v[4], v[8]);
+ v[5] = xorv(v[5], v[9]);
+ v[6] = xorv(v[6], v[10]);
+ v[7] = xorv(v[7], v[11]);
+ v[4] = rot12(v[4]);
+ v[5] = rot12(v[5]);
+ v[6] = rot12(v[6]);
+ v[7] = rot12(v[7]);
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
+ v[0] = addv(v[0], v[4]);
+ v[1] = addv(v[1], v[5]);
+ v[2] = addv(v[2], v[6]);
+ v[3] = addv(v[3], v[7]);
+ v[12] = xorv(v[12], v[0]);
+ v[13] = xorv(v[13], v[1]);
+ v[14] = xorv(v[14], v[2]);
+ v[15] = xorv(v[15], v[3]);
+ v[12] = rot8(v[12]);
+ v[13] = rot8(v[13]);
+ v[14] = rot8(v[14]);
+ v[15] = rot8(v[15]);
+ v[8] = addv(v[8], v[12]);
+ v[9] = addv(v[9], v[13]);
+ v[10] = addv(v[10], v[14]);
+ v[11] = addv(v[11], v[15]);
+ v[4] = xorv(v[4], v[8]);
+ v[5] = xorv(v[5], v[9]);
+ v[6] = xorv(v[6], v[10]);
+ v[7] = xorv(v[7], v[11]);
+ v[4] = rot7(v[4]);
+ v[5] = rot7(v[5]);
+ v[6] = rot7(v[6]);
+ v[7] = rot7(v[7]);
+
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
+ v[0] = addv(v[0], v[5]);
+ v[1] = addv(v[1], v[6]);
+ v[2] = addv(v[2], v[7]);
+ v[3] = addv(v[3], v[4]);
+ v[15] = xorv(v[15], v[0]);
+ v[12] = xorv(v[12], v[1]);
+ v[13] = xorv(v[13], v[2]);
+ v[14] = xorv(v[14], v[3]);
+ v[15] = rot16(v[15]);
+ v[12] = rot16(v[12]);
+ v[13] = rot16(v[13]);
+ v[14] = rot16(v[14]);
+ v[10] = addv(v[10], v[15]);
+ v[11] = addv(v[11], v[12]);
+ v[8] = addv(v[8], v[13]);
+ v[9] = addv(v[9], v[14]);
+ v[5] = xorv(v[5], v[10]);
+ v[6] = xorv(v[6], v[11]);
+ v[7] = xorv(v[7], v[8]);
+ v[4] = xorv(v[4], v[9]);
+ v[5] = rot12(v[5]);
+ v[6] = rot12(v[6]);
+ v[7] = rot12(v[7]);
+ v[4] = rot12(v[4]);
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
+ v[0] = addv(v[0], v[5]);
+ v[1] = addv(v[1], v[6]);
+ v[2] = addv(v[2], v[7]);
+ v[3] = addv(v[3], v[4]);
+ v[15] = xorv(v[15], v[0]);
+ v[12] = xorv(v[12], v[1]);
+ v[13] = xorv(v[13], v[2]);
+ v[14] = xorv(v[14], v[3]);
+ v[15] = rot8(v[15]);
+ v[12] = rot8(v[12]);
+ v[13] = rot8(v[13]);
+ v[14] = rot8(v[14]);
+ v[10] = addv(v[10], v[15]);
+ v[11] = addv(v[11], v[12]);
+ v[8] = addv(v[8], v[13]);
+ v[9] = addv(v[9], v[14]);
+ v[5] = xorv(v[5], v[10]);
+ v[6] = xorv(v[6], v[11]);
+ v[7] = xorv(v[7], v[8]);
+ v[4] = xorv(v[4], v[9]);
+ v[5] = rot7(v[5]);
+ v[6] = rot7(v[6]);
+ v[7] = rot7(v[7]);
+ v[4] = rot7(v[4]);
+}
+
+INLINE void transpose_vecs(__m128i vecs[DEGREE]) {
+ // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is
+ // 22/33. Note that this doesn't split the vector into two lanes, as the
+ // AVX2 counterparts do.
+ __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
+ __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
+ __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
+ __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
+
+ // Interleave 64-bit lanes.
+ __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01);
+ __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01);
+ __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23);
+ __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23);
+
+ vecs[0] = abcd_0;
+ vecs[1] = abcd_1;
+ vecs[2] = abcd_2;
+ vecs[3] = abcd_3;
+}
+
+INLINE void transpose_msg_vecs(const uint8_t *const *inputs,
+ size_t block_offset, __m128i out[16]) {
+ out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]);
+ out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]);
+ out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]);
+ out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]);
+ out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]);
+ out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]);
+ out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]);
+ out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]);
+ out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]);
+ out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]);
+ out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]);
+ out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]);
+ out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]);
+ out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]);
+ out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]);
+ out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]);
+ for (size_t i = 0; i < 4; ++i) {
+ _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
+ }
+ transpose_vecs(&out[0]);
+ transpose_vecs(&out[4]);
+ transpose_vecs(&out[8]);
+ transpose_vecs(&out[12]);
+}
+
+INLINE void load_counters(uint64_t counter, bool increment_counter,
+ __m128i *out_lo, __m128i *out_hi) {
+ const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter);
+ const __m128i add0 = _mm_set_epi32(3, 2, 1, 0);
+ const __m128i add1 = _mm_and_si128(mask, add0);
+ __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1);
+ __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)),
+ _mm_xor_si128( l, _mm_set1_epi32(0x80000000)));
+ __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry);
+ *out_lo = l;
+ *out_hi = h;
+}
+
+static
+void blake3_hash4_sse2(const uint8_t *const *inputs, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ bool increment_counter, uint8_t flags,
+ uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
+ __m128i h_vecs[8] = {
+ set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]),
+ set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]),
+ };
+ __m128i counter_low_vec, counter_high_vec;
+ load_counters(counter, increment_counter, &counter_low_vec,
+ &counter_high_vec);
+ uint8_t block_flags = flags | flags_start;
+
+ for (size_t block = 0; block < blocks; block++) {
+ if (block + 1 == blocks) {
+ block_flags |= flags_end;
+ }
+ __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN);
+ __m128i block_flags_vec = set1(block_flags);
+ __m128i msg_vecs[16];
+ transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
+
+ __m128i v[16] = {
+ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
+ h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
+ set1(B3_IV_0), set1(B3_IV_1), set1(B3_IV_2), set1(B3_IV_3),
+ counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
+ };
+ round_fn(v, msg_vecs, 0);
+ round_fn(v, msg_vecs, 1);
+ round_fn(v, msg_vecs, 2);
+ round_fn(v, msg_vecs, 3);
+ round_fn(v, msg_vecs, 4);
+ round_fn(v, msg_vecs, 5);
+ round_fn(v, msg_vecs, 6);
+ h_vecs[0] = xorv(v[0], v[8]);
+ h_vecs[1] = xorv(v[1], v[9]);
+ h_vecs[2] = xorv(v[2], v[10]);
+ h_vecs[3] = xorv(v[3], v[11]);
+ h_vecs[4] = xorv(v[4], v[12]);
+ h_vecs[5] = xorv(v[5], v[13]);
+ h_vecs[6] = xorv(v[6], v[14]);
+ h_vecs[7] = xorv(v[7], v[15]);
+
+ block_flags = flags;
+ }
+
+ transpose_vecs(&h_vecs[0]);
+ transpose_vecs(&h_vecs[4]);
+ // The first four vecs now contain the first half of each output, and the
+ // second four vecs contain the second half of each output.
+ storeu(h_vecs[0], &out[0 * sizeof(__m128i)]);
+ storeu(h_vecs[4], &out[1 * sizeof(__m128i)]);
+ storeu(h_vecs[1], &out[2 * sizeof(__m128i)]);
+ storeu(h_vecs[5], &out[3 * sizeof(__m128i)]);
+ storeu(h_vecs[2], &out[4 * sizeof(__m128i)]);
+ storeu(h_vecs[6], &out[5 * sizeof(__m128i)]);
+ storeu(h_vecs[3], &out[6 * sizeof(__m128i)]);
+ storeu(h_vecs[7], &out[7 * sizeof(__m128i)]);
+}
+
+INLINE void hash_one_sse2(const uint8_t *input, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) {
+ uint32_t cv[8];
+ memcpy(cv, key, BLAKE3_KEY_LEN);
+ uint8_t block_flags = flags | flags_start;
+ while (blocks > 0) {
+ if (blocks == 1) {
+ block_flags |= flags_end;
+ }
+ blake3_compress_in_place_sse2(cv, input, BLAKE3_BLOCK_LEN, counter,
+ block_flags);
+ input = &input[BLAKE3_BLOCK_LEN];
+ blocks -= 1;
+ block_flags = flags;
+ }
+ memcpy(out, cv, BLAKE3_OUT_LEN);
+}
+
+void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out) {
+ while (num_inputs >= DEGREE) {
+ blake3_hash4_sse2(inputs, blocks, key, counter, increment_counter, flags,
+ flags_start, flags_end, out);
+ if (increment_counter) {
+ counter += DEGREE;
+ }
+ inputs += DEGREE;
+ num_inputs -= DEGREE;
+ out = &out[DEGREE * BLAKE3_OUT_LEN];
+ }
+ while (num_inputs > 0) {
+ hash_one_sse2(inputs[0], blocks, key, counter, flags, flags_start,
+ flags_end, out);
+ if (increment_counter) {
+ counter += 1;
+ }
+ inputs += 1;
+ num_inputs -= 1;
+ out = &out[BLAKE3_OUT_LEN];
+ }
+}
--- /dev/null
+#if defined(__ELF__) && defined(__linux__)
+.section .note.GNU-stack,"",%progbits
+#endif
+
+#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
+#if __has_include(<cet.h>)
+#include <cet.h>
+#endif
+#endif
+
+#if !defined(_CET_ENDBR)
+#define _CET_ENDBR
+#endif
+
+.intel_syntax noprefix
+.global blake3_hash_many_sse2_asm
+.global _blake3_hash_many_sse2_asm
+.global blake3_compress_in_place_sse2_asm
+.global _blake3_compress_in_place_sse2_asm
+.global blake3_compress_xof_sse2_asm
+.global _blake3_compress_xof_sse2_asm
+#ifdef __APPLE__
+.text
+#else
+.section .text
+#endif
+ .p2align 6
+_blake3_hash_many_sse2_asm:
+blake3_hash_many_sse2_asm:
+ _CET_ENDBR
+ push r15
+ push r14
+ push r13
+ push r12
+ push rbx
+ push rbp
+ mov rbp, rsp
+ sub rsp, 360
+ and rsp, 0xFFFFFFFFFFFFFFC0
+ neg r9d
+ movd xmm0, r9d
+ pshufd xmm0, xmm0, 0x00
+ movdqa xmmword ptr [rsp+0x130], xmm0
+ movdqa xmm1, xmm0
+ pand xmm1, xmmword ptr [ADD0+rip]
+ pand xmm0, xmmword ptr [ADD1+rip]
+ movdqa xmmword ptr [rsp+0x150], xmm0
+ movd xmm0, r8d
+ pshufd xmm0, xmm0, 0x00
+ paddd xmm0, xmm1
+ movdqa xmmword ptr [rsp+0x110], xmm0
+ pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip]
+ pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip]
+ pcmpgtd xmm1, xmm0
+ shr r8, 32
+ movd xmm2, r8d
+ pshufd xmm2, xmm2, 0x00
+ psubd xmm2, xmm1
+ movdqa xmmword ptr [rsp+0x120], xmm2
+ mov rbx, qword ptr [rbp+0x50]
+ mov r15, rdx
+ shl r15, 6
+ movzx r13d, byte ptr [rbp+0x38]
+ movzx r12d, byte ptr [rbp+0x48]
+ cmp rsi, 4
+ jc 3f
+2:
+ movdqu xmm3, xmmword ptr [rcx]
+ pshufd xmm0, xmm3, 0x00
+ pshufd xmm1, xmm3, 0x55
+ pshufd xmm2, xmm3, 0xAA
+ pshufd xmm3, xmm3, 0xFF
+ movdqu xmm7, xmmword ptr [rcx+0x10]
+ pshufd xmm4, xmm7, 0x00
+ pshufd xmm5, xmm7, 0x55
+ pshufd xmm6, xmm7, 0xAA
+ pshufd xmm7, xmm7, 0xFF
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ mov r10, qword ptr [rdi+0x10]
+ mov r11, qword ptr [rdi+0x18]
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+9:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ movdqu xmm8, xmmword ptr [r8+rdx-0x40]
+ movdqu xmm9, xmmword ptr [r9+rdx-0x40]
+ movdqu xmm10, xmmword ptr [r10+rdx-0x40]
+ movdqu xmm11, xmmword ptr [r11+rdx-0x40]
+ movdqa xmm12, xmm8
+ punpckldq xmm8, xmm9
+ punpckhdq xmm12, xmm9
+ movdqa xmm14, xmm10
+ punpckldq xmm10, xmm11
+ punpckhdq xmm14, xmm11
+ movdqa xmm9, xmm8
+ punpcklqdq xmm8, xmm10
+ punpckhqdq xmm9, xmm10
+ movdqa xmm13, xmm12
+ punpcklqdq xmm12, xmm14
+ punpckhqdq xmm13, xmm14
+ movdqa xmmword ptr [rsp], xmm8
+ movdqa xmmword ptr [rsp+0x10], xmm9
+ movdqa xmmword ptr [rsp+0x20], xmm12
+ movdqa xmmword ptr [rsp+0x30], xmm13
+ movdqu xmm8, xmmword ptr [r8+rdx-0x30]
+ movdqu xmm9, xmmword ptr [r9+rdx-0x30]
+ movdqu xmm10, xmmword ptr [r10+rdx-0x30]
+ movdqu xmm11, xmmword ptr [r11+rdx-0x30]
+ movdqa xmm12, xmm8
+ punpckldq xmm8, xmm9
+ punpckhdq xmm12, xmm9
+ movdqa xmm14, xmm10
+ punpckldq xmm10, xmm11
+ punpckhdq xmm14, xmm11
+ movdqa xmm9, xmm8
+ punpcklqdq xmm8, xmm10
+ punpckhqdq xmm9, xmm10
+ movdqa xmm13, xmm12
+ punpcklqdq xmm12, xmm14
+ punpckhqdq xmm13, xmm14
+ movdqa xmmword ptr [rsp+0x40], xmm8
+ movdqa xmmword ptr [rsp+0x50], xmm9
+ movdqa xmmword ptr [rsp+0x60], xmm12
+ movdqa xmmword ptr [rsp+0x70], xmm13
+ movdqu xmm8, xmmword ptr [r8+rdx-0x20]
+ movdqu xmm9, xmmword ptr [r9+rdx-0x20]
+ movdqu xmm10, xmmword ptr [r10+rdx-0x20]
+ movdqu xmm11, xmmword ptr [r11+rdx-0x20]
+ movdqa xmm12, xmm8
+ punpckldq xmm8, xmm9
+ punpckhdq xmm12, xmm9
+ movdqa xmm14, xmm10
+ punpckldq xmm10, xmm11
+ punpckhdq xmm14, xmm11
+ movdqa xmm9, xmm8
+ punpcklqdq xmm8, xmm10
+ punpckhqdq xmm9, xmm10
+ movdqa xmm13, xmm12
+ punpcklqdq xmm12, xmm14
+ punpckhqdq xmm13, xmm14
+ movdqa xmmword ptr [rsp+0x80], xmm8
+ movdqa xmmword ptr [rsp+0x90], xmm9
+ movdqa xmmword ptr [rsp+0xA0], xmm12
+ movdqa xmmword ptr [rsp+0xB0], xmm13
+ movdqu xmm8, xmmword ptr [r8+rdx-0x10]
+ movdqu xmm9, xmmword ptr [r9+rdx-0x10]
+ movdqu xmm10, xmmword ptr [r10+rdx-0x10]
+ movdqu xmm11, xmmword ptr [r11+rdx-0x10]
+ movdqa xmm12, xmm8
+ punpckldq xmm8, xmm9
+ punpckhdq xmm12, xmm9
+ movdqa xmm14, xmm10
+ punpckldq xmm10, xmm11
+ punpckhdq xmm14, xmm11
+ movdqa xmm9, xmm8
+ punpcklqdq xmm8, xmm10
+ punpckhqdq xmm9, xmm10
+ movdqa xmm13, xmm12
+ punpcklqdq xmm12, xmm14
+ punpckhqdq xmm13, xmm14
+ movdqa xmmword ptr [rsp+0xC0], xmm8
+ movdqa xmmword ptr [rsp+0xD0], xmm9
+ movdqa xmmword ptr [rsp+0xE0], xmm12
+ movdqa xmmword ptr [rsp+0xF0], xmm13
+ movdqa xmm9, xmmword ptr [BLAKE3_IV_1+rip]
+ movdqa xmm10, xmmword ptr [BLAKE3_IV_2+rip]
+ movdqa xmm11, xmmword ptr [BLAKE3_IV_3+rip]
+ movdqa xmm12, xmmword ptr [rsp+0x110]
+ movdqa xmm13, xmmword ptr [rsp+0x120]
+ movdqa xmm14, xmmword ptr [BLAKE3_BLOCK_LEN+rip]
+ movd xmm15, eax
+ pshufd xmm15, xmm15, 0x00
+ prefetcht0 [r8+rdx+0x80]
+ prefetcht0 [r9+rdx+0x80]
+ prefetcht0 [r10+rdx+0x80]
+ prefetcht0 [r11+rdx+0x80]
+ paddd xmm0, xmmword ptr [rsp]
+ paddd xmm1, xmmword ptr [rsp+0x20]
+ paddd xmm2, xmmword ptr [rsp+0x40]
+ paddd xmm3, xmmword ptr [rsp+0x60]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ movdqa xmm8, xmmword ptr [BLAKE3_IV_0+rip]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x10]
+ paddd xmm1, xmmword ptr [rsp+0x30]
+ paddd xmm2, xmmword ptr [rsp+0x50]
+ paddd xmm3, xmmword ptr [rsp+0x70]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x80]
+ paddd xmm1, xmmword ptr [rsp+0xA0]
+ paddd xmm2, xmmword ptr [rsp+0xC0]
+ paddd xmm3, xmmword ptr [rsp+0xE0]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x90]
+ paddd xmm1, xmmword ptr [rsp+0xB0]
+ paddd xmm2, xmmword ptr [rsp+0xD0]
+ paddd xmm3, xmmword ptr [rsp+0xF0]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x20]
+ paddd xmm1, xmmword ptr [rsp+0x30]
+ paddd xmm2, xmmword ptr [rsp+0x70]
+ paddd xmm3, xmmword ptr [rsp+0x40]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x60]
+ paddd xmm1, xmmword ptr [rsp+0xA0]
+ paddd xmm2, xmmword ptr [rsp]
+ paddd xmm3, xmmword ptr [rsp+0xD0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x10]
+ paddd xmm1, xmmword ptr [rsp+0xC0]
+ paddd xmm2, xmmword ptr [rsp+0x90]
+ paddd xmm3, xmmword ptr [rsp+0xF0]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xB0]
+ paddd xmm1, xmmword ptr [rsp+0x50]
+ paddd xmm2, xmmword ptr [rsp+0xE0]
+ paddd xmm3, xmmword ptr [rsp+0x80]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x30]
+ paddd xmm1, xmmword ptr [rsp+0xA0]
+ paddd xmm2, xmmword ptr [rsp+0xD0]
+ paddd xmm3, xmmword ptr [rsp+0x70]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x40]
+ paddd xmm1, xmmword ptr [rsp+0xC0]
+ paddd xmm2, xmmword ptr [rsp+0x20]
+ paddd xmm3, xmmword ptr [rsp+0xE0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x60]
+ paddd xmm1, xmmword ptr [rsp+0x90]
+ paddd xmm2, xmmword ptr [rsp+0xB0]
+ paddd xmm3, xmmword ptr [rsp+0x80]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x50]
+ paddd xmm1, xmmword ptr [rsp]
+ paddd xmm2, xmmword ptr [rsp+0xF0]
+ paddd xmm3, xmmword ptr [rsp+0x10]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xA0]
+ paddd xmm1, xmmword ptr [rsp+0xC0]
+ paddd xmm2, xmmword ptr [rsp+0xE0]
+ paddd xmm3, xmmword ptr [rsp+0xD0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x70]
+ paddd xmm1, xmmword ptr [rsp+0x90]
+ paddd xmm2, xmmword ptr [rsp+0x30]
+ paddd xmm3, xmmword ptr [rsp+0xF0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x40]
+ paddd xmm1, xmmword ptr [rsp+0xB0]
+ paddd xmm2, xmmword ptr [rsp+0x50]
+ paddd xmm3, xmmword ptr [rsp+0x10]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp]
+ paddd xmm1, xmmword ptr [rsp+0x20]
+ paddd xmm2, xmmword ptr [rsp+0x80]
+ paddd xmm3, xmmword ptr [rsp+0x60]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xC0]
+ paddd xmm1, xmmword ptr [rsp+0x90]
+ paddd xmm2, xmmword ptr [rsp+0xF0]
+ paddd xmm3, xmmword ptr [rsp+0xE0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xD0]
+ paddd xmm1, xmmword ptr [rsp+0xB0]
+ paddd xmm2, xmmword ptr [rsp+0xA0]
+ paddd xmm3, xmmword ptr [rsp+0x80]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x70]
+ paddd xmm1, xmmword ptr [rsp+0x50]
+ paddd xmm2, xmmword ptr [rsp]
+ paddd xmm3, xmmword ptr [rsp+0x60]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x20]
+ paddd xmm1, xmmword ptr [rsp+0x30]
+ paddd xmm2, xmmword ptr [rsp+0x10]
+ paddd xmm3, xmmword ptr [rsp+0x40]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x90]
+ paddd xmm1, xmmword ptr [rsp+0xB0]
+ paddd xmm2, xmmword ptr [rsp+0x80]
+ paddd xmm3, xmmword ptr [rsp+0xF0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xE0]
+ paddd xmm1, xmmword ptr [rsp+0x50]
+ paddd xmm2, xmmword ptr [rsp+0xC0]
+ paddd xmm3, xmmword ptr [rsp+0x10]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xD0]
+ paddd xmm1, xmmword ptr [rsp]
+ paddd xmm2, xmmword ptr [rsp+0x20]
+ paddd xmm3, xmmword ptr [rsp+0x40]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x30]
+ paddd xmm1, xmmword ptr [rsp+0xA0]
+ paddd xmm2, xmmword ptr [rsp+0x60]
+ paddd xmm3, xmmword ptr [rsp+0x70]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xB0]
+ paddd xmm1, xmmword ptr [rsp+0x50]
+ paddd xmm2, xmmword ptr [rsp+0x10]
+ paddd xmm3, xmmword ptr [rsp+0x80]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xF0]
+ paddd xmm1, xmmword ptr [rsp]
+ paddd xmm2, xmmword ptr [rsp+0x90]
+ paddd xmm3, xmmword ptr [rsp+0x60]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xE0]
+ paddd xmm1, xmmword ptr [rsp+0x20]
+ paddd xmm2, xmmword ptr [rsp+0x30]
+ paddd xmm3, xmmword ptr [rsp+0x70]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ pshuflw xmm15, xmm15, 0xB1
+ pshufhw xmm15, xmm15, 0xB1
+ pshuflw xmm12, xmm12, 0xB1
+ pshufhw xmm12, xmm12, 0xB1
+ pshuflw xmm13, xmm13, 0xB1
+ pshufhw xmm13, xmm13, 0xB1
+ pshuflw xmm14, xmm14, 0xB1
+ pshufhw xmm14, xmm14, 0xB1
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xA0]
+ paddd xmm1, xmmword ptr [rsp+0xC0]
+ paddd xmm2, xmmword ptr [rsp+0x40]
+ paddd xmm3, xmmword ptr [rsp+0xD0]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmm15
+ psrld xmm15, 8
+ pslld xmm8, 24
+ pxor xmm15, xmm8
+ movdqa xmm8, xmm12
+ psrld xmm12, 8
+ pslld xmm8, 24
+ pxor xmm12, xmm8
+ movdqa xmm8, xmm13
+ psrld xmm13, 8
+ pslld xmm8, 24
+ pxor xmm13, xmm8
+ movdqa xmm8, xmm14
+ psrld xmm14, 8
+ pslld xmm8, 24
+ pxor xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ pxor xmm0, xmm8
+ pxor xmm1, xmm9
+ pxor xmm2, xmm10
+ pxor xmm3, xmm11
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ pxor xmm4, xmm12
+ pxor xmm5, xmm13
+ pxor xmm6, xmm14
+ pxor xmm7, xmm15
+ mov eax, r13d
+ jne 9b
+ movdqa xmm9, xmm0
+ punpckldq xmm0, xmm1
+ punpckhdq xmm9, xmm1
+ movdqa xmm11, xmm2
+ punpckldq xmm2, xmm3
+ punpckhdq xmm11, xmm3
+ movdqa xmm1, xmm0
+ punpcklqdq xmm0, xmm2
+ punpckhqdq xmm1, xmm2
+ movdqa xmm3, xmm9
+ punpcklqdq xmm9, xmm11
+ punpckhqdq xmm3, xmm11
+ movdqu xmmword ptr [rbx], xmm0
+ movdqu xmmword ptr [rbx+0x20], xmm1
+ movdqu xmmword ptr [rbx+0x40], xmm9
+ movdqu xmmword ptr [rbx+0x60], xmm3
+ movdqa xmm9, xmm4
+ punpckldq xmm4, xmm5
+ punpckhdq xmm9, xmm5
+ movdqa xmm11, xmm6
+ punpckldq xmm6, xmm7
+ punpckhdq xmm11, xmm7
+ movdqa xmm5, xmm4
+ punpcklqdq xmm4, xmm6
+ punpckhqdq xmm5, xmm6
+ movdqa xmm7, xmm9
+ punpcklqdq xmm9, xmm11
+ punpckhqdq xmm7, xmm11
+ movdqu xmmword ptr [rbx+0x10], xmm4
+ movdqu xmmword ptr [rbx+0x30], xmm5
+ movdqu xmmword ptr [rbx+0x50], xmm9
+ movdqu xmmword ptr [rbx+0x70], xmm7
+ movdqa xmm1, xmmword ptr [rsp+0x110]
+ movdqa xmm0, xmm1
+ paddd xmm1, xmmword ptr [rsp+0x150]
+ movdqa xmmword ptr [rsp+0x110], xmm1
+ pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip]
+ pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip]
+ pcmpgtd xmm0, xmm1
+ movdqa xmm1, xmmword ptr [rsp+0x120]
+ psubd xmm1, xmm0
+ movdqa xmmword ptr [rsp+0x120], xmm1
+ add rbx, 128
+ add rdi, 32
+ sub rsi, 4
+ cmp rsi, 4
+ jnc 2b
+ test rsi, rsi
+ jnz 3f
+4:
+ mov rsp, rbp
+ pop rbp
+ pop rbx
+ pop r12
+ pop r13
+ pop r14
+ pop r15
+ ret
+.p2align 5
+3:
+ test esi, 0x2
+ je 3f
+ movups xmm0, xmmword ptr [rcx]
+ movups xmm1, xmmword ptr [rcx+0x10]
+ movaps xmm8, xmm0
+ movaps xmm9, xmm1
+ movd xmm13, dword ptr [rsp+0x110]
+ movd xmm14, dword ptr [rsp+0x120]
+ punpckldq xmm13, xmm14
+ movaps xmmword ptr [rsp], xmm13
+ movd xmm14, dword ptr [rsp+0x114]
+ movd xmm13, dword ptr [rsp+0x124]
+ punpckldq xmm14, xmm13
+ movaps xmmword ptr [rsp+0x10], xmm14
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+2:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ movaps xmm2, xmmword ptr [BLAKE3_IV+rip]
+ movaps xmm10, xmm2
+ movups xmm4, xmmword ptr [r8+rdx-0x40]
+ movups xmm5, xmmword ptr [r8+rdx-0x30]
+ movaps xmm3, xmm4
+ shufps xmm4, xmm5, 136
+ shufps xmm3, xmm5, 221
+ movaps xmm5, xmm3
+ movups xmm6, xmmword ptr [r8+rdx-0x20]
+ movups xmm7, xmmword ptr [r8+rdx-0x10]
+ movaps xmm3, xmm6
+ shufps xmm6, xmm7, 136
+ pshufd xmm6, xmm6, 0x93
+ shufps xmm3, xmm7, 221
+ pshufd xmm7, xmm3, 0x93
+ movups xmm12, xmmword ptr [r9+rdx-0x40]
+ movups xmm13, xmmword ptr [r9+rdx-0x30]
+ movaps xmm11, xmm12
+ shufps xmm12, xmm13, 136
+ shufps xmm11, xmm13, 221
+ movaps xmm13, xmm11
+ movups xmm14, xmmword ptr [r9+rdx-0x20]
+ movups xmm15, xmmword ptr [r9+rdx-0x10]
+ movaps xmm11, xmm14
+ shufps xmm14, xmm15, 136
+ pshufd xmm14, xmm14, 0x93
+ shufps xmm11, xmm15, 221
+ pshufd xmm15, xmm11, 0x93
+ shl rax, 0x20
+ or rax, 0x40
+ movq xmm3, rax
+ movdqa xmmword ptr [rsp+0x20], xmm3
+ movaps xmm3, xmmword ptr [rsp]
+ movaps xmm11, xmmword ptr [rsp+0x10]
+ punpcklqdq xmm3, xmmword ptr [rsp+0x20]
+ punpcklqdq xmm11, xmmword ptr [rsp+0x20]
+ mov al, 7
+9:
+ paddd xmm0, xmm4
+ paddd xmm8, xmm12
+ movaps xmmword ptr [rsp+0x20], xmm4
+ movaps xmmword ptr [rsp+0x30], xmm12
+ paddd xmm0, xmm1
+ paddd xmm8, xmm9
+ pxor xmm3, xmm0
+ pxor xmm11, xmm8
+ pshuflw xmm3, xmm3, 0xB1
+ pshufhw xmm3, xmm3, 0xB1
+ pshuflw xmm11, xmm11, 0xB1
+ pshufhw xmm11, xmm11, 0xB1
+ paddd xmm2, xmm3
+ paddd xmm10, xmm11
+ pxor xmm1, xmm2
+ pxor xmm9, xmm10
+ movdqa xmm4, xmm1
+ pslld xmm1, 20
+ psrld xmm4, 12
+ por xmm1, xmm4
+ movdqa xmm4, xmm9
+ pslld xmm9, 20
+ psrld xmm4, 12
+ por xmm9, xmm4
+ paddd xmm0, xmm5
+ paddd xmm8, xmm13
+ movaps xmmword ptr [rsp+0x40], xmm5
+ movaps xmmword ptr [rsp+0x50], xmm13
+ paddd xmm0, xmm1
+ paddd xmm8, xmm9
+ pxor xmm3, xmm0
+ pxor xmm11, xmm8
+ movdqa xmm13, xmm3
+ psrld xmm3, 8
+ pslld xmm13, 24
+ pxor xmm3, xmm13
+ movdqa xmm13, xmm11
+ psrld xmm11, 8
+ pslld xmm13, 24
+ pxor xmm11, xmm13
+ paddd xmm2, xmm3
+ paddd xmm10, xmm11
+ pxor xmm1, xmm2
+ pxor xmm9, xmm10
+ movdqa xmm4, xmm1
+ pslld xmm1, 25
+ psrld xmm4, 7
+ por xmm1, xmm4
+ movdqa xmm4, xmm9
+ pslld xmm9, 25
+ psrld xmm4, 7
+ por xmm9, xmm4
+ pshufd xmm0, xmm0, 0x93
+ pshufd xmm8, xmm8, 0x93
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm11, xmm11, 0x4E
+ pshufd xmm2, xmm2, 0x39
+ pshufd xmm10, xmm10, 0x39
+ paddd xmm0, xmm6
+ paddd xmm8, xmm14
+ paddd xmm0, xmm1
+ paddd xmm8, xmm9
+ pxor xmm3, xmm0
+ pxor xmm11, xmm8
+ pshuflw xmm3, xmm3, 0xB1
+ pshufhw xmm3, xmm3, 0xB1
+ pshuflw xmm11, xmm11, 0xB1
+ pshufhw xmm11, xmm11, 0xB1
+ paddd xmm2, xmm3
+ paddd xmm10, xmm11
+ pxor xmm1, xmm2
+ pxor xmm9, xmm10
+ movdqa xmm4, xmm1
+ pslld xmm1, 20
+ psrld xmm4, 12
+ por xmm1, xmm4
+ movdqa xmm4, xmm9
+ pslld xmm9, 20
+ psrld xmm4, 12
+ por xmm9, xmm4
+ paddd xmm0, xmm7
+ paddd xmm8, xmm15
+ paddd xmm0, xmm1
+ paddd xmm8, xmm9
+ pxor xmm3, xmm0
+ pxor xmm11, xmm8
+ movdqa xmm13, xmm3
+ psrld xmm3, 8
+ pslld xmm13, 24
+ pxor xmm3, xmm13
+ movdqa xmm13, xmm11
+ psrld xmm11, 8
+ pslld xmm13, 24
+ pxor xmm11, xmm13
+ paddd xmm2, xmm3
+ paddd xmm10, xmm11
+ pxor xmm1, xmm2
+ pxor xmm9, xmm10
+ movdqa xmm4, xmm1
+ pslld xmm1, 25
+ psrld xmm4, 7
+ por xmm1, xmm4
+ movdqa xmm4, xmm9
+ pslld xmm9, 25
+ psrld xmm4, 7
+ por xmm9, xmm4
+ pshufd xmm0, xmm0, 0x39
+ pshufd xmm8, xmm8, 0x39
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm11, xmm11, 0x4E
+ pshufd xmm2, xmm2, 0x93
+ pshufd xmm10, xmm10, 0x93
+ dec al
+ je 9f
+ movdqa xmm12, xmmword ptr [rsp+0x20]
+ movdqa xmm5, xmmword ptr [rsp+0x40]
+ pshufd xmm13, xmm12, 0x0F
+ shufps xmm12, xmm5, 214
+ pshufd xmm4, xmm12, 0x39
+ movdqa xmm12, xmm6
+ shufps xmm12, xmm7, 250
+ pand xmm13, xmmword ptr [PBLENDW_0x33_MASK+rip]
+ pand xmm12, xmmword ptr [PBLENDW_0xCC_MASK+rip]
+ por xmm13, xmm12
+ movdqa xmmword ptr [rsp+0x20], xmm13
+ movdqa xmm12, xmm7
+ punpcklqdq xmm12, xmm5
+ movdqa xmm13, xmm6
+ pand xmm12, xmmword ptr [PBLENDW_0x3F_MASK+rip]
+ pand xmm13, xmmword ptr [PBLENDW_0xC0_MASK+rip]
+ por xmm12, xmm13
+ pshufd xmm12, xmm12, 0x78
+ punpckhdq xmm5, xmm7
+ punpckldq xmm6, xmm5
+ pshufd xmm7, xmm6, 0x1E
+ movdqa xmmword ptr [rsp+0x40], xmm12
+ movdqa xmm5, xmmword ptr [rsp+0x30]
+ movdqa xmm13, xmmword ptr [rsp+0x50]
+ pshufd xmm6, xmm5, 0x0F
+ shufps xmm5, xmm13, 214
+ pshufd xmm12, xmm5, 0x39
+ movdqa xmm5, xmm14
+ shufps xmm5, xmm15, 250
+ pand xmm6, xmmword ptr [PBLENDW_0x33_MASK+rip]
+ pand xmm5, xmmword ptr [PBLENDW_0xCC_MASK+rip]
+ por xmm6, xmm5
+ movdqa xmm5, xmm15
+ punpcklqdq xmm5, xmm13
+ movdqa xmmword ptr [rsp+0x30], xmm2
+ movdqa xmm2, xmm14
+ pand xmm5, xmmword ptr [PBLENDW_0x3F_MASK+rip]
+ pand xmm2, xmmword ptr [PBLENDW_0xC0_MASK+rip]
+ por xmm5, xmm2
+ movdqa xmm2, xmmword ptr [rsp+0x30]
+ pshufd xmm5, xmm5, 0x78
+ punpckhdq xmm13, xmm15
+ punpckldq xmm14, xmm13
+ pshufd xmm15, xmm14, 0x1E
+ movdqa xmm13, xmm6
+ movdqa xmm14, xmm5
+ movdqa xmm5, xmmword ptr [rsp+0x20]
+ movdqa xmm6, xmmword ptr [rsp+0x40]
+ jmp 9b
+9:
+ pxor xmm0, xmm2
+ pxor xmm1, xmm3
+ pxor xmm8, xmm10
+ pxor xmm9, xmm11
+ mov eax, r13d
+ cmp rdx, r15
+ jne 2b
+ movups xmmword ptr [rbx], xmm0
+ movups xmmword ptr [rbx+0x10], xmm1
+ movups xmmword ptr [rbx+0x20], xmm8
+ movups xmmword ptr [rbx+0x30], xmm9
+ mov eax, dword ptr [rsp+0x130]
+ neg eax
+ mov r10d, dword ptr [rsp+0x110+8*rax]
+ mov r11d, dword ptr [rsp+0x120+8*rax]
+ mov dword ptr [rsp+0x110], r10d
+ mov dword ptr [rsp+0x120], r11d
+ add rdi, 16
+ add rbx, 64
+ sub rsi, 2
+3:
+ test esi, 0x1
+ je 4b
+ movups xmm0, xmmword ptr [rcx]
+ movups xmm1, xmmword ptr [rcx+0x10]
+ movd xmm13, dword ptr [rsp+0x110]
+ movd xmm14, dword ptr [rsp+0x120]
+ punpckldq xmm13, xmm14
+ mov r8, qword ptr [rdi]
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+2:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ movaps xmm2, xmmword ptr [BLAKE3_IV+rip]
+ shl rax, 32
+ or rax, 64
+ movq xmm12, rax
+ movdqa xmm3, xmm13
+ punpcklqdq xmm3, xmm12
+ movups xmm4, xmmword ptr [r8+rdx-0x40]
+ movups xmm5, xmmword ptr [r8+rdx-0x30]
+ movaps xmm8, xmm4
+ shufps xmm4, xmm5, 136
+ shufps xmm8, xmm5, 221
+ movaps xmm5, xmm8
+ movups xmm6, xmmword ptr [r8+rdx-0x20]
+ movups xmm7, xmmword ptr [r8+rdx-0x10]
+ movaps xmm8, xmm6
+ shufps xmm6, xmm7, 136
+ pshufd xmm6, xmm6, 0x93
+ shufps xmm8, xmm7, 221
+ pshufd xmm7, xmm8, 0x93
+ mov al, 7
+9:
+ paddd xmm0, xmm4
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshuflw xmm3, xmm3, 0xB1
+ pshufhw xmm3, xmm3, 0xB1
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm5
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ movdqa xmm14, xmm3
+ psrld xmm3, 8
+ pslld xmm14, 24
+ pxor xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x93
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x39
+ paddd xmm0, xmm6
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshuflw xmm3, xmm3, 0xB1
+ pshufhw xmm3, xmm3, 0xB1
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm7
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ movdqa xmm14, xmm3
+ psrld xmm3, 8
+ pslld xmm14, 24
+ pxor xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x39
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x93
+ dec al
+ jz 9f
+ movdqa xmm8, xmm4
+ shufps xmm8, xmm5, 214
+ pshufd xmm9, xmm4, 0x0F
+ pshufd xmm4, xmm8, 0x39
+ movdqa xmm8, xmm6
+ shufps xmm8, xmm7, 250
+ pand xmm9, xmmword ptr [PBLENDW_0x33_MASK+rip]
+ pand xmm8, xmmword ptr [PBLENDW_0xCC_MASK+rip]
+ por xmm9, xmm8
+ movdqa xmm8, xmm7
+ punpcklqdq xmm8, xmm5
+ movdqa xmm10, xmm6
+ pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip]
+ pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip]
+ por xmm8, xmm10
+ pshufd xmm8, xmm8, 0x78
+ punpckhdq xmm5, xmm7
+ punpckldq xmm6, xmm5
+ pshufd xmm7, xmm6, 0x1E
+ movdqa xmm5, xmm9
+ movdqa xmm6, xmm8
+ jmp 9b
+9:
+ pxor xmm0, xmm2
+ pxor xmm1, xmm3
+ mov eax, r13d
+ cmp rdx, r15
+ jne 2b
+ movups xmmword ptr [rbx], xmm0
+ movups xmmword ptr [rbx+0x10], xmm1
+ jmp 4b
+
+.p2align 6
+blake3_compress_in_place_sse2_asm:
+_blake3_compress_in_place_sse2_asm:
+ _CET_ENDBR
+ movups xmm0, xmmword ptr [rdi]
+ movups xmm1, xmmword ptr [rdi+0x10]
+ movaps xmm2, xmmword ptr [BLAKE3_IV+rip]
+ shl r8, 32
+ add rdx, r8
+ movq xmm3, rcx
+ movq xmm4, rdx
+ punpcklqdq xmm3, xmm4
+ movups xmm4, xmmword ptr [rsi]
+ movups xmm5, xmmword ptr [rsi+0x10]
+ movaps xmm8, xmm4
+ shufps xmm4, xmm5, 136
+ shufps xmm8, xmm5, 221
+ movaps xmm5, xmm8
+ movups xmm6, xmmword ptr [rsi+0x20]
+ movups xmm7, xmmword ptr [rsi+0x30]
+ movaps xmm8, xmm6
+ shufps xmm6, xmm7, 136
+ pshufd xmm6, xmm6, 0x93
+ shufps xmm8, xmm7, 221
+ pshufd xmm7, xmm8, 0x93
+ mov al, 7
+9:
+ paddd xmm0, xmm4
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshuflw xmm3, xmm3, 0xB1
+ pshufhw xmm3, xmm3, 0xB1
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm5
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ movdqa xmm14, xmm3
+ psrld xmm3, 8
+ pslld xmm14, 24
+ pxor xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x93
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x39
+ paddd xmm0, xmm6
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshuflw xmm3, xmm3, 0xB1
+ pshufhw xmm3, xmm3, 0xB1
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm7
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ movdqa xmm14, xmm3
+ psrld xmm3, 8
+ pslld xmm14, 24
+ pxor xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x39
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x93
+ dec al
+ jz 9f
+ movdqa xmm8, xmm4
+ shufps xmm8, xmm5, 214
+ pshufd xmm9, xmm4, 0x0F
+ pshufd xmm4, xmm8, 0x39
+ movdqa xmm8, xmm6
+ shufps xmm8, xmm7, 250
+ pand xmm9, xmmword ptr [PBLENDW_0x33_MASK+rip]
+ pand xmm8, xmmword ptr [PBLENDW_0xCC_MASK+rip]
+ por xmm9, xmm8
+ movdqa xmm8, xmm7
+ punpcklqdq xmm8, xmm5
+ movdqa xmm10, xmm6
+ pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip]
+ pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip]
+ por xmm8, xmm10
+ pshufd xmm8, xmm8, 0x78
+ punpckhdq xmm5, xmm7
+ punpckldq xmm6, xmm5
+ pshufd xmm7, xmm6, 0x1E
+ movdqa xmm5, xmm9
+ movdqa xmm6, xmm8
+ jmp 9b
+9:
+ pxor xmm0, xmm2
+ pxor xmm1, xmm3
+ movups xmmword ptr [rdi], xmm0
+ movups xmmword ptr [rdi+0x10], xmm1
+ ret
+
+.p2align 6
+blake3_compress_xof_sse2_asm:
+_blake3_compress_xof_sse2_asm:
+ _CET_ENDBR
+ movups xmm0, xmmword ptr [rdi]
+ movups xmm1, xmmword ptr [rdi+0x10]
+ movaps xmm2, xmmword ptr [BLAKE3_IV+rip]
+ movzx eax, r8b
+ movzx edx, dl
+ shl rax, 32
+ add rdx, rax
+ movq xmm3, rcx
+ movq xmm4, rdx
+ punpcklqdq xmm3, xmm4
+ movups xmm4, xmmword ptr [rsi]
+ movups xmm5, xmmword ptr [rsi+0x10]
+ movaps xmm8, xmm4
+ shufps xmm4, xmm5, 136
+ shufps xmm8, xmm5, 221
+ movaps xmm5, xmm8
+ movups xmm6, xmmword ptr [rsi+0x20]
+ movups xmm7, xmmword ptr [rsi+0x30]
+ movaps xmm8, xmm6
+ shufps xmm6, xmm7, 136
+ pshufd xmm6, xmm6, 0x93
+ shufps xmm8, xmm7, 221
+ pshufd xmm7, xmm8, 0x93
+ mov al, 7
+9:
+ paddd xmm0, xmm4
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshuflw xmm3, xmm3, 0xB1
+ pshufhw xmm3, xmm3, 0xB1
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm5
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ movdqa xmm14, xmm3
+ psrld xmm3, 8
+ pslld xmm14, 24
+ pxor xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x93
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x39
+ paddd xmm0, xmm6
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshuflw xmm3, xmm3, 0xB1
+ pshufhw xmm3, xmm3, 0xB1
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm7
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ movdqa xmm14, xmm3
+ psrld xmm3, 8
+ pslld xmm14, 24
+ pxor xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x39
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x93
+ dec al
+ jz 9f
+ movdqa xmm8, xmm4
+ shufps xmm8, xmm5, 214
+ pshufd xmm9, xmm4, 0x0F
+ pshufd xmm4, xmm8, 0x39
+ movdqa xmm8, xmm6
+ shufps xmm8, xmm7, 250
+ pand xmm9, xmmword ptr [PBLENDW_0x33_MASK+rip]
+ pand xmm8, xmmword ptr [PBLENDW_0xCC_MASK+rip]
+ por xmm9, xmm8
+ movdqa xmm8, xmm7
+ punpcklqdq xmm8, xmm5
+ movdqa xmm10, xmm6
+ pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip]
+ pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip]
+ por xmm8, xmm10
+ pshufd xmm8, xmm8, 0x78
+ punpckhdq xmm5, xmm7
+ punpckldq xmm6, xmm5
+ pshufd xmm7, xmm6, 0x1E
+ movdqa xmm5, xmm9
+ movdqa xmm6, xmm8
+ jmp 9b
+9:
+ movdqu xmm4, xmmword ptr [rdi]
+ movdqu xmm5, xmmword ptr [rdi+0x10]
+ pxor xmm0, xmm2
+ pxor xmm1, xmm3
+ pxor xmm2, xmm4
+ pxor xmm3, xmm5
+ movups xmmword ptr [r9], xmm0
+ movups xmmword ptr [r9+0x10], xmm1
+ movups xmmword ptr [r9+0x20], xmm2
+ movups xmmword ptr [r9+0x30], xmm3
+ ret
+
+
+#ifdef __APPLE__
+.static_data
+#else
+.section .rodata
+#endif
+.p2align 6
+BLAKE3_IV:
+ .long 0x6A09E667, 0xBB67AE85
+ .long 0x3C6EF372, 0xA54FF53A
+ADD0:
+ .long 0, 1, 2, 3
+ADD1:
+ .long 4, 4, 4, 4
+BLAKE3_IV_0:
+ .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667
+BLAKE3_IV_1:
+ .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85
+BLAKE3_IV_2:
+ .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372
+BLAKE3_IV_3:
+ .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A
+BLAKE3_BLOCK_LEN:
+ .long 64, 64, 64, 64
+CMP_MSB_MASK:
+ .long 0x80000000, 0x80000000, 0x80000000, 0x80000000
+PBLENDW_0x33_MASK:
+ .long 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000
+PBLENDW_0xCC_MASK:
+ .long 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF
+PBLENDW_0x3F_MASK:
+ .long 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000
+PBLENDW_0xC0_MASK:
+ .long 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF
--- /dev/null
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "blake3_impl.h"
+
+#include <immintrin.h>
+
+/* define it here instead of the header */
+static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF };
+
+#define DEGREE 4
+
+#define _mm_shuffle_ps2(a, b, c) \
+ (_mm_castps_si128( \
+ _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c))))
+
+INLINE __m128i loadu(const uint8_t src[16]) {
+ return _mm_loadu_si128((const __m128i *)src);
+}
+
+INLINE void storeu(__m128i src, uint8_t dest[16]) {
+ _mm_storeu_si128((__m128i *)dest, src);
+}
+
+INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); }
+
+// Note that clang-format doesn't like the name "xor" for some reason.
+INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); }
+
+INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); }
+
+INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
+ return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d);
+}
+
+INLINE __m128i rot16(__m128i x) {
+ return _mm_shuffle_epi8(
+ x, _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2));
+}
+
+INLINE __m128i rot12(__m128i x) {
+ return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12));
+}
+
+INLINE __m128i rot8(__m128i x) {
+ return _mm_shuffle_epi8(
+ x, _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1));
+}
+
+INLINE __m128i rot7(__m128i x) {
+ return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7));
+}
+
+INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
+ __m128i m) {
+ *row0 = addv(addv(*row0, m), *row1);
+ *row3 = xorv(*row3, *row0);
+ *row3 = rot16(*row3);
+ *row2 = addv(*row2, *row3);
+ *row1 = xorv(*row1, *row2);
+ *row1 = rot12(*row1);
+}
+
+INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3,
+ __m128i m) {
+ *row0 = addv(addv(*row0, m), *row1);
+ *row3 = xorv(*row3, *row0);
+ *row3 = rot8(*row3);
+ *row2 = addv(*row2, *row3);
+ *row1 = xorv(*row1, *row2);
+ *row1 = rot7(*row1);
+}
+
+// Note the optimization here of leaving row1 as the unrotated row, rather than
+// row0. All the message loads below are adjusted to compensate for this. See
+// discussion at https://github.com/sneves/blake2-avx2/pull/4
+INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) {
+ *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3));
+ *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
+ *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1));
+}
+
+INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) {
+ *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1));
+ *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2));
+ *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3));
+}
+
+INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter, uint8_t flags) {
+ rows[0] = loadu((uint8_t *)&cv[0]);
+ rows[1] = loadu((uint8_t *)&cv[4]);
+ rows[2] = set4(B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3);
+ rows[3] = set4(counter_low(counter), counter_high(counter),
+ (uint32_t)block_len, (uint32_t)flags);
+
+ __m128i m0 = loadu(&block[sizeof(__m128i) * 0]);
+ __m128i m1 = loadu(&block[sizeof(__m128i) * 1]);
+ __m128i m2 = loadu(&block[sizeof(__m128i) * 2]);
+ __m128i m3 = loadu(&block[sizeof(__m128i) * 3]);
+
+ __m128i t0, t1, t2, t3, tt;
+
+ // Round 1. The first round permutes the message words from the original
+ // input order, into the groups that get mixed in parallel.
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8
+ t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9
+ t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 2. This round and all following rounds apply a fixed permutation
+ // to the message words from the round before.
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 3
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 4
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 5
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 6
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+ m0 = t0;
+ m1 = t1;
+ m2 = t2;
+ m3 = t3;
+
+ // Round 7
+ t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2));
+ t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t0);
+ t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2));
+ tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3));
+ t1 = _mm_blend_epi16(tt, t1, 0xCC);
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t1);
+ diagonalize(&rows[0], &rows[2], &rows[3]);
+ t2 = _mm_unpacklo_epi64(m3, m1);
+ tt = _mm_blend_epi16(t2, m2, 0xC0);
+ t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0));
+ g1(&rows[0], &rows[1], &rows[2], &rows[3], t2);
+ t3 = _mm_unpackhi_epi32(m1, m3);
+ tt = _mm_unpacklo_epi32(m2, t3);
+ t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2));
+ g2(&rows[0], &rows[1], &rows[2], &rows[3], t3);
+ undiagonalize(&rows[0], &rows[2], &rows[3]);
+}
+
+void blake3_compress_in_place_sse41(uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags) {
+ __m128i rows[4];
+ compress_pre(rows, cv, block, block_len, counter, flags);
+ storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]);
+ storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]);
+}
+
+void blake3_compress_xof_sse41(const uint32_t cv[8],
+ const uint8_t block[BLAKE3_BLOCK_LEN],
+ uint8_t block_len, uint64_t counter,
+ uint8_t flags, uint8_t out[64]) {
+ __m128i rows[4];
+ compress_pre(rows, cv, block, block_len, counter, flags);
+ storeu(xorv(rows[0], rows[2]), &out[0]);
+ storeu(xorv(rows[1], rows[3]), &out[16]);
+ storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]);
+ storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]);
+}
+
+INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) {
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]);
+ v[0] = addv(v[0], v[4]);
+ v[1] = addv(v[1], v[5]);
+ v[2] = addv(v[2], v[6]);
+ v[3] = addv(v[3], v[7]);
+ v[12] = xorv(v[12], v[0]);
+ v[13] = xorv(v[13], v[1]);
+ v[14] = xorv(v[14], v[2]);
+ v[15] = xorv(v[15], v[3]);
+ v[12] = rot16(v[12]);
+ v[13] = rot16(v[13]);
+ v[14] = rot16(v[14]);
+ v[15] = rot16(v[15]);
+ v[8] = addv(v[8], v[12]);
+ v[9] = addv(v[9], v[13]);
+ v[10] = addv(v[10], v[14]);
+ v[11] = addv(v[11], v[15]);
+ v[4] = xorv(v[4], v[8]);
+ v[5] = xorv(v[5], v[9]);
+ v[6] = xorv(v[6], v[10]);
+ v[7] = xorv(v[7], v[11]);
+ v[4] = rot12(v[4]);
+ v[5] = rot12(v[5]);
+ v[6] = rot12(v[6]);
+ v[7] = rot12(v[7]);
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]);
+ v[0] = addv(v[0], v[4]);
+ v[1] = addv(v[1], v[5]);
+ v[2] = addv(v[2], v[6]);
+ v[3] = addv(v[3], v[7]);
+ v[12] = xorv(v[12], v[0]);
+ v[13] = xorv(v[13], v[1]);
+ v[14] = xorv(v[14], v[2]);
+ v[15] = xorv(v[15], v[3]);
+ v[12] = rot8(v[12]);
+ v[13] = rot8(v[13]);
+ v[14] = rot8(v[14]);
+ v[15] = rot8(v[15]);
+ v[8] = addv(v[8], v[12]);
+ v[9] = addv(v[9], v[13]);
+ v[10] = addv(v[10], v[14]);
+ v[11] = addv(v[11], v[15]);
+ v[4] = xorv(v[4], v[8]);
+ v[5] = xorv(v[5], v[9]);
+ v[6] = xorv(v[6], v[10]);
+ v[7] = xorv(v[7], v[11]);
+ v[4] = rot7(v[4]);
+ v[5] = rot7(v[5]);
+ v[6] = rot7(v[6]);
+ v[7] = rot7(v[7]);
+
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]);
+ v[0] = addv(v[0], v[5]);
+ v[1] = addv(v[1], v[6]);
+ v[2] = addv(v[2], v[7]);
+ v[3] = addv(v[3], v[4]);
+ v[15] = xorv(v[15], v[0]);
+ v[12] = xorv(v[12], v[1]);
+ v[13] = xorv(v[13], v[2]);
+ v[14] = xorv(v[14], v[3]);
+ v[15] = rot16(v[15]);
+ v[12] = rot16(v[12]);
+ v[13] = rot16(v[13]);
+ v[14] = rot16(v[14]);
+ v[10] = addv(v[10], v[15]);
+ v[11] = addv(v[11], v[12]);
+ v[8] = addv(v[8], v[13]);
+ v[9] = addv(v[9], v[14]);
+ v[5] = xorv(v[5], v[10]);
+ v[6] = xorv(v[6], v[11]);
+ v[7] = xorv(v[7], v[8]);
+ v[4] = xorv(v[4], v[9]);
+ v[5] = rot12(v[5]);
+ v[6] = rot12(v[6]);
+ v[7] = rot12(v[7]);
+ v[4] = rot12(v[4]);
+ v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]);
+ v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]);
+ v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]);
+ v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]);
+ v[0] = addv(v[0], v[5]);
+ v[1] = addv(v[1], v[6]);
+ v[2] = addv(v[2], v[7]);
+ v[3] = addv(v[3], v[4]);
+ v[15] = xorv(v[15], v[0]);
+ v[12] = xorv(v[12], v[1]);
+ v[13] = xorv(v[13], v[2]);
+ v[14] = xorv(v[14], v[3]);
+ v[15] = rot8(v[15]);
+ v[12] = rot8(v[12]);
+ v[13] = rot8(v[13]);
+ v[14] = rot8(v[14]);
+ v[10] = addv(v[10], v[15]);
+ v[11] = addv(v[11], v[12]);
+ v[8] = addv(v[8], v[13]);
+ v[9] = addv(v[9], v[14]);
+ v[5] = xorv(v[5], v[10]);
+ v[6] = xorv(v[6], v[11]);
+ v[7] = xorv(v[7], v[8]);
+ v[4] = xorv(v[4], v[9]);
+ v[5] = rot7(v[5]);
+ v[6] = rot7(v[6]);
+ v[7] = rot7(v[7]);
+ v[4] = rot7(v[4]);
+}
+
+INLINE void transpose_vecs(__m128i vecs[DEGREE]) {
+ // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is
+ // 22/33. Note that this doesn't split the vector into two lanes, as the
+ // AVX2 counterparts do.
+ __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]);
+ __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]);
+ __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]);
+ __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]);
+
+ // Interleave 64-bit lanes.
+ __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01);
+ __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01);
+ __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23);
+ __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23);
+
+ vecs[0] = abcd_0;
+ vecs[1] = abcd_1;
+ vecs[2] = abcd_2;
+ vecs[3] = abcd_3;
+}
+
+INLINE void transpose_msg_vecs(const uint8_t *const *inputs,
+ size_t block_offset, __m128i out[16]) {
+ out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]);
+ out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]);
+ out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]);
+ out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]);
+ out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]);
+ out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]);
+ out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]);
+ out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]);
+ out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]);
+ out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]);
+ out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]);
+ out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]);
+ out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]);
+ out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]);
+ out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]);
+ out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]);
+ for (size_t i = 0; i < 4; ++i) {
+ _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0);
+ }
+ transpose_vecs(&out[0]);
+ transpose_vecs(&out[4]);
+ transpose_vecs(&out[8]);
+ transpose_vecs(&out[12]);
+}
+
+INLINE void load_counters(uint64_t counter, bool increment_counter,
+ __m128i *out_lo, __m128i *out_hi) {
+ const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter);
+ const __m128i add0 = _mm_set_epi32(3, 2, 1, 0);
+ const __m128i add1 = _mm_and_si128(mask, add0);
+ __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1);
+ __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)),
+ _mm_xor_si128( l, _mm_set1_epi32(0x80000000)));
+ __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry);
+ *out_lo = l;
+ *out_hi = h;
+}
+
+static
+void blake3_hash4_sse41(const uint8_t *const *inputs, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ bool increment_counter, uint8_t flags,
+ uint8_t flags_start, uint8_t flags_end, uint8_t *out) {
+ __m128i h_vecs[8] = {
+ set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]),
+ set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]),
+ };
+ __m128i counter_low_vec, counter_high_vec;
+ load_counters(counter, increment_counter, &counter_low_vec,
+ &counter_high_vec);
+ uint8_t block_flags = flags | flags_start;
+
+ for (size_t block = 0; block < blocks; block++) {
+ if (block + 1 == blocks) {
+ block_flags |= flags_end;
+ }
+ __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN);
+ __m128i block_flags_vec = set1(block_flags);
+ __m128i msg_vecs[16];
+ transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs);
+
+ __m128i v[16] = {
+ h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3],
+ h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7],
+ set1(B3_IV_0), set1(B3_IV_1), set1(B3_IV_2), set1(B3_IV_3),
+ counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec,
+ };
+ round_fn(v, msg_vecs, 0);
+ round_fn(v, msg_vecs, 1);
+ round_fn(v, msg_vecs, 2);
+ round_fn(v, msg_vecs, 3);
+ round_fn(v, msg_vecs, 4);
+ round_fn(v, msg_vecs, 5);
+ round_fn(v, msg_vecs, 6);
+ h_vecs[0] = xorv(v[0], v[8]);
+ h_vecs[1] = xorv(v[1], v[9]);
+ h_vecs[2] = xorv(v[2], v[10]);
+ h_vecs[3] = xorv(v[3], v[11]);
+ h_vecs[4] = xorv(v[4], v[12]);
+ h_vecs[5] = xorv(v[5], v[13]);
+ h_vecs[6] = xorv(v[6], v[14]);
+ h_vecs[7] = xorv(v[7], v[15]);
+
+ block_flags = flags;
+ }
+
+ transpose_vecs(&h_vecs[0]);
+ transpose_vecs(&h_vecs[4]);
+ // The first four vecs now contain the first half of each output, and the
+ // second four vecs contain the second half of each output.
+ storeu(h_vecs[0], &out[0 * sizeof(__m128i)]);
+ storeu(h_vecs[4], &out[1 * sizeof(__m128i)]);
+ storeu(h_vecs[1], &out[2 * sizeof(__m128i)]);
+ storeu(h_vecs[5], &out[3 * sizeof(__m128i)]);
+ storeu(h_vecs[2], &out[4 * sizeof(__m128i)]);
+ storeu(h_vecs[6], &out[5 * sizeof(__m128i)]);
+ storeu(h_vecs[3], &out[6 * sizeof(__m128i)]);
+ storeu(h_vecs[7], &out[7 * sizeof(__m128i)]);
+}
+
+INLINE void hash_one_sse41(const uint8_t *input, size_t blocks,
+ const uint32_t key[8], uint64_t counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) {
+ uint32_t cv[8];
+ memcpy(cv, key, BLAKE3_KEY_LEN);
+ uint8_t block_flags = flags | flags_start;
+ while (blocks > 0) {
+ if (blocks == 1) {
+ block_flags |= flags_end;
+ }
+ blake3_compress_in_place_sse41(cv, input, BLAKE3_BLOCK_LEN, counter,
+ block_flags);
+ input = &input[BLAKE3_BLOCK_LEN];
+ blocks -= 1;
+ block_flags = flags;
+ }
+ memcpy(out, cv, BLAKE3_OUT_LEN);
+}
+
+void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,
+ size_t blocks, const uint32_t key[8],
+ uint64_t counter, bool increment_counter,
+ uint8_t flags, uint8_t flags_start,
+ uint8_t flags_end, uint8_t *out) {
+ while (num_inputs >= DEGREE) {
+ blake3_hash4_sse41(inputs, blocks, key, counter, increment_counter, flags,
+ flags_start, flags_end, out);
+ if (increment_counter) {
+ counter += DEGREE;
+ }
+ inputs += DEGREE;
+ num_inputs -= DEGREE;
+ out = &out[DEGREE * BLAKE3_OUT_LEN];
+ }
+ while (num_inputs > 0) {
+ hash_one_sse41(inputs[0], blocks, key, counter, flags, flags_start,
+ flags_end, out);
+ if (increment_counter) {
+ counter += 1;
+ }
+ inputs += 1;
+ num_inputs -= 1;
+ out = &out[BLAKE3_OUT_LEN];
+ }
+}
--- /dev/null
+#if defined(__ELF__) && defined(__linux__)
+.section .note.GNU-stack,"",%progbits
+#endif
+
+#if defined(__ELF__) && defined(__CET__) && defined(__has_include)
+#if __has_include(<cet.h>)
+#include <cet.h>
+#endif
+#endif
+
+#if !defined(_CET_ENDBR)
+#define _CET_ENDBR
+#endif
+
+.intel_syntax noprefix
+.global blake3_hash_many_sse41_asm
+.global _blake3_hash_many_sse41_asm
+.global blake3_compress_in_place_sse41_asm
+.global _blake3_compress_in_place_sse41_asm
+.global blake3_compress_xof_sse41_asm
+.global _blake3_compress_xof_sse41_asm
+#ifdef __APPLE__
+.text
+#else
+.section .text
+#endif
+ .p2align 6
+_blake3_hash_many_sse41_asm:
+blake3_hash_many_sse41_asm:
+ _CET_ENDBR
+ push r15
+ push r14
+ push r13
+ push r12
+ push rbx
+ push rbp
+ mov rbp, rsp
+ sub rsp, 360
+ and rsp, 0xFFFFFFFFFFFFFFC0
+ neg r9d
+ movd xmm0, r9d
+ pshufd xmm0, xmm0, 0x00
+ movdqa xmmword ptr [rsp+0x130], xmm0
+ movdqa xmm1, xmm0
+ pand xmm1, xmmword ptr [ADD0+rip]
+ pand xmm0, xmmword ptr [ADD1+rip]
+ movdqa xmmword ptr [rsp+0x150], xmm0
+ movd xmm0, r8d
+ pshufd xmm0, xmm0, 0x00
+ paddd xmm0, xmm1
+ movdqa xmmword ptr [rsp+0x110], xmm0
+ pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip]
+ pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip]
+ pcmpgtd xmm1, xmm0
+ shr r8, 32
+ movd xmm2, r8d
+ pshufd xmm2, xmm2, 0x00
+ psubd xmm2, xmm1
+ movdqa xmmword ptr [rsp+0x120], xmm2
+ mov rbx, qword ptr [rbp+0x50]
+ mov r15, rdx
+ shl r15, 6
+ movzx r13d, byte ptr [rbp+0x38]
+ movzx r12d, byte ptr [rbp+0x48]
+ cmp rsi, 4
+ jc 3f
+2:
+ movdqu xmm3, xmmword ptr [rcx]
+ pshufd xmm0, xmm3, 0x00
+ pshufd xmm1, xmm3, 0x55
+ pshufd xmm2, xmm3, 0xAA
+ pshufd xmm3, xmm3, 0xFF
+ movdqu xmm7, xmmword ptr [rcx+0x10]
+ pshufd xmm4, xmm7, 0x00
+ pshufd xmm5, xmm7, 0x55
+ pshufd xmm6, xmm7, 0xAA
+ pshufd xmm7, xmm7, 0xFF
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ mov r10, qword ptr [rdi+0x10]
+ mov r11, qword ptr [rdi+0x18]
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+9:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ movdqu xmm8, xmmword ptr [r8+rdx-0x40]
+ movdqu xmm9, xmmword ptr [r9+rdx-0x40]
+ movdqu xmm10, xmmword ptr [r10+rdx-0x40]
+ movdqu xmm11, xmmword ptr [r11+rdx-0x40]
+ movdqa xmm12, xmm8
+ punpckldq xmm8, xmm9
+ punpckhdq xmm12, xmm9
+ movdqa xmm14, xmm10
+ punpckldq xmm10, xmm11
+ punpckhdq xmm14, xmm11
+ movdqa xmm9, xmm8
+ punpcklqdq xmm8, xmm10
+ punpckhqdq xmm9, xmm10
+ movdqa xmm13, xmm12
+ punpcklqdq xmm12, xmm14
+ punpckhqdq xmm13, xmm14
+ movdqa xmmword ptr [rsp], xmm8
+ movdqa xmmword ptr [rsp+0x10], xmm9
+ movdqa xmmword ptr [rsp+0x20], xmm12
+ movdqa xmmword ptr [rsp+0x30], xmm13
+ movdqu xmm8, xmmword ptr [r8+rdx-0x30]
+ movdqu xmm9, xmmword ptr [r9+rdx-0x30]
+ movdqu xmm10, xmmword ptr [r10+rdx-0x30]
+ movdqu xmm11, xmmword ptr [r11+rdx-0x30]
+ movdqa xmm12, xmm8
+ punpckldq xmm8, xmm9
+ punpckhdq xmm12, xmm9
+ movdqa xmm14, xmm10
+ punpckldq xmm10, xmm11
+ punpckhdq xmm14, xmm11
+ movdqa xmm9, xmm8
+ punpcklqdq xmm8, xmm10
+ punpckhqdq xmm9, xmm10
+ movdqa xmm13, xmm12
+ punpcklqdq xmm12, xmm14
+ punpckhqdq xmm13, xmm14
+ movdqa xmmword ptr [rsp+0x40], xmm8
+ movdqa xmmword ptr [rsp+0x50], xmm9
+ movdqa xmmword ptr [rsp+0x60], xmm12
+ movdqa xmmword ptr [rsp+0x70], xmm13
+ movdqu xmm8, xmmword ptr [r8+rdx-0x20]
+ movdqu xmm9, xmmword ptr [r9+rdx-0x20]
+ movdqu xmm10, xmmword ptr [r10+rdx-0x20]
+ movdqu xmm11, xmmword ptr [r11+rdx-0x20]
+ movdqa xmm12, xmm8
+ punpckldq xmm8, xmm9
+ punpckhdq xmm12, xmm9
+ movdqa xmm14, xmm10
+ punpckldq xmm10, xmm11
+ punpckhdq xmm14, xmm11
+ movdqa xmm9, xmm8
+ punpcklqdq xmm8, xmm10
+ punpckhqdq xmm9, xmm10
+ movdqa xmm13, xmm12
+ punpcklqdq xmm12, xmm14
+ punpckhqdq xmm13, xmm14
+ movdqa xmmword ptr [rsp+0x80], xmm8
+ movdqa xmmword ptr [rsp+0x90], xmm9
+ movdqa xmmword ptr [rsp+0xA0], xmm12
+ movdqa xmmword ptr [rsp+0xB0], xmm13
+ movdqu xmm8, xmmword ptr [r8+rdx-0x10]
+ movdqu xmm9, xmmword ptr [r9+rdx-0x10]
+ movdqu xmm10, xmmword ptr [r10+rdx-0x10]
+ movdqu xmm11, xmmword ptr [r11+rdx-0x10]
+ movdqa xmm12, xmm8
+ punpckldq xmm8, xmm9
+ punpckhdq xmm12, xmm9
+ movdqa xmm14, xmm10
+ punpckldq xmm10, xmm11
+ punpckhdq xmm14, xmm11
+ movdqa xmm9, xmm8
+ punpcklqdq xmm8, xmm10
+ punpckhqdq xmm9, xmm10
+ movdqa xmm13, xmm12
+ punpcklqdq xmm12, xmm14
+ punpckhqdq xmm13, xmm14
+ movdqa xmmword ptr [rsp+0xC0], xmm8
+ movdqa xmmword ptr [rsp+0xD0], xmm9
+ movdqa xmmword ptr [rsp+0xE0], xmm12
+ movdqa xmmword ptr [rsp+0xF0], xmm13
+ movdqa xmm9, xmmword ptr [BLAKE3_IV_1+rip]
+ movdqa xmm10, xmmword ptr [BLAKE3_IV_2+rip]
+ movdqa xmm11, xmmword ptr [BLAKE3_IV_3+rip]
+ movdqa xmm12, xmmword ptr [rsp+0x110]
+ movdqa xmm13, xmmword ptr [rsp+0x120]
+ movdqa xmm14, xmmword ptr [BLAKE3_BLOCK_LEN+rip]
+ movd xmm15, eax
+ pshufd xmm15, xmm15, 0x00
+ prefetcht0 [r8+rdx+0x80]
+ prefetcht0 [r9+rdx+0x80]
+ prefetcht0 [r10+rdx+0x80]
+ prefetcht0 [r11+rdx+0x80]
+ paddd xmm0, xmmword ptr [rsp]
+ paddd xmm1, xmmword ptr [rsp+0x20]
+ paddd xmm2, xmmword ptr [rsp+0x40]
+ paddd xmm3, xmmword ptr [rsp+0x60]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [BLAKE3_IV_0+rip]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x10]
+ paddd xmm1, xmmword ptr [rsp+0x30]
+ paddd xmm2, xmmword ptr [rsp+0x50]
+ paddd xmm3, xmmword ptr [rsp+0x70]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x80]
+ paddd xmm1, xmmword ptr [rsp+0xA0]
+ paddd xmm2, xmmword ptr [rsp+0xC0]
+ paddd xmm3, xmmword ptr [rsp+0xE0]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x90]
+ paddd xmm1, xmmword ptr [rsp+0xB0]
+ paddd xmm2, xmmword ptr [rsp+0xD0]
+ paddd xmm3, xmmword ptr [rsp+0xF0]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x20]
+ paddd xmm1, xmmword ptr [rsp+0x30]
+ paddd xmm2, xmmword ptr [rsp+0x70]
+ paddd xmm3, xmmword ptr [rsp+0x40]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x60]
+ paddd xmm1, xmmword ptr [rsp+0xA0]
+ paddd xmm2, xmmword ptr [rsp]
+ paddd xmm3, xmmword ptr [rsp+0xD0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x10]
+ paddd xmm1, xmmword ptr [rsp+0xC0]
+ paddd xmm2, xmmword ptr [rsp+0x90]
+ paddd xmm3, xmmword ptr [rsp+0xF0]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xB0]
+ paddd xmm1, xmmword ptr [rsp+0x50]
+ paddd xmm2, xmmword ptr [rsp+0xE0]
+ paddd xmm3, xmmword ptr [rsp+0x80]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x30]
+ paddd xmm1, xmmword ptr [rsp+0xA0]
+ paddd xmm2, xmmword ptr [rsp+0xD0]
+ paddd xmm3, xmmword ptr [rsp+0x70]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x40]
+ paddd xmm1, xmmword ptr [rsp+0xC0]
+ paddd xmm2, xmmword ptr [rsp+0x20]
+ paddd xmm3, xmmword ptr [rsp+0xE0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x60]
+ paddd xmm1, xmmword ptr [rsp+0x90]
+ paddd xmm2, xmmword ptr [rsp+0xB0]
+ paddd xmm3, xmmword ptr [rsp+0x80]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x50]
+ paddd xmm1, xmmword ptr [rsp]
+ paddd xmm2, xmmword ptr [rsp+0xF0]
+ paddd xmm3, xmmword ptr [rsp+0x10]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xA0]
+ paddd xmm1, xmmword ptr [rsp+0xC0]
+ paddd xmm2, xmmword ptr [rsp+0xE0]
+ paddd xmm3, xmmword ptr [rsp+0xD0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x70]
+ paddd xmm1, xmmword ptr [rsp+0x90]
+ paddd xmm2, xmmword ptr [rsp+0x30]
+ paddd xmm3, xmmword ptr [rsp+0xF0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x40]
+ paddd xmm1, xmmword ptr [rsp+0xB0]
+ paddd xmm2, xmmword ptr [rsp+0x50]
+ paddd xmm3, xmmword ptr [rsp+0x10]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp]
+ paddd xmm1, xmmword ptr [rsp+0x20]
+ paddd xmm2, xmmword ptr [rsp+0x80]
+ paddd xmm3, xmmword ptr [rsp+0x60]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xC0]
+ paddd xmm1, xmmword ptr [rsp+0x90]
+ paddd xmm2, xmmword ptr [rsp+0xF0]
+ paddd xmm3, xmmword ptr [rsp+0xE0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xD0]
+ paddd xmm1, xmmword ptr [rsp+0xB0]
+ paddd xmm2, xmmword ptr [rsp+0xA0]
+ paddd xmm3, xmmword ptr [rsp+0x80]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x70]
+ paddd xmm1, xmmword ptr [rsp+0x50]
+ paddd xmm2, xmmword ptr [rsp]
+ paddd xmm3, xmmword ptr [rsp+0x60]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x20]
+ paddd xmm1, xmmword ptr [rsp+0x30]
+ paddd xmm2, xmmword ptr [rsp+0x10]
+ paddd xmm3, xmmword ptr [rsp+0x40]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x90]
+ paddd xmm1, xmmword ptr [rsp+0xB0]
+ paddd xmm2, xmmword ptr [rsp+0x80]
+ paddd xmm3, xmmword ptr [rsp+0xF0]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xE0]
+ paddd xmm1, xmmword ptr [rsp+0x50]
+ paddd xmm2, xmmword ptr [rsp+0xC0]
+ paddd xmm3, xmmword ptr [rsp+0x10]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xD0]
+ paddd xmm1, xmmword ptr [rsp]
+ paddd xmm2, xmmword ptr [rsp+0x20]
+ paddd xmm3, xmmword ptr [rsp+0x40]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0x30]
+ paddd xmm1, xmmword ptr [rsp+0xA0]
+ paddd xmm2, xmmword ptr [rsp+0x60]
+ paddd xmm3, xmmword ptr [rsp+0x70]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xB0]
+ paddd xmm1, xmmword ptr [rsp+0x50]
+ paddd xmm2, xmmword ptr [rsp+0x10]
+ paddd xmm3, xmmword ptr [rsp+0x80]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xF0]
+ paddd xmm1, xmmword ptr [rsp]
+ paddd xmm2, xmmword ptr [rsp+0x90]
+ paddd xmm3, xmmword ptr [rsp+0x60]
+ paddd xmm0, xmm4
+ paddd xmm1, xmm5
+ paddd xmm2, xmm6
+ paddd xmm3, xmm7
+ pxor xmm12, xmm0
+ pxor xmm13, xmm1
+ pxor xmm14, xmm2
+ pxor xmm15, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ pshufb xmm15, xmm8
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm12
+ paddd xmm9, xmm13
+ paddd xmm10, xmm14
+ paddd xmm11, xmm15
+ pxor xmm4, xmm8
+ pxor xmm5, xmm9
+ pxor xmm6, xmm10
+ pxor xmm7, xmm11
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xE0]
+ paddd xmm1, xmmword ptr [rsp+0x20]
+ paddd xmm2, xmmword ptr [rsp+0x30]
+ paddd xmm3, xmmword ptr [rsp+0x70]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT16+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ movdqa xmmword ptr [rsp+0x100], xmm8
+ movdqa xmm8, xmm5
+ psrld xmm8, 12
+ pslld xmm5, 20
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 12
+ pslld xmm6, 20
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 12
+ pslld xmm7, 20
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 12
+ pslld xmm4, 20
+ por xmm4, xmm8
+ paddd xmm0, xmmword ptr [rsp+0xA0]
+ paddd xmm1, xmmword ptr [rsp+0xC0]
+ paddd xmm2, xmmword ptr [rsp+0x40]
+ paddd xmm3, xmmword ptr [rsp+0xD0]
+ paddd xmm0, xmm5
+ paddd xmm1, xmm6
+ paddd xmm2, xmm7
+ paddd xmm3, xmm4
+ pxor xmm15, xmm0
+ pxor xmm12, xmm1
+ pxor xmm13, xmm2
+ pxor xmm14, xmm3
+ movdqa xmm8, xmmword ptr [ROT8+rip]
+ pshufb xmm15, xmm8
+ pshufb xmm12, xmm8
+ pshufb xmm13, xmm8
+ pshufb xmm14, xmm8
+ paddd xmm10, xmm15
+ paddd xmm11, xmm12
+ movdqa xmm8, xmmword ptr [rsp+0x100]
+ paddd xmm8, xmm13
+ paddd xmm9, xmm14
+ pxor xmm5, xmm10
+ pxor xmm6, xmm11
+ pxor xmm7, xmm8
+ pxor xmm4, xmm9
+ pxor xmm0, xmm8
+ pxor xmm1, xmm9
+ pxor xmm2, xmm10
+ pxor xmm3, xmm11
+ movdqa xmm8, xmm5
+ psrld xmm8, 7
+ pslld xmm5, 25
+ por xmm5, xmm8
+ movdqa xmm8, xmm6
+ psrld xmm8, 7
+ pslld xmm6, 25
+ por xmm6, xmm8
+ movdqa xmm8, xmm7
+ psrld xmm8, 7
+ pslld xmm7, 25
+ por xmm7, xmm8
+ movdqa xmm8, xmm4
+ psrld xmm8, 7
+ pslld xmm4, 25
+ por xmm4, xmm8
+ pxor xmm4, xmm12
+ pxor xmm5, xmm13
+ pxor xmm6, xmm14
+ pxor xmm7, xmm15
+ mov eax, r13d
+ jne 9b
+ movdqa xmm9, xmm0
+ punpckldq xmm0, xmm1
+ punpckhdq xmm9, xmm1
+ movdqa xmm11, xmm2
+ punpckldq xmm2, xmm3
+ punpckhdq xmm11, xmm3
+ movdqa xmm1, xmm0
+ punpcklqdq xmm0, xmm2
+ punpckhqdq xmm1, xmm2
+ movdqa xmm3, xmm9
+ punpcklqdq xmm9, xmm11
+ punpckhqdq xmm3, xmm11
+ movdqu xmmword ptr [rbx], xmm0
+ movdqu xmmword ptr [rbx+0x20], xmm1
+ movdqu xmmword ptr [rbx+0x40], xmm9
+ movdqu xmmword ptr [rbx+0x60], xmm3
+ movdqa xmm9, xmm4
+ punpckldq xmm4, xmm5
+ punpckhdq xmm9, xmm5
+ movdqa xmm11, xmm6
+ punpckldq xmm6, xmm7
+ punpckhdq xmm11, xmm7
+ movdqa xmm5, xmm4
+ punpcklqdq xmm4, xmm6
+ punpckhqdq xmm5, xmm6
+ movdqa xmm7, xmm9
+ punpcklqdq xmm9, xmm11
+ punpckhqdq xmm7, xmm11
+ movdqu xmmword ptr [rbx+0x10], xmm4
+ movdqu xmmword ptr [rbx+0x30], xmm5
+ movdqu xmmword ptr [rbx+0x50], xmm9
+ movdqu xmmword ptr [rbx+0x70], xmm7
+ movdqa xmm1, xmmword ptr [rsp+0x110]
+ movdqa xmm0, xmm1
+ paddd xmm1, xmmword ptr [rsp+0x150]
+ movdqa xmmword ptr [rsp+0x110], xmm1
+ pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip]
+ pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip]
+ pcmpgtd xmm0, xmm1
+ movdqa xmm1, xmmword ptr [rsp+0x120]
+ psubd xmm1, xmm0
+ movdqa xmmword ptr [rsp+0x120], xmm1
+ add rbx, 128
+ add rdi, 32
+ sub rsi, 4
+ cmp rsi, 4
+ jnc 2b
+ test rsi, rsi
+ jnz 3f
+4:
+ mov rsp, rbp
+ pop rbp
+ pop rbx
+ pop r12
+ pop r13
+ pop r14
+ pop r15
+ ret
+.p2align 5
+3:
+ test esi, 0x2
+ je 3f
+ movups xmm0, xmmword ptr [rcx]
+ movups xmm1, xmmword ptr [rcx+0x10]
+ movaps xmm8, xmm0
+ movaps xmm9, xmm1
+ movd xmm13, dword ptr [rsp+0x110]
+ pinsrd xmm13, dword ptr [rsp+0x120], 1
+ pinsrd xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2
+ movaps xmmword ptr [rsp], xmm13
+ movd xmm14, dword ptr [rsp+0x114]
+ pinsrd xmm14, dword ptr [rsp+0x124], 1
+ pinsrd xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2
+ movaps xmmword ptr [rsp+0x10], xmm14
+ mov r8, qword ptr [rdi]
+ mov r9, qword ptr [rdi+0x8]
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+2:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ movaps xmm2, xmmword ptr [BLAKE3_IV+rip]
+ movaps xmm10, xmm2
+ movups xmm4, xmmword ptr [r8+rdx-0x40]
+ movups xmm5, xmmword ptr [r8+rdx-0x30]
+ movaps xmm3, xmm4
+ shufps xmm4, xmm5, 136
+ shufps xmm3, xmm5, 221
+ movaps xmm5, xmm3
+ movups xmm6, xmmword ptr [r8+rdx-0x20]
+ movups xmm7, xmmword ptr [r8+rdx-0x10]
+ movaps xmm3, xmm6
+ shufps xmm6, xmm7, 136
+ pshufd xmm6, xmm6, 0x93
+ shufps xmm3, xmm7, 221
+ pshufd xmm7, xmm3, 0x93
+ movups xmm12, xmmword ptr [r9+rdx-0x40]
+ movups xmm13, xmmword ptr [r9+rdx-0x30]
+ movaps xmm11, xmm12
+ shufps xmm12, xmm13, 136
+ shufps xmm11, xmm13, 221
+ movaps xmm13, xmm11
+ movups xmm14, xmmword ptr [r9+rdx-0x20]
+ movups xmm15, xmmword ptr [r9+rdx-0x10]
+ movaps xmm11, xmm14
+ shufps xmm14, xmm15, 136
+ pshufd xmm14, xmm14, 0x93
+ shufps xmm11, xmm15, 221
+ pshufd xmm15, xmm11, 0x93
+ movaps xmm3, xmmword ptr [rsp]
+ movaps xmm11, xmmword ptr [rsp+0x10]
+ pinsrd xmm3, eax, 3
+ pinsrd xmm11, eax, 3
+ mov al, 7
+9:
+ paddd xmm0, xmm4
+ paddd xmm8, xmm12
+ movaps xmmword ptr [rsp+0x20], xmm4
+ movaps xmmword ptr [rsp+0x30], xmm12
+ paddd xmm0, xmm1
+ paddd xmm8, xmm9
+ pxor xmm3, xmm0
+ pxor xmm11, xmm8
+ movaps xmm12, xmmword ptr [ROT16+rip]
+ pshufb xmm3, xmm12
+ pshufb xmm11, xmm12
+ paddd xmm2, xmm3
+ paddd xmm10, xmm11
+ pxor xmm1, xmm2
+ pxor xmm9, xmm10
+ movdqa xmm4, xmm1
+ pslld xmm1, 20
+ psrld xmm4, 12
+ por xmm1, xmm4
+ movdqa xmm4, xmm9
+ pslld xmm9, 20
+ psrld xmm4, 12
+ por xmm9, xmm4
+ paddd xmm0, xmm5
+ paddd xmm8, xmm13
+ movaps xmmword ptr [rsp+0x40], xmm5
+ movaps xmmword ptr [rsp+0x50], xmm13
+ paddd xmm0, xmm1
+ paddd xmm8, xmm9
+ pxor xmm3, xmm0
+ pxor xmm11, xmm8
+ movaps xmm13, xmmword ptr [ROT8+rip]
+ pshufb xmm3, xmm13
+ pshufb xmm11, xmm13
+ paddd xmm2, xmm3
+ paddd xmm10, xmm11
+ pxor xmm1, xmm2
+ pxor xmm9, xmm10
+ movdqa xmm4, xmm1
+ pslld xmm1, 25
+ psrld xmm4, 7
+ por xmm1, xmm4
+ movdqa xmm4, xmm9
+ pslld xmm9, 25
+ psrld xmm4, 7
+ por xmm9, xmm4
+ pshufd xmm0, xmm0, 0x93
+ pshufd xmm8, xmm8, 0x93
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm11, xmm11, 0x4E
+ pshufd xmm2, xmm2, 0x39
+ pshufd xmm10, xmm10, 0x39
+ paddd xmm0, xmm6
+ paddd xmm8, xmm14
+ paddd xmm0, xmm1
+ paddd xmm8, xmm9
+ pxor xmm3, xmm0
+ pxor xmm11, xmm8
+ pshufb xmm3, xmm12
+ pshufb xmm11, xmm12
+ paddd xmm2, xmm3
+ paddd xmm10, xmm11
+ pxor xmm1, xmm2
+ pxor xmm9, xmm10
+ movdqa xmm4, xmm1
+ pslld xmm1, 20
+ psrld xmm4, 12
+ por xmm1, xmm4
+ movdqa xmm4, xmm9
+ pslld xmm9, 20
+ psrld xmm4, 12
+ por xmm9, xmm4
+ paddd xmm0, xmm7
+ paddd xmm8, xmm15
+ paddd xmm0, xmm1
+ paddd xmm8, xmm9
+ pxor xmm3, xmm0
+ pxor xmm11, xmm8
+ pshufb xmm3, xmm13
+ pshufb xmm11, xmm13
+ paddd xmm2, xmm3
+ paddd xmm10, xmm11
+ pxor xmm1, xmm2
+ pxor xmm9, xmm10
+ movdqa xmm4, xmm1
+ pslld xmm1, 25
+ psrld xmm4, 7
+ por xmm1, xmm4
+ movdqa xmm4, xmm9
+ pslld xmm9, 25
+ psrld xmm4, 7
+ por xmm9, xmm4
+ pshufd xmm0, xmm0, 0x39
+ pshufd xmm8, xmm8, 0x39
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm11, xmm11, 0x4E
+ pshufd xmm2, xmm2, 0x93
+ pshufd xmm10, xmm10, 0x93
+ dec al
+ je 9f
+ movdqa xmm12, xmmword ptr [rsp+0x20]
+ movdqa xmm5, xmmword ptr [rsp+0x40]
+ pshufd xmm13, xmm12, 0x0F
+ shufps xmm12, xmm5, 214
+ pshufd xmm4, xmm12, 0x39
+ movdqa xmm12, xmm6
+ shufps xmm12, xmm7, 250
+ pblendw xmm13, xmm12, 0xCC
+ movdqa xmm12, xmm7
+ punpcklqdq xmm12, xmm5
+ pblendw xmm12, xmm6, 0xC0
+ pshufd xmm12, xmm12, 0x78
+ punpckhdq xmm5, xmm7
+ punpckldq xmm6, xmm5
+ pshufd xmm7, xmm6, 0x1E
+ movdqa xmmword ptr [rsp+0x20], xmm13
+ movdqa xmmword ptr [rsp+0x40], xmm12
+ movdqa xmm5, xmmword ptr [rsp+0x30]
+ movdqa xmm13, xmmword ptr [rsp+0x50]
+ pshufd xmm6, xmm5, 0x0F
+ shufps xmm5, xmm13, 214
+ pshufd xmm12, xmm5, 0x39
+ movdqa xmm5, xmm14
+ shufps xmm5, xmm15, 250
+ pblendw xmm6, xmm5, 0xCC
+ movdqa xmm5, xmm15
+ punpcklqdq xmm5, xmm13
+ pblendw xmm5, xmm14, 0xC0
+ pshufd xmm5, xmm5, 0x78
+ punpckhdq xmm13, xmm15
+ punpckldq xmm14, xmm13
+ pshufd xmm15, xmm14, 0x1E
+ movdqa xmm13, xmm6
+ movdqa xmm14, xmm5
+ movdqa xmm5, xmmword ptr [rsp+0x20]
+ movdqa xmm6, xmmword ptr [rsp+0x40]
+ jmp 9b
+9:
+ pxor xmm0, xmm2
+ pxor xmm1, xmm3
+ pxor xmm8, xmm10
+ pxor xmm9, xmm11
+ mov eax, r13d
+ cmp rdx, r15
+ jne 2b
+ movups xmmword ptr [rbx], xmm0
+ movups xmmword ptr [rbx+0x10], xmm1
+ movups xmmword ptr [rbx+0x20], xmm8
+ movups xmmword ptr [rbx+0x30], xmm9
+ movdqa xmm0, xmmword ptr [rsp+0x130]
+ movdqa xmm1, xmmword ptr [rsp+0x110]
+ movdqa xmm2, xmmword ptr [rsp+0x120]
+ movdqu xmm3, xmmword ptr [rsp+0x118]
+ movdqu xmm4, xmmword ptr [rsp+0x128]
+ blendvps xmm1, xmm3, xmm0
+ blendvps xmm2, xmm4, xmm0
+ movdqa xmmword ptr [rsp+0x110], xmm1
+ movdqa xmmword ptr [rsp+0x120], xmm2
+ add rdi, 16
+ add rbx, 64
+ sub rsi, 2
+3:
+ test esi, 0x1
+ je 4b
+ movups xmm0, xmmword ptr [rcx]
+ movups xmm1, xmmword ptr [rcx+0x10]
+ movd xmm13, dword ptr [rsp+0x110]
+ pinsrd xmm13, dword ptr [rsp+0x120], 1
+ pinsrd xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2
+ movaps xmm14, xmmword ptr [ROT8+rip]
+ movaps xmm15, xmmword ptr [ROT16+rip]
+ mov r8, qword ptr [rdi]
+ movzx eax, byte ptr [rbp+0x40]
+ or eax, r13d
+ xor edx, edx
+2:
+ mov r14d, eax
+ or eax, r12d
+ add rdx, 64
+ cmp rdx, r15
+ cmovne eax, r14d
+ movaps xmm2, xmmword ptr [BLAKE3_IV+rip]
+ movaps xmm3, xmm13
+ pinsrd xmm3, eax, 3
+ movups xmm4, xmmword ptr [r8+rdx-0x40]
+ movups xmm5, xmmword ptr [r8+rdx-0x30]
+ movaps xmm8, xmm4
+ shufps xmm4, xmm5, 136
+ shufps xmm8, xmm5, 221
+ movaps xmm5, xmm8
+ movups xmm6, xmmword ptr [r8+rdx-0x20]
+ movups xmm7, xmmword ptr [r8+rdx-0x10]
+ movaps xmm8, xmm6
+ shufps xmm6, xmm7, 136
+ pshufd xmm6, xmm6, 0x93
+ shufps xmm8, xmm7, 221
+ pshufd xmm7, xmm8, 0x93
+ mov al, 7
+9:
+ paddd xmm0, xmm4
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm15
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm5
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x93
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x39
+ paddd xmm0, xmm6
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm15
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm7
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x39
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x93
+ dec al
+ jz 9f
+ movdqa xmm8, xmm4
+ shufps xmm8, xmm5, 214
+ pshufd xmm9, xmm4, 0x0F
+ pshufd xmm4, xmm8, 0x39
+ movdqa xmm8, xmm6
+ shufps xmm8, xmm7, 250
+ pblendw xmm9, xmm8, 0xCC
+ movdqa xmm8, xmm7
+ punpcklqdq xmm8, xmm5
+ pblendw xmm8, xmm6, 0xC0
+ pshufd xmm8, xmm8, 0x78
+ punpckhdq xmm5, xmm7
+ punpckldq xmm6, xmm5
+ pshufd xmm7, xmm6, 0x1E
+ movdqa xmm5, xmm9
+ movdqa xmm6, xmm8
+ jmp 9b
+9:
+ pxor xmm0, xmm2
+ pxor xmm1, xmm3
+ mov eax, r13d
+ cmp rdx, r15
+ jne 2b
+ movups xmmword ptr [rbx], xmm0
+ movups xmmword ptr [rbx+0x10], xmm1
+ jmp 4b
+
+.p2align 6
+blake3_compress_in_place_sse41_asm:
+_blake3_compress_in_place_sse41_asm:
+ _CET_ENDBR
+ movups xmm0, xmmword ptr [rdi]
+ movups xmm1, xmmword ptr [rdi+0x10]
+ movaps xmm2, xmmword ptr [BLAKE3_IV+rip]
+ shl r8, 32
+ add rdx, r8
+ movq xmm3, rcx
+ movq xmm4, rdx
+ punpcklqdq xmm3, xmm4
+ movups xmm4, xmmword ptr [rsi]
+ movups xmm5, xmmword ptr [rsi+0x10]
+ movaps xmm8, xmm4
+ shufps xmm4, xmm5, 136
+ shufps xmm8, xmm5, 221
+ movaps xmm5, xmm8
+ movups xmm6, xmmword ptr [rsi+0x20]
+ movups xmm7, xmmword ptr [rsi+0x30]
+ movaps xmm8, xmm6
+ shufps xmm6, xmm7, 136
+ pshufd xmm6, xmm6, 0x93
+ shufps xmm8, xmm7, 221
+ pshufd xmm7, xmm8, 0x93
+ movaps xmm14, xmmword ptr [ROT8+rip]
+ movaps xmm15, xmmword ptr [ROT16+rip]
+ mov al, 7
+9:
+ paddd xmm0, xmm4
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm15
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm5
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x93
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x39
+ paddd xmm0, xmm6
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm15
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm7
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x39
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x93
+ dec al
+ jz 9f
+ movdqa xmm8, xmm4
+ shufps xmm8, xmm5, 214
+ pshufd xmm9, xmm4, 0x0F
+ pshufd xmm4, xmm8, 0x39
+ movdqa xmm8, xmm6
+ shufps xmm8, xmm7, 250
+ pblendw xmm9, xmm8, 0xCC
+ movdqa xmm8, xmm7
+ punpcklqdq xmm8, xmm5
+ pblendw xmm8, xmm6, 0xC0
+ pshufd xmm8, xmm8, 0x78
+ punpckhdq xmm5, xmm7
+ punpckldq xmm6, xmm5
+ pshufd xmm7, xmm6, 0x1E
+ movdqa xmm5, xmm9
+ movdqa xmm6, xmm8
+ jmp 9b
+9:
+ pxor xmm0, xmm2
+ pxor xmm1, xmm3
+ movups xmmword ptr [rdi], xmm0
+ movups xmmword ptr [rdi+0x10], xmm1
+ ret
+
+.p2align 6
+blake3_compress_xof_sse41_asm:
+_blake3_compress_xof_sse41_asm:
+ _CET_ENDBR
+ movups xmm0, xmmword ptr [rdi]
+ movups xmm1, xmmword ptr [rdi+0x10]
+ movaps xmm2, xmmword ptr [BLAKE3_IV+rip]
+ movzx eax, r8b
+ movzx edx, dl
+ shl rax, 32
+ add rdx, rax
+ movq xmm3, rcx
+ movq xmm4, rdx
+ punpcklqdq xmm3, xmm4
+ movups xmm4, xmmword ptr [rsi]
+ movups xmm5, xmmword ptr [rsi+0x10]
+ movaps xmm8, xmm4
+ shufps xmm4, xmm5, 136
+ shufps xmm8, xmm5, 221
+ movaps xmm5, xmm8
+ movups xmm6, xmmword ptr [rsi+0x20]
+ movups xmm7, xmmword ptr [rsi+0x30]
+ movaps xmm8, xmm6
+ shufps xmm6, xmm7, 136
+ pshufd xmm6, xmm6, 0x93
+ shufps xmm8, xmm7, 221
+ pshufd xmm7, xmm8, 0x93
+ movaps xmm14, xmmword ptr [ROT8+rip]
+ movaps xmm15, xmmword ptr [ROT16+rip]
+ mov al, 7
+9:
+ paddd xmm0, xmm4
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm15
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm5
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x93
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x39
+ paddd xmm0, xmm6
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm15
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 20
+ psrld xmm11, 12
+ por xmm1, xmm11
+ paddd xmm0, xmm7
+ paddd xmm0, xmm1
+ pxor xmm3, xmm0
+ pshufb xmm3, xmm14
+ paddd xmm2, xmm3
+ pxor xmm1, xmm2
+ movdqa xmm11, xmm1
+ pslld xmm1, 25
+ psrld xmm11, 7
+ por xmm1, xmm11
+ pshufd xmm0, xmm0, 0x39
+ pshufd xmm3, xmm3, 0x4E
+ pshufd xmm2, xmm2, 0x93
+ dec al
+ jz 9f
+ movdqa xmm8, xmm4
+ shufps xmm8, xmm5, 214
+ pshufd xmm9, xmm4, 0x0F
+ pshufd xmm4, xmm8, 0x39
+ movdqa xmm8, xmm6
+ shufps xmm8, xmm7, 250
+ pblendw xmm9, xmm8, 0xCC
+ movdqa xmm8, xmm7
+ punpcklqdq xmm8, xmm5
+ pblendw xmm8, xmm6, 0xC0
+ pshufd xmm8, xmm8, 0x78
+ punpckhdq xmm5, xmm7
+ punpckldq xmm6, xmm5
+ pshufd xmm7, xmm6, 0x1E
+ movdqa xmm5, xmm9
+ movdqa xmm6, xmm8
+ jmp 9b
+9:
+ movdqu xmm4, xmmword ptr [rdi]
+ movdqu xmm5, xmmword ptr [rdi+0x10]
+ pxor xmm0, xmm2
+ pxor xmm1, xmm3
+ pxor xmm2, xmm4
+ pxor xmm3, xmm5
+ movups xmmword ptr [r9], xmm0
+ movups xmmword ptr [r9+0x10], xmm1
+ movups xmmword ptr [r9+0x20], xmm2
+ movups xmmword ptr [r9+0x30], xmm3
+ ret
+
+
+#ifdef __APPLE__
+.static_data
+#else
+.section .rodata
+#endif
+.p2align 6
+BLAKE3_IV:
+ .long 0x6A09E667, 0xBB67AE85
+ .long 0x3C6EF372, 0xA54FF53A
+ROT16:
+ .byte 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13
+ROT8:
+ .byte 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12
+ADD0:
+ .long 0, 1, 2, 3
+ADD1:
+ .long 4, 4, 4, 4
+BLAKE3_IV_0:
+ .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667
+BLAKE3_IV_1:
+ .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85
+BLAKE3_IV_2:
+ .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372
+BLAKE3_IV_3:
+ .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A
+BLAKE3_BLOCK_LEN:
+ .long 64, 64, 64, 64
+CMP_MSB_MASK:
+ .long 0x80000000, 0x80000000, 0x80000000, 0x80000000
--- /dev/null
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <alloca.h>
+#include <stdlib.h>
+#include <errno.h>
+
+#include "blake3.h"
+#include "blake3_impl.h"
+
+#include <libfyaml.h>
+
+struct fy_blake3_hasher {
+ uint8_t output[FY_BLAKE3_OUT_LEN] FY_CACHELINE_ALIGN;
+ struct fy_blake3_hasher_cfg cfg;
+ struct blake3_host_state *hs;
+ struct blake3_hasher *hasher;
+};
+
+struct fy_blake3_hasher *fy_blake3_hasher_create(const struct fy_blake3_hasher_cfg *cfg)
+{
+ struct fy_blake3_hasher *fyh = NULL;
+ blake3_host_config hs_cfg;
+
+ fyh = malloc(sizeof(*fyh));
+ if (!fyh)
+ goto err_out;
+
+ memset(fyh, 0, sizeof(*fyh));
+
+ /* copy config - if it exists */
+ if (cfg)
+ fyh->cfg = *cfg;
+
+ memset(&hs_cfg, 0, sizeof(hs_cfg));
+ hs_cfg.debug = false;
+ hs_cfg.backend = cfg->backend;
+ if (!cfg->tp) {
+ if (cfg->num_threads >= 0)
+ hs_cfg.num_threads = (unsigned int)cfg->num_threads;
+ else
+ hs_cfg.no_mthread = true;
+ } else
+ hs_cfg.tp = cfg->tp;
+ hs_cfg.no_mmap = cfg->no_mmap;
+ hs_cfg.file_io_bufsz = cfg->file_buffer;
+ hs_cfg.mmap_min_chunk = cfg->mmap_min_chunk;
+ hs_cfg.mmap_max_chunk = cfg->mmap_max_chunk;
+ fyh->hs = blake3_host_state_create(&hs_cfg);
+ if (!fyh->hs)
+ goto err_out;
+
+ fyh->hasher = blake3_hasher_create(fyh->hs, fyh->cfg.key, fyh->cfg.context, fyh->cfg.context ? fyh->cfg.context_len : 0);
+ if (!fyh->hasher)
+ goto err_out;
+
+ return fyh;
+
+err_out:
+ fy_blake3_hasher_destroy(fyh);
+ return NULL;
+}
+
+void fy_blake3_hasher_destroy(struct fy_blake3_hasher *fyh)
+{
+ if (!fyh)
+ return;
+ if (fyh->hasher)
+ blake3_hasher_destroy(fyh->hasher);
+ if (fyh->hs)
+ blake3_host_state_destroy(fyh->hs);
+ free(fyh);
+}
+
+void fy_blake3_hasher_update(struct fy_blake3_hasher *fyh, const void *input, size_t input_len)
+{
+ if (!fyh)
+ return;
+ blake3_hasher_update(fyh->hasher, input, input_len);
+}
+
+const uint8_t *fy_blake3_hasher_finalize(struct fy_blake3_hasher *fyh)
+{
+ if (!fyh)
+ return NULL;
+ blake3_hasher_finalize(fyh->hasher, fyh->output, FY_BLAKE3_OUT_LEN);
+ return fyh->output;
+}
+
+void fy_blake3_hasher_reset(struct fy_blake3_hasher *fyh)
+{
+ if (!fyh)
+ return;
+ blake3_hasher_reset(fyh->hasher);
+}
+
+const uint8_t *fy_blake3_hash_file(struct fy_blake3_hasher *fyh, const char *filename)
+{
+ int rc;
+
+ if (!fyh || !filename)
+ return NULL;
+
+ rc = blake3_hash_file(fyh->hasher, filename, fyh->output);
+ if (rc)
+ return NULL;
+
+ return fyh->output;
+}
+
+const uint8_t *fy_blake3_hash(struct fy_blake3_hasher *fyh, const void *mem, size_t size)
+{
+ if (!fyh || !mem)
+ return NULL;
+
+ blake3_hash(fyh->hasher, mem, size, fyh->output);
+
+ return fyh->output;
+}
+
+const char *fy_blake3_backend_iterate(const char **prevp)
+{
+ const char *backend_names[64]; /* maximum 64 backends + 1 for NULL */
+ const blake3_backend_info *bei;
+ uint64_t backends, avail;
+ unsigned int i, count;
+
+ if (!prevp)
+ return NULL;
+
+ avail = blake3_get_selectable_backends() &
+ blake3_get_detected_backends();
+
+ /* first pass, fill the backend name array */
+ for (i = 0, backends = avail, count = 0; backends && i < B3BID_COUNT; i++) {
+ if (!(backends & ((uint64_t)1 << i)))
+ continue;
+ bei = blake3_get_backend_info(i);
+ if (!bei)
+ continue;
+ backends &= ~((uint64_t)1 << i);
+ backend_names[count++] = bei->name;
+ }
+
+ /* start of the iterator */
+ if (!*prevp) {
+ if (!count)
+ return NULL;
+ i = 0;
+
+ } else {
+
+ /* ok, find the current spot */
+ for (i = 0; i < count; i++) {
+ if (!strcmp(*prevp, backend_names[i]))
+ break;
+ }
+
+ /* increase for next, last or not found? */
+ if (++i >= count) {
+ *prevp = NULL;
+ return NULL;
+ }
+ }
+
+ return *prevp = backend_names[i];
+}
--- /dev/null
+/*
+ * fy-b3sum.c - blake3 utility for testing within fyaml
+ *
+ * Copyright (c) 2023 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <alloca.h>
+#include <stdbool.h>
+#include <getopt.h>
+#include <ctype.h>
+#include <assert.h>
+
+#include <stdatomic.h>
+#include <blake3.h>
+
+#define OPT_NO_MMAP 128
+#define OPT_NO_MTHREAD 129
+#define OPT_NUM_THREADS 130
+#define OPT_BUFFER_SIZE 131
+#define OPT_MT_DEGREE 132
+#define OPT_LIST_BACKENDS 133
+#define OPT_ENABLE_CPUSIMD 134
+#define OPT_CPUSIMD_NUM_CPUS 135
+#define OPT_CPUSIMD_MULT_FACT 136
+#define OPT_DERIVE_KEY 137
+#define OPT_NO_NAMES 138
+#define OPT_RAW 139
+#define OPT_QUIET 140
+#define OPT_KEYED 141
+
+static struct option lopts[] = {
+ {"check", no_argument, 0, 'c' },
+ {"derive-key", required_argument, 0, OPT_DERIVE_KEY },
+ {"no-names", no_argument, 0, OPT_NO_NAMES },
+ {"raw", no_argument, 0, OPT_RAW },
+ {"length", required_argument, 0, 'l' },
+ {"quiet", no_argument, 0, OPT_QUIET },
+ {"keyed", no_argument, 0, OPT_KEYED },
+
+ {"num-threads", required_argument, 0, OPT_NUM_THREADS },
+ {"no-mmap", no_argument, 0, OPT_NO_MMAP },
+ {"no-mthread", no_argument, 0, OPT_NO_MTHREAD },
+ {"buffer-size", required_argument, 0, OPT_BUFFER_SIZE },
+ {"mt-degree", required_argument, 0, OPT_MT_DEGREE },
+ {"backend", required_argument, 0, 'b' },
+ {"enable-cpusimd", no_argument, 0, OPT_ENABLE_CPUSIMD },
+ {"cpusimd-num-cpus", required_argument, 0, OPT_CPUSIMD_NUM_CPUS },
+ {"cpusimd-mult-fact", required_argument, 0, OPT_CPUSIMD_MULT_FACT },
+
+ {"list-backends", no_argument, 0, OPT_LIST_BACKENDS },
+ {"help", no_argument, 0, 'h' },
+ {"debug", no_argument, 0, 'd' },
+ {0, 0, 0, 0 },
+};
+
+static void display_usage(FILE *fp, const char *progname)
+{
+ const char *s;
+
+ s = strrchr(progname, '/');
+ if (s != NULL)
+ progname = s + 1;
+
+ fprintf(fp, "Usage:\n\t%s [options] [args]\n", progname);
+ fprintf(fp, "\noptions:\n");
+ fprintf(fp, "\t--derive-key <context> : Key derivation mode, with the given context string\n");
+ fprintf(fp, "\t--no-names : Omit filenames\n");
+ fprintf(fp, "\t--raw : Output result in raw bytes (single input allowed)\n");
+ fprintf(fp, "\t--length <n>, -l <n> : Output only this amount of bytes per output (max %u)\n", BLAKE3_OUT_LEN);
+ fprintf(fp, "\t--check, -c : Read files with BLAKE3 checksums and check files\n");
+ fprintf(fp, "\t--quiet : Do not print OK for checked files that are correct\n");
+ fprintf(fp, "\t--keyed : Keyed mode with secret key read from <stdin> (32 raw bytes)\n");
+ fprintf(fp, "\ntuning options:\n");
+ fprintf(fp, "\t--num-threads <n> : Number of threads to use (default: number of CPUs * 3 / 2)\n");
+ fprintf(fp, "\t--no-mmap : Disable file mmap\n");
+ fprintf(fp, "\t--no-mthread : Disable multithreading\n");
+ fprintf(fp, "\t--buffer-size <n> : Buffer size for file I/O\n");
+ fprintf(fp, "\t--mt-degree <n> : Set the multi-thread degree (default 128)\n");
+ fprintf(fp, "\t--backend <arg>, -b <arg> : Backend selection\n");
+ fprintf(fp, "\t--enable-cpusimd : Enable experimental CPUSIMD support\n");
+ fprintf(fp, "\t--cpusimd-num-cpus : Number of CPUs assigned to CPUSIMD\n");
+ fprintf(fp, "\t--cpusimd-mult-fact : Multiplication factor for CPUSIMD\n");
+ fprintf(fp, "\ninformational options:\n");
+ fprintf(fp, "\t--list-backends : List available backends\n");
+ fprintf(fp, "\t--debug : Enable debug messages\n");
+ fprintf(fp, "\t--help, -h : Display help message\n");
+ fprintf(fp, "\n");
+ fprintf(fp, "\nargs:\n");
+ fprintf(fp, "\t<file>... Files to hash or checkfiles to check.\n\tIf no file given (or file is '-' hash stdin\n");
+}
+
+static void list_backends(const char *name)
+{
+ const blake3_backend_info *bei;
+ uint64_t backends;
+ bool selected;
+ unsigned int i;
+
+ backends = blake3_get_selectable_backends() &
+ blake3_get_detected_backends();
+ for (i = 0; backends && i < B3BID_COUNT; i++) {
+ if (!(backends & ((uint64_t)1 << i)))
+ continue;
+ bei = blake3_get_backend_info(i);
+ if (!bei)
+ continue;
+ backends &= ~((uint64_t)1 << i);
+
+ if (!name)
+ selected = backends == 0;
+ else
+ selected = !strcmp(name, bei->name);
+
+ printf("%c %-12s\t%s\n",
+ selected ? '*' : ' ', bei->name, bei->description);
+ }
+}
+
+static int do_hash_file(struct blake3_hasher *hasher, const char *filename, bool no_names, bool raw, unsigned int length)
+{
+ static const char *hexb = "0123456789abcdef";
+ uint8_t output[BLAKE3_OUT_LEN];
+ size_t filename_sz, line_sz, outsz;
+ ssize_t wrn;
+ uint8_t v;
+ void *outp;
+ char *line, *s;
+ unsigned int i;
+ int rc;
+
+ filename_sz = strlen(filename);
+
+ rc = blake3_hash_file(hasher, filename, output);
+ if (rc) {
+ fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno));
+ return -1;
+ }
+
+ if (!raw) {
+ /* output line (optimized) */
+ line_sz = (length * 2); /* the hex output */
+ if (!no_names)
+ line_sz += 2 + filename_sz; /* 2 spaces + filename */
+ line_sz++; /* '\n' */
+ line = alloca(line_sz + 1);
+ s = line;
+ for (i = 0; i < length; i++) {
+ v = output[i];
+ *s++ = hexb[v >> 4];
+ *s++ = hexb[v & 15];
+ }
+ if (!no_names) {
+ *s++ = ' ';
+ *s++ = ' ';
+ memcpy(s, filename, filename_sz);
+ s += filename_sz;
+ }
+ *s++ = '\n';
+ *s = '\0';
+ outp = line;
+ outsz = (size_t)(s - line);
+ } else {
+ outp = output;
+ outsz = length;
+ }
+
+ wrn = fwrite(outp, 1, outsz, stdout);
+ if ((size_t)wrn != outsz) {
+ fprintf(stderr, "Unable to write to stdout! error: %s\n", strerror(errno));
+ return -1;
+ }
+
+ return 0;
+}
+
+static int do_check_file(struct blake3_hasher *hasher, const char *check_filename, bool quiet)
+{
+ char *hash, *filename;
+ FILE *fp = NULL;
+ char linebuf[8192]; /* maximum size for a line is 8K, should be enough (PATH_MAX is 4K at linux) */
+ uint8_t read_hash[BLAKE3_OUT_LEN];
+ uint8_t computed_hash[BLAKE3_OUT_LEN];
+ uint8_t v;
+ char *s;
+ char c;
+ unsigned int i, j, length;
+ size_t linesz;
+ int line, rc, exit_code;
+
+ if (check_filename && strcmp(check_filename, "-")) {
+ fp = fopen(check_filename, "ra");
+ if (!fp) {
+ fprintf(stderr, "Failed to open check file: \"%s\", error: %s\n", check_filename, strerror(errno));
+ goto err_out;
+ }
+ } else {
+ fp = stdin;
+ }
+
+ /* default error code if all is fine */
+ exit_code = 0;
+
+ line = 0;
+ while (fgets(linebuf, sizeof(linebuf), fp)) {
+ /* '\0' terminate always */
+ linebuf[(sizeof(linebuf)/sizeof(linebuf[0]))-1] = '\0';
+
+ linesz = strlen(linebuf);
+ while (linesz > 0 && linebuf[linesz-1] == '\n')
+ linesz--;
+
+ if (!linesz) {
+ fprintf(stderr, "Empty line found at file \"%s\" line #%d\n", check_filename, line);
+ goto err_out;
+ }
+ linebuf[linesz] = '\0';
+
+ length = 0;
+ s = linebuf;
+ while (isxdigit(*s))
+ s++;
+
+ length = s - linebuf;
+
+ if (length == 0 || length > (BLAKE3_OUT_LEN * 2) || (length % 1) || !isspace(*s)) {
+ fprintf(stderr, "Bad line found at file \"%s\" line #%d\n", check_filename, line);
+ fprintf(stderr, "%s\n", linebuf);
+ goto err_out;
+ }
+
+ *s++ = '\0';
+
+ while (isspace(*s))
+ s++;
+
+ length >>= 1; /* to bytes */
+ hash = linebuf;
+ filename = s;
+
+ for (i = 0, s = hash; i < length; i++) {
+ v = 0;
+ for (j = 0; j < 2; j++) {
+ v <<= 4;
+ c = *s++;
+ if (c >= '0' && c <= '9')
+ v |= c - '0';
+ else if (c >= 'a' && c <= 'f')
+ v |= c - 'a' + 10;
+ else if (c >= 'A' && c <= 'F')
+ v |= c - 'A' + 10;
+ else
+ v = 0;
+ }
+ read_hash[i] = v;
+ }
+
+ rc = blake3_hash_file(hasher, filename, computed_hash);
+ if (rc) {
+ fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno));
+ goto err_out;
+ }
+
+ /* constant time comparison */
+ v = 0;
+ for (i = 0; i < length; i++)
+ v |= (read_hash[i] ^ computed_hash[i]);
+
+ if (v) {
+ printf("%s: FAILED\n", filename);
+ exit_code = -1;
+ } else if (!quiet)
+ printf("%s: OK\n", filename);
+ }
+
+out:
+ if (fp && fp != stdin)
+ fclose(fp);
+
+ return exit_code;
+
+err_out:
+ exit_code = -1;
+ goto out;
+}
+
+int main(int argc, char *argv[])
+{
+ blake3_host_config host_cfg;
+ struct blake3_host_state *host_state = NULL;
+ struct blake3_hasher *hasher = NULL;
+ const char *filename;
+ int i, opt, lidx, rc, num_inputs, num_ok;
+ int exitcode = EXIT_FAILURE, opti;
+ size_t buffer_size = 0;
+ bool no_names = false, raw = false, quiet = false, keyed = false;
+ bool no_mmap = false, no_mthread = false, debug = false, enable_cpusimd = false;
+ bool do_list_backends = false, check = false, derive_key = false;
+ unsigned int mt_degree = 0, num_threads = 0, cpusimd_num_cpus = 0, cpusimd_mult_fact = 0, length = BLAKE3_OUT_LEN;
+ const char *backend = NULL, *context = NULL;
+ uint8_t key[BLAKE3_OUT_LEN];
+ ssize_t rdn;
+
+ while ((opt = getopt_long_only(argc, argv, "cl:b:dh", lopts, &lidx)) != -1) {
+ switch (opt) {
+
+ case OPT_DERIVE_KEY:
+ derive_key = true;
+ context = optarg;
+ break;
+ case OPT_NO_NAMES:
+ no_names = true;
+ break;
+ case OPT_RAW:
+ raw = true;
+ break;
+ case 'c':
+ check = true;
+ break;
+ case OPT_QUIET:
+ quiet = true;
+ break;
+ case OPT_KEYED:
+ keyed = true;
+ break;
+
+ case 'l':
+ opti = atoi(optarg);;
+ if (opti <= 0 || opti > BLAKE3_OUT_LEN) {
+ fprintf(stderr, "Error: bad length=%d (must be > 0 and <= %u)\n\n", opti, BLAKE3_OUT_LEN);
+ goto err_out_usage;
+ }
+ length = (unsigned int)opti;
+ break;
+
+ case OPT_NO_MMAP:
+ no_mmap = true;
+ break;
+ case OPT_NO_MTHREAD:
+ no_mthread = true;
+ break;
+ case OPT_NUM_THREADS:
+ opti = atoi(optarg);
+ if (opti < 0) {
+ fprintf(stderr, "Error: bad num_threads=%d (must be >= 0)\n\n", opti);
+ goto err_out_usage;
+ }
+ num_threads = (unsigned int)opti;
+ break;
+ case OPT_BUFFER_SIZE:
+ opti = atoi(optarg);
+ if (opti <= 0) {
+ fprintf(stderr, "Error: bad buffer-size=%d (must be > 0)\n\n", opti);
+ goto err_out_usage;
+ }
+ buffer_size = (unsigned int)opti;
+ break;
+ case OPT_MT_DEGREE:
+ opti = atoi(optarg);
+ if (opti < 0) {
+ fprintf(stderr, "Error: bad mt_degree=%d (must be >= 0)\n\n", opti);
+ goto err_out_usage;
+ }
+ mt_degree = (unsigned int)opti;
+ break;
+
+ case OPT_LIST_BACKENDS:
+ do_list_backends = true;
+ break;
+
+ case OPT_ENABLE_CPUSIMD:
+ enable_cpusimd = true;
+ break;
+ case OPT_CPUSIMD_NUM_CPUS:
+ opti = atoi(optarg);
+ if (opti < 0) {
+ fprintf(stderr, "Error: bad cpusimd_num_cpus=%d (must be >= 0)\n\n", opti);
+ goto err_out_usage;
+ }
+ cpusimd_num_cpus = (unsigned int)opti;
+ break;
+ case OPT_CPUSIMD_MULT_FACT:
+ opti = atoi(optarg);
+ if (opti < 0) {
+ fprintf(stderr, "Error: bad cpusimd_mult_fact=%d (must be >= 0)\n\n", opti);
+ goto err_out_usage;
+ }
+ cpusimd_mult_fact = (unsigned int)opti;
+ break;
+
+ case 'b':
+ backend = optarg;
+ break;
+
+ case 'd':
+ debug = true;
+ break;
+ case 'h' :
+ display_usage(stdout, argv[0]);
+ goto ok_out;
+ default:
+ goto err_out_usage;
+ }
+ }
+
+ if (enable_cpusimd) {
+ rc = blake3_backend_cpusimd_setup(cpusimd_num_cpus, cpusimd_mult_fact);
+ if (rc) {
+ fprintf(stderr, "Unable to enable CPUSIMD\n");
+ goto err_out;
+ }
+ }
+
+ if (do_list_backends) {
+ list_backends(backend);
+ goto ok_out;
+ }
+
+ if (quiet && !check) {
+ fprintf(stderr, "Error: --quiet may only be used together with --check\n\n");
+ goto err_out_usage;
+ }
+
+ if (keyed && derive_key) {
+ fprintf(stderr, "Error: --keyed and --derive-key may not be used together\n\n");
+ goto err_out_usage;
+ }
+
+ if (check && length != BLAKE3_OUT_LEN) {
+ fprintf(stderr, "Error: --check and --length may not be used together\n\n");
+ goto err_out_usage;
+ }
+
+ if (keyed) {
+ rdn = fread(key, 1, BLAKE3_KEY_LEN, stdin);
+ if (rdn != BLAKE3_KEY_LEN) {
+ if (rdn >= 0 && rdn < BLAKE3_KEY_LEN)
+ fprintf(stderr, "Error: could not read secret key from <stdin>: short key\n\n");
+ else
+ fprintf(stderr, "Error: could not read secret key from <stdin>: error %s\n\n", strerror(errno));
+ goto err_out_usage;
+ }
+ rc = fgetc(stdin);
+ if (rc != EOF) {
+ fprintf(stderr, "Error: garbage trailing secret key from <stdin>\n\n");
+ goto err_out_usage;
+ }
+ }
+
+ memset(&host_cfg, 0, sizeof(host_cfg));
+ host_cfg.debug = debug;
+ host_cfg.no_mthread = no_mthread;
+ host_cfg.no_mmap = no_mmap;
+ host_cfg.num_threads = num_threads;
+ host_cfg.backend = backend;
+ host_cfg.mt_degree = mt_degree;
+ host_cfg.file_io_bufsz = buffer_size;
+
+ host_state = blake3_host_state_create(&host_cfg);
+ if (!host_state) {
+ fprintf(stderr, "unable to create blake3 host state\n");
+ goto err_out;
+ }
+
+ hasher = blake3_hasher_create(host_state,
+ keyed ? key : NULL,
+ derive_key ? context : NULL, 0);
+ if (!hasher) {
+ fprintf(stderr, "unable to create blake3 hasher\n");
+ goto err_out;
+ }
+
+ num_inputs = argc - optind;
+ if (num_inputs <= 0)
+ num_inputs = 1; /* stdin mode */
+
+ if (raw && num_inputs > 1) {
+ fprintf(stderr, "Error: Raw output mode is only supported with a single input\n\n");
+ goto err_out_usage;
+ }
+
+ if (!length)
+ length = BLAKE3_OUT_LEN;
+
+ /* we will get in the loop even when no arguments (we'll do stdin instead) */
+ num_ok = 0;
+ i = optind;
+ do {
+ /* if no arguments, use stdin */
+ filename = i < argc ? argv[i] : "-";
+
+ /* we can't handle '-' in keyed mode */
+ if (keyed && !strcmp(filename, "-")) {
+ fprintf(stderr, "Cannot use <stdin> in keyed mode\n");
+ goto err_out_usage;
+ }
+
+ if (!check)
+ rc = do_hash_file(hasher, filename, no_names, raw, length);
+ else
+ rc = do_check_file(hasher, filename, quiet);
+ if (!rc)
+ num_ok++;
+
+ } while (++i < argc);
+
+ if (num_inputs != num_ok)
+ goto err_out;
+
+ok_out:
+ exitcode = EXIT_SUCCESS;
+
+out:
+ if (hasher)
+ blake3_hasher_destroy(hasher);
+
+ if (host_state)
+ blake3_host_state_destroy(host_state);
+
+ if (enable_cpusimd)
+ blake3_backend_cpusimd_cleanup();
+
+ return exitcode;
+
+err_out_usage:
+ display_usage(stderr, argv[0]);
+err_out:
+ exitcode = EXIT_FAILURE;
+ goto out;
+}
--- /dev/null
+/*
+ * fy-thread.c - thread testing internal utility
+ *
+ * Copyright (c) 2023 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <alloca.h>
+#include <stdbool.h>
+#include <getopt.h>
+#include <ctype.h>
+#include <assert.h>
+#include <time.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <stdatomic.h>
+
+#include <libfyaml.h>
+
+#include "fy-thread.h"
+
+static void test_worker_thread_fn(void *arg)
+{
+ _Atomic(int) *p = arg;
+ int v, exp_v;
+
+ /* atomically increase the counter */
+ v = atomic_load(p);
+ for (;;) {
+ exp_v = v;
+ if (atomic_compare_exchange_strong(p, &exp_v, v + 1))
+ return;
+ v = exp_v;
+ }
+}
+
+void test_worker_threads(unsigned int num_threads)
+{
+ struct fy_thread_pool_cfg tp_cfg;
+ struct fy_thread_pool *tp;
+ struct fy_thread **threads, *t;
+ struct fy_thread_work *works;
+ long scval;
+ unsigned int i, count, num_cpus;
+ int rc, test_count;
+
+ (void)rc;
+
+ if (num_threads == 0) {
+ scval = sysconf(_SC_NPROCESSORS_ONLN);
+ assert(scval > 0);
+ num_cpus = (unsigned int)scval;
+ } else
+ num_cpus = num_threads;
+
+ memset(&tp_cfg, 0, sizeof(tp_cfg));
+ tp_cfg.flags = 0;
+ tp_cfg.num_threads = num_cpus;
+ tp_cfg.userdata = NULL;
+
+ fprintf(stderr, "calling: fy_thread_pool_create()\n");
+
+ tp = fy_thread_pool_create(&tp_cfg);
+ assert(tp);
+
+ count = tp->num_threads;
+ threads = alloca(count * sizeof(*threads));
+ works = alloca(count * sizeof(*works));
+
+ test_count = 0;
+
+ for (i = 0; i < count; i++) {
+ fprintf(stderr, "calling: fy_thread_reserve(#%u)\n", i);
+ threads[i] = fy_thread_reserve(tp);
+ assert(threads[i]);
+ t = threads[i];
+ assert(t->id == i);
+ }
+
+ for (i = 0; i < count; i++) {
+ t = threads[i];
+ fprintf(stderr, "calling: fy_thread_submit_work(#%u)\n", i);
+ works[i].fn = test_worker_thread_fn;
+ works[i].arg = &test_count;
+ fy_thread_submit_work(t, &works[i]);
+ }
+
+ for (i = 0; i < count; i++) {
+ t = threads[i];
+ fprintf(stderr, "calling: fy_thread_wait_work(#%u)\n", i);
+ fy_thread_wait_work(t);
+ }
+
+ fprintf(stderr, "%s: test_count=%d\n", __func__, test_count);
+ if (test_count != (int)num_cpus) {
+ fprintf(stderr, "error: test_count=%d expected %d\n", test_count, (int)num_cpus);
+ abort();
+ }
+
+ for (i = 0; i < count; i++) {
+ t = threads[i];
+ fprintf(stderr, "calling: fy_thread_pool_unreserve(#%u)\n", i);
+ fy_thread_unreserve(t);
+ }
+
+ fprintf(stderr, "calling: fy_thread_pool_destroy()\n");
+ fy_thread_pool_destroy(tp);
+}
+
+void test_thread_join(unsigned int num_threads)
+{
+ struct fy_thread_pool_cfg tp_cfg;
+ struct fy_thread_pool *tp;
+ void **args;
+ long scval;
+ unsigned int count, num_cpus;
+ int rc, test_count;
+
+ (void)rc;
+
+ if (num_threads == 0) {
+ scval = sysconf(_SC_NPROCESSORS_ONLN);
+ assert(scval > 0);
+ num_cpus = (unsigned int)scval;
+ } else
+ num_cpus = num_threads;
+
+ memset(&tp_cfg, 0, sizeof(tp_cfg));
+ tp_cfg.flags = 0;
+ tp_cfg.num_threads = num_cpus;
+ tp_cfg.userdata = NULL;
+
+ fprintf(stderr, "calling: fy_thread_pool_create()\n");
+ tp = fy_thread_pool_create(&tp_cfg);
+ assert(tp);
+
+ count = tp->num_threads;
+ args = alloca(count * sizeof(*args));
+
+ test_count = 0;
+
+ fy_thread_arg_join(tp, test_worker_thread_fn, NULL, &test_count, count);
+
+ fprintf(stderr, "%s: test_count=%d\n", __func__, test_count);
+ assert(test_count == (int)num_cpus);
+
+ fprintf(stderr, "calling: fy_thread_pool_destroy()\n");
+ fy_thread_pool_destroy(tp);
+}
+
+struct thread_latency_state {
+ struct timespec reserve;
+ struct timespec reserve_done;
+ struct timespec submit;
+ struct timespec execute;
+ struct timespec wait;
+ struct timespec wait_done;
+ struct timespec unreserve;
+ struct timespec unreserve_done;
+};
+
+long long delta_ns(struct timespec before, struct timespec after)
+{
+ if ((before.tv_sec == 0 && before.tv_nsec == 0) ||
+ (after.tv_sec == 0 && after.tv_nsec == 0))
+ return -1;
+ return (long long)((int64_t)(after.tv_sec - before.tv_sec) * (int64_t)1000000000UL + (int64_t)(after.tv_nsec - before.tv_nsec));
+}
+
+static void test_latency_worker_thread_fn(void *arg)
+{
+ struct thread_latency_state *s = arg;
+
+ clock_gettime(CLOCK_MONOTONIC, &s->execute);
+}
+
+void test_thread_latency(unsigned int num_threads)
+{
+ struct fy_thread_pool_cfg tp_cfg;
+ struct fy_thread_pool *tp;
+ struct fy_thread **threads, *t;
+ struct fy_thread_work *works;
+ long scval;
+ unsigned int i, count, num_cpus;
+ int rc;
+ struct thread_latency_state *states, *s;
+
+ (void)rc;
+
+ if (num_threads == 0) {
+ scval = sysconf(_SC_NPROCESSORS_ONLN);
+ assert(scval > 0);
+ num_cpus = (unsigned int)scval;
+ } else
+ num_cpus = num_threads;
+
+ memset(&tp_cfg, 0, sizeof(tp_cfg));
+ tp_cfg.flags = 0;
+ tp_cfg.num_threads = num_cpus;
+ tp_cfg.userdata = NULL;
+
+ tp = fy_thread_pool_create(&tp_cfg);
+ assert(tp);
+
+
+ count = tp->num_threads;
+ threads = alloca(count * sizeof(*threads));
+ works = alloca(count * sizeof(*works));
+ states = alloca(count * sizeof(*states));
+
+ memset(states, 0, count * sizeof(*states));
+
+ for (i = 0; i < count; i++) {
+ s = &states[i];
+
+ clock_gettime(CLOCK_MONOTONIC, &s->reserve);
+
+ threads[i] = fy_thread_reserve(tp);
+ assert(threads[i]);
+ t = threads[i];
+ assert(t->id == i);
+
+ clock_gettime(CLOCK_MONOTONIC, &s->reserve_done);
+
+ }
+
+ for (i = 0; i < count; i++) {
+ s = &states[i];
+ clock_gettime(CLOCK_MONOTONIC, &s->submit);
+
+ t = threads[i];
+ works[i].fn = test_latency_worker_thread_fn;
+ works[i].arg = s;
+ fy_thread_submit_work(t, &works[i]);
+ }
+
+ for (i = 0; i < count; i++) {
+ s = &states[i];
+ clock_gettime(CLOCK_MONOTONIC, &s->wait);
+
+ t = threads[i];
+ fy_thread_wait_work(t);
+
+ clock_gettime(CLOCK_MONOTONIC, &s->wait_done);
+ }
+
+ for (i = 0; i < count; i++) {
+
+ s = &states[i];
+ clock_gettime(CLOCK_MONOTONIC, &s->unreserve);
+
+ t = threads[i];
+ fy_thread_unreserve(t);
+
+ clock_gettime(CLOCK_MONOTONIC, &s->unreserve_done);
+ }
+
+ fy_thread_pool_destroy(tp);
+
+ fprintf(stderr, "latency results\n");
+ for (i = 0; i < count; i++) {
+ s = &states[i];
+
+ fprintf(stderr, "#%2u: reserve:%10lld submit-execute:%10lld execute-waitdone:%10lld wait:%10lld unreserve:%10lld\n", i,
+ delta_ns(s->reserve, s->reserve_done),
+ delta_ns(s->submit, s->execute),
+ delta_ns(s->execute, s->wait_done),
+ delta_ns(s->wait, s->wait_done),
+ delta_ns(s->unreserve, s->unreserve_done));
+ }
+}
+
+// #define STEAL_LOOP_COUNT 100000000
+#define STEAL_LOOP_COUNT 10000
+static void test_worker_thread_steal_fn(void *arg)
+{
+ _Atomic(int) *p = arg;
+ int v, exp_v;
+ unsigned int i;
+
+ /* atomically increase the counter STEAL_LOOP_COUNT times */
+ for (i = 0; i < STEAL_LOOP_COUNT; i++) {
+ v = atomic_load(p);
+ for (;;) {
+ exp_v = v;
+ if (atomic_compare_exchange_strong(p, &exp_v, v + 1))
+ break;
+ v = exp_v;
+ }
+ }
+}
+
+void test_thread_join_steal(unsigned int num_threads)
+{
+ struct fy_thread_pool_cfg tp_cfg;
+ struct fy_thread_pool *tp;
+ void **args;
+ long scval;
+ unsigned int count, num_cpus;
+ int rc, test_count;
+
+ (void)rc;
+
+ if (num_threads == 0) {
+ scval = sysconf(_SC_NPROCESSORS_ONLN);
+ assert(scval > 0);
+ num_cpus = (unsigned int)scval;
+ } else
+ num_cpus = num_threads;
+
+
+ tp_cfg.flags = FYTPCF_STEAL_MODE;
+ tp_cfg.num_threads = num_cpus;
+ tp_cfg.userdata = NULL;
+
+ fprintf(stderr, "calling: fy_thread_pool_create()\n");
+ tp = fy_thread_pool_create(&tp_cfg);
+ assert(tp);
+
+ count = tp->num_threads * 4;
+ args = alloca(count * sizeof(*args));
+
+ test_count = 0;
+
+ fy_thread_arg_join(tp, test_worker_thread_steal_fn, NULL, &test_count, count);
+
+ fprintf(stderr, "%s: test_count=%d\n", __func__, test_count);
+ if (test_count != (int)num_cpus * STEAL_LOOP_COUNT * 4) {
+ fprintf(stderr, "error: test_count=%d expected %d\n", test_count, (int)num_cpus * STEAL_LOOP_COUNT);
+ abort();
+ }
+
+ fprintf(stderr, "calling: fy_thread_pool_destroy()\n");
+ fy_thread_pool_destroy(tp);
+}
+
+struct sum_args {
+ struct fy_thread_pool *tp;
+ const uint8_t *values_start;
+ unsigned int count_start;
+ const uint8_t *values;
+ unsigned int count;
+ uint64_t sum;
+};
+
+static uint64_t calc_sum(const uint8_t *values, unsigned int count)
+{
+ unsigned int i;
+ uint64_t sum = 0;
+
+ for (i = 0; i < count; i++)
+ sum += values[i];
+
+ return sum;
+}
+
+static void test_worker_thread_sum_fn(void *arg)
+{
+ struct sum_args *s = arg;
+ struct sum_args args[2];
+ uint64_t sum;
+ unsigned int pos;
+
+ pos = s->values - s->values_start;
+ assert(pos <= s->count_start);
+ assert(pos + s->count <= s->count_start);
+
+ (void)pos;
+
+ // if (s->count <= (1 << 20) / 8) {
+ if (s->count <= 4096) {
+ // fprintf(stderr, "S<%06x-%06x>\n", pos, pos + s->count - 1);
+ sum = calc_sum(s->values, s->count);
+ } else {
+ memset(args, 0, sizeof(args));
+ args[0].tp = args[1].tp = s->tp;
+ args[0].values_start = args[1].values_start = s->values_start;
+ args[0].count_start = args[1].count_start = s->count_start;
+ args[0].sum = args[1].sum = 0;
+ args[0].values = s->values;
+ args[0].count = s->count / 2;
+ args[1].values = s->values + args[0].count;
+ args[1].count = s->count - args[0].count;
+
+ // fprintf(stderr, "M<%06x-%06x,%06x-%06x>\n",
+ // pos,
+ // pos + args[0].count - 1,
+ // pos + args[0].count,
+ // pos + args[0].count + args[1].count - 1);
+
+ fy_thread_arg_array_join(s->tp, test_worker_thread_sum_fn, NULL, &args, sizeof(args[0]), sizeof(args)/sizeof(args[0]));
+ sum = args[0].sum + args[1].sum;
+ }
+ s->sum = sum;
+}
+
+void test_thread_join_sum(unsigned int num_threads, unsigned int count, bool steal_mode, unsigned int times)
+{
+ struct fy_thread_pool_cfg tp_cfg;
+ struct fy_thread_pool *tp;
+ struct timespec before, after;
+ unsigned int i, num_cpus;
+ long scval;
+ uint8_t *values;
+ int rc;
+ uint64_t sum_single, sum_multi;
+ struct sum_args args[2];
+ long long table_multi[times];
+ long long ns;
+
+ (void)rc;
+
+ fprintf(stderr, "**********************************************************************\n");
+ fprintf(stderr, "%s: steal_mode=%s\n", __func__, steal_mode ? "true" : "false");
+
+ values = malloc(count * sizeof(*values));
+ assert(values);
+
+ clock_gettime(CLOCK_MONOTONIC, &before);
+ for (i = 0; i < count; i++)
+ values[i] = (uint8_t)rand();
+
+ clock_gettime(CLOCK_MONOTONIC, &after);
+ fprintf(stderr, "%s: seeding done in %lldus\n", __func__, delta_ns(before, after) / 1000);
+
+ clock_gettime(CLOCK_MONOTONIC, &before);
+ sum_single = calc_sum(values, count);
+ clock_gettime(CLOCK_MONOTONIC, &after);
+ ns = delta_ns(before, after);
+ fprintf(stderr, "%s: calculated sum=%"PRIu64" (single threaded) done in %lldus\n", __func__, sum_single, ns / 1000);
+
+ if (num_threads == 0) {
+ scval = sysconf(_SC_NPROCESSORS_ONLN);
+ assert(scval > 0);
+ num_cpus = (unsigned int)scval;
+ } else
+ num_cpus = num_threads;
+
+ memset(&tp_cfg, 0, sizeof(tp_cfg));
+ tp_cfg.flags = steal_mode ? FYTPCF_STEAL_MODE : 0;
+ tp_cfg.num_threads = num_cpus;
+ tp_cfg.userdata = NULL;
+
+ tp = fy_thread_pool_create(&tp_cfg);
+ assert(tp);
+
+ fprintf(stderr, "%s: calculating (multi threaded) -", __func__);
+ for (i = 0; i < times; i++) {
+ clock_gettime(CLOCK_MONOTONIC, &before);
+
+ memset(args, 0, sizeof(args));
+ args[0].tp = args[1].tp = tp;
+ args[0].values_start = args[1].values_start = values;
+ args[0].count_start = args[1].count_start = count;
+ args[0].sum = args[1].sum = 0;
+
+ args[0].values = values;
+ args[0].count = count / 2;
+ args[1].values = values + args[0].count;
+ args[1].count = count - args[0].count;
+
+ // fprintf(stderr, "M<%06x-%06x,%06x-%06x>\n",
+ // 0,
+ // args[0].count - 1,
+ // args[0].count,
+ // args[0].count + args[1].count - 1);
+
+ fy_thread_arg_array_join(tp, test_worker_thread_sum_fn, NULL, &args, sizeof(args[0]), sizeof(args)/sizeof(args[0]));
+
+ sum_multi = args[0].sum + args[1].sum;
+ if (sum_multi != sum_single) {
+ fprintf(stderr, "\nFailed sum_multi %"PRIu64" should be %"PRIu64"\n", sum_multi, sum_single);
+ abort();
+ }
+
+ clock_gettime(CLOCK_MONOTONIC, &after);
+ ns = delta_ns(before, after);
+ fprintf(stderr, " %lldus", ns / 1000);
+ fflush(stderr);
+ table_multi[i] = ns;
+ }
+ ns = 0;
+ for (i = 0; i < times; i++)
+ ns += table_multi[i];
+ ns /= times;
+ fprintf(stderr, " : average %lldus\n", ns / 1000);
+
+ fy_thread_pool_destroy(tp);
+
+ free(values);
+}
+
+int thread_test(unsigned int num_threads)
+{
+#if 0
+ test_worker_threads(num_threads);
+ test_thread_join(num_threads);
+ test_thread_latency(num_threads);
+ test_thread_join_steal(num_threads);
+#endif
+ test_thread_join_sum(num_threads, 1 << 20, false, 10); /* 1M of values */
+ test_thread_join_sum(num_threads, 1 << 20, true, 10); /* 1M of values */
+
+ return 0;
+}
+
+#define OPT_NUM_THREADS 128
+
+static struct option lopts[] = {
+ {"num-threads", required_argument, 0, OPT_NUM_THREADS },
+ {"help", no_argument, 0, 'h' },
+ {0, 0, 0, 0 },
+};
+
+static void display_usage(FILE *fp, const char *progname)
+{
+ const char *s;
+
+ s = strrchr(progname, '/');
+ if (s != NULL)
+ progname = s + 1;
+
+ fprintf(fp, "Usage:\n\t%s [options]\n", progname);
+ fprintf(fp, "\noptions:\n");
+ fprintf(fp, "\t--num-threads <n> : Number of threads to use (default: number of CPUs * 3 / 2)\n");
+ fprintf(fp, "\t--help, -h : Display help message\n");
+ fprintf(fp, "\n");
+}
+
+int main(int argc, char *argv[])
+{
+ int opt, lidx, rc;
+ unsigned int num_threads = 0;
+ int exitcode = EXIT_FAILURE, opti;
+
+ while ((opt = getopt_long_only(argc, argv, "h", lopts, &lidx)) != -1) {
+ switch (opt) {
+ case OPT_NUM_THREADS:
+ opti = atoi(optarg);
+ if (opti < 0) {
+ fprintf(stderr, "Error: bad num_threads=%d (must be >= 0)\n\n", opti);
+ goto err_out_usage;
+ }
+ num_threads = (unsigned int)opti;
+ break;
+ case 'h' :
+ display_usage(stdout, argv[0]);
+ goto ok_out;
+ default:
+ goto err_out_usage;
+ }
+ }
+
+ rc = thread_test(num_threads);
+ if (rc) {
+ fprintf(stderr, "Error: thread_test() failed\n");
+ goto err_out;
+ }
+
+ok_out:
+ exitcode = EXIT_SUCCESS;
+
+out:
+ return exitcode;
+
+err_out_usage:
+ display_usage(stderr, argv[0]);
+err_out:
+ exitcode = EXIT_FAILURE;
+ goto out;
+}
#include <unistd.h>
#include <limits.h>
#include <ctype.h>
+#include <time.h>
+#include <inttypes.h>
#include <libfyaml.h>
#include "fy-parse.h"
#include "fy-walk.h"
+#include "fy-blob.h"
#include "fy-valgrind.h"
#define LIBYAML_MODES ""
#endif
-#define MODES "parse|scan|copy|testsuite|dump|dump2|build|walk|reader|compose|iterate|comment|pathspec" LIBYAML_MODES
+#define MODES "parse|scan|copy|testsuite|dump|dump2|build|walk|reader|compose|iterate|comment|pathspec|shell-split|parse-timing" LIBYAML_MODES
static void display_usage(FILE *fp, char *progname)
{
struct fy_path_exec_cfg xcfg_local, *xcfg = &xcfg_local;
char *path;
unsigned int flags;
- int rc, count;
+ int rc;
flags = 0;
if (sort)
fypx = fy_path_exec_create(xcfg);
assert(fypx);
- count = 0;
while ((fyd = fy_parse_load_document(fyp)) != NULL) {
if (resolve) {
fy_walk_result_free_rl(NULL, result);
fy_parse_document_destroy(fyp, fyd);
-
- count++;
}
fy_path_exec_unref(fypx);
return 0;
}
+int do_shell_split(int in_argc, char *in_argv[])
+{
+ char buf[256], line[256 + 1];
+ char *s, *e;
+ const char * const *argv;
+ int i, argc;
+ void *mem;
+
+ printf("shell split; Ctrl-D to exit\n");
+ buf[sizeof(buf)-1] = '\0';
+ while (fgets(buf, sizeof(buf) - 1, stdin)) {
+ buf[sizeof(buf)-1] = '\0';
+ strcpy(line, buf);
+ s = line;
+ e = s + strlen(line);
+ while (e > s && e[-1] == '\n')
+ *--e = '\0';
+
+ printf("input: '%s'\n", line);
+
+ mem = fy_utf8_split_posix(line, &argc, &argv);
+ if (!mem) {
+ fprintf(stderr, "Bad input '%s'\n", line);
+ } else {
+ for (i = 0; i < argc; i++) {
+ fprintf(stderr, "%d: %s\n", i, argv[i]);
+ }
+ assert(argv[argc] == NULL);
+ free(mem);
+ }
+ }
+ return 0;
+}
+
+int do_parse_timing(int argc, char *argv[], bool disable_mmap)
+{
+ void *blob;
+ size_t blob_size;
+ int i, c, w;
+ struct timespec before, after;
+ int64_t ns;
+ const uint8_t *s, *e, *ss;
+ size_t count;
+ struct fy_utf8_result res;
+
+#undef BEFORE
+#define BEFORE() \
+ do { \
+ clock_gettime(CLOCK_MONOTONIC, &before); \
+ } while(0)
+
+#undef AFTER
+#define AFTER() \
+ ({ \
+ clock_gettime(CLOCK_MONOTONIC, &after); \
+ (int64_t)(after.tv_sec - before.tv_sec) * (int64_t)1000000000UL + (int64_t)(after.tv_nsec - before.tv_nsec); \
+ })
+
+ for (i = optind; i < argc; i++) {
+
+ printf("file=%s\n", argv[i]);
+
+ BEFORE();
+ blob = fy_blob_read(argv[i], &blob_size);
+ assert(blob);
+ ns = AFTER();
+
+ printf("read %zu bytes in %"PRId64"ns\n", blob_size, ns);
+
+ BEFORE();
+ s = blob;
+ e = s + blob_size;
+ count = 0;
+ while (s < e) {
+ if (*s++ == 'e')
+ count++;
+ }
+ ns = AFTER();
+ printf("%zu 'e' chars method 1 in %"PRId64"ns\n", count, ns);
+
+ BEFORE();
+ s = blob;
+ e = s + blob_size;
+ count = 0;
+ while ((ss = memchr(s, 'e', e - s)) != NULL) {
+ count++;
+ s = ss + 1;
+ }
+ ns = AFTER();
+ printf("%zu 'e' chars method 2 in %"PRId64"ns\n", count, ns);
+
+ BEFORE();
+ s = blob;
+ e = s + blob_size;
+ count = 0;
+ while ((c = fy_utf8_get(s, e - s, &w)) >= 0) {
+ if (c == 'e')
+ count++;
+ s += w;
+ }
+ ns = AFTER();
+ printf("%zu 'e' utf8 characters using method 1 in %"PRId64"ns\n", count, ns);
+
+ BEFORE();
+ s = blob;
+ e = s + blob_size;
+ count = 0;
+ while ((c = fy_utf8_get_s(s, e, &w)) >= 0) {
+ if (c == 'e')
+ count++;
+ s += w;
+ }
+ ns = AFTER();
+ printf("%zu 'e' utf8 characters using method 2 in %"PRId64"ns\n", count, ns);
+
+ BEFORE();
+ s = blob;
+ e = s + blob_size - FY_UTF8_MAX_WIDTH;
+ count = 0;
+ while (s < e && (c = fy_utf8_get_s_nocheck(s, &w)) >= 0) {
+ if (c == 'e')
+ count++;
+ s += w;
+ }
+ while ((c = fy_utf8_get_s(s, e, &w)) >= 0) {
+ if (c == 'e')
+ count++;
+ s += w;
+ }
+ ns = AFTER();
+ printf("%zu 'e' utf8 characters using method 3 in %"PRId64"ns\n", count, ns);
+
+ BEFORE();
+ s = blob;
+ e = s + blob_size;
+ count = 0;
+ while ((res = fy_utf8_get_s_res(s, e)).c >= 0) {
+ if (res.c == 'e')
+ count++;
+ s += res.w;
+ }
+ ns = AFTER();
+ printf("%zu 'e' utf8 characters using method 4 in %"PRId64"ns\n", count, ns);
+
+ free(blob);
+
+ }
+
+ return 0;
+}
+
int apply_flags_option(const char *arg, unsigned int *flagsp,
int (*modify_flags)(const char *what, unsigned int *flagsp))
{
strcmp(mode, "pathspec") &&
strcmp(mode, "bypath") &&
strcmp(mode, "crash") &&
- strcmp(mode, "badutf8")
+ strcmp(mode, "badutf8") &&
+ strcmp(mode, "shell-split") &&
+ strcmp(mode, "parse-timing")
#if defined(HAVE_LIBYAML) && HAVE_LIBYAML
&& strcmp(mode, "libyaml-scan")
&& strcmp(mode, "libyaml-parse")
/* fprintf(stderr, "do_bypath() error %d\n", rc); */
goto cleanup;
}
+ } else if (!strcmp(mode, "shell-split")) {
+ rc = do_shell_split(argc, argv);
+ if (rc < 0) {
+ /* fprintf(stderr, "do_shell_split() error %d\n", rc); */
+ goto cleanup;
+ }
+ } else if (!strcmp(mode, "parse-timing")) {
+ rc = do_parse_timing(argc, argv, !!(cfg.flags & FYPCF_DISABLE_MMAP_OPT));
+ if (rc < 0) {
+ /* fprintf(stderr, "do_parse_timing() error %d\n", rc); */
+ goto cleanup;
+ }
}
#if defined(HAVE_LIBYAML) && HAVE_LIBYAML
if (!strcmp(mode, "libyaml-diff")) {
return -1;
/* first try the pushed ungetc */
- if (iter->unget_c != -1) {
+ if (iter->unget_c >= 0) {
c = iter->unget_c;
+ /* unmatched getc/ungetc */
+ if (fy_utf8_width(c) != 1)
+ return -1;
iter->unget_c = -1;
- return c & 0xff;
+ return c;
}
/* read first octet */
int fy_atom_iter_ungetc(struct fy_atom_iter *iter, int c)
{
- if (!iter)
+ if (!iter || c >= 0x80)
return -1;
- if (iter->unget_c != -1)
+ if (iter->unget_c >= 0)
return -1;
- if (c == -1) {
+ if (c < 0) {
iter->unget_c = -1;
return 0;
}
- iter->unget_c = c & 0xff;
- return c & 0xff;
+ iter->unget_c = c;
+ return c;
}
int fy_atom_iter_peekc(struct fy_atom_iter *iter)
return -1;
/* first try the pushed ungetc */
- if (iter->unget_c != -1) {
+ if (iter->unget_c >= 0) {
c = iter->unget_c;
iter->unget_c = -1;
- return c & 0xff;
+ return c;
}
/* read first octet */
return -1;
/* first try the pushed ungetc */
- if (iter->unget_c != -1) {
+ if (iter->unget_c >= 0) {
c = iter->unget_c;
iter->unget_c = -1;
*lenp = 0;
- return c & 0xff;
+ return c;
}
/* read first octet */
int fy_atom_iter_utf8_unget(struct fy_atom_iter *iter, int c)
{
- if (iter->unget_c != -1)
+ if (!iter)
return -1;
- if (c == -1) {
+
+ if (iter->unget_c >= 0)
+ return -1;
+ if (c < 0) {
iter->unget_c = -1;
return 0;
}
+ if (!fy_utf8_is_valid(c))
+ return -1;
iter->unget_c = c;
return c;
}
fy_atom_iter_start(atom, &iter);
/* skip minus sign if it's there */
- c = fy_atom_iter_peekc(&iter);
+ c = fy_atom_iter_utf8_peek(&iter);
if (c == '-') {
- (void)fy_atom_iter_getc(&iter);
+ (void)fy_atom_iter_utf8_get(&iter);
len++;
}
/* skip digits */
first_zero = false;
dec = 0;
- while ((c = fy_atom_iter_peekc(&iter)) >= 0 && isdigit(c)) {
+ while ((c = fy_atom_iter_utf8_peek(&iter)) >= 0 && fy_is_digit(c)) {
if (dec == 0 && c == '0')
first_zero = true;
else if (dec == 1 && first_zero)
goto err_out; /* 0[0-9] is bad */
- (void)fy_atom_iter_getc(&iter);
+ (void)fy_atom_iter_utf8_get(&iter);
dec++;
len++;
}
fract = 0;
/* dot? */
- c = fy_atom_iter_peekc(&iter);
+ c = fy_atom_iter_utf8_peek(&iter);
if (c == '.') {
- (void)fy_atom_iter_getc(&iter);
+ (void)fy_atom_iter_utf8_get(&iter);
len++;
/* skip decimal part */
- while ((c = fy_atom_iter_peekc(&iter)) >= 0 && isdigit(c)) {
- (void)fy_atom_iter_getc(&iter);
+ while ((c = fy_atom_iter_utf8_peek(&iter)) >= 0 && fy_is_digit(c)) {
+ (void)fy_atom_iter_utf8_get(&iter);
len++;
fract++;
}
enot = 0;
/* scientific notation */
- c = fy_atom_iter_peekc(&iter);
+ c = fy_atom_iter_utf8_peek(&iter);
if (c == 'e' || c == 'E') {
- (void)fy_atom_iter_getc(&iter);
+ (void)fy_atom_iter_utf8_get(&iter);
len++;
/* skip sign if it's there */
- c = fy_atom_iter_peekc(&iter);
+ c = fy_atom_iter_utf8_peek(&iter);
if (c == '+' || c == '-') {
- (void)fy_atom_iter_getc(&iter);
+ (void)fy_atom_iter_utf8_get(&iter);
len++;
}
/* skip exponent part */
- while ((c = fy_atom_iter_peekc(&iter)) >= 0 && isdigit(c)) {
- (void)fy_atom_iter_getc(&iter);
+ while ((c = fy_atom_iter_utf8_peek(&iter)) >= 0 && fy_is_digit(c)) {
+ (void)fy_atom_iter_utf8_get(&iter);
len++;
enot++;
}
goto err_out;
}
- c = fy_atom_iter_peekc(&iter);
+ c = fy_atom_iter_utf8_peek(&iter);
fy_atom_iter_finish(&iter);
return NULL;
while (s > iter->is) {
- c = fy_utf8_get_right(iter->is, (int)(s - iter->is), &w);
+ c = fy_utf8_get_right(iter->is, (size_t)(s - iter->is), &w);
if (c <= 0 || fy_is_lb_m(c, iter->atom->lb_mode))
break;
s -= w;
/* track until the start of the content */
while (s < iter->as) {
- c = fy_utf8_get(s, (int)(iter->ae - s), &w);
+ c = fy_utf8_get(s, (size_t)(iter->ae - s), &w);
/* we should never hit that */
if (c <= 0)
return NULL;
/* track until the end of the content (or lb) */
while (s < iter->ae) {
- c = fy_utf8_get(s, (int)(iter->ae - s), &w);
+ c = fy_utf8_get(s, (size_t)(iter->ae - s), &w);
/* we should never hit that */
if (c <= 0)
return NULL;
/* if the stop was due to end of the atom */
if (s >= iter->ae) {
while (s < iter->ie) {
- c = fy_utf8_get(s, (int)(iter->ie - s), &w);
+ c = fy_utf8_get(s, (size_t)(iter->ie - s), &w);
/* just end of input */
if (c <= 0)
break;
bool valid_anchor : 1; /* atom is a valid anchor */
bool json_mode : 1; /* atom was read in json mode */
bool ends_with_eof : 1; /* atom ends at EOF of input */
+ bool is_merge_key: 1; /* atom is just << */
};
};
};
return handle->lb_mode;
}
+static inline bool fy_atom_is_merge_key(struct fy_atom *handle)
+{
+ return handle && handle->is_merge_key;
+}
+
static inline enum fy_flow_ws_mode fy_atom_flow_ws_mode(struct fy_atom *handle)
{
if (!handle)
return FYCR_ERROR;
}
+static void
+fy_composer_halt(struct fy_composer *fyc, struct fy_path *fypp, enum fy_composer_return rc)
+{
+ const struct fy_composer_ops *ops;
+ struct fy_eventp ev_none;
+ struct fy_path_component *fypc;
+
+ /* for normal stop, we don't teardown */
+ if (rc == FYCR_OK_STOP)
+ return;
+
+ ops = fyc->cfg.ops;
+ assert(ops);
+
+ memset(&ev_none, 0, sizeof(ev_none));
+ ev_none.e.type = FYET_NONE;
+
+ /* pump FYET_NONEs to clean the stack */
+
+ while ((fypc = fy_path_component_list_tail(&fypp->components)) != NULL) {
+
+ ops->process_event(fyc, fypp, &ev_none.e);
+
+ fy_path_component_list_del(&fypp->components, fypc);
+ fy_path_component_free(fypc);
+ }
+
+ /* and the final one to clean the root */
+ ops->process_event(fyc, fypp, &ev_none.e);
+}
+
enum fy_composer_return
fy_composer_process_event(struct fy_composer *fyc, struct fy_event *fye)
{
rc = fy_composer_process_event_private(fyc, fye, fypp);
+ if (rc == FYCR_ERROR || rc == FYCR_OK_STOP)
+ fy_composer_halt(fyc, fypp, rc);
+
return rc;
}
+++ /dev/null
-/*
- * fy-ctype.c - ctype utilities
- *
- * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
- *
- * SPDX-License-Identifier: MIT
- *
- */
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <libfyaml.h>
-
-#include "fy-ctype.h"
-
-const char *fy_uri_esc(const char *s, size_t len, uint8_t *code, int *code_len)
-{
- const char *e = s + len;
- int j, k, width;
- uint8_t octet;
- char c;
-
- width = 0;
- k = 0;
- do {
- /* check for enough space for %XX */
- if ((e - s) < 3)
- return NULL;
-
- /* if more than one run, expect '%' */
- if (s[0] != '%')
- return NULL;
-
- octet = 0;
- for (j = 0; j < 2; j++) {
- c = s[1 + j];
- octet <<= 4;
- if (c >= '0' && c <= '9')
- octet |= c - '0';
- else if (c >= 'a' && c <= 'f')
- octet |= 10 + c - 'a';
- else
- octet |= 10 + c - 'A';
- }
- if (!width) {
- width = fy_utf8_width_by_first_octet(octet);
- if (!width)
- return NULL;
- k = 0;
- }
- if (k >= *code_len)
- return NULL;
-
- code[k++] = octet;
-
- /* skip over the 3 character escape */
- s += 3;
-
- } while (--width > 0);
-
- *code_len = k;
-
- return s;
-}
-
+++ /dev/null
-/*
- * fy-ctype.h - ctype like macros header
- *
- * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
- *
- * SPDX-License-Identifier: MIT
- */
-#ifndef FY_CTYPE_H
-#define FY_CTYPE_H
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdint.h>
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-
-#include <libfyaml.h>
-
-#include "fy-utf8.h"
-
-enum fy_lb_mode {
- fylb_cr_nl, /* only \r, \n (json + >= yaml1.2 */
- fylb_cr_nl_N_L_P, /* NEL/LS/PS (yaml1.1) */
-};
-
-enum fy_flow_ws_mode {
- fyfws_space_tab, /* space + TAB (yaml) */
- fyfws_space, /* only space (json) */
-};
-
-static inline bool fy_is_first_alpha(int c)
-{
- return (c >= 'a' && c <= 'z') ||
- (c >= 'A' && c <= 'Z') ||
- c == '_';
-}
-
-static inline bool fy_is_alpha(int c)
-{
- return fy_is_first_alpha(c) || c == '-';
-}
-
-static inline bool fy_is_num(int c)
-{
- return c >= '0' && c <= '9';
-}
-
-static inline bool fy_is_first_alnum(int c)
-{
- return fy_is_first_alpha(c);
-}
-
-static inline bool fy_is_alnum(int c)
-{
- return fy_is_alpha(c) || fy_is_num(c);
-}
-
-static inline bool fy_is_space(int c)
-{
- return c == ' ';
-}
-
-static inline bool fy_is_tab(int c)
-{
- return c == '\t';
-}
-
-static inline bool fy_is_ws(int c)
-{
- return fy_is_space(c) || fy_is_tab(c);
-}
-
-static inline bool fy_is_hex(int c)
-{
- return (c >= '0' && c <= '9') ||
- (c >= 'a' && c <= 'f') ||
- (c >= 'A' && c <= 'F');
-}
-
-static inline bool fy_is_uri(int c)
-{
- return fy_is_alnum(c) || fy_utf8_strchr(";/?:@&=+$,.!~*\'()[]%", c);
-}
-
-static inline bool fy_is_lb_r_n(int c)
-{
- return c == '\r' || c == '\n';
-}
-
-static inline bool fy_is_lb_NEL(int c)
-{
- return c == 0x85;
-}
-
-static inline bool fy_is_lb_LS_PS(int c)
-{
- return c == 0x2028 || c == 0x2029;
-}
-
-static inline bool fy_is_unicode_lb(int c)
-{
- /* note that YAML1.1 supports NEL #x85, LS #x2028 and PS #x2029 as linebreaks */
- /* YAML1.2 and higher does not */
- return fy_is_lb_NEL(c) || fy_is_lb_LS_PS(c);
-}
-
-static inline bool fy_is_any_lb(int c)
-{
- return fy_is_lb_r_n(c) || fy_is_unicode_lb(c);
-}
-
-static inline bool fy_is_z(int c)
-{
- return c <= 0;
-}
-
-static inline bool fy_is_blank(int c)
-{
- return c == ' ' || c == '\t';
-}
-
-#define FY_UTF8_BOM 0xfeff
-
-static inline bool fy_is_print(int c)
-{
- return c == '\n' || c == '\r' ||
- (c >= 0x0020 && c <= 0x007e) ||
- (c >= 0x00a0 && c <= 0xd7ff) ||
- (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM);
-}
-
-static inline bool fy_is_printq(int c)
-{
- return c != '\t' && c != 0xa0 && !fy_is_any_lb(c) && fy_is_print(c);
-}
-
-static inline bool fy_is_nb_char(int c)
-{
- return (c >= 0x0020 && c <= 0x007e) ||
- (c >= 0x00a0 && c <= 0xd7ff) ||
- (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM);
-}
-
-static inline bool fy_is_ns_char(int c)
-{
- return fy_is_nb_char(c) && !fy_is_ws(c);
-}
-
-static inline bool fy_is_start_indicator(int c)
-{
- return !!fy_utf8_strchr(",[]{}#&*!|>'\"%%@`", c);
-}
-
-static inline bool fy_is_indicator_before_space(int c)
-{
- return !!fy_utf8_strchr("-:?`", c);
-}
-
-static inline bool fy_is_flow_indicator(int c)
-{
- return !!fy_utf8_strchr(",[]{}", c);
-}
-
-static inline bool fy_is_path_flow_scalar_start(int c)
-{
- return c == '\'' || c == '"';
-}
-
-static inline bool fy_is_path_flow_key_start(int c)
-{
- return c == '"' || c == '\'' || c == '{' || c == '[';
-}
-
-static inline bool fy_is_path_flow_key_end(int c)
-{
- return c == '"' || c == '\'' || c == '}' || c == ']';
-}
-
-static inline bool fy_is_unicode_control(int c)
-{
- return (c >= 0 && c <= 0x1f) || (c >= 0x80 && c <= 0x9f);
-}
-
-static inline bool fy_is_unicode_space(int c)
-{
- return c == 0x20 || c == 0xa0 ||
- (c >= 0x2000 && c <= 0x200a) ||
- c == 0x202f || c == 0x205f || c == 0x3000;
-}
-
-static inline bool fy_is_json_unescaped(int c)
-{
- return c >= 0x20 && c <= 0x110000 && c != '"' && c != '\\';
-}
-
-static inline bool fy_is_json_unescaped_range_only(int c)
-{
- return c >= 0x20 && c <= 0x110000;
-}
-
-static inline bool fy_is_lb_m(int c, enum fy_lb_mode lb_mode)
-{
- if (fy_is_lb_r_n(c))
- return true;
- return lb_mode == fylb_cr_nl_N_L_P && fy_is_unicode_lb(c);
-}
-
-static inline bool fy_is_generic_lb_m(int c, enum fy_lb_mode lb_mode)
-{
- if (fy_is_lb_r_n(c))
- return true;
- return lb_mode == fylb_cr_nl_N_L_P && fy_is_lb_NEL(c);
-}
-
-static inline bool fy_is_lbz_m(int c, enum fy_lb_mode lb_mode)
-{
- return fy_is_lb_m(c, lb_mode) || fy_is_z(c);
-}
-
-static inline bool fy_is_generic_lbz_m(int c, enum fy_lb_mode lb_mode)
-{
- return fy_is_generic_lb_m(c, lb_mode) || fy_is_z(c);
-}
-
-static inline bool fy_is_blankz_m(int c, enum fy_lb_mode lb_mode)
-{
- return fy_is_ws(c) || fy_is_lbz_m(c, lb_mode);
-}
-
-static inline bool fy_is_generic_blankz_m(int c, enum fy_lb_mode lb_mode)
-{
- return fy_is_ws(c) || fy_is_generic_lbz_m(c, lb_mode);
-}
-
-static inline bool fy_is_flow_ws_m(int c, enum fy_flow_ws_mode fws_mode)
-{
- return fy_is_space(c) || (fws_mode == fyfws_space_tab && fy_is_tab(c));
-}
-
-#define FY_CTYPE_AT_BUILDER(_kind) \
-static inline const void * \
-fy_find_ ## _kind (const void *s, size_t len) \
-{ \
- const void *e = s + len; \
- int c, w; \
- for (; s < e && (c = fy_utf8_get(s, e - s, &w)) >= 0; s += w) { \
- assert(w); \
- if (fy_is_ ## _kind (c)) \
- return s; \
- } \
- return NULL; \
-} \
-static inline const void * \
-fy_find_non_ ## _kind (const void *s, size_t len) \
-{ \
- const void *e = s + len; \
- int c, w; \
- for (; s < e && (c = fy_utf8_get(s, e - s, &w)) >= 0; s += w) { \
- assert(w); \
- if (!(fy_is_ ## _kind (c))) \
- return s; \
- assert(w); \
- } \
- return NULL; \
-} \
-struct useless_struct_for_semicolon
-
-FY_CTYPE_AT_BUILDER(first_alpha);
-FY_CTYPE_AT_BUILDER(alpha);
-FY_CTYPE_AT_BUILDER(num);
-FY_CTYPE_AT_BUILDER(first_alnum);
-FY_CTYPE_AT_BUILDER(alnum);
-FY_CTYPE_AT_BUILDER(space);
-FY_CTYPE_AT_BUILDER(tab);
-FY_CTYPE_AT_BUILDER(ws);
-FY_CTYPE_AT_BUILDER(hex);
-FY_CTYPE_AT_BUILDER(uri);
-FY_CTYPE_AT_BUILDER(z);
-FY_CTYPE_AT_BUILDER(any_lb);
-FY_CTYPE_AT_BUILDER(blank);
-FY_CTYPE_AT_BUILDER(print);
-FY_CTYPE_AT_BUILDER(printq);
-FY_CTYPE_AT_BUILDER(nb_char);
-FY_CTYPE_AT_BUILDER(ns_char);
-FY_CTYPE_AT_BUILDER(start_indicator);
-FY_CTYPE_AT_BUILDER(indicator_before_space);
-FY_CTYPE_AT_BUILDER(flow_indicator);
-FY_CTYPE_AT_BUILDER(path_flow_key_start);
-FY_CTYPE_AT_BUILDER(path_flow_key_end);
-FY_CTYPE_AT_BUILDER(unicode_control);
-FY_CTYPE_AT_BUILDER(unicode_space);
-FY_CTYPE_AT_BUILDER(json_unescaped);
-
-/*
- * Very special linebreak/ws methods
- * Things get interesting due to \r\n and
- * unicode linebreaks/spaces
- */
-
-/* skip for a _single_ linebreak */
-static inline const void *fy_skip_lb(const void *ptr, int left)
-{
- int c, width;
-
- /* get the utf8 character at this point */
- c = fy_utf8_get(ptr, left, &width);
- if (c < 0 || !fy_is_any_lb(c))
- return NULL;
-
- /* MS-DOS: check if next character is '\n' */
- if (c == '\r' && left > width && *(char *)ptr == '\n')
- width++;
-
- return ptr + width;
-}
-
-/* given a pointer to a chunk of memory, return pointer to first
- * ws character after the last non-ws character, or the end
- * of the chunk
- */
-static inline const void *fy_last_non_ws(const void *ptr, int left)
-{
- const char *s, *e;
- int c;
-
- s = ptr;
- e = s + left;
- while (e > s) {
- c = e[-1];
- if (c != ' ' && c != '\t')
- return e;
- e--;
-
- }
- return NULL;
-}
-
-const char *fy_uri_esc(const char *s, size_t len, uint8_t *code, int *code_len);
-
-#endif
struct fy_raw_line l_tmp;
struct fy_atom_raw_line_iter iter;
int content_start_col, content_end_col, content_width;
- int pass, cols, min_col, max_col, total_lines, max_line_count, max_line_col8, max_width;
+ int pass, cols, min_col, max_col, max_line_count, max_line_col8, max_width;
int start_col, end_col;
const char *color_start, *color_end, *white;
bool first_line, last_line;
max_col = -1;
max_line_count = -1;
max_line_col8 = -1;
- total_lines = 0;
line_shift = -1;
for (pass = 0; pass < 2; pass++) {
rbe = rowbuf + rowbufsz;
/* if the maximum column number is less than the terminal
- * width everything fits, and we're fine */
+ * width everything fits, and we're fine */
if (max_line_col8 < cols) {
line_shift = 0;
} else {
if (pass == 0) {
- total_lines++;
-
if (min_col < 0 || content_start_col < min_col)
min_col = content_start_col;
if (max_col < 0 || content_end_col > max_col)
va_end(ap);
}
+void fy_diag_token_vreport(struct fy_diag *diag, struct fy_token *fyt,
+ enum fy_error_type type, const char *fmt, va_list ap)
+{
+ struct fy_diag_report_ctx drc;
+ bool save_on_error;
+
+ if (!fyt || !diag)
+ return;
+
+ save_on_error = diag->on_error;
+ diag->on_error = false;
+
+ memset(&drc, 0, sizeof(drc));
+ drc.type = type;
+ drc.module = FYEM_UNKNOWN;
+ drc.fyt = fyt;
+ fy_diag_vreport(diag, &drc, fmt, ap);
+
+ diag->on_error = save_on_error;
+}
+
+void fy_diag_token_report(struct fy_diag *diag, struct fy_token *fyt,
+ enum fy_error_type type, const char *fmt, ...)
+{
+ va_list ap;
+
+ va_start(ap, fmt);
+ fy_diag_token_vreport(diag, fyt, type, fmt, ap);
+ va_end(ap);
+}
+
+void fy_diag_token_override_vreport(struct fy_diag *diag, struct fy_token *fyt,
+ enum fy_error_type type, const char *file,
+ int line, int column,
+ const char *fmt, va_list ap)
+{
+ struct fy_diag_report_ctx drc;
+ bool save_on_error;
+
+ if (!fyt || !diag)
+ return;
+
+ save_on_error = diag->on_error;
+ diag->on_error = false;
+
+ memset(&drc, 0, sizeof(drc));
+ drc.type = type;
+ drc.module = FYEM_UNKNOWN;
+ drc.fyt = fyt;
+ drc.has_override = true;
+ drc.override_file = file;
+ drc.override_line = line;
+ drc.override_column = column;
+ fy_diag_vreport(diag, &drc, fmt, ap);
+
+ diag->on_error = save_on_error;
+}
+
+void fy_diag_token_override_report(struct fy_diag *diag, struct fy_token *fyt,
+ enum fy_error_type type, const char *file,
+ int line, int column, const char *fmt, ...)
+{
+ va_list ap;
+
+ va_start(ap, fmt);
+ fy_diag_token_override_vreport(diag, fyt, type, file, line, column, fmt, ap);
+ va_end(ap);
+}
+
void fy_diag_node_vreport(struct fy_diag *diag, struct fy_node *fyn,
enum fy_error_type type, const char *fmt, va_list ap)
{
fy_node_override_vreport(fyn, type, file, line, column, fmt, ap);
va_end(ap);
}
+
+void fy_parser_vlog(struct fy_parser *fyp, enum fy_error_type type,
+ const char *fmt, va_list ap)
+{
+ struct fy_diag_ctx ctx = {
+ .level = type,
+ .module = FYEM_UNKNOWN,
+ .source_func = "",
+ .source_file = "",
+ .source_line = 0,
+ .file = NULL,
+ .line = 0,
+ .column = 0,
+ };
+
+ if (!fyp || !fyp->diag)
+ return;
+
+ (void)fy_vdiag(fyp->diag, &ctx, fmt, ap);
+}
+
+void fy_parser_log(struct fy_parser *fyp, enum fy_error_type type,
+ const char *fmt, ...)
+{
+ va_list ap;
+
+ va_start(ap, fmt);
+ fy_parser_vlog(fyp, type, fmt, ap);
+ va_end(ap);
+}
+
+void fy_parser_vreport(struct fy_parser *fyp, enum fy_error_type type,
+ struct fy_token *fyt, const char *fmt, va_list ap)
+{
+ struct fy_diag_report_ctx fydrc_local, *fydrc = &fydrc_local;
+
+ if (!fyp || !fyp->diag || !fyt)
+ return;
+
+ memset(fydrc, 0, sizeof(*fydrc));
+ fydrc->type = type;
+ fydrc->module = FYEM_UNKNOWN;
+ fydrc->fyt = fy_token_ref(fyt);
+
+ fy_parser_diag_vreport(fyp, fydrc, fmt, ap);
+}
+
+void fy_parser_report(struct fy_parser *fyp, enum fy_error_type type,
+ struct fy_token *fyt, const char *fmt, ...)
+{
+ va_list ap;
+
+ va_start(ap, fmt);
+ fy_parser_vreport(fyp, type, fyt, fmt, ap);
+ va_end(ap);
+}
#ifdef FY_DEVMODE
#define __FY_DEBUG_UNUSED__ /* nothing */
#else
+#if defined(__GNUC__) && __GNUC__ >= 4
#define __FY_DEBUG_UNUSED__ __attribute__((__unused__))
+#else
+#define __FY_DEBUG_UNUSED__ /* nothing */
+#endif
#endif
/* parser diagnostics */
-
struct fy_parser;
void fy_diag_cfg_from_parser_flags(struct fy_diag_cfg *cfg, enum fy_parse_cfg_flags pflags);
#include <ctype.h>
#include <errno.h>
#include <unistd.h>
+#include <limits.h>
#include <libfyaml.h>
void fy_parse_document_destroy(struct fy_parser *fyp, struct fy_document *fyd)
{
struct fy_node *fyn;
- struct fy_anchor *fya;
- struct fy_anchor *fyan;
- struct fy_accel_entry *xle;
if (!fyd)
return;
fyd->root = NULL;
fy_node_detach_and_free(fyn);
- /* remove all anchors */
- for (fya = fy_anchor_list_head(&fyd->anchors); fya; fya = fyan) {
- fyan = fy_anchor_next(&fyd->anchors, fya);
- fy_anchor_list_del(&fyd->anchors, fya);
-
- if (fy_document_is_accelerated(fyd)) {
- xle = fy_accel_entry_lookup_key_value(fyd->axl, fya->anchor, fya);
- fy_accel_entry_remove(fyd->axl, xle);
-
- xle = fy_accel_entry_lookup_key_value(fyd->naxl, fya->fyn, fya);
- fy_accel_entry_remove(fyd->naxl, xle);
- }
-
- fy_anchor_destroy(fya);
- }
-
- if (fy_document_is_accelerated(fyd)) {
- fy_accel_cleanup(fyd->axl);
- free(fyd->axl);
-
- fy_accel_cleanup(fyd->naxl);
- free(fyd->naxl);
- }
+ fy_document_purge_anchors(fyd);
fy_document_state_unref(fyd->fyds);
}
}
-typedef void (*fy_node_applyf)(struct fy_node *fyn);
+void fy_document_purge_anchors(struct fy_document *fyd)
+{
+ struct fy_anchor *fya;
+ struct fy_anchor *fyan;
+ struct fy_accel_entry *xle;
-void fy_node_apply(struct fy_node *fyn, fy_node_applyf func)
+ /* remove all anchors */
+ for (fya = fy_anchor_list_head(&fyd->anchors); fya; fya = fyan) {
+ fyan = fy_anchor_next(&fyd->anchors, fya);
+ fy_anchor_list_del(&fyd->anchors, fya);
+
+ if (fy_document_is_accelerated(fyd)) {
+ xle = fy_accel_entry_lookup_key_value(fyd->axl, fya->anchor, fya);
+ fy_accel_entry_remove(fyd->axl, xle);
+
+ xle = fy_accel_entry_lookup_key_value(fyd->naxl, fya->fyn, fya);
+ fy_accel_entry_remove(fyd->naxl, xle);
+ }
+
+ fy_anchor_destroy(fya);
+ }
+
+ if (fy_document_is_accelerated(fyd)) {
+ fy_accel_cleanup(fyd->axl);
+ free(fyd->axl);
+
+ fy_accel_cleanup(fyd->naxl);
+ free(fyd->naxl);
+ }
+}
+
+typedef void (*fy_node_applyf)(struct fy_node *fyn, void *user);
+
+void fy_node_apply(struct fy_node *fyn, fy_node_applyf func, void *user)
{
struct fy_node *fyni;
struct fy_node_pair *fynp;
if (!fyn || !func)
return;
- (*func)(fyn);
+ (*func)(fyn, user);
switch (fyn->type) {
case FYNT_SCALAR:
case FYNT_SEQUENCE:
for (fyni = fy_node_list_head(&fyn->sequence); fyni;
fyni = fy_node_next(&fyn->sequence, fyni))
- fy_node_apply(fyni, func);
+ fy_node_apply(fyni, func, user);
break;
case FYNT_MAPPING:
for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp;
fynp = fy_node_pair_next(&fyn->mapping, fynp)) {
- fy_node_apply(fynp->key, func);
- fy_node_apply(fynp->value, func);
+ fy_node_apply(fynp->key, func, user);
+ fy_node_apply(fynp->value, func, user);
}
break;
}
}
-static void clear_system_marks(struct fy_node *fyn)
+static void clear_system_marks(struct fy_node *fyn, void *user)
{
fyn->marks &= ~FYNWF_SYSTEM_MARKS;
}
/* clear all the system markers */
void fy_node_clear_system_marks(struct fy_node *fyn)
{
- fy_node_apply(fyn, clear_system_marks);
+ fy_node_apply(fyn, clear_system_marks, NULL);
+}
+
+static void count_aliases(struct fy_node *fyn, void *user)
+{
+ int *countp = user;
+
+ if (fyn->style == FYNS_ALIAS)
+ (*countp)++;
+}
+
+int fy_node_count_aliases(struct fy_node *fyn)
+{
+ int count;
+
+ count = 0;
+
+ fy_node_apply(fyn, count_aliases, &count);
+
+ return count;
}
int fy_document_resolve(struct fy_document *fyd)
{
- int rc;
+ int rc, num_aliases, num_aliases_prev;
bool ret;
if (!fyd)
return 0;
- fy_node_clear_system_marks(fyd->root);
+ num_aliases_prev = INT_MAX;
+ do {
+ fy_node_clear_system_marks(fyd->root);
- /* for resolution to work, no reference loops should exist */
- ret = fy_check_ref_loop(fyd, fyd->root,
- FYNWF_MAXDEPTH_DEFAULT | FYNWF_FOLLOW, NULL);
+ /* for resolution to work, no reference loops should exist */
+ ret = fy_check_ref_loop(fyd, fyd->root,
+ FYNWF_MAXDEPTH_DEFAULT | FYNWF_FOLLOW, NULL);
- fy_node_clear_system_marks(fyd->root);
+ fy_node_clear_system_marks(fyd->root);
- if (ret)
- goto err_out;
+ if (ret)
+ goto err_out;
+ /* now resolve any anchor nodes */
+ rc = fy_resolve_anchor_node(fyd, fyd->root);
+ if (rc)
+ goto err_out_rc;
- /* now resolve any anchor nodes */
- rc = fy_resolve_anchor_node(fyd, fyd->root);
- if (rc)
- goto err_out_rc;
+ /* redo parent resolution */
+ fy_resolve_parent_node(fyd, fyd->root, NULL);
- /* redo parent resolution */
- fy_resolve_parent_node(fyd, fyd->root, NULL);
+ /* count the remaining aliases */
+ num_aliases = fy_node_count_aliases(fyd->root);
+ if (num_aliases == num_aliases_prev)
+ goto err_out;
+
+ } while (num_aliases > 0);
+
+ /* remove all anchors after resolution */
+ fy_document_purge_anchors(fyd);
return 0;
return fyn ? fyn->style : FYNS_PLAIN;
}
+struct fy_token *fy_node_get_start_token(struct fy_node *fyn)
+{
+ if (!fyn)
+ return NULL;
+
+ switch (fyn->type) {
+ case FYNT_MAPPING:
+ return fyn->mapping_start;
+ case FYNT_SEQUENCE:
+ return fyn->sequence_start;
+ case FYNT_SCALAR:
+ return fyn->scalar;
+ default:
+ /* should not happen, but play it safe */
+ break;
+ }
+ return NULL;
+}
+
+struct fy_token *fy_node_get_end_token(struct fy_node *fyn)
+{
+ if (!fyn)
+ return NULL;
+
+ switch (fyn->type) {
+ case FYNT_MAPPING:
+ return fyn->mapping_end;
+ case FYNT_SEQUENCE:
+ return fyn->sequence_end;
+ case FYNT_SCALAR:
+ return fyn->scalar;
+ default:
+ /* should not happen, but play it safe */
+ break;
+ }
+ return NULL;
+}
+
bool fy_node_is_null(struct fy_node *fyn)
{
if (!fyn)
return NULL;
}
+struct fy_document *
+fy_block_document_build_from_string(const struct fy_parse_cfg *cfg,
+ const char *str, size_t len, size_t *consumed)
+{
+ struct flow_reader_container frc;
+ struct fy_reader *fyr = NULL;
+ struct fy_parser fyp_data, *fyp = &fyp_data;
+ struct fy_parse_cfg cfg_data;
+ struct fy_input *fyi;
+ struct fy_document *fyd;
+ struct fy_mark mark;
+ int rc;
+
+ if (!str)
+ return NULL;
+
+ if (consumed)
+ *consumed = 0;
+
+ if (!cfg) {
+ memset(&cfg_data, 0, sizeof(cfg_data));
+ cfg_data.flags = FYPCF_DEFAULT_PARSE;
+ cfg = &cfg_data;
+ }
+
+ memset(&frc, 0, sizeof(frc));
+ fyr = &frc.reader;
+ frc.cfg = cfg;
+
+ fy_reader_setup(fyr, &reader_ops);
+
+ rc = fy_parse_setup(fyp, cfg);
+ if (rc)
+ goto err_no_parse;
+
+ fyi = fy_input_from_data(str, len, NULL, false);
+ if (!fyi)
+ goto err_no_input;
+
+ rc = fy_reader_input_open(fyr, fyi, NULL);
+ if (rc)
+ goto err_no_input_open;
+
+ fy_parser_set_reader(fyp, fyr);
+ fy_parser_set_block_only_mode(fyp, true);
+
+ fyd = fy_parse_load_document(fyp);
+
+ fy_parse_cleanup(fyp);
+
+ if (fyd && consumed) {
+ fy_reader_get_mark(fyr, &mark);
+ *consumed = mark.input_pos;
+ }
+
+ fy_reader_cleanup(fyr);
+ fy_input_unref(fyi);
+
+ return fyd;
+
+err_no_input_open:
+ fy_input_unref(fyi);
+err_no_input:
+ fy_parse_cleanup(fyp);
+err_no_parse:
+ fy_reader_cleanup(fyr);
+ return NULL;
+}
+
int fy_node_vscanf(struct fy_node *fyn, const char *fmt, va_list ap)
{
size_t len;
FY_TYPE_DECL_LIST(document);
struct fy_document *fy_parse_document_create(struct fy_parser *fyp, struct fy_eventp *fyep);
+void fy_document_purge_anchors(struct fy_document *fyd);
struct fy_node_mapping_sort_ctx {
fy_node_mapping_sort_fn key_cmp;
/* get ownership of the document */
return fy_document_builder_take_document(fydb);
}
+
+struct fy_document *
+fy_document_builder_event_document(struct fy_document_builder *fydb, struct fy_eventp_list *evpl)
+{
+ struct fy_eventp *fyep = NULL;
+ int rc;
+
+ if (!fydb || !evpl)
+ return NULL;
+
+ for (fyep = fy_eventp_list_head(evpl); fyep; fyep = fy_eventp_next(evpl, fyep)) {
+
+ if (fy_document_builder_is_document_complete(fydb))
+ break;
+
+ rc = fy_document_builder_process_event(fydb, fyep);
+ if (rc < 0)
+ return NULL;
+
+ }
+
+ /* get ownership of the document */
+ return fy_document_builder_take_document(fydb);
+}
fy_document_builder_load_document(struct fy_document_builder *fydb,
struct fy_parser *fyp);
+struct fy_document *
+fy_document_builder_event_document(struct fy_document_builder *fydb, struct fy_eventp_list *evpl);
+
struct fy_document *
fy_parse_load_document_with_builder(struct fy_parser *fyp);
return tag;
}
+
+struct fy_tag **
+fy_document_state_tag_directives(struct fy_document_state *fyds)
+{
+ struct fy_tag *tags, **tagsp;
+ const struct fy_tag *fytag;
+ size_t size, len;
+ void *iter;
+ char *s;
+ int i;
+
+ if (!fyds)
+ return NULL;
+
+ /* first find the extents of the area we're going to need */
+ i = 0;
+ size = 0;
+ iter = NULL;
+ while ((fytag = fy_document_state_tag_directive_iterate(fyds, &iter)) != NULL) {
+ size += strlen(fytag->handle) + 1 + strlen(fytag->prefix) + 1;
+ i++;
+ }
+ size += sizeof(struct fy_tag) * i;
+ size += sizeof(struct fy_tag *) * (i + 1);
+
+ tagsp = malloc(size);
+ if (!tagsp)
+ return NULL;
+ memset(tagsp, 0, size);
+ tags = (void *)(tagsp + i + 1);
+ s = (void *)(tags + i);
+
+ i = 0;
+ iter = NULL;
+ while ((fytag = fy_document_state_tag_directive_iterate(fyds, &iter)) != NULL) {
+ tags[i].handle = s;
+ len = strlen(fytag->handle);
+ memcpy(s, fytag->handle, len + 1);
+ s += len + 1;
+
+ tags[i].prefix = s;
+ len = strlen(fytag->prefix);
+ memcpy(s, fytag->prefix, len + 1);
+ s += len + 1;
+
+ tagsp[i] = &tags[i];
+
+ i++;
+ }
+ tagsp[i] = NULL; /* NULL terminated */
+
+ return tagsp;
+}
#include "fy-parse.h"
#include "fy-emit.h"
+static inline struct fy_token_list *token_recycle_list(struct fy_emitter *emit, struct fy_parser *fyp)
+{
+ if (fyp && fyp->recycled_token_list)
+ return fyp->recycled_token_list;
+ if (emit && emit->recycled_token_list)
+ return emit->recycled_token_list;
+ return NULL;
+}
+
+static inline void fy_emit_token_unref(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_token *fyt)
+{
+ fy_token_unref_rl(token_recycle_list(emit, fyp), fyt);
+}
+
/* fwd decl */
void fy_emit_write(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int len);
void fy_emit_printf(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *fmt, ...)
{
enum fy_emitter_cfg_flags flags = emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK);
- return flags == FYECF_MODE_FLOW || flags == FYECF_MODE_FLOW_ONELINE;
+ return flags == FYECF_MODE_FLOW || flags == FYECF_MODE_FLOW_ONELINE || fy_emit_is_json_mode(emit);
}
static inline bool fy_emit_is_block_mode(const struct fy_emitter *emit)
int flags, int indent, enum fy_node_style style,
struct fy_token *fyt_tag)
{
- const char *value = NULL;
- size_t len = 0;
bool json, flow, is_null_scalar, is_json_plain;
struct fy_atom *atom;
int aflags = -1;
if (flow && (style == FYNS_ANY || style == FYNS_LITERAL || style == FYNS_FOLDED)) {
- if (fyt && !value)
- value = fy_token_get_text(fyt, &len);
-
/* if there's a linebreak, use double quoted style */
- if (fy_find_any_lb(value, len)) {
+ if (aflags & FYTTAF_HAS_ANY_LB) {
style = FYNS_DOUBLE_QUOTED;
goto out;
}
- /* check if there's a non printable */
- if (!fy_find_non_print(value, len)) {
+ if (!(aflags & FYTTAF_HAS_NON_PRINT)) {
style = FYNS_SINGLE_QUOTED;
goto out;
}
}
out:
- if (style == FYNS_ANY) {
- if (fyt)
- value = fy_token_get_text(fyt, &len);
-
- style = (aflags & FYTTAF_CAN_BE_PLAIN) ?
- FYNS_PLAIN : FYNS_DOUBLE_QUOTED;
- }
+ if (style == FYNS_ANY)
+ style = (aflags & FYTTAF_CAN_BE_PLAIN) ? FYNS_PLAIN : FYNS_DOUBLE_QUOTED;
if (style == FYNS_PLAIN) {
/* plains in flow mode not being able to be plains
return 0;
}
+void fy_emit_save_ctx_cleanup(struct fy_emitter *emit, struct fy_emit_save_ctx *sc)
+{
+ sc = &emit->s_sc;
+ fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_key);
+ fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_value);
+ memset(sc, 0, sizeof(*sc));
+}
+
void fy_emit_cleanup(struct fy_emitter *emit)
{
struct fy_eventp *fyep;
if (emit->finalizer)
emit->finalizer(emit);
+ fy_emit_save_ctx_cleanup(emit, &emit->s_sc);
+ while (emit->sc_stack_top > 0)
+ fy_emit_save_ctx_cleanup(emit, &emit->sc_stack[--emit->sc_stack_top]);
+
while ((fyt = fy_token_list_pop(&emit->recycled_token)) != NULL)
fy_token_free(fyt);
return buf;
}
+/* we can consume events only if the lookaheads
+ * are satisfied. The lookaheads depends on the
+ * styling mode. Some styling modes do not require
+ * a lookahead at all but we do it anyway.
+ */
static bool fy_emit_ready(struct fy_emitter *emit)
{
struct fy_eventp *fyep;
- int need, count, level;
-
- /* no events in the list, not ready */
- fyep = fy_eventp_list_head(&emit->queued_events);
- if (!fyep)
- return false;
+ int need, count;
- /* some events need more than one */
- switch (fyep->e.type) {
- case FYET_DOCUMENT_START:
- need = 1;
- break;
- case FYET_SEQUENCE_START:
- need = 2;
- break;
- case FYET_MAPPING_START:
- need = 3;
- break;
- default:
- need = 0;
- break;
- }
-
- /* if we don't need any more, that's enough */
- if (!need)
- return true;
-
- level = 0;
count = 0;
- for (; fyep; fyep = fy_eventp_next(&emit->queued_events, fyep)) {
- count++;
-
- if (count > need)
- return true;
+ need = -1;
+ for (fyep = fy_eventp_list_head(&emit->queued_events); fyep;
+ fyep = fy_eventp_next(&emit->queued_events, fyep)) {
+ count++;
switch (fyep->e.type) {
- case FYET_STREAM_START:
case FYET_DOCUMENT_START:
+ need++;
+ break;
+
case FYET_SEQUENCE_START:
- case FYET_MAPPING_START:
- level++;
+ if (need < 0)
+ need = 2;
break;
- case FYET_STREAM_END:
- case FYET_DOCUMENT_END:
- case FYET_SEQUENCE_END:
- case FYET_MAPPING_END:
- level--;
+
+ case FYET_MAPPING_START:
+ if (need < 0)
+ need = 2;
break;
+
default:
+ need = 0;
break;
}
- if (!level)
+ if (count >= need)
return true;
+
}
return false;
struct fy_eventp *
fy_emit_peek_next_event(struct fy_emitter *emit)
{
- if (!fy_emit_ready(emit))
- return NULL;
-
return fy_eventp_list_head(&emit->queued_events);
}
bool fy_emit_streaming_sequence_empty(struct fy_emitter *emit)
{
- enum fy_emitter_cfg_flags flags = emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK);
struct fy_eventp *fyepn;
- struct fy_event *fyen;
-
- if (flags == FYECF_MODE_BLOCK || flags == FYECF_MODE_DEJSON)
- return false;
fyepn = fy_emit_peek_next_event(emit);
- fyen = fyepn ? &fyepn->e : NULL;
- return !fyen || fyen->type == FYET_SEQUENCE_END;
+ /* should never happen, check on debugging mode */
+ assert(fyepn);
+ if (!fyepn)
+ return false;
+
+ return fyepn->e.type == FYET_SEQUENCE_END;
}
bool fy_emit_streaming_mapping_empty(struct fy_emitter *emit)
{
struct fy_eventp *fyepn;
- struct fy_event *fyen;
fyepn = fy_emit_peek_next_event(emit);
- fyen = fyepn ? &fyepn->e : NULL;
- return !fyen || fyen->type == FYET_MAPPING_END;
+ /* should never happen, check on debugging mode */
+ assert(fyepn);
+ if (!fyepn)
+ return false;
+
+ return fyepn->e.type == FYET_MAPPING_END;
}
static void fy_emit_goto_state(struct fy_emitter *emit, enum fy_emitter_state state)
return 0;
}
-static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_eventp *fyep, int flags)
+static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, int flags)
{
struct fy_event *fye = &fyep->e;
struct fy_emit_save_ctx *sc = &emit->s_sc;
s_flags = emit->s_flags;
s_indent = emit->s_indent;
- xstyle = fye->sequence_start.sequence_start->type == FYTT_BLOCK_SEQUENCE_START ?
- FYNS_BLOCK : FYNS_FLOW;
+ if (fye->sequence_start.sequence_start)
+ xstyle = fye->sequence_start.sequence_start->type == FYTT_BLOCK_SEQUENCE_START ?
+ FYNS_BLOCK : FYNS_FLOW;
+ else
+ xstyle = FYNS_ANY;
fy_emit_common_node_preamble(emit, fye->sequence_start.anchor, fye->sequence_start.tag, emit->s_flags, emit->s_indent);
sc->old_indent = sc->indent;
sc->s_flags = s_flags;
sc->s_indent = s_indent;
- sc->s_flags = s_flags;
- sc->s_indent = s_indent;
fy_emit_sequence_prolog(emit, sc);
sc->flags &= ~DDNF_MAP;
s_flags = emit->s_flags;
s_indent = emit->s_indent;
- xstyle = fye->mapping_start.mapping_start->type == FYTT_BLOCK_MAPPING_START ?
- FYNS_BLOCK : FYNS_FLOW;
+ if (fye->mapping_start.mapping_start)
+ xstyle = fye->mapping_start.mapping_start->type == FYTT_BLOCK_MAPPING_START ?
+ FYNS_BLOCK : FYNS_FLOW;
+ else
+ xstyle = FYNS_ANY;
fy_emit_common_node_preamble(emit, fye->mapping_start.anchor, fye->mapping_start.tag, emit->s_flags, emit->s_indent);
sc->old_indent = sc->indent;
sc->s_flags = s_flags;
sc->s_indent = s_indent;
- sc->s_flags = s_flags;
- sc->s_indent = s_indent;
fy_emit_mapping_prolog(emit, sc);
sc->flags &= ~DDNF_SEQ;
return 0;
}
-static int fy_emit_handle_stream_start(struct fy_emitter *emit, struct fy_eventp *fyep)
+static int fy_emit_handle_stream_start(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep)
{
struct fy_event *fye = &fyep->e;
return 0;
}
-static int fy_emit_handle_document_start(struct fy_emitter *emit, struct fy_eventp *fyep, bool first)
+static int fy_emit_handle_document_start(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool first)
{
struct fy_event *fye = &fyep->e;
struct fy_document_state *fyds;
return 0;
}
-static int fy_emit_handle_document_end(struct fy_emitter *emit, struct fy_eventp *fyep)
+static int fy_emit_handle_document_end(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep)
{
struct fy_document_state *fyds;
struct fy_event *fye = &fyep->e;
return 0;
}
-static int fy_emit_handle_document_content(struct fy_emitter *emit, struct fy_eventp *fyep)
+static int fy_emit_handle_document_content(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep)
{
struct fy_event *fye = &fyep->e;
int ret;
/* empty document? */
if (fye->type == FYET_DOCUMENT_END)
- return fy_emit_handle_document_end(emit, fyep);
+ return fy_emit_handle_document_end(emit, fyp, fyep);
ret = fy_emit_push_state(emit, FYES_DOCUMENT_END);
if (ret)
return ret;
- return fy_emit_streaming_node(emit, fyep, DDNF_ROOT);
+ return fy_emit_streaming_node(emit, fyp, fyep, DDNF_ROOT);
}
-static int fy_emit_handle_sequence_item(struct fy_emitter *emit, struct fy_eventp *fyep, bool first)
+static int fy_emit_handle_sequence_item(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool first)
{
struct fy_event *fye = &fyep->e;
struct fy_emit_save_ctx *sc = &emit->s_sc;
struct fy_token *fyt_item = NULL;
int ret;
- fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_value);
+ fy_emit_token_unref(emit, fyp, sc->fyt_last_value);
sc->fyt_last_value = NULL;
switch (fye->type) {
/* emit epilog */
fy_emit_sequence_epilog(emit, sc);
- /* pop state */
- ret = fy_emit_pop_sc(emit, sc);
- /* pop state */
- fy_emit_goto_state(emit, fy_emit_pop_state(emit));
/* restore indent and flags */
emit->s_indent = sc->s_indent;
emit->s_flags = sc->s_flags;
+
+ /* pop state */
+ ret = fy_emit_pop_sc(emit, sc);
+ assert(!ret);
+
+ /* pop state */
+ fy_emit_goto_state(emit, fy_emit_pop_state(emit));
return ret;
case FYET_ALIAS:
fy_emit_sequence_item_prolog(emit, sc, fyt_item);
- ret = fy_emit_streaming_node(emit, fyep, sc->flags);
+ ret = fy_emit_streaming_node(emit, fyp, fyep, sc->flags);
switch (fye->type) {
case FYET_ALIAS:
return ret;
}
-static int fy_emit_handle_mapping_key(struct fy_emitter *emit, struct fy_eventp *fyep, bool first)
+static int fy_emit_handle_mapping_key(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool first)
{
struct fy_event *fye = &fyep->e;
struct fy_emit_save_ctx *sc = &emit->s_sc;
int ret, aflags;
bool simple_key;
- fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_key);
+ fy_emit_token_unref(emit, fyp, sc->fyt_last_key);
+ fy_emit_token_unref(emit, fyp, sc->fyt_last_value);
sc->fyt_last_key = NULL;
- fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_value);
sc->fyt_last_value = NULL;
simple_key = false;
/* emit epilog */
fy_emit_mapping_epilog(emit, sc);
- /* pop state */
- ret = fy_emit_pop_sc(emit, sc);
- /* pop state */
- fy_emit_goto_state(emit, fy_emit_pop_state(emit));
/* restore indent and flags */
emit->s_indent = sc->s_indent;
emit->s_flags = sc->s_flags;
+
+ /* pop state */
+ ret = fy_emit_pop_sc(emit, sc);
+ assert(!ret);
+
+ /* pop state */
+ fy_emit_goto_state(emit, fy_emit_pop_state(emit));
return ret;
case FYET_ALIAS:
fy_emit_mapping_key_prolog(emit, sc, fyt_key, simple_key);
- ret = fy_emit_streaming_node(emit, fyep, sc->flags);
+ ret = fy_emit_streaming_node(emit, fyp, fyep, sc->flags);
switch (fye->type) {
case FYET_ALIAS:
return ret;
}
-static int fy_emit_handle_mapping_value(struct fy_emitter *emit, struct fy_eventp *fyep, bool simple)
+static int fy_emit_handle_mapping_value(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool simple)
{
struct fy_event *fye = &fyep->e;
struct fy_emit_save_ctx *sc = &emit->s_sc;
fy_emit_mapping_value_prolog(emit, sc, fyt_value);
- ret = fy_emit_streaming_node(emit, fyep, sc->flags);
+ ret = fy_emit_streaming_node(emit, fyp, fyep, sc->flags);
switch (fye->type) {
case FYET_ALIAS:
switch (emit->state) {
case FYES_STREAM_START:
- ret = fy_emit_handle_stream_start(emit, fyep);
+ ret = fy_emit_handle_stream_start(emit, fyp, fyep);
break;
case FYES_FIRST_DOCUMENT_START:
case FYES_DOCUMENT_START:
- ret = fy_emit_handle_document_start(emit, fyep,
+ ret = fy_emit_handle_document_start(emit, fyp, fyep,
emit->state == FYES_FIRST_DOCUMENT_START);
break;
case FYES_DOCUMENT_CONTENT:
- ret = fy_emit_handle_document_content(emit, fyep);
+ ret = fy_emit_handle_document_content(emit, fyp, fyep);
break;
case FYES_DOCUMENT_END:
- ret = fy_emit_handle_document_end(emit, fyep);
+ ret = fy_emit_handle_document_end(emit, fyp, fyep);
break;
case FYES_SEQUENCE_FIRST_ITEM:
case FYES_SEQUENCE_ITEM:
- ret = fy_emit_handle_sequence_item(emit, fyep,
+ ret = fy_emit_handle_sequence_item(emit, fyp, fyep,
emit->state == FYES_SEQUENCE_FIRST_ITEM);
break;
case FYES_MAPPING_FIRST_KEY:
case FYES_MAPPING_KEY:
- ret = fy_emit_handle_mapping_key(emit, fyep,
+ ret = fy_emit_handle_mapping_key(emit, fyp, fyep,
emit->state == FYES_MAPPING_FIRST_KEY);
break;
case FYES_MAPPING_SIMPLE_VALUE:
case FYES_MAPPING_VALUE:
- ret = fy_emit_handle_mapping_value(emit, fyep,
+ ret = fy_emit_handle_mapping_value(emit, fyp, fyep,
emit->state == FYES_MAPPING_SIMPLE_VALUE);
break;
goto err_out;
}
- fyi = fy_input_from_data(data, len, &handle, true);
+ fyi = fy_input_from_malloc_data(data, len, &handle, true);
if (!fyi)
goto err_out;
data = NULL;
return fye;
}
+int
+fy_emit_vevent(struct fy_emitter *emit, enum fy_event_type type, va_list ap)
+{
+ struct fy_event *fye;
+
+ if (!emit)
+ return -1;
+
+ fye = fy_emit_event_vcreate(emit, type, ap);
+ if (!fye)
+ return -1;
+
+ return fy_emit_event(emit, fye);
+}
+
+int
+fy_emit_eventf(struct fy_emitter *emit, enum fy_event_type type, ...)
+{
+ int rc;
+ va_list ap;
+
+ va_start(ap, type);
+ rc = fy_emit_vevent(emit, type, ap);
+ va_end(ap);
+
+ return rc;
+}
+
+int
+fy_emit_scalar_write(struct fy_emitter *fye, enum fy_scalar_style style,
+ const char *anchor, const char *tag,
+ const char *text, size_t len)
+{
+ return fy_emit_eventf(fye, FYET_SCALAR, style, text, len, anchor, tag);
+}
+
+int
+fy_emit_scalar_vprintf(struct fy_emitter *fye, enum fy_scalar_style style,
+ const char *anchor, const char *tag,
+ const char *fmt, va_list ap)
+{
+ char *buf;
+ size_t len;
+ int rc;
+
+ rc = vasprintf(&buf, fmt, ap);
+ if (rc < 0)
+ return -1;
+ len = (size_t)rc;
+
+ rc = fy_emit_scalar_write(fye, style, anchor, tag, buf, len);
+ free(buf);
+
+ return rc;
+}
+
+int
+fy_emit_scalar_printf(struct fy_emitter *fye, enum fy_scalar_style style,
+ const char *anchor, const char *tag,
+ const char *fmt, ...)
+{
+ va_list ap;
+ int rc;
+
+ va_start(ap, fmt);
+ rc = fy_emit_scalar_vprintf(fye, style, anchor, tag, fmt, ap);
+ va_end(ap);
+
+ return rc;
+}
+
struct fy_event *
fy_parse_event_vcreate(struct fy_parser *fyp, enum fy_event_type type, va_list ap)
{
return NULL;
}
+struct fy_token *fy_event_get_and_clear_anchor_token(struct fy_event *fye)
+{
+ struct fy_token *fyt = NULL;
+
+ if (!fye)
+ return NULL;
+
+ switch (fye->type) {
+ case FYET_MAPPING_START:
+ fyt = fye->mapping_start.anchor;
+ fye->mapping_start.anchor = NULL;
+ break;
+
+ case FYET_SEQUENCE_START:
+ fyt = fye->sequence_start.anchor;
+ fye->sequence_start.anchor = NULL;
+ break;
+
+ case FYET_SCALAR:
+ fyt = fye->scalar.anchor;
+ fye->scalar.anchor = NULL;
+ break;
+
+ default:
+ break;
+ }
+
+ return fyt;
+}
+
struct fy_token *fy_event_get_tag_token(struct fy_event *fye)
{
if (!fye)
return FYNS_ANY;
}
+struct fy_eventp *fy_parse_eventp_clone(struct fy_parser *fyp, struct fy_eventp *fyep_src, bool strip_anchors)
+{
+ struct fy_eventp *fyep = NULL;
+ struct fy_event *fye, *fye_src;
+
+ if (!fyp || !fyep_src)
+ return NULL;
+
+ if (fyp->recycled_eventp_list)
+ fyep = fy_eventp_list_pop(fyp->recycled_eventp_list);
+ if (!fyep)
+ fyep = fy_eventp_alloc();
+ if (!fyep)
+ return NULL;
+
+ fye_src = &fyep_src->e;
+ fye = &fyep->e;
+
+ fye->type = fye_src->type;
+ switch (fye->type) {
+ case FYET_NONE:
+ break;
+ case FYET_STREAM_START:
+ fye->stream_start.stream_start = fy_token_ref(fye_src->stream_start.stream_start);
+ break;
+ case FYET_STREAM_END:
+ fye->stream_end.stream_end = fy_token_ref(fye_src->stream_end.stream_end);
+ break;
+ case FYET_DOCUMENT_START:
+ fye->document_start.document_start = fy_token_ref(fye_src->document_start.document_start);
+ fye->document_start.document_state = fy_document_state_ref(fye_src->document_start.document_state);
+ break;
+ case FYET_DOCUMENT_END:
+ fye->document_end.document_end = fy_token_ref(fye_src->document_end.document_end);
+ break;
+ case FYET_MAPPING_START:
+ if (!strip_anchors)
+ fye->mapping_start.anchor = fy_token_ref(fye_src->mapping_start.anchor);
+ else
+ fye->mapping_start.anchor = NULL;
+ fye->mapping_start.tag = fy_token_ref(fye_src->mapping_start.tag);
+ fye->mapping_start.mapping_start = fy_token_ref(fye_src->mapping_start.mapping_start);
+ break;
+ case FYET_MAPPING_END:
+ fye->mapping_end.mapping_end = fy_token_ref(fye_src->mapping_end.mapping_end);
+ break;
+ case FYET_SEQUENCE_START:
+ if (!strip_anchors)
+ fye->sequence_start.anchor = fy_token_ref(fye_src->sequence_start.anchor);
+ else
+ fye->sequence_start.anchor = NULL;
+ fye->sequence_start.tag = fy_token_ref(fye_src->sequence_start.tag);
+ fye->sequence_start.sequence_start = fy_token_ref(fye_src->sequence_start.sequence_start);
+ break;
+ case FYET_SEQUENCE_END:
+ fye->sequence_end.sequence_end = fy_token_ref(fye_src->sequence_end.sequence_end);
+ break;
+ case FYET_SCALAR:
+ if (!strip_anchors)
+ fye->scalar.anchor = fy_token_ref(fye_src->scalar.anchor);
+ else
+ fye->scalar.anchor = NULL;
+ fye->scalar.tag = fy_token_ref(fye_src->scalar.tag);
+ fye->scalar.value = fy_token_ref(fye_src->scalar.value);
+ break;
+ case FYET_ALIAS:
+ fye->alias.anchor = fy_token_ref(fye_src->alias.anchor);
+ break;
+ }
+ return fyep;
+}
+
+const char *fy_event_get_anchor(struct fy_event *fye, size_t *anchor_lenp)
+{
+ struct fy_token *anchor;
+
+ anchor = fy_event_get_anchor_token(fye);
+ if (!anchor)
+ return NULL;
+
+ return fy_token_get_text(anchor, anchor_lenp);
+}
+
const struct fy_version *
fy_document_start_event_version(struct fy_event *fye)
{
struct fy_event e;
};
FY_TYPE_DECL_LIST(eventp);
-FY_PARSE_TYPE_DECL_ALLOC(eventp);
struct fy_eventp *fy_eventp_alloc(void);
void fy_eventp_free(struct fy_eventp *fyep);
struct fy_eventp *fy_emit_eventp_alloc(struct fy_emitter *fye);
void fy_emit_eventp_recycle(struct fy_emitter *emit, struct fy_eventp *fyep);
+struct fy_eventp *fy_parse_eventp_clone(struct fy_parser *fyp, struct fy_eventp *fyep_src, bool strip_anchors);
+struct fy_token *fy_event_get_and_clear_anchor_token(struct fy_event *fye);
+const char *fy_event_get_anchor(struct fy_event *fye, size_t *anchor_lenp);
+
struct fy_document_iterator;
struct fy_eventp *fy_document_iterator_eventp_alloc(struct fy_document_iterator *fydi);
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
+#include <limits.h>
#include <errno.h>
#include <libfyaml.h>
}
}
+ssize_t fy_input_estimate_queued_size(const struct fy_input *fyi)
+{
+ struct stat sb;
+ int fd, rc;
+
+ if (!fyi || fyi->state != FYIS_QUEUED)
+ return 0;
+
+ memset(&sb, 0, sizeof(sb));
+
+ switch (fyi->cfg.type) {
+ case fyit_file:
+ rc = stat(fyi->cfg.file.filename, &sb);
+ if (rc)
+ return -1;
+ break;
+
+ case fyit_stream:
+ fd = fileno(fyi->cfg.stream.fp);
+ if (fd < 0)
+ return -1;
+ rc = fstat(fd, &sb);
+ if (rc)
+ return -1;
+ break;
+
+ case fyit_memory:
+ return (ssize_t)fyi->cfg.memory.size;
+
+ case fyit_alloc:
+ return (ssize_t)fyi->cfg.alloc.size;
+
+ case fyit_fd:
+ rc = fstat(fyi->cfg.fd.fd, &sb);
+ if (rc)
+ return -1;
+ break;
+
+ case fyit_callback:
+ default:
+ return SSIZE_MAX; /* cannot determine */
+ }
+
+ /* only do it for regular files */
+ if ((sb.st_mode & S_IFMT) != S_IFREG)
+ return SSIZE_MAX;
+
+ /* check for very impossible roll-over */
+ if ((size_t)sb.st_size > (size_t)SSIZE_MAX)
+ return SSIZE_MAX;
+
+ /* ok, we did find it */
+ return (ssize_t)sb.st_size;
+}
+
struct fy_diag *fy_reader_get_diag(struct fy_reader *fyr)
{
if (fyr && fyr->ops && fyr->ops->get_diag)
{
struct fy_input *fyi;
const void *p;
- int left;
+ size_t left;
if (fyr->current_ptr) {
if (leftp)
fyi->refs--;
}
+ssize_t fy_input_estimate_queued_size(const struct fy_input *fyi);
+
struct fy_reader;
enum fy_reader_mode {
+++ /dev/null
-/*
- * fy-list.h - slightly modified Linux kernel list.h
- *
- * All copyrights owned by their respective holders.
- *
- * SPDX-License-Identifier: GPL-2.0
- */
-#ifndef _FY_LIST_H
-#define _FY_LIST_H
-
-#include <stddef.h>
-
-/**
- * container_of - cast a member of a structure out to the containing structure
- * @ptr: the pointer to the member.
- * @type: the type of the container struct this is embedded in.
- * @member: the name of the member within the struct.
- *
- */
-#define container_of(ptr, type, member) ({ \
- const typeof( ((type *)0)->member ) *__mptr = (ptr); \
- (type *)( (char *)__mptr - offsetof(type,member) );})
-
-#define LIST_POISON1 NULL
-#define LIST_POISON2 NULL
-
-#ifndef ARCH_HAS_PREFETCH
-#define ARCH_HAS_PREFETCH
-static inline void prefetch(const void *x) {;}
-#endif
-
-/*
- * Simple doubly linked list implementation.
- *
- * Some of the internal functions ("__xxx") are useful when
- * manipulating whole lists rather than single entries, as
- * sometimes we already know the next/prev entries and we can
- * generate better code by using them directly rather than
- * using the generic single-entry routines.
- */
-
-struct list_head {
- struct list_head *next, *prev;
-};
-
-#define LIST_HEAD_INIT(name) { &(name), &(name) }
-
-#define LIST_HEAD(name) \
- struct list_head name = LIST_HEAD_INIT(name)
-
-static inline void INIT_LIST_HEAD(struct list_head *list)
-{
- list->next = list;
- list->prev = list;
-}
-
-/*
- * Insert a new entry between two known consecutive entries.
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_add(struct list_head *new,
- struct list_head *prev,
- struct list_head *next)
-{
- next->prev = new;
- new->next = next;
- new->prev = prev;
- prev->next = new;
-}
-
-/**
- * list_add - add a new entry
- * @new: new entry to be added
- * @head: list head to add it after
- *
- * Insert a new entry after the specified head.
- * This is good for implementing stacks.
- */
-static inline void list_add(struct list_head *new, struct list_head *head)
-{
- __list_add(new, head, head->next);
-}
-
-/**
- * list_add_tail - add a new entry
- * @new: new entry to be added
- * @head: list head to add it before
- *
- * Insert a new entry before the specified head.
- * This is useful for implementing queues.
- */
-static inline void list_add_tail(struct list_head *new, struct list_head *head)
-{
- __list_add(new, head->prev, head);
-}
-
-/*
- * Delete a list entry by making the prev/next entries
- * point to each other.
- *
- * This is only for internal list manipulation where we know
- * the prev/next entries already!
- */
-static inline void __list_del(struct list_head *prev, struct list_head *next)
-{
- next->prev = prev;
- prev->next = next;
-}
-
-/**
- * list_del - deletes entry from list.
- * @entry: the element to delete from the list.
- * Note: list_empty() on entry does not return true after this, the entry is
- * in an undefined state.
- */
-static inline void list_del(struct list_head *entry)
-{
- __list_del(entry->prev, entry->next);
- entry->next = LIST_POISON1;
- entry->prev = LIST_POISON2;
-}
-
-/**
- * list_replace - replace old entry by new one
- * @old : the element to be replaced
- * @new : the new element to insert
- *
- * If @old was empty, it will be overwritten.
- */
-static inline void list_replace(struct list_head *old,
- struct list_head *new)
-{
- new->next = old->next;
- new->next->prev = new;
- new->prev = old->prev;
- new->prev->next = new;
-}
-
-static inline void list_replace_init(struct list_head *old,
- struct list_head *new)
-{
- list_replace(old, new);
- INIT_LIST_HEAD(old);
-}
-
-/**
- * list_del_init - deletes entry from list and reinitialize it.
- * @entry: the element to delete from the list.
- */
-static inline void list_del_init(struct list_head *entry)
-{
- __list_del(entry->prev, entry->next);
- INIT_LIST_HEAD(entry);
-}
-
-/**
- * list_move - delete from one list and add as another's head
- * @list: the entry to move
- * @head: the head that will precede our entry
- */
-static inline void list_move(struct list_head *list, struct list_head *head)
-{
- __list_del(list->prev, list->next);
- list_add(list, head);
-}
-
-/**
- * list_move_tail - delete from one list and add as another's tail
- * @list: the entry to move
- * @head: the head that will follow our entry
- */
-static inline void list_move_tail(struct list_head *list,
- struct list_head *head)
-{
- __list_del(list->prev, list->next);
- list_add_tail(list, head);
-}
-
-/**
- * list_is_last - tests whether @list is the last entry in list @head
- * @list: the entry to test
- * @head: the head of the list
- */
-static inline int list_is_last(const struct list_head *list,
- const struct list_head *head)
-{
- return list->next == head;
-}
-
-/**
- * list_empty - tests whether a list is empty
- * @head: the list to test.
- */
-static inline int list_empty(const struct list_head *head)
-{
- return head->next == head;
-}
-
-/**
- * list_empty_careful - tests whether a list is empty and not being modified
- * @head: the list to test
- *
- * Description:
- * tests whether a list is empty _and_ checks that no other CPU might be
- * in the process of modifying either member (next or prev)
- *
- * NOTE: using list_empty_careful() without synchronization
- * can only be safe if the only activity that can happen
- * to the list entry is list_del_init(). Eg. it cannot be used
- * if another CPU could re-list_add() it.
- */
-static inline int list_empty_careful(const struct list_head *head)
-{
- struct list_head *next = head->next;
- return (next == head) && (next == head->prev);
-}
-
-/**
- * list_is_singular - tests whether a list has just one entry.
- * @head: the list to test.
- */
-static inline int list_is_singular(const struct list_head *head)
-{
- return !list_empty(head) && (head->next == head->prev);
-}
-
-static inline void __list_cut_position(struct list_head *list,
- struct list_head *head, struct list_head *entry)
-{
- struct list_head *new_first = entry->next;
- list->next = head->next;
- list->next->prev = list;
- list->prev = entry;
- entry->next = list;
- head->next = new_first;
- new_first->prev = head;
-}
-
-/**
- * list_cut_position - cut a list into two
- * @list: a new list to add all removed entries
- * @head: a list with entries
- * @entry: an entry within head, could be the head itself
- * and if so we won't cut the list
- *
- * This helper moves the initial part of @head, up to and
- * including @entry, from @head to @list. You should
- * pass on @entry an element you know is on @head. @list
- * should be an empty list or a list you do not care about
- * losing its data.
- *
- */
-static inline void list_cut_position(struct list_head *list,
- struct list_head *head, struct list_head *entry)
-{
- if (list_empty(head))
- return;
- if (list_is_singular(head) &&
- (head->next != entry && head != entry))
- return;
- if (entry == head)
- INIT_LIST_HEAD(list);
- else
- __list_cut_position(list, head, entry);
-}
-
-static inline void __list_splice(const struct list_head *list,
- struct list_head *prev,
- struct list_head *next)
-{
- struct list_head *first = list->next;
- struct list_head *last = list->prev;
-
- first->prev = prev;
- prev->next = first;
-
- last->next = next;
- next->prev = last;
-}
-
-/**
- * list_splice - join two lists, this is designed for stacks
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- */
-static inline void list_splice(const struct list_head *list,
- struct list_head *head)
-{
- if (!list_empty(list))
- __list_splice(list, head, head->next);
-}
-
-/**
- * list_splice_tail - join two lists, each list being a queue
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- */
-static inline void list_splice_tail(struct list_head *list,
- struct list_head *head)
-{
- if (!list_empty(list))
- __list_splice(list, head->prev, head);
-}
-
-/**
- * list_splice_init - join two lists and reinitialise the emptied list.
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- *
- * The list at @list is reinitialised
- */
-static inline void list_splice_init(struct list_head *list,
- struct list_head *head)
-{
- if (!list_empty(list)) {
- __list_splice(list, head, head->next);
- INIT_LIST_HEAD(list);
- }
-}
-
-/**
- * list_splice_tail_init - join two lists and reinitialise the emptied list
- * @list: the new list to add.
- * @head: the place to add it in the first list.
- *
- * Each of the lists is a queue.
- * The list at @list is reinitialised
- */
-static inline void list_splice_tail_init(struct list_head *list,
- struct list_head *head)
-{
- if (!list_empty(list)) {
- __list_splice(list, head->prev, head);
- INIT_LIST_HEAD(list);
- }
-}
-
-/**
- * list_entry - get the struct for this entry
- * @ptr: the &struct list_head pointer.
- * @type: the type of the struct this is embedded in.
- * @member: the name of the list_struct within the struct.
- */
-#define list_entry(ptr, type, member) \
- container_of(ptr, type, member)
-
-/**
- * list_first_entry - get the first element from a list
- * @ptr: the list head to take the element from.
- * @type: the type of the struct this is embedded in.
- * @member: the name of the list_struct within the struct.
- *
- * Note, that list is expected to be not empty.
- */
-#define list_first_entry(ptr, type, member) \
- list_entry((ptr)->next, type, member)
-
-/**
- * list_last_entry - get the last element from a list
- * @ptr: the list head to take the element from.
- * @type: the type of the struct this is embedded in.
- * @member: the name of the list_struct within the struct.
- *
- * Note, that list is expected to be not empty.
- */
-#define list_last_entry(ptr, type, member) \
- list_entry((ptr)->prev, type, member)
-
-/**
- * list_for_each - iterate over a list
- * @pos: the &struct list_head to use as a loop cursor.
- * @head: the head for your list.
- */
-#define list_for_each(pos, head) \
- for (pos = (head)->next; prefetch(pos->next), pos != (head); \
- pos = pos->next)
-
-/**
- * __list_for_each - iterate over a list
- * @pos: the &struct list_head to use as a loop cursor.
- * @head: the head for your list.
- *
- * This variant differs from list_for_each() in that it's the
- * simplest possible list iteration code, no prefetching is done.
- * Use this for code that knows the list to be very short (empty
- * or 1 entry) most of the time.
- */
-#define __list_for_each(pos, head) \
- for (pos = (head)->next; pos != (head); pos = pos->next)
-
-/**
- * list_for_each_prev - iterate over a list backwards
- * @pos: the &struct list_head to use as a loop cursor.
- * @head: the head for your list.
- */
-#define list_for_each_prev(pos, head) \
- for (pos = (head)->prev; prefetch(pos->prev), pos != (head); \
- pos = pos->prev)
-
-/**
- * list_for_each_safe - iterate over a list safe against removal of list entry
- * @pos: the &struct list_head to use as a loop cursor.
- * @n: another &struct list_head to use as temporary storage
- * @head: the head for your list.
- */
-#define list_for_each_safe(pos, n, head) \
- for (pos = (head)->next, n = pos->next; pos != (head); \
- pos = n, n = pos->next)
-
-/**
- * list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry
- * @pos: the &struct list_head to use as a loop cursor.
- * @n: another &struct list_head to use as temporary storage
- * @head: the head for your list.
- */
-#define list_for_each_prev_safe(pos, n, head) \
- for (pos = (head)->prev, n = pos->prev; \
- prefetch(pos->prev), pos != (head); \
- pos = n, n = pos->prev)
-
-/**
- * list_for_each_entry - iterate over list of given type
- * @pos: the type * to use as a loop cursor.
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- */
-#define list_for_each_entry(pos, head, member) \
- for (pos = list_entry((head)->next, typeof(*pos), member); \
- prefetch(pos->member.next), &pos->member != (head); \
- pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_reverse - iterate backwards over list of given type.
- * @pos: the type * to use as a loop cursor.
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- */
-#define list_for_each_entry_reverse(pos, head, member) \
- for (pos = list_entry((head)->prev, typeof(*pos), member); \
- prefetch(pos->member.prev), &pos->member != (head); \
- pos = list_entry(pos->member.prev, typeof(*pos), member))
-
-/**
- * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue()
- * @pos: the type * to use as a start point
- * @head: the head of the list
- * @member: the name of the list_struct within the struct.
- *
- * Prepares a pos entry for use as a start point in list_for_each_entry_continue().
- */
-#define list_prepare_entry(pos, head, member) \
- ((pos) ? : list_entry(head, typeof(*pos), member))
-
-/**
- * list_for_each_entry_continue - continue iteration over list of given type
- * @pos: the type * to use as a loop cursor.
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- *
- * Continue to iterate over list of given type, continuing after
- * the current position.
- */
-#define list_for_each_entry_continue(pos, head, member) \
- for (pos = list_entry(pos->member.next, typeof(*pos), member); \
- prefetch(pos->member.next), &pos->member != (head); \
- pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_continue_reverse - iterate backwards from the given point
- * @pos: the type * to use as a loop cursor.
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- *
- * Start to iterate over list of given type backwards, continuing after
- * the current position.
- */
-#define list_for_each_entry_continue_reverse(pos, head, member) \
- for (pos = list_entry(pos->member.prev, typeof(*pos), member); \
- prefetch(pos->member.prev), &pos->member != (head); \
- pos = list_entry(pos->member.prev, typeof(*pos), member))
-
-/**
- * list_for_each_entry_from - iterate over list of given type from the current point
- * @pos: the type * to use as a loop cursor.
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- *
- * Iterate over list of given type, continuing from current position.
- */
-#define list_for_each_entry_from(pos, head, member) \
- for (; prefetch(pos->member.next), &pos->member != (head); \
- pos = list_entry(pos->member.next, typeof(*pos), member))
-
-/**
- * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
- * @pos: the type * to use as a loop cursor.
- * @n: another type * to use as temporary storage
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- */
-#define list_for_each_entry_safe(pos, n, head, member) \
- for (pos = list_entry((head)->next, typeof(*pos), member), \
- n = list_entry(pos->member.next, typeof(*pos), member); \
- &pos->member != (head); \
- pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-/**
- * list_for_each_entry_safe_continue
- * @pos: the type * to use as a loop cursor.
- * @n: another type * to use as temporary storage
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- *
- * Iterate over list of given type, continuing after current point,
- * safe against removal of list entry.
- */
-#define list_for_each_entry_safe_continue(pos, n, head, member) \
- for (pos = list_entry(pos->member.next, typeof(*pos), member), \
- n = list_entry(pos->member.next, typeof(*pos), member); \
- &pos->member != (head); \
- pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-/**
- * list_for_each_entry_safe_from
- * @pos: the type * to use as a loop cursor.
- * @n: another type * to use as temporary storage
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- *
- * Iterate over list of given type from current point, safe against
- * removal of list entry.
- */
-#define list_for_each_entry_safe_from(pos, n, head, member) \
- for (n = list_entry(pos->member.next, typeof(*pos), member); \
- &pos->member != (head); \
- pos = n, n = list_entry(n->member.next, typeof(*n), member))
-
-/**
- * list_for_each_entry_safe_reverse
- * @pos: the type * to use as a loop cursor.
- * @n: another type * to use as temporary storage
- * @head: the head for your list.
- * @member: the name of the list_struct within the struct.
- *
- * Iterate backwards over list of given type, safe against removal
- * of list entry.
- */
-#define list_for_each_entry_safe_reverse(pos, n, head, member) \
- for (pos = list_entry((head)->prev, typeof(*pos), member), \
- n = list_entry(pos->member.prev, typeof(*pos), member); \
- &pos->member != (head); \
- pos = n, n = list_entry(n->member.prev, typeof(*n), member))
-
-/*
- * Double linked lists with a single pointer list head.
- * Mostly useful for hash tables where the two pointer list head is
- * too wasteful.
- * You lose the ability to access the tail in O(1).
- */
-
-struct hlist_head {
- struct hlist_node *first;
-};
-
-struct hlist_node {
- struct hlist_node *next, **pprev;
-};
-
-#define HLIST_HEAD_INIT { .first = NULL }
-#define HLIST_HEAD(name) struct hlist_head name = { .first = NULL }
-#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL)
-static inline void INIT_HLIST_NODE(struct hlist_node *h)
-{
- h->next = NULL;
- h->pprev = NULL;
-}
-
-static inline int hlist_unhashed(const struct hlist_node *h)
-{
- return !h->pprev;
-}
-
-static inline int hlist_empty(const struct hlist_head *h)
-{
- return !h->first;
-}
-
-static inline void __hlist_del(struct hlist_node *n)
-{
- struct hlist_node *next = n->next;
- struct hlist_node **pprev = n->pprev;
- *pprev = next;
- if (next)
- next->pprev = pprev;
-}
-
-static inline void hlist_del(struct hlist_node *n)
-{
- __hlist_del(n);
- n->next = LIST_POISON1;
- n->pprev = LIST_POISON2;
-}
-
-static inline void hlist_del_init(struct hlist_node *n)
-{
- if (!hlist_unhashed(n)) {
- __hlist_del(n);
- INIT_HLIST_NODE(n);
- }
-}
-
-static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
-{
- struct hlist_node *first = h->first;
- n->next = first;
- if (first)
- first->pprev = &n->next;
- h->first = n;
- n->pprev = &h->first;
-}
-
-/* next must be != NULL */
-static inline void hlist_add_before(struct hlist_node *n,
- struct hlist_node *next)
-{
- n->pprev = next->pprev;
- n->next = next;
- next->pprev = &n->next;
- *(n->pprev) = n;
-}
-
-static inline void hlist_add_after(struct hlist_node *n,
- struct hlist_node *next)
-{
- next->next = n->next;
- n->next = next;
- next->pprev = &n->next;
-
- if(next->next)
- next->next->pprev = &next->next;
-}
-
-#define hlist_entry(ptr, type, member) container_of(ptr,type,member)
-
-#define hlist_for_each(pos, head) \
- for (pos = (head)->first; pos && ({ prefetch(pos->next); 1; }); \
- pos = pos->next)
-
-#define hlist_for_each_safe(pos, n, head) \
- for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \
- pos = n)
-
-/**
- * hlist_for_each_entry - iterate over list of given type
- * @tpos: the type * to use as a loop cursor.
- * @pos: the &struct hlist_node to use as a loop cursor.
- * @head: the head for your list.
- * @member: the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry(tpos, pos, head, member) \
- for (pos = (head)->first; \
- pos && ({ prefetch(pos->next); 1;}) && \
- ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
- pos = pos->next)
-
-/**
- * hlist_for_each_entry_continue - iterate over a hlist continuing after current point
- * @tpos: the type * to use as a loop cursor.
- * @pos: the &struct hlist_node to use as a loop cursor.
- * @member: the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_continue(tpos, pos, member) \
- for (pos = (pos)->next; \
- pos && ({ prefetch(pos->next); 1;}) && \
- ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
- pos = pos->next)
-
-/**
- * hlist_for_each_entry_from - iterate over a hlist continuing from current point
- * @tpos: the type * to use as a loop cursor.
- * @pos: the &struct hlist_node to use as a loop cursor.
- * @member: the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_from(tpos, pos, member) \
- for (; pos && ({ prefetch(pos->next); 1;}) && \
- ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
- pos = pos->next)
-
-/**
- * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry
- * @tpos: the type * to use as a loop cursor.
- * @pos: the &struct hlist_node to use as a loop cursor.
- * @n: another &struct hlist_node to use as temporary storage
- * @head: the head for your list.
- * @member: the name of the hlist_node within the struct.
- */
-#define hlist_for_each_entry_safe(tpos, pos, n, head, member) \
- for (pos = (head)->first; \
- pos && ({ n = pos->next; 1; }) && \
- ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
- pos = n)
-
-#endif
return -1;
}
+ssize_t fy_parse_estimate_queued_input_size(struct fy_parser *fyp)
+{
+ struct fy_input *fyi;
+ ssize_t size, total;
+
+ if (!fyp)
+ return 0;
+
+ total = 0;
+ for (fyi = fy_input_list_head(&fyp->queued_inputs); fyi; fyi = fy_input_next(&fyp->queued_inputs, fyi)) {
+ if (fyi->state != FYIS_QUEUED)
+ continue;
+
+ size = fy_input_estimate_queued_size(fyi);
+ if (size < 0)
+ return -1;
+
+ /* something that's indetermined */
+ if (size == SSIZE_MAX)
+ return SSIZE_MAX;
+
+ total += size;
+
+ /* roll-over check */
+ if (total < 0)
+ return SSIZE_MAX;
+ }
+
+ return total;
+}
+
static inline void
fy_token_queue_epilogue(struct fy_parser *fyp, struct fy_token *fyt)
{
fyp->flow = FYFT_NONE;
fy_parse_flow_list_recycle_all(fyp, &fyp->flow_stack);
+ /* and reset all aliases (TODO look for cross document aliases) */
+ fy_parse_streaming_aliases_reset(fyp);
+
return 0;
err_out:
fy_indent_list_init(&fyp->recycled_indent);
fyp->indent = -2;
fyp->indent_line = -1;
+ fyp->starting_indent = -1;
fyp->generated_block_map = false;
fyp->last_was_comma = false;
fyp->pending_complex_key_column = -1;
fyp->last_block_mapping_key_line = -1;
+ fy_streaming_alias_list_init(&fyp->streaming_aliases);
+ fy_streaming_alias_list_init(&fyp->recycled_streaming_alias);
+ fy_eventp_list_init(&fyp->mks.args);
+
fyp->suppress_recycling = !!(fyp->cfg.flags & FYPCF_DISABLE_RECYCLING) ||
(getenv("FY_VALGRIND") &&
!getenv("FY_VALGRIND_RECYCLING"));
struct fy_eventp *fyep;
struct fy_token *fyt;
+ fy_parse_streaming_aliases_reset(fyp);
+
fy_input_unref(fyp->last_event_handle.fyi);
fy_atom_reset(&fyp->last_event_handle);
fy_parse_parse_state_log_list_recycle_all(fyp, &fyp->state_stack);
fy_parse_flow_list_recycle_all(fyp, &fyp->flow_stack);
+ fy_parse_streaming_alias_list_recycle_all(fyp, &fyp->streaming_aliases);
fy_token_unref_rl(fyp->recycled_token_list, fyp->stream_end_token);
fy_parse_simple_key_vacuum(fyp);
fy_parse_parse_state_log_vacuum(fyp);
fy_parse_flow_vacuum(fyp);
+ fy_parse_streaming_alias_vacuum(fyp);
/* free the recycled events */
while ((fyep = fy_eventp_list_pop(&fyp->recycled_eventp)) != NULL) {
fyp->parent_indent = fyp->indent;
fyp->indent = indent;
fyp->indent_line = indent_line;
+ if (fyp->parse_block_only && fyp->starting_indent < 0)
+ fyp->starting_indent = indent;
fyp->generated_block_map = generated_block_map;
fyp_scan_debug(fyp, "push_indent %d -> %d - generated_block_map=%s\n",
int rc;
/* only reset the stream in regular mode */
- if (!fyp->parse_flow_only)
+ if (!fyp->parse_flow_only && !fyp->parse_block_only)
fy_reader_stream_end(fyp->reader);
fy_remove_all_simple_keys(fyp);
fyp_error_check(fyp, fyt, err_out,
"fy_token_queue_simple() failed");
- rc = fy_save_simple_key_mark(fyp, &skm, type, NULL);
- fyp_error_check(fyp, !rc, err_out_rc,
- "fy_save_simple_key_mark() failed");
+ /* json does not have complex keys */
+ if (!fyp_json_mode(fyp)) {
+ rc = fy_save_simple_key_mark(fyp, &skm, type, NULL);
+ fyp_error_check(fyp, !rc, err_out_rc,
+ "fy_save_simple_key_mark() failed");
+ }
/* increase flow level */
fyp->flow_level++;
line_length += fy_utf8_width(c);
}
+ /* keep the variables without warning */
+ (void)trailing_ws;
+ (void)leading_ws;
if (doc_start_end_detected)
break;
handle.fws_mode = fyp_fws_mode(fyp);
handle.tabsize = fyp_tabsize(fyp);
handle.ends_with_eof = ends_with_eof;
+ handle.is_merge_key = false;
#ifdef ATOM_SIZE_CHECK
tlength = fy_atom_format_text_length(&handle);
handle->fws_mode = fy_reader_flow_ws_mode(fyr);
handle->tabsize = fy_reader_tabsize(fyr);
handle->ends_with_eof = false; /* flow scalars never end with EOF and be valid */
+ handle->is_merge_key = false;
/* skip over block scalar end */
fy_reader_advance_by(fyr, 1);
bool has_leading_blanks;
bool last_ptr;
struct fy_mark mark, last_mark;
- bool is_multiline, has_lb, has_ws, ends_with_eof;
+ bool is_multiline, has_lb, has_ws, ends_with_eof, is_merge_key;
bool has_json_esc;
#ifdef ATOM_SIZE_CHECK
size_t tlength;
flow_level == 0 || !(c == '-' && fy_utf8_strchr(",[]{}", fy_reader_peek_at(fyr, 1))), err_out,
"plain scalar cannot start with '%c' followed by ,[]{} (in flow context)", c);
+ /* we can prime is_merge_key here */
+ is_merge_key = c == '<' && fy_reader_peek_at(fyr, 1) == '<';
+
fy_reader_get_mark(fyr, &mark);
fy_reader_fill_atom_start(fyr, handle);
handle->fws_mode = fy_reader_flow_ws_mode(fyr);
handle->tabsize = fy_reader_tabsize(fyr);
handle->ends_with_eof = ends_with_eof;
+ handle->is_merge_key = is_merge_key && length == 2;
#ifdef ATOM_SIZE_CHECK
tlength = fy_atom_format_text_length(handle);
}
s++;
} else {
- c = fy_utf8_get(s, (int)(e - s), &w);
+ c = fy_utf8_get(s, (size_t)(e - s), &w);
if (c == FYUG_PARTIAL) {
/* get the width (from the first octet */
"fy_scan_to_next_token() failed");
if (fyp_block_mode(fyp)) {
+ if (fyp->parse_block_only && fyp->starting_indent >= 0 &&
+ fyp_column(fyp) < fyp->starting_indent) {
+ rc = fy_fetch_stream_end(fyp);
+ fyp_error_check(fyp, !rc, err_out_rc,
+ "fy_fetch_stream_end() failed");
+ return 0;
+ }
rc = fy_parse_unroll_indent(fyp, fyp_column(fyp));
fyp_error_check(fyp, !rc, err_out_rc,
"fy_parse_unroll_indent() failed");
fyp_scan_debug(fyp, "setting stream_end_produced to true");
fyp->stream_end_produced = true;
- if (!fyp->parse_flow_only) {
+ if (!fyp->parse_flow_only && !fyp->parse_block_only) {
rc = fy_reader_input_done(fyp->reader);
fyp_error_check(fyp, !rc, err_out,
"fy_parse_input_done() failed");
{
fyp->indent = -2;
fyp->indent_line = -1;
+ fyp->starting_indent = -1;
fyp->generated_block_map = false;
fyp->last_was_comma = false;
fyp->flow = FYFT_NONE;
struct fy_event *fy_parser_parse(struct fy_parser *fyp)
{
- struct fy_eventp *fyep;
+ struct fy_eventp *fyep, *fyep2;
enum fy_composer_return ret;
if (!fyp)
return NULL;
+ if ((fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT) && fy_reader_get_mode(fyp->reader) != fyrm_json) {
+ fyep = fy_parser_parse_resolve_prolog(fyp);
+ if (fyep)
+ return &fyep->e;
+ }
+
fyep = fy_parse_private(fyp);
if (!fyep)
return NULL;
+ if ((fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT) && fy_reader_get_mode(fyp->reader) != fyrm_json) {
+ fyep2 = fy_parser_parse_resolve_epilog(fyp, fyep);
+ if (!fyep2)
+ return NULL;
+ fyep = fyep2;
+ }
+
if (fyp->fyc) {
ret = fy_composer_process_event(fyp->fyc, &fyep->e);
if (ret == FYCR_ERROR) {
int fy_parse_set_composer(struct fy_parser *fyp, fy_parse_composer_cb cb, void *userdata)
{
struct fy_composer_cfg ccfg;
+ bool can_change_composer;
if (!fyp)
return -1;
+ can_change_composer = fyp->state == FYPS_NONE ||
+ fyp->state == FYPS_END ||
+ fyp->state == FYPS_DOCUMENT_START;
+
/* must not be in the middle of something */
- fyp_error_check(fyp, fyp->state == FYPS_NONE || fyp->state == FYPS_END,
- err_out, "cannot change composer state at state '%s'",
- state_txt[fyp->state]);
+ fyp_error_check(fyp, can_change_composer, err_out,
+ "cannot change composer state at state '%s'", state_txt[fyp->state]);
/* clear */
if (!cb) {
rc_out = 0;
/* reset the parser; the composer clear must always succeed */
- fy_parser_reset(fyp);
+ if (ret == FYCR_ERROR)
+ fy_parser_reset(fyp);
/* clear composer */
rc = fy_parse_set_composer(fyp, NULL, NULL);
err_out:
return -1;
}
+
+struct fy_streaming_alias *
+fy_parser_streaming_alias_lookup(struct fy_parser *fyp, struct fy_token *fyt_anchor)
+{
+ struct fy_streaming_alias *fysa;
+
+ /* XXX todo hashing */
+ for (fysa = fy_streaming_alias_list_head(&fyp->streaming_aliases); fysa != NULL; fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa)) {
+ if (fy_token_cmp(fyt_anchor, fysa->anchor) == 0)
+ return fysa;
+ }
+ return NULL;
+}
+
+struct fy_streaming_alias *
+fy_parser_streaming_alias_lookup_pivot(struct fy_parser *fyp, struct fy_streaming_alias *fysa_pivot, struct fy_token *fyt_anchor)
+{
+ struct fy_streaming_alias *fysa;
+
+ assert(fysa_pivot);
+ /* at first do forwards starting at the pivot */
+ for (fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa_pivot); fysa != NULL; fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa)) {
+ if (fysa->anchor && fy_token_cmp(fyt_anchor, fysa->anchor) == 0)
+ return fysa;
+ }
+ /* not found? do backwards (more recent) */
+ for (fysa = fy_streaming_alias_prev(&fyp->streaming_aliases, fysa_pivot); fysa != NULL; fysa = fy_streaming_alias_prev(&fyp->streaming_aliases, fysa)) {
+ if (fysa->anchor && fy_token_cmp(fyt_anchor, fysa->anchor) == 0)
+ return fysa;
+ }
+ return NULL;
+}
+
+struct fy_streaming_alias *
+fy_parse_streaming_alias_create(struct fy_parser *fyp, struct fy_token *fyt_anchor)
+{
+ struct fy_streaming_alias *fysa;
+
+ fysa = fy_parse_streaming_alias_alloc(fyp);
+ fyp_error_check(fyp, fysa != NULL, err_out,
+ "fy_parse_streaming_alias_alloc() failed!");
+ fysa->anchor = fyt_anchor;
+ fysa->collecting = false;
+ fysa->mapping_nest = 0;
+ fysa->sequence_nest = 0;
+ fy_eventp_list_init(&fysa->events);
+
+ return fysa;
+
+err_out:
+ return NULL;
+}
+
+void
+fy_parse_streaming_alias_clean(struct fy_parser *fyp, struct fy_streaming_alias *fysa)
+{
+ struct fy_eventp *fyep;
+
+ if (!fyp || !fysa)
+ return;
+
+ fy_token_unref(fysa->anchor);
+ fysa->anchor = NULL;
+ fysa->collecting = false;
+ fysa->mapping_nest = 0;
+ fysa->sequence_nest = 0;
+ while ((fyep = fy_eventp_list_pop(&fysa->events)) != NULL)
+ fy_parse_eventp_recycle(fyp, fyep);
+}
+
+void fy_parse_streaming_aliases_reset(struct fy_parser *fyp)
+{
+ struct fy_streaming_alias *fysa;
+ struct fy_eventp *fyep;
+
+ if (!fyp)
+ return;
+
+ while ((fysa = fy_streaming_alias_list_pop(&fyp->streaming_aliases)) != NULL) {
+ fy_parse_streaming_alias_clean(fyp, fysa);
+ fy_parse_streaming_alias_recycle(fyp, fysa);
+ }
+ if (fyp->sas.stack != NULL && fyp->sas.stack != fyp->sas.local)
+ free(fyp->sas.stack);
+ memset(&fyp->sas, 0, sizeof(fyp->sas));
+ if (fyp->cts.stack != NULL && fyp->cts.stack != fyp->cts.local)
+ free(fyp->cts.stack);
+ memset(&fyp->cts, 0, sizeof(fyp->cts));
+
+ while ((fyep = fy_eventp_list_pop(&fyp->mks.args)) != NULL)
+ fy_parse_eventp_recycle(fyp, fyep);
+ memset(&fyp->mks, 0, sizeof(fyp->mks));
+ fy_eventp_list_init(&fyp->mks.args);
+}
+
+struct fy_streaming_alias_state *
+fy_parse_streaming_alias_state_push(struct fy_parser *fyp, struct fy_streaming_alias *fysa)
+{
+ struct fy_streaming_alias_state *new_stack;
+ struct fy_streaming_alias_state *fysas;
+
+ /* if first one, just point to the local area */
+ if (fyp->sas.stack == NULL) {
+ fyp->sas.stack = fyp->sas.local;
+ fyp->sas.top = 0;
+ fyp->sas.alloc = ARRAY_SIZE(fyp->sas.local);
+ }
+ /* grow the stack if required */
+ if (fyp->sas.top >= fyp->sas.alloc) {
+ assert(fyp->sas.alloc > 0);
+ assert(fyp->sas.stack);
+ new_stack = malloc(sizeof(*new_stack) * fyp->sas.alloc * 2);
+ fyp_error_check(fyp, new_stack != NULL, err_out,
+ "realloc() failed!");
+ memcpy(new_stack, fyp->sas.stack, sizeof(*new_stack) * fyp->sas.alloc);
+ fyp->sas.alloc *= 2;
+ if (fyp->sas.stack != fyp->sas.local)
+ free(fyp->sas.stack);
+ fyp->sas.stack = new_stack;
+ }
+ /* finally push it */
+ fysas = &fyp->sas.stack[fyp->sas.top++];
+
+ fysas->fysa = fysa;
+ fysas->next = fy_eventp_list_head(&fysa->events);
+
+ return fysas;
+err_out:
+ return NULL;
+}
+
+struct fy_streaming_alias_state *
+fy_parse_streaming_alias_state_pop(struct fy_parser *fyp)
+{
+ struct fy_streaming_alias_state *fysas;
+
+ if (fyp->sas.top <= 0)
+ return NULL;
+
+ fysas = &fyp->sas.stack[--fyp->sas.top];
+ fysas->fysa = NULL;
+ fysas->next = NULL;
+ if (fyp->sas.top <= 0)
+ return NULL;
+ return &fyp->sas.stack[fyp->sas.top - 1];
+}
+
+struct fy_streaming_alias_state *
+fy_parse_streaming_alias_state_top(struct fy_parser *fyp)
+{
+ if (fyp->sas.top <= 0)
+ return NULL;
+ return &fyp->sas.stack[fyp->sas.top - 1];
+}
+
+void
+fy_parse_streaming_alias_state_next_event(struct fy_parser *fyp)
+{
+ struct fy_streaming_alias_state *fysas;
+
+ fysas = fy_parse_streaming_alias_state_top(fyp);
+ if (!fysas)
+ return;
+
+ fysas->next = fy_eventp_next(&fysas->fysa->events, fysas->next);
+ while (!fysas->next) {
+ fysas = fy_parse_streaming_alias_state_pop(fyp);
+ if (!fysas)
+ break;
+ }
+}
+
+static inline int
+fy_parse_streaming_alias_collection_state_set_top(struct fy_parser *fyp, int cts)
+{
+ int pos, off, old_cts;
+
+ if (fyp->cts.top < 2)
+ return -1;
+
+ cts &= 3; /* keep 2 bits */
+
+ /* get position and offset */
+ pos = (fyp->cts.top - 2) / (sizeof(*fyp->cts.stack) * 8);
+ off = (fyp->cts.top - 2) % (sizeof(*fyp->cts.stack) * 8);
+
+ old_cts = (int)((fyp->cts.stack[pos] >> off) & 3);
+
+ /* clear the bits first */
+ fyp->cts.stack[pos] &= ~(3 << off);
+ /* set the state */
+ fyp->cts.stack[pos] |= (cts & 3) << off;
+
+ return old_cts;
+}
+
+static inline int
+fy_parse_streaming_alias_collection_state_top(struct fy_parser *fyp)
+{
+ int pos, off;
+
+ if (fyp->cts.top < 2)
+ return -1;
+
+ /* get position and offset */
+ pos = (fyp->cts.top - 2) / (sizeof(*fyp->cts.stack) * 8);
+ off = (fyp->cts.top - 2) % (sizeof(*fyp->cts.stack) * 8);
+
+ return (int)((fyp->cts.stack[pos] >> off) & 3);
+}
+
+static inline bool
+fy_parse_streaming_alias_collection_state_in_map_key(struct fy_parser *fyp)
+{
+ int cts = fy_parse_streaming_alias_collection_state_top(fyp);
+ return cts == FYCTS_MAP_KEY;
+}
+
+static inline void
+fy_parse_streaming_alias_collection_state_toggle_map(struct fy_parser *fyp)
+{
+ int cts = fy_parse_streaming_alias_collection_state_top(fyp);
+ if (cts >= 0 && (cts & FYCTS_MAP))
+ fy_parse_streaming_alias_collection_state_set_top(fyp, cts ^ FYCTS_MAP_TOGGLE);
+}
+
+int
+fy_parse_streaming_alias_collection_state_push(struct fy_parser *fyp, int cts)
+{
+ uint32_t *new_stack;
+ int size;
+
+ assert(cts >= 0);
+
+ /* if first one, just point to the local area */
+ if (fyp->cts.stack == NULL) {
+ fyp->cts.stack = fyp->cts.local;
+ fyp->cts.top = 0;
+ fyp->cts.alloc = sizeof(fyp->cts.local) * 8; /* in bits */
+ }
+ /* grow the stack if required */
+ if (fyp->cts.top + 2 >= fyp->cts.alloc) {
+ assert(fyp->cts.alloc > 0);
+ assert(fyp->cts.stack);
+ size = fyp->cts.alloc / 8; /* convert to bytes */
+ new_stack = malloc(size * 2);
+ fyp_error_check(fyp, new_stack != NULL, err_out,
+ "realloc() failed!");
+ memcpy(new_stack, fyp->cts.stack, size);
+ memset((uint8_t *)new_stack + size, 0, size);
+ fyp->cts.alloc *= 2;
+ if (fyp->cts.stack != fyp->cts.local)
+ free(fyp->cts.stack);
+ fyp->cts.stack = new_stack;
+ }
+
+ fyp->cts.top += 2;
+ fy_parse_streaming_alias_collection_state_set_top(fyp, cts);
+ return cts;
+err_out:
+ return -1;
+}
+
+int
+fy_parse_streaming_alias_collection_state_pop(struct fy_parser *fyp)
+{
+ int ret;
+
+ ret = fy_parse_streaming_alias_collection_state_top(fyp);
+ if (ret >= 0)
+ fyp->cts.top -= 2;
+ return ret;
+}
+
+static int
+fy_parser_event_resolve_hook_collect(struct fy_parser *fyp, struct fy_eventp *fyep)
+{
+ struct fy_eventp *fyep_clone = NULL;
+ struct fy_streaming_alias *fysa;
+ long map_add, seq_add;
+
+ if (!fyp || !fyep)
+ return -1;
+
+ map_add = seq_add = 0;
+ switch (fyep->e.type) {
+ case FYET_MAPPING_START:
+ map_add = 1;
+ break;
+ case FYET_MAPPING_END:
+ map_add = -1;
+ break;
+ case FYET_SEQUENCE_START:
+ seq_add = 1;
+ break;
+ case FYET_SEQUENCE_END:
+ seq_add = -1;
+ break;
+ default:
+ break;
+ }
+
+ /* for all streaming aliases that are collecting... */
+ for (fysa = fy_streaming_alias_list_head(&fyp->streaming_aliases); fysa != NULL; fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa)) {
+ if (!fysa->collecting)
+ continue;
+
+ /* clone event, stripping the anchors */
+ fyep_clone = fy_parse_eventp_clone(fyp, fyep, true);
+ fyp_error_check(fyp, fyep_clone != NULL, err_out,
+ "fy_parse_eventp_clone() failed!");
+
+ /* add to the list */
+ fy_eventp_list_add_tail(&fysa->events, fyep_clone);
+ fyep_clone = NULL;
+
+ /* the scalar will be collected every time */
+ fysa->mapping_nest += map_add;
+ fysa->sequence_nest += seq_add;
+
+ if (fysa->mapping_nest == 0 && fysa->sequence_nest == 0)
+ fysa->collecting = false;
+ }
+
+ return 0;
+
+err_out:
+ fy_parse_eventp_recycle(fyp, fyep_clone);
+ fyp->stream_error = true;
+ return -1;
+}
+
+struct fy_eventp *fy_parser_event_resolve_hook_alias(struct fy_parser *fyp, struct fy_eventp *fyep)
+{
+ struct fy_token *fyt_anchor = NULL;
+ struct fy_streaming_alias *fysa;
+ struct fy_streaming_alias_state *fysas;
+
+ if (!fyp || !fyep)
+ return NULL;
+
+ /* if it's not an alias, continue normal processing */
+ if (fyep->e.type != FYET_ALIAS)
+ return fyep;
+
+ /* it's an alias, get the token and dispose the event */
+ fyt_anchor = fyep->e.alias.anchor;
+ fyep->e.alias.anchor = NULL;
+ fy_parse_eventp_recycle(fyp, fyep);
+ fyep = NULL;
+
+ /* start generation of alias events */
+ fysa = fy_parser_streaming_alias_lookup(fyp, fyt_anchor);
+
+ /* we don't support forward aliases for now like we do in document mode ... */
+ FYP_TOKEN_ERROR_CHECK(fyp, fyt_anchor, FYEM_PARSE,
+ fysa, err_out,
+ "alias not found; note streaming mode does not support forward alias references");
+
+ /* if this is a reference to a collecting alias, then it's recursive */
+ FYP_TOKEN_ERROR_CHECK(fyp, fyt_anchor, FYEM_PARSE,
+ !fysa->collecting, err_out,
+ "Recursive alias reference detected");
+
+ fy_token_unref(fyt_anchor);
+ fyt_anchor = NULL;
+
+ fysas = fy_parse_streaming_alias_state_push(fyp, fysa);
+ fyp_error_check(fyp, fysas, err_out,
+ "fy_parse_streaming_alias_push() failed!");
+ assert(fysas->next);
+
+ fyep = fy_parse_eventp_clone(fyp, fysas->next, true);
+ fyp_error_check(fyp, fyep != NULL, err_out,
+ "fy_parse_eventp_clone() failed!");
+
+ /* advance event */
+ fy_parse_streaming_alias_state_next_event(fyp);
+
+ return fyep;
+
+err_out:
+ fy_token_unref(fyt_anchor);
+ fyp->stream_error = true;
+ return NULL;
+}
+
+struct fy_eventp *fy_parser_event_resolve_hook_anchor_start(struct fy_parser *fyp, struct fy_eventp *fyep)
+{
+ struct fy_token *fyt_anchor = NULL;
+ struct fy_streaming_alias *fysa = NULL;
+
+ if (!fyep)
+ return NULL;
+
+ /* is there an anchor?, is so get it and clear it */
+ fyt_anchor = fy_event_get_and_clear_anchor_token(&fyep->e);
+ if (!fyt_anchor)
+ return fyep;
+
+ fysa = fy_parse_streaming_alias_create(fyp, fyt_anchor);
+ fyp_error_check(fyp, fysa != NULL, err_out,
+ "fy_parser_streaming_alias_create() failed!");
+ fyt_anchor = NULL;
+
+ /* we just mark that we're collecting
+ * the collection hook will pick up the first event
+ * including the scalar one
+ */
+ fysa->collecting = true;
+ fysa->sequence_nest = 0;
+ fysa->mapping_nest = 0;
+
+ /* always add to the head of the list (overrides what follows with the same name) */
+ fy_streaming_alias_list_add(&fyp->streaming_aliases, fysa);
+
+ return fyep;
+
+err_out:
+ fy_token_unref(fyt_anchor);
+ fy_parse_streaming_alias_clean(fyp, fysa);
+ fy_parse_streaming_alias_recycle(fyp, fysa);
+ fy_parse_eventp_recycle(fyp, fyep);
+ return NULL;
+}
+
+static struct fy_document *
+fy_parser_get_merge_key_argument(struct fy_parser *fyp, struct fy_document_builder *fydb, struct fy_node *fyn)
+{
+ struct fy_streaming_alias *fysa;
+ struct fy_document *fyd_new = NULL;
+ struct fy_document_state *fyds = NULL;
+ int rc;
+
+ if (!fy_node_is_alias(fyn))
+ return NULL;
+
+ /* get the alias */
+ fysa = fy_parser_streaming_alias_lookup(fyp, fyn->scalar);
+ FYP_TOKEN_ERROR_CHECK(fyp, fyn->scalar, FYEM_PARSE,
+ fysa, err_out,
+ "Unable to find merge key argument alias");
+
+ /* if this is a reference to a collecting alias, then it's recursive */
+ FYP_TOKEN_ERROR_CHECK(fyp, fyn->scalar, FYEM_PARSE,
+ !fysa->collecting, err_out,
+ "merge key recursive alias reference detected");
+
+ fyds = fy_parser_get_document_state(fyp);
+ fyp_error_check(fyp, fyds, err_out,
+ "fy_parser_get_document_state() failed");
+
+ rc = fy_document_builder_set_in_document(fydb, fyds, true);
+ fyp_error_check(fyp, !rc, err_out,
+ "fy_document_builder_set_target_document() failed\n");
+
+ /* build a document from the events of the alias (must be the same) */
+ fyd_new = fy_document_builder_event_document(fydb, &fysa->events);
+ fyp_error_check(fyp, fyd_new, err_out,
+ "fy_document_builder_event_document() failed\n");
+
+ FYP_TOKEN_ERROR_CHECK(fyp, fyn->scalar, FYEM_PARSE,
+ fy_node_is_mapping(fyd_new->root), err_out,
+ "alias argument does not refer to mapping");
+
+ return fyd_new;
+
+err_out:
+ fy_document_destroy(fyd_new);
+ return NULL;
+}
+
+static struct fy_document *
+fy_parser_get_merge_key_document(struct fy_parser *fyp, struct fy_eventp *fyep)
+{
+ struct fy_event *fye;
+ struct fy_document_builder_cfg cfg;
+ struct fy_document_builder *fydb = NULL;
+ struct fy_document *fyd = NULL, *fyd_arg = NULL;
+ struct fy_document_state *fyds;
+ struct fy_node *fyn = NULL, *fyn_src = NULL;
+ struct fy_node *new_root = NULL;
+ struct fy_node_pair *fynp, *fynp_exists;
+ void *iters, *iterm;
+ int rc;
+
+ fye = &fyep->e;
+
+ memset(&cfg, 0, sizeof(cfg));
+ cfg.parse_cfg = fyp->cfg;
+ cfg.diag = fy_diag_ref(fyp->diag);
+
+ fydb = fy_document_builder_create(&cfg);
+ fyp_error_check(fyp, fydb, err_out,
+ "fy_document_builder_create() failed\n");
+
+ fyds = fy_parser_get_document_state(fyp);
+ /* start with this document state */
+ fyds = fy_parser_get_document_state(fyp);
+ rc = fy_document_builder_set_in_document(fydb, fyds, true);
+ fyp_error_check(fyp, !rc, err_out,
+ "fy_document_builder_set_in_document() failed\n");
+
+ fyd = fy_document_builder_load_document(fydb, fyp);
+ fyp_error_check(fyp, fyd, err_out,
+ "fy_document_builder_load_document() failed\n");
+
+ FYP_TOKEN_ERROR_CHECK(fyp, fy_event_get_token(fye), FYEM_PARSE,
+ fyd->root, err_out,
+ "merge key arguments must exist");
+
+ /* single alias, replace */
+ if (fy_node_is_alias(fyd->root)) {
+ fyd_arg = fy_parser_get_merge_key_argument(fyp, fydb, fyd->root);
+ fyp_error_check(fyp, fyd_arg, err_out,
+ "fy_parser_get_merge_key_argument() failed\n");
+ fy_node_free(fyd->root);
+ fyd->root = fy_node_copy(fyd, fyd_arg->root);
+
+ fy_document_destroy(fyd_arg);
+ fyd_arg = NULL;
+
+ } else if (fy_node_is_mapping(fyd->root)) {
+
+ /* nothing, already a mapping */
+
+ } else if (fy_node_is_sequence(fyd->root)) {
+
+ new_root = fy_node_create_mapping(fyd);
+ fyp_error_check(fyp, new_root, err_out,
+ "fy_node_create_mapping() failed\n");
+
+ iters = NULL;
+ while ((fyn_src = fy_node_sequence_iterate(fyd->root, &iters)) != NULL) {
+
+ if (fy_node_is_alias(fyn_src)) {
+ fyd_arg = fy_parser_get_merge_key_argument(fyp, fydb, fyn_src);
+ fyp_error_check(fyp, fyd_arg, err_out,
+ "fy_parser_get_merge_key_argument() failed\n");
+
+ fyn = fy_node_copy(fyd, fyd_arg->root);
+ fyp_error_check(fyp, fyn, err_out,
+ "fy_node_copy() failed\n");
+ fy_document_destroy(fyd_arg);
+ fyd_arg = NULL;
+
+ } else {
+ FYP_NODE_ERROR_CHECK(fyp, fyn_src, FYEM_PARSE,
+ fy_node_is_mapping(fyn_src), err_out,
+ "merge key argument is not a mapping or an alias to such");
+ fyn = fy_node_copy(fyd, fyn_src);
+ fyp_error_check(fyp, fyn, err_out,
+ "fy_node_copy() failed\n");
+ }
+
+ iterm = NULL;
+ while ((fynp = fy_node_mapping_iterate(fyn, &iterm)) != NULL) {
+ fynp_exists = fy_node_mapping_lookup_pair(new_root, fynp->key);
+
+ /* do not override existing keys */
+ if (fynp_exists)
+ continue;
+
+ /* add it to the mapping */
+ rc = fy_node_mapping_append(new_root, fy_node_copy(fyd, fynp->key), fy_node_copy(fyd, fynp->value));
+ fyp_error_check(fyp, !rc, err_out,
+ "fy_node_mapping_append() failed\n");
+ }
+
+ /* and we're done with it */
+ fy_node_free(fyn);
+ fyn = NULL;
+ }
+ fy_node_free(fyd->root);
+ fyd->root = new_root;
+
+ } else {
+ FYP_TOKEN_ERROR(fyp, fy_event_get_token(fye), FYEM_PARSE,
+ "merge key argument is neither an alias or a mapping");
+ goto err_out;
+ }
+
+ fy_document_builder_destroy(fydb);
+ fydb = NULL;
+
+ return fyd;
+
+err_out:
+ fy_node_free(new_root);
+ fy_document_destroy(fyd_arg);
+ fy_document_destroy(fyd);
+ fy_document_builder_destroy(fydb);
+ return NULL;
+}
+
+struct fy_eventp *fy_parser_event_resolve_hook_merge_key_start(struct fy_parser *fyp, struct fy_eventp *fyep)
+{
+ struct fy_document *fyd = NULL;
+ struct fy_document_iterator *fydi = NULL;
+ struct fy_eventp *fyep_next = NULL;
+ struct fy_event *fye;
+ struct fy_streaming_alias *fysa;
+ struct fy_streaming_alias_state *fysas;
+
+ FYP_TOKEN_ERROR_CHECK(fyp, fyep->e.scalar.value, FYEM_PARSE,
+ !fyp->mks.active, err_out,
+ "No recursive merge key processing is supported");
+
+ fyd = fy_parser_get_merge_key_document(fyp, fyep);
+ FYP_TOKEN_ERROR_CHECK(fyp, fyep->e.scalar.value, FYEM_PARSE,
+ fyd, err_out,
+ "No merge key argument document");
+
+ fysa = fy_parse_streaming_alias_create(fyp, NULL);
+ fyp_error_check(fyp, fysa, err_out,
+ "fy_parse_streaming_alias_create() failed");
+
+ fydi = fy_document_iterator_create();
+ fyp_error_check(fyp, fydi, err_out,
+ "fy_document_iterator_create() failed");
+
+ fy_document_iterator_node_start(fydi, fyd->root);
+
+ /* now append all to the (fake) streaming anchor */
+ while ((fye = fy_document_iterator_body_next(fydi)) != NULL) {
+ fyep_next = container_of(fye, struct fy_eventp, e);
+ fy_eventp_list_add_tail(&fysa->events, fyep_next);
+ }
+
+ fy_document_iterator_destroy(fydi);
+ fy_document_destroy(fyd);
+ fyd = NULL;
+
+ /* and add it to the start of the list */
+ fy_streaming_alias_list_add(&fyp->streaming_aliases, fysa);
+
+ /* OK, trim mapping start and end */
+ fyep_next = fy_eventp_list_head(&fysa->events);
+ fyp_error_check(fyp, fyep_next->e.type == FYET_MAPPING_START, err_out,
+ "not a mapping merge key start");
+ fy_eventp_list_del(&fysa->events, fyep_next);
+ fy_eventp_free(fyep_next);
+
+ fyep_next = fy_eventp_list_tail(&fysa->events);
+ fyp_error_check(fyp, fyep_next->e.type == FYET_MAPPING_END, err_out,
+ "not a mapping merge key end");
+ fy_eventp_list_del(&fysa->events, fyep_next);
+ fy_eventp_free(fyep_next);
+
+ /* OK, push the new state */
+ fysas = fy_parse_streaming_alias_state_push(fyp, fysa);
+ fyp_error_check(fyp, fysas, err_out,
+ "fy_parse_streaming_alias_push() failed!");
+ assert(fysas->next);
+
+ /* dispose of the event */
+ fy_parse_eventp_recycle(fyp, fyep);
+ fyep = NULL;
+
+ fyep = fy_parse_eventp_clone(fyp, fysas->next, true);
+ fyp_error_check(fyp, fyep != NULL, err_out,
+ "fy_parse_eventp_clone() failed!");
+
+ /* advance event */
+ fy_parse_streaming_alias_state_next_event(fyp);
+
+ return fyep;
+
+err_out:
+ fy_document_iterator_destroy(fydi);
+ fy_document_destroy(fyd);
+ fy_parse_eventp_recycle(fyp, fyep_next);
+ fy_parse_eventp_recycle(fyp, fyep);
+ return NULL;
+}
+
+struct fy_eventp *fy_parser_event_resolve_hook_merge_key(struct fy_parser *fyp, struct fy_eventp *fyep)
+{
+ /* will toggle the map key bit if it is in a mapping */
+ fy_parse_streaming_alias_collection_state_toggle_map(fyp);
+ switch (fyep->e.type) {
+ case FYET_SCALAR:
+ /* if a merge key is detected do the dance */
+ if (fy_parse_streaming_alias_collection_state_in_map_key(fyp) &&
+ fy_token_scalar_style(fyep->e.scalar.value) == FYSS_PLAIN &&
+ fy_plain_atom_streq(fy_token_atom(fyep->e.scalar.value), "<<"))
+ return fy_parser_event_resolve_hook_merge_key_start(fyp, fyep);
+ break;
+ case FYET_ALIAS:
+ /* nothing for alias */
+ break;
+ case FYET_MAPPING_START:
+ fy_parse_streaming_alias_collection_state_push(fyp, FYCTS_MAP_VAL); /* it will toggle to key at the first one */
+ break;
+ case FYET_SEQUENCE_START:
+ fy_parse_streaming_alias_collection_state_push(fyp, FYCTS_SEQ);
+ break;
+ case FYET_MAPPING_END:
+ case FYET_SEQUENCE_END:
+ fy_parse_streaming_alias_collection_state_pop(fyp);
+ break;
+ default:
+ break;
+ }
+
+ return fyep;
+}
+
+/* NOTE: order is _very_ important, do not re-arrange if you're not sure */
+struct fy_eventp *fy_parser_event_resolve_hook(struct fy_parser *fyp, struct fy_eventp *fyep)
+{
+ int rc;
+
+ /* merge key processing for version 1.1 only! */
+ if (fy_reader_get_mode(fyp->reader) == fyrm_yaml_1_1) {
+ fyep = fy_parser_event_resolve_hook_merge_key(fyp, fyep);
+ fyp_error_check(fyp, fyep != NULL, err_out,
+ "fy_parser_event_resolve_hook_collection_tracking() failed!");
+ }
+
+ /* anchor marking */
+ fyep = fy_parser_event_resolve_hook_anchor_start(fyp, fyep);
+ fyp_error_check(fyp, fyep != NULL, err_out,
+ "fy_parser_event_resolve_hook_anchor_start() failed!");
+
+ /* collect afterwards */
+ rc = fy_parser_event_resolve_hook_collect(fyp, fyep);
+ fyp_error_check(fyp, !rc, err_out,
+ "fy_parser_event_resolve_hook_collect() failed!");
+
+ /* finally do first entry into alias processing */
+ fyep = fy_parser_event_resolve_hook_alias(fyp, fyep);
+ fyp_error_check(fyp, fyep != NULL, err_out,
+ "fy_parser_event_resolve_hook_alias() failed!");
+
+ return fyep;
+
+err_out:
+ return NULL;
+}
+
+struct fy_eventp *fy_parser_parse_resolve_prolog(struct fy_parser *fyp)
+{
+ struct fy_streaming_alias *fysa;
+ struct fy_streaming_alias_state *fysas;
+ struct fy_eventp *fyep, *fyep_src;
+ bool was_merge_key;
+ int rc;
+
+ if (!(fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT))
+ return NULL;
+
+ fysas = fy_parse_streaming_alias_state_top(fyp);
+ if (!fysas)
+ return NULL;
+
+ was_merge_key = false;
+ for (;;) {
+ fysa = fysas->fysa;
+ assert(fysa);
+
+ assert(fysas->next != NULL);
+
+ /* keep track of it */
+ fyep_src = fysas->next;
+
+ /* merge key entries don't have an anchor */
+ was_merge_key = fysa->anchor == NULL;
+
+ /* advance event */
+ fy_parse_streaming_alias_state_next_event(fyp);
+
+ /* not an alias, continue normally */
+ if (fyep_src->e.type != FYET_ALIAS)
+ break;
+
+ /* get the anchor that was defined closest to this one */
+ fysa = fy_parser_streaming_alias_lookup_pivot(fyp, fysa, fyep_src->e.alias.anchor);
+ fyp_error_check(fyp, fysa, err_out,
+ "fy_parser_streaming_alias_lookup_pivot() failed!");
+
+ /* and push */
+ fysas = fy_parse_streaming_alias_state_push(fyp, fysa);
+ fyp_error_check(fyp, fysas, err_out,
+ "fy_parse_streaming_alias_state_push() failed!");
+ }
+
+ fyep = fy_parse_eventp_clone(fyp, fyep_src, true);
+ fyp_error_check(fyp, fyep != NULL, err_out,
+ "fy_parse_eventp_clone() failed!");
+
+ if (was_merge_key) {
+ rc = fy_parser_event_resolve_hook_collect(fyp, fyep_src);
+ fyp_error_check(fyp, !rc, err_out,
+ "fy_parser_event_resolve_hook_collect() failed!");
+ }
+
+ return fyep;
+
+err_out:
+ fyp->stream_error = true;
+ return NULL;
+}
+
+struct fy_eventp *fy_parser_parse_resolve_epilog(struct fy_parser *fyp, struct fy_eventp *fyep)
+{
+ if (!(fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT))
+ return fyep;
+
+ fyep = fy_parser_event_resolve_hook(fyp, fyep);
+ if (!fyep)
+ fyp->stream_error = true;
+
+ return fyep;
+}
struct fy_parser;
struct fy_input;
+/* for the event */
+FY_PARSE_TYPE_DECL_ALLOC(eventp);
+
enum fy_flow_type {
FYFT_NONE,
FYFT_MAP,
};
FY_PARSE_TYPE_DECL(parse_state_log);
+struct fy_streaming_alias {
+ struct list_head node;
+ struct fy_token *anchor;
+ bool collecting;
+ long mapping_nest;
+ long sequence_nest;
+ struct fy_eventp_list events;
+};
+FY_PARSE_TYPE_DECL(streaming_alias);
+
+struct fy_streaming_alias_state {
+ struct fy_streaming_alias *fysa;
+ struct fy_eventp *next;
+};
+
+/* 2 bits per collection tracking state
+ * pos: * 2
+ * bit 0 - 0 map, 1 seq
+ * bit 1 - 0 key, 1 val (if map)
+ */
+#define FYCTS_SCALAR 0
+#define FYCTS_SEQ 1
+#define FYCTS_MAP 2
+#define FYCTS_MAP_KEY 2
+#define FYCTS_MAP_VAL 3
+#define FYCTS_MAP_TOGGLE 1
+
struct fy_parser {
struct fy_parse_cfg cfg;
bool bare_document_only : 1; /* no document start indicators allowed, no directives */
bool stream_has_content : 1;
bool parse_flow_only : 1; /* document is in flow form, and stop parsing at the end */
+ bool parse_block_only : 1; /* document is in embedded block form, and stop parsing at the end */
bool colon_follows_colon : 1; /* "foo"::bar -> "foo": :bar */
bool had_directives : 1; /* document had directives */
int flow_level;
int indent;
int parent_indent;
int indent_line;
+ int starting_indent;
/* simple key stack */
struct fy_simple_key_list simple_keys;
/* state stack */
struct fy_simple_key_list recycled_simple_key;
struct fy_parse_state_log_list recycled_parse_state_log;
struct fy_flow_list recycled_flow;
+ struct fy_streaming_alias_list recycled_streaming_alias;
struct fy_eventp_list recycled_eventp;
struct fy_token_list recycled_token;
/* last generated event atom */
struct fy_atom last_event_handle;
+
+ struct fy_streaming_alias_list streaming_aliases;
+ /* streaming alias state */
+ struct {
+ int alloc;
+ int top;
+ struct fy_streaming_alias_state *stack;
+ struct fy_streaming_alias_state local[8];
+ } sas;
+ /* collection tracking state */
+ struct {
+ int alloc;
+ int top;
+ uint32_t *stack;
+ uint32_t local[8]; /* 8 * 32 = 256 / 2 = 128 levels before needing dynamic allocation */
+ } cts;
+ /* merge key state */
+ struct {
+ bool active;
+ struct fy_eventp_list args;
+ struct fy_document *fyd;
+ } mks;
};
static inline struct fy_input *
fyp->parse_flow_only = flow_only_mode;
}
+static inline void
+fy_parser_set_block_only_mode(struct fy_parser *fyp, bool block_only_mode)
+{
+ fyp->parse_block_only = block_only_mode;
+}
+
#define fy_fill_atom_a(_fyp, _advance) \
fy_fill_atom((_fyp), (_advance), alloca(sizeof(struct fy_atom)))
void fy_parse_cleanup(struct fy_parser *fyp);
int fy_parse_input_append(struct fy_parser *fyp, const struct fy_input_cfg *fyic);
+ssize_t fy_parse_estimate_queued_input_size(struct fy_parser *fyp);
struct fy_eventp *fy_parse_private(struct fy_parser *fyp);
int fy_parse_set_composer(struct fy_parser *fyp, fy_parse_composer_cb cb, void *userdata);
+struct fy_streaming_alias *
+fy_parse_streaming_alias_create(struct fy_parser *fyp, struct fy_token *fyt_anchor);
+void fy_parse_streaming_alias_clean(struct fy_parser *fyp, struct fy_streaming_alias *fysa);
+void fy_parse_streaming_aliases_reset(struct fy_parser *fyp);
+
+struct fy_eventp *fy_parser_parse_resolve_prolog(struct fy_parser *fyp);
+struct fy_eventp *fy_parser_parse_resolve_epilog(struct fy_parser *fyp, struct fy_eventp *fyep);
+
#endif
fypc->user_data = data;
}
+
+void *fy_path_get_parent_user_data(struct fy_path *fypp)
+{
+ struct fy_path_component *parent;
+
+ if (fy_path_in_root(fypp))
+ return fy_path_get_root_user_data(fypp);
+
+ parent = fy_path_last_not_collection_root_component(fypp);
+ if (fy_path_in_sequence(fypp))
+ return fy_path_component_get_sequence_user_data(parent);
+ else
+ return fy_path_component_get_mapping_user_data(parent);
+}
+
+void fy_path_set_parent_user_data(struct fy_path *fypp, void *data)
+{
+ struct fy_path_component *parent;
+
+ if (fy_path_in_root(fypp)) {
+ fy_path_set_root_user_data(fypp, data);
+ } else {
+ parent = fy_path_last_not_collection_root_component(fypp);
+ if (fy_path_in_sequence(fypp))
+ fy_path_component_set_sequence_user_data(parent, data);
+ else
+ fy_path_component_set_mapping_user_data(parent, data);
+ }
+}
+
+void *fy_path_get_last_user_data(struct fy_path *fypp)
+{
+ struct fy_path_component *last;
+
+ last = fy_path_last_component(fypp);
+ if (!last)
+ return NULL;
+
+ if (last->type == FYPCT_SEQ)
+ return fy_path_component_get_sequence_user_data(last);
+ else
+ return fy_path_component_get_mapping_user_data(last);
+}
+
+void fy_path_set_last_user_data(struct fy_path *fypp, void *data)
+{
+ struct fy_path_component *last;
+
+ last = fy_path_last_component(fypp);
+ if (!last)
+ return;
+
+ if (last->type == FYPCT_SEQ)
+ fy_path_component_set_sequence_user_data(last, data);
+ else
+ fy_path_component_set_mapping_user_data(last, data);
+}
free(fyt->tag.suffix0);
fyt->tag.suffix0 = NULL;
}
+ if (fyt->tag.short0) {
+ free(fyt->tag.short0);
+ fyt->tag.short0 = NULL;
+ }
break;
case FYTT_TAG_DIRECTIVE:
fyt->tag.fyt_td = fy_token_ref(fyt_td);
fyt->tag.handle0 = NULL;
fyt->tag.suffix0 = NULL;
+ fyt->tag.short0 = NULL;
break;
case FYTT_VERSION_DIRECTIVE:
int fy_token_text_analyze(struct fy_token *fyt)
{
- const char *s, *e;
- const char *value = NULL;
+ struct fy_atom_iter iter;
enum fy_atom_style style;
- int c, w, cn, cnn, cp, col;
- size_t len;
+ int c, cn, cnn, cp, col;
+ uint8_t col0si, col0ei; /* mask for --- ... at indent 0 */
int flags;
if (!fyt)
if (!fy_atom_style_is_block(style))
flags |= FYTTAF_DIRECT_OUTPUT;
- /* get value */
- value = fy_token_get_text(fyt, &len);
- if (!value || len == 0) {
+ fy_atom_iter_start(&fyt->handle, &iter);
+
+ col = 0;
+
+ /* get first character */
+ cn = fy_atom_iter_utf8_get(&iter);
+ if (cn < 0) {
+ /* empty? */
flags |= FYTTAF_EMPTY | FYTTAF_CAN_BE_DOUBLE_QUOTED | FYTTAF_CAN_BE_UNQUOTED_PATH_KEY;
- fyt->analyze_flags = flags;
- return flags;
+ goto out;
}
flags |= FYTTAF_CAN_BE_PLAIN |
FYTTAF_CAN_BE_PLAIN_FLOW |
FYTTAF_CAN_BE_UNQUOTED_PATH_KEY;
- /* start with document indicators must be quoted at indent 0 */
- if (len >= 3 && (!memcmp(value, "---", 3) || !memcmp(value, "...", 3)))
- flags |= FYTTAF_QUOTE_AT_0;
-
- s = value;
- e = value + len;
-
- col = 0;
-
- /* get first character */
- cn = fy_utf8_get(s, e - s, &w);
- s += w;
- col = fy_token_is_lb(fyt, cn) ? 0 : (col + 1);
+ col0si = col0ei = 0;
/* disable folded right off the bat, it's a pain */
flags &= ~FYTTAF_CAN_BE_FOLDED;
flags &= ~FYTTAF_CAN_BE_PLAIN_FLOW;
if ((flags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_CAN_BE_PLAIN_FLOW))) {
- cnn = fy_utf8_get(s, e - s, &w);
- if (fy_is_ws(cnn) && fy_is_indicator_before_space(cn))
+ cnn = fy_atom_iter_utf8_peek(&iter);
+ if (fy_is_blankz_m(cnn, fy_token_atom_lb_mode(fyt)) && fy_is_indicator_before_space(cn))
flags &= ~(FYTTAF_CAN_BE_PLAIN | FYTTAF_CAN_BE_PLAIN_FLOW);
}
flags &= ~FYTTAF_CAN_BE_UNQUOTED_PATH_KEY;
cp = -1;
- for (c = cn; c >= 0; s += w, cp = c, c = cn) {
+ for (c = cn; c >= 0; cp = c, c = cn) {
+
+ if (col <= 2) {
+ if (cn == '-') {
+ col0si |= (uint8_t)1 << col;
+ if (col0si == 7)
+ flags |= FYTTAF_HAS_START_IND | FYTTAF_QUOTE_AT_0;
+ } else if (cn == '.') {
+ col0ei |= (uint8_t)1 << col;
+ if (col0ei == 7)
+ flags |= FYTTAF_HAS_END_IND | FYTTAF_QUOTE_AT_0;
+ }
+ }
/* can be -1 on end */
- cn = fy_utf8_get(s, e - s, &w);
+ cn = fy_atom_iter_utf8_get(&iter);
/* zero can't be output, only in double quoted mode */
if (c == 0) {
flags &= ~FYTTAF_CAN_BE_PLAIN_FLOW;
/* non printable characters, turn off these styles */
- if ((flags & (FYTTAF_CAN_BE_SINGLE_QUOTED |
- FYTTAF_CAN_BE_LITERAL |
- FYTTAF_CAN_BE_FOLDED)) && !fy_is_print(c))
- flags &= ~(FYTTAF_CAN_BE_SINGLE_QUOTED |
- FYTTAF_CAN_BE_LITERAL |
+ if (!fy_is_print(c)) {
+ flags &= ~(FYTTAF_CAN_BE_SINGLE_QUOTED | FYTTAF_CAN_BE_LITERAL |
FYTTAF_CAN_BE_FOLDED);
+ flags |= FYTTAF_HAS_NON_PRINT;
+ }
/* if there's an escape, it can't be direct */
if ((flags & FYTTAF_DIRECT_OUTPUT) &&
(style == FYAS_DOUBLE_QUOTED && c == '\\')))
flags &= ~FYTTAF_DIRECT_OUTPUT;
- col = fy_token_is_lb(fyt, c) ? 0 : (col + 1);
+ if (fy_token_is_lb(fyt, c)) {
+ col = 0;
+ col0si = col0ei = 0;
+ } else
+ col++;
+
+ if (fy_is_any_lb(c))
+ flags |= FYTTAF_HAS_ANY_LB;
/* last character */
if (cn < 0) {
/* if ends with whitespace or linebreak, can't be plain */
- if (fy_is_ws(cn) || fy_token_is_lb(fyt, cn))
+ if (fy_is_ws(c) || fy_token_is_lb(fyt, c))
flags &= ~(FYTTAF_CAN_BE_PLAIN |
FYTTAF_CAN_BE_PLAIN_FLOW);
}
}
-
+out:
+ fy_atom_iter_finish(&iter);
fyt->analyze_flags = flags;
return flags;
}
return fyt->tag.suffix0;
}
+const char *fy_tag_token_short(struct fy_token *fyt, size_t *lenp)
+{
+ const char *handle, *suffix;
+ size_t handle_len, suffix_len;
+ char *text0;
+
+ if (!fyt || fyt->type != FYTT_TAG)
+ return NULL;
+
+ /* use the cache if it's there (and doesn't need a rebuild) */
+ if (fyt->tag.short0 && !fy_token_text_needs_rebuild(fyt))
+ return fyt->tag.short0;
+
+ if (fyt->tag.short0) {
+ free(fyt->tag.short0);
+ fyt->tag.short0 = NULL;
+ }
+
+ handle = fy_tag_token_handle(fyt, &handle_len);
+ if (!handle)
+ return NULL;
+
+ suffix = fy_tag_token_suffix(fyt, &suffix_len);
+ if (!suffix)
+ return NULL;
+
+ text0 = malloc(handle_len + suffix_len + 1);
+ if (!text0)
+ return NULL;
+ memcpy(text0, handle, handle_len);
+ memcpy(text0 + handle_len, suffix, suffix_len);
+ text0[handle_len + suffix_len] = '\0';
+
+ fyt->tag.short0 = text0;
+ fyt->tag.short_length = handle_len + suffix_len;
+
+ *lenp = fyt->tag.short_length;
+
+ return fyt->tag.short0;
+}
+
+const char *fy_tag_token_short0(struct fy_token *fyt)
+{
+ const char *text;
+ size_t len;
+
+ text = fy_tag_token_short(fyt, &len);
+ if (!text)
+ return NULL;
+ return text;
+}
+
const struct fy_version * fy_version_directive_token_version(struct fy_token *fyt)
{
if (!fyt || fyt->type != FYTT_VERSION_DIRECTIVE)
const char *t1, *t2;
size_t l1, l2, l;
int ret;
+ bool aoa; /* anchor or alias */
/* handles both NULL */
if (fyt1 == fyt2)
if (fyt1 && !fyt2)
return 1;
+ /* special case for comparing anchors and aliases */
+ aoa = (fyt1->type == FYTT_ANCHOR || fyt1->type == FYTT_ALIAS) &&
+ (fyt2->type == FYTT_ANCHOR || fyt2->type == FYTT_ALIAS);
+
/* tokens with different types can't be equal */
- if (fyt1->type != fyt2->type)
+ if (!aoa && fyt1->type != fyt2->type)
return fyt2->type > fyt1->type ? -1 : 1;
/* special case, these can't use the atom comparisons */
return -1;
/* first try the pushed ungetc */
- if (iter->unget_c != -1) {
+ if (iter->unget_c >= 0) {
c = iter->unget_c;
+ /* unmatched getc/ungetc */
+ if (fy_utf8_width(c) != 1)
+ return -1;
iter->unget_c = -1;
return c;
}
int fy_token_iter_ungetc(struct fy_token_iter *iter, int c)
{
- if (iter->unget_c != -1)
+ if (!iter || c >= 0x80)
+ return -1;
+
+ if (iter->unget_c >= 0)
return -1;
- if (c == -1) {
+
+ if (c < 0) {
iter->unget_c = -1;
return 0;
}
- iter->unget_c = c & 0xff;
- return c & 0xff;
+ iter->unget_c = c;
+ return c;
}
int fy_token_iter_peekc(struct fy_token_iter *iter)
{
int c, w, w1;
+ if (!iter)
+ return -1;
+
/* first try the pushed ungetc */
- if (iter->unget_c != -1) {
+ if (iter->unget_c >= 0) {
c = iter->unget_c;
iter->unget_c = -1;
return c;
int fy_token_iter_utf8_unget(struct fy_token_iter *iter, int c)
{
- if (iter->unget_c != -1)
+ if (!iter)
+ return -1;
+
+ if (iter->unget_c >= 0)
return -1;
- if (c == -1) {
+ if (c < 0) {
iter->unget_c = -1;
return 0;
}
+ if (!fy_utf8_is_valid(c))
+ return -1;
iter->unget_c = c;
return c;
struct fy_token *fyt_td;
char *handle0; /* zero terminated and allocated, only used by binding */
char *suffix0;
+ unsigned int short_length;
+ char *short0; /* zero terminated and allocated for when the short tag is requested */
struct fy_tag tag; /* prefix is now suffix */
} tag;
struct {
#define FYTTAF_CAN_BE_PLAIN_FLOW FY_BIT(15)
#define FYTTAF_QUOTE_AT_0 FY_BIT(16)
#define FYTTAF_CAN_BE_UNQUOTED_PATH_KEY FY_BIT(17)
+#define FYTTAF_HAS_ANY_LB FY_BIT(18) /* any LB including unicode, not per input */
+#define FYTTAF_HAS_START_IND FY_BIT(19) /* has --- at col 0 */
+#define FYTTAF_HAS_END_IND FY_BIT(20) /* has ... at col 0 */
+#define FYTTAF_HAS_NON_PRINT FY_BIT(21) /* has any non printable utf8 */
int fy_token_text_analyze(struct fy_token *fyt);
+++ /dev/null
-/*
- * fy-typelist.h - typed list method builders
- *
- * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
- *
- * SPDX-License-Identifier: MIT
- */
-#ifndef FY_TYPELIST_H
-#define FY_TYPELIST_H
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdint.h>
-#include <stdbool.h>
-#include <stdio.h>
-
-#include <libfyaml.h>
-
-#include "fy-list.h"
-
-/* declare type methods */
-
-#define FY_TYPE_FWD_DECL_LIST(_type) \
-/* type safe list wrapper */ \
-struct fy_ ## _type ## _list { struct list_head _lh; }; \
-\
-struct __useless_struct_to_allow_semicolon
-
-#define FY_TYPE_DECL_LIST(_type) \
-static inline void fy_ ## _type ## _list_init(struct fy_ ## _type ## _list *_l) \
-{ \
- INIT_LIST_HEAD(&_l->_lh); \
-} \
-static inline void fy_ ## _type ## _list_add(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
-{ \
- if (_l && _n) \
- list_add(&_n->node, &_l->_lh); \
-} \
-static inline void fy_ ## _type ## _list_add_tail(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
-{ \
- if (_l && _n) \
- list_add_tail(&_n->node, &_l->_lh); \
-} \
-static inline void fy_ ## _type ## _list_push(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
-{ \
- if (_l && _n) \
- fy_ ## _type ## _list_add(_l, _n); \
-} \
-static inline void fy_ ## _type ## _list_push_tail(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
-{ \
- if (_l && _n) \
- fy_ ## _type ## _list_add_tail(_l, _n); \
-} \
-static inline bool fy_ ## _type ## _list_empty(struct fy_ ## _type ## _list *_l) \
-{ \
- return _l ? list_empty(&_l->_lh) : true; \
-} \
-static inline bool fy_ ## _type ## _list_is_singular(struct fy_ ## _type ## _list *_l) \
-{ \
- return _l ? list_is_singular(&_l->_lh) : true; \
-} \
-static inline void fy_ ## _type ## _list_del(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
-{ \
- if (_l && _n) { \
- list_del(&_n->node); \
- INIT_LIST_HEAD(&_n->node); \
- } \
-} \
-static inline void fy_ ## _type ## _list_insert_after(struct fy_ ## _type ## _list *_l, \
- struct fy_ ## _type *_p, struct fy_ ## _type *_n) \
-{ \
- if (_l && _p && _n) \
- list_add(&_n->node, &_p->node); \
-} \
-static inline void fy_ ## _type ## _list_insert_before(struct fy_ ## _type ## _list *_l, \
- struct fy_ ## _type *_p, struct fy_ ## _type *_n) \
-{ \
- if (_l && _p && _n) \
- list_add_tail(&_n->node, &_p->node); \
-} \
-static inline struct fy_ ## _type *fy_ ## _type ## _list_head(struct fy_ ## _type ## _list *_l) \
-{ \
- return !fy_ ## _type ## _list_empty(_l) ? list_first_entry(&_l->_lh, struct fy_ ## _type, node) : NULL; \
-} \
-static inline struct fy_ ## _type *fy_ ## _type ## _list_tail(struct fy_ ## _type ## _list *_l) \
-{ \
- return !fy_ ## _type ## _list_empty(_l) ? list_last_entry(&_l->_lh, struct fy_ ## _type, node) : NULL; \
-} \
-static inline struct fy_ ## _type *fy_ ## _type ## _list_first(struct fy_ ## _type ## _list *_l) \
-{ \
- return fy_ ## _type ## _list_head(_l); \
-} \
-static inline struct fy_ ## _type *fy_ ## _type ## _list_last(struct fy_ ## _type ## _list *_l) \
-{ \
- return fy_ ## _type ## _list_tail(_l); \
-} \
-static inline struct fy_ ## _type *fy_ ## _type ## _list_pop(struct fy_ ## _type ## _list *_l) \
-{ \
- struct fy_ ## _type *_n; \
- \
- _n = fy_ ## _type ## _list_head(_l); \
- if (!_n) \
- return NULL; \
- fy_ ## _type ## _list_del(_l, _n); \
- return _n; \
-} \
-static inline struct fy_ ## _type *fy_ ## _type ## _list_pop_tail(struct fy_ ## _type ## _list *_l) \
-{ \
- struct fy_ ## _type *_n; \
- \
- _n = fy_ ## _type ## _list_tail(_l); \
- if (!_n) \
- return NULL; \
- fy_ ## _type ## _list_del(_l, _n); \
- return _n; \
-} \
-static inline struct fy_ ## _type *fy_ ## _type ## _next(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
-{ \
- if (!_n || !_l || _n->node.next == &_l->_lh) \
- return NULL; \
- return list_entry(_n->node.next, struct fy_ ## _type, node); \
-} \
-static inline struct fy_ ## _type *fy_ ## _type ## _prev(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
-{ \
- if (!_n || !_l || _n->node.prev == &_l->_lh) \
- return NULL; \
- return list_entry(_n->node.prev, struct fy_ ## _type, node); \
-} \
-static inline void fy_ ## _type ## _lists_splice( \
- struct fy_ ## _type ## _list *_l, \
- struct fy_ ## _type ## _list *_lfrom) \
-{ \
- /* check arguments for sanity and lists are not empty */ \
- if (!_l || !_lfrom || \
- fy_ ## _type ## _list_empty(_lfrom)) \
- return; \
- list_splice(&_lfrom->_lh, &_l->_lh); \
-} \
-static inline void fy_ ## _type ## _list_splice_after( \
- struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n, \
- struct fy_ ## _type ## _list *_lfrom) \
-{ \
- /* check arguments for sanity and lists are not empty */ \
- if (!_l || !_n || !_lfrom || \
- fy_ ## _type ## _list_empty(_lfrom)) \
- return; \
- list_splice(&_lfrom->_lh, &_n->node); \
-} \
-static inline void fy_ ## _type ## _list_splice_before( \
- struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n, \
- struct fy_ ## _type ## _list *_lfrom) \
-{ \
- /* check arguments for sanity and lists are not empty */ \
- if (!_l || !_n || !_lfrom || \
- fy_ ## _type ## _list_empty(_lfrom)) \
- return; \
- list_splice(&_lfrom->_lh, _n->node.prev); \
-} \
-struct __useless_struct_to_allow_semicolon
-
-#endif
FY_PARSE_TYPE_DEFINE_SIMPLE(simple_key);
FY_PARSE_TYPE_DEFINE_SIMPLE(parse_state_log);
FY_PARSE_TYPE_DEFINE_SIMPLE(flow);
+FY_PARSE_TYPE_DEFINE_SIMPLE(streaming_alias);
+++ /dev/null
-/*
- * fy-utf8.c - utf8 handling methods
- *
- * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
- *
- * SPDX-License-Identifier: MIT
- */
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdbool.h>
-#include <string.h>
-#include <stdlib.h>
-
-#include <libfyaml.h>
-
-#include "fy-utf8.h"
-
-const int8_t fy_utf8_width_table[32] = {
- 1, 1, 1, 1, 1, 1, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1,
- 0, 0, 0, 0, 0, 0, 0, 0,
- 2, 2, 2, 2, 3, 3, 4, 0,
-};
-
-int fy_utf8_get_generic(const void *ptr, int left, int *widthp)
-{
- const uint8_t *p = ptr;
- int i, width, value;
-
- if (left < 1)
- return FYUG_EOF;
-
- /* this is the slow path */
- width = fy_utf8_width_by_first_octet(p[0]);
- if (!width)
- return FYUG_INV;
- if (width > left)
- return FYUG_PARTIAL;
-
- /* initial value */
- value = *p++ & (0xff >> width);
- for (i = 1; i < width; i++) {
- if ((*p & 0xc0) != 0x80)
- return FYUG_INV;
- value = (value << 6) | (*p++ & 0x3f);
- }
-
- /* check for validity */
- if ((width == 4 && value < 0x10000) ||
- (width == 3 && value < 0x800) ||
- (width == 2 && value < 0x80) ||
- (value >= 0xd800 && value <= 0xdfff) || value >= 0x110000)
- return FYUG_INV;
-
- *widthp = width;
-
- return value;
-}
-
-int fy_utf8_get_right_generic(const void *ptr, int left, int *widthp)
-{
- const uint8_t *s, *e;
- uint8_t v;
-
- s = ptr;
- e = s + left;
-
- if (left < 1)
- return FYUG_EOF;
-
- /* single byte sequence */
- v = e[-1];
- if ((v & 0x80) == 0) {
- if (widthp)
- *widthp = 1;
- return (int)v & 0x7f;
- }
-
- /* the last byte must be & 0xc0 == 0x80 */
- if ((v & 0xc0) != 0x80)
- return FYUG_INV;
-
- /* at least two byte sequence */
- if (left < 2)
- return FYUG_EOF;
-
- v = e[-2];
- /* the first byte of the sequence (must be a two byte sequence) */
- if ((v & 0xc0) != 0x80) {
- /* two byte start is 110x_xxxx */
- if ((v & 0xe0) != 0xc0)
- return FYUG_INV;
- return fy_utf8_get(e - 2, 2, widthp);
- }
-
- /* at least three byte sequence */
- if (left < 3)
- return FYUG_EOF;
-
- v = e[-3];
- /* the first byte of the sequence (must be a three byte sequence) */
- if ((v & 0xc0) != 0x80) {
- /* three byte start is 1110_xxxx */
- if ((v & 0xf0) != 0xe0)
- return FYUG_INV;
- return fy_utf8_get(e - 3, 3, widthp);
- }
-
- /* at least four byte sequence */
- if (left < 4)
- return FYUG_EOF;
-
- v = e[-4];
-
- /* the first byte of the sequence (must be a four byte sequence) */
- /* four byte start is 1111_0xxx */
- if ((v & 0xf8) != 0xf0) {
- return FYUG_INV;
- }
- return fy_utf8_get(e - 4, 4, widthp);
-}
-
-struct fy_utf8_fmt_esc_map {
- const int *ch;
- const int *map;
-};
-
-static const struct fy_utf8_fmt_esc_map esc_all = {
- .ch = (const int []){ '\\', '\0', '\b', '\r', '\t', '\f', '\n', '\v', '\a', '\e', 0x85, 0xa0, 0x2028, 0x2029, -1 },
- .map = (const int []){ '\\', '0', 'b', 'r', 't', 'f', 'n', 'v', 'a', 'e', 'N', '_', 'L', 'P', 0 }
-};
-
-static inline int esc_map(const struct fy_utf8_fmt_esc_map *esc_map, int c)
-{
- const int *ch;
- int cc;
-
- ch = esc_map->ch;
- while ((cc = *ch++) >= 0) {
- if (cc == c)
- return esc_map->map[(ch - esc_map->ch) - 1];
- }
- return -1;
-}
-
-static inline int fy_utf8_esc_map(int c, enum fy_utf8_escape esc)
-{
- if (esc == fyue_none)
- return -1;
- if (esc == fyue_singlequote && c == '\'')
- return '\'';
- if (fy_utf8_escape_is_any_doublequote(esc) && c == '"')
- return '"';
- return esc_map(&esc_all, c);
-}
-
-int fy_utf8_format_text_length(const char *buf, size_t len,
- enum fy_utf8_escape esc)
-{
- int c, w, l;
- const char *s, *e;
-
- s = buf;
- e = buf + len;
- l = 0;
- while (s < e) {
- c = fy_utf8_get(s, e - s, &w);
- if (!w || c < 0)
- break;
- s += w;
- if (fy_utf8_esc_map(c, esc))
- w = 2;
- l += w;
- }
- return l + 1;
-}
-
-char *fy_utf8_format_text(const char *buf, size_t len,
- char *out, size_t maxsz,
- enum fy_utf8_escape esc)
-{
- int c, w, cc;
- const char *s, *e;
- char *os, *oe;
-
- s = buf;
- e = buf + len;
- os = out;
- oe = out + maxsz - 1;
- while (s < e) {
- c = fy_utf8_get(s, e - s, &w);
- if (!w || c < 0)
- break;
- s += w;
-
- if ((cc = fy_utf8_esc_map(c, esc)) > 0) {
- if (os + 2 > oe)
- break;
- *os++ = '\\';
- *os++ = cc;
- continue;
- }
-
- if (os + w > oe)
- break;
-
- os = fy_utf8_put_unchecked(os, c);
- }
- *os++ = '\0';
- return out;
-}
-
-char *fy_utf8_format(int c, char *buf, enum fy_utf8_escape esc)
-{
- int cc;
- char *s;
-
- if (!fy_utf8_is_valid(c)) {
- *buf = '\0';
- return buf;
- }
-
- s = buf;
- if ((cc = fy_utf8_esc_map(c, esc)) > 0) {
- *s++ = '\\';
- *s++ = cc;
- } else
- s = fy_utf8_put_unchecked(s, c);
- *s = '\0';
- return buf;
-}
-
-char *fy_utf8_format_text_alloc(const char *buf, size_t len, enum fy_utf8_escape esc)
-{
- int outsz;
- char *out;
-
- outsz = fy_utf8_format_text_length(buf, len, esc);
- if (outsz < 0)
- return NULL;
- out = malloc(outsz);
- if (!out)
- return NULL;
- fy_utf8_format_text(buf, len, out, outsz, esc);
-
- return out;
-}
-
-const void *fy_utf8_memchr_generic(const void *s, int c, size_t n)
-{
- int cc, w;
- const void *e;
-
- e = s + n;
- while (s < e && (cc = fy_utf8_get(s, e - s, &w)) >= 0) {
- if (c == cc)
- return s;
- s += w;
- }
-
- return NULL;
-}
-
-/* parse an escape and return utf8 value */
-int fy_utf8_parse_escape(const char **strp, size_t len, enum fy_utf8_escape esc)
-{
- const char *s, *e;
- char c;
- int i, value, code_length, cc, w;
- unsigned int hi_surrogate, lo_surrogate;
-
- /* why do you bother us? */
- if (esc == fyue_none)
- return -1;
-
- if (!strp || !*strp || len < 2)
- return -1;
-
- value = -1;
-
- s = *strp;
- e = s + len;
-
- c = *s++;
-
- if (esc == fyue_singlequote) {
- if (c != '\'')
- goto out;
- c = *s++;
- if (c != '\'')
- goto out;
-
- value = '\'';
- goto out;
- }
-
- /* get '\\' */
- if (c != '\\')
- goto out;
-
- c = *s++;
-
- /* common YAML & JSON escapes */
- switch (c) {
- case 'b':
- value = '\b';
- break;
- case 'f':
- value = '\f';
- break;
- case 'n':
- value = '\n';
- break;
- case 'r':
- value = '\r';
- break;
- case 't':
- value = '\t';
- break;
- case '"':
- value = '"';
- break;
- case '/':
- value = '/';
- break;
- case '\\':
- value = '\\';
- break;
- default:
- break;
- }
-
- if (value >= 0)
- goto out;
-
- if (esc == fyue_doublequote || esc == fyue_doublequote_yaml_1_1) {
- switch (c) {
- case '0':
- value = '\0';
- break;
- case 'a':
- value = '\a';
- break;
- case '\t':
- value = '\t';
- break;
- case 'v':
- value = '\v';
- break;
- case 'e':
- value = '\e';
- break;
- case ' ':
- value = ' ';
- break;
- case 'N':
- value = 0x85; /* NEL */
- break;
- case '_':
- value = 0xa0;
- break;
- case 'L':
- value = 0x2028; /* LS */
- break;
- case 'P': /* PS 0x2029 */
- value = 0x2029; /* PS */
- break;
- default:
- /* weird unicode escapes */
- if ((uint8_t)c >= 0x80) {
- /* in non yaml-1.1 mode we don't allow this craziness */
- if (esc == fyue_doublequote)
- goto out;
-
- cc = fy_utf8_get(s - 1, e - (s - 1), &w);
- switch (cc) {
- case 0x2028:
- case 0x2029:
- case 0x85:
- case 0xa0:
- value = cc;
- break;
- default:
- break;
- }
- }
- break;
- }
- if (value >= 0)
- goto out;
- }
-
- /* finally try the unicode escapes */
- code_length = 0;
-
- if (esc == fyue_doublequote || esc == fyue_doublequote_yaml_1_1) {
- switch (c) {
- case 'x':
- code_length = 2;
- break;
- case 'u':
- code_length = 4;
- break;
- case 'U':
- code_length = 8;
- break;
- default:
- return -1;
- }
- } else if (esc == fyue_doublequote_json && c == 'u')
- code_length = 4;
-
- if (!code_length || code_length > (e - s))
- goto out;
-
- value = 0;
- for (i = 0; i < code_length; i++) {
- c = *s++;
- value <<= 4;
- if (c >= '0' && c <= '9')
- value |= c - '0';
- else if (c >= 'a' && c <= 'f')
- value |= 10 + c - 'a';
- else if (c >= 'A' && c <= 'F')
- value |= 10 + c - 'A';
- else
- goto out;
- }
-
- /* hi/lo surrogate pair */
- if (code_length == 4 && value >= 0xd800 && value <= 0xdbff &&
- (e - s) >= 6 && s[0] == '\\' && s[1] == 'u') {
- hi_surrogate = value;
-
- s += 2;
-
- value = 0;
- for (i = 0; i < code_length; i++) {
- c = *s++;
- value <<= 4;
- if (c >= '0' && c <= '9')
- value |= c - '0';
- else if (c >= 'a' && c <= 'f')
- value |= 10 + c - 'a';
- else if (c >= 'A' && c <= 'F')
- value |= 10 + c - 'A';
- else
- return -1;
- }
- lo_surrogate = value;
- value = 0x10000 + (hi_surrogate - 0xd800) * 0x400 + (lo_surrogate - 0xdc00);
- }
-
-out:
- *strp = s;
-
- return value;
-}
-
-uint8_t fy_utf8_low_ascii_flags[0x80] = {
- [0x00] = F_NON_PRINT, // NUL '\0' (null character)
- [0x01] = F_NON_PRINT, // SOH (start of heading)
- [0x02] = F_NON_PRINT, // STX (start of text)
- [0x03] = F_NON_PRINT, // ETX (end of text)
- [0x04] = F_NON_PRINT, // EOT (end of transmission)
- [0x05] = F_NON_PRINT, // ENQ (enquiry)
- [0x06] = F_NON_PRINT, // ACK (acknowledge)
- [0x07] = F_NON_PRINT | F_QUOTE_ESC, // BEL '\a' (bell)
- [0x08] = F_NON_PRINT | F_QUOTE_ESC, // BS '\b' (backspace)
- [0x09] = F_NON_PRINT | F_QUOTE_ESC | F_WS, // HT '\t' (horizontal tab)
- [0x0A] = F_NON_PRINT | F_QUOTE_ESC | F_LB, // LF '\n' (new line)
- [0x0B] = F_NON_PRINT | F_QUOTE_ESC, // VT '\v' (vertical tab)
- [0x0C] = F_NON_PRINT | F_QUOTE_ESC, // FF '\f' (form feed)
- [0x0D] = F_NON_PRINT | F_QUOTE_ESC | F_LB, // CR '\r' (carriage ret)
- [0x0E] = F_NON_PRINT, // SO (shift out)
- [0x0F] = F_NON_PRINT, // SI (shift in)
- [0x10] = F_NON_PRINT, // DLE (data link escape)
- [0x11] = F_NON_PRINT, // DC1 (device control 1)
- [0x12] = F_NON_PRINT, // DC2 (device control 2)
- [0x13] = F_NON_PRINT, // DC3 (device control 3)
- [0x14] = F_NON_PRINT, // DC4 (device control 4)
- [0x15] = F_NON_PRINT, // NAK (negative ack.)
- [0x16] = F_NON_PRINT, // SYN (synchronous idle)
- [0x17] = F_NON_PRINT, // ETB (end of trans. blk)
- [0x18] = F_NON_PRINT, // CAN (cancel)
- [0x19] = F_NON_PRINT, // EM (end of medium)
- [0x1A] = F_NON_PRINT, // SUB (substitute)
- [0x1B] = F_NON_PRINT, // ESC (escape)
- [0x1C] = F_NON_PRINT, // FS (file separator)
- [0x1D] = F_NON_PRINT, // GS (group separator)
- [0x1E] = F_NON_PRINT, // RS (record separator)
- [0x1F] = F_NON_PRINT, // US (unit separator)
- [' '] = F_WS,
- ['!'] = F_PUNCT,
- ['"'] = F_PUNCT,
- ['#'] = F_PUNCT,
- ['$'] = F_PUNCT,
- ['%'] = F_PUNCT,
- ['&'] = F_PUNCT,
- ['\''] = F_PUNCT,
- ['('] = F_PUNCT,
- [')'] = F_PUNCT,
- ['*'] = F_PUNCT,
- ['+'] = F_PUNCT,
- [','] = F_PUNCT,
- ['-'] = F_PUNCT,
- ['.'] = F_PUNCT,
- ['/'] = F_PUNCT,
- ['0'] = F_DIGIT | F_SIMPLE_SCALAR,
- ['1'] = F_DIGIT | F_SIMPLE_SCALAR,
- ['2'] = F_DIGIT | F_SIMPLE_SCALAR,
- ['3'] = F_DIGIT | F_SIMPLE_SCALAR,
- ['4'] = F_DIGIT | F_SIMPLE_SCALAR,
- ['5'] = F_DIGIT | F_SIMPLE_SCALAR,
- ['6'] = F_DIGIT | F_SIMPLE_SCALAR,
- ['7'] = F_DIGIT | F_SIMPLE_SCALAR,
- ['8'] = F_DIGIT | F_SIMPLE_SCALAR,
- ['9'] = F_DIGIT | F_SIMPLE_SCALAR,
- [':'] = F_PUNCT,
- [';'] = F_PUNCT,
- ['<'] = F_PUNCT,
- ['='] = F_PUNCT,
- ['>'] = F_PUNCT,
- ['?'] = F_PUNCT,
- ['@'] = F_PUNCT,
- ['A'] = F_LETTER | F_SIMPLE_SCALAR,
- ['B'] = F_LETTER | F_SIMPLE_SCALAR,
- ['C'] = F_LETTER | F_SIMPLE_SCALAR,
- ['D'] = F_LETTER | F_SIMPLE_SCALAR,
- ['E'] = F_LETTER | F_SIMPLE_SCALAR,
- ['F'] = F_LETTER | F_SIMPLE_SCALAR,
- ['G'] = F_LETTER | F_SIMPLE_SCALAR,
- ['H'] = F_LETTER | F_SIMPLE_SCALAR,
- ['I'] = F_LETTER | F_SIMPLE_SCALAR,
- ['J'] = F_LETTER | F_SIMPLE_SCALAR,
- ['K'] = F_LETTER | F_SIMPLE_SCALAR,
- ['L'] = F_LETTER | F_SIMPLE_SCALAR,
- ['M'] = F_LETTER | F_SIMPLE_SCALAR,
- ['N'] = F_LETTER | F_SIMPLE_SCALAR,
- ['O'] = F_LETTER | F_SIMPLE_SCALAR,
- ['P'] = F_LETTER | F_SIMPLE_SCALAR,
- ['Q'] = F_LETTER | F_SIMPLE_SCALAR,
- ['R'] = F_LETTER | F_SIMPLE_SCALAR,
- ['S'] = F_LETTER | F_SIMPLE_SCALAR,
- ['T'] = F_LETTER | F_SIMPLE_SCALAR,
- ['U'] = F_LETTER | F_SIMPLE_SCALAR,
- ['V'] = F_LETTER | F_SIMPLE_SCALAR,
- ['W'] = F_LETTER | F_SIMPLE_SCALAR,
- ['X'] = F_LETTER | F_SIMPLE_SCALAR,
- ['Y'] = F_LETTER | F_SIMPLE_SCALAR,
- ['Z'] = F_LETTER | F_SIMPLE_SCALAR,
- ['['] = F_PUNCT,
- ['\\'] = F_PUNCT, // '\\'
- [']'] = F_PUNCT,
- ['^'] = F_PUNCT,
- ['_'] = F_PUNCT | F_SIMPLE_SCALAR,
- ['`'] = F_PUNCT,
- ['a'] = F_LETTER | F_SIMPLE_SCALAR,
- ['b'] = F_LETTER | F_SIMPLE_SCALAR,
- ['c'] = F_LETTER | F_SIMPLE_SCALAR,
- ['d'] = F_LETTER | F_SIMPLE_SCALAR,
- ['e'] = F_LETTER | F_SIMPLE_SCALAR,
- ['f'] = F_LETTER | F_SIMPLE_SCALAR,
- ['g'] = F_LETTER | F_SIMPLE_SCALAR,
- ['h'] = F_LETTER | F_SIMPLE_SCALAR,
- ['i'] = F_LETTER | F_SIMPLE_SCALAR,
- ['j'] = F_LETTER | F_SIMPLE_SCALAR,
- ['k'] = F_LETTER | F_SIMPLE_SCALAR,
- ['l'] = F_LETTER | F_SIMPLE_SCALAR,
- ['m'] = F_LETTER | F_SIMPLE_SCALAR,
- ['n'] = F_LETTER | F_SIMPLE_SCALAR,
- ['o'] = F_LETTER | F_SIMPLE_SCALAR,
- ['p'] = F_LETTER | F_SIMPLE_SCALAR,
- ['q'] = F_LETTER | F_SIMPLE_SCALAR,
- ['r'] = F_LETTER | F_SIMPLE_SCALAR,
- ['s'] = F_LETTER | F_SIMPLE_SCALAR,
- ['t'] = F_LETTER | F_SIMPLE_SCALAR,
- ['u'] = F_LETTER | F_SIMPLE_SCALAR,
- ['v'] = F_LETTER | F_SIMPLE_SCALAR,
- ['w'] = F_LETTER | F_SIMPLE_SCALAR,
- ['x'] = F_LETTER | F_SIMPLE_SCALAR,
- ['y'] = F_LETTER | F_SIMPLE_SCALAR,
- ['z'] = F_LETTER | F_SIMPLE_SCALAR,
- ['{'] = F_PUNCT,
- ['|'] = F_PUNCT,
- ['}'] = F_PUNCT,
- ['~'] = F_PUNCT,
- [0x7F] = F_NON_PRINT, // DEL
-};
+++ /dev/null
-/*
- * fy-utf8.h - UTF-8 methods
- *
- * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
- *
- * SPDX-License-Identifier: MIT
- */
-#ifndef FY_UTF8_H
-#define FY_UTF8_H
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdint.h>
-#include <stdlib.h>
-#include <assert.h>
-
-#include <libfyaml.h>
-
-#include "fy-utils.h"
-
-extern const int8_t fy_utf8_width_table[32];
-
-static inline int
-fy_utf8_width_by_first_octet_no_table(uint8_t c)
-{
- return (c & 0x80) == 0x00 ? 1 :
- (c & 0xe0) == 0xc0 ? 2 :
- (c & 0xf0) == 0xe0 ? 3 :
- (c & 0xf8) == 0xf0 ? 4 : 0;
-}
-
-static inline FY_ALWAYS_INLINE int
-fy_utf8_width_by_first_octet(uint8_t c)
-{
- return fy_utf8_width_table[c >> 3];
-}
-
-/* assumes valid utf8 character */
-static inline size_t
-fy_utf8_width(int c)
-{
- return c < 0x80 ? 1 :
- c < 0x800 ? 2 :
- c < 0x10000 ? 3 : 4;
-}
-
-static inline bool
-fy_utf8_is_valid(int c)
-{
- return c >= 0 && !((c >= 0xd800 && c <= 0xdfff) || c >= 0x110000);
-}
-
-static inline bool
-fy_utf8_is_printable_ascii(int c)
-{
- return c >= 0x20 && c <= 0x7e;
-}
-
-/* generic utf8 decoder (not inlined) */
-int fy_utf8_get_generic(const void *ptr, int left, int *widthp);
-
-/* -1 for end of input, -2 for invalid character, -3 for partial */
-#define FYUG_EOF -1
-#define FYUG_INV -2
-#define FYUG_PARTIAL -3
-
-static inline int fy_utf8_get(const void *ptr, int left, int *widthp)
-{
- const uint8_t *p = ptr;
-
- /* single byte (hot path) */
- if (left <= 0) {
- *widthp = 0;
- return FYUG_EOF;
- }
-
- if (!(p[0] & 0x80)) {
- *widthp = 1;
- return p[0] & 0x7f;
- }
- return fy_utf8_get_generic(ptr, left, widthp);
-}
-
-int fy_utf8_get_right_generic(const void *ptr, int left, int *widthp);
-
-static inline int fy_utf8_get_right(const void *ptr, int left, int *widthp)
-{
- const uint8_t *p = ptr + left;
-
- /* single byte (hot path) */
- if (left > 0 && !(p[-1] & 0x80)) {
- if (widthp)
- *widthp = 1;
- return p[-1] & 0x7f;
- }
- return fy_utf8_get_right_generic(ptr, left, widthp);
-}
-
-
-/* for when you _know_ that there's enough room and c is valid */
-static inline void *fy_utf8_put_unchecked(void *ptr, int c)
-{
- uint8_t *s = ptr;
-
- assert(c >= 0);
- if (c < 0x80)
- *s++ = c;
- else if (c < 0x800) {
- *s++ = (c >> 6) | 0xc0;
- *s++ = (c & 0x3f) | 0x80;
- } else if (c < 0x10000) {
- *s++ = (c >> 12) | 0xe0;
- *s++ = ((c >> 6) & 0x3f) | 0x80;
- *s++ = (c & 0x3f) | 0x80;
- } else {
- *s++ = (c >> 18) | 0xf0;
- *s++ = ((c >> 12) & 0x3f) | 0x80;
- *s++ = ((c >> 6) & 0x3f) | 0x80;
- *s++ = (c & 0x3f) | 0x80;
- }
- return s;
-}
-
-static inline void *fy_utf8_put(void *ptr, size_t left, int c)
-{
- if (!fy_utf8_is_valid(c) || fy_utf8_width(c) > left)
- return NULL;
-
- return fy_utf8_put_unchecked(ptr, c);
-}
-
-/* buffer must contain at least 5 characters */
-#define FY_UTF8_FORMAT_BUFMIN 5
-enum fy_utf8_escape {
- fyue_none,
- fyue_singlequote,
- fyue_doublequote,
- fyue_doublequote_json,
- fyue_doublequote_yaml_1_1,
-};
-
-static inline bool fy_utf8_escape_is_any_doublequote(enum fy_utf8_escape esc)
-{
- return esc >= fyue_doublequote && esc <= fyue_doublequote_yaml_1_1;
-}
-
-char *fy_utf8_format(int c, char *buf, enum fy_utf8_escape esc);
-
-#define fy_utf8_format_a(_c, _esc) \
- ({ \
- char *_buf = alloca(FY_UTF8_FORMAT_BUFMIN); \
- fy_utf8_format((_c), _buf, _esc); \
- })
-
-int fy_utf8_format_text_length(const char *buf, size_t len,
- enum fy_utf8_escape esc);
-char *fy_utf8_format_text(const char *buf, size_t len,
- char *out, size_t maxsz,
- enum fy_utf8_escape esc);
-
-#define fy_utf8_format_text_a(_buf, _len, _esc) \
- ({ \
- const char *__buf = (_buf); \
- size_t __len = (_len); \
- enum fy_utf8_escape __esc = (_esc); \
- size_t _outsz = fy_utf8_format_text_length(__buf, __len, __esc); \
- char *_out = alloca(_outsz + 1); \
- fy_utf8_format_text(__buf, __len, _out, _outsz, __esc); \
- })
-
-char *fy_utf8_format_text_alloc(const char *buf, size_t len, enum fy_utf8_escape esc);
-
-const void *fy_utf8_memchr_generic(const void *s, int c, size_t n);
-
-static inline const void *fy_utf8_memchr(const void *s, int c, size_t n)
-{
- if (c < 0 || !n)
- return NULL;
- if (c < 0x80)
- return memchr(s, c, n);
- return fy_utf8_memchr_generic(s, c, n);
-}
-
-static inline const void *fy_utf8_strchr(const void *s, int c)
-{
- if (c < 0)
- return NULL;
- if (c < 0x80)
- return strchr(s, c);
- return fy_utf8_memchr_generic(s, c, strlen(s));
-}
-
-static inline int fy_utf8_count(const void *ptr, size_t len)
-{
- const uint8_t *s = ptr, *e = ptr + len;
- int w, count;
-
- count = 0;
- while (s < e) {
- w = fy_utf8_width_by_first_octet(*s);
-
- /* malformed? */
- if (!w || s + w > e)
- break;
- s += w;
-
- count++;
- }
-
- return count;
-}
-
-int fy_utf8_parse_escape(const char **strp, size_t len, enum fy_utf8_escape esc);
-
-#define F_NONE 0
-#define F_NON_PRINT FY_BIT(0) /* non printable */
-#define F_SIMPLE_SCALAR FY_BIT(1) /* part of simple scalar */
-#define F_QUOTE_ESC FY_BIT(2) /* escape form, i.e \n */
-#define F_LB FY_BIT(3) /* is a linebreak */
-#define F_WS FY_BIT(4) /* is a whitespace */
-#define F_PUNCT FY_BIT(5) /* is a punctuation mark */
-#define F_LETTER FY_BIT(6) /* is a letter a..z A..Z */
-#define F_DIGIT FY_BIT(7) /* is a digit 0..9 */
-
-extern uint8_t fy_utf8_low_ascii_flags[0x80];
-
-#endif
+++ /dev/null
-/*
- * fy-utils.c - Generic utilities for functionality that's missing
- * from platforms.
- *
- * For now only used to implement memstream for Apple platforms.
- *
- * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
- *
- * SPDX-License-Identifier: MIT
- */
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <termios.h>
-#include <unistd.h>
-#include <sys/select.h>
-#include <sys/time.h>
-#include <sys/types.h>
-
-#include "fy-utf8.h"
-#include "fy-ctype.h"
-#include "fy-utils.h"
-
-#if defined(__APPLE__) && (_POSIX_C_SOURCE < 200809L)
-
-/*
- * adapted from http://piumarta.com/software/memstream/
- *
- * Under the MIT license.
- */
-
-/*
- * ----------------------------------------------------------------------------
- *
- * OPEN_MEMSTREAM(3) BSD and Linux Library Functions OPEN_MEMSTREAM(3)
- *
- * SYNOPSIS
- * #include "memstream.h"
- *
- * FILE *open_memstream(char **bufp, size_t *sizep);
- *
- * DESCRIPTION
- * The open_memstream() function opens a stream for writing to a buffer.
- * The buffer is dynamically allocated (as with malloc(3)), and
- * automatically grows as required. After closing the stream, the caller
- * should free(3) this buffer.
- *
- * When the stream is closed (fclose(3)) or flushed (fflush(3)), the
- * locations pointed to by bufp and sizep are updated to contain,
- * respectively, a pointer to the buffer and the current size of the
- * buffer. These values remain valid only as long as the caller performs
- * no further output on the stream. If further output is performed, then
- * the stream must again be flushed before trying to access these
- * variables.
- *
- * A null byte is maintained at the end of the buffer. This byte is not
- * included in the size value stored at sizep.
- *
- * The stream's file position can be changed with fseek(3) or fseeko(3).
- * Moving the file position past the end of the data already written fills
- * the intervening space with zeros.
- *
- * RETURN VALUE
- * Upon successful completion open_memstream() returns a FILE pointer.
- * Otherwise, NULL is returned and errno is set to indicate the error.
- *
- * CONFORMING TO
- * POSIX.1-2008
- *
- * ----------------------------------------------------------------------------
- */
-
-#ifndef min
-#define min(X, Y) (((X) < (Y)) ? (X) : (Y))
-#endif
-
-struct memstream {
- size_t position;
- size_t size;
- size_t capacity;
- char *contents;
- char **ptr;
- size_t *sizeloc;
-};
-
-static int memstream_grow(struct memstream *ms, size_t minsize)
-{
- size_t newcap;
- char *newcontents;
-
- newcap = ms->capacity * 2;
- while (newcap <= minsize + 1)
- newcap *= 2;
- newcontents = realloc(ms->contents, newcap);
- if (!newcontents)
- return -1;
- ms->contents = newcontents;
- memset(ms->contents + ms->capacity, 0, newcap - ms->capacity);
- ms->capacity = newcap;
- *ms->ptr = ms->contents;
- return 0;
-}
-
-static int memstream_read(void *cookie, char *buf, int count)
-{
- struct memstream *ms = cookie;
- size_t n;
-
- n = min(ms->size - ms->position, (size_t)count);
- if (n < 1)
- return 0;
- memcpy(buf, ms->contents, n);
- ms->position += n;
- return n;
-}
-
-static int memstream_write(void *cookie, const char *buf, int count)
-{
- struct memstream *ms = cookie;
-
- if (ms->capacity <= ms->position + (size_t)count &&
- memstream_grow(ms, ms->position + (size_t)count) < 0)
- return -1;
- memcpy(ms->contents + ms->position, buf, count);
- ms->position += count;
- ms->contents[ms->position] = '\0';
- if (ms->size < ms->position)
- *ms->sizeloc = ms->size = ms->position;
-
- return count;
-}
-
-static fpos_t memstream_seek(void *cookie, fpos_t offset, int whence)
-{
- struct memstream *ms = cookie;
- fpos_t pos= 0;
-
- switch (whence) {
- case SEEK_SET:
- pos = offset;
- break;
- case SEEK_CUR:
- pos = ms->position + offset;
- break;
- case SEEK_END:
- pos = ms->size + offset;
- break;
- default:
- errno= EINVAL;
- return -1;
- }
- if (pos >= (fpos_t)ms->capacity && memstream_grow(ms, pos) < 0)
- return -1;
- ms->position = pos;
- if (ms->size < ms->position)
- *ms->sizeloc = ms->size = ms->position;
- return pos;
-}
-
-static int memstream_close(void *cookie)
-{
- struct memstream *ms = cookie;
-
- ms->size = min(ms->size, ms->position);
- *ms->ptr = ms->contents;
- *ms->sizeloc = ms->size;
- ms->contents[ms->size]= 0;
- /* ms->contents is what's returned */
- free(ms);
- return 0;
-}
-
-FILE *open_memstream(char **ptr, size_t *sizeloc)
-{
- struct memstream *ms;
- FILE *fp;
-
- if (!ptr || !sizeloc) {
- errno= EINVAL;
- goto err_out;
- }
-
- ms = calloc(1, sizeof(struct memstream));
- if (!ms)
- goto err_out;
-
- ms->position = ms->size= 0;
- ms->capacity = 4096;
- ms->contents = calloc(ms->capacity, 1);
- if (!ms->contents)
- goto err_free_ms;
- ms->ptr = ptr;
- ms->sizeloc = sizeloc;
- fp= funopen(ms, memstream_read, memstream_write,
- memstream_seek, memstream_close);
- if (!fp)
- goto err_free_all;
- *ptr = ms->contents;
- *sizeloc = ms->size;
- return fp;
-
-err_free_all:
- free(ms->contents);
-err_free_ms:
- free(ms);
-err_out:
- return NULL;
-}
-
-#endif /* __APPLE__ && _POSIX_C_SOURCE < 200809L */
-
-bool fy_tag_uri_is_valid(const char *data, size_t len)
-{
- const char *s, *e;
- int w, j, k, width, c;
- uint8_t octet, esc_octets[4];
-
- s = data;
- e = s + len;
-
- while ((c = fy_utf8_get(s, e - s, &w)) >= 0) {
- if (c != '%') {
- s += w;
- continue;
- }
-
- width = 0;
- k = 0;
- do {
- /* short URI escape */
- if ((e - s) < 3)
- return false;
-
- if (width > 0) {
- c = fy_utf8_get(s, e - s, &w);
- if (c != '%')
- return false;
- }
-
- s += w;
-
- octet = 0;
-
- for (j = 0; j < 2; j++) {
- c = fy_utf8_get(s, e - s, &w);
- if (!fy_is_hex(c))
- return false;
- s += w;
-
- octet <<= 4;
- if (c >= '0' && c <= '9')
- octet |= c - '0';
- else if (c >= 'a' && c <= 'f')
- octet |= 10 + c - 'a';
- else
- octet |= 10 + c - 'A';
- }
- if (!width) {
- width = fy_utf8_width_by_first_octet(octet);
-
- if (width < 1 || width > 4)
- return false;
- k = 0;
- }
- esc_octets[k++] = octet;
-
- } while (--width > 0);
-
- /* now convert to utf8 */
- c = fy_utf8_get(esc_octets, k, &w);
-
- if (c < 0)
- return false;
- }
-
- return true;
-}
-
-int fy_tag_handle_length(const char *data, size_t len)
-{
- const char *s, *e;
- int c, w;
-
- s = data;
- e = s + len;
-
- c = fy_utf8_get(s, e - s, &w);
- if (c != '!')
- return -1;
- s += w;
-
- c = fy_utf8_get(s, e - s, &w);
- if (fy_is_ws(c))
- return s - data;
- /* if first character is !, empty handle */
- if (c == '!') {
- s += w;
- return s - data;
- }
- if (!fy_is_first_alpha(c))
- return -1;
- s += w;
- while (fy_is_alnum(c = fy_utf8_get(s, e - s, &w)))
- s += w;
- if (c == '!')
- s += w;
-
- return s - data;
-}
-
-int fy_tag_uri_length(const char *data, size_t len)
-{
- const char *s, *e;
- int c, w, cn, wn, uri_length;
-
- s = data;
- e = s + len;
-
- while (fy_is_uri(c = fy_utf8_get(s, e - s, &w))) {
- cn = fy_utf8_get(s + w, e - (s + w), &wn);
- if ((fy_is_z(cn) || fy_is_blank(cn) || fy_is_any_lb(cn)) && fy_utf8_strchr(",}]", c))
- break;
- s += w;
- }
- uri_length = s - data;
-
- if (!fy_tag_uri_is_valid(data, uri_length))
- return -1;
-
- return uri_length;
-}
-
-int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info)
-{
- const char *s, *e;
- int total_length, handle_length, uri_length, prefix_length, suffix_length;
- int c, cn, w, wn;
-
- s = data;
- e = s + len;
-
- prefix_length = 0;
-
- /* it must start with '!' */
- c = fy_utf8_get(s, e - s, &w);
- if (c != '!')
- return -1;
- cn = fy_utf8_get(s + w, e - (s + w), &wn);
- if (cn == '<') {
- prefix_length = 2;
- suffix_length = 1;
- } else
- prefix_length = suffix_length = 0;
-
- if (prefix_length) {
- handle_length = 0; /* set the handle to '' */
- s += prefix_length;
- } else {
- /* either !suffix or !handle!suffix */
- /* we scan back to back, and split handle/suffix */
- handle_length = fy_tag_handle_length(s, e - s);
- if (handle_length <= 0)
- return -1;
- s += handle_length;
- }
-
- uri_length = fy_tag_uri_length(s, e - s);
- if (uri_length < 0)
- return -1;
-
- /* a handle? */
- if (!prefix_length && (handle_length == 0 || data[handle_length - 1] != '!')) {
- /* special case, '!', handle set to '' and suffix to '!' */
- if (handle_length == 1 && uri_length == 0) {
- handle_length = 0;
- uri_length = 1;
- } else {
- uri_length = handle_length - 1 + uri_length;
- handle_length = 1;
- }
- }
- total_length = prefix_length + handle_length + uri_length + suffix_length;
-
- if (total_length != (int)len)
- return -1;
-
- info->total_length = total_length;
- info->handle_length = handle_length;
- info->uri_length = uri_length;
- info->prefix_length = prefix_length;
- info->suffix_length = suffix_length;
-
- return 0;
-}
-
-/* simple terminal methods; mainly for getting size of terminal */
-int fy_term_set_raw(int fd, struct termios *oldt)
-{
- struct termios newt, t;
- int ret;
-
- /* must be a terminal */
- if (!isatty(fd))
- return -1;
-
- ret = tcgetattr(fd, &t);
- if (ret != 0)
- return ret;
-
- newt = t;
-
- cfmakeraw(&newt);
-
- ret = tcsetattr(fd, TCSANOW, &newt);
- if (ret != 0)
- return ret;
-
- if (oldt)
- *oldt = t;
-
- return 0;
-}
-
-int fy_term_restore(int fd, const struct termios *oldt)
-{
- /* must be a terminal */
- if (!isatty(fd))
- return -1;
-
- return tcsetattr(fd, TCSANOW, oldt);
-}
-
-ssize_t fy_term_write(int fd, const void *data, size_t count)
-{
- ssize_t wrn, r;
-
- if (!isatty(fd))
- return -1;
- r = 0;
- wrn = 0;
- while (count > 0) {
- do {
- r = write(fd, data, count);
- } while (r == -1 && errno == EAGAIN);
- if (r < 0)
- break;
- wrn += r;
- data += r;
- count -= r;
- }
-
- /* return the amount written, or the last error code */
- return wrn > 0 ? wrn : r;
-}
-
-int fy_term_safe_write(int fd, const void *data, size_t count)
-{
- if (!isatty(fd))
- return -1;
-
- return fy_term_write(fd, data, count) == (ssize_t)count ? 0 : -1;
-}
-
-ssize_t fy_term_read(int fd, void *data, size_t count, int timeout_us)
-{
- ssize_t rdn, r;
- struct timeval tv, tvto, *tvp;
- fd_set rdfds;
-
- if (!isatty(fd))
- return -1;
-
- FD_ZERO(&rdfds);
-
- memset(&tvto, 0, sizeof(tvto));
- memset(&tv, 0, sizeof(tv));
-
- if (timeout_us >= 0) {
- tvto.tv_sec = timeout_us / 1000000;
- tvto.tv_usec = timeout_us % 1000000;
- tvp = &tv;
- } else {
- tvp = NULL;
- }
-
- r = 0;
- rdn = 0;
- while (count > 0) {
- do {
- FD_SET(fd, &rdfds);
- if (tvp)
- *tvp = tvto;
- r = select(fd + 1, &rdfds, NULL, NULL, tvp);
- } while (r == -1 && errno == EAGAIN);
-
- /* select ends, or something weird */
- if (r <= 0 || !FD_ISSET(fd, &rdfds))
- break;
-
- /* now read */
- do {
- r = read(fd, data, count);
- } while (r == -1 && errno == EAGAIN);
- if (r < 0)
- break;
-
- rdn += r;
- data += r;
- count -= r;
- }
-
- /* return the amount written, or the last error code */
- return rdn > 0 ? rdn : r;
-}
-
-ssize_t fy_term_read_escape(int fd, void *buf, size_t count)
-{
- char *p;
- int r, rdn;
- char c;
-
- /* at least 3 characters */
- if (count < 3)
- return -1;
-
- p = buf;
- rdn = 0;
-
- /* ESC */
- r = fy_term_read(fd, &c, 1, 100 * 1000);
- if (r != 1 || c != '\x1b')
- return -1;
- *p++ = c;
- count--;
- rdn++;
-
- /* [ */
- r = fy_term_read(fd, &c, 1, 100 * 1000);
- if (r != 1 || c != '[')
- return rdn;
- *p++ = c;
- count--;
- rdn++;
-
- /* read until error, out of buffer, or < 0x40 || > 0x7e */
- r = -1;
- while (count > 0) {
- r = fy_term_read(fd, &c, 1, 100 * 1000);
- if (r != 1)
- r = -1;
- if (r != 1)
- break;
- *p++ = c;
- count--;
- rdn++;
-
- /* end of escape */
- if (c >= 0x40 && c <= 0x7e)
- break;
- }
-
- return rdn;
-}
-
-int fy_term_query_size_raw(int fd, int *rows, int *cols)
-{
- char buf[32];
- char *s, *e;
- ssize_t r;
-
- /* must be a terminal */
- if (!isatty(fd))
- return -1;
-
- *rows = *cols = 0;
-
- /* query text area */
- r = fy_term_safe_write(fd, "\x1b[18t", 5);
- if (r != 0)
- return r;
-
- /* read a character */
- r = fy_term_read_escape(fd, buf, sizeof(buf));
-
- /* return must be ESC[8;<height>;<width>;t */
-
- if (r < 8 || r >= (int)sizeof(buf) - 2) /* minimum ESC[8;1;1t */
- return -1;
-
- s = buf;
- e = s + r;
-
- /* correct response? starts with ESC[8; */
- if (s[0] != '\x1b' || s[1] != '[' || s[2] != '8' || s[3] != ';')
- return -1;
- s += 4;
-
- /* must end with t */
- if (e[-1] != 't')
- return -1;
- *--e = '\0'; /* remove trailing t, and zero terminate */
-
- /* scan two ints separated by ; */
- r = sscanf(s, "%d;%d", rows, cols);
- if (r != 2)
- return -1;
-
- return 0;
-}
-
-int fy_term_query_size(int fd, int *rows, int *cols)
-{
- struct termios old_term;
- int ret, r;
-
- if (!isatty(fd))
- return -1;
-
- r = fy_term_set_raw(fd, &old_term);
- if (r != 0)
- return -1;
-
- ret = fy_term_query_size_raw(fd, rows, cols);
-
- r = fy_term_restore(fd, &old_term);
- if (r != 0)
- return -1;
-
- return ret;
-}
+++ /dev/null
-/*
- * fy-utils.h - internal utilities header file
- *
- * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
- *
- * SPDX-License-Identifier: MIT
- */
-
-#ifndef FY_UTILS_H
-#define FY_UTILS_H
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <stdio.h>
-#include <stdbool.h>
-#include <unistd.h>
-#include <termios.h>
-
-#if defined(__APPLE__) && (_POSIX_C_SOURCE < 200809L)
-FILE *open_memstream(char **ptr, size_t *sizeloc);
-#endif
-
-int fy_tag_handle_length(const char *data, size_t len);
-bool fy_tag_uri_is_valid(const char *data, size_t len);
-int fy_tag_uri_length(const char *data, size_t len);
-
-struct fy_tag_scan_info {
- int total_length;
- int handle_length;
- int uri_length;
- int prefix_length;
- int suffix_length;
-};
-
-int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info);
-
-#ifndef ARRAY_SIZE
-#define ARRAY_SIZE(x) ((sizeof(x)/sizeof((x)[0])))
-#endif
-
-#if !defined(NDEBUG) && (defined(__GNUC__) && __GNUC__ >= 4)
-#define FY_ALWAYS_INLINE __attribute__((always_inline))
-#else
-#define FY_ALWAYS_INLINE /* nothing */
-#endif
-
-#if defined(__GNUC__) && __GNUC__ >= 4
-#define FY_UNUSED __attribute__((unused))
-#else
-#define FY_UNUSED /* nothing */
-#endif
-
-int fy_term_set_raw(int fd, struct termios *oldt);
-int fy_term_restore(int fd, const struct termios *oldt);
-ssize_t fy_term_write(int fd, const void *data, size_t count);
-int fy_term_safe_write(int fd, const void *data, size_t count);
-ssize_t fy_term_read(int fd, void *data, size_t count, int timeout_us);
-ssize_t fy_term_read_escape(int fd, void *buf, size_t count);
-
-/* the raw methods require the terminal to be in raw mode */
-int fy_term_query_size_raw(int fd, int *rows, int *cols);
-
-/* the non raw methods will set the terminal to raw and then restore */
-int fy_term_query_size(int fd, int *rows, int *cols);
-
-#endif
--- /dev/null
+/*
+ * fy-thread.c - Lighting fast thread pool implementation
+ *
+ * Copyright (c) 2023 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <assert.h>
+#include <stdbool.h>
+#include <string.h>
+#include <stdio.h>
+#include <alloca.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <limits.h>
+
+#if defined(__linux__)
+#include <sys/syscall.h>
+#include <linux/futex.h>
+#endif
+
+#include "fy-diag.h"
+#include "fy-utils.h"
+#include "fy-thread.h"
+
+#undef WORK_SHUTDOWN
+#define WORK_SHUTDOWN ((struct fy_thread_work *)(void *)-1)
+
+#ifdef FY_THREAD_DEBUG
+#define TDBG(_fmt, ...) \
+ do { fprintf(stderr, (_fmt) __VA_OPT__(,) __VA_ARGS__); } while(0)
+#else
+#define TDBG(_fmt, ...) \
+ do { /* nothing */ } while(0)
+#endif
+
+static void *fy_worker_thread_standard(void *arg);
+static void *fy_worker_thread_steal(void *arg);
+
+static void fy_thread_work_join_standard(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn);
+static void fy_thread_work_join_steal(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn);
+static void fy_thread_work_join_steal_2(struct fy_thread_pool *tp, struct fy_thread_work works[2], fy_work_check_fn check_fn);
+
+#if defined(__linux__) && !defined(FY_THREAD_PORTABLE)
+
+/* linux pedal to the metal implementation */
+static inline int futex(_Atomic(uint32_t) *uaddr, int futex_op, uint32_t val, const struct timespec *timeout, uint32_t *uaddr2, uint32_t val3)
+{
+ return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3);
+}
+
+static inline int fwait(_Atomic(uint32_t) *futexp)
+{
+ long s;
+ uint32_t one = 1;
+
+ while (!atomic_compare_exchange_strong(futexp, &one, 0)) {
+ s = futex(futexp, FUTEX_WAIT_PRIVATE, 0, NULL, NULL, 0);
+ if (s == -1 && errno != EAGAIN)
+ return -1;
+ }
+ return 0;
+}
+
+static inline int fpost(_Atomic(uint32_t) *futexp)
+{
+ long s;
+ uint32_t zero = 0;
+
+ if (atomic_compare_exchange_strong(futexp, &zero, 1)) {
+ s = futex(futexp, FUTEX_WAKE_PRIVATE, 1, NULL, NULL, 0);
+ if (s == -1)
+ return -1;
+ }
+ return 0;
+}
+
+static inline void fy_thread_init_sync(struct fy_thread *t)
+{
+ /* nothing more needed for futexes */
+ atomic_store(&t->submit, 0);
+ atomic_store(&t->done, 0);
+}
+
+static inline struct fy_thread_work *fy_worker_wait_for_work(struct fy_thread *t)
+{
+ struct fy_thread_work *w;
+ int rc __FY_DEBUG_UNUSED__;
+
+ while ((w = atomic_load(&t->work)) == NULL) {
+ rc = fwait(&t->submit);
+ assert(!rc);
+ }
+ return w;
+}
+
+static inline void fy_worker_signal_work_done(struct fy_thread *t, struct fy_thread_work *work)
+{
+ struct fy_thread_work *exp_work;
+ bool ok __FY_DEBUG_UNUSED__;
+ int rc __FY_DEBUG_UNUSED__;
+
+ /* note that the work won't be replaced if it's a shutdown */
+ exp_work = work;
+ ok = atomic_compare_exchange_strong(&t->work, &exp_work, NULL);
+ assert(ok);
+
+ rc = fpost(&t->done);
+ assert(!rc);
+}
+
+static inline int fy_thread_submit_work_internal(struct fy_thread *t, struct fy_thread_work *work)
+{
+ struct fy_thread_work *exp_work;
+ int rc __FY_DEBUG_UNUSED__;
+
+ /* atomically update the work */
+ exp_work = NULL;
+ if (!atomic_compare_exchange_strong(&t->work, &exp_work, work))
+ return -1;
+
+ rc = fpost(&t->submit);
+ assert(!rc);
+
+ return 0;
+}
+
+static inline int fy_thread_wait_work_internal(struct fy_thread *t)
+{
+ const struct fy_thread_work *work;
+
+ while ((work = atomic_load(&t->work)) != NULL)
+ fwait(&t->done);
+
+ atomic_store(&t->done, 0);
+
+ return 0;
+}
+
+void fy_worker_thread_shutdown(struct fy_thread *t)
+{
+ int rc __FY_DEBUG_UNUSED__;
+
+ atomic_store(&t->work, WORK_SHUTDOWN);
+ rc = fpost(&t->submit);
+ assert(!rc);
+
+ rc = pthread_join(t->tid, NULL);
+ assert(!rc);
+
+ if (!(t->tp->cfg.flags & FYTPCF_STEAL_MODE))
+ atomic_store(&t->work, NULL);
+}
+
+#else
+
+/* portable pthread implementation */
+
+static inline void fy_thread_init_sync(struct fy_thread *t)
+{
+ pthread_mutex_init(&t->lock, NULL);
+ pthread_cond_init(&t->cond, NULL);
+
+ pthread_mutex_init(&t->wait_lock, NULL);
+ pthread_cond_init(&t->wait_cond, NULL);
+}
+
+static inline struct fy_thread_work *fy_worker_wait_for_work(struct fy_thread *t)
+{
+ struct fy_thread_work *work;
+
+ pthread_mutex_lock(&t->lock);
+ while ((work = atomic_load(&t->work)) == NULL)
+ pthread_cond_wait(&t->cond, &t->lock);
+ pthread_mutex_unlock(&t->lock);
+
+ return work;
+}
+
+static inline void fy_worker_signal_work_done(struct fy_thread *t, struct fy_thread_work *work)
+{
+ struct fy_thread_work *exp_work;
+
+ /* clear the work, so that the user knows we're done */
+ pthread_mutex_lock(&t->wait_lock);
+
+ /* note that the work won't be replaced if it's a shutdown */
+ exp_work = work;
+ if (!atomic_compare_exchange_strong(&t->work, &exp_work, NULL))
+ assert(exp_work == WORK_SHUTDOWN);
+ pthread_cond_signal(&t->wait_cond);
+ pthread_mutex_unlock(&t->wait_lock);
+}
+
+static inline int fy_thread_submit_work_internal(struct fy_thread *t, struct fy_thread_work *work)
+{
+ struct fy_thread_work *exp_work;
+ int ret;
+
+ /* atomically update the work */
+
+ assert(t);
+ assert(work);
+
+ pthread_mutex_lock(&t->lock);
+ exp_work = NULL;
+ if (!atomic_compare_exchange_strong(&t->work, &exp_work, work)) {
+ assert(exp_work == WORK_SHUTDOWN);
+ ret = -1;
+ } else {
+ pthread_cond_signal(&t->cond);
+ ret = 0;
+ }
+ pthread_mutex_unlock(&t->lock);
+
+ return ret;
+}
+
+static inline int fy_thread_wait_work_internal(struct fy_thread *t)
+{
+ const struct fy_thread_work *work;
+
+ pthread_mutex_lock(&t->wait_lock);
+ while ((work = atomic_load(&t->work)) != NULL)
+ pthread_cond_wait(&t->wait_cond, &t->wait_lock);
+ pthread_mutex_unlock(&t->wait_lock);
+
+ return 0;
+}
+
+void fy_worker_thread_shutdown(struct fy_thread *t)
+{
+ pthread_mutex_lock(&t->lock);
+ atomic_store(&t->work, WORK_SHUTDOWN);
+ pthread_cond_signal(&t->cond);
+ pthread_mutex_unlock(&t->lock);
+ pthread_join(t->tid, NULL);
+}
+
+#endif
+
+static inline struct fy_thread *fy_thread_reserve_internal(struct fy_thread_pool *tp)
+{
+ struct fy_thread *t;
+ unsigned int slot;
+ _Atomic(uint64_t) *free;
+ uint64_t exp, v;
+ unsigned int i, num_threads_words;
+
+ t = NULL;
+
+ num_threads_words = FY_BIT64_COUNT(tp->num_threads);
+ for (i = 0, free = tp->freep; i < num_threads_words; i++, free++) {
+ v = atomic_load(free);
+ while (v) {
+ slot = FY_BIT64_LOWEST(v);
+ assert(v & FY_BIT64(slot));
+ exp = v; /* expecting the previous value */
+ v &= ~FY_BIT64(slot); /* clear this bit */
+ if (atomic_compare_exchange_strong(free, &exp, v)) {
+ slot += i * 64;
+ t = tp->threads + slot;
+ assert(slot == t->id);
+ return t;
+ }
+ v = exp;
+ }
+ }
+
+ return NULL;
+}
+
+static inline void fy_thread_unreserve_internal(struct fy_thread *t)
+{
+ struct fy_thread_pool *tp;
+ _Atomic(uint64_t) *free;
+
+ tp = t->tp;
+ free = tp->freep + (unsigned int)(t->id / 64);
+ atomic_fetch_or(free, FY_BIT64(t->id & 63));
+}
+
+static inline bool fy_thread_is_reserved_internal(struct fy_thread *t)
+{
+ struct fy_thread_pool *tp = t->tp;
+ _Atomic(uint64_t) *free;
+
+ free = tp->freep + (unsigned int)(t->id / 64);
+ return !(atomic_load(free) & FY_BIT64(t->id & 63));
+}
+
+static inline bool fy_thread_pool_are_all_reserved_internal(struct fy_thread_pool *tp)
+{
+ _Atomic(uint64_t) *free;
+ uint64_t v, m;
+ unsigned int i, num_threads_words;
+
+ num_threads_words = FY_BIT64_COUNT(tp->num_threads);
+ for (i = 0, free = tp->freep; i < num_threads_words - 1; i++, free++) {
+ v = atomic_load(free);
+ if (v != (uint64_t)-1)
+ return false;
+ }
+ v = atomic_load(free);
+ m = FY_BIT64(tp->num_threads & 63) - 1;
+ return (v & m) == m;
+}
+
+static inline bool fy_thread_pool_is_any_reserved_internal(struct fy_thread_pool *tp)
+{
+ _Atomic(uint64_t) *free;
+ uint64_t v, m;
+ unsigned int i, num_threads_words;
+
+ num_threads_words = FY_BIT64_COUNT(tp->num_threads);
+ for (i = 0, free = tp->freep; i < num_threads_words - 1; i++, free++) {
+ v = atomic_load(free);
+ if (!v)
+ return true;
+ }
+ v = atomic_load(free);
+ m = FY_BIT64(tp->num_threads & 63) - 1;
+ return !(v & m);
+}
+
+struct fy_thread *fy_thread_reserve(struct fy_thread_pool *tp)
+{
+ if (!tp)
+ return NULL;
+
+ /* only valid for non-work stealing thread pools */
+ if (tp->cfg.flags & FYTPCF_STEAL_MODE)
+ return NULL;
+
+ return fy_thread_reserve_internal(tp);
+}
+
+void fy_thread_unreserve(struct fy_thread *t)
+{
+ struct fy_thread_pool *tp;
+
+ if (!t)
+ return;
+
+ tp = t->tp;
+ assert(tp);
+
+ /* only valid for non-work stealing thread pools */
+ if (tp->cfg.flags & FYTPCF_STEAL_MODE)
+ return;
+
+ fy_thread_unreserve_internal(t);
+}
+
+bool fy_thread_is_reserved(struct fy_thread *t)
+{
+ if (!t)
+ return false;
+
+ return fy_thread_is_reserved_internal(t);
+}
+
+bool fy_thread_pool_are_all_reserved(struct fy_thread_pool *tp)
+{
+ return fy_thread_pool_are_all_reserved_internal(tp);
+}
+
+bool fy_thread_pool_is_any_reserved(struct fy_thread_pool *tp)
+{
+ return fy_thread_pool_is_any_reserved_internal(tp);
+}
+
+int fy_thread_submit_work(struct fy_thread *t, struct fy_thread_work *work)
+{
+ if (!t || !work)
+ return -1;
+
+ if (t->tp->cfg.flags & FYTPCF_STEAL_MODE)
+ return -1;
+
+ return fy_thread_submit_work_internal(t, work);
+}
+
+int fy_thread_wait_work(struct fy_thread *t)
+{
+ if (!t)
+ return -1;
+
+ if (t->tp->cfg.flags & FYTPCF_STEAL_MODE)
+ return -1;
+
+ return fy_thread_wait_work_internal(t);
+}
+
+void fy_thread_pool_cleanup(struct fy_thread_pool *tp)
+{
+ struct fy_thread *t;
+ unsigned int i;
+
+ if (!tp)
+ return;
+
+ if (tp->threads) {
+ /* get out of steal mode */
+ for (i = 0, t = tp->threads; i < tp->num_threads; i++, t++) {
+ fy_worker_thread_shutdown(t);
+ }
+
+ fy_cacheline_free(tp->threads);
+ }
+
+ memset(tp, 0, sizeof(*tp));
+}
+
+int fy_thread_pool_setup(struct fy_thread_pool *tp, const struct fy_thread_pool_cfg *cfg)
+{
+ struct fy_thread *t;
+ unsigned int i, num_threads, num_threads_words;
+ size_t size, free_offset, loot_offset, thread_bitmask_size;
+ void *(*start_routine)(void *);
+ long scval;
+ int rc __FY_DEBUG_UNUSED__;
+
+ assert(tp);
+
+ memset(tp, 0, sizeof(*tp));
+
+ if (!cfg) {
+ tp->cfg.flags = 0;
+ tp->cfg.num_threads = 0;
+ } else
+ tp->cfg = *cfg;
+
+ if (!tp->cfg.num_threads) {
+ scval = sysconf(_SC_NPROCESSORS_ONLN);
+ assert(scval > 0);
+ num_threads = (unsigned int)scval;
+ } else
+ num_threads = tp->cfg.num_threads;
+
+ tp->num_threads = num_threads;
+
+ num_threads_words = FY_BIT64_COUNT(tp->num_threads);
+ thread_bitmask_size = FY_BIT64_SIZE(tp->num_threads);
+
+ /* the size of the threads, aligned to a cache line */
+ size = FY_CACHELINE_SIZE_ALIGN(sizeof(*tp->threads) * tp->num_threads);
+
+ /* the free bitmask array offset */
+ free_offset = size;
+ size = FY_CACHELINE_SIZE_ALIGN(size + thread_bitmask_size);
+
+ /* the loot bitmask array offset, aligned to a 64 byte cacheline */
+ loot_offset = size;
+ size = FY_CACHELINE_SIZE_ALIGN(size + thread_bitmask_size);
+
+ /* allocate everything in one go */
+ tp->threads = fy_cacheline_alloc(size);
+ if (!tp->threads)
+ goto err_out;
+
+ memset(tp->threads, 0, size);
+
+ rc = pthread_key_create(&tp->key, NULL);
+ assert(!rc);
+
+ tp->freep = (void *)tp->threads + free_offset;
+ tp->lootp = (void *)tp->threads + loot_offset;
+
+ /* prime the thread free */
+ for (i = 0; i < num_threads_words - 1; i++)
+ tp->freep[i] = (uint64_t)-1;
+ tp->freep[i] = FY_BIT64(tp->num_threads & 63) - 1;
+
+ /* the lootp's are zero */
+
+ for (i = 0, t = tp->threads; i < tp->num_threads; i++, t++) {
+
+ t->tp = tp;
+ t->id = i;
+
+ fy_thread_init_sync(t);
+ }
+
+ start_routine = !(tp->cfg.flags & FYTPCF_STEAL_MODE) ?
+ fy_worker_thread_standard :
+ fy_worker_thread_steal;
+
+ for (i = 0, t = tp->threads; i < tp->num_threads; i++, t++) {
+ rc = pthread_create(&t->tid, NULL, start_routine, t);
+ if (rc)
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ fy_thread_pool_cleanup(tp);
+ return -1;
+}
+
+struct fy_thread_pool *fy_thread_pool_create(const struct fy_thread_pool_cfg *cfg)
+{
+ struct fy_thread_pool *tp;
+ int rc;
+
+ tp = malloc(sizeof(*tp));
+ if (!tp)
+ return NULL;
+
+ rc = fy_thread_pool_setup(tp, cfg);
+ if (rc) {
+ free(tp);
+ return NULL;
+ }
+
+ return tp;
+}
+
+void fy_thread_pool_destroy(struct fy_thread_pool *tp)
+{
+ if (!tp)
+ return;
+
+ fy_thread_pool_cleanup(tp);
+ free(tp);
+}
+
+int fy_thread_pool_get_num_threads(struct fy_thread_pool *tp)
+{
+ if (!tp)
+ return -1;
+
+ return (int)tp->num_threads;
+}
+
+const struct fy_thread_pool_cfg *fy_thread_pool_get_cfg(struct fy_thread_pool *tp)
+{
+ if (!tp)
+ return NULL;
+ return &tp->cfg;
+}
+
+void fy_thread_work_join(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn)
+{
+ if (!(tp->cfg.flags & FYTPCF_STEAL_MODE))
+ fy_thread_work_join_standard(tp, works, work_count, check_fn);
+ else if (work_count == 2)
+ fy_thread_work_join_steal_2(tp, works, check_fn);
+ else
+ fy_thread_work_join_steal(tp, works, work_count, check_fn);
+}
+
+void fy_thread_args_join(struct fy_thread_pool *tp, fy_work_exec_fn fn, fy_work_check_fn check_fn, void **args, size_t count)
+{
+ struct fy_thread_work *works;
+ size_t i;
+
+ if (!count)
+ return;
+
+ works = alloca(sizeof(*works) * count);
+ memset(works, 0, sizeof(*works) * count);
+ for (i = 0; i < count; i++) {
+ works[i].fn = fn;
+ works[i].arg = args ? args[i] : NULL;
+ }
+
+ fy_thread_work_join(tp, works, count, check_fn);
+}
+
+void fy_thread_arg_array_join(struct fy_thread_pool *tp, fy_work_exec_fn fn, fy_work_check_fn check_fn, void *args, size_t argsize, size_t count)
+{
+ struct fy_thread_work *works;
+ size_t i;
+
+ if (!count)
+ return;
+
+ works = alloca(sizeof(*works) * count);
+ memset(works, 0, sizeof(*works) * count);
+ for (i = 0; i < count; i++) {
+ works[i].fn = fn;
+ works[i].arg = args;
+ args += argsize;
+ }
+
+ fy_thread_work_join(tp, works, count, check_fn);
+}
+
+void fy_thread_arg_join(struct fy_thread_pool *tp, fy_work_exec_fn fn, fy_work_check_fn check_fn, void *arg, size_t count)
+{
+ struct fy_thread_work *works;
+ size_t i;
+
+ if (!count)
+ return;
+
+ works = alloca(sizeof(*works) * count);
+ memset(works, 0, sizeof(*works) * count);
+ for (i = 0; i < count; i++) {
+ works[i].fn = fn;
+ works[i].arg = arg;
+ }
+
+ fy_thread_work_join(tp, works, count, check_fn);
+}
+
+/*
+ * the standard (non-stealing implementation)
+ */
+static void *fy_worker_thread_standard(void *arg)
+{
+ struct fy_thread *t = arg;
+ struct fy_thread_pool *tp;
+ struct fy_thread_work *work;
+
+ tp = t->tp;
+
+ /* store per thread info */
+ pthread_setspecific(tp->key, t);
+
+ while ((work = fy_worker_wait_for_work(t)) != WORK_SHUTDOWN) {
+ work->fn(work->arg);
+ fy_worker_signal_work_done(t, work);
+ }
+
+ return NULL;
+}
+
+static void fy_thread_work_join_standard(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn)
+{
+ struct fy_thread_work **direct_work, **thread_work, *w;
+ struct fy_thread **threads, *t;
+ size_t i, direct_work_count, thread_work_count;
+ int rc;
+
+ /* just a single (or no) work, or no threads? execute directly */
+ if (work_count <= 1 || !tp || !tp->num_threads) {
+ for (i = 0, w = works; i < work_count; i++, w++)
+ w->fn(w->arg);
+ return;
+ }
+
+ /* allocate the keeper of direct work */
+ direct_work = alloca(work_count * sizeof(*direct_work));
+ direct_work_count = 0;
+
+ threads = alloca(work_count * sizeof(*threads));
+ thread_work = alloca(work_count * sizeof(*thread_work));
+ thread_work_count = 0;
+
+ for (i = 0, w = works; i < work_count; i++, w++) {
+
+ t = NULL;
+ if (!check_fn || check_fn(w->arg))
+ t = fy_thread_reserve_internal(tp);
+
+ if (t) {
+ threads[thread_work_count] = t;
+ thread_work[thread_work_count++] = w;
+ } else
+ direct_work[direct_work_count++] = w;
+ }
+
+ /* if we don't have any direct_work, steal the last threaded work as direct */
+ if (!direct_work_count) {
+ assert(thread_work_count > 0);
+ t = threads[thread_work_count - 1];
+ w = thread_work[thread_work_count - 1];
+ thread_work_count--;
+
+ /* unreserve this */
+ fy_thread_unreserve_internal(t);
+ direct_work[direct_work_count++] = w;
+ }
+
+ /* submit the threaded work */
+ for (i = 0; i < thread_work_count; i++) {
+ t = threads[i];
+ w = thread_work[i];
+ rc = fy_thread_submit_work_internal(t, w);
+ if (rc) {
+ /* unable to submit? remove work, and move to direct */
+ threads[i] = NULL;
+ thread_work[i] = NULL;
+ fy_thread_unreserve_internal(t);
+ direct_work[direct_work_count++] = w;
+ }
+ }
+
+ /* now perform the direct work while the threaded work is being performed in parallel */
+ for (i = 0; i < direct_work_count; i++) {
+ w = direct_work[i];
+ w->fn(w->arg);
+ }
+
+ /* finally wait for all threaded work to complete */
+ for (i = 0; i < thread_work_count; i++) {
+ t = threads[i];
+ assert(t);
+ fy_thread_wait_work_internal(t);
+ fy_thread_unreserve_internal(t);
+ }
+}
+
+/*
+ * the stealing implementation
+ */
+static inline struct fy_work_pool *fy_work_pool_init(struct fy_work_pool *wp, size_t work_count)
+{
+#if !(defined(__linux__) && !defined(FY_THREAD_PORTABLE))
+ int rc __FY_DEBUG_UNUSED__;
+#endif
+
+ if (!wp)
+ return NULL;
+
+ atomic_store(&wp->work_left, work_count);
+#if defined(__linux__) && !defined(FY_THREAD_PORTABLE)
+ atomic_store(&wp->done, !work_count);
+#elif defined(__APPLE__)
+ wp->sem = dispatch_semaphore_create(!work_count);
+ assert(wp->sem != NULL);
+ (void)rc;
+#else
+ rc = sem_init(&wp->sem, 0, !work_count);
+ assert(!rc);
+#endif
+ return wp;
+}
+
+static inline void fy_work_pool_cleanup(struct fy_work_pool *wp)
+{
+#if !(defined(__linux__) && !defined(FY_THREAD_PORTABLE))
+ int rc __FY_DEBUG_UNUSED__;
+#endif
+
+ if (!wp)
+ return;
+
+#if defined(__linux__) && !defined(FY_THREAD_PORTABLE)
+ /* nothing */
+#elif defined(__APPLE__)
+ /* nothing */
+ (void)rc;
+#else
+ rc = sem_destroy(&wp->sem);
+ assert(!rc);
+#endif
+}
+
+static inline bool fy_work_pool_signal(struct fy_work_pool *wp)
+{
+ size_t prev_work_left;
+ int rc __FY_DEBUG_UNUSED__;
+
+ if (!wp)
+ return false;
+
+ prev_work_left = atomic_fetch_sub(&wp->work_left, 1);
+ if (prev_work_left == 1) {
+#if defined(__linux__) && !defined(FY_THREAD_PORTABLE)
+ rc = fpost(&wp->done);
+ assert(!rc);
+#elif defined(__APPLE__)
+ dispatch_semaphore_signal(wp->sem);
+ (void)rc;
+#else
+ rc = sem_post(&wp->sem);
+ assert(!rc);
+#endif
+ return true;
+ }
+ return false;
+}
+
+static inline void fy_work_pool_wait(struct fy_work_pool *wp)
+{
+ int rc __FY_DEBUG_UNUSED__;
+
+ if (!wp)
+ return;
+
+ /* if there's any work left, wait for it */
+ while (atomic_load(&wp->work_left) > 0) {
+#if defined(__linux__) && !defined(FY_THREAD_PORTABLE)
+ rc = fwait(&wp->done);
+#elif defined(__APPLE__)
+ dispatch_semaphore_wait(wp->sem, DISPATCH_TIME_FOREVER);
+ (void)rc;
+#else
+ rc = sem_wait(&wp->sem);
+#endif
+ assert(!rc || (rc == -1 && errno == EAGAIN));
+ }
+}
+
+static inline void fy_worker_thread_steal_execute(struct fy_thread *t, struct fy_thread_work *w)
+{
+ struct fy_work_pool *wp;
+ bool signalled __FY_DEBUG_UNUSED__;
+
+ TDBG("%s: T#%u worker executing W:%p\n", __func__, t->id, w);
+ wp = w->wp;
+ assert(wp);
+ w->fn(w->arg);
+ TDBG("%s: T#%u worker executed W:%p\n", __func__, t->id, w);
+
+ signalled = fy_work_pool_signal(wp);
+ (void)signalled;
+ TDBG("%s: T#%u W:%p WP:%p signalled=%s\n", __func__,
+ t->id, w, wp, signalled ? "true" : "false");
+}
+
+static inline struct fy_thread_work *fy_worker_thread_steal_work(struct fy_thread_pool *tp, struct fy_thread *t_thief)
+{
+ struct fy_thread *t;
+ unsigned int slot;
+ _Atomic(uint64_t) *loot;
+ uint64_t exp, v;
+ unsigned int i, num_threads_words;
+ struct fy_thread_work *w, *w_exp;
+
+ /* the threads that have work to steal, have the loot bit set */
+ t = NULL;
+
+ num_threads_words = FY_BIT64_COUNT(tp->num_threads);
+ for (i = 0, loot = tp->lootp; i < num_threads_words; i++, loot++) {
+ v = atomic_load(loot);
+ while (v) {
+ slot = FY_BIT64_LOWEST(v);
+ assert(v & FY_BIT64(slot));
+ exp = v; /* expecting the previous value */
+ v &= ~FY_BIT64(slot); /* clear this bit */
+
+ if (atomic_compare_exchange_strong(loot, &exp, v)) {
+ slot += i * 64;
+ t = tp->threads + slot;
+ if ((w = atomic_load(&t->next_work)) != NULL) {
+ w_exp = w;
+ if (atomic_compare_exchange_strong(&t->next_work, &w_exp, NULL))
+ return w;
+ }
+ }
+ v = exp;
+ }
+ }
+
+ return NULL;
+}
+
+static void *fy_worker_thread_steal(void *arg)
+{
+ struct fy_thread *t = arg;
+ struct fy_thread_pool *tp;
+ struct fy_thread_work *w, *w_exp, *w_stolen, *w_last;
+
+ tp = t->tp;
+
+ /* store per thread info */
+ pthread_setspecific(tp->key, t);
+
+ TDBG("%s: T#%u in steal mode\n", __func__, t->id);
+
+ while ((w = fy_worker_wait_for_work(t)) != WORK_SHUTDOWN) {
+
+ assert(fy_thread_is_reserved_internal(t));
+
+ for (;;) {
+ fy_worker_thread_steal_execute(t, w);
+ w_last = w;
+ w_stolen = fy_worker_thread_steal_work(tp, t);
+ w = NULL;
+ if (!w_stolen)
+ break;
+
+ TDBG("%s: T#%u stole W:%p\n", __func__, t->id, w_stolen);
+ w_exp = w_last;
+ if (!atomic_compare_exchange_strong(&t->work, &w_exp, w_stolen)) {
+ assert(w_exp != WORK_SHUTDOWN);
+ TDBG("%s: T#%u t->work:%p w:%p w_stolen:%p\n", __func__, t->id, atomic_load(&t->work), w, w_stolen);
+ abort();
+ }
+ w = w_stolen;
+ }
+
+ /* unreserve first */
+ fy_thread_unreserve_internal(t);
+
+ w_exp = w_last;
+ if (!atomic_compare_exchange_strong(&t->work, &w_exp, NULL)) {
+ assert(w_exp != WORK_SHUTDOWN);
+ break;
+ }
+ }
+ TDBG("%s: T#%u leaving steal mode\n", __func__, t->id);
+
+ return NULL;
+}
+
+static void fy_thread_work_join_steal(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn)
+{
+ struct fy_work_pool wp_local, *wp;
+ struct fy_thread_work *dw, *expw;
+ struct fy_thread *t, *tw;
+ bool has_loot, resolved_t;
+ int rc __FY_DEBUG_UNUSED__;
+ int tid __FY_DEBUG_UNUSED__;
+
+ t = NULL;
+ resolved_t = false;
+ tid = -1;
+ (void)tid;
+
+#ifdef FY_THREAD_DEBUG
+ t = pthread_getspecific(tp->key);
+ tid = t ? (int)t->id : -1;
+ resolved_t = true;
+#else
+ t = NULL;
+ tid = -1;
+ resolved_t = false;
+#endif
+
+ dw = NULL;
+ wp = NULL;
+
+ while (work_count > 0) {
+ if (!dw) {
+ dw = works++;
+ work_count--;
+
+ TDBG("%s: T#%d sdir W:%p\n", __func__, tid, dw);
+
+ continue;
+ }
+
+ has_loot = false;
+ if (work_count > 0 && (!check_fn || check_fn(works->arg))) {
+ while (work_count > 0 && (tw = fy_thread_reserve_internal(tp)) != NULL) {
+
+ assert(!works->wp);
+
+ if (!wp)
+ wp = fy_work_pool_init(&wp_local, work_count + !!dw);
+
+ works->wp = wp;
+ expw = NULL;
+
+ rc = fy_thread_submit_work_internal(tw, works);
+ assert(!rc);
+
+ TDBG("%s: T#%d post W:%p to T#%u\n", __func__, tid, works, tw->id);
+
+ works++;
+ work_count--;
+ }
+
+ if (work_count > 0) {
+
+ if (!resolved_t) {
+ t = pthread_getspecific(tp->key);
+ tid = t ? (int)t->id : -1;
+ resolved_t = true;
+ }
+
+ if (t && atomic_load(&t->next_work) == NULL) {
+ TDBG("%s: T#%d could not post, available to steal W:%p\n", __func__, tid, works);
+ assert(works->wp == NULL);
+
+ if (!wp)
+ wp = fy_work_pool_init(&wp_local, work_count + !!dw);
+
+ works->wp = wp;
+
+ expw = NULL;
+ if (!atomic_compare_exchange_strong(&t->next_work, &expw, works)) {
+ TDBG("%s: T#%d could not update next_work: W:%p\n", __func__, tid, works);
+ abort();
+ }
+
+ /* set the has loot bit */
+ atomic_fetch_or(tp->lootp + (unsigned int)(t->id / 64), FY_BIT64(t->id & 63));
+
+ has_loot = true;
+ }
+ }
+ }
+
+ if (dw) {
+ TDBG("%s: T#%d exec W:%p\n", __func__, tid, dw);
+
+ /* execute the direct work */
+ dw->fn(dw->arg);
+ dw = NULL;
+
+ fy_work_pool_signal(wp);
+ }
+
+ if (has_loot) {
+ assert(t);
+ expw = works;
+
+ /* clear the has loot bit unconditionally */
+ atomic_fetch_and(tp->lootp + (unsigned int)(t->id / 64), ~FY_BIT64(t->id & 63));
+
+ if (!atomic_compare_exchange_strong(&t->next_work, &expw, NULL)) {
+ TDBG("%s: T#%d had W:%p stolen, good\n", __func__, tid, works);
+ work_count--;
+ works++;
+ } else {
+ TDBG("%s: T#%d had W:%p not stolen, bad\n", __func__, tid, works);
+ }
+ }
+ }
+
+ /* last out and direct work */
+ if (dw) {
+ TDBG("%s: T#%d executing final direct W:%p\n", __func__, tid, dw);
+
+ dw->fn(dw->arg);
+ dw = NULL;
+
+ fy_work_pool_signal(wp);
+ }
+
+ TDBG("%s: T#%d wait WP:%p\n", __func__, tid, wp);
+
+ fy_work_pool_wait(wp);
+ fy_work_pool_cleanup(wp);
+
+ TDBG("%s: T#%d done WP:%p\n", __func__, tid, wp);
+}
+
+static void fy_thread_work_join_steal_2(struct fy_thread_pool *tp, struct fy_thread_work works[2], fy_work_check_fn check_fn)
+{
+ struct fy_work_pool wp_local, *wp;
+ struct fy_thread_work *expw;
+ struct fy_thread *t, *tw;
+ bool has_loot, pushed, resolved_t;
+ int rc __FY_DEBUG_UNUSED__;
+ int tid __FY_DEBUG_UNUSED__;
+
+ t = NULL;
+ resolved_t = false;
+ tid = -1;
+
+ (void)tid;
+
+#ifdef FY_THREAD_DEBUG
+ t = pthread_getspecific(tp->key);
+ tid = t ? (int)t->id : -1;
+ resolved_t = true;
+#else
+ t = NULL;
+ tid = -1;
+ resolved_t = false;
+#endif
+
+ wp = NULL;
+
+ pushed = false;
+ has_loot = false;
+ if (!check_fn || check_fn(works->arg)) {
+ if ((tw = fy_thread_reserve_internal(tp)) != NULL) {
+
+ assert(!works[1].wp);
+
+ if (!wp)
+ wp = fy_work_pool_init(&wp_local, 1);
+
+ works[1].wp = wp;
+ expw = NULL;
+
+ rc = fy_thread_submit_work_internal(tw, &works[1]);
+ assert(!rc);
+
+ TDBG("%s: T#%d post W:%p to T#%u\n", __func__, tid, &works[1], tw->id);
+
+ pushed = true;
+
+ } else {
+ if (!resolved_t) {
+ t = pthread_getspecific(tp->key);
+ tid = t ? (int)t->id : -1;
+ resolved_t = true;
+ }
+
+ if (t && atomic_load(&t->next_work) == NULL) {
+ TDBG("%s: T#%d could not post, available to steal W:%p\n", __func__, tid, &works[1]);
+ assert(works[1].wp == NULL);
+
+ if (!wp)
+ wp = fy_work_pool_init(&wp_local, 1);
+
+ works[1].wp = wp;
+
+ expw = NULL;
+ if (!atomic_compare_exchange_strong(&t->next_work, &expw, &works[1])) {
+ TDBG("%s: T#%d could not update next_work: W:%p\n", __func__, tid, &works[1]);
+ abort();
+ }
+
+ /* set the has loot bit */
+ atomic_fetch_or(tp->lootp + (unsigned int)(t->id / 64), FY_BIT64(t->id & 63));
+
+ has_loot = true;
+ }
+ }
+ }
+
+ TDBG("%s: T#%d exec W:%p (left)\n", __func__, tid, &works[0]);
+
+ /* execute the direct work */
+ works[0].fn(works[0].arg);
+
+ if (has_loot) {
+ assert(t);
+ expw = &works[1];
+
+ /* clear the has loot bit unconditionally */
+ atomic_fetch_and(tp->lootp + (unsigned int)(t->id / 64), ~FY_BIT64(t->id & 63));
+
+ if (!atomic_compare_exchange_strong(&t->next_work, &expw, NULL)) {
+ TDBG("%s: T#%d had W:%p stolen\n", __func__, tid, &works[1]);
+ } else {
+ TDBG("%s: T#%d had W:%p not stolen\n", __func__, tid, &works[1]);
+
+ TDBG("%s: T#%d exec W:%p (right)\n", __func__, tid, &works[1]);
+ /* execute the direct work */
+ works[1].fn(works[1].arg);
+ fy_work_pool_signal(wp);
+ }
+
+ } else if (!pushed) {
+
+ TDBG("%s: T#%d exec W:%p (right)\n", __func__, tid, &works[1]);
+ /* execute the direct work */
+ works[1].fn(works[1].arg);
+ fy_work_pool_signal(wp);
+ }
+
+ TDBG("%s: T#%d wait WP:%p\n", __func__, tid, wp);
+
+ fy_work_pool_wait(wp);
+ fy_work_pool_cleanup(wp);
+
+ TDBG("%s: T#%d done WP:%p\n", __func__, tid, wp);
+}
--- /dev/null
+/*
+ * fy-thread.h - Lighting fast thread pool implementation
+ *
+ * Copyright (c) 2023 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef FY_THREAD_H
+#define FY_THREAD_H
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+#include <stdatomic.h>
+#include <pthread.h>
+
+#ifndef __APPLE__
+#include <semaphore.h>
+#else
+#include <dispatch/dispatch.h>
+#endif
+
+#include "fy-bit64.h"
+#include "fy-align.h"
+
+#include <libfyaml.h>
+
+// #define FY_THREAD_DEBUG /* define to enable debugging information to stderr */
+// #define FY_THREAD_PORTABLE /* define to use the portable implementation even on linux */
+
+struct fy_work_pool {
+ _Atomic(size_t) work_left;
+#if defined(__linux__) && !defined(FY_THREAD_PORTABLE)
+ _Atomic(uint32_t) done;
+#elif defined(__APPLE__)
+ dispatch_semaphore_t sem;
+#else
+ sem_t sem;
+#endif
+};
+
+struct fy_thread {
+ struct fy_thread_pool *tp;
+ unsigned int id;
+ pthread_t tid;
+ _Atomic(struct fy_thread_work *)work;
+ _Atomic(struct fy_thread_work *)next_work;
+#if defined(__linux__) && !defined(FY_THREAD_PORTABLE)
+ _Atomic(uint32_t) submit;
+ _Atomic(uint32_t) done;
+#else
+ pthread_mutex_t lock;
+ pthread_cond_t cond;
+ pthread_mutex_t wait_lock;
+ pthread_cond_t wait_cond;
+#endif
+};
+
+struct fy_thread_pool {
+ struct fy_thread_pool_cfg cfg;
+ unsigned int num_threads;
+ struct fy_thread *threads;
+ _Atomic(uint64_t) *freep;
+ _Atomic(uint64_t) *lootp;
+ pthread_key_t key;
+};
+
+/* those are internal only */
+int fy_thread_pool_setup(struct fy_thread_pool *tp, const struct fy_thread_pool_cfg *cfg);
+void fy_thread_pool_cleanup(struct fy_thread_pool *tp);
+
+#endif
#include <getopt.h>
#include <ctype.h>
#include <unistd.h>
+#include <errno.h>
#include <libfyaml.h>
#define OPT_PARSE_DUMP 1007
#define OPT_YAML_VERSION_DUMP 1008
#define OPT_COMPOSE 1009
+#define OPT_B3SUM 1010
#define OPT_STRIP_LABELS 2000
#define OPT_STRIP_TAGS 2001
#define OPT_YAML_1_2 4001
#define OPT_YAML_1_3 4002
+/* b3sum options */
+#define OPT_CHECK 5000
+#define OPT_DERIVE_KEY 5001
+#define OPT_NO_NAMES 5002
+#define OPT_RAW 5003
+#define OPT_KEYED 5005
+#define OPT_LENGTH 5006
+#define OPT_LIST_BACKENDS 5007
+#define OPT_BACKEND 5008
+#define OPT_NUM_THREADS 5009
+#define OPT_FILE_BUFFER 5010
+#define OPT_MMAP_MIN_CHUNK 5011
+#define OPT_MMAP_MAX_CHUNK 5012
+
static struct option lopts[] = {
{"include", required_argument, 0, 'I' },
{"debug-level", required_argument, 0, 'd' },
{"compose", no_argument, 0, OPT_COMPOSE },
{"dump-path", no_argument, 0, OPT_DUMP_PATH },
{"yaml-version-dump", no_argument, 0, OPT_YAML_VERSION_DUMP },
+ {"b3sum", no_argument, 0, OPT_B3SUM },
{"strip-labels", no_argument, 0, OPT_STRIP_LABELS },
{"strip-tags", no_argument, 0, OPT_STRIP_TAGS },
{"strip-doc", no_argument, 0, OPT_STRIP_DOC },
{"to", required_argument, 0, 'T' },
{"from", required_argument, 0, 'F' },
{"quiet", no_argument, 0, 'q' },
+
+ {"check", no_argument, 0, OPT_CHECK },
+ {"derive-key", required_argument, 0, OPT_DERIVE_KEY },
+ {"no-names", no_argument, 0, OPT_NO_NAMES },
+ {"raw", no_argument, 0, OPT_RAW },
+ {"length", required_argument, 0, OPT_LENGTH },
+ {"keyed", no_argument, 0, OPT_KEYED },
+ {"list-backends", no_argument, 0, OPT_LIST_BACKENDS },
+ {"backend", required_argument, 0, OPT_BACKEND },
+ {"num-threads", required_argument, 0, OPT_NUM_THREADS },
+ {"file-buffer", required_argument, 0, OPT_FILE_BUFFER },
+ {"mmap-min-chunk", required_argument, 0, OPT_MMAP_MIN_CHUNK },
+ {"mmap-max-chunk", required_argument, 0, OPT_MMAP_MAX_CHUNK },
+
{"help", no_argument, 0, 'h' },
{"version", no_argument, 0, 'v' },
{0, 0, 0, 0 },
fprintf(fp, "\t{\n\t \"foo\": \"bar\"\n\t}\n");
break;
case OPT_YAML_VERSION_DUMP:
- fprintf(fp, "\tDisplay information about the YAML versions libfyaml supports)\n");
+ fprintf(fp, "\tDisplay information about the YAML versions libfyaml supports\n");
+ fprintf(fp, "\n");
+ break;
+
+ case OPT_B3SUM:
+ fprintf(fp, "\tBLAKE3 hash b3sum utility\n");
+ fprintf(fp, "\t--derive-key <context> : Key derivation mode, with the given context string\n");
+ fprintf(fp, "\t--no-names : Omit filenames\n");
+ fprintf(fp, "\t--raw : Output result in raw bytes (single input allowed)\n");
+ fprintf(fp, "\t--length <n> : Output only this amount of bytes per output (max %u)\n", FY_BLAKE3_OUT_LEN);
+ fprintf(fp, "\t--check : Read files with BLAKE3 checksums and check files\n");
+ fprintf(fp, "\t--keyed : Keyed mode with secret key read from <stdin> (32 raw bytes)\n");
+ fprintf(fp, "\t--backend <backend> : Select a BLAKE3 backend instead of the default\n");
+ fprintf(fp, "\t--list-backends : Print out a list of available backends\n");
+ fprintf(fp, "\t--num-threads <n> : Number of threads, -1 disable, 0 let system decide, >= 1 explicit\n");
+ fprintf(fp, "\t--file-buffer <n> : Size of file I/O buffer (non-mmap case), 0 let system decide\n");
+ fprintf(fp, "\t--mmap-min-chunk <n> : Size of minimum mmap chunk, 0 let system decide\n");
+ fprintf(fp, "\t--mmap-max-chunk <n> : Size of maximum mmap chunk, 0 let system decide\n");
fprintf(fp, "\n");
break;
}
return FYCR_ERROR;
}
+struct b3sum_config {
+ bool no_names : 1,
+ raw : 1,
+ keyed : 1,
+ check : 1,
+ derive_key : 1,
+ quiet : 1,
+ list_backends : 1,
+ no_mmap : 1;
+ size_t file_buffer;
+ size_t mmap_min_chunk;
+ size_t mmap_max_chunk;
+ unsigned int length;
+ const char *context;
+ size_t context_len;
+ const char *backend;
+ unsigned int num_threads;
+};
+
+struct b3sum_config default_b3sum_cfg = {
+ .length = FY_BLAKE3_OUT_LEN,
+};
+
+static int do_b3sum_hash_file(struct fy_blake3_hasher *hasher, const char *filename, bool no_names, bool raw, unsigned int length)
+{
+ static const char *hexb = "0123456789abcdef";
+ const uint8_t *output;
+ size_t filename_sz, line_sz, outsz;
+ ssize_t wrn;
+ uint8_t v;
+ const void *outp;
+ char *line, *s;
+ unsigned int i;
+
+ filename_sz = strlen(filename);
+
+ output = fy_blake3_hash_file(hasher, filename);
+ if (!output) {
+ fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno));
+ return -1;
+ }
+
+ if (!raw) {
+ /* output line (optimized) */
+ line_sz = (length * 2); /* the hex output */
+ if (!no_names)
+ line_sz += 2 + filename_sz; /* 2 spaces + filename */
+ line_sz++; /* '\n' */
+ line = alloca(line_sz + 1);
+ s = line;
+ for (i = 0; i < length; i++) {
+ v = output[i];
+ *s++ = hexb[v >> 4];
+ *s++ = hexb[v & 15];
+ }
+ if (!no_names) {
+ *s++ = ' ';
+ *s++ = ' ';
+ memcpy(s, filename, filename_sz);
+ s += filename_sz;
+ }
+ *s++ = '\n';
+ *s = '\0';
+ outp = line;
+ outsz = (size_t)(s - line);
+ } else {
+ outp = output;
+ outsz = length;
+ }
+
+ wrn = fwrite(outp, 1, outsz, stdout);
+ if ((size_t)wrn != outsz) {
+ fprintf(stderr, "Unable to write to stdout! error: %s\n", strerror(errno));
+ return -1;
+ }
+
+ return 0;
+}
+
+static int do_b3sum_check_file(struct fy_blake3_hasher *hasher, const char *check_filename, bool quiet)
+{
+ char *hash, *filename;
+ FILE *fp = NULL;
+ char linebuf[8192]; /* maximum size for a line is 8K, should be enough (PATH_MAX is 4K at linux) */
+ uint8_t read_hash[FY_BLAKE3_OUT_LEN], v;
+ const uint8_t *computed_hash;
+ char *s;
+ char c;
+ unsigned int i, j, length;
+ size_t linesz;
+ int line, exit_code;
+
+ if (check_filename && strcmp(check_filename, "-")) {
+ fp = fopen(check_filename, "ra");
+ if (!fp) {
+ fprintf(stderr, "Failed to open check file: \"%s\", error: %s\n", check_filename, strerror(errno));
+ goto err_out;
+ }
+ } else {
+ fp = stdin;
+ }
+
+ /* default error code if all is fine */
+ exit_code = 0;
+
+ line = 0;
+ while (fgets(linebuf, sizeof(linebuf), fp)) {
+ /* '\0' terminate always */
+ linebuf[(sizeof(linebuf)/sizeof(linebuf[0]))-1] = '\0';
+
+ linesz = strlen(linebuf);
+ while (linesz > 0 && linebuf[linesz-1] == '\n')
+ linesz--;
+
+ if (!linesz) {
+ fprintf(stderr, "Empty line found at file \"%s\" line #%d\n", check_filename, line);
+ goto err_out;
+ }
+ linebuf[linesz] = '\0';
+
+ length = 0;
+ s = linebuf;
+ while (isxdigit(*s))
+ s++;
+
+ length = s - linebuf;
+
+ if (length == 0 || length > (FY_BLAKE3_OUT_LEN * 2) || (length % 1) || !isspace(*s)) {
+ fprintf(stderr, "Bad line found at file \"%s\" line #%d\n", check_filename, line);
+ fprintf(stderr, "%s\n", linebuf);
+ goto err_out;
+ }
+
+ *s++ = '\0';
+
+ while (isspace(*s))
+ s++;
+
+ length >>= 1; /* to bytes */
+ hash = linebuf;
+ filename = s;
+
+ for (i = 0, s = hash; i < length; i++) {
+ v = 0;
+ for (j = 0; j < 2; j++) {
+ v <<= 4;
+ c = *s++;
+ if (c >= '0' && c <= '9')
+ v |= c - '0';
+ else if (c >= 'a' && c <= 'f')
+ v |= c - 'a' + 10;
+ else if (c >= 'A' && c <= 'F')
+ v |= c - 'A' + 10;
+ else
+ v = 0;
+ }
+ read_hash[i] = v;
+ }
+
+ computed_hash = fy_blake3_hash_file(hasher, filename);
+ if (!computed_hash) {
+ fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno));
+ goto err_out;
+ }
+
+ /* constant time comparison */
+ v = 0;
+ for (i = 0; i < length; i++)
+ v |= (read_hash[i] ^ computed_hash[i]);
+
+ if (v) {
+ printf("%s: FAILED\n", filename);
+ exit_code = -1;
+ } else if (!quiet)
+ printf("%s: OK\n", filename);
+ }
+
+out:
+ if (fp && fp != stdin)
+ fclose(fp);
+
+ return exit_code;
+
+err_out:
+ exit_code = -1;
+ goto out;
+}
+
+static int
+do_b3sum(int argc, char *argv[], int optind, const struct b3sum_config *cfg)
+{
+ struct fy_blake3_hasher_cfg hcfg;
+ struct fy_blake3_hasher *hasher;
+ uint8_t key[FY_BLAKE3_OUT_LEN];
+ const char *filename;
+ int rc, num_inputs, num_ok, i;
+ size_t rdn;
+ const char *backend, *prev;
+
+ if (cfg->list_backends) {
+ prev = NULL;
+ while ((backend = fy_blake3_backend_iterate(&prev)) != NULL)
+ printf("%s\n", backend);
+ return 0;
+ }
+
+ if (cfg->quiet && !cfg->check) {
+ fprintf(stderr, "Error: --quiet may only be used together with --check\n\n");
+ return 1;
+ }
+
+ if (cfg->keyed && cfg->derive_key) {
+ fprintf(stderr, "Error: --keyed and --derive-key may not be used together\n\n");
+ return 1;
+ }
+
+ if (cfg->check && cfg->length != FY_BLAKE3_OUT_LEN) {
+ fprintf(stderr, "Error: --check and --length may not be used together\n\n");
+ return 1;
+ }
+
+ if (cfg->keyed) {
+ rdn = fread(key, 1, FY_BLAKE3_KEY_LEN, stdin);
+ if (rdn != FY_BLAKE3_KEY_LEN) {
+ if (rdn >= 0 && rdn < FY_BLAKE3_KEY_LEN)
+ fprintf(stderr, "Error: could not read secret key from <stdin>: short key\n\n");
+ else
+ fprintf(stderr, "Error: could not read secret key from <stdin>: error %s\n\n", strerror(errno));
+ return 1;
+ }
+ rc = fgetc(stdin);
+ if (rc != EOF) {
+ fprintf(stderr, "Error: garbage trailing secret key from <stdin>\n\n");
+ return -1;
+ }
+ }
+
+ num_inputs = argc - optind;
+ if (num_inputs <= 0)
+ num_inputs = 1; /* stdin mode */
+
+ if (cfg->raw && num_inputs > 1) {
+ fprintf(stderr, "Error: Raw output mode is only supported with a single input\n\n");
+ return 1;
+ }
+
+ /* we can't handle '-' in keyed mode */
+ if (cfg->keyed) {
+ for (i = optind; i < argc; i++) {
+ if (!strcmp(argv[i], "-")) {
+ fprintf(stderr, "Cannot use <stdin> in keyed mode\n");
+ return 1;
+ }
+ }
+ }
+
+ memset(&hcfg, 0, sizeof(hcfg));
+ hcfg.key = cfg->keyed ? key : NULL;
+ hcfg.context = cfg->derive_key ? cfg->context : NULL;
+ hcfg.context_len = cfg->derive_key ? cfg->context_len : 0;
+ hcfg.backend = cfg->backend;
+ hcfg.no_mmap = cfg->no_mmap;
+ hcfg.file_buffer = cfg->file_buffer;
+ hcfg.mmap_min_chunk = cfg->mmap_min_chunk;
+ hcfg.mmap_max_chunk = cfg->mmap_max_chunk;
+ hcfg.num_threads = cfg->num_threads;
+ hasher = fy_blake3_hasher_create(&hcfg);
+ if (!hasher) {
+ fprintf(stderr, "unable to create blake3 hasher\n");
+ return -1;
+ }
+
+ /* we will get in the loop even when no arguments (we'll do stdin instead) */
+ num_ok = 0;
+ i = optind;
+ do {
+ /* if no arguments, use stdin */
+ filename = i < argc ? argv[i] : "-";
+
+ if (!cfg->check)
+ rc = do_b3sum_hash_file(hasher, filename, cfg->no_names, cfg->raw, cfg->length);
+ else
+ rc = do_b3sum_check_file(hasher, filename, cfg->quiet);
+ if (!rc)
+ num_ok++;
+
+ } while (++i < argc);
+
+ fy_blake3_hasher_destroy(hasher);
+
+ return num_inputs == num_ok ? 0 : -1;
+}
+
int main(int argc, char *argv[])
{
struct fy_parse_cfg cfg = {
struct fy_emitter_cfg emit_cfg;
struct fy_parser *fyp = NULL;
struct fy_emitter *fye = NULL;
- int rc, exitcode = EXIT_FAILURE, opt, lidx, count, i, j, step = 1;
+ int rc, exitcode = EXIT_FAILURE, opt, lidx, i, j, step = 1;
enum fy_error_module errmod;
unsigned int errmod_mask;
bool show;
struct composer_data cd;
bool dump_path = DUMP_PATH_DEFAULT;
const char *input_arg;
+ /* b3sum */
+ int opti;
+ struct b3sum_config b3cfg = default_b3sum_cfg;
fy_valgrind_check(&argc, &argv);
tool_mode = OPT_COMPOSE;
else if (!strcmp(progname, "fy-yaml-version-dump"))
tool_mode = OPT_YAML_VERSION_DUMP;
+ else if (!strcmp(progname, "fy-b3sum"))
+ tool_mode = OPT_B3SUM;
else
tool_mode = OPT_TOOL;
indent = atoi(optarg);
if (indent < 0 || indent > FYECF_INDENT_MASK) {
fprintf(stderr, "bad indent option %s\n", optarg);
- display_usage(stderr, progname, tool_mode);
- return EXIT_FAILURE;
+ goto err_out_usage;
}
break;
width = atoi(optarg);
if (width < 0 || width > FYECF_WIDTH_MASK) {
fprintf(stderr, "bad width option %s\n", optarg);
- display_usage(stderr, progname, tool_mode);
- return EXIT_FAILURE;
+ goto err_out_usage;
}
break;
case 'd':
dcfg.level = fy_string_to_error_type(optarg);
if (dcfg.level == FYET_MAX) {
fprintf(stderr, "bad debug level option %s\n", optarg);
- display_usage(stderr, progname, tool_mode);
- return EXIT_FAILURE;
+ goto err_out_usage;
}
break;
case OPT_DISABLE_DIAG:
errmod = fy_string_to_error_module(optarg);
if (errmod == FYEM_MAX) {
fprintf(stderr, "bad error module option %s\n", optarg);
- display_usage(stderr, progname, tool_mode);
- return EXIT_FAILURE;
+ goto err_out_usage;
}
errmod_mask = FY_BIT(errmod);
}
} else {
fprintf(stderr, "bad %s option %s\n",
show ? "show" : "hide", optarg);
- display_usage(stderr, progname, tool_mode);
- return EXIT_FAILURE;
+ goto err_out_usage;
}
break;
du.colorize = false;
} else {
fprintf(stderr, "bad color option %s\n", optarg);
- display_usage(stderr, progname, tool_mode);
- return EXIT_FAILURE;
+ goto err_out_usage;
}
break;
case 'm':
rc = apply_mode_flags(optarg, &emit_flags);
if (rc) {
fprintf(stderr, "bad mode option %s\n", optarg);
- display_usage(stderr, progname, tool_mode);
- return EXIT_FAILURE;
+ goto err_out_usage;
}
break;
case 'V':
dcfg.output_fn = no_diag_output_fn;
dcfg.fp = NULL;
dcfg.colorize = false;
+ b3cfg.quiet = true;
break;
case 'f':
file = optarg;
case OPT_PARSE_DUMP:
case OPT_COMPOSE:
case OPT_YAML_VERSION_DUMP:
+ case OPT_B3SUM:
tool_mode = opt;
break;
case OPT_STRIP_LABELS:
cfg.flags |= FYPCF_JSON_FORCE;
else {
fprintf(stderr, "bad json option %s\n", optarg);
- display_usage(stderr, progname, tool_mode);
- return EXIT_FAILURE;
+ goto err_out_usage;
}
break;
case OPT_DISABLE_ACCEL:
break;
case OPT_DISABLE_MMAP:
cfg.flags |= FYPCF_DISABLE_MMAP_OPT;
+ b3cfg.no_mmap = true;
break;
case OPT_DUMP_PATHEXPR:
dump_pathexpr = true;
case OPT_TSV_FORMAT:
tsv_format = true;
break;
+
+ case OPT_DERIVE_KEY:
+ b3cfg.derive_key = true;
+ b3cfg.context = optarg;
+ b3cfg.context_len = strlen(optarg);
+ break;
+ case OPT_NO_NAMES:
+ b3cfg.no_names = true;
+ break;
+ case OPT_RAW:
+ b3cfg.raw = true;
+ break;
+ case OPT_CHECK:
+ b3cfg.check = true;
+ break;
+ case OPT_KEYED:
+ b3cfg.keyed = true;
+ break;
+
+ case OPT_LENGTH:
+ opti = atoi(optarg);
+ if (opti <= 0 || opti > FY_BLAKE3_OUT_LEN) {
+ fprintf(stderr, "Error: bad length=%d (must be > 0 and <= %u)\n\n", opti, FY_BLAKE3_OUT_LEN);
+ goto err_out_usage;
+ }
+ b3cfg.length = (unsigned int)opti;
+ break;
+
+ case OPT_LIST_BACKENDS:
+ b3cfg.list_backends = true;
+ break;
+
+ case OPT_BACKEND:
+ b3cfg.backend = optarg;
+ break;
+
+ case OPT_NUM_THREADS:
+ b3cfg.num_threads = atoi(optarg);
+ break;
+
+ case OPT_FILE_BUFFER:
+ opti = atoi(optarg);
+ if (opti < 0) {
+ fprintf(stderr, "Error: bad file-buffer=%d (must be >= 0)\n\n", opti);
+ goto err_out_usage;
+ }
+ b3cfg.file_buffer = (size_t)opti;
+ break;
+
+ case OPT_MMAP_MIN_CHUNK:
+ opti = atoi(optarg);
+ if (opti < 0) {
+ fprintf(stderr, "Error: bad mmap-min-chunk=%d (must be >= 0)\n\n", opti);
+ goto err_out_usage;
+ }
+ b3cfg.mmap_min_chunk = (size_t)opti;
+ break;
+
+ case OPT_MMAP_MAX_CHUNK:
+ opti = atoi(optarg);
+ if (opti < 0) {
+ fprintf(stderr, "Error: bad mmap-max-chunk=%d (must be >= 0)\n\n", opti);
+ goto err_out_usage;
+ }
+ b3cfg.mmap_max_chunk = (size_t)opti;
+ break;
+
+
case 'h' :
default:
if (opt != 'h')
}
}
+ if (tool_mode == OPT_B3SUM) {
+
+ rc = do_b3sum(argc, argv, optind, &b3cfg);
+ if (rc == 1) {
+ /* display usage */
+ goto err_out_usage;
+ }
+ exitcode = !rc ? EXIT_SUCCESS : EXIT_FAILURE;
+ goto cleanup;
+ }
+
if (tool_mode == OPT_YAML_VERSION_DUMP) {
const struct fy_version *vers;
void *iter;
break;
case OPT_DUMP:
- count = 0;
for (i = optind; ; i++) {
if (optind < argc) {
if (rc)
goto cleanup;
- count++;
}
} else {
while ((fyev = fy_parser_parse(fyp)) != NULL) {
if (!null_output) {
- rc = fy_emit_event(fye, fyev);
+ rc = fy_emit_event_from_parser(fye, fyp, fyev);
if (rc)
goto cleanup;
} else {
fy_parser_event_free(fyp, fyev);
}
}
- count++;
}
if (fy_parser_get_stream_error(fyp))
goto cleanup;
}
- count = 0;
while ((fyd = fy_parse_load_document(fyp)) != NULL) {
for (i = optind, j = 0; i < argc; i += step, j++) {
if (rc)
goto cleanup;
- count++;
}
fy_parse_document_destroy(fyp, fyd);
} else
stdin_input = false;
- count = 0;
for (;;) {
if (!stdin_input) {
goto cleanup;
}
- count = 0;
for (i = optind; i < argc; i++) {
rc = set_parser_input(fyp, argv[i], false);
if (rc) {
fy_parser_event_free(fyp, fyev);
}
}
- count++;
if (fy_parser_get_stream_error(fyp))
goto cleanup;
cd.single_document = false;
cd.verbose = dump_path;
- count = 0;
for (i = optind; i < argc; i++) {
rc = set_parser_input(fyp, argv[i], false);
if (rc) {
fprintf(stderr, "failed to set parser input to '%s' for dump\n", argv[i]);
goto cleanup;
}
-
- count++;
}
/* ignore errors for now */
}
return exitcode;
+
+err_out_usage:
+ exitcode = EXIT_FAILURE;
+ display_usage(stderr, progname, tool_mode);
+ goto cleanup;
}
--- /dev/null
+/*
+ * fy-align.h - various align related stuff
+ *
+ * Copyright (c) 2023 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef FY_ALIGN_H
+#define FY_ALIGN_H
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#if defined(__GNUC__) || defined(__clang__)
+#define FY_ALIGNED_TO(x) __attribute__ ((aligned(x)))
+#elif defined(_MSC_VER)
+#define FY_ALIGNED_TO(x) __declspec(align(x))
+#else
+#define FY_ALIGNED_TO(x) /* nothing */
+#endif
+
+#define FY_ALIGN(_align, _x) (((_x) + ((_align) - 1)) & ~((_align) - 1))
+
+/* cachelines are universally 64 bytes */
+#define FY_CACHELINE_SIZE 64
+#define FY_CACHELINE_SIZE_ALIGN(_x) FY_ALIGN(FY_CACHELINE_SIZE, _x)
+#define FY_CACHELINE_ALIGN FY_ALIGNED_TO(FY_CACHELINE_SIZE)
+
+/* provide posix_memalign for platforms that don't have it */
+#ifdef _WIN32
+static inline int posix_memalign(void **ptr, size_t align, size_t size)
+{
+ void *p;
+
+ /* must be a power of two */
+ if ((size & (size -1)) != 0) {
+ *ptr = NULL;
+ return EINVAL;
+ }
+
+ p = _aligned_malloc(size, align);
+ if (!p) {
+ *ptr = NULL;
+ return ENOMEM;
+ }
+ return p;
+}
+
+static inline void posix_memalign_free(void *p)
+{
+ _aligned_free(p);
+}
+
+#else
+/* normal implementations just use free */
+static inline void posix_memalign_free(void *p)
+{
+ free(p);
+}
+#endif
+
+static inline void *fy_align_alloc(size_t align, size_t size)
+{
+ void *p;
+ int rc;
+
+ size = FY_ALIGN(align, size);
+ rc = posix_memalign(&p, align, size);
+ if (rc)
+ return NULL;
+ return p;
+}
+
+static inline void fy_align_free(void *p)
+{
+ if (p)
+ posix_memalign_free(p);
+}
+
+static inline void *fy_cacheline_alloc(size_t size)
+{
+ return fy_align_alloc(FY_CACHELINE_SIZE, size);
+}
+
+static inline void fy_cacheline_free(void *p)
+{
+ fy_align_free(p);
+}
+
+#endif
--- /dev/null
+/*
+ * fy-bit64.h - various bit64 methods
+ *
+ * Copyright (c) 2023 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef FY_BIT64_H
+#define FY_BIT64_H
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdint.h>
+
+#define FY_BIT64(x) ((uint64_t)1 << (x))
+#define FY_BIT64_COUNT(_x) (((_x) + 63) / 64)
+#define FY_BIT64_SIZE(_x) ((size_t)((((_x) + 63) / 64) * sizeof(uint64_t)))
+
+#if (defined(__GNUC__) && __GNUC__ >= 4) || defined(__clang__)
+
+#define FY_BIT64_LOWEST(_x) ((unsigned int)__builtin_ctzll((unsigned long long)(_x)))
+#define FY_BIT64_HIGHEST(_x) ((unsigned int)__builtin_clzll((unsigned long long)(_x)))
+#define FY_BIT64_POPCNT(_x) ((unsigned int)__builtin_popcountl((unsigned long long)(_x)))
+
+static inline unsigned int fy_bit64_lowest(uint64_t x)
+{
+ return FY_BIT64_LOWEST(x);
+}
+
+static inline unsigned int fy_bit64_highest(uint64_t x)
+{
+ return FY_BIT64_HIGHEST(x);
+}
+
+static inline unsigned int FY_bit64_popcnt(uint64_t x)
+{
+ return FY_BIT64_POPCNT(x);
+}
+
+#else /* portable implementation */
+
+static inline unsigned int FY_BIT64_LOWEST(uint64_t x)
+{
+ unsigned int c = 0;
+
+ if (!(x & 0x00000000ffffffffULL)) { x >>= 32; c += 32; }
+ if (!(x & 0x000000000000ffffULL)) { x >>= 16; c += 16; }
+ if (!(x & 0x00000000000000ffULL)) { x >>= 8; c += 8; }
+ if (!(x & 0x000000000000000fULL)) { x >>= 4; c += 4; }
+ if (!(x & 0x0000000000000003ULL)) { x >>= 2; c += 2; }
+ if (!(x & 0x0000000000000001ULL)) { c += 1; }
+ return c;
+}
+
+static inline unsigned int FY_BIT64_HIGHEST(uint64_t x)
+{
+ unsigned int c = 0;
+
+ if (x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }
+ if (x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }
+ if (x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }
+ if (x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }
+ if (x & 0x000000000000000cULL) { x >>= 2; c += 2; }
+ if (x & 0x0000000000000002ULL) { c += 1; }
+ return c;
+}
+
+static inline unsigned int FY_BIT64_POPCNT(uint64_t x)
+{
+ unsigned int count;
+
+ for (count = 0; x; x &= x - 1)
+ count++;
+ return count;
+}
+
+#define FY_BIT64_LOWEST(_x) fy_bit64_lowest(_x)
+#define FY_BIT64_HIGEST(_x) fy_bit64_highest(_x)
+#define FY_BIT64_POPCNT(_x) fy_bit64_popcnt(_x)
+
+#endif
+
+#endif
--- /dev/null
+/*
+ * fy-blob.c - binary blob handling
+ *
+ * Copyright (c) 2023 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <stdlib.h>
+
+#include "fy-blob.h"
+
+/* define optimized methods for probing */
+BR_DEFINE_WX(br_probe_w, 8, false, false);
+BR_DEFINE_WX(br_probe_w, 16, false, false);
+BR_DEFINE_WX(br_probe_w, 32, false, false);
+BR_DEFINE_WX(br_probe_w, 64, false, false);
+
+/* define optimized methods for writing */
+BR_DEFINE_WX(br_native_w, 8, false, true);
+BR_DEFINE_WX(br_native_w, 16, false, true);
+BR_DEFINE_WX(br_native_w, 32, false, true);
+BR_DEFINE_WX(br_native_w, 64, false, true);
+
+BR_DEFINE_WX(br_bswap_w, 8, true, true);
+BR_DEFINE_WX(br_bswap_w, 16, true, true);
+BR_DEFINE_WX(br_bswap_w, 32, true, true);
+BR_DEFINE_WX(br_bswap_w, 64, true, true);
+
+void *fy_blob_read(const char *file, size_t *sizep)
+{
+ struct stat sb;
+ void *blob = NULL;
+ uint8_t *p;
+ size_t size, left;
+ ssize_t rdn;
+ int fd = -1, rc;
+
+ if (!file || !sizep)
+ goto err_out;
+
+ fd = open(file, O_RDONLY);
+ if (fd < 0)
+ goto err_out;
+
+ rc = fstat(fd, &sb);
+ if (rc < 0)
+ goto err_out;
+
+ size = sb.st_size;
+ blob = malloc(size);
+ if (!blob)
+ goto err_out;
+
+ p = blob;
+ left = size;
+ while (left > 0) {
+ do {
+ rdn = read(fd, p, left);
+ } while (rdn == -1 && errno == EAGAIN);
+ if (rdn < 0)
+ goto err_out;
+ p += rdn;
+ left -= rdn;
+ }
+ close(fd);
+
+ *sizep = size;
+ return blob;
+
+err_out:
+ if (blob)
+ free(blob);
+ if (fd >= 0)
+ close(fd);
+ return NULL;
+}
+
+int fy_blob_write(const char *file, const void *blob, size_t size)
+{
+ int fd = -1;
+ const uint8_t *p;
+ size_t left;
+ ssize_t wrn;
+
+ if (!file || !blob)
+ return -1;
+
+ fd = open(file, O_WRONLY|O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IRGRP | S_IROTH);
+ if (fd < 0)
+ goto err_out;
+
+ p = blob;
+ left = size;
+ while (left > 0) {
+ do {
+ wrn = write(fd, p, left);
+ } while (wrn == -1 && errno == EAGAIN);
+ if (wrn < 0)
+ goto err_out;
+ p += wrn;
+ left -= wrn;
+ }
+ close(fd);
+
+ return 0;
+
+err_out:
+ if (fd >= 0)
+ close(fd);
+ return -1;
+}
--- /dev/null
+/*
+ * fy-blob.h - binary blob handling support
+ *
+ * Copyright (c) 2023 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef FY_BLOB_H
+#define FY_BLOB_H
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <string.h>
+#include <stdint.h>
+#include <assert.h>
+#ifdef HAVE_BYTESWAP_H
+#include <byteswap.h>
+#else
+#ifdef HAVE___BUILTIN_BSWAP16
+#define bswap_16(value) __builtin_bswap16(value)
+#else
+#define bswap_16(value) ((((value)&0xff) << 8) | ((value) >> 8))
+#endif
+
+#ifdef HAVE___BUILTIN_BSWAP32
+#define bswap_32(value) __builtin_bswap32(value)
+#else
+#define bswap_32(value) \
+ (((uint32_t)bswap_16((uint16_t)((value)&0xffff)) << 16) | \
+ (uint32_t)bswap_16((uint16_t)((value) >> 16)))
+#endif
+
+#ifdef HAVE___BUILTIN_BSWAP64
+#define bswap_64(value) __builtin_bswap64(value)
+#else
+#define bswap_64(value) \
+ (((uint64_t)bswap_32((uint32_t)((value)&0xffffffff)) << 32) | \
+ (uint64_t)bswap_32((uint32_t)((value) >> 32)))
+#endif
+#endif
+
+#include <stdbool.h>
+#include <time.h>
+
+#include "fy-endian.h"
+
+/* special unaligned types for pointer accesses */
+
+#if defined(__GNUC__) && __GNUC__ >= 4
+typedef union { uint64_t v; } __attribute__((__packed__)) br_64;
+typedef union { uint32_t v; } __attribute__((__packed__)) br_32;
+typedef union { uint16_t v; } __attribute__((__packed__)) br_16;
+typedef union { uint8_t v; } br_8;
+#elif defined(_MSC_VER)
+#pragma pack(push, 1)
+typedef union { uint64_t v; } br_64;
+typedef union { uint32_t v; } br_32;
+typedef union { uint16_t v; } br_16;
+typedef union { uint8_t v; } br_8;
+#pragma pack(pop)
+#else
+#error Unsupported compiler
+#endif
+
+enum blob_id_size {
+ BID_U8,
+ BID_U16,
+ BID_U32,
+ BID_U64,
+};
+
+static inline enum blob_id_size
+blob_count_to_id_size(unsigned long long count)
+{
+ if (count < (1LLU << 8))
+ return BID_U8;
+ if (count < (1LLU << 16))
+ return BID_U16;
+ if (count < (1LLU << 32))
+ return BID_U32;
+ return BID_U64;
+}
+
+static inline unsigned int
+blob_id_size_to_byte_size(enum blob_id_size size)
+{
+ return 1U << size;
+}
+
+static inline unsigned int
+blob_id_size_to_bit_size(enum blob_id_size size)
+{
+ return blob_id_size_to_byte_size(size) * 8;
+}
+
+enum blob_endian_type {
+ BET_NATIVE_ENDIAN,
+ BET_LITTLE_ENDIAN,
+ BET_BIG_ENDIAN
+};
+
+struct blob_region {
+ union {
+ uint8_t *wstart;
+ const uint8_t *rstart;
+ };
+ size_t size;
+ enum blob_endian_type endian;
+ bool bswap;
+ size_t curr;
+};
+
+#define BR_DEFINE_WX(_pfx, _bits, _bswap, _write) \
+size_t _pfx ## _bits (struct blob_region *br, uint ## _bits ## _t v) \
+{ \
+ size_t pos; \
+\
+ assert(br->curr < br->size); \
+ if (_write) { \
+ if (_bswap) \
+ v = bswap_ ## _bits (v); \
+ ((br_ ## _bits *)(br->wstart + br->curr))->v = v; \
+ } \
+ pos = br->curr; \
+ br->curr += sizeof(v); \
+ return pos; \
+} \
+struct __useless_struct_to_allow_semicolon
+
+#define BR_DEFINE_RX(_pfx, _bits, _bswap) \
+uint ## _bits ## _t _pfx ## _bits (struct blob_region *br) \
+{ \
+ uint ## _bits ## _t v; \
+\
+ assert(br->curr + sizeof(v) <= br->size); \
+ assert(br->rstart); \
+ v = ((const br_ ## _bits *)(br->rstart + br->curr))->v; \
+ if (_bswap) \
+ v = bswap_ ## _bits (v); \
+ br->curr += sizeof(v); \
+ return v; \
+} \
+struct __useless_struct_to_allow_semicolon
+
+static inline void br_setup_common(struct blob_region *br, size_t size, enum blob_endian_type endian)
+{
+ memset(br, 0, sizeof(*br));
+ br->size = size;
+ br->endian = endian;
+
+ switch (br->endian) {
+ case BET_NATIVE_ENDIAN:
+ /* native endian never needs swap */
+ br->bswap = false;
+ break;
+
+ case BET_LITTLE_ENDIAN:
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+ br->bswap = false;
+#else
+ br->bswap = true;
+#endif
+ break;
+
+ case BET_BIG_ENDIAN:
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+ br->bswap = true;
+#else
+ br->bswap = false;
+#endif
+ break;
+ }
+
+ br->curr = 0;
+}
+
+static inline void br_wsetup(struct blob_region *br, void *data, size_t size, enum blob_endian_type endian)
+{
+ /* write with null data means probe */
+ br_setup_common(br, size, endian);
+ br->wstart = data;
+}
+
+static inline void br_rsetup(struct blob_region *br, const void *data, size_t size, enum blob_endian_type endian)
+{
+ /* read makes no sense without data */
+ assert(data);
+ br_setup_common(br, size, endian);
+ br->rstart = data;
+}
+
+static inline void br_reset(struct blob_region *br)
+{
+ br->curr = 0;
+}
+
+static inline size_t br_curr(struct blob_region *br)
+{
+ return br->curr;
+}
+
+static inline size_t br_write(struct blob_region *br, const void *data, size_t size)
+{
+ size_t pos;
+
+ assert(br->curr + size <= br->size);
+ if (br->wstart)
+ memcpy(br->wstart + br->curr, data, size);
+ pos = br->curr;
+ br->curr += size;
+ return pos;
+}
+
+static inline size_t br_w0(struct blob_region *br, size_t size)
+{
+ size_t pos;
+
+ assert(br->curr + size <= br->size);
+ if (br->wstart)
+ memset(br->wstart + br->curr, 0, size);
+ pos = br->curr;
+ br->curr += size;
+ return pos;
+}
+
+static inline size_t br_wskip(struct blob_region *br, size_t size)
+{
+ return br_w0(br, size);
+}
+
+static inline size_t br_wskip_to(struct blob_region *br, size_t offset)
+{
+ assert(br->curr <= offset);
+ return br_w0(br, offset - br->curr);
+}
+
+static inline BR_DEFINE_WX(br_w, 8, br->bswap, br->wstart != NULL);
+static inline BR_DEFINE_WX(br_w, 16, br->bswap, br->wstart != NULL);
+static inline BR_DEFINE_WX(br_w, 32, br->bswap, br->wstart != NULL);
+static inline BR_DEFINE_WX(br_w, 64, br->bswap, br->wstart != NULL);
+
+typedef size_t (*br_wid_func)(struct blob_region *br, int id);
+
+static inline size_t br_wid8(struct blob_region *br, int id)
+{
+ assert((unsigned int)id < ((unsigned int)1 << 8));
+ return br_w8(br, (uint8_t)id);
+}
+
+static inline size_t br_wid16(struct blob_region *br, int id)
+{
+ assert((unsigned int)id < ((unsigned int)1 << 16));
+ return br_w16(br, (uint16_t)id);
+}
+
+static inline size_t br_wid32(struct blob_region *br, int id)
+{
+ assert((unsigned long)id < ((unsigned long)1 << 32));
+ return br_w32(br, (uint32_t)id);
+}
+
+static inline size_t br_wid64(struct blob_region *br, int id)
+{
+ assert(id >= 0);
+ return br_w64(br, (uint64_t)id);
+}
+
+static inline br_wid_func br_wid_get_func(enum blob_id_size id_size)
+{
+ switch (id_size) {
+ case BID_U8:
+ return br_wid8;
+
+ case BID_U16:
+ return br_wid16;
+
+ case BID_U32:
+ return br_wid32;
+
+ case BID_U64:
+ return br_wid64;
+ }
+ return NULL;
+}
+
+static inline size_t br_wstr(struct blob_region *br, const char *str, size_t len)
+{
+ if (len == (size_t)-1)
+ len = strlen(str);
+ return br_write(br, str, len + 1);
+}
+
+static inline size_t br_wX(struct blob_region *br, enum blob_id_size x_size, uint64_t x)
+{
+ switch (x_size) {
+ case BID_U8:
+ assert(x < ((uint64_t)1 << 8));
+ return br_w8(br, (uint8_t)x);
+
+ case BID_U16:
+ assert(x < ((uint64_t)1 << 16));
+ return br_w16(br, (uint16_t)x);
+
+ case BID_U32:
+ assert(x < ((uint64_t)1 << 32));
+ return br_w32(br, (uint32_t)x);
+
+ case BID_U64:
+ return br_w64(br, x);
+ }
+ return (size_t)-1;
+}
+
+static inline size_t br_wid(struct blob_region *br, enum blob_id_size id_size, int id)
+{
+ assert(id >= 0);
+ switch (id_size) {
+ case BID_U8:
+ return br_wid8(br, id);
+
+ case BID_U16:
+ return br_wid16(br, id);
+
+ case BID_U32:
+ return br_wid32(br, id);
+
+ case BID_U64:
+ return br_wid64(br, id);
+ }
+ return br->curr;
+}
+
+static inline size_t br_read(struct blob_region *br, void *data, size_t size)
+{
+ size_t pos;
+
+ assert(br->curr + size <= br->size);
+ assert(br->rstart);
+ memcpy(data, br->rstart + br->curr, size);
+ pos = br->curr;
+ br->curr += size;
+ return pos;
+}
+
+static inline void br_rskip(struct blob_region *br, size_t size)
+{
+ assert(br->curr + size <= br->size);
+ br->curr += size;
+}
+
+static inline void br_rskip_to(struct blob_region *br, size_t offset)
+{
+ assert(br->curr <= offset);
+ assert(offset <= br->size);
+ br->curr = offset;
+}
+
+static inline BR_DEFINE_RX(br_r, 8, br->bswap);
+static inline BR_DEFINE_RX(br_r, 16, br->bswap);
+static inline BR_DEFINE_RX(br_r, 32, br->bswap);
+static inline BR_DEFINE_RX(br_r, 64, br->bswap);
+
+typedef int (*br_rid_func)(struct blob_region *br);
+
+static inline int br_rid8(struct blob_region *br)
+{
+ return (int)br_r8(br);
+}
+
+static inline int br_rid16(struct blob_region *br)
+{
+ return (int)br_r16(br);
+}
+
+static inline int br_rid32(struct blob_region *br)
+{
+ return (int)br_r32(br);
+}
+
+static inline int br_rid64(struct blob_region *br)
+{
+ return (int)br_r64(br);
+}
+
+static inline br_rid_func br_rid_get_func(enum blob_id_size id_size)
+{
+ switch (id_size) {
+ case BID_U8:
+ return br_rid8;
+
+ case BID_U16:
+ return br_rid16;
+
+ case BID_U32:
+ return br_rid32;
+
+ case BID_U64:
+ return br_rid64;
+ }
+ return NULL;
+}
+
+static inline uint64_t br_rX(struct blob_region *br, enum blob_id_size x_size)
+{
+ switch (x_size) {
+ case BID_U8:
+ return br_r8(br);
+
+ case BID_U16:
+ return br_r16(br);
+
+ case BID_U32:
+ return br_r32(br);
+
+ case BID_U64:
+ return br_r64(br);
+ }
+ return (uint64_t)-1;
+}
+
+static inline int br_rid(struct blob_region *br, enum blob_id_size id_size)
+{
+ int id;
+
+ switch (id_size) {
+ case BID_U8:
+ id = br_rid8(br);
+ break;
+
+ case BID_U16:
+ id = br_rid16(br);
+ break;
+
+ case BID_U32:
+ id = br_rid32(br);
+ break;
+
+ case BID_U64:
+ id = br_rid64(br);
+ break;
+
+ default:
+ id = -1;
+ break;
+ }
+ assert(id >= 0);
+ return id;
+}
+
+void *fy_blob_read(const char *file, size_t *sizep);
+int fy_blob_write(const char *file, const void *blob, size_t size);
+
+#endif
--- /dev/null
+/*
+ * fy-ctype.c - ctype utilities
+ *
+ * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "fy-ctype.h"
+
+const char *fy_uri_esc(const char *s, size_t len, uint8_t *code, int *code_len)
+{
+ const char *e = s + len;
+ int j, k, width;
+ uint8_t octet;
+ char c;
+
+ width = 0;
+ k = 0;
+ do {
+ /* check for enough space for %XX */
+ if ((e - s) < 3)
+ return NULL;
+
+ /* if more than one run, expect '%' */
+ if (s[0] != '%')
+ return NULL;
+
+ octet = 0;
+ for (j = 0; j < 2; j++) {
+ c = s[1 + j];
+ octet <<= 4;
+ if (c >= '0' && c <= '9')
+ octet |= c - '0';
+ else if (c >= 'a' && c <= 'f')
+ octet |= 10 + c - 'a';
+ else
+ octet |= 10 + c - 'A';
+ }
+ if (!width) {
+ width = fy_utf8_width_by_first_octet(octet);
+ if (!width)
+ return NULL;
+ k = 0;
+ }
+ if (k >= *code_len)
+ return NULL;
+
+ code[k++] = octet;
+
+ /* skip over the 3 character escape */
+ s += 3;
+
+ } while (--width > 0);
+
+ *code_len = k;
+
+ return s;
+}
+
--- /dev/null
+/*
+ * fy-ctype.h - ctype like macros header
+ *
+ * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef FY_CTYPE_H
+#define FY_CTYPE_H
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include "fy-utf8.h"
+
+enum fy_lb_mode {
+ fylb_cr_nl, /* only \r, \n (json + >= yaml1.2 */
+ fylb_cr_nl_N_L_P, /* NEL/LS/PS (yaml1.1) */
+};
+
+enum fy_flow_ws_mode {
+ fyfws_space_tab, /* space + TAB (yaml) */
+ fyfws_space, /* only space (json) */
+};
+
+static inline bool fy_is_first_alpha(int c)
+{
+ return (c >= 'a' && c <= 'z') ||
+ (c >= 'A' && c <= 'Z') ||
+ c == '_';
+}
+
+static inline bool fy_is_alpha(int c)
+{
+ return fy_is_first_alpha(c) || c == '-';
+}
+
+static inline bool fy_is_num(int c)
+{
+ return c >= '0' && c <= '9';
+}
+
+static inline bool fy_is_digit(int c)
+{
+ return c >= '0' && c <= '9';
+}
+
+static inline bool fy_is_xdigit(int c)
+{
+ return fy_is_digit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+}
+
+static inline bool fy_is_first_alnum(int c)
+{
+ return fy_is_first_alpha(c);
+}
+
+static inline bool fy_is_alnum(int c)
+{
+ return fy_is_alpha(c) || fy_is_num(c);
+}
+
+static inline bool fy_is_space(int c)
+{
+ return c == ' ';
+}
+
+static inline bool fy_is_tab(int c)
+{
+ return c == '\t';
+}
+
+static inline bool fy_is_ws(int c)
+{
+ return fy_is_space(c) || fy_is_tab(c);
+}
+
+static inline bool fy_is_hex(int c)
+{
+ return (c >= '0' && c <= '9') ||
+ (c >= 'a' && c <= 'f') ||
+ (c >= 'A' && c <= 'F');
+}
+
+static inline bool fy_is_uri(int c)
+{
+ return fy_is_alnum(c) || fy_utf8_strchr(";/?:@&=+$,.!~*\'()[]%", c);
+}
+
+static inline bool fy_is_lb_r_n(int c)
+{
+ return c == '\r' || c == '\n';
+}
+
+static inline bool fy_is_lb_NEL(int c)
+{
+ return c == 0x85;
+}
+
+static inline bool fy_is_lb_LS_PS(int c)
+{
+ return c == 0x2028 || c == 0x2029;
+}
+
+static inline bool fy_is_unicode_lb(int c)
+{
+ /* note that YAML1.1 supports NEL #x85, LS #x2028 and PS #x2029 as linebreaks */
+ /* YAML1.2 and higher does not */
+ return fy_is_lb_NEL(c) || fy_is_lb_LS_PS(c);
+}
+
+static inline bool fy_is_any_lb(int c)
+{
+ return fy_is_lb_r_n(c) || fy_is_unicode_lb(c);
+}
+
+static inline bool fy_is_z(int c)
+{
+ return c <= 0;
+}
+
+static inline bool fy_is_blank(int c)
+{
+ return c == ' ' || c == '\t';
+}
+
+#define FY_UTF8_BOM 0xfeff
+
+static inline bool fy_is_print(int c)
+{
+ return c == '\n' || c == '\r' ||
+ (c >= 0x0020 && c <= 0x007e) ||
+ (c >= 0x00a0 && c <= 0xd7ff) ||
+ (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM);
+}
+
+static inline bool fy_is_printq(int c)
+{
+ return c != '\t' && c != 0xa0 && !fy_is_any_lb(c) && fy_is_print(c);
+}
+
+static inline bool fy_is_nb_char(int c)
+{
+ return (c >= 0x0020 && c <= 0x007e) ||
+ (c >= 0x00a0 && c <= 0xd7ff) ||
+ (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM);
+}
+
+static inline bool fy_is_ns_char(int c)
+{
+ return fy_is_nb_char(c) && !fy_is_ws(c);
+}
+
+static inline bool fy_is_start_indicator(int c)
+{
+ return !!fy_utf8_strchr(",[]{}#&*!|>'\"%%@`", c);
+}
+
+static inline bool fy_is_indicator_before_space(int c)
+{
+ return !!fy_utf8_strchr("-:?`", c);
+}
+
+static inline bool fy_is_flow_indicator(int c)
+{
+ return !!fy_utf8_strchr(",[]{}", c);
+}
+
+static inline bool fy_is_path_flow_scalar_start(int c)
+{
+ return c == '\'' || c == '"';
+}
+
+static inline bool fy_is_path_flow_key_start(int c)
+{
+ return c == '"' || c == '\'' || c == '{' || c == '[';
+}
+
+static inline bool fy_is_path_flow_key_end(int c)
+{
+ return c == '"' || c == '\'' || c == '}' || c == ']';
+}
+
+static inline bool fy_is_unicode_control(int c)
+{
+ return (c >= 0 && c <= 0x1f) || (c >= 0x80 && c <= 0x9f);
+}
+
+static inline bool fy_is_unicode_space(int c)
+{
+ return c == 0x20 || c == 0xa0 ||
+ (c >= 0x2000 && c <= 0x200a) ||
+ c == 0x202f || c == 0x205f || c == 0x3000;
+}
+
+static inline bool fy_is_json_unescaped(int c)
+{
+ return c >= 0x20 && c <= 0x110000 && c != '"' && c != '\\';
+}
+
+static inline bool fy_is_json_unescaped_range_only(int c)
+{
+ return c >= 0x20 && c <= 0x110000;
+}
+
+static inline bool fy_is_lb_m(int c, enum fy_lb_mode lb_mode)
+{
+ if (fy_is_lb_r_n(c))
+ return true;
+ return lb_mode == fylb_cr_nl_N_L_P && fy_is_unicode_lb(c);
+}
+
+static inline bool fy_is_generic_lb_m(int c, enum fy_lb_mode lb_mode)
+{
+ if (fy_is_lb_r_n(c))
+ return true;
+ return lb_mode == fylb_cr_nl_N_L_P && fy_is_lb_NEL(c);
+}
+
+static inline bool fy_is_lbz_m(int c, enum fy_lb_mode lb_mode)
+{
+ return fy_is_lb_m(c, lb_mode) || fy_is_z(c);
+}
+
+static inline bool fy_is_generic_lbz_m(int c, enum fy_lb_mode lb_mode)
+{
+ return fy_is_generic_lb_m(c, lb_mode) || fy_is_z(c);
+}
+
+static inline bool fy_is_blankz_m(int c, enum fy_lb_mode lb_mode)
+{
+ return fy_is_ws(c) || fy_is_lbz_m(c, lb_mode);
+}
+
+static inline bool fy_is_generic_blankz_m(int c, enum fy_lb_mode lb_mode)
+{
+ return fy_is_ws(c) || fy_is_generic_lbz_m(c, lb_mode);
+}
+
+static inline bool fy_is_flow_ws_m(int c, enum fy_flow_ws_mode fws_mode)
+{
+ return fy_is_space(c) || (fws_mode == fyfws_space_tab && fy_is_tab(c));
+}
+
+#define FY_CTYPE_AT_BUILDER(_kind) \
+static inline const void * \
+fy_find_ ## _kind (const void *s, size_t len) \
+{ \
+ const void *e = s + len; \
+ int c, w; \
+ for (; s < e && (c = fy_utf8_get(s, e - s, &w)) >= 0; s += w) { \
+ assert(w); \
+ if (fy_is_ ## _kind (c)) \
+ return s; \
+ } \
+ return NULL; \
+} \
+static inline const void * \
+fy_find_non_ ## _kind (const void *s, size_t len) \
+{ \
+ const void *e = s + len; \
+ int c, w; \
+ for (; s < e && (c = fy_utf8_get(s, e - s, &w)) >= 0; s += w) { \
+ assert(w); \
+ if (!(fy_is_ ## _kind (c))) \
+ return s; \
+ assert(w); \
+ } \
+ return NULL; \
+} \
+struct useless_struct_for_semicolon
+
+FY_CTYPE_AT_BUILDER(first_alpha);
+FY_CTYPE_AT_BUILDER(alpha);
+FY_CTYPE_AT_BUILDER(num);
+FY_CTYPE_AT_BUILDER(digit);
+FY_CTYPE_AT_BUILDER(xdigit);
+FY_CTYPE_AT_BUILDER(first_alnum);
+FY_CTYPE_AT_BUILDER(alnum);
+FY_CTYPE_AT_BUILDER(space);
+FY_CTYPE_AT_BUILDER(tab);
+FY_CTYPE_AT_BUILDER(ws);
+FY_CTYPE_AT_BUILDER(hex);
+FY_CTYPE_AT_BUILDER(uri);
+FY_CTYPE_AT_BUILDER(z);
+FY_CTYPE_AT_BUILDER(any_lb);
+FY_CTYPE_AT_BUILDER(blank);
+FY_CTYPE_AT_BUILDER(print);
+FY_CTYPE_AT_BUILDER(printq);
+FY_CTYPE_AT_BUILDER(nb_char);
+FY_CTYPE_AT_BUILDER(ns_char);
+FY_CTYPE_AT_BUILDER(start_indicator);
+FY_CTYPE_AT_BUILDER(indicator_before_space);
+FY_CTYPE_AT_BUILDER(flow_indicator);
+FY_CTYPE_AT_BUILDER(path_flow_key_start);
+FY_CTYPE_AT_BUILDER(path_flow_key_end);
+FY_CTYPE_AT_BUILDER(unicode_control);
+FY_CTYPE_AT_BUILDER(unicode_space);
+FY_CTYPE_AT_BUILDER(json_unescaped);
+
+/*
+ * Very special linebreak/ws methods
+ * Things get interesting due to \r\n and
+ * unicode linebreaks/spaces
+ */
+
+/* skip for a _single_ linebreak */
+static inline const void *fy_skip_lb(const void *ptr, size_t left)
+{
+ int c, width;
+
+ /* get the utf8 character at this point */
+ c = fy_utf8_get(ptr, left, &width);
+ if (c < 0 || !fy_is_any_lb(c))
+ return NULL;
+
+ /* MS-DOS: check if next character is '\n' */
+ if (c == '\r' && left > (size_t)width && *(char *)ptr == '\n')
+ width++;
+
+ return ptr + width;
+}
+
+/* given a pointer to a chunk of memory, return pointer to first
+ * ws character after the last non-ws character, or the end
+ * of the chunk
+ */
+static inline const void *fy_last_non_ws(const void *ptr, size_t left)
+{
+ const char *s, *e;
+ int c;
+
+ s = ptr;
+ e = s + left;
+ while (e > s) {
+ c = e[-1];
+ if (c != ' ' && c != '\t')
+ return e;
+ e--;
+
+ }
+ return NULL;
+}
+
+const char *fy_uri_esc(const char *s, size_t len, uint8_t *code, int *code_len);
+
+#endif
--- /dev/null
+/*
+ * fy-endian.h - simple system endian header wrapper
+ *
+ * Copyright (c) 2023 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef FY_ENDIAN_H
+#define FY_ENDIAN_H
+
+#if defined(__linux__) || defined(__CYGWIN__) || defined(__OpenBSD__)
+# include <endian.h>
+#elif defined(__APPLE__)
+# include <libkern/OSByteOrder.h>
+# include <machine/endian.h>
+#elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__)
+# include <sys/endian.h>
+#elif defined(_MSC_VER)
+# include <winsock2.h>
+# ifdef __GNUC__
+# include <sys/param.h>
+# endif
+#else
+# error unsupported platform
+#endif
+
+#if !defined(__BYTE_ORDER) && defined(BYTE_ORDER)
+#define __BYTE_ORDER BYTE_ORDER
+#endif
+
+#if !defined(__BIG_ENDIAN) && defined(BIG_ENDIAN)
+#define __BIG_ENDIAN BIG_ENDIAN
+#endif
+
+#if !defined(__LITTLE_ENDIAN) && defined(LITTLE_ENDIAN)
+#define __LITTLE_ENDIAN LITTLE_ENDIAN
+#endif
+
+#if !defined(__BYTE_ORDER) || !defined(__BIG_ENDIAN) || !defined(__LITTLE_ENDIAN)
+# error Platform does not define endian macros
+#endif
+
+/* no-one cares about PDP endian anymore */
+
+/* make the macros work for 8 bit too */
+#ifndef bswap_8
+#define bswap_8(x) (x)
+#endif
+
+#endif
--- /dev/null
+/*
+ * fy-id.h - small ID allocator
+ *
+ * Copyright (c) 2023 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef FY_ID_H
+#define FY_ID_H
+
+#include <stdlib.h>
+#include <strings.h>
+#include <string.h>
+#include <assert.h>
+
+#include <stdint.h>
+
+typedef unsigned int fy_id_bits;
+
+#define FY_ID_BITS_SZ ((size_t)(sizeof(fy_id_bits)))
+#define FY_ID_BITS_BITS (FY_ID_BITS_SZ * 8) /* yes, every character is 8 bits in 2023 */
+#define FY_ID_BITS_MASK ((size_t)(FY_ID_BITS_BITS - 1))
+
+#define FY_ID_BITS_ARRAY_COUNT_BITS(_bits) \
+ (((_bits) + FY_ID_BITS_MASK) & ~FY_ID_BITS_MASK)
+
+#define FY_ID_BITS_ARRAY_COUNT(_bits) \
+ (FY_ID_BITS_ARRAY_COUNT_BITS(_bits) / FY_ID_BITS_BITS)
+
+#define FY_ID_OFFSET(_id) ((_id) / FY_ID_BITS_BITS)
+#define FY_ID_BIT_MASK(_id) ((fy_id_bits)1 << ((_id) & FY_ID_BITS_MASK))
+
+#if defined(__GNUC__) && __GNUC__ >= 4
+static inline int fy_id_ffs(const fy_id_bits id_bit)
+{
+ if (!id_bit)
+ return -1;
+ return __builtin_ffs(id_bit) - 1;
+}
+
+#else
+static inline int fy_id_ffs(const fy_id_bits id_bit)
+{
+ return ffs((int)id_bits) - 1;
+}
+#endif
+
+#if defined(__GNUC__) && __GNUC__ >= 4
+static inline int fy_id_popcount(const fy_id_bits id_bit)
+{
+ return __builtin_popcount(id_bit);
+}
+
+#else
+static inline int fy_id_popcount(fy_id_bits id_bit)
+{
+ int count, pos;
+
+ count = 0;
+ while ((pos = fy_id_ffs(id_bit)) >= 0) {
+ count++;
+ id_bit &= ~((fy_id_bits)1 << pos);
+ }
+ return count;
+}
+#endif
+
+static inline void fy_id_reset(fy_id_bits *bits, const size_t count)
+{
+ memset(bits, 0, sizeof(*bits) * count);
+}
+
+static inline int fy_id_alloc(fy_id_bits *bits, const size_t count)
+{
+ size_t i;
+ int pos;
+
+ for (i = 0; i < count; i++, bits++) {
+ pos = fy_id_ffs(~*bits);
+ if (pos >= 0) {
+ *bits |= FY_ID_BIT_MASK(pos);
+ return (i * FY_ID_BITS_BITS) + pos;
+ }
+ }
+ return -1;
+}
+
+static inline int fy_id_alloc_fixed(fy_id_bits *bits, const size_t count, const int id)
+{
+ size_t offset;
+ fy_id_bits m;
+
+ offset = FY_ID_OFFSET(id);
+ if (offset >= count)
+ return -1;
+ bits += offset;
+ m = FY_ID_BIT_MASK(id);
+ if (*bits & m)
+ return -1;
+ *bits |= m;
+ return id;
+}
+
+static inline bool fy_id_is_used(fy_id_bits *bits, const size_t count, const int id)
+{
+ size_t offset;
+
+ offset = FY_ID_OFFSET(id);
+ if (offset >= count)
+ return false;
+ return !!(bits[offset] & FY_ID_BIT_MASK(id));
+}
+
+static inline bool fy_id_is_used_no_check(fy_id_bits *bits, const size_t count, const int id)
+{
+ size_t offset;
+
+ offset = FY_ID_OFFSET(id);
+ if (offset >= count)
+ return false;
+ return !!(bits[offset] & FY_ID_BIT_MASK(id));
+}
+
+static inline bool fy_id_is_free(fy_id_bits *bits, const size_t count, const int id)
+{
+ size_t offset;
+
+ offset = FY_ID_OFFSET(id);
+ if (offset >= count)
+ return false;
+ return !(bits[offset] & FY_ID_BIT_MASK(id));
+}
+
+static inline void fy_id_free(fy_id_bits *bits, const size_t count, int id)
+{
+ size_t offset;
+
+ offset = FY_ID_OFFSET(id);
+ if (offset >= count)
+ return;
+ bits += offset;
+ *bits &= ~FY_ID_BIT_MASK(id); /* no need to check, if bit is free, it is still free */
+}
+
+static inline void fy_id_set_used(fy_id_bits *bits, const size_t count, const int id)
+{
+ assert(FY_ID_OFFSET(id) < count);
+ bits[FY_ID_OFFSET(id)] |= FY_ID_BIT_MASK(id);
+}
+
+static inline void fy_id_set_free(fy_id_bits *bits, const size_t count, const int id)
+{
+ assert(FY_ID_OFFSET(id) < count);
+ bits[FY_ID_OFFSET(id)] &= ~FY_ID_BIT_MASK(id);
+}
+
+static inline size_t fy_id_count_used(fy_id_bits *bits, const size_t count)
+{
+ size_t i, popcount;
+
+ popcount = 0;
+ for (i = 0; i < count; i++, bits++)
+ popcount += fy_id_popcount(*bits);
+ return popcount;
+}
+
+static inline size_t fy_id_count_free(fy_id_bits *bits, const size_t count)
+{
+ size_t i, popcount;
+
+ popcount = 0;
+ for (i = 0; i < count; i++, bits++)
+ popcount += fy_id_popcount(~*bits);
+ return popcount;
+}
+
+struct fy_id_iter {
+ const fy_id_bits *s;
+ fy_id_bits m;
+};
+
+static inline void fy_id_iter_begin(const fy_id_bits *bits, const size_t count, struct fy_id_iter *iterp)
+{
+ iterp->s = bits;
+ iterp->m = ~(fy_id_bits)0;
+}
+
+static inline int fy_id_iter_next(const fy_id_bits *bits, const size_t count, struct fy_id_iter *iterp)
+{
+ const fy_id_bits *s, *e;
+ fy_id_bits m;
+ int pos;
+
+ s = iterp->s;
+ if (!s)
+ return -1;
+
+ m = iterp->m;
+ e = bits + count;
+
+ if (s >= e)
+ return -1;
+
+ /* scan until we find a set bit or hit the end */
+ pos = -1;
+ while (s < e && (pos = fy_id_ffs(*s & m)) < 0) {
+ s++;
+ m = ~(fy_id_bits)0;
+ }
+
+ /* not found */
+ if (pos < 0) {
+ iterp->s = NULL;
+ iterp->m = 0;
+ return -1;
+ }
+
+ /* remove this bit from the mask */
+ m &= ~((fy_id_bits)1 << pos);
+
+ /* advance by the number of fy_id_bits we scanned */
+ pos += (int)(s - bits) * FY_ID_BITS_BITS;
+
+ /* if we run out of mask, advance */
+ if (!m) {
+ s++;
+ m = ~(fy_id_bits)0;
+ }
+ iterp->s = s;
+ iterp->m = m;
+ return pos;
+}
+
+static inline void fy_id_iter_end(fy_id_bits *bits, const size_t count, const struct fy_id_iter *iterp)
+{
+ /* nothing */
+}
+
+#endif
--- /dev/null
+/*
+ * fy-list.h - slightly modified Linux kernel list.h
+ *
+ * All copyrights owned by their respective holders.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ */
+#ifndef _FY_LIST_H
+#define _FY_LIST_H
+
+#include <stddef.h>
+
+/**
+ * container_of - cast a member of a structure out to the containing structure
+ * @ptr: the pointer to the member.
+ * @type: the type of the container struct this is embedded in.
+ * @member: the name of the member within the struct.
+ *
+ */
+#define container_of(ptr, type, member) ({ \
+ const typeof( ((type *)0)->member ) *__mptr = (ptr); \
+ (type *)( (char *)__mptr - offsetof(type,member) );})
+
+#define LIST_POISON1 NULL
+#define LIST_POISON2 NULL
+
+#ifndef ARCH_HAS_PREFETCH
+#define ARCH_HAS_PREFETCH
+static inline void prefetch(const void *x) {;}
+#endif
+
+/*
+ * Simple doubly linked list implementation.
+ *
+ * Some of the internal functions ("__xxx") are useful when
+ * manipulating whole lists rather than single entries, as
+ * sometimes we already know the next/prev entries and we can
+ * generate better code by using them directly rather than
+ * using the generic single-entry routines.
+ */
+
+struct list_head {
+ struct list_head *next, *prev;
+};
+
+#define LIST_HEAD_INIT(name) { &(name), &(name) }
+
+#define LIST_HEAD(name) \
+ struct list_head name = LIST_HEAD_INIT(name)
+
+static inline void INIT_LIST_HEAD(struct list_head *list)
+{
+ list->next = list;
+ list->prev = list;
+}
+
+/*
+ * Insert a new entry between two known consecutive entries.
+ *
+ * This is only for internal list manipulation where we know
+ * the prev/next entries already!
+ */
+static inline void __list_add(struct list_head *new,
+ struct list_head *prev,
+ struct list_head *next)
+{
+ next->prev = new;
+ new->next = next;
+ new->prev = prev;
+ prev->next = new;
+}
+
+/**
+ * list_add - add a new entry
+ * @new: new entry to be added
+ * @head: list head to add it after
+ *
+ * Insert a new entry after the specified head.
+ * This is good for implementing stacks.
+ */
+static inline void list_add(struct list_head *new, struct list_head *head)
+{
+ __list_add(new, head, head->next);
+}
+
+/**
+ * list_add_tail - add a new entry
+ * @new: new entry to be added
+ * @head: list head to add it before
+ *
+ * Insert a new entry before the specified head.
+ * This is useful for implementing queues.
+ */
+static inline void list_add_tail(struct list_head *new, struct list_head *head)
+{
+ __list_add(new, head->prev, head);
+}
+
+/*
+ * Delete a list entry by making the prev/next entries
+ * point to each other.
+ *
+ * This is only for internal list manipulation where we know
+ * the prev/next entries already!
+ */
+static inline void __list_del(struct list_head *prev, struct list_head *next)
+{
+ next->prev = prev;
+ prev->next = next;
+}
+
+/**
+ * list_del - deletes entry from list.
+ * @entry: the element to delete from the list.
+ * Note: list_empty() on entry does not return true after this, the entry is
+ * in an undefined state.
+ */
+static inline void list_del(struct list_head *entry)
+{
+ __list_del(entry->prev, entry->next);
+ entry->next = LIST_POISON1;
+ entry->prev = LIST_POISON2;
+}
+
+/**
+ * list_replace - replace old entry by new one
+ * @old : the element to be replaced
+ * @new : the new element to insert
+ *
+ * If @old was empty, it will be overwritten.
+ */
+static inline void list_replace(struct list_head *old,
+ struct list_head *new)
+{
+ new->next = old->next;
+ new->next->prev = new;
+ new->prev = old->prev;
+ new->prev->next = new;
+}
+
+static inline void list_replace_init(struct list_head *old,
+ struct list_head *new)
+{
+ list_replace(old, new);
+ INIT_LIST_HEAD(old);
+}
+
+/**
+ * list_del_init - deletes entry from list and reinitialize it.
+ * @entry: the element to delete from the list.
+ */
+static inline void list_del_init(struct list_head *entry)
+{
+ __list_del(entry->prev, entry->next);
+ INIT_LIST_HEAD(entry);
+}
+
+/**
+ * list_move - delete from one list and add as another's head
+ * @list: the entry to move
+ * @head: the head that will precede our entry
+ */
+static inline void list_move(struct list_head *list, struct list_head *head)
+{
+ __list_del(list->prev, list->next);
+ list_add(list, head);
+}
+
+/**
+ * list_move_tail - delete from one list and add as another's tail
+ * @list: the entry to move
+ * @head: the head that will follow our entry
+ */
+static inline void list_move_tail(struct list_head *list,
+ struct list_head *head)
+{
+ __list_del(list->prev, list->next);
+ list_add_tail(list, head);
+}
+
+/**
+ * list_is_last - tests whether @list is the last entry in list @head
+ * @list: the entry to test
+ * @head: the head of the list
+ */
+static inline int list_is_last(const struct list_head *list,
+ const struct list_head *head)
+{
+ return list->next == head;
+}
+
+/**
+ * list_empty - tests whether a list is empty
+ * @head: the list to test.
+ */
+static inline int list_empty(const struct list_head *head)
+{
+ return head->next == head;
+}
+
+/**
+ * list_empty_careful - tests whether a list is empty and not being modified
+ * @head: the list to test
+ *
+ * Description:
+ * tests whether a list is empty _and_ checks that no other CPU might be
+ * in the process of modifying either member (next or prev)
+ *
+ * NOTE: using list_empty_careful() without synchronization
+ * can only be safe if the only activity that can happen
+ * to the list entry is list_del_init(). Eg. it cannot be used
+ * if another CPU could re-list_add() it.
+ */
+static inline int list_empty_careful(const struct list_head *head)
+{
+ struct list_head *next = head->next;
+ return (next == head) && (next == head->prev);
+}
+
+/**
+ * list_is_singular - tests whether a list has just one entry.
+ * @head: the list to test.
+ */
+static inline int list_is_singular(const struct list_head *head)
+{
+ return !list_empty(head) && (head->next == head->prev);
+}
+
+static inline void __list_cut_position(struct list_head *list,
+ struct list_head *head, struct list_head *entry)
+{
+ struct list_head *new_first = entry->next;
+ list->next = head->next;
+ list->next->prev = list;
+ list->prev = entry;
+ entry->next = list;
+ head->next = new_first;
+ new_first->prev = head;
+}
+
+/**
+ * list_cut_position - cut a list into two
+ * @list: a new list to add all removed entries
+ * @head: a list with entries
+ * @entry: an entry within head, could be the head itself
+ * and if so we won't cut the list
+ *
+ * This helper moves the initial part of @head, up to and
+ * including @entry, from @head to @list. You should
+ * pass on @entry an element you know is on @head. @list
+ * should be an empty list or a list you do not care about
+ * losing its data.
+ *
+ */
+static inline void list_cut_position(struct list_head *list,
+ struct list_head *head, struct list_head *entry)
+{
+ if (list_empty(head))
+ return;
+ if (list_is_singular(head) &&
+ (head->next != entry && head != entry))
+ return;
+ if (entry == head)
+ INIT_LIST_HEAD(list);
+ else
+ __list_cut_position(list, head, entry);
+}
+
+static inline void __list_splice(const struct list_head *list,
+ struct list_head *prev,
+ struct list_head *next)
+{
+ struct list_head *first = list->next;
+ struct list_head *last = list->prev;
+
+ first->prev = prev;
+ prev->next = first;
+
+ last->next = next;
+ next->prev = last;
+}
+
+/**
+ * list_splice - join two lists, this is designed for stacks
+ * @list: the new list to add.
+ * @head: the place to add it in the first list.
+ */
+static inline void list_splice(const struct list_head *list,
+ struct list_head *head)
+{
+ if (!list_empty(list))
+ __list_splice(list, head, head->next);
+}
+
+/**
+ * list_splice_tail - join two lists, each list being a queue
+ * @list: the new list to add.
+ * @head: the place to add it in the first list.
+ */
+static inline void list_splice_tail(struct list_head *list,
+ struct list_head *head)
+{
+ if (!list_empty(list))
+ __list_splice(list, head->prev, head);
+}
+
+/**
+ * list_splice_init - join two lists and reinitialise the emptied list.
+ * @list: the new list to add.
+ * @head: the place to add it in the first list.
+ *
+ * The list at @list is reinitialised
+ */
+static inline void list_splice_init(struct list_head *list,
+ struct list_head *head)
+{
+ if (!list_empty(list)) {
+ __list_splice(list, head, head->next);
+ INIT_LIST_HEAD(list);
+ }
+}
+
+/**
+ * list_splice_tail_init - join two lists and reinitialise the emptied list
+ * @list: the new list to add.
+ * @head: the place to add it in the first list.
+ *
+ * Each of the lists is a queue.
+ * The list at @list is reinitialised
+ */
+static inline void list_splice_tail_init(struct list_head *list,
+ struct list_head *head)
+{
+ if (!list_empty(list)) {
+ __list_splice(list, head->prev, head);
+ INIT_LIST_HEAD(list);
+ }
+}
+
+/**
+ * list_entry - get the struct for this entry
+ * @ptr: the &struct list_head pointer.
+ * @type: the type of the struct this is embedded in.
+ * @member: the name of the list_struct within the struct.
+ */
+#define list_entry(ptr, type, member) \
+ container_of(ptr, type, member)
+
+/**
+ * list_first_entry - get the first element from a list
+ * @ptr: the list head to take the element from.
+ * @type: the type of the struct this is embedded in.
+ * @member: the name of the list_struct within the struct.
+ *
+ * Note, that list is expected to be not empty.
+ */
+#define list_first_entry(ptr, type, member) \
+ list_entry((ptr)->next, type, member)
+
+/**
+ * list_last_entry - get the last element from a list
+ * @ptr: the list head to take the element from.
+ * @type: the type of the struct this is embedded in.
+ * @member: the name of the list_struct within the struct.
+ *
+ * Note, that list is expected to be not empty.
+ */
+#define list_last_entry(ptr, type, member) \
+ list_entry((ptr)->prev, type, member)
+
+/**
+ * list_for_each - iterate over a list
+ * @pos: the &struct list_head to use as a loop cursor.
+ * @head: the head for your list.
+ */
+#define list_for_each(pos, head) \
+ for (pos = (head)->next; prefetch(pos->next), pos != (head); \
+ pos = pos->next)
+
+/**
+ * __list_for_each - iterate over a list
+ * @pos: the &struct list_head to use as a loop cursor.
+ * @head: the head for your list.
+ *
+ * This variant differs from list_for_each() in that it's the
+ * simplest possible list iteration code, no prefetching is done.
+ * Use this for code that knows the list to be very short (empty
+ * or 1 entry) most of the time.
+ */
+#define __list_for_each(pos, head) \
+ for (pos = (head)->next; pos != (head); pos = pos->next)
+
+/**
+ * list_for_each_prev - iterate over a list backwards
+ * @pos: the &struct list_head to use as a loop cursor.
+ * @head: the head for your list.
+ */
+#define list_for_each_prev(pos, head) \
+ for (pos = (head)->prev; prefetch(pos->prev), pos != (head); \
+ pos = pos->prev)
+
+/**
+ * list_for_each_safe - iterate over a list safe against removal of list entry
+ * @pos: the &struct list_head to use as a loop cursor.
+ * @n: another &struct list_head to use as temporary storage
+ * @head: the head for your list.
+ */
+#define list_for_each_safe(pos, n, head) \
+ for (pos = (head)->next, n = pos->next; pos != (head); \
+ pos = n, n = pos->next)
+
+/**
+ * list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry
+ * @pos: the &struct list_head to use as a loop cursor.
+ * @n: another &struct list_head to use as temporary storage
+ * @head: the head for your list.
+ */
+#define list_for_each_prev_safe(pos, n, head) \
+ for (pos = (head)->prev, n = pos->prev; \
+ prefetch(pos->prev), pos != (head); \
+ pos = n, n = pos->prev)
+
+/**
+ * list_for_each_entry - iterate over list of given type
+ * @pos: the type * to use as a loop cursor.
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ */
+#define list_for_each_entry(pos, head, member) \
+ for (pos = list_entry((head)->next, typeof(*pos), member); \
+ prefetch(pos->member.next), &pos->member != (head); \
+ pos = list_entry(pos->member.next, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_reverse - iterate backwards over list of given type.
+ * @pos: the type * to use as a loop cursor.
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ */
+#define list_for_each_entry_reverse(pos, head, member) \
+ for (pos = list_entry((head)->prev, typeof(*pos), member); \
+ prefetch(pos->member.prev), &pos->member != (head); \
+ pos = list_entry(pos->member.prev, typeof(*pos), member))
+
+/**
+ * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue()
+ * @pos: the type * to use as a start point
+ * @head: the head of the list
+ * @member: the name of the list_struct within the struct.
+ *
+ * Prepares a pos entry for use as a start point in list_for_each_entry_continue().
+ */
+#define list_prepare_entry(pos, head, member) \
+ ((pos) ? : list_entry(head, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_continue - continue iteration over list of given type
+ * @pos: the type * to use as a loop cursor.
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ *
+ * Continue to iterate over list of given type, continuing after
+ * the current position.
+ */
+#define list_for_each_entry_continue(pos, head, member) \
+ for (pos = list_entry(pos->member.next, typeof(*pos), member); \
+ prefetch(pos->member.next), &pos->member != (head); \
+ pos = list_entry(pos->member.next, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_continue_reverse - iterate backwards from the given point
+ * @pos: the type * to use as a loop cursor.
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ *
+ * Start to iterate over list of given type backwards, continuing after
+ * the current position.
+ */
+#define list_for_each_entry_continue_reverse(pos, head, member) \
+ for (pos = list_entry(pos->member.prev, typeof(*pos), member); \
+ prefetch(pos->member.prev), &pos->member != (head); \
+ pos = list_entry(pos->member.prev, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_from - iterate over list of given type from the current point
+ * @pos: the type * to use as a loop cursor.
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ *
+ * Iterate over list of given type, continuing from current position.
+ */
+#define list_for_each_entry_from(pos, head, member) \
+ for (; prefetch(pos->member.next), &pos->member != (head); \
+ pos = list_entry(pos->member.next, typeof(*pos), member))
+
+/**
+ * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry
+ * @pos: the type * to use as a loop cursor.
+ * @n: another type * to use as temporary storage
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ */
+#define list_for_each_entry_safe(pos, n, head, member) \
+ for (pos = list_entry((head)->next, typeof(*pos), member), \
+ n = list_entry(pos->member.next, typeof(*pos), member); \
+ &pos->member != (head); \
+ pos = n, n = list_entry(n->member.next, typeof(*n), member))
+
+/**
+ * list_for_each_entry_safe_continue
+ * @pos: the type * to use as a loop cursor.
+ * @n: another type * to use as temporary storage
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ *
+ * Iterate over list of given type, continuing after current point,
+ * safe against removal of list entry.
+ */
+#define list_for_each_entry_safe_continue(pos, n, head, member) \
+ for (pos = list_entry(pos->member.next, typeof(*pos), member), \
+ n = list_entry(pos->member.next, typeof(*pos), member); \
+ &pos->member != (head); \
+ pos = n, n = list_entry(n->member.next, typeof(*n), member))
+
+/**
+ * list_for_each_entry_safe_from
+ * @pos: the type * to use as a loop cursor.
+ * @n: another type * to use as temporary storage
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ *
+ * Iterate over list of given type from current point, safe against
+ * removal of list entry.
+ */
+#define list_for_each_entry_safe_from(pos, n, head, member) \
+ for (n = list_entry(pos->member.next, typeof(*pos), member); \
+ &pos->member != (head); \
+ pos = n, n = list_entry(n->member.next, typeof(*n), member))
+
+/**
+ * list_for_each_entry_safe_reverse
+ * @pos: the type * to use as a loop cursor.
+ * @n: another type * to use as temporary storage
+ * @head: the head for your list.
+ * @member: the name of the list_struct within the struct.
+ *
+ * Iterate backwards over list of given type, safe against removal
+ * of list entry.
+ */
+#define list_for_each_entry_safe_reverse(pos, n, head, member) \
+ for (pos = list_entry((head)->prev, typeof(*pos), member), \
+ n = list_entry(pos->member.prev, typeof(*pos), member); \
+ &pos->member != (head); \
+ pos = n, n = list_entry(n->member.prev, typeof(*n), member))
+
+/*
+ * Double linked lists with a single pointer list head.
+ * Mostly useful for hash tables where the two pointer list head is
+ * too wasteful.
+ * You lose the ability to access the tail in O(1).
+ */
+
+struct hlist_head {
+ struct hlist_node *first;
+};
+
+struct hlist_node {
+ struct hlist_node *next, **pprev;
+};
+
+#define HLIST_HEAD_INIT { .first = NULL }
+#define HLIST_HEAD(name) struct hlist_head name = { .first = NULL }
+#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL)
+static inline void INIT_HLIST_NODE(struct hlist_node *h)
+{
+ h->next = NULL;
+ h->pprev = NULL;
+}
+
+static inline int hlist_unhashed(const struct hlist_node *h)
+{
+ return !h->pprev;
+}
+
+static inline int hlist_empty(const struct hlist_head *h)
+{
+ return !h->first;
+}
+
+static inline void __hlist_del(struct hlist_node *n)
+{
+ struct hlist_node *next = n->next;
+ struct hlist_node **pprev = n->pprev;
+ *pprev = next;
+ if (next)
+ next->pprev = pprev;
+}
+
+static inline void hlist_del(struct hlist_node *n)
+{
+ __hlist_del(n);
+ n->next = LIST_POISON1;
+ n->pprev = LIST_POISON2;
+}
+
+static inline void hlist_del_init(struct hlist_node *n)
+{
+ if (!hlist_unhashed(n)) {
+ __hlist_del(n);
+ INIT_HLIST_NODE(n);
+ }
+}
+
+static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h)
+{
+ struct hlist_node *first = h->first;
+ n->next = first;
+ if (first)
+ first->pprev = &n->next;
+ h->first = n;
+ n->pprev = &h->first;
+}
+
+/* next must be != NULL */
+static inline void hlist_add_before(struct hlist_node *n,
+ struct hlist_node *next)
+{
+ n->pprev = next->pprev;
+ n->next = next;
+ next->pprev = &n->next;
+ *(n->pprev) = n;
+}
+
+static inline void hlist_add_after(struct hlist_node *n,
+ struct hlist_node *next)
+{
+ next->next = n->next;
+ n->next = next;
+ next->pprev = &n->next;
+
+ if(next->next)
+ next->next->pprev = &next->next;
+}
+
+#define hlist_entry(ptr, type, member) container_of(ptr,type,member)
+
+#define hlist_for_each(pos, head) \
+ for (pos = (head)->first; pos && ({ prefetch(pos->next); 1; }); \
+ pos = pos->next)
+
+#define hlist_for_each_safe(pos, n, head) \
+ for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \
+ pos = n)
+
+/**
+ * hlist_for_each_entry - iterate over list of given type
+ * @tpos: the type * to use as a loop cursor.
+ * @pos: the &struct hlist_node to use as a loop cursor.
+ * @head: the head for your list.
+ * @member: the name of the hlist_node within the struct.
+ */
+#define hlist_for_each_entry(tpos, pos, head, member) \
+ for (pos = (head)->first; \
+ pos && ({ prefetch(pos->next); 1;}) && \
+ ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
+ pos = pos->next)
+
+/**
+ * hlist_for_each_entry_continue - iterate over a hlist continuing after current point
+ * @tpos: the type * to use as a loop cursor.
+ * @pos: the &struct hlist_node to use as a loop cursor.
+ * @member: the name of the hlist_node within the struct.
+ */
+#define hlist_for_each_entry_continue(tpos, pos, member) \
+ for (pos = (pos)->next; \
+ pos && ({ prefetch(pos->next); 1;}) && \
+ ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
+ pos = pos->next)
+
+/**
+ * hlist_for_each_entry_from - iterate over a hlist continuing from current point
+ * @tpos: the type * to use as a loop cursor.
+ * @pos: the &struct hlist_node to use as a loop cursor.
+ * @member: the name of the hlist_node within the struct.
+ */
+#define hlist_for_each_entry_from(tpos, pos, member) \
+ for (; pos && ({ prefetch(pos->next); 1;}) && \
+ ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
+ pos = pos->next)
+
+/**
+ * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry
+ * @tpos: the type * to use as a loop cursor.
+ * @pos: the &struct hlist_node to use as a loop cursor.
+ * @n: another &struct hlist_node to use as temporary storage
+ * @head: the head for your list.
+ * @member: the name of the hlist_node within the struct.
+ */
+#define hlist_for_each_entry_safe(tpos, pos, n, head, member) \
+ for (pos = (head)->first; \
+ pos && ({ n = pos->next; 1; }) && \
+ ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
+ pos = n)
+
+#endif
--- /dev/null
+/*
+ * fy-typelist.h - typed list method builders
+ *
+ * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef FY_TYPELIST_H
+#define FY_TYPELIST_H
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+
+#include "fy-list.h"
+
+/* declare type methods */
+
+#define FY_TYPE_FWD_DECL_LIST(_type) \
+/* type safe list wrapper */ \
+struct fy_ ## _type ## _list { struct list_head _lh; }; \
+\
+struct __useless_struct_to_allow_semicolon
+
+#define FY_TYPE_DECL_LIST(_type) \
+static inline void fy_ ## _type ## _list_init(struct fy_ ## _type ## _list *_l) \
+{ \
+ INIT_LIST_HEAD(&_l->_lh); \
+} \
+static inline void fy_ ## _type ## _list_add(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
+{ \
+ if (_l && _n) \
+ list_add(&_n->node, &_l->_lh); \
+} \
+static inline void fy_ ## _type ## _list_add_tail(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
+{ \
+ if (_l && _n) \
+ list_add_tail(&_n->node, &_l->_lh); \
+} \
+static inline void fy_ ## _type ## _list_push(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
+{ \
+ if (_l && _n) \
+ fy_ ## _type ## _list_add(_l, _n); \
+} \
+static inline void fy_ ## _type ## _list_push_tail(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
+{ \
+ if (_l && _n) \
+ fy_ ## _type ## _list_add_tail(_l, _n); \
+} \
+static inline bool fy_ ## _type ## _list_empty(struct fy_ ## _type ## _list *_l) \
+{ \
+ return _l ? list_empty(&_l->_lh) : true; \
+} \
+static inline bool fy_ ## _type ## _list_is_singular(struct fy_ ## _type ## _list *_l) \
+{ \
+ return _l ? list_is_singular(&_l->_lh) : true; \
+} \
+static inline void fy_ ## _type ## _list_del(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
+{ \
+ if (_l && _n) { \
+ list_del(&_n->node); \
+ INIT_LIST_HEAD(&_n->node); \
+ } \
+} \
+static inline void fy_ ## _type ## _list_insert_after(struct fy_ ## _type ## _list *_l, \
+ struct fy_ ## _type *_p, struct fy_ ## _type *_n) \
+{ \
+ if (_l && _p && _n) \
+ list_add(&_n->node, &_p->node); \
+} \
+static inline void fy_ ## _type ## _list_insert_before(struct fy_ ## _type ## _list *_l, \
+ struct fy_ ## _type *_p, struct fy_ ## _type *_n) \
+{ \
+ if (_l && _p && _n) \
+ list_add_tail(&_n->node, &_p->node); \
+} \
+static inline struct fy_ ## _type *fy_ ## _type ## _list_head(struct fy_ ## _type ## _list *_l) \
+{ \
+ return !fy_ ## _type ## _list_empty(_l) ? list_first_entry(&_l->_lh, struct fy_ ## _type, node) : NULL; \
+} \
+static inline struct fy_ ## _type *fy_ ## _type ## _list_tail(struct fy_ ## _type ## _list *_l) \
+{ \
+ return !fy_ ## _type ## _list_empty(_l) ? list_last_entry(&_l->_lh, struct fy_ ## _type, node) : NULL; \
+} \
+static inline struct fy_ ## _type *fy_ ## _type ## _list_first(struct fy_ ## _type ## _list *_l) \
+{ \
+ return fy_ ## _type ## _list_head(_l); \
+} \
+static inline struct fy_ ## _type *fy_ ## _type ## _list_last(struct fy_ ## _type ## _list *_l) \
+{ \
+ return fy_ ## _type ## _list_tail(_l); \
+} \
+static inline struct fy_ ## _type *fy_ ## _type ## _list_pop(struct fy_ ## _type ## _list *_l) \
+{ \
+ struct fy_ ## _type *_n; \
+ \
+ _n = fy_ ## _type ## _list_head(_l); \
+ if (!_n) \
+ return NULL; \
+ fy_ ## _type ## _list_del(_l, _n); \
+ return _n; \
+} \
+static inline struct fy_ ## _type *fy_ ## _type ## _list_pop_tail(struct fy_ ## _type ## _list *_l) \
+{ \
+ struct fy_ ## _type *_n; \
+ \
+ _n = fy_ ## _type ## _list_tail(_l); \
+ if (!_n) \
+ return NULL; \
+ fy_ ## _type ## _list_del(_l, _n); \
+ return _n; \
+} \
+static inline struct fy_ ## _type *fy_ ## _type ## _next(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
+{ \
+ if (!_n || !_l || _n->node.next == &_l->_lh) \
+ return NULL; \
+ return list_entry(_n->node.next, struct fy_ ## _type, node); \
+} \
+static inline struct fy_ ## _type *fy_ ## _type ## _prev(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \
+{ \
+ if (!_n || !_l || _n->node.prev == &_l->_lh) \
+ return NULL; \
+ return list_entry(_n->node.prev, struct fy_ ## _type, node); \
+} \
+static inline void fy_ ## _type ## _lists_splice( \
+ struct fy_ ## _type ## _list *_l, \
+ struct fy_ ## _type ## _list *_lfrom) \
+{ \
+ /* check arguments for sanity and lists are not empty */ \
+ if (!_l || !_lfrom || \
+ fy_ ## _type ## _list_empty(_lfrom)) \
+ return; \
+ list_splice(&_lfrom->_lh, &_l->_lh); \
+} \
+static inline void fy_ ## _type ## _list_splice_after( \
+ struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n, \
+ struct fy_ ## _type ## _list *_lfrom) \
+{ \
+ /* check arguments for sanity and lists are not empty */ \
+ if (!_l || !_n || !_lfrom || \
+ fy_ ## _type ## _list_empty(_lfrom)) \
+ return; \
+ list_splice(&_lfrom->_lh, &_n->node); \
+} \
+static inline void fy_ ## _type ## _list_splice_before( \
+ struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n, \
+ struct fy_ ## _type ## _list *_lfrom) \
+{ \
+ /* check arguments for sanity and lists are not empty */ \
+ if (!_l || !_n || !_lfrom || \
+ fy_ ## _type ## _list_empty(_lfrom)) \
+ return; \
+ list_splice(&_lfrom->_lh, _n->node.prev); \
+} \
+struct __useless_struct_to_allow_semicolon
+
+#endif
--- /dev/null
+/*
+ * fy-utf8.c - utf8 handling methods
+ *
+ * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdbool.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "fy-utf8.h"
+
+/* to avoid dragging in libfyaml.h */
+#ifndef FY_BIT
+#define FY_BIT(x) (1U << (x))
+#endif
+
+const int8_t fy_utf8_width_table[32] = {
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 2, 2, 2, 2, 3, 3, 4, 0,
+};
+
+int fy_utf8_get_generic(const void *ptr, size_t left, int *widthp)
+{
+ const uint8_t *p = ptr;
+ int i, width, value;
+
+ if (left < 1)
+ return FYUG_EOF;
+
+ /* this is the slow path */
+ width = fy_utf8_width_by_first_octet(p[0]);
+ if (!width)
+ return FYUG_INV;
+ if ((size_t)width > left)
+ return FYUG_PARTIAL;
+
+ /* initial value */
+ value = *p++ & (0xff >> width);
+ for (i = 1; i < width; i++) {
+ if ((*p & 0xc0) != 0x80)
+ return FYUG_INV;
+ value = (value << 6) | (*p++ & 0x3f);
+ }
+
+ /* check for validity */
+ if ((width == 4 && value < 0x10000) ||
+ (width == 3 && value < 0x800) ||
+ (width == 2 && value < 0x80) ||
+ (value >= 0xd800 && value <= 0xdfff) || value >= 0x110000)
+ return FYUG_INV;
+
+ *widthp = width;
+
+ return value;
+}
+
+int fy_utf8_get_right_generic(const void *ptr, size_t left, int *widthp)
+{
+ const uint8_t *s, *e;
+ uint8_t v;
+
+ s = ptr;
+ e = s + left;
+
+ if (left < 1)
+ return FYUG_EOF;
+
+ /* single byte sequence */
+ v = e[-1];
+ if ((v & 0x80) == 0) {
+ if (widthp)
+ *widthp = 1;
+ return (int)v & 0x7f;
+ }
+
+ /* the last byte must be & 0xc0 == 0x80 */
+ if ((v & 0xc0) != 0x80)
+ return FYUG_INV;
+
+ /* at least two byte sequence */
+ if (left < 2)
+ return FYUG_EOF;
+
+ v = e[-2];
+ /* the first byte of the sequence (must be a two byte sequence) */
+ if ((v & 0xc0) != 0x80) {
+ /* two byte start is 110x_xxxx */
+ if ((v & 0xe0) != 0xc0)
+ return FYUG_INV;
+ return fy_utf8_get(e - 2, 2, widthp);
+ }
+
+ /* at least three byte sequence */
+ if (left < 3)
+ return FYUG_EOF;
+
+ v = e[-3];
+ /* the first byte of the sequence (must be a three byte sequence) */
+ if ((v & 0xc0) != 0x80) {
+ /* three byte start is 1110_xxxx */
+ if ((v & 0xf0) != 0xe0)
+ return FYUG_INV;
+ return fy_utf8_get(e - 3, 3, widthp);
+ }
+
+ /* at least four byte sequence */
+ if (left < 4)
+ return FYUG_EOF;
+
+ v = e[-4];
+
+ /* the first byte of the sequence (must be a four byte sequence) */
+ /* four byte start is 1111_0xxx */
+ if ((v & 0xf8) != 0xf0) {
+ return FYUG_INV;
+ }
+ return fy_utf8_get(e - 4, 4, widthp);
+}
+
+struct fy_utf8_fmt_esc_map {
+ const int *ch;
+ const int *map;
+};
+
+static const struct fy_utf8_fmt_esc_map esc_all = {
+ .ch = (const int []){ '\\', '\0', '\b', '\r', '\t', '\f', '\n', '\v', '\a', '\e', 0x85, 0xa0, 0x2028, 0x2029, -1 },
+ .map = (const int []){ '\\', '0', 'b', 'r', 't', 'f', 'n', 'v', 'a', 'e', 'N', '_', 'L', 'P', 0 }
+};
+
+static inline int esc_map(const struct fy_utf8_fmt_esc_map *esc_map, int c)
+{
+ const int *ch;
+ int cc;
+
+ ch = esc_map->ch;
+ while ((cc = *ch++) >= 0) {
+ if (cc == c)
+ return esc_map->map[(ch - esc_map->ch) - 1];
+ }
+ return -1;
+}
+
+static inline int fy_utf8_esc_map(int c, enum fy_utf8_escape esc)
+{
+ if (esc == fyue_none)
+ return -1;
+ if (esc == fyue_singlequote && c == '\'')
+ return '\'';
+ if (fy_utf8_escape_is_any_doublequote(esc) && c == '"')
+ return '"';
+ return esc_map(&esc_all, c);
+}
+
+size_t fy_utf8_format_text_length(const char *buf, size_t len,
+ enum fy_utf8_escape esc)
+{
+ int c, w;
+ ssize_t l;
+ const char *s, *e;
+
+ s = buf;
+ e = buf + len;
+ l = 0;
+ while (s < e) {
+ c = fy_utf8_get(s, e - s, &w);
+ if (!w || c < 0)
+ break;
+ s += w;
+ if (fy_utf8_esc_map(c, esc))
+ w = 2;
+ l += w;
+ }
+ return l + 1;
+}
+
+char *fy_utf8_format_text(const char *buf, size_t len,
+ char *out, size_t maxsz,
+ enum fy_utf8_escape esc)
+{
+ int c, w, cc;
+ const char *s, *e;
+ char *os, *oe;
+
+ s = buf;
+ e = buf + len;
+ os = out;
+ oe = out + maxsz - 1;
+ while (s < e) {
+ c = fy_utf8_get(s, e - s, &w);
+ if (!w || c < 0)
+ break;
+ s += w;
+
+ if ((cc = fy_utf8_esc_map(c, esc)) > 0) {
+ if (os + 2 > oe)
+ break;
+ *os++ = '\\';
+ *os++ = cc;
+ continue;
+ }
+
+ if (os + w > oe)
+ break;
+
+ os = fy_utf8_put_unchecked(os, c);
+ }
+ *os++ = '\0';
+ return out;
+}
+
+char *fy_utf8_format(int c, char *buf, enum fy_utf8_escape esc)
+{
+ int cc;
+ char *s;
+
+ if (!fy_utf8_is_valid(c)) {
+ *buf = '\0';
+ return buf;
+ }
+
+ s = buf;
+ if ((cc = fy_utf8_esc_map(c, esc)) > 0) {
+ *s++ = '\\';
+ *s++ = cc;
+ } else
+ s = fy_utf8_put_unchecked(s, c);
+ *s = '\0';
+ return buf;
+}
+
+char *fy_utf8_format_text_alloc(const char *buf, size_t len, enum fy_utf8_escape esc)
+{
+ size_t outsz;
+ char *out;
+
+ outsz = fy_utf8_format_text_length(buf, len, esc);
+ if ((ssize_t)outsz < 0)
+ return NULL;
+ out = malloc(outsz);
+ if (!out)
+ return NULL;
+ fy_utf8_format_text(buf, len, out, outsz, esc);
+
+ return out;
+}
+
+const void *fy_utf8_memchr_generic(const void *s, int c, size_t n)
+{
+ int cc, w;
+ const void *e;
+
+ e = s + n;
+ while (s < e && (cc = fy_utf8_get(s, e - s, &w)) >= 0) {
+ if (c == cc)
+ return s;
+ s += w;
+ }
+
+ return NULL;
+}
+
+/* parse an escape and return utf8 value */
+int fy_utf8_parse_escape(const char **strp, size_t len, enum fy_utf8_escape esc)
+{
+ const char *s, *e;
+ char c;
+ int i, value, code_length, cc, w;
+ unsigned int hi_surrogate, lo_surrogate;
+
+ /* why do you bother us? */
+ if (esc == fyue_none)
+ return -1;
+
+ if (!strp || !*strp || len < 2)
+ return -1;
+
+ value = -1;
+
+ s = *strp;
+ e = s + len;
+
+ c = *s++;
+
+ if (esc == fyue_singlequote) {
+ if (c != '\'')
+ goto out;
+ c = *s++;
+ if (c != '\'')
+ goto out;
+
+ value = '\'';
+ goto out;
+ }
+
+ /* get '\\' */
+ if (c != '\\')
+ goto out;
+
+ c = *s++;
+
+ /* common YAML & JSON escapes */
+ switch (c) {
+ case 'b':
+ value = '\b';
+ break;
+ case 'f':
+ value = '\f';
+ break;
+ case 'n':
+ value = '\n';
+ break;
+ case 'r':
+ value = '\r';
+ break;
+ case 't':
+ value = '\t';
+ break;
+ case '"':
+ value = '"';
+ break;
+ case '/':
+ value = '/';
+ break;
+ case '\\':
+ value = '\\';
+ break;
+ default:
+ break;
+ }
+
+ if (value >= 0)
+ goto out;
+
+ if (esc == fyue_doublequote || esc == fyue_doublequote_yaml_1_1) {
+ switch (c) {
+ case '0':
+ value = '\0';
+ break;
+ case 'a':
+ value = '\a';
+ break;
+ case '\t':
+ value = '\t';
+ break;
+ case 'v':
+ value = '\v';
+ break;
+ case 'e':
+ value = '\e';
+ break;
+ case ' ':
+ value = ' ';
+ break;
+ case 'N':
+ value = 0x85; /* NEL */
+ break;
+ case '_':
+ value = 0xa0;
+ break;
+ case 'L':
+ value = 0x2028; /* LS */
+ break;
+ case 'P': /* PS 0x2029 */
+ value = 0x2029; /* PS */
+ break;
+ default:
+ /* weird unicode escapes */
+ if ((uint8_t)c >= 0x80) {
+ /* in non yaml-1.1 mode we don't allow this craziness */
+ if (esc == fyue_doublequote)
+ goto out;
+
+ cc = fy_utf8_get(s - 1, e - (s - 1), &w);
+ switch (cc) {
+ case 0x2028:
+ case 0x2029:
+ case 0x85:
+ case 0xa0:
+ value = cc;
+ break;
+ default:
+ break;
+ }
+ }
+ break;
+ }
+ if (value >= 0)
+ goto out;
+ }
+
+ /* finally try the unicode escapes */
+ code_length = 0;
+
+ if (esc == fyue_doublequote || esc == fyue_doublequote_yaml_1_1) {
+ switch (c) {
+ case 'x':
+ code_length = 2;
+ break;
+ case 'u':
+ code_length = 4;
+ break;
+ case 'U':
+ code_length = 8;
+ break;
+ default:
+ return -1;
+ }
+ } else if (esc == fyue_doublequote_json && c == 'u')
+ code_length = 4;
+
+ if (!code_length || code_length > (e - s))
+ goto out;
+
+ value = 0;
+ for (i = 0; i < code_length; i++) {
+ c = *s++;
+ value <<= 4;
+ if (c >= '0' && c <= '9')
+ value |= c - '0';
+ else if (c >= 'a' && c <= 'f')
+ value |= 10 + c - 'a';
+ else if (c >= 'A' && c <= 'F')
+ value |= 10 + c - 'A';
+ else
+ goto out;
+ }
+
+ /* hi/lo surrogate pair */
+ if (code_length == 4 && value >= 0xd800 && value <= 0xdbff &&
+ (e - s) >= 6 && s[0] == '\\' && s[1] == 'u') {
+ hi_surrogate = value;
+
+ s += 2;
+
+ value = 0;
+ for (i = 0; i < code_length; i++) {
+ c = *s++;
+ value <<= 4;
+ if (c >= '0' && c <= '9')
+ value |= c - '0';
+ else if (c >= 'a' && c <= 'f')
+ value |= 10 + c - 'a';
+ else if (c >= 'A' && c <= 'F')
+ value |= 10 + c - 'A';
+ else
+ return -1;
+ }
+ lo_surrogate = value;
+ value = 0x10000 + (hi_surrogate - 0xd800) * 0x400 + (lo_surrogate - 0xdc00);
+ }
+
+out:
+ *strp = s;
+
+ return value;
+}
+
+uint8_t fy_utf8_low_ascii_flags[0x80] = {
+ [0x00] = F_NON_PRINT, // NUL '\0' (null character)
+ [0x01] = F_NON_PRINT, // SOH (start of heading)
+ [0x02] = F_NON_PRINT, // STX (start of text)
+ [0x03] = F_NON_PRINT, // ETX (end of text)
+ [0x04] = F_NON_PRINT, // EOT (end of transmission)
+ [0x05] = F_NON_PRINT, // ENQ (enquiry)
+ [0x06] = F_NON_PRINT, // ACK (acknowledge)
+ [0x07] = F_NON_PRINT | F_QUOTE_ESC, // BEL '\a' (bell)
+ [0x08] = F_NON_PRINT | F_QUOTE_ESC, // BS '\b' (backspace)
+ [0x09] = F_NON_PRINT | F_QUOTE_ESC | F_WS, // HT '\t' (horizontal tab)
+ [0x0A] = F_NON_PRINT | F_QUOTE_ESC | F_LB, // LF '\n' (new line)
+ [0x0B] = F_NON_PRINT | F_QUOTE_ESC, // VT '\v' (vertical tab)
+ [0x0C] = F_NON_PRINT | F_QUOTE_ESC, // FF '\f' (form feed)
+ [0x0D] = F_NON_PRINT | F_QUOTE_ESC | F_LB, // CR '\r' (carriage ret)
+ [0x0E] = F_NON_PRINT, // SO (shift out)
+ [0x0F] = F_NON_PRINT, // SI (shift in)
+ [0x10] = F_NON_PRINT, // DLE (data link escape)
+ [0x11] = F_NON_PRINT, // DC1 (device control 1)
+ [0x12] = F_NON_PRINT, // DC2 (device control 2)
+ [0x13] = F_NON_PRINT, // DC3 (device control 3)
+ [0x14] = F_NON_PRINT, // DC4 (device control 4)
+ [0x15] = F_NON_PRINT, // NAK (negative ack.)
+ [0x16] = F_NON_PRINT, // SYN (synchronous idle)
+ [0x17] = F_NON_PRINT, // ETB (end of trans. blk)
+ [0x18] = F_NON_PRINT, // CAN (cancel)
+ [0x19] = F_NON_PRINT, // EM (end of medium)
+ [0x1A] = F_NON_PRINT, // SUB (substitute)
+ [0x1B] = F_NON_PRINT, // ESC (escape)
+ [0x1C] = F_NON_PRINT, // FS (file separator)
+ [0x1D] = F_NON_PRINT, // GS (group separator)
+ [0x1E] = F_NON_PRINT, // RS (record separator)
+ [0x1F] = F_NON_PRINT, // US (unit separator)
+ [' '] = F_WS,
+ ['!'] = F_PUNCT,
+ ['"'] = F_PUNCT,
+ ['#'] = F_PUNCT,
+ ['$'] = F_PUNCT,
+ ['%'] = F_PUNCT,
+ ['&'] = F_PUNCT,
+ ['\''] = F_PUNCT,
+ ['('] = F_PUNCT,
+ [')'] = F_PUNCT,
+ ['*'] = F_PUNCT,
+ ['+'] = F_PUNCT,
+ [','] = F_PUNCT,
+ ['-'] = F_PUNCT,
+ ['.'] = F_PUNCT,
+ ['/'] = F_PUNCT,
+ ['0'] = F_DIGIT | F_SIMPLE_SCALAR,
+ ['1'] = F_DIGIT | F_SIMPLE_SCALAR,
+ ['2'] = F_DIGIT | F_SIMPLE_SCALAR,
+ ['3'] = F_DIGIT | F_SIMPLE_SCALAR,
+ ['4'] = F_DIGIT | F_SIMPLE_SCALAR,
+ ['5'] = F_DIGIT | F_SIMPLE_SCALAR,
+ ['6'] = F_DIGIT | F_SIMPLE_SCALAR,
+ ['7'] = F_DIGIT | F_SIMPLE_SCALAR,
+ ['8'] = F_DIGIT | F_SIMPLE_SCALAR,
+ ['9'] = F_DIGIT | F_SIMPLE_SCALAR,
+ [':'] = F_PUNCT,
+ [';'] = F_PUNCT,
+ ['<'] = F_PUNCT,
+ ['='] = F_PUNCT,
+ ['>'] = F_PUNCT,
+ ['?'] = F_PUNCT,
+ ['@'] = F_PUNCT,
+ ['A'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['B'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['C'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['D'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['E'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['F'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['G'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['H'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['I'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['J'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['K'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['L'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['M'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['N'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['O'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['P'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['Q'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['R'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['S'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['T'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['U'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['V'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['W'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['X'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['Y'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['Z'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['['] = F_PUNCT,
+ ['\\'] = F_PUNCT, // '\\'
+ [']'] = F_PUNCT,
+ ['^'] = F_PUNCT,
+ ['_'] = F_PUNCT | F_SIMPLE_SCALAR,
+ ['`'] = F_PUNCT,
+ ['a'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['b'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['c'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['d'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['e'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['f'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['g'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['h'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['i'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['j'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['k'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['l'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['m'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['n'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['o'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['p'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['q'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['r'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['s'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['t'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['u'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['v'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['w'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['x'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['y'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['z'] = F_LETTER | F_SIMPLE_SCALAR,
+ ['{'] = F_PUNCT,
+ ['|'] = F_PUNCT,
+ ['}'] = F_PUNCT,
+ ['~'] = F_PUNCT,
+ [0x7F] = F_NON_PRINT, // DEL
+};
+
+void *fy_utf8_split_posix(const char *str, int *argcp, const char * const *argvp[])
+{
+ enum split_state {
+ SS_WS, /* at whitespace */
+ SS_WS_BS, /* backslash at whitespace */
+ SS_UQ, /* at unquoted */
+ SS_UQ_BS, /* backslash at unquoted */
+ SS_SQ, /* at single quote */
+ SS_SQ_BS, /* backslash at single quote */
+ SS_DQ, /* at double quote */
+ SS_DQ_BS, /* backslash at double quote */
+ SS_DQ_BS_OCT1, /* in \nnn octal escape digit 1 */
+ SS_DQ_BS_OCT2, /* in \nnn octal escape digit 2 */
+ SS_DQ_BS_HEX0, /* in \xNN hex escape digit 0 */
+ SS_DQ_BS_HEX1, /* in \xNN hex escape digit 1 */
+ SS_DQ_BS_HEX2, /* in \xNN hex escape digit 2 */
+ SS_DQ_BS_C, /* in \cN control character escape */
+ SS_WS_BS_ERR_EOL, /* EOL while waiting for char after \ */
+ SS_UQ_BS_ERR_EOL, /* EOL while waiting for char after \ */
+ SS_SQ_BS_ERR_EOL, /* EOL while waiting for char after \ */
+ SS_DQ_BS_ERR_EOL, /* EOL while waiting for char after \ */
+ SS_DQ_BS_ERR_BAD, /* bad escape */
+ SS_SQ_NC_ERR, /* no closing single quote */
+ SS_DQ_NC_ERR, /* no closing double quote */
+ SS_DQ_BS_ERR_OCT_BAD, /* bad octal escape */
+ SS_DQ_BS_ERR_HEX_BAD, /* bad hex escape */
+ SS_DQ_BS_ERR_C_BAD, /* bad control char */
+ SS_DONE /* final state */
+ };
+ const char *s, *e;
+ enum split_state ss;
+ int c, w, val, tmp, outc, i;
+ size_t arg_alloc = strlen(str) + 1, tmplen;
+ char *arg = NULL, *args, *arge, *tmparg;
+ int argv_alloc = 64, argv_count = 0;
+ char **argv = NULL, **tmpargv;
+ void *mem;
+
+ if (!str || !argcp || !argvp)
+ return NULL;
+
+ /* the temporary is guaranteed to fit one split */
+ arg = alloca(arg_alloc);
+ args = arg;
+ arge = arg + arg_alloc - 1;
+
+ argv = alloca(sizeof(*argv) * argv_alloc);
+
+ e = str + strlen(str);
+
+#undef OUTC
+#define OUTC(_c) \
+ do { \
+ assert(args < arge); \
+ args = fy_utf8_put(args, arge - args, (_c)); \
+ assert(args); \
+ } while(0)
+
+#undef NEW_ARGV
+#define NEW_ARGV() \
+ do { \
+ if (args > arg) { \
+ if (argv_count + 1 >= argv_alloc) { \
+ argv_alloc *= 2; \
+ tmpargv = alloca((sizeof(*tmpargv) * argv_alloc)); \
+ memcpy(tmpargv, argv, argv_count * sizeof(*tmpargv)); \
+ } \
+ tmplen = args - arg; \
+ tmparg = alloca(tmplen + 1); \
+ memcpy(tmparg, arg, tmplen); \
+ tmparg[tmplen] = '\0'; \
+ argv[argv_count] = tmparg; \
+ argv[++argv_count] = NULL; \
+ } \
+ args = arg; \
+ } while(0)
+
+#define GOTO(_ss) \
+ do { \
+ ss = (_ss); \
+ } while(0)
+
+
+ ss = SS_WS;
+ s = str;
+ val = 0;
+ while (!(ss >= SS_WS_BS_ERR_EOL && ss <= SS_DONE)) {
+ c = fy_utf8_get(s, e - s, &w);
+ switch (ss) {
+ case SS_WS:
+ if (c < 0) {
+ NEW_ARGV();
+ GOTO(SS_DONE);
+ break;
+ }
+ switch (c) {
+ case ' ':
+ case '\t':
+ break; /* no change */
+ case '\\':
+ GOTO(SS_WS_BS);
+ break;
+ case '"':
+ GOTO(SS_DQ); /* start double quoted */
+ NEW_ARGV();
+ break;
+ case '\'':
+ GOTO(SS_SQ); /* start single quoted */
+ NEW_ARGV();
+ break;
+ default:
+ GOTO(SS_UQ); /* start unquoted */
+ NEW_ARGV();
+ OUTC(c);
+ break;
+ }
+ break;
+
+ case SS_WS_BS:
+ if (c < 0) {
+ GOTO(SS_WS_BS_ERR_EOL);
+ break;
+ }
+ if (c == '\n') { /* backslash newline, continuation */
+ GOTO(SS_WS);
+ break;
+ }
+ GOTO(SS_UQ);
+ NEW_ARGV();
+ OUTC(c);
+ break;
+
+ case SS_UQ:
+ if (c <= 0) {
+ NEW_ARGV();
+ GOTO(SS_DONE);
+ break;
+ }
+
+ switch (c) {
+ case ' ':
+ case '\t':
+ GOTO(SS_WS);
+ NEW_ARGV();
+ break; /* no change */
+ case '\\':
+ GOTO(SS_UQ_BS);
+ break;
+ case '"':
+ GOTO(SS_DQ); /* double quoted */
+ break;
+ case '\'':
+ GOTO(SS_SQ); /* single quoted */
+ break;
+ default:
+ OUTC(c);
+ break;
+ }
+ break;
+
+ case SS_UQ_BS:
+ if (c < 0) {
+ GOTO(SS_UQ_BS_ERR_EOL);
+ break;
+ }
+ if (c == '\n') { /* backslash new line only */
+ GOTO(SS_UQ);
+ break;
+ }
+ GOTO(SS_UQ);
+ OUTC(c);
+ break;
+
+ case SS_SQ:
+ if (c < 0) {
+ GOTO(SS_SQ_NC_ERR); /* no closing ' */
+ break;
+ }
+ switch (c) {
+ case '\'': /* end of quote, back to unquoted */
+ GOTO(SS_UQ);
+ break;
+ case '\\':
+ OUTC(c);
+ GOTO(SS_SQ_BS); /* backslash */
+ break;
+ default:
+ OUTC(c);
+ break;
+ }
+ break;
+
+ case SS_SQ_BS:
+ if (c < 0) {
+ GOTO(SS_SQ_BS_ERR_EOL);
+ break;
+ }
+ if (c == '\n') { /* backslash new line only */
+ GOTO(SS_SQ); /* back to single quoted */
+ break;
+ }
+ GOTO(SS_UQ); /* always back to */
+ OUTC(c);
+ break;
+
+ case SS_DQ:
+ if (c < 0) {
+ GOTO(SS_DQ_NC_ERR); /* no closing ' */
+ break;
+ }
+ switch (c) {
+ case '"': /* end of quote, back to unquoted */
+ GOTO(SS_UQ);
+ break;
+ case '\\':
+ GOTO(SS_DQ_BS); /* backslash */
+ break;
+ default:
+ OUTC(c);
+ break;
+ }
+ break;
+
+ case SS_DQ_BS:
+ if (c < 0) {
+ GOTO(SS_DQ_BS_ERR_EOL);
+ break;
+ }
+ outc = -1;
+ switch (c) {
+ case 'a':
+ outc = '\a';
+ break;
+ case 'b':
+ outc = '\b';
+ break;
+ case 'e':
+ outc = '\x1b'; /* escape */
+ break;
+ case 'n':
+ outc = '\n';
+ break;
+ case 'r':
+ outc = '\r';
+ break;
+ case 't':
+ outc = '\t';
+ break;
+ case 'v':
+ outc = '\v';
+ break;
+ case '\\':
+ outc = '\\';
+ break;
+ case '\'':
+ outc = '\'';
+ break;
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ val = c - '0';
+ GOTO(SS_DQ_BS_OCT1);
+ break;
+ case 'x':
+ val = 0;
+ GOTO(SS_DQ_BS_HEX0);
+ break;
+ case 'c':
+ val = 0;
+ GOTO(SS_DQ_BS_C);
+ break;
+ default:
+ GOTO(SS_DQ_BS_ERR_BAD); /* unknown escape */
+ break;
+ }
+ if (outc > 0) {
+ OUTC(outc);
+ GOTO(SS_DQ);
+ }
+ break;
+
+ case SS_DQ_BS_OCT1:
+ case SS_DQ_BS_OCT2:
+ if (c < 0) {
+ OUTC(val);
+ NEW_ARGV();
+ GOTO(SS_DONE);
+ break;
+ }
+ if (!(c >= '0' && c <= '7')) {
+ OUTC(val);
+ GOTO(SS_DQ);
+ w = 0; /* redo, same */
+ break;
+ }
+ val *= 8;
+ val += c - '0';
+ if (ss == SS_DQ_BS_OCT2) {
+ OUTC(val);
+ GOTO(SS_DQ);
+ } else
+ GOTO(SS_DQ_BS_OCT2);
+ break;
+
+ case SS_DQ_BS_HEX0:
+ case SS_DQ_BS_HEX1:
+ case SS_DQ_BS_HEX2:
+ tmp = -1;
+ if (c >= '0' && c <= '9')
+ tmp = c - '0';
+ else if (c >= 'a' && c <= 'f')
+ tmp = 10 + c - 'a';
+ else if (c >= 'A' && c <= 'F')
+ tmp = 10 + c - 'A';
+
+ switch (ss) {
+ case SS_DQ_BS_HEX0:
+ if (tmp < 0) {
+ fprintf(stderr, "tmp=%d c=%c\n", tmp, c);
+ GOTO(SS_DQ_BS_ERR_HEX_BAD);
+ break;
+ }
+ val = tmp;
+ GOTO(SS_DQ_BS_HEX1);
+ break;
+
+ case SS_DQ_BS_HEX1:
+ case SS_DQ_BS_HEX2:
+ if (tmp < 0) {
+ GOTO(SS_DQ);
+ OUTC(val);
+ w = 0; /* redo */
+ break;
+ }
+ val *= 16;
+ val += tmp;
+ if (ss == SS_DQ_BS_HEX1)
+ GOTO(SS_DQ_BS_HEX2);
+ else {
+ OUTC(val);
+ GOTO(SS_DQ);
+ }
+ break;
+
+ default:
+ abort();
+ }
+ break;
+
+ case SS_DQ_BS_C:
+ outc = -1;
+ if (c >= 'a' && c <= 'z')
+ outc = c - 'a' + 1;
+ else if (c >= 'A' && c <= 'Z')
+ outc = c - 'Z' + 1;
+ if (outc > 0x20) {
+ outc = -1;
+ GOTO(SS_DQ_BS_ERR_C_BAD);
+ } else
+ GOTO(SS_DQ);
+ if (outc > 0)
+ OUTC(outc);
+ break;
+ default:
+ abort();
+ break;
+ }
+ s += w;
+ }
+
+ /* someething went wrong */
+ if (ss != SS_DONE)
+ return NULL;
+
+ tmplen = (argv_count + 1) * sizeof(*argv);
+ for (i = 0; i < argv_count; i++)
+ tmplen += strlen(argv[i]) + 1;
+
+ mem = malloc(tmplen);
+ if (!mem)
+ return NULL;
+
+ tmpargv = mem;
+ tmparg = mem + (argv_count + 1) * sizeof(*tmpargv);
+ for (i = 0; i < argv_count; i++) {
+ tmpargv[i] = tmparg;
+ strcpy(tmparg, argv[i]);
+ tmparg += strlen(argv[i]) + 1;
+ }
+ tmpargv[i] = NULL;
+
+ *argcp = i;
+ *argvp = mem;
+
+ return mem;
+}
+
+int fy_utf8_get_generic_s(const void *ptr, const void *ptr_end, int *widthp)
+{
+ const uint8_t *p = ptr;
+ int i, width, value;
+
+ if (ptr >= ptr_end)
+ return FYUG_EOF;
+
+ /* this is the slow path */
+ width = fy_utf8_width_by_first_octet(p[0]);
+ if (!width)
+ return FYUG_INV;
+ if (ptr + width > ptr_end)
+ return FYUG_PARTIAL;
+
+ /* initial value */
+ value = *p++ & (0xff >> width);
+ for (i = 1; i < width; i++) {
+ if ((*p & 0xc0) != 0x80)
+ return FYUG_INV;
+ value = (value << 6) | (*p++ & 0x3f);
+ }
+
+ /* check for validity */
+ if ((width == 4 && value < 0x10000) ||
+ (width == 3 && value < 0x800) ||
+ (width == 2 && value < 0x80) ||
+ (value >= 0xd800 && value <= 0xdfff) || value >= 0x110000)
+ return FYUG_INV;
+
+ *widthp = width;
+
+ return value;
+}
+
+int fy_utf8_get_generic_s_nocheck(const void *ptr, int *widthp)
+{
+ const uint8_t *p = ptr;
+ int i, width, value;
+
+ /* this is the slow path */
+ width = fy_utf8_width_by_first_octet(p[0]);
+ if (!width)
+ return FYUG_INV;
+
+ /* initial value */
+ value = *p++ & (0xff >> width);
+ for (i = 1; i < width; i++) {
+ if ((*p & 0xc0) != 0x80)
+ return FYUG_INV;
+ value = (value << 6) | (*p++ & 0x3f);
+ }
+
+ /* check for validity */
+ if ((width == 4 && value < 0x10000) ||
+ (width == 3 && value < 0x800) ||
+ (width == 2 && value < 0x80) ||
+ (value >= 0xd800 && value <= 0xdfff) || value >= 0x110000)
+ return FYUG_INV;
+
+ *widthp = width;
+
+ return value;
+}
--- /dev/null
+/*
+ * fy-utf8.h - UTF-8 methods
+ *
+ * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifndef FY_UTF8_H
+#define FY_UTF8_H
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <alloca.h>
+
+#include "fy-utils.h"
+
+#define FY_UTF8_MIN_WIDTH 1
+#define FY_UTF8_MAX_WIDTH 4
+
+extern const int8_t fy_utf8_width_table[32];
+
+static inline int
+fy_utf8_width_by_first_octet_no_table(uint8_t c)
+{
+ return (c & 0x80) == 0x00 ? 1 :
+ (c & 0xe0) == 0xc0 ? 2 :
+ (c & 0xf0) == 0xe0 ? 3 :
+ (c & 0xf8) == 0xf0 ? 4 : 0;
+}
+
+static inline FY_ALWAYS_INLINE int
+fy_utf8_width_by_first_octet(uint8_t c)
+{
+ return fy_utf8_width_table[c >> 3];
+}
+
+/* assumes valid utf8 character */
+static inline size_t
+fy_utf8_width(int c)
+{
+ return c < 0x80 ? 1 :
+ c < 0x800 ? 2 :
+ c < 0x10000 ? 3 : 4;
+}
+
+static inline bool
+fy_utf8_is_valid(int c)
+{
+ return c >= 0 && !((c >= 0xd800 && c <= 0xdfff) || c >= 0x110000);
+}
+
+static inline bool
+fy_utf8_is_printable_ascii(int c)
+{
+ return c >= 0x20 && c <= 0x7e;
+}
+
+/* generic utf8 decoder (not inlined) */
+int fy_utf8_get_generic(const void *ptr, size_t left, int *widthp);
+
+/* -1 for end of input, -2 for invalid character, -3 for partial */
+#define FYUG_EOF -1
+#define FYUG_INV -2
+#define FYUG_PARTIAL -3
+
+static inline int fy_utf8_get(const void *ptr, size_t left, int *widthp)
+{
+ const uint8_t *p = ptr;
+
+ /* single byte (hot path) */
+ if (left <= 0) {
+ *widthp = 0;
+ return FYUG_EOF;
+ }
+
+ if (!(p[0] & 0x80)) {
+ *widthp = 1;
+ return p[0] & 0x7f;
+ }
+ return fy_utf8_get_generic(ptr, left, widthp);
+}
+
+int fy_utf8_get_right_generic(const void *ptr, size_t left, int *widthp);
+
+static inline int fy_utf8_get_right(const void *ptr, size_t left, int *widthp)
+{
+ const uint8_t *p = ptr + left;
+
+ /* single byte (hot path) */
+ if (left > 0 && !(p[-1] & 0x80)) {
+ if (widthp)
+ *widthp = 1;
+ return p[-1] & 0x7f;
+ }
+ return fy_utf8_get_right_generic(ptr, left, widthp);
+}
+
+
+/* for when you _know_ that there's enough room and c is valid */
+static inline void *fy_utf8_put_unchecked(void *ptr, int c)
+{
+ uint8_t *s = ptr;
+
+ assert(c >= 0);
+ if (c < 0x80)
+ *s++ = c;
+ else if (c < 0x800) {
+ *s++ = (c >> 6) | 0xc0;
+ *s++ = (c & 0x3f) | 0x80;
+ } else if (c < 0x10000) {
+ *s++ = (c >> 12) | 0xe0;
+ *s++ = ((c >> 6) & 0x3f) | 0x80;
+ *s++ = (c & 0x3f) | 0x80;
+ } else {
+ *s++ = (c >> 18) | 0xf0;
+ *s++ = ((c >> 12) & 0x3f) | 0x80;
+ *s++ = ((c >> 6) & 0x3f) | 0x80;
+ *s++ = (c & 0x3f) | 0x80;
+ }
+ return s;
+}
+
+static inline void *fy_utf8_put(void *ptr, size_t left, int c)
+{
+ if (!fy_utf8_is_valid(c) || fy_utf8_width(c) > left)
+ return NULL;
+
+ return fy_utf8_put_unchecked(ptr, c);
+}
+
+/* buffer must contain at least 5 characters */
+#define FY_UTF8_FORMAT_BUFMIN 5
+enum fy_utf8_escape {
+ fyue_none,
+ fyue_singlequote,
+ fyue_doublequote,
+ fyue_doublequote_json,
+ fyue_doublequote_yaml_1_1,
+};
+
+static inline bool fy_utf8_escape_is_any_doublequote(enum fy_utf8_escape esc)
+{
+ return esc >= fyue_doublequote && esc <= fyue_doublequote_yaml_1_1;
+}
+
+char *fy_utf8_format(int c, char *buf, enum fy_utf8_escape esc);
+
+#define fy_utf8_format_a(_c, _esc) \
+ ({ \
+ char *_buf = alloca(FY_UTF8_FORMAT_BUFMIN); \
+ fy_utf8_format((_c), _buf, _esc); \
+ })
+
+size_t fy_utf8_format_text_length(const char *buf, size_t len,
+ enum fy_utf8_escape esc);
+char *fy_utf8_format_text(const char *buf, size_t len,
+ char *out, size_t maxsz,
+ enum fy_utf8_escape esc);
+
+#define fy_utf8_format_text_a(_buf, _len, _esc) \
+ ({ \
+ const char *__buf = (_buf); \
+ size_t __len = (_len); \
+ enum fy_utf8_escape __esc = (_esc); \
+ size_t _outsz = fy_utf8_format_text_length(__buf, __len, __esc); \
+ char *_out = alloca(_outsz + 1); \
+ fy_utf8_format_text(__buf, __len, _out, _outsz, __esc); \
+ })
+
+char *fy_utf8_format_text_alloc(const char *buf, size_t len, enum fy_utf8_escape esc);
+
+const void *fy_utf8_memchr_generic(const void *s, int c, size_t n);
+
+static inline const void *fy_utf8_memchr(const void *s, int c, size_t n)
+{
+ if (c < 0 || !n)
+ return NULL;
+ if (c < 0x80)
+ return memchr(s, c, n);
+ return fy_utf8_memchr_generic(s, c, n);
+}
+
+static inline const void *fy_utf8_strchr(const void *s, int c)
+{
+ if (c < 0)
+ return NULL;
+ if (c < 0x80)
+ return strchr(s, c);
+ return fy_utf8_memchr_generic(s, c, strlen(s));
+}
+
+static inline int fy_utf8_count(const void *ptr, size_t len)
+{
+ const uint8_t *s = ptr, *e = ptr + len;
+ int w, count;
+
+ count = 0;
+ while (s < e) {
+ w = fy_utf8_width_by_first_octet(*s);
+
+ /* malformed? */
+ if (!w || s + w > e)
+ break;
+ s += w;
+
+ count++;
+ }
+
+ return count;
+}
+
+int fy_utf8_parse_escape(const char **strp, size_t len, enum fy_utf8_escape esc);
+
+#define F_NONE 0
+#define F_NON_PRINT FY_BIT(0) /* non printable */
+#define F_SIMPLE_SCALAR FY_BIT(1) /* part of simple scalar */
+#define F_QUOTE_ESC FY_BIT(2) /* escape form, i.e \n */
+#define F_LB FY_BIT(3) /* is a linebreak */
+#define F_WS FY_BIT(4) /* is a whitespace */
+#define F_PUNCT FY_BIT(5) /* is a punctuation mark */
+#define F_LETTER FY_BIT(6) /* is a letter a..z A..Z */
+#define F_DIGIT FY_BIT(7) /* is a digit 0..9 */
+
+extern uint8_t fy_utf8_low_ascii_flags[0x80];
+
+void *fy_utf8_split_posix(const char *str, int *argcp, const char * const *argvp[]);
+
+int fy_utf8_get_generic_s(const void *ptr, const void *ptr_end, int *widthp);
+int fy_utf8_get_generic_s_nocheck(const void *ptr, int *widthp);
+
+static inline int fy_utf8_get_s(const void *ptr, const void *ptr_end, int *widthp)
+{
+ const uint8_t *p = ptr;
+
+ /* single byte (hot path) */
+ if (ptr >= ptr_end) {
+ *widthp = 0;
+ return FYUG_EOF;
+ }
+
+ if (!(p[0] & 0x80)) {
+ *widthp = 1;
+ return p[0];
+ }
+ return fy_utf8_get_generic_s(ptr, ptr_end, widthp);
+}
+
+static inline int fy_utf8_get_s_nocheck(const void *ptr, int *widthp)
+{
+ const uint8_t *p = ptr;
+
+ if (!(p[0] & 0x80)) {
+ *widthp = 1;
+ return p[0];
+ }
+ return fy_utf8_get_generic_s_nocheck(ptr, widthp);
+}
+
+/* for most 64 bit arches this will fit in a single register */
+struct fy_utf8_result {
+ int c;
+ int w;
+};
+
+/* probably the most performant version */
+static inline struct fy_utf8_result fy_utf8_get_s_res(const void *ptr, const void *ptr_end)
+{
+ const uint8_t *p = ptr;
+ int c, width;
+
+ /* single byte (hot path) */
+ if (ptr >= ptr_end)
+ return (struct fy_utf8_result){ FYUG_EOF, 0 };
+
+ if (!(p[0] & 0x80))
+ return (struct fy_utf8_result){ p[0], 1 };
+
+ c = fy_utf8_get_generic_s(ptr, ptr_end, &width);
+ if (c < 0)
+ return (struct fy_utf8_result){ c, 0 };
+ return (struct fy_utf8_result){ c, width };
+}
+
+#endif
--- /dev/null
+/*
+ * fy-utils.c - Generic utilities for functionality that's missing
+ * from platforms.
+ *
+ * For now only used to implement memstream for Apple platforms.
+ *
+ * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <termios.h>
+#include <unistd.h>
+#include <sys/select.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <ctype.h>
+
+#include "fy-utf8.h"
+#include "fy-ctype.h"
+#include "fy-utils.h"
+
+#if defined(__APPLE__) && (_POSIX_C_SOURCE < 200809L)
+
+/*
+ * adapted from http://piumarta.com/software/memstream/
+ *
+ * Under the MIT license.
+ */
+
+/*
+ * ----------------------------------------------------------------------------
+ *
+ * OPEN_MEMSTREAM(3) BSD and Linux Library Functions OPEN_MEMSTREAM(3)
+ *
+ * SYNOPSIS
+ * #include "memstream.h"
+ *
+ * FILE *open_memstream(char **bufp, size_t *sizep);
+ *
+ * DESCRIPTION
+ * The open_memstream() function opens a stream for writing to a buffer.
+ * The buffer is dynamically allocated (as with malloc(3)), and
+ * automatically grows as required. After closing the stream, the caller
+ * should free(3) this buffer.
+ *
+ * When the stream is closed (fclose(3)) or flushed (fflush(3)), the
+ * locations pointed to by bufp and sizep are updated to contain,
+ * respectively, a pointer to the buffer and the current size of the
+ * buffer. These values remain valid only as long as the caller performs
+ * no further output on the stream. If further output is performed, then
+ * the stream must again be flushed before trying to access these
+ * variables.
+ *
+ * A null byte is maintained at the end of the buffer. This byte is not
+ * included in the size value stored at sizep.
+ *
+ * The stream's file position can be changed with fseek(3) or fseeko(3).
+ * Moving the file position past the end of the data already written fills
+ * the intervening space with zeros.
+ *
+ * RETURN VALUE
+ * Upon successful completion open_memstream() returns a FILE pointer.
+ * Otherwise, NULL is returned and errno is set to indicate the error.
+ *
+ * CONFORMING TO
+ * POSIX.1-2008
+ *
+ * ----------------------------------------------------------------------------
+ */
+
+#ifndef min
+#define min(X, Y) (((X) < (Y)) ? (X) : (Y))
+#endif
+
+struct memstream {
+ size_t position;
+ size_t size;
+ size_t capacity;
+ char *contents;
+ char **ptr;
+ size_t *sizeloc;
+};
+
+static int memstream_grow(struct memstream *ms, size_t minsize)
+{
+ size_t newcap;
+ char *newcontents;
+
+ newcap = ms->capacity * 2;
+ while (newcap <= minsize + 1)
+ newcap *= 2;
+ newcontents = realloc(ms->contents, newcap);
+ if (!newcontents)
+ return -1;
+ ms->contents = newcontents;
+ memset(ms->contents + ms->capacity, 0, newcap - ms->capacity);
+ ms->capacity = newcap;
+ *ms->ptr = ms->contents;
+ return 0;
+}
+
+static int memstream_read(void *cookie, char *buf, int count)
+{
+ struct memstream *ms = cookie;
+ size_t n;
+
+ n = min(ms->size - ms->position, (size_t)count);
+ if (n < 1)
+ return 0;
+ memcpy(buf, ms->contents, n);
+ ms->position += n;
+ return n;
+}
+
+static int memstream_write(void *cookie, const char *buf, int count)
+{
+ struct memstream *ms = cookie;
+
+ if (ms->capacity <= ms->position + (size_t)count &&
+ memstream_grow(ms, ms->position + (size_t)count) < 0)
+ return -1;
+ memcpy(ms->contents + ms->position, buf, count);
+ ms->position += count;
+ ms->contents[ms->position] = '\0';
+ if (ms->size < ms->position)
+ *ms->sizeloc = ms->size = ms->position;
+
+ return count;
+}
+
+static fpos_t memstream_seek(void *cookie, fpos_t offset, int whence)
+{
+ struct memstream *ms = cookie;
+ fpos_t pos= 0;
+
+ switch (whence) {
+ case SEEK_SET:
+ pos = offset;
+ break;
+ case SEEK_CUR:
+ pos = ms->position + offset;
+ break;
+ case SEEK_END:
+ pos = ms->size + offset;
+ break;
+ default:
+ errno= EINVAL;
+ return -1;
+ }
+ if (pos >= (fpos_t)ms->capacity && memstream_grow(ms, pos) < 0)
+ return -1;
+ ms->position = pos;
+ if (ms->size < ms->position)
+ *ms->sizeloc = ms->size = ms->position;
+ return pos;
+}
+
+static int memstream_close(void *cookie)
+{
+ struct memstream *ms = cookie;
+
+ ms->size = min(ms->size, ms->position);
+ *ms->ptr = ms->contents;
+ *ms->sizeloc = ms->size;
+ ms->contents[ms->size]= 0;
+ /* ms->contents is what's returned */
+ free(ms);
+ return 0;
+}
+
+FILE *open_memstream(char **ptr, size_t *sizeloc)
+{
+ struct memstream *ms;
+ FILE *fp;
+
+ if (!ptr || !sizeloc) {
+ errno= EINVAL;
+ goto err_out;
+ }
+
+ ms = calloc(1, sizeof(struct memstream));
+ if (!ms)
+ goto err_out;
+
+ ms->position = ms->size= 0;
+ ms->capacity = 4096;
+ ms->contents = calloc(ms->capacity, 1);
+ if (!ms->contents)
+ goto err_free_ms;
+ ms->ptr = ptr;
+ ms->sizeloc = sizeloc;
+ fp= funopen(ms, memstream_read, memstream_write,
+ memstream_seek, memstream_close);
+ if (!fp)
+ goto err_free_all;
+ *ptr = ms->contents;
+ *sizeloc = ms->size;
+ return fp;
+
+err_free_all:
+ free(ms->contents);
+err_free_ms:
+ free(ms);
+err_out:
+ return NULL;
+}
+
+#endif /* __APPLE__ && _POSIX_C_SOURCE < 200809L */
+
+bool fy_tag_uri_is_valid(const char *data, size_t len)
+{
+ const char *s, *e;
+ int w, j, k, width, c;
+ uint8_t octet, esc_octets[4];
+
+ s = data;
+ e = s + len;
+
+ while ((c = fy_utf8_get(s, e - s, &w)) >= 0) {
+ if (c != '%') {
+ s += w;
+ continue;
+ }
+
+ width = 0;
+ k = 0;
+ do {
+ /* short URI escape */
+ if ((e - s) < 3)
+ return false;
+
+ if (width > 0) {
+ c = fy_utf8_get(s, e - s, &w);
+ if (c != '%')
+ return false;
+ }
+
+ s += w;
+
+ octet = 0;
+
+ for (j = 0; j < 2; j++) {
+ c = fy_utf8_get(s, e - s, &w);
+ if (!fy_is_hex(c))
+ return false;
+ s += w;
+
+ octet <<= 4;
+ if (c >= '0' && c <= '9')
+ octet |= c - '0';
+ else if (c >= 'a' && c <= 'f')
+ octet |= 10 + c - 'a';
+ else
+ octet |= 10 + c - 'A';
+ }
+ if (!width) {
+ width = fy_utf8_width_by_first_octet(octet);
+
+ if (width < 1 || width > 4)
+ return false;
+ k = 0;
+ }
+ esc_octets[k++] = octet;
+
+ } while (--width > 0);
+
+ /* now convert to utf8 */
+ c = fy_utf8_get(esc_octets, k, &w);
+
+ if (c < 0)
+ return false;
+ }
+
+ return true;
+}
+
+int fy_tag_handle_length(const char *data, size_t len)
+{
+ const char *s, *e;
+ int c, w;
+
+ s = data;
+ e = s + len;
+
+ c = fy_utf8_get(s, e - s, &w);
+ if (c != '!')
+ return -1;
+ s += w;
+
+ c = fy_utf8_get(s, e - s, &w);
+ if (fy_is_ws(c))
+ return s - data;
+ /* if first character is !, empty handle */
+ if (c == '!') {
+ s += w;
+ return s - data;
+ }
+ if (!fy_is_first_alpha(c))
+ return -1;
+ s += w;
+ while (fy_is_alnum(c = fy_utf8_get(s, e - s, &w)))
+ s += w;
+ if (c == '!')
+ s += w;
+
+ return s - data;
+}
+
+int fy_tag_uri_length(const char *data, size_t len)
+{
+ const char *s, *e;
+ int c, w, cn, wn, uri_length;
+
+ s = data;
+ e = s + len;
+
+ while (fy_is_uri(c = fy_utf8_get(s, e - s, &w))) {
+ cn = fy_utf8_get(s + w, e - (s + w), &wn);
+ if ((fy_is_z(cn) || fy_is_blank(cn) || fy_is_any_lb(cn)) && fy_utf8_strchr(",}]", c))
+ break;
+ s += w;
+ }
+ uri_length = s - data;
+
+ if (!fy_tag_uri_is_valid(data, uri_length))
+ return -1;
+
+ return uri_length;
+}
+
+int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info)
+{
+ const char *s, *e;
+ int total_length, handle_length, uri_length, prefix_length, suffix_length;
+ int c, cn, w, wn;
+
+ s = data;
+ e = s + len;
+
+ prefix_length = 0;
+
+ /* it must start with '!' */
+ c = fy_utf8_get(s, e - s, &w);
+ if (c != '!')
+ return -1;
+ cn = fy_utf8_get(s + w, e - (s + w), &wn);
+ if (cn == '<') {
+ prefix_length = 2;
+ suffix_length = 1;
+ } else
+ prefix_length = suffix_length = 0;
+
+ if (prefix_length) {
+ handle_length = 0; /* set the handle to '' */
+ s += prefix_length;
+ } else {
+ /* either !suffix or !handle!suffix */
+ /* we scan back to back, and split handle/suffix */
+ handle_length = fy_tag_handle_length(s, e - s);
+ if (handle_length <= 0)
+ return -1;
+ s += handle_length;
+ }
+
+ uri_length = fy_tag_uri_length(s, e - s);
+ if (uri_length < 0)
+ return -1;
+
+ /* a handle? */
+ if (!prefix_length && (handle_length == 0 || data[handle_length - 1] != '!')) {
+ /* special case, '!', handle set to '' and suffix to '!' */
+ if (handle_length == 1 && uri_length == 0) {
+ handle_length = 0;
+ uri_length = 1;
+ } else {
+ uri_length = handle_length - 1 + uri_length;
+ handle_length = 1;
+ }
+ }
+ total_length = prefix_length + handle_length + uri_length + suffix_length;
+
+ if (total_length != (int)len)
+ return -1;
+
+ info->total_length = total_length;
+ info->handle_length = handle_length;
+ info->uri_length = uri_length;
+ info->prefix_length = prefix_length;
+ info->suffix_length = suffix_length;
+
+ return 0;
+}
+
+/* simple terminal methods; mainly for getting size of terminal */
+int fy_term_set_raw(int fd, struct termios *oldt)
+{
+ struct termios newt, t;
+ int ret;
+
+ /* must be a terminal */
+ if (!isatty(fd))
+ return -1;
+
+ ret = tcgetattr(fd, &t);
+ if (ret != 0)
+ return ret;
+
+ newt = t;
+
+ cfmakeraw(&newt);
+
+ ret = tcsetattr(fd, TCSANOW, &newt);
+ if (ret != 0)
+ return ret;
+
+ if (oldt)
+ *oldt = t;
+
+ return 0;
+}
+
+int fy_term_restore(int fd, const struct termios *oldt)
+{
+ /* must be a terminal */
+ if (!isatty(fd))
+ return -1;
+
+ return tcsetattr(fd, TCSANOW, oldt);
+}
+
+ssize_t fy_term_write(int fd, const void *data, size_t count)
+{
+ ssize_t wrn, r;
+
+ if (!isatty(fd))
+ return -1;
+ r = 0;
+ wrn = 0;
+ while (count > 0) {
+ do {
+ r = write(fd, data, count);
+ } while (r == -1 && errno == EAGAIN);
+ if (r < 0)
+ break;
+ wrn += r;
+ data += r;
+ count -= r;
+ }
+
+ /* return the amount written, or the last error code */
+ return wrn > 0 ? wrn : r;
+}
+
+int fy_term_safe_write(int fd, const void *data, size_t count)
+{
+ if (!isatty(fd))
+ return -1;
+
+ return fy_term_write(fd, data, count) == (ssize_t)count ? 0 : -1;
+}
+
+ssize_t fy_term_read(int fd, void *data, size_t count, int timeout_us)
+{
+ ssize_t rdn, r;
+ struct timeval tv, tvto, *tvp;
+ fd_set rdfds;
+
+ if (!isatty(fd))
+ return -1;
+
+ FD_ZERO(&rdfds);
+
+ memset(&tvto, 0, sizeof(tvto));
+ memset(&tv, 0, sizeof(tv));
+
+ if (timeout_us >= 0) {
+ tvto.tv_sec = timeout_us / 1000000;
+ tvto.tv_usec = timeout_us % 1000000;
+ tvp = &tv;
+ } else {
+ tvp = NULL;
+ }
+
+ r = 0;
+ rdn = 0;
+ while (count > 0) {
+ do {
+ FD_SET(fd, &rdfds);
+ if (tvp)
+ *tvp = tvto;
+ r = select(fd + 1, &rdfds, NULL, NULL, tvp);
+ } while (r == -1 && errno == EAGAIN);
+
+ /* select ends, or something weird */
+ if (r <= 0 || !FD_ISSET(fd, &rdfds))
+ break;
+
+ /* now read */
+ do {
+ r = read(fd, data, count);
+ } while (r == -1 && errno == EAGAIN);
+ if (r < 0)
+ break;
+
+ rdn += r;
+ data += r;
+ count -= r;
+ }
+
+ /* return the amount written, or the last error code */
+ return rdn > 0 ? rdn : r;
+}
+
+ssize_t fy_term_read_escape(int fd, void *buf, size_t count)
+{
+ char *p;
+ int r, rdn;
+ char c;
+
+ /* at least 3 characters */
+ if (count < 3)
+ return -1;
+
+ p = buf;
+ rdn = 0;
+
+ /* ESC */
+ r = fy_term_read(fd, &c, 1, 100 * 1000);
+ if (r != 1 || c != '\x1b')
+ return -1;
+ *p++ = c;
+ count--;
+ rdn++;
+
+ /* [ */
+ r = fy_term_read(fd, &c, 1, 100 * 1000);
+ if (r != 1 || c != '[')
+ return rdn;
+ *p++ = c;
+ count--;
+ rdn++;
+
+ /* read until error, out of buffer, or < 0x40 || > 0x7e */
+ r = -1;
+ while (count > 0) {
+ r = fy_term_read(fd, &c, 1, 100 * 1000);
+ if (r != 1)
+ r = -1;
+ if (r != 1)
+ break;
+ *p++ = c;
+ count--;
+ rdn++;
+
+ /* end of escape */
+ if (c >= 0x40 && c <= 0x7e)
+ break;
+ }
+
+ return rdn;
+}
+
+int fy_term_query_size_raw(int fd, int *rows, int *cols)
+{
+ char buf[32];
+ char *s, *e;
+ ssize_t r;
+
+ /* must be a terminal */
+ if (!isatty(fd))
+ return -1;
+
+ *rows = *cols = 0;
+
+ /* query text area */
+ r = fy_term_safe_write(fd, "\x1b[18t", 5);
+ if (r != 0)
+ return r;
+
+ /* read a character */
+ r = fy_term_read_escape(fd, buf, sizeof(buf));
+
+ /* return must be ESC[8;<height>;<width>;t */
+
+ if (r < 8 || r >= (int)sizeof(buf) - 2) /* minimum ESC[8;1;1t */
+ return -1;
+
+ s = buf;
+ e = s + r;
+
+ /* correct response? starts with ESC[8; */
+ if (s[0] != '\x1b' || s[1] != '[' || s[2] != '8' || s[3] != ';')
+ return -1;
+ s += 4;
+
+ /* must end with t */
+ if (e[-1] != 't')
+ return -1;
+ *--e = '\0'; /* remove trailing t, and zero terminate */
+
+ /* scan two ints separated by ; */
+ r = sscanf(s, "%d;%d", rows, cols);
+ if (r != 2)
+ return -1;
+
+ return 0;
+}
+
+int fy_term_query_size(int fd, int *rows, int *cols)
+{
+ struct termios old_term;
+ int ret, r;
+
+ if (!isatty(fd))
+ return -1;
+
+ r = fy_term_set_raw(fd, &old_term);
+ if (r != 0)
+ return -1;
+
+ ret = fy_term_query_size_raw(fd, rows, cols);
+
+ r = fy_term_restore(fd, &old_term);
+ if (r != 0)
+ return -1;
+
+ return ret;
+}
+
+int fy_comment_iter_begin(const char *comment, size_t size, struct fy_comment_iter *iter)
+{
+ if (!comment || !iter)
+ return -1;
+
+ memset(iter, 0, sizeof(*iter));
+ iter->start = comment;
+ iter->size = size == (size_t)-1 ? strlen(comment) : size;
+ iter->end = iter->start + iter->size;
+
+ if (!iter->size)
+ return -1;
+
+ iter->next = iter->start;
+ iter->line = 0;
+
+ return 0;
+}
+
+const char *fy_comment_iter_next_line(struct fy_comment_iter *iter, size_t *lenp)
+{
+ const char *s, *e, *le, *t;
+
+ if (!iter || !lenp || !iter->start)
+ return NULL;
+
+ /* no more */
+ if (iter->next >= iter->end)
+ return NULL;
+again:
+ *lenp = 0;
+
+ s = iter->next;
+ e = iter->end;
+
+ /* skip whitespace */
+ while (s < e && isblank(*s))
+ s++;
+
+ if (s >= e)
+ return NULL;
+
+ /* find end of line */
+ le = memchr(s, '\n', e - s);
+ if (!le) {
+ le = e;
+ iter->next = e;
+ } else {
+ iter->next = le + 1;
+ }
+
+ /* final? check if ends in *\/ */
+ if (le >= e && (le - s) > 2 && le[-2] == '*' && le[-1] == '/')
+ le -= 2;
+
+ /* backtrack while there's space at the end of line */
+ while (le > s && isblank(le[-1]))
+ le--;
+
+ /* check if the whole line is punctuation so it's formatting */
+ t = s;
+ while (t < le && ispunct(*t))
+ t++;
+
+ /* everything is punctuation? */
+ if (t > s && t >= le) {
+ if (((le - s) == 2 && s[0] == '/' && s[1] == '/') || /* '//' -> empty line */
+ ((le - s) == 1 && s[0] == '*')) { /* '*' -> empty line */
+ iter->line++;
+ return "";
+ }
+ /* for anything else, just try again */
+ goto again;
+ }
+
+ /* something is there, skip over '// ' or '* ' */
+ if ((le - s) > 3 && s[0] == '/' && s[1] == '/' && isblank(s[2]))
+ s += 3;
+ else if ((le - s) > 2 && s[0] == '*' && isblank(s[1]))
+ s += 2;
+ else if (iter->line == 0 && (le - s) > 3 && s[0] == '/' && s[1] == '*' && isblank(s[2]))
+ s += 3;
+
+ iter->line++;
+ *lenp = le - s;
+ return s;
+}
+
+void fy_comment_iter_end(struct fy_comment_iter *iter)
+{
+ /* nothing */
+}
+
+char *fy_get_cooked_comment(const char *raw_comment, size_t size)
+{
+ struct fy_comment_iter iter;
+ FILE *fp;
+ char *buf;
+ const char *line;
+ size_t len, line_len;
+ int ret;
+
+ if (!raw_comment)
+ return NULL;
+
+ fp = open_memstream(&buf, &len);
+ if (!fp)
+ return NULL;
+
+ ret = 0;
+ fy_comment_iter_begin(raw_comment, size, &iter);
+ while ((line = fy_comment_iter_next_line(&iter, &line_len)) != NULL) {
+ ret = fprintf(fp, "%.*s\n", (int)line_len, line);
+ if (ret < 0)
+ break;
+ }
+ fy_comment_iter_end(&iter);
+
+ fclose(fp);
+
+ if (ret < 0) {
+ if (buf)
+ free(buf);
+ return NULL;
+ }
+
+ /* must be freed */
+ return buf;
+}
+
+int fy_keyword_iter_begin(const char *text, size_t size, const char *keyword, struct fy_keyword_iter *iter)
+{
+ if (!text || !size || !keyword || !iter)
+ return -1;
+
+ memset(iter, 0, sizeof(*iter));
+ iter->keyword = keyword;
+ iter->keyword_len = strlen(keyword);
+ iter->start = text;
+ iter->size = size == (size_t)-1 ? strlen(text) : size;
+ iter->end = iter->start + iter->size;
+ iter->pc = '\n';
+
+ if (!iter->size)
+ return -1;
+
+ iter->next = iter->start;
+
+ return 0;
+}
+
+const char *fy_keyword_iter_next(struct fy_keyword_iter *iter)
+{
+ const char *keyword;
+ size_t keyword_len;
+ const char *s, *e;
+ int c, w, pc, mode;
+
+ if (!iter || !iter->start)
+ return NULL;
+
+ /* no more */
+ if (iter->next >= iter->end)
+ return NULL;
+
+ s = iter->next;
+ e = iter->end;
+
+ keyword = iter->keyword;
+ keyword_len = iter->keyword_len;
+
+ mode = 0;
+ pc = iter->pc;
+ while ((c = fy_utf8_get_s(s, e, &w)) >= 0) {
+
+ /* simple state machine to handle quoting */
+ switch (mode) {
+ case 0: /* unquoted */
+ if (c == keyword[0] && (fy_is_any_lb(pc) || fy_is_ws(pc)) &&
+ (size_t)(e - s) > (keyword_len + 1) && !memcmp(s, keyword, keyword_len) &&
+ (fy_is_ws(s[keyword_len]) || fy_is_any_lb(s[keyword_len]))) {
+
+ iter->next = s;
+ iter->pc = pc;
+ return s;
+ }
+
+ if (c == '\'')
+ mode = 1;
+ else if (c == '"')
+ mode = 3;
+ break;
+ case 1: /* single quote */
+ if (c == '\\')
+ mode = 2; /* escaped single quote? */
+ else if (c == '\'')
+ mode = 0; /* back to unquoted */
+ break;
+ case 2: /* single quote backslash */
+ mode = 1; /* back to single quote mode always */
+ break;
+ case 3: /* double quote */
+ if (c == '\\')
+ mode = 4; /* escaped quote? */
+ else if (c == '"')
+ mode = 0; /* back to unquoted */
+ break;
+ case 4:
+ mode = 3; /* back to double quote mode always */
+ break;
+ }
+ s += w;
+ pc = c;
+ }
+
+ return NULL;
+}
+
+void fy_keyword_iter_advance(struct fy_keyword_iter *iter, size_t advance)
+{
+ int w;
+ const char *prev;
+
+ if (!iter || !iter->next)
+ return;
+
+ prev = iter->next;
+
+ iter->next += advance;
+ if (iter->next >= iter->end)
+ iter->next = iter->end;
+
+ iter->pc = fy_utf8_get_right(prev, (size_t)(iter->next - prev), &w);
+ if (iter->pc < 0)
+ iter->pc = '\n';
+}
+
+void fy_keyword_iter_end(struct fy_keyword_iter *iter)
+{
+ /* nothing */
+}
--- /dev/null
+/*
+ * fy-utils.h - internal utilities header file
+ *
+ * Copyright (c) 2019 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef FY_UTILS_H
+#define FY_UTILS_H
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <unistd.h>
+#include <termios.h>
+
+#if defined(__linux__)
+#include <sys/sysmacros.h>
+#endif
+
+#if defined(__APPLE__) && (_POSIX_C_SOURCE < 200809L)
+FILE *open_memstream(char **ptr, size_t *sizeloc);
+#endif
+
+int fy_tag_handle_length(const char *data, size_t len);
+bool fy_tag_uri_is_valid(const char *data, size_t len);
+int fy_tag_uri_length(const char *data, size_t len);
+
+struct fy_tag_scan_info {
+ int total_length;
+ int handle_length;
+ int uri_length;
+ int prefix_length;
+ int suffix_length;
+};
+
+int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info);
+
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(x) ((sizeof(x)/sizeof((x)[0])))
+#endif
+
+#if !defined(NDEBUG) && (defined(__GNUC__) && __GNUC__ >= 4)
+#define FY_ALWAYS_INLINE __attribute__((always_inline))
+#else
+#define FY_ALWAYS_INLINE /* nothing */
+#endif
+
+#if defined(__GNUC__) && __GNUC__ >= 4
+#define FY_UNUSED __attribute__((unused))
+#else
+#define FY_UNUSED /* nothing */
+#endif
+
+int fy_term_set_raw(int fd, struct termios *oldt);
+int fy_term_restore(int fd, const struct termios *oldt);
+ssize_t fy_term_write(int fd, const void *data, size_t count);
+int fy_term_safe_write(int fd, const void *data, size_t count);
+ssize_t fy_term_read(int fd, void *data, size_t count, int timeout_us);
+ssize_t fy_term_read_escape(int fd, void *buf, size_t count);
+
+/* the raw methods require the terminal to be in raw mode */
+int fy_term_query_size_raw(int fd, int *rows, int *cols);
+
+/* the non raw methods will set the terminal to raw and then restore */
+int fy_term_query_size(int fd, int *rows, int *cols);
+
+struct fy_comment_iter {
+ const char *start;
+ size_t size;
+ const char *end;
+ const char *next;
+ int line;
+};
+
+int fy_comment_iter_begin(const char *comment, size_t size, struct fy_comment_iter *iter);
+const char *fy_comment_iter_next_line(struct fy_comment_iter *iter, size_t *lenp);
+void fy_comment_iter_end(struct fy_comment_iter *iter);
+
+char *fy_get_cooked_comment(const char *raw_comment, size_t size);
+
+struct fy_keyword_iter {
+ const char *keyword;
+ size_t keyword_len;
+ const char *start;
+ size_t size;
+ const char *end;
+ const char *next;
+ int pc;
+};
+
+int fy_keyword_iter_begin(const char *text, size_t size, const char *keyword, struct fy_keyword_iter *iter);
+const char *fy_keyword_iter_next(struct fy_keyword_iter *iter);
+void fy_keyword_iter_advance(struct fy_keyword_iter *iter, size_t advance);
+void fy_keyword_iter_end(struct fy_keyword_iter *iter);
+
+#if !defined(S_ISREG) && defined(S_IFMT) && defined(S_IFREG)
+#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
+#endif
+
+#if !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR)
+#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
+#endif
+
+#endif
--- /dev/null
+*.timestamp
check_PROGRAMS = libfyaml-test
libfyaml_test_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/valgrind/ \
- -I$(top_srcdir)/src/lib/
+ -I$(top_srcdir)/src/lib/ \
+ -I$(top_srcdir)/src/util
libfyaml_test_LDADD = $(AM_LDADD) $(CHECK_LIBS) $(top_builddir)/src/libfyaml.la
libfyaml_test_CFLAGS = $(AM_CFLAGS) $(CHECK_CFLAGS)
libfyaml_test_LDFLAGS = $(AM_LDFLAGS) $(CHECK_LDFLAGS)
# the private tests require static compilation
# if we're compiling shared only, disable the private parts
if HAVE_STATIC
-libfyaml_test_SOURCES += libfyaml-test-private.c
+libfyaml_test_SOURCES += \
+ libfyaml-test-private.c \
+ libfyaml-test-private-id.c
libfyaml_test_LDFLAGS += -static
endif
# normal YAML testsuite using document event stream
TESTS += testsuite-evstream.test
-testsuite.test: test-suite-data json-test-suite-data
+# resolution equivalence
+TESTS += testsuite-resolution.test
-test-suite-data:
- @GIT@ clone "@TESTSUITEURL@" -- $@ && \
- cd $@ && \
- @GIT@ checkout --detach @TESTSUITECHECKOUT@
+testsuite.test: test-suite-data.checkout.timestamp
+if HAVE_JQ
+testsuite-json.test: test-suite-data.checkout.timestamp
+endif
+testsuite-evstream.test: test-suite-data.checkout.timestamp
+testsuite-resolution.test: test-suite-data.checkout.timestamp
-jsontestsuite.test: json-test-suite-data
+test-suite-data.checkout.timestamp:
+ @GIT@ clone -q "@TESTSUITEURL@" -- test-suite-data && \
+ @GIT@ -C test-suite-data checkout -q --detach @TESTSUITECHECKOUT@ && \
+ touch test-suite-data.checkout.timestamp
-json-test-suite-data:
- @GIT@ clone "@JSONTESTSUITEURL@" -- $@ && \
- cd $@ && \
- @GIT@ checkout --detach @JSONTESTSUITECHECKOUT@
+jsontestsuite.test: json-test-suite-data.checkout.timestamp
-check-local: test-suite-data json-test-suite-data
+json-test-suite-data.checkout.timestamp:
+ @GIT@ clone -q "@JSONTESTSUITEURL@" -- json-test-suite-data && \
+ @GIT@ -C json-test-suite-data checkout -q --detach @JSONTESTSUITECHECKOUT@ && \
+ touch json-test-suite-data.checkout.timestamp
distclean-local:
- @rm -rf test-suite-data json-test-suite-data
+ @rm -rf test-suite-data test-suite-data.checkout.timestamp
+ @rm -rf json-test-suite-data json-test-suite-data.checkout.timestamp
endif
endif
}
END_TEST
+START_TEST(token_test) {
+ struct fy_document *fyd;
+ struct fy_node *fyn_sequence, *fyn_mapping, *fyn_scalar;
+ struct fy_token *fyn_sequence_start, *fyn_sequence_end,
+ *fyn_mapping_start, *fyn_mapping_end, *fyn_scalar_token;
+ struct fy_parse_cfg cfg = {
+ .flags = FYPCF_PARSE_COMMENTS
+ };
+
+ /* build document */
+
+ fyd = fy_document_build_from_string(&cfg, "- name: key\n"
+ " value: value\n", FY_NT);
+
+
+ /* Root (sequence) */
+ fyn_sequence = fy_document_root(fyd);
+ ck_assert_ptr_ne(fyn_sequence, NULL);
+
+ fyn_sequence_start = fy_node_get_start_token(fyn_sequence);
+ ck_assert_ptr_ne(fyn_sequence_start, NULL);
+
+ ck_assert_ptr_ne(fy_token_start_mark(fyn_sequence_start), NULL);
+
+ ck_assert_int_eq(fy_token_start_mark(fyn_sequence_start)->line, 0);
+ ck_assert_int_eq(fy_token_start_mark(fyn_sequence_start)->column, 0);
+
+ fyn_sequence_end = fy_node_get_end_token(fyn_sequence);
+ ck_assert_ptr_ne(fyn_sequence_end, NULL);
+
+ ck_assert_int_eq(fy_token_end_mark(fyn_sequence_end)->line, 2);
+ ck_assert_int_eq(fy_token_end_mark(fyn_sequence_end)->column, 0);
+
+ /* Mapping (sequence item) */
+ fyn_mapping = fy_node_sequence_get_by_index(fyn_sequence, 0);
+ ck_assert_ptr_ne(fyn_mapping, NULL);
+
+ fyn_mapping_start = fy_node_get_start_token(fyn_mapping);
+ ck_assert_ptr_ne(fyn_mapping_start, NULL);
+
+ ck_assert_ptr_ne(fy_token_start_mark(fyn_mapping_start), NULL);
+
+ ck_assert_int_eq(fy_token_start_mark(fyn_mapping_start)->line, 0);
+ ck_assert_int_eq(fy_token_start_mark(fyn_mapping_start)->column, 2);
+
+ fyn_mapping_end = fy_node_get_end_token(fyn_mapping);
+ ck_assert_ptr_ne(fyn_mapping_end, NULL);
+
+ ck_assert_ptr_ne(fy_token_start_mark(fyn_mapping_end), NULL);
+
+ ck_assert_int_eq(fy_token_start_mark(fyn_mapping_end)->line, 2);
+ ck_assert_int_eq(fy_token_start_mark(fyn_mapping_end)->column, 0);
+
+ /* Scalar (key) */
+ fyn_scalar = fy_node_pair_key(fy_node_mapping_get_by_index(fyn_mapping, 0));
+ ck_assert_ptr_ne(fyn_scalar, NULL);
+
+ fyn_scalar_token = fy_node_get_start_token(fyn_scalar);
+ ck_assert_ptr_ne(fyn_scalar_token, NULL);
+ ck_assert_ptr_eq(fyn_scalar_token, fy_node_get_end_token(fyn_scalar));
+
+ ck_assert_ptr_ne(fy_token_start_mark(fyn_scalar_token), NULL);
+ ck_assert_int_eq(fy_token_start_mark(fyn_scalar_token)->line, 0);
+ ck_assert_int_eq(fy_token_start_mark(fyn_scalar_token)->column, 2);
+
+ ck_assert_ptr_ne(fy_token_end_mark(fyn_scalar_token), NULL);
+ ck_assert_int_eq(fy_token_end_mark(fyn_scalar_token)->line, 0);
+ ck_assert_int_eq(fy_token_end_mark(fyn_scalar_token)->column, 6);
+
+ /* cleanup */
+ fy_document_destroy(fyd);
+}
+END_TEST
+
TCase *libfyaml_case_core(void)
{
TCase *tc;
tcase_add_test(tc, scanf_check);
+ tcase_add_test(tc, token_test);
+
return tc;
}
--- /dev/null
+/*
+ * libfyaml-test-private-id.c - libfyaml id allocation and handling
+ *
+ * Copyright (c) 2023 Pantelis Antoniou <pantelis.antoniou@konsulko.com>
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <getopt.h>
+#include <unistd.h>
+#include <limits.h>
+#include <alloca.h>
+
+#include <check.h>
+
+#include <libfyaml.h>
+
+#include "fy-utils.h"
+#include "fy-id.h"
+
+/* check ffs works */
+START_TEST(id_ffs)
+{
+ static const struct {
+ fy_id_bits v;
+ int r;
+ } ffs_check[] = {
+ { .v = 0, .r = -1 },
+ { .v = ~(fy_id_bits)0, .r = 0 },
+ { .v = ((fy_id_bits)1 << 0), .r = 0 },
+ { .v = ((fy_id_bits)1 << (FY_ID_BITS_BITS - 1)), .r = (FY_ID_BITS_BITS - 1) },
+ { .v = ((fy_id_bits)1 << 0) | ((fy_id_bits)1 << (FY_ID_BITS_BITS - 1)), .r = 0 },
+ { .v = ((fy_id_bits)1 << (FY_ID_BITS_BITS / 2)), .r = (FY_ID_BITS_BITS / 2), },
+ };
+ unsigned int i;
+ int r;
+
+ for (i = 0; i < ARRAY_SIZE(ffs_check); i++) {
+ r = fy_id_ffs(ffs_check[i].v);
+ ck_assert_int_eq(r, ffs_check[i].r);
+ }
+}
+END_TEST
+
+/* a random bit number for the following tests, not a power of 2 */
+#define BA_BITS 67
+
+#define BA_DECL \
+ size_t ba_count = FY_ID_BITS_ARRAY_COUNT(BA_BITS); \
+ fy_id_bits *ba = alloca(ba_count * sizeof(*ba))
+
+/* verify that reset clears everything */
+START_TEST(id_reset)
+{
+ BA_DECL;
+ unsigned int i;
+
+ fy_id_reset(ba, ba_count);
+ for (i = 0; i < ba_count; i++)
+ ck_assert_int_eq(ba[i], 0);
+}
+END_TEST
+
+/* verify that allocation to the full works, and then fails */
+START_TEST(id_alloc_full)
+{
+ BA_DECL;
+ unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS);
+ unsigned int i;
+ int id, expected_id;
+
+ fy_id_reset(ba, ba_count);
+
+ /* allocate all, verify that we get numbers in sequence */
+ expected_id = 0;
+ for (i = 0; i < ba_bits_actual; i++) {
+ id = fy_id_alloc(ba, ba_count);
+ ck_assert_int_eq(id, expected_id);
+ expected_id++;
+ }
+
+ /* full, it must fail now */
+ id = fy_id_alloc(ba, ba_count);
+ ck_assert_int_eq(id, -1);
+}
+END_TEST
+
+/* verify that allocation when almost full works, and then fails */
+START_TEST(id_alloc_almost_full)
+{
+ BA_DECL;
+ unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS);
+ unsigned int i;
+ int id, expected_id;
+
+ /* fill the array */
+ fy_id_reset(ba, ba_count);
+ for (i = 0; i < ba_bits_actual; i++)
+ fy_id_set_used(ba, ba_count, i);
+
+ /* allocate all, verify that we get numbers in sequence */
+ expected_id = 0;
+ for (i = 0; i < ba_bits_actual; i++) {
+ /* free one, and allocate, it must succeed at the exact spot */
+ fy_id_free(ba, ba_count, i);
+ id = fy_id_alloc(ba, ba_count);
+ ck_assert_int_eq(id, expected_id);
+ expected_id++;
+
+ /* now it must fail */
+ id = fy_id_alloc(ba, ba_count);
+ ck_assert_int_eq(id, -1);
+ }
+}
+END_TEST
+
+/* verify that allocation of even bits works */
+START_TEST(id_alloc_even)
+{
+ BA_DECL;
+ unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS);
+ unsigned int i, j;
+ int id, expected_id;
+
+ /* fill the array */
+ fy_id_reset(ba, ba_count);
+ for (i = 0; i < ba_bits_actual; i++)
+ fy_id_set_used(ba, ba_count, i);
+
+ /* free the even ids */
+ j = ba_bits_actual / 2;
+ for (i = 0; i < j; i++)
+ fy_id_free(ba, ba_count, i * 2);
+
+ /* allocate all, verify that we get numbers in sequence */
+ expected_id = 0;
+ for (i = 0; i < j; i++) {
+ id = fy_id_alloc(ba, ba_count);
+ ck_assert_int_eq(id, expected_id);
+ expected_id += 2;
+ }
+}
+END_TEST
+
+/* verify that allocation of odd bits works */
+START_TEST(id_alloc_odd)
+{
+ BA_DECL;
+ unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS);
+ unsigned int i, j;
+ int id, expected_id;
+
+ /* fill the array */
+ fy_id_reset(ba, ba_count);
+ for (i = 0; i < ba_bits_actual; i++)
+ fy_id_set_used(ba, ba_count, i);
+
+ /* free the even ids */
+ j = ba_bits_actual / 2;
+ for (i = 0; i < j; i++)
+ fy_id_free(ba, ba_count, i * 2 + 1);
+
+ /* allocate all, verify that we get numbers in sequence */
+ expected_id = 1;
+ for (i = 0; i < j; i++) {
+ id = fy_id_alloc(ba, ba_count);
+ ck_assert_int_eq(id, expected_id);
+ expected_id += 2;
+ }
+}
+END_TEST
+
+/* verify that a specific allocation sequence works */
+START_TEST(id_alloc_seq)
+{
+ BA_DECL;
+ unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS);
+ unsigned int i;
+ int id, expected_id;
+ int check_ids[] = {
+ 0,
+ ba_bits_actual / 2 - 1,
+ ba_bits_actual / 2,
+ ba_bits_actual / 2 + 1,
+ ba_bits_actual - 1
+ };
+
+ /* fill the array */
+ fy_id_reset(ba, ba_count);
+ for (i = 0; i < ba_bits_actual; i++)
+ fy_id_set_used(ba, ba_count, i);
+
+ /* free those specific bits */
+ for (i = 0; i < ARRAY_SIZE(check_ids); i++)
+ fy_id_free(ba, ba_count, check_ids[i]);
+
+ /* now allocate in sequence */
+ expected_id = 0;
+ for (i = 0; i < ARRAY_SIZE(check_ids); i++) {
+ expected_id = check_ids[i];
+ id = fy_id_alloc(ba, ba_count);
+ ck_assert_int_eq(id, expected_id);
+ }
+}
+END_TEST
+
+/* verify that a iterator works for a single bit in array */
+START_TEST(id_iter_single)
+{
+ BA_DECL;
+ unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS);
+ unsigned int i;
+ int id, expected_id, found_id;
+ struct fy_id_iter iter;
+
+ for (i = 0; i < ba_bits_actual; i++) {
+ fy_id_reset(ba, ba_count);
+ fy_id_set_used(ba, ba_count, i);
+
+ expected_id = i;
+ found_id = -1;
+ fy_id_iter_begin(ba, ba_count, &iter);
+ while ((id = fy_id_iter_next(ba, ba_count, &iter)) >= 0) {
+ /* must find a single one */
+ ck_assert_int_lt(found_id, 0);
+ found_id = id;
+ ck_assert_int_eq(id, expected_id);
+ }
+ fy_id_iter_end(ba, ba_count, &iter);
+ }
+
+}
+END_TEST
+
+/* verify that a iterator works for a full array */
+START_TEST(id_iter_full)
+{
+ BA_DECL;
+ unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS);
+ unsigned int i;
+ int id, expected_id;
+ struct fy_id_iter iter;
+
+ fy_id_reset(ba, ba_count);
+ for (i = 0; i < ba_bits_actual; i++)
+ fy_id_set_used(ba, ba_count, i);
+
+ expected_id = 0;
+ fy_id_iter_begin(ba, ba_count, &iter);
+ while ((id = fy_id_iter_next(ba, ba_count, &iter)) >= 0) {
+ ck_assert_int_eq(id, expected_id);
+ expected_id++;
+ }
+
+ /* we must have run through the whole array */
+ ck_assert_int_eq(expected_id, ba_bits_actual);
+
+ fy_id_iter_end(ba, ba_count, &iter);
+}
+END_TEST
+
+/* verify that a iterator works for sequences up to 3 bits */
+START_TEST(id_iter_seq)
+{
+ BA_DECL;
+ unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS);
+ int idtab[][3] = {
+ { 0, ba_bits_actual - 1, -1 },
+ { 0, 1, -1 },
+ { ba_bits_actual - 2, ba_bits_actual - 1, -1 },
+ { 0, FY_ID_BITS_BITS - 1, -1 },
+ { 0, FY_ID_BITS_BITS, -1 },
+ { FY_ID_BITS_BITS - 1, FY_ID_BITS_BITS, -1 },
+ { FY_ID_BITS_BITS, FY_ID_BITS_BITS + 1, -1 },
+ { 0, 1, 2 },
+ { 0, ba_bits_actual - 2, ba_bits_actual - 1 },
+ { ba_bits_actual - 3, ba_bits_actual - 2, ba_bits_actual - 1 },
+ };
+ unsigned int i;
+ int id, expected_id;
+ int p0, p1, p2;
+ struct fy_id_iter iter;
+
+ for (i = 0; i < ARRAY_SIZE(idtab); i++) {
+
+ p0 = idtab[i][0];
+ p1 = idtab[i][1];
+ p2 = idtab[i][2];
+
+ fy_id_reset(ba, ba_count);
+ if (p0 >= 0)
+ fy_id_set_used(ba, ba_count, p0);
+ if (p1 >= 0)
+ fy_id_set_used(ba, ba_count, p1);
+ if (p2 >= 0)
+ fy_id_set_used(ba, ba_count, p2);
+
+ fy_id_iter_begin(ba, ba_count, &iter);
+
+ if (p0 >= 0) {
+ id = fy_id_iter_next(ba, ba_count, &iter);
+ expected_id = p0;
+ ck_assert_int_eq(id, expected_id);
+ fy_id_set_free(ba, ba_count, expected_id);
+ }
+
+ if (p1 >= 0) {
+ id = fy_id_iter_next(ba, ba_count, &iter);
+ expected_id = p1;
+ ck_assert_int_eq(id, expected_id);
+ fy_id_set_free(ba, ba_count, expected_id);
+ }
+
+ if (p2 >= 0) {
+ id = fy_id_iter_next(ba, ba_count, &iter);
+ expected_id = p2;
+ ck_assert_int_eq(id, expected_id);
+ fy_id_set_free(ba, ba_count, expected_id);
+ }
+ fy_id_iter_end(ba, ba_count, &iter);
+
+ /* and now it should be empty */
+ fy_id_iter_begin(ba, ba_count, &iter);
+ id = fy_id_iter_next(ba, ba_count, &iter);
+ ck_assert_int_eq(id, -1);
+ fy_id_iter_end(ba, ba_count, &iter);
+ }
+}
+END_TEST
+
+TCase *libfyaml_case_private_id(void)
+{
+ TCase *tc;
+
+ tc = tcase_create("private-id");
+
+ tcase_add_test(tc, id_ffs);
+
+ tcase_add_test(tc, id_reset);
+ tcase_add_test(tc, id_alloc_full);
+ tcase_add_test(tc, id_alloc_almost_full);
+ tcase_add_test(tc, id_alloc_even);
+ tcase_add_test(tc, id_alloc_odd);
+ tcase_add_test(tc, id_alloc_seq);
+
+ tcase_add_test(tc, id_iter_single);
+ tcase_add_test(tc, id_iter_full);
+ tcase_add_test(tc, id_iter_seq);
+
+ return tc;
+}
#if defined(HAVE_STATIC) && HAVE_STATIC
extern TCase *libfyaml_case_private(void);
+extern TCase *libfyaml_case_private_id(void);
#endif
extern TCase *libfyaml_case_core(void);
extern TCase *libfyaml_case_meta(void);
#if defined(HAVE_STATIC) && HAVE_STATIC
suite_add_tcase(s, libfyaml_case_private());
+ suite_add_tcase(s, libfyaml_case_private_id());
#endif
suite_add_tcase(s, libfyaml_case_core());
suite_add_tcase(s, libfyaml_case_meta());
--- /dev/null
+#!/bin/bash
+
+count=0
+for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do
+ for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do
+ # there must be a test there
+ if [ ! -e "$tst/===" ]; then
+ continue
+ fi
+ # skip tests that are expected to fail
+ if [ -e "$tst/error" ]; then
+ continue
+ fi
+ # there must be an alias in the test
+ grep -q '\*[A-Za-z]' "$tst/in.yaml"
+ if [ $? -ne 0 ]; then
+ continue
+ fi
+ count=`expr $count + 1`
+ done
+done
+
+# output plan
+echo 1..$count
+
+# X38W has duplicate keys after resolution
+skiplist="2JQS X38W"
+xfaillist=""
+
+i=0
+for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do
+
+ for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do
+
+ # there must be a test there
+ if [ ! -e "$tst/===" ]; then
+ continue
+ fi
+
+ # skip tests that are expected to fail
+ if [ -e "$tst/error" ]; then
+ continue
+ fi
+
+ # there must be an alias in the test
+ grep -q '\*[A-Za-z]' "$tst/in.yaml"
+ if [ $? -ne 0 ]; then
+ continue
+ fi
+
+ i=`expr $i + 1`
+
+ # strip trailing /
+ t=${tst%/}
+ # remove test-suite-data/
+ test_subtest_id=`echo $t | cut -d/ -f2-`
+ test_id=`echo $test_subtest_id | cut -d/ -f1`
+ subtest_id=`echo $test_subtest_id | cut -s -d/ -f2`
+
+ desctxt=`cat 2>/dev/null "$tst/==="`
+
+ t1=`mktemp`
+ t2=`mktemp`
+
+ directive=""
+ for skip in $skiplist; do
+ if [ "$test_subtest_id" == "$skip" ]; then
+ directive=" # skip: duplicate keys in testcase; cannot load as document"
+ break
+ fi
+ done
+
+ res="ok"
+ if [ "x$directive" == "x" ]; then
+ res="not ok"
+
+ ${TOP_BUILDDIR}/src/fy-tool --dump --resolve "$tst/in.yaml" | ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t1"
+ ${TOP_BUILDDIR}/src/fy-tool --dump --resolve --streaming "$tst/in.yaml" | ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t2"
+
+ diff -u "$t1" "$t2"
+ if [ $? -eq 0 ]; then
+ res="ok"
+ else
+ res="not ok"
+ fi
+
+ rm -f "$t1" "$t2"
+
+ for xfail in $xfaillist; do
+ if [ "$test_subtest_id" == "$xfail" ]; then
+ directive=" # TODO: known failure."
+ break
+ fi
+ done
+ fi
+
+ echo "$res $i $test_subtest_id - $desctxt$directive"
+ done
+done
projects / libfyaml.git / commitdiff