From: Timo Röhling Date: Tue, 27 Jan 2026 10:46:11 +0000 (+0100) Subject: Import libfyaml_0.9.3.orig.tar.gz X-Git-Tag: archive/raspbian/0.9.4-1+rpi1~4 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=902865e63b994ce3266eda3b437632a91c22e168;p=libfyaml.git Import libfyaml_0.9.3.orig.tar.gz [dgit import orig libfyaml_0.9.3.orig.tar.gz] --- 902865e63b994ce3266eda3b437632a91c22e168 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bc30ee7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,78 @@ +**/*.dirstamp +**/*.la +**/*.lo +**/*.o +**/.deps/ +**/.libs/ +**/Makefile +**/Makefile.in +aclocal.m4 +ar-lib +autom4te.cache +build-aux/shave +build-aux/shave-libtool +compile +config.guess +config.h +config.h.in +config.h.in~ +config.log +config.status +config.sub +configure +depcomp +doc/_build +examples/*.diff +examples/*.log +examples/*.scan +install/ +install-sh +libfyaml.pc +libfyaml-*.tar.gz +libtool +ltmain.sh +m4/libtool.m4 +m4/lt~obsolete.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +Makefile +Makefile.in +missing +output/ +sphinx/ +src/.deps/ +src/fy-tool +src/internal/.deps/ +src/internal/.libs/ +src/lib/.deps/ +src/libfyaml-parser +src/lib/.libs/ +src/.libs/ +src/tool/.deps/ +src/tool/.libs/ +stamp-h1 +tags +TAGS +tap-driver.sh +.tarball-version +test/.deps +test-driver +test/libfyaml-test +test/.libs +test/*.log +test/test-suite-data/ +test/*.trs +test/*.txt +.version +libfyaml-*/ +libfyaml[\-_]*.deb +libfyaml[\-_]*.tar.* +libfyaml[\-_]*.build +libfyaml[\-_]*.dsc +libfyaml[\-_]*.changes +libfyaml_*.orig.tar.gz +debian/changelog +debian/control +debian/copyright +artifacts/ diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..4291c19 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,29 @@ +name: Standard Automake CI + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: apt-deps + run: sudo apt-get update -qq && sudo apt-get install --no-install-recommends -y gcc autoconf automake libtool git make libyaml-dev libltdl-dev pkg-config check python3 python3-pip python3-setuptools + + - name: boostrap + run: ./bootstrap.sh + + - name: configure + run: ./configure + + - name: make + run: make + + - name: make check + run: make check + + - name: make distcheck + run: make distcheck diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2ec5596 --- /dev/null +++ b/.gitignore @@ -0,0 +1,72 @@ +*.dirstamp +*.la +*.lo +*.o +.deps/ +.libs/ +aclocal.m4 +ar-lib +autom4te.cache/ +build-aux/shave +build-aux/shave-libtool +compile +config.guess +config.h +config.h.in +config.h.in~ +config.log +config.status +config.sub +configure +depcomp +doc/_build +examples/*.diff +examples/*.log +examples/*.scan +install/ +install-sh +libfyaml.pc +libfyaml-*.tar.gz +libtool +ltmain.sh +m4/libtool.m4 +m4/lt~obsolete.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +Makefile +Makefile.in +missing +output/ +sphinx/ +src/fy-tool +src/libfyaml-parser +src/fy-thread +src/fy-b3sum +src/fy-allocators +src/*.a +stamp-h1 +tags +TAGS +tap-driver.sh +test-driver +test/libfyaml-test +test/*.log +test/test-suite-data/ +test/json-test-suite-data/ +test/*.trs +test/*.txt +.version +libfyaml-*/ +libfyaml[_-]*.deb +libfyaml[_-]*.tar.* +libfyaml[_-]*.build +libfyaml[_-]*.dsc +libfyaml[_-]*.changes +libfyaml_*.orig.tar.gz +debian/*.install +debian/changelog +debian/control +debian/copyright +artifacts/ +build diff --git a/.libtool-version b/.libtool-version new file mode 100644 index 0000000..7e093fb --- /dev/null +++ b/.libtool-version @@ -0,0 +1 @@ +2:0:0 diff --git a/.tarball-version b/.tarball-version new file mode 100644 index 0000000..965065d --- /dev/null +++ b/.tarball-version @@ -0,0 +1 @@ +0.9.3 diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..ffb66c7 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Pantelis Antoniou diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ae5157b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,180 @@ +# Changelog + +All notable changes to libfyaml will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.9.3] - 2026-01-14 + +### Added + +- fy-tool: `-winf` option for infinite width output +- fy-tool: `--no-output-newline` option (useful with oneline mode) +- Emitter: `FYEXCF_OUTPUT_FILENAME` extended option for direct file output + +### Changed + +- JSON emit mode now works like YAML flow mode (less special-casing) +- Compact emit mode now produces truly compact output (no indentation) +- Emitter only changes plain scalar style when space or linebreak is present (preserves numeric scalars better) + +### Fixed + +- **#170**: Wrong length return when buffer ends mid UTF-8 sequence +- **#167**: Oneline styles now correctly use infinite width +- **#139**: Walk path parser now always creates diagnostics (fixes token reference leak) +- **#137**: Walk memory leak on parse error path +- **#136**: Walk memory leak when no output is generated +- **#131**: Walk memory leak when expression error occurs +- Composer: Memory leak when back-to-back complex keys exist +- Emitter: Broken flow mode output +- Emitter: Unnecessary plain scalar style changes + +### Platform Support + +**Supported platforms**: Linux, macOS, FreeBSD, OpenBSD, and NetBSD. + +- **NetBSD**: Disabled mremap (different semantics than Linux) +- **NetBSD**: CMake shared libraries now correctly built with -fPIC +- **NetBSD**: Only include alloca.h if it exists +- **NetBSD**: Fixed ctype(3) argument casting (unsigned char) +- **GNU/Hurd**: Use `` for endianness detection +- Use `getopt_long` instead of `getopt_long_only` for portability +- Fix test(1) operator in test suite (use POSIX `=` not bash `==`) +- Fix `-Wformat` warning (use PRIx64 for uint64_t) + +### Statistics + +- 25 commits since v0.9.2 +- 6 bug fix issues closed + +## [0.9.2] - 2025-12-29 + +### Fixed +- **#156**: automake: Respect --disable-static and don't build internals +- **#157**: dist tarball now includes the missing CMake files + +## [0.9.1] - 2025-12-28 + +### CRITICAL: Licensing Change + +**GPL Code Removed - Library Now Fully MIT Licensed** + +Replaced GPL-licensed list implementation with clean-room MIT-licensed minimal +implementation. The entire library is now consistently MIT licensed without any +GPL components. This is a major change for users requiring permissive licensing +for commercial or proprietary projects. + +### Added + +- Thread-safe lockless allocator infrastructure (linear, malloc, mremap, dedup, auto) +- Parser checkpointing API: `fy_parser_parse_peek()`, `fy_parser_skip()`, `fy_parser_count_sequence_items()`, `fy_parser_count_mapping_items()` +- Public composer and document builder interfaces +- YPath set operators: `@` for inclusion, `!` for exclusion +- Compact emitter formats (flow and JSON) +- `fy_emit_body_node()` for emitting single nodes without stream/doc events +- `fy_event_report()` methods for error diagnostics in composer interface +- `fy_node_get_tag0()` and `fy_node_get_tag_length()` tag utility methods +- `fy_node_delete()` method for explicit node removal +- `fy_allocator_contains()` for pointer containment checks +- `fy_event_get_type()` convenience wrapper +- CMake documentation build targets (doc-html, doc-latexpdf, doc-man) +- Cross-compilation support for both CMake and autotools build systems +- Optional libclang dependency detection for type-aware features +- `--enable-static-tools` configuration option +- `examples/` directory with library usage examples +- Comprehensive allocator and parser tests + +### Changed + +- **fy-tool now uses streaming mode by default** for dump operations (up to 24x faster on large files: 105MB in 0.776s vs 18.3s; document mode available via `--no-streaming`) +- Emission performance improved up to 2x through token analysis optimization +- UTF-8 handling significantly faster +- Plain scalar parsing and fetch reworked for better performance +- fy-tool no longer wraps output when stdout is not a TTY +- fy-tool now properly calls `fy_shutdown()` at exit for clean valgrind runs +- CMake build system now at feature parity with autotools +- Stream YAML version handling now per-stream (not per-document) +- Split diagnostics into separate modules by type +- Reorganized fy-tool directory structure +- Renamed `alloca_vsprintf` to `fy_vsprintfa` +- Improved block scalar end linebreak and whitespace handling + +### Fixed + +- **#133**: Segmentation fault in `fy_reader_*` when all input processed +- **#135**: Use-after-free when document acceleration fails +- **#134**: Double-free in `fy_node_setup_path_expr_data` +- **#132**: Use-after-free after document resolution +- **#123**: Crash when assertions compiled out in release mode +- **#122**: Crash when addressing expression as node incorrectly +- **#120**: Crash on error during walk evaluation +- **#119**: Segmentation fault on garbage input in `evaluate_new()` +- **#118**: Segmentation fault when block scalar indicators last in input +- **#115**: Assertion preventing node deletion via `fy_document_insert_at()` +- **#107**: Crash when comparing mappings with NULL values +- **#102**: Portability crash on NEC Aurora (enums in bitfields) +- Infinite loop in scan directive for malformed UTF-8 input +- Use-after-free in auto allocator (sub-allocator freeing order) +- Use-after-free in anchor setup before resolution +- Composer halt teardown event pumping +- Document state lifecycle in presence of errors +- Parser reset queued inputs initialization +- Blank/whitespace-only scalar creation via emit event API +- Zero-length scalars as plain mapping keys +- Prefix-only tags passing scan validation +- Plain scalars with linebreaks creation +- Pretty mode document end marker forcing +- MacOS `LIST_HEAD` warning and linker options +- Numerous compiler warnings at high optimization levels +- Testsuite failures with latest jq versions + +### Performance + +- fy-tool: Up to 24x faster on large files (streaming mode by default) +- Emission: ~2x faster (optimized plain scalar token analysis) +- Parsing: Faster plain scalar fetch with inline specialization +- UTF-8: Significantly improved validation and processing speed + +### Internal + +- New atomic helpers (fy-atomics) based on C atomics with fallbacks +- fy-shutdown mechanism for library cleanup +- FY_IMPOSSIBLE_ABORT() macro for impossible conditions +- Variable size encoding header (fy-vlsize) +- CPP varargs macros for recursive expansion +- iovec copy/from/xxhash methods +- FY_LAMBDA define for lambda support detection +- Enhanced blob infrastructure +- Improved utility functions (overflow checks, alloca sprintf, constructors) + +### API Compatibility + +- **Minor breakage**: Added `minimum_bucket_occupancy` to dedup allocator config + (no known external users affected) +- **Behavioral change**: fy-tool now defaults to streaming mode instead of document + mode (24x faster, but loses forward anchor references and strict duplicate key + checking; use `--no-streaming` for old behavior) +- All other changes are backwards compatible additions + +### Contributors + +Pantelis Antoniou, Alessandro Astone, Kevin Wooten, Henry Qin, Roland Wirth, +Yurii Rashkovskii, Alexandre Detiste, Benjamin Rodenberg, +Jose Luis Blanco-Claraco, Andrey Somov, Orange_233, Martin Diehl + +### Statistics + +- 171 commits since v0.9 +- 12 bug fix issues closed +- 12 contributors + +## [0.9] - 2023-09-28 + +Initial public release with comprehensive YAML 1.2 support. + +[0.9.3]: https://github.com/pantoniou/libfyaml/compare/v0.9.2...v0.9.3 +[0.9.2]: https://github.com/pantoniou/libfyaml/compare/v0.9.1...v0.9.2 +[0.9.1]: https://github.com/pantoniou/libfyaml/compare/v0.9...v0.9.1 +[0.9]: https://github.com/pantoniou/libfyaml/releases/tag/v0.9 diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..135e3d2 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,1578 @@ +cmake_minimum_required(VERSION 3.10) + +if (POLICY CMP0048) + cmake_policy(SET CMP0048 NEW) # Allow project(xxx VERSION a.b.c) +endif() + +if (POLICY CMP0074) + cmake_policy(SET CMP0074 NEW) # find_package uses _ROOT variables +endif() + +if (POLICY CMP0110) + cmake_policy(SET CMP0110 NEW) # add_test() supports arbitrary characters in test names +endif() + +# Get version from git-version-gen +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.tarball-version") + file(READ "${CMAKE_CURRENT_SOURCE_DIR}/.tarball-version" PROJECT_VERSION) + string(STRIP "${PROJECT_VERSION}" PROJECT_VERSION) +else() + execute_process( + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/build-aux/git-version-gen" "${CMAKE_CURRENT_SOURCE_DIR}/.tarball-version" + OUTPUT_VARIABLE PROJECT_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +endif() + +# Parse version components +string(REGEX MATCH "^([0-9]+)(\\.([0-9]+)(\\.([0-9]+))?)?(-(.*))?$" _ "${PROJECT_VERSION}") +set(VERSION_MAJOR "${CMAKE_MATCH_1}") +if(CMAKE_MATCH_3) + set(VERSION_MINOR "${CMAKE_MATCH_3}") +else() + set(VERSION_MINOR "0") +endif() +if(CMAKE_MATCH_5) + set(VERSION_PATCH "${CMAKE_MATCH_5}") +else() + set(VERSION_PATCH "0") +endif() +if(CMAKE_MATCH_7) + set(VERSION_EXTRA "${CMAKE_MATCH_7}") +else() + set(VERSION_EXTRA "") +endif() + +project(libfyaml LANGUAGES C ASM VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}) + +# Must use GNUInstallDirs to install libraries into correct locations on all platforms +include(GNUInstallDirs) +include(CheckIncludeFile) +include(CheckFunctionExists) +include(CheckCSourceCompiles) +include(CheckCCompilerFlag) +include(CheckLibraryExists) +include(CheckSymbolExists) +include(CMakeDependentOption) +include(CTest) +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/CheckLibClang.cmake) + +# does this CMake version understand generator expressions? +if(${CMAKE_VERSION} VERSION_LESS "3.27") + set(HAVE_CMAKE_GEXPR 0) +else() + set(HAVE_CMAKE_GEXPR 1) +endif() + +# Macro to add libclang support to a target +macro(add_libclang_to_target target_name link_scope) + if(HAVE_LIBCLANG) + # Split LIBCLANG_CFLAGS into list and add as compile options + string(REPLACE " " ";" LIBCLANG_CFLAGS_LIST "${LIBCLANG_CFLAGS}") + target_compile_options(${target_name} PRIVATE ${LIBCLANG_CFLAGS_LIST}) + + # Split LIBCLANG_LDFLAGS into list and add as link directories/options + # LIBCLANG_LDFLAGS typically contains -L/path/to/lib and -Wl,flags + # Extract only the -L paths + string(REGEX MATCHALL "-L([^ ]+)" LIBCLANG_L_FLAGS "${LIBCLANG_LDFLAGS}") + set(LIBCLANG_LINK_DIRS_LOCAL "") + foreach(flag ${LIBCLANG_L_FLAGS}) + string(REGEX REPLACE "^-L" "" dir "${flag}") + list(APPEND LIBCLANG_LINK_DIRS_LOCAL "${dir}") + endforeach() + if(LIBCLANG_LINK_DIRS_LOCAL) + target_link_directories(${target_name} ${link_scope} ${LIBCLANG_LINK_DIRS_LOCAL}) + endif() + + # Add libclang libraries + target_link_libraries(${target_name} ${link_scope} ${LIBCLANG_LIBS}) + endif() +endmacro() + +# Options +option(BUILD_SHARED_LIBS "Build shared libraries" ON) +option(ENABLE_PORTABLE_TARGET "Enable portable mode (disable per-target optimizations)" OFF) +option(ENABLE_STATIC_TOOLS "Tools will be compiled as static executables" OFF) +option(ENABLE_ASAN "Enable ASAN support" OFF) +option(ENABLE_NETWORK "Enable tests requiring network access" ON) +option(ENABLE_DEVMODE "Enable development mode only debugging" OFF) +option(BUILD_TESTING "Build tests" ON) +option(ENABLE_LIBCLANG "Enable libclang support for reflection" ON) + +# Path to LLVM installation (optional, overrides automatic detection) +set(LLVM_ROOT "" CACHE PATH "Path to LLVM installation directory (e.g., /usr/lib/llvm-15)") + +# Read libtool version +file(READ "${CMAKE_CURRENT_SOURCE_DIR}/.libtool-version" LIBTOOL_VERSION) +string(STRIP "${LIBTOOL_VERSION}" LIBTOOL_VERSION) + +# Parse libtool version components (format is current:revision:age) +string(REGEX REPLACE "^([0-9]+):.*" "\\1" LIBTOOL_CURRENT "${LIBTOOL_VERSION}") +string(REGEX REPLACE "^[0-9]+:([0-9]+):.*" "\\1" LIBTOOL_REVISION "${LIBTOOL_VERSION}") +string(REGEX REPLACE "^[0-9]+:[0-9]+:([0-9]+)" "\\1" LIBTOOL_AGE "${LIBTOOL_VERSION}") + +# Calculate SO version (current - age) +math(EXPR SO_VERSION "${LIBTOOL_CURRENT} - ${LIBTOOL_AGE}") +set(SO_VERSION_FULL "${SO_VERSION}.${LIBTOOL_AGE}.${LIBTOOL_REVISION}") + +# Platform detection +if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + set(TARGET_CPU_X86_64 TRUE) + set(TARGET_CPU_ANY_X86 TRUE) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "i.86|x86") + set(TARGET_CPU_X86 TRUE) + set(TARGET_CPU_ANY_X86 TRUE) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(TARGET_CPU_ARM64 TRUE) + set(TARGET_CPU_ANY_ARM TRUE) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm") + set(TARGET_CPU_ARM TRUE) + set(TARGET_CPU_ANY_ARM TRUE) +endif() + +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILING 1) + message(STATUS "Cross-compiling detected:") + message(STATUS " Build system: ${CMAKE_HOST_SYSTEM_NAME} ${CMAKE_HOST_SYSTEM_PROCESSOR}") + message(STATUS " Target system: ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}") +else() + set(CROSS_COMPILING 0) +endif() + +# Check for required dependencies +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_package(Threads REQUIRED) +if(NOT CMAKE_USE_PTHREADS_INIT) + message(FATAL_ERROR "Missing required pthread support") +endif() + +# Check for headers and functions +check_include_file("alloca.h" HAVE_ALLOCA_H) +check_include_file("byteswap.h" HAVE_BYTESWAP_H) +check_function_exists(__builtin_bswap16 HAVE___BUILTIN_BSWAP16) +check_function_exists(__builtin_bswap32 HAVE___BUILTIN_BSWAP32) +check_function_exists(__builtin_bswap64 HAVE___BUILTIN_BSWAP64) +check_function_exists(qsort_r HAVE_QSORT_R) +check_function_exists(mremap HAVE_MREMAP) + +# Check for environment variables +check_c_source_compiles(" + extern char **environ; + int main() { return 0; } +" HAVE_DECL_ENVIRON) + +# Check for optional dependencies + +# libyaml detection - try CMake config first, then pkg-config +set(HAVE_LIBYAML 0) +find_package(yaml QUIET CONFIG) +if(yaml_FOUND) + set(HAVE_LIBYAML 1) + # CMake config found, use imported target + set(LIBYAML_LIBRARIES yaml) + message(STATUS "Found libyaml via CMake config") +else() + find_package(PkgConfig) + if(PKG_CONFIG_FOUND) + pkg_check_modules(LIBYAML yaml-0.1) + if(LIBYAML_FOUND) + set(HAVE_LIBYAML 1) + message(STATUS "Found libyaml via pkg-config") + else() + message(WARNING "failed to find libyaml; compatibility disabled") + endif() + endif() +endif() + +# check framework detection - try CMake config first, then pkg-config +set(HAVE_CHECK 0) +set(HAVE_COMPATIBLE_CHECK 0) +if(BUILD_TESTING) + find_package(check QUIET CONFIG) + if(check_FOUND) + set(HAVE_CHECK 1) + set(CHECK_LIBRARIES check) + get_target_property(CHECK_INCLUDE_DIRS check INTERFACE_INCLUDE_DIRECTORIES) + # Get library location to extract directory + get_target_property(CHECK_LIB_LOCATION check IMPORTED_LOCATION) + if(NOT CHECK_LIB_LOCATION) + get_target_property(CHECK_LIB_LOCATION check LOCATION) + endif() + if(CHECK_LIB_LOCATION) + get_filename_component(CHECK_LIBRARY_DIRS "${CHECK_LIB_LOCATION}" DIRECTORY) + endif() + message(STATUS "Found check framework via CMake config") + + # Check if libcheck has srunner_set_tap (jessie has outdated libcheck) + set(_TEMP_CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + set(_TEMP_CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES}") + # For CMake config targets, use the target directly + set(CMAKE_REQUIRED_LIBRARIES check) + set(CMAKE_REQUIRED_INCLUDES ${CHECK_INCLUDE_DIRS}) + + check_symbol_exists(srunner_set_tap "check.h" HAVE_SRUNNER_SET_TAP) + + set(CMAKE_REQUIRED_LIBRARIES "${_TEMP_CMAKE_REQUIRED_LIBRARIES}") + set(CMAKE_REQUIRED_INCLUDES "${_TEMP_CMAKE_REQUIRED_INCLUDES}") + unset(_TEMP_CMAKE_REQUIRED_LIBRARIES) + unset(_TEMP_CMAKE_REQUIRED_INCLUDES) + + if(HAVE_SRUNNER_SET_TAP) + set(HAVE_COMPATIBLE_CHECK 1) + endif() + else() + # Fall back to pkg-config + find_package(PkgConfig) + if(PKG_CONFIG_FOUND) + pkg_check_modules(CHECK check) + if(CHECK_FOUND) + set(HAVE_CHECK 1) + message(STATUS "Found check framework via pkg-config") + + # Check if libcheck has srunner_set_tap (jessie has outdated libcheck) + set(_TEMP_CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + set(_TEMP_CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES}") + set(_TEMP_CMAKE_REQUIRED_LINK_OPTIONS "${CMAKE_REQUIRED_LINK_OPTIONS}") + set(CMAKE_REQUIRED_LIBRARIES ${CHECK_LIBRARIES}) + set(CMAKE_REQUIRED_INCLUDES ${CHECK_INCLUDE_DIRS}) + # Add library directories to linker flags + foreach(libdir ${CHECK_LIBRARY_DIRS}) + list(APPEND CMAKE_REQUIRED_LINK_OPTIONS "-L${libdir}") + endforeach() + + check_symbol_exists(srunner_set_tap "check.h" HAVE_SRUNNER_SET_TAP) + + set(CMAKE_REQUIRED_LIBRARIES "${_TEMP_CMAKE_REQUIRED_LIBRARIES}") + set(CMAKE_REQUIRED_INCLUDES "${_TEMP_CMAKE_REQUIRED_INCLUDES}") + set(CMAKE_REQUIRED_LINK_OPTIONS "${_TEMP_CMAKE_REQUIRED_LINK_OPTIONS}") + unset(_TEMP_CMAKE_REQUIRED_LIBRARIES) + unset(_TEMP_CMAKE_REQUIRED_INCLUDES) + unset(_TEMP_CMAKE_REQUIRED_LINK_OPTIONS) + + if(HAVE_SRUNNER_SET_TAP) + set(HAVE_COMPATIBLE_CHECK 1) + endif() + endif() + endif() + endif() +endif() + +# Check for libclang (for reflection support) +set(HAVE_LIBCLANG 0) +set(LIBCLANG_CFLAGS "") +set(LIBCLANG_LDFLAGS "") +set(LIBCLANG_LIBS "") +set(LIBCLANG_FOUND FALSE) +set(LIBCLANG_DETECTION_METHOD "") + +if(NOT HAVE_CMAKE_GEXPR) + message(STATUS "libclang support disabled (CMake version is too old (<=3.27)") + set(ENABLE_LIBCLANG OFF CACHE BOOL "Disable my feature" FORCE) +endif() + +if(NOT ENABLE_LIBCLANG) + message(STATUS "libclang support disabled (ENABLE_LIBCLANG=OFF)") +else() + + # If user specified LLVM_ROOT, add it to CMAKE_PREFIX_PATH for this search + if(LLVM_ROOT) + message(STATUS "Using user-specified LLVM_ROOT: ${LLVM_ROOT}") + list(PREPEND CMAKE_PREFIX_PATH "${LLVM_ROOT}") + endif() + + # Try CMake config mode first + find_package(LLVM QUIET CONFIG) + + # If LLVM was found, help CMake find Clang in the same installation + if(LLVM_FOUND) + # Clang's config is typically in ${LLVM_INSTALL_PREFIX}/lib/cmake/clang + set(Clang_DIR "${LLVM_INSTALL_PREFIX}/lib/cmake/clang") + endif() + + find_package(Clang QUIET CONFIG) + + if(LLVM_FOUND AND Clang_FOUND) + + message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION} via CMake config") + message(STATUS " LLVM_INSTALL_PREFIX: ${LLVM_INSTALL_PREFIX}") + message(STATUS " LLVM_INCLUDE_DIRS: ${LLVM_INCLUDE_DIRS}") + + # Use LLVM's CMake variables directly + set(LIBCLANG_CFLAGS_LIST ${LLVM_DEFINITIONS}) + list(APPEND LIBCLANG_CFLAGS_LIST "-I${LLVM_INCLUDE_DIRS}") + list(APPEND LIBCLANG_CFLAGS_LIST "-I${CLANG_INCLUDE_DIRS}") + + # Set up library directories + set(LIBCLANG_LINK_DIRS ${LLVM_LIBRARY_DIRS}) + + # Link against libclang (C API) and LLVM core libs + # Need to link against LLVM core for LLVMShutdown() and other LLVM APIs + if(TARGET libclang) + set(LIBCLANG_LIBS_LIST libclang LLVM) + else() + set(LIBCLANG_LIBS_LIST clang LLVM) + endif() + + set(LIBCLANG_INCLUDES "${LLVM_INCLUDE_DIRS};${CLANG_INCLUDE_DIRS}") + set(LIBCLANG_FOUND TRUE) + set(LIBCLANG_DETECTION_METHOD "CMake config") + elseif(NOT CROSS_COMPILING) + # Fallback to llvm-config method (only when not cross-compiling) + message(STATUS "LLVM CMake config not found, trying llvm-config") + + # Try to find llvm-config (versions 20 down to 10) + set(LLVM_CONFIG_NAMES llvm-config) + foreach(ver RANGE 20 10 -1) + list(APPEND LLVM_CONFIG_NAMES llvm-config-${ver}) + endforeach() + + find_program(LLVM_CONFIG NAMES ${LLVM_CONFIG_NAMES}) + + if(LLVM_CONFIG) + execute_process(COMMAND ${LLVM_CONFIG} --cflags OUTPUT_VARIABLE LIBCLANG_CFLAGS_TEMP OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${LLVM_CONFIG} --ldflags OUTPUT_VARIABLE LIBCLANG_LDFLAGS_TEMP OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${LLVM_CONFIG} --libs OUTPUT_VARIABLE LLVM_LIBS_TEMP OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${LLVM_CONFIG} --system-libs OUTPUT_VARIABLE LLVM_SYSLIBS_TEMP OUTPUT_STRIP_TRAILING_WHITESPACE) + + set(LIBCLANG_CFLAGS_LIST "${LIBCLANG_CFLAGS_TEMP} ${LIBCLANG_LDFLAGS_TEMP}") + set(LIBCLANG_LIBS_LIST "-lclang;${LLVM_LIBS_TEMP};${LLVM_SYSLIBS_TEMP}") + set(LIBCLANG_CFLAGS "${LIBCLANG_CFLAGS_TEMP}") + set(LIBCLANG_LDFLAGS "${LIBCLANG_LDFLAGS_TEMP}") + set(LIBCLANG_FOUND TRUE) + set(LIBCLANG_DETECTION_METHOD "llvm-config: ${LLVM_CONFIG}") + endif() + else() + message(STATUS "Cross-compiling: LLVM CMake config not found") + message(STATUS " Set LLVM_ROOT to LLVM install directory (e.g., /usr/lib/llvm-18)") + message(STATUS " Or set CMAKE_PREFIX_PATH or LLVM_DIR to the directory containing LLVMConfig.cmake") + endif() + + # Test if libclang actually works (single test for all detection methods) + if(LIBCLANG_FOUND) + if(NOT CROSS_COMPILING) + check_libclang_works(LIBCLANG_WORKS + CFLAGS "${LIBCLANG_CFLAGS_LIST}" + INCLUDES "${LIBCLANG_INCLUDES}" + LINK_DIRS "${LIBCLANG_LINK_DIRS}" + LIBRARIES "${LIBCLANG_LIBS_LIST}" + ) + else() + # When cross-compiling, disable libclang by default to avoid linking host libraries + # User can set LLVM_ROOT, CMAKE_PREFIX_PATH, or LLVM_DIR to point to target-specific LLVM installation + message(STATUS "Cross-compiling: disabling libclang") + message(STATUS " To enable, set LLVM_ROOT or CMAKE_PREFIX_PATH to target LLVM installation") + set(LIBCLANG_WORKS FALSE) + endif() + + if(LIBCLANG_WORKS) + set(HAVE_LIBCLANG 1) + # Convert list to string for LIBCLANG_CFLAGS if needed (CMake config path) + if(LIBCLANG_DETECTION_METHOD STREQUAL "CMake config") + string(REPLACE ";" " " LIBCLANG_CFLAGS "${LIBCLANG_CFLAGS_LIST}") + endif() + set(LIBCLANG_LIBS "${LIBCLANG_LIBS_LIST}") + message(STATUS "Found libclang via ${LIBCLANG_DETECTION_METHOD}") + else() + message(STATUS "libclang not found or not working") + message(STATUS " Reflection support will be disabled") + message(STATUS " To specify libclang location: cmake -DLLVM_ROOT=/path/to/llvm") + message(STATUS " To disable this check: cmake -DENABLE_LIBCLANG=OFF") + endif() + else() + if(NOT LIBCLANG_FOUND) + message(STATUS "libclang not found") + message(STATUS " Reflection support will be disabled") + message(STATUS " To specify libclang location: cmake -DLLVM_ROOT=/path/to/llvm") + message(STATUS " To disable this check: cmake -DENABLE_LIBCLANG=OFF") + endif() + endif() + +endif() # ENABLE_LIBCLANG + +# Check for tools +find_program(GIT_EXECUTABLE git) +if(GIT_EXECUTABLE) + set(HAVE_GIT 1) +else() + set(HAVE_GIT 0) +endif() + +find_program(JQ_EXECUTABLE jq) +if(JQ_EXECUTABLE) + set(HAVE_JQ 1) +else() + set(HAVE_JQ 0) +endif() + +find_program(DOCKER_EXECUTABLE docker) +if(DOCKER_EXECUTABLE) + set(HAVE_DOCKER 1) +else() + set(HAVE_DOCKER 0) +endif() + +# Test suite URLs +if(NOT TESTSUITEURL) + set(TESTSUITEURL "https://github.com/yaml/yaml-test-suite") +endif() +if(NOT TESTSUITECHECKOUT) + set(TESTSUITECHECKOUT "6e6c296ae9c9d2d5c4134b4b64d01b29ac19ff6f") +endif() +if(NOT JSONTESTSUITEURL) + set(JSONTESTSUITEURL "https://github.com/nst/JSONTestSuite") +endif() +if(NOT JSONTESTSUITECHECKOUT) + set(JSONTESTSUITECHECKOUT "d64aefb55228d9584d3e5b2433f720ea8fd00c82") +endif() + +# SIMD capability detection +set(TARGET_HAS_SSE2 FALSE) +set(TARGET_HAS_SSE41 FALSE) +set(TARGET_HAS_AVX2 FALSE) +set(TARGET_HAS_AVX512 FALSE) +set(TARGET_HAS_NEON FALSE) + +if(TARGET_CPU_ANY_X86 AND NOT ENABLE_PORTABLE_TARGET) + check_c_source_compiles(" + #include + int main() { __m128i a = _mm_setzero_si128(); return 0; } + " COMPILER_SUPPORTS_SSE2) + if(COMPILER_SUPPORTS_SSE2) + set(TARGET_HAS_SSE2 TRUE) + endif() + + check_c_source_compiles(" + #include + int main() { __m128i a = _mm_setzero_si128(); return 0; } + " COMPILER_SUPPORTS_SSE41) + if(COMPILER_SUPPORTS_SSE41) + set(TARGET_HAS_SSE41 TRUE) + endif() + + set(CMAKE_REQUIRED_FLAGS "-mavx2") + check_c_source_compiles(" + #include + int main() { __m256i a = _mm256_setzero_si256(); return 0; } + " COMPILER_SUPPORTS_AVX2) + set(CMAKE_REQUIRED_FLAGS "") + if(COMPILER_SUPPORTS_AVX2) + set(TARGET_HAS_AVX2 TRUE) + endif() + + set(CMAKE_REQUIRED_FLAGS "-mavx512f -mavx512vl") + check_c_source_compiles(" + #include + int main() { __m512i a = _mm512_setzero_si512(); return 0; } + " COMPILER_SUPPORTS_AVX512) + set(CMAKE_REQUIRED_FLAGS "") + if(COMPILER_SUPPORTS_AVX512) + set(TARGET_HAS_AVX512 TRUE) + endif() +endif() + +if(TARGET_CPU_ANY_ARM AND NOT ENABLE_PORTABLE_TARGET) + if(TARGET_CPU_ARM64) + set(TARGET_HAS_NEON TRUE) + else() + check_c_source_compiles(" + #include + int main() { uint8x16_t a = vdupq_n_u8(0); return 0; } + " COMPILER_SUPPORTS_NEON) + if(COMPILER_SUPPORTS_NEON) + set(TARGET_HAS_NEON TRUE) + endif() + endif() +endif() + +# ASAN support +set(HAVE_ASAN 0) +if(ENABLE_ASAN) + check_c_source_compiles(" + int main() { return 0; } + " ASAN_WORKS) + if(ASAN_WORKS) + set(HAVE_ASAN 1) + endif() +endif() + +# Build common compiler flags for all targets +set(COMMON_C_FLAGS) +set(COMMON_C_DEFINITIONS _GNU_SOURCE) + +if(NOT MSVC) + list(APPEND COMMON_C_FLAGS -Wall -Wsign-compare) + + check_c_compiler_flag(-Wno-unused-function COMPILER_SUPPORTS_WNO_UNUSED_FUNCTION) + if(COMPILER_SUPPORTS_WNO_UNUSED_FUNCTION) + list(APPEND COMMON_C_FLAGS -Wno-unused-function) + endif() + + check_c_compiler_flag(-Wno-stringop-overflow COMPILER_SUPPORTS_WNO_STRINGOP_OVERFLOW) + if(COMPILER_SUPPORTS_WNO_STRINGOP_OVERFLOW) + list(APPEND COMMON_C_FLAGS -Wno-stringop-overflow) + endif() + + check_c_compiler_flag(-Wno-tautological-constant-out-of-range-compare + COMPILER_SUPPORTS_WNO_TAUTOLOGICAL_CONSTANT_OUT_OF_RANGE_COMPARE) + if(COMPILER_SUPPORTS_WNO_TAUTOLOGICAL_CONSTANT_OUT_OF_RANGE_COMPARE) + list(APPEND COMMON_C_FLAGS -Wno-tautological-constant-out-of-range-compare) + endif() + + set(USE_STD 0) + + if(NOT USE_STD) + check_c_compiler_flag(-std=c2x COMPILER_SUPPORTS_GNU2X) + if(COMPILER_SUPPORTS_GNU2X) + list(APPEND COMMON_C_FLAGS -std=gnu2x) + set(USE_STD 1) + endif() + endif() + + if(NOT USE_STD) + check_c_compiler_flag(-std=c2x COMPILER_SUPPORTS_C2X) + if(COMPILER_SUPPORTS_C2X) + list(APPEND COMMON_C_FLAGS -std=c2x) + set(USE_STD 1) + endif() + endif() + + # GCC heap trampolines support (for nested functions) + # Heap trampolines are more secure than executable stack trampolines + # Available in GCC 14+ with -ftrampoline-impl=heap + # Supported targets: x86_64/i386/aarch64 Linux, x86_64/i386 Darwin + set(HAVE_HEAP_TRAMPOLINES 0) + if(CMAKE_C_COMPILER_ID STREQUAL "GNU") + check_c_compiler_flag(-ftrampoline-impl=heap COMPILER_SUPPORTS_HEAP_TRAMPOLINES) + if(COMPILER_SUPPORTS_HEAP_TRAMPOLINES) + set(HAVE_HEAP_TRAMPOLINES 1) + list(APPEND COMMON_C_FLAGS -ftrampoline-impl=heap) + message(STATUS "GCC heap trampolines enabled (-ftrampoline-impl=heap)") + else() + message(STATUS "GCC heap trampolines not available (GCC 14+ required), using stack trampolines") + endif() + endif() + +endif() + +# the options for the libs +set(LIB_OPTS "") +if(BUILD_SHARED_LIBS) + list(APPEND LIB_OPTS "-fPIC") +endif() + +# Network tests +if(ENABLE_NETWORK) + set(HAVE_NETWORK 1) +else() + set(HAVE_NETWORK 0) +endif() + +# Dev mode +if(ENABLE_DEVMODE) + set(HAVE_DEVMODE 1) +else() + set(HAVE_DEVMODE 0) +endif() + +# Static support +if(NOT BUILD_SHARED_LIBS) + set(HAVE_STATIC 1) +else() + set(HAVE_STATIC 0) +endif() + +# Library sources +set(LIBHDRS + src/lib/fy-accel.h + src/lib/fy-atom.h + src/lib/fy-composer.h + src/lib/fy-diag.h + src/lib/fy-doc.h + src/lib/fy-docbuilder.h + src/lib/fy-docstate.h + src/lib/fy-dump.h + src/lib/fy-emit.h + src/lib/fy-emit-accum.h + src/lib/fy-event.h + src/lib/fy-input.h + src/lib/fy-parse.h + src/lib/fy-path.h + src/lib/fy-token.h + src/lib/fy-types.h + src/lib/fy-walk.h +) + +set(UTILHDRS + src/util/fy-blob.h + src/util/fy-ctype.h + src/util/fy-endian.h + src/util/fy-id.h + src/util/fy-list.h + src/util/fy-typelist.h + src/util/fy-utf8.h + src/util/fy-utils.h + src/util/fy-align.h + src/util/fy-bit64.h + src/util/fy-vlsize.h +) + +set(LIBSRCS + src/lib/fy-accel.c + src/lib/fy-atom.c + src/lib/fy-composer.c + src/lib/fy-diag.c + src/lib/fy-doc.c + src/lib/fy-docbuilder.c + src/lib/fy-docstate.c + src/lib/fy-dump.c + src/lib/fy-emit.c + src/lib/fy-event.c + src/lib/fy-input.c + src/lib/fy-parse.c + src/lib/fy-path.c + src/lib/fy-token.c + src/lib/fy-types.c + src/lib/fy-walk.c + src/util/fy-blob.c + src/util/fy-ctype.c + src/util/fy-utf8.c + src/util/fy-utils.c + src/xxhash/xxhash.c + src/thread/fy-thread.c + src/allocator/fy-allocator.c + src/allocator/fy-allocator-linear.c + src/allocator/fy-allocator-malloc.c + src/allocator/fy-allocator-mremap.c + src/allocator/fy-allocator-dedup.c + src/allocator/fy-allocator-auto.c + src/blake3/blake3_host_state.c + src/blake3/blake3_backend.c + src/blake3/blake3_be_cpusimd.c + src/blake3/fy-blake3.c + src/lib/fy-composer-diag.c + src/lib/fy-doc-diag.c + src/lib/fy-docbuilder-diag.c + src/lib/fy-input-diag.c + src/lib/fy-parse-diag.c +) + +set(LIBHDRSPUB + include/libfyaml.h +) + +# BLAKE3 portable library +add_library(b3portable OBJECT + src/blake3/blake3_portable.c + src/blake3/blake3.c +) +target_include_directories(b3portable PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 +) +target_compile_definitions(b3portable PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=portable + SIMD_DEGREE=1 +) +target_compile_options(b3portable PRIVATE ${COMMON_C_FLAGS} ${LIB_OPTS}) + +# BLAKE3 SIMD libraries +set(BLAKE3_TARGETS b3portable) + +if(TARGET_HAS_SSE2) + add_library(b3sse2 OBJECT + src/blake3/blake3_sse2.c + src/blake3/blake3_sse2_x86-64_unix.S + src/blake3/blake3.c + ) + target_include_directories(b3sse2 PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ) + target_compile_definitions(b3sse2 PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=sse2 + SIMD_DEGREE=4 + ) + target_compile_options(b3sse2 PRIVATE ${COMMON_C_FLAGS} -msse2 ${LIB_OPTS}) + list(APPEND BLAKE3_TARGETS b3sse2) +endif() + +if(TARGET_HAS_SSE41) + add_library(b3sse41 OBJECT + src/blake3/blake3_sse41.c + src/blake3/blake3_sse41_x86-64_unix.S + src/blake3/blake3.c + ) + target_include_directories(b3sse41 PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ) + target_compile_definitions(b3sse41 PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=sse41 + SIMD_DEGREE=4 + ) + target_compile_options(b3sse41 PRIVATE ${COMMON_C_FLAGS} -msse4.1 ${LIB_OPTS}) + list(APPEND BLAKE3_TARGETS b3sse41) +endif() + +if(TARGET_HAS_AVX2) + add_library(b3avx2 OBJECT + src/blake3/blake3_avx2.c + src/blake3/blake3_avx2_x86-64_unix.S + src/blake3/blake3.c + ) + target_include_directories(b3avx2 PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ) + target_compile_definitions(b3avx2 PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=avx2 + SIMD_DEGREE=8 + ) + target_compile_options(b3avx2 PRIVATE ${COMMON_C_FLAGS} -mavx2 ${LIB_OPTS}) + list(APPEND BLAKE3_TARGETS b3avx2) +endif() + +if(TARGET_HAS_AVX512) + add_library(b3avx512 OBJECT + src/blake3/blake3_avx512.c + src/blake3/blake3_avx512_x86-64_unix.S + src/blake3/blake3.c + ) + target_include_directories(b3avx512 PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ) + target_compile_definitions(b3avx512 PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=avx512 + SIMD_DEGREE=16 + ) + target_compile_options(b3avx512 PRIVATE ${COMMON_C_FLAGS} -mavx512f -mavx512vl ${LIB_OPTS}) + list(APPEND BLAKE3_TARGETS b3avx512) +endif() + +if(TARGET_HAS_NEON) + add_library(b3neon OBJECT + src/blake3/blake3_neon.c + src/blake3/blake3.c + ) + target_include_directories(b3neon PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ) + target_compile_definitions(b3neon PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=neon + SIMD_DEGREE=4 + ) + target_compile_options(b3neon PRIVATE ${COMMON_C_FLAGS} ${LIB_OPTS}) + if(TARGET_CPU_ARM) + target_compile_options(b3neon PRIVATE -mfpu=neon) + endif() + list(APPEND BLAKE3_TARGETS b3neon) +endif() + +# Main library +add_library(fyaml + ${LIBHDRS} + ${UTILHDRS} + ${LIBSRCS} + ${LIBHDRSPUB} +) + +# Link BLAKE3 object libraries +foreach(blake3_target ${BLAKE3_TARGETS}) + target_sources(fyaml PRIVATE $) +endforeach() + +add_library(libfyaml::libfyaml ALIAS fyaml) + +# Always build a static library for internal tools (they need access to internal APIs) +# This is separate from BUILD_SHARED_LIBS and matches autotools behavior +add_library(fyaml_static STATIC + ${LIBHDRS} + ${UTILHDRS} + ${LIBSRCS} + ${LIBHDRSPUB} +) + +# Link BLAKE3 object libraries to static version +foreach(blake3_target ${BLAKE3_TARGETS}) + target_sources(fyaml_static PRIVATE $) +endforeach() + +set_target_properties(fyaml_static PROPERTIES + OUTPUT_NAME fyaml_static + POSITION_INDEPENDENT_CODE OFF + FOLDER "Libraries" +) + +# Configure static library with same settings as main library +target_include_directories(fyaml_static + PUBLIC + $ + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/xxhash + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ${CMAKE_CURRENT_BINARY_DIR} +) + +target_link_libraries(fyaml_static + PUBLIC + Threads::Threads +) + +# Add libclang support if available +add_libclang_to_target(fyaml_static INTERFACE) + +# check if blocks are available, and if so link them +include(${CMAKE_SOURCE_DIR}/cmake/CheckClangBlocks.cmake) + +# Apply flags if available +if (HAVE_CLANG_BLOCKS) + add_compile_options(-fblocks) + list(APPEND COMMON_C_FLAGS -fblocks) + + if (CLANG_BLOCKS_LIB) + link_libraries(${CLANG_BLOCKS_LIB}) + endif() +endif() + +target_compile_definitions(fyaml_static PRIVATE + HAVE_CONFIG_H + VERSION="${PROJECT_VERSION}" + $<$>:NDEBUG> +) + +target_compile_definitions(fyaml_static PRIVATE ${COMMON_C_DEFINITIONS}) +target_compile_options(fyaml_static PRIVATE ${COMMON_C_FLAGS}) + +if(NOT MSVC) + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_options(fyaml_static PRIVATE -O2) + endif() + + if(HAVE_ASAN) + target_compile_options(fyaml_static PRIVATE -fsanitize=address -fno-omit-frame-pointer) + # Don't add link options here - executables will link dynamically against libasan + endif() +endif() + +set_target_properties(fyaml PROPERTIES + OUTPUT_NAME fyaml + VERSION ${PROJECT_VERSION} + SOVERSION ${VERSION_MAJOR} + FOLDER "Libraries" + PUBLIC_HEADER "${LIBHDRSPUB}" +) + +target_include_directories(fyaml + PUBLIC + $ + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/xxhash + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ${CMAKE_CURRENT_SOURCE_DIR}/src/generic + ${CMAKE_CURRENT_SOURCE_DIR}/src/reflection +) + +target_link_libraries(fyaml + PUBLIC + Threads::Threads +) + +# Add libclang support if available +add_libclang_to_target(fyaml PRIVATE) + +# Generate config.h +configure_file( + ${CMAKE_SOURCE_DIR}/cmake/config.h.in + ${CMAKE_BINARY_DIR}/config.h +) + +# Generate config.h +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/config.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/config.h" +) +target_include_directories(fyaml PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + +target_compile_definitions(fyaml PRIVATE + HAVE_CONFIG_H + VERSION="${PROJECT_VERSION}" + $<$>:NDEBUG> +) + +target_compile_definitions(fyaml PRIVATE ${COMMON_C_DEFINITIONS}) +target_compile_options(fyaml PRIVATE ${COMMON_C_FLAGS}) + +# Shared options between GCC and CLANG +if(NOT MSVC) + target_compile_options(fyaml PRIVATE ${LIB_OPTS}) + target_compile_options(fyaml PRIVATE -fvisibility=hidden) + + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_options(fyaml PRIVATE -O2) + endif() + + if(HAVE_ASAN) + target_compile_options(fyaml PRIVATE -fsanitize=address -fno-omit-frame-pointer) + # Use INTERFACE to propagate link flags to executables linking against this library + target_link_options(fyaml INTERFACE -fsanitize=address) + endif() +endif() + +# Build fy-tool +add_executable(fy-tool + src/tool/fy-tool.c + src/tool/fy-tool-util.c + src/tool/fy-tool-dump.c + src/valgrind/fy-valgrind.h +) + +target_include_directories(fy-tool PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind +) + +target_link_libraries(fy-tool PRIVATE fyaml) + +if(ENABLE_STATIC_TOOLS AND NOT MSVC) + target_link_options(fy-tool PRIVATE -static) +endif() + +# Build internal tools - these are never installed, built for internal testing +# These tools use private/internal APIs and embed libfyaml code via fyaml_static +# System libraries are dynamically linked (works with or without ASAN) +# Skip when cross-compiling since these tools cannot be executed on build host + +if(NOT CROSS_COMPILING) + +# libfyaml-parser requires libyaml +if(HAVE_LIBYAML) + add_executable(libfyaml-parser + src/internal/libfyaml-parser.c + src/valgrind/fy-valgrind.h + ) + target_compile_definitions(libfyaml-parser PRIVATE ${COMMON_C_DEFINITIONS}) + target_compile_options(libfyaml-parser PRIVATE ${COMMON_C_FLAGS}) + + target_include_directories(libfyaml-parser PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/xxhash + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_BINARY_DIR} + ${LIBYAML_INCLUDE_DIRS} + ) + target_link_directories(libfyaml-parser PRIVATE ${LIBYAML_LIBRARY_DIRS}) + if(NOT MSVC) + # Embed libfyaml code statically, allow dynamic linking for system libraries + if(APPLE) + target_link_libraries(libfyaml-parser PRIVATE + -Wl,-force_load,$ + ${LIBYAML_LIBRARIES} + $<$:${LIBCLANG_LIBS}> + ) + else() + target_link_libraries(libfyaml-parser PRIVATE + -Wl,--whole-archive fyaml_static -Wl,--no-whole-archive + ${LIBYAML_LIBRARIES} + $<$:${LIBCLANG_LIBS}> + ) + endif() + if(HAVE_ASAN) + target_link_options(libfyaml-parser PRIVATE -fsanitize=address) + endif() + else() + target_link_libraries(libfyaml-parser PRIVATE fyaml_static ${LIBYAML_LIBRARIES}) + endif() + if(HAVE_LIBCLANG) + target_link_directories(libfyaml-parser PRIVATE ${LIBCLANG_LINK_DIRS}) + endif() + add_dependencies(libfyaml-parser fyaml_static) +endif() + +add_executable(fy-thread + src/internal/fy-thread.c + src/valgrind/fy-valgrind.h +) +target_include_directories(fy-thread PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_BINARY_DIR} +) +if(NOT MSVC) + if(APPLE) + target_link_libraries(fy-thread PRIVATE + -Wl,-force_load,$ + $<$:${LIBCLANG_LIBS}> + ) + else() + target_link_libraries(fy-thread PRIVATE + -Wl,--whole-archive fyaml_static -Wl,--no-whole-archive + $<$:${LIBCLANG_LIBS}> + ) + endif() + if(HAVE_ASAN) + target_link_options(fy-thread PRIVATE -fsanitize=address) + endif() +else() + target_link_libraries(fy-thread PRIVATE fyaml_static) +endif() +if(HAVE_LIBCLANG) + target_link_directories(fy-thread PRIVATE ${LIBCLANG_LINK_DIRS}) +endif() +add_dependencies(fy-thread fyaml_static) + +add_executable(fy-b3sum + src/internal/fy-b3sum.c + src/valgrind/fy-valgrind.h +) +target_include_directories(fy-b3sum PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_BINARY_DIR} +) +if(NOT MSVC) + if(APPLE) + target_link_libraries(fy-b3sum PRIVATE + -Wl,-force_load,$ + $<$:${LIBCLANG_LIBS}> + ) + else() + target_link_libraries(fy-b3sum PRIVATE + -Wl,--whole-archive fyaml_static -Wl,--no-whole-archive + $<$:${LIBCLANG_LIBS}> + ) + endif() + if(HAVE_ASAN) + target_link_options(fy-b3sum PRIVATE -fsanitize=address) + endif() +else() + target_link_libraries(fy-b3sum PRIVATE fyaml_static) +endif() +if(HAVE_LIBCLANG) + target_link_directories(fy-b3sum PRIVATE ${LIBCLANG_LINK_DIRS}) +endif() +add_dependencies(fy-b3sum fyaml_static) + +add_executable(fy-allocators + src/internal/fy-allocators.c + src/valgrind/fy-valgrind.h +) +target_include_directories(fy-allocators PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/xxhash + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_BINARY_DIR} +) +if(NOT MSVC) + if(APPLE) + target_link_libraries(fy-allocators PRIVATE + -Wl,-force_load,$ + $<$:${LIBCLANG_LIBS}> + ) + else() + target_link_libraries(fy-allocators PRIVATE + -Wl,--whole-archive fyaml_static -Wl,--no-whole-archive + $<$:${LIBCLANG_LIBS}> + ) + endif() + if(HAVE_ASAN) + target_link_options(fy-allocators PRIVATE -fsanitize=address) + endif() +else() + target_link_libraries(fy-allocators PRIVATE fyaml_static) +endif() +if(HAVE_LIBCLANG) + target_link_directories(fy-allocators PRIVATE ${LIBCLANG_LINK_DIRS}) +endif() +add_dependencies(fy-allocators fyaml_static) + +endif() # NOT CROSS_COMPILING (internal tools) + +# Testing +if(BUILD_TESTING AND NOT CROSS_COMPILING) + # Include TAP subtest registration functions + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/add-tap-subtests.cmake) + + # Create test directory + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test) + + # Create src directory for compatibility with test scripts + # Tests expect tools at build/src/ + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/src) + add_custom_command(TARGET fy-tool POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_BINARY_DIR}/fy-tool + ${CMAKE_CURRENT_BINARY_DIR}/src/fy-tool + COMMENT "Creating symlink for test compatibility" + ) + + # Create symlinks for internal tools + add_custom_command(TARGET fy-thread POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_BINARY_DIR}/fy-thread + ${CMAKE_CURRENT_BINARY_DIR}/src/fy-thread + ) + add_custom_command(TARGET fy-b3sum POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_BINARY_DIR}/fy-b3sum + ${CMAKE_CURRENT_BINARY_DIR}/src/fy-b3sum + ) + add_custom_command(TARGET fy-allocators POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_BINARY_DIR}/fy-allocators + ${CMAKE_CURRENT_BINARY_DIR}/src/fy-allocators + ) + if(HAVE_LIBYAML) + add_custom_command(TARGET libfyaml-parser POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_BINARY_DIR}/libfyaml-parser + ${CMAKE_CURRENT_BINARY_DIR}/src/libfyaml-parser + ) + endif() + + # Build libfyaml-test if check is available + if(HAVE_COMPATIBLE_CHECK) + + add_executable(libfyaml-test + test/libfyaml-test.c + test/libfyaml-test-core.c + test/libfyaml-test-meta.c + test/libfyaml-test-emit.c + test/libfyaml-test-allocator.c + test/libfyaml-test-private.c + test/libfyaml-test-private-id.c + test/libfyaml-test-parser.c + test/libfyaml-test-thread.c + ) + + target_compile_options(libfyaml-test PRIVATE ${COMMON_C_FLAGS}) + target_compile_definitions(libfyaml-test PRIVATE ${COMMON_C_DEFINITIONS}) + + target_include_directories(libfyaml-test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_BINARY_DIR} + ${CHECK_INCLUDE_DIRS} + ) + + # Link against static library to access private symbols + # Use same pattern as internal tools + if(NOT MSVC) + if(APPLE) + target_link_libraries(libfyaml-test PRIVATE + -Wl,-force_load,$ + ${CHECK_LIBRARIES} + $<$:${LIBCLANG_LIBS}> + ) + else() + target_link_libraries(libfyaml-test PRIVATE + -Wl,--whole-archive fyaml_static -Wl,--no-whole-archive + ${CHECK_LIBRARIES} + $<$:${LIBCLANG_LIBS}> + ) + endif() + if(HAVE_ASAN) + target_link_options(libfyaml-test PRIVATE -fsanitize=address) + endif() + else() + target_link_libraries(libfyaml-test PRIVATE fyaml_static ${CHECK_LIBRARIES}) + endif() + if(CHECK_LIBRARY_DIRS) + target_link_directories(libfyaml-test PRIVATE ${CHECK_LIBRARY_DIRS}) + endif() + if(HAVE_LIBCLANG) + target_link_directories(libfyaml-test PRIVATE ${LIBCLANG_LINK_DIRS}) + endif() + + if (HAVE_CLANG_BLOCKS) + target_compile_options(libfyaml-test PRIVATE -fblocks) + if (CLANG_BLOCKS_LIB) + target_link_libraries(libfyaml-test PRIVATE ${CLANG_BLOCKS_LIB}) + endif() + endif() + + add_dependencies(libfyaml-test fyaml_static) + + # Set output directory to match autoconf behavior + set_target_properties(libfyaml-test PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Add libfyaml.test + add_test(NAME libfyaml + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test/libfyaml.test + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + set_tests_properties(libfyaml PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test" + ) + endif() + + # Error tests - register individual subtests + add_testerrors_tests() + + # Emitter tests - register individual subtests + add_testemitter_tests(testemitter "") + add_testemitter_tests(testemitter-streaming "--streaming") + add_testemitter_tests(testemitter-restreaming "--streaming --recreating") + + # Network and git-dependent tests + if(HAVE_NETWORK AND HAVE_GIT) + # YAML test suite - setup fixture + add_test(NAME yaml-test-suite-setup + COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR}/test + ${CMAKE_COMMAND} + -DGIT_EXECUTABLE=${GIT_EXECUTABLE} + -DTESTSUITEURL=${TESTSUITEURL} + -DTESTSUITECHECKOUT=${TESTSUITECHECKOUT} + -DTEST_DIR=${CMAKE_CURRENT_BINARY_DIR}/test + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/setup-yaml-test-suite.cmake + ) + set_tests_properties(yaml-test-suite-setup PROPERTIES + FIXTURES_SETUP yaml-test-suite-data + ) + + # YAML test suite tests - register individual subtests + # Three modes (handled automatically by add_testsuite_tests): + # 1. If test-suite-data exists: scan filesystem and register tests + # 2. If cmake/testsuite-tests.cmake exists: use pre-generated list + # 3. Otherwise: use monolithic test (re-run cmake after downloading data) + # To generate cmake/testsuite-tests.cmake, see cmake/README-testsuite-generation.md + add_testsuite_tests(testsuite testsuite.test) + add_testsuite_tests(testsuite-evstream testsuite-evstream.test) + add_testsuite_tests(testsuite-resolution testsuite-resolution.test) + if(HAVE_JQ) + add_testsuite_tests(testsuite-json testsuite-json.test) + endif() + + # JSON test suite - setup fixture + add_test(NAME json-test-suite-setup + COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR}/test + ${CMAKE_COMMAND} + -DGIT_EXECUTABLE=${GIT_EXECUTABLE} + -DTESTSUITEURL=${JSONTESTSUITEURL} + -DTESTSUITECHECKOUT=${JSONTESTSUITECHECKOUT} + -DTEST_DIR=${CMAKE_CURRENT_BINARY_DIR}/test + -DREPO_NAME=json-test-suite-data + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/setup-yaml-test-suite.cmake + ) + set_tests_properties(json-test-suite-setup PROPERTIES + FIXTURES_SETUP json-test-suite-data + ) + + # JSON test suite - register individual subtests + # Three modes (handled automatically by add_jsontestsuite_tests): + # 1. If json-test-suite-data exists: scan filesystem and register tests + # 2. If cmake/jsontestsuite-tests.cmake exists: use pre-generated list + # 3. Otherwise: use monolithic test (re-run cmake after downloading data) + # To generate cmake/jsontestsuite-tests.cmake, see cmake/README-testsuite-generation.md + add_jsontestsuite_tests() + endif() + + set(TEST_DRIVER ${CMAKE_SOURCE_DIR}/build-aux/tap-driver.sh) + + # Build all test dependencies as part of default 'all' target + # Note: CMake's 'make test' target doesn't trigger builds by design. + # Use 'make all test' or 'make check-am' or 'make build_and_test' for complete workflow. + set(TEST_DEPENDENCIES fy-tool fy-thread fy-b3sum fy-allocators) + if(HAVE_COMPATIBLE_CHECK) + list(APPEND TEST_DEPENDENCIES libfyaml-test) + endif() + if(HAVE_LIBYAML) + list(APPEND TEST_DEPENDENCIES libfyaml-parser) + endif() + + add_custom_target(build_tests ALL + DEPENDS ${TEST_DEPENDENCIES} + COMMENT "Building test dependencies" + ) + + # Convenience target that builds and runs tests in one command + add_custom_target(build_and_test + COMMAND ${CMAKE_CTEST_COMMAND} + DEPENDS ${TEST_DEPENDENCIES} + COMMENT "Building and running tests" + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + + # Autotools-compatible test runner using test-driver + # NOTE: Named 'check-am' instead of 'check' to avoid conflicts with check library target + add_custom_target(check-am + COMMENT "Running tests via Automake test-driver" + ) + + if(HAVE_COMPATIBLE_CHECK) + set(TEST_EXECUTABLES + libfyaml-test + ) + + foreach(exe IN LISTS TEST_EXECUTABLES) + add_custom_target(check_${exe} + COMMAND ${CMAKE_COMMAND} -E env TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR} TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR} SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test + ${TEST_DRIVER} + --test-name ${exe} + --log-file ${exe}.log + --trs-file ${exe}.trs + --comments + --color-tests yes + --expect-failure 0 + -- ${CMAKE_CURRENT_BINARY_DIR}/test/${exe} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + COMMENT "Running ${exe}" + ) + add_dependencies(check_${exe} ${exe}) + add_dependencies(check-am check_${exe}) + endforeach() + endif() + + set(TEST_SUITES + testsuite.test + jsontestsuite.test + testsuite-evstream.test + testsuite-resolution.test + testerrors.test + testemitter.test + testemitter-streaming.test + testemitter-restreaming.test + ) + + if(HAVE_JQ) + list(APPEND TEST_SUITES testsuite-json.test) + endif() + + if(HAVE_LIBCLANG) + list(APPEND TEST_SUITES testreflection.test) + endif() + + foreach(suite IN LISTS TEST_SUITES) + add_custom_target(check_${suite} + COMMAND ${CMAKE_COMMAND} -E env TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR} TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR} SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test + ${TEST_DRIVER} + --test-name ${suite} + --log-file ${suite}.log + --trs-file ${suite}.trs + --comments + --color-tests yes + --expect-failure 0 + -- ${CMAKE_SOURCE_DIR}/test/${suite} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + COMMENT "Running ${suite}" + ) + add_dependencies(check_${suite} fy-tool) + add_dependencies(check-am check_${suite}) + endforeach() + +endif() + +# Documentation +find_program(SPHINX_EXECUTABLE NAMES sphinx-build) + +if(SPHINX_EXECUTABLE) + set(HAVE_SPHINX TRUE) + set(SPHINX_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/doc") + set(SPHINX_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/doc/_build") + + # Add custom targets for documentation + add_custom_target(doc-html + COMMAND ${SPHINX_EXECUTABLE} -M html "${SPHINX_SOURCE_DIR}" "${SPHINX_BUILD_DIR}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Building HTML documentation with Sphinx" + VERBATIM + ) + + add_custom_target(doc-man + COMMAND ${SPHINX_EXECUTABLE} -M man "${SPHINX_SOURCE_DIR}" "${SPHINX_BUILD_DIR}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Building man pages with Sphinx" + VERBATIM + ) + + add_custom_target(doc-latexpdf + COMMAND ${SPHINX_EXECUTABLE} -M latexpdf "${SPHINX_SOURCE_DIR}" "${SPHINX_BUILD_DIR}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Building PDF documentation with Sphinx" + VERBATIM + ) + + add_custom_target(doc-clean + COMMAND ${CMAKE_COMMAND} -E remove_directory "${SPHINX_BUILD_DIR}" + COMMENT "Cleaning documentation build directory" + ) + + message(STATUS "Sphinx found: ${SPHINX_EXECUTABLE}") + message(STATUS " make doc-html - Build HTML documentation") + message(STATUS " make doc-man - Build man pages") + message(STATUS " make doc-latexpdf - Build PDF documentation") +else() + set(HAVE_SPHINX FALSE) + message(STATUS "Sphinx not found - documentation targets disabled") + message(STATUS " Install: pip3 install sphinx sphinx_rtd_theme sphinx-markdown-builder linuxdoc") +endif() + +# Install targets +install(TARGETS fyaml fy-tool + EXPORT libfyaml-targets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) + +# Create symlinks for fy-tool +install(CODE " + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-dump WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-filter WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-testsuite WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-join WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-ypath WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-compose WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) +") + +install(EXPORT libfyaml-targets + FILE libfyaml-targets.cmake + NAMESPACE libfyaml:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libfyaml +) + +export( + TARGETS fyaml + FILE "${CMAKE_CURRENT_BINARY_DIR}/cmake/libfyaml-targets.cmake" + NAMESPACE libfyaml:: +) + +# Package config +include(CMakePackageConfigHelpers) +configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/libfyaml-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/cmake/libfyaml-config.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libfyaml +) + +write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/cmake/libfyaml-config-version.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY AnyNewerVersion +) + +install( + FILES + ${CMAKE_CURRENT_BINARY_DIR}/cmake/libfyaml-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/cmake/libfyaml-config-version.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libfyaml +) + +# pkg-config file +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/libfyaml.pc.in + ${CMAKE_CURRENT_BINARY_DIR}/libfyaml.pc + @ONLY +) + +install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/libfyaml.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig +) + +# tags and cscope support +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/tags.cmake) + +# Print configuration summary +message(STATUS "") +message(STATUS "---{ ${PROJECT_NAME} ${PROJECT_VERSION} }---") +message(STATUS "") +message(STATUS "VERSION: ${PROJECT_VERSION}") +message(STATUS "MAJOR.MINOR: ${VERSION_MAJOR}.${VERSION_MINOR}") +message(STATUS "PATCH: ${VERSION_PATCH}") +message(STATUS "EXTRA: ${VERSION_EXTRA}") +message(STATUS "LIBTOOL_VERSION: ${LIBTOOL_VERSION}") +message(STATUS "prefix: ${CMAKE_INSTALL_PREFIX}") +message(STATUS "Build system: ${CMAKE_HOST_SYSTEM_NAME} ${CMAKE_HOST_SYSTEM_PROCESSOR}") +message(STATUS "Target system: ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}") +message(STATUS "Cross compiling: ${CROSS_COMPILING}") +message(STATUS "C compiler: ${CMAKE_C_COMPILER}") +message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") +message(STATUS "HAVE_CHECK: ${HAVE_CHECK}") +message(STATUS "HAVE_COMPATIBLE_CHECK: ${HAVE_COMPATIBLE_CHECK}") +message(STATUS "HAVE_LIBYAML: ${HAVE_LIBYAML}") +message(STATUS "HAVE_CMAKE_GEXPR: ${HAVE_CMAKE_GEXPR}") +message(STATUS "HAVE_LIBCLANG: ${HAVE_LIBCLANG}") +message(STATUS "HAVE_CLANG_BLOCKS: ${HAVE_CLANG_BLOCKS}") +message(STATUS "HAVE_NETWORK: ${HAVE_NETWORK}") +message(STATUS "HAVE_DEVMODE: ${HAVE_DEVMODE}") +message(STATUS "HAVE_GIT: ${HAVE_GIT}") +message(STATUS "HAVE_JQ: ${HAVE_JQ}") +message(STATUS "HAVE_DOCKER: ${HAVE_DOCKER}") +message(STATUS "HAVE_SPHINX: ${HAVE_SPHINX}") +message(STATUS "HAVE_ASAN: ${HAVE_ASAN}") +message(STATUS "TARGET_HAS_SSE2: ${TARGET_HAS_SSE2}") +message(STATUS "TARGET_HAS_SSE41: ${TARGET_HAS_SSE41}") +message(STATUS "TARGET_HAS_AVX2: ${TARGET_HAS_AVX2}") +message(STATUS "TARGET_HAS_AVX512: ${TARGET_HAS_AVX512}") +message(STATUS "TARGET_HAS_NEON: ${TARGET_HAS_NEON}") +message(STATUS "HAVE_HEAP_TRAMPOLINES: ${HAVE_HEAP_TRAMPOLINES}") +message(STATUS "TESTSUITEURL: ${TESTSUITEURL}") +message(STATUS "TESTSUITECHECKOUT: ${TESTSUITECHECKOUT}") +message(STATUS "JSONTESTSUITEURL: ${JSONTESTSUITEURL}") +message(STATUS "JSONTESTSUITECHECKOUT: ${JSONTESTSUITECHECKOUT}") +message(STATUS "") +if(BUILD_TESTING AND NOT CROSS_COMPILING) + message(STATUS "Test targets:") + message(STATUS " make all test - Build everything then run CTest (2464 individual tests)") + message(STATUS " make build_and_test - Convenience target: build and test in one command") + message(STATUS " make check-am - Autotools-compatible TAP test runner (~10 test scripts)") + message(STATUS "") + message(STATUS "NOTE: 'make test' alone will fail if binaries aren't built. Use 'make all test'.") + message(STATUS "") +elseif(CROSS_COMPILING) + message(STATUS "Tests disabled when cross-compiling (target binaries cannot execute on build host)") + message(STATUS "") +endif() diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6af6f28 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +ARG IMAGE=ubuntu +FROM ${IMAGE} +# install build dependencies +RUN apt-get update -qq +RUN apt-get install --no-install-recommends -y \ + gcc autoconf automake libtool git make libyaml-dev libltdl-dev \ + pkg-config check python3 python3-pip python3-setuptools +# install sphinx doc dependencies globally (without venv) +RUN pip3 install --break-system-packages wheel sphinx git+http://github.com/return42/linuxdoc.git sphinx_rtd_theme sphinx-markdown-builder +# configure argument +ARG CONFIG_ARGS +ENV CONFIG_ARGS=${CONFIG_ARGS:-"--enable-debug --prefix=/usr"} +COPY . /build +WORKDIR /build +# do a maintainer clean if the directory was unclean (it can fail) +RUN make maintainer-clean >/dev/null 2>&1|| true +RUN ./bootstrap.sh 2>&1 +RUN ./configure 2>&1 ${CONFIG_ARGS} +RUN make +RUN make check +RUN make distcheck +RUN make doc-html diff --git a/Dockerfile-build-deb b/Dockerfile-build-deb new file mode 100644 index 0000000..268fb3d --- /dev/null +++ b/Dockerfile-build-deb @@ -0,0 +1,21 @@ +ARG IMAGE=ubuntu +FROM ${IMAGE} +# install build dependencies +RUN apt-get update -qq +RUN apt-get install --no-install-recommends --fix-missing -y \ + gcc autoconf automake libtool git make libyaml-dev libltdl-dev \ + pkg-config check python3 python3-pip python3-setuptools \ + devscripts build-essential lintian debhelper dh-autoreconf fakeroot \ + gnupg +# install sphinx doc dependencies +RUN pip3 install wheel sphinx git+http://github.com/return42/linuxdoc.git sphinx_rtd_theme sphinx-markdown-builder +# configure argument +ARG CONFIG_ARGS +ENV CONFIG_ARGS=${CONFIG_ARGS} +COPY . /build +WORKDIR /build +# do a maintainer clean if the directory was unclean (it can fail) +RUN make maintainer-clean >/dev/null 2>&1|| true +RUN ./bootstrap.sh 2>&1 +RUN ./configure 2>&1 ${CONFIG_ARGS} +RUN make deb diff --git a/Dockerfile.alpine b/Dockerfile.alpine new file mode 100644 index 0000000..5aa6105 --- /dev/null +++ b/Dockerfile.alpine @@ -0,0 +1,16 @@ +ARG IMAGE=alpine +FROM ${IMAGE} +# install build dependencies +RUN apk update +RUN apk add musl-dev gcc autoconf automake libtool git make pkgconf bash +# configure argument +ARG CONFIG_ARGS +ENV CONFIG_ARGS=${CONFIG_ARGS:-"--enable-debug --prefix=/usr"} +COPY . /build +WORKDIR /build +# do a maintainer clean if the directory was unclean (it can fail) +RUN make maintainer-clean >/dev/null 2>&1|| true +RUN ./bootstrap.sh 2>&1 +RUN ./configure 2>&1 ${CONFIG_ARGS} +RUN make +# NOTE: no check, since alpine it's only a build test distro diff --git a/Dockerfile.centos b/Dockerfile.centos new file mode 100644 index 0000000..cb9495d --- /dev/null +++ b/Dockerfile.centos @@ -0,0 +1,16 @@ +ARG IMAGE=centos +FROM ${IMAGE} +# install build dependencies +RUN yum update -y +RUN yum install -y gcc autoconf automake libtool git make pkgconf +# configure argument +ARG CONFIG_ARGS +ENV CONFIG_ARGS=${CONFIG_ARGS:-"--enable-debug --prefix=/usr"} +COPY . /build +WORKDIR /build +# do a maintainer clean if the directory was unclean (it can fail) +RUN make maintainer-clean >/dev/null 2>&1|| true +RUN ./bootstrap.sh 2>&1 +RUN ./configure 2>&1 ${CONFIG_ARGS} +RUN make +# NOTE: no check, since alpine it's only a build test distro diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7403da9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2019 Pantelis Antoniou + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..e593fea --- /dev/null +++ b/Makefile.am @@ -0,0 +1,68 @@ +BUILT_SOURCES = .version +.version: + echo $(VERSION) > $@-t && mv $@-t $@ +dist-hook: + echo $(VERSION) > $(distdir)/.tarball-version +tarball-version: + echo $(VERSION) > .tarball-version + +# files to keep in the distribution (in case you want to boostrap) +EXTRA_DIST=bootstrap.sh \ + build-aux/shave.in build-aux/shave-libtool.in \ + build-aux/git-version-gen \ + README.md LICENSE \ + Dockerfile Dockerfile-build-deb \ + Dockerfile.alpine Dockerfile.centos \ + cmake CMakeLists.txt \ + examples + +MAINTAINERCLEANFILES = \ + Makefile.in src/Makefile.in config.h.in configure \ + install-sh ltmain.sh missing mkinstalldirs \ + config.log config.status config.guess config.sub config.h \ + build-stamp compile depcomp acinclude.m4 aclocal.m4 \ + stamp-h1 \ + ar-lib m4/libtool.m4 m4/ltoptions.m4 m4/ltsugar.m4 m4/ltversion.m4 \ + m4/lt~obsolete.m4 src/mock/.dirstamp src/mock/Makefile.in \ + config.h.in~ \ + test-driver test/Makefile.in \ + build-aux/ar-lib build-aux/compile build-aux/config.guess \ + build-aux/config.sub build-aux/depcomp build-aux/install-sh \ + build-aux/ltmain.sh build-aux/missing build-aux/tap-driver.sh + +DISTCLEANFILES = \ + .version + +clean-local: + +SUBDIRS = src test doc + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = libfyaml.pc + +maintainer-clean-local: + @rm -rf install sphinx + @rm -rf install artifacts + +# extra file to put in the distribution +EXTRA_DIST += \ + scripts/create-virtual-env \ + scripts/install-linuxdoc.sh \ + scripts/run-check-errors.sh \ + scripts/run-compare-dump.sh \ + scripts/run-compare-examples.sh \ + scripts/run-compare-parse.sh \ + scripts/run-compare-scan.sh \ + scripts/run-compare-testsuite.sh \ + scripts/run-emit-check.sh \ + scripts/run-kcachegrind.sh \ + scripts/run-list-testsuite.sh \ + scripts/run-massif.sh \ + scripts/run-test.sh \ + scripts/run-valgrind.sh \ + scripts/show-desc.sh + +if HAVE_DOCKER +docker: Dockerfile + @DOCKER@ build -t libfyaml:$(VERSION) $(top_srcdir) +endif diff --git a/README.md b/README.md new file mode 100644 index 0000000..bdd7af2 --- /dev/null +++ b/README.md @@ -0,0 +1,660 @@ +# libfyaml + +[![CI Status](https://github.com/pantoniou/libfyaml/workflows/Standard%20Automake%20CI/badge.svg)](https://github.com/pantoniou/libfyaml/actions) +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) +[![Language: C](https://img.shields.io/badge/language-C-brightgreen.svg)](https://en.wikipedia.org/wiki/C_(programming_language)) + +**A fully-featured YAML 1.2 and JSON parser/writer with zero-copy operation and no artificial limits.** + +libfyaml is designed for high performance with zero content duplication, supporting arbitrarily large documents without the 1024-character limit on implicit keys found in other parsers. It passes the complete YAML test suite and provides both event-based (streaming) and document tree APIs for maximum flexibility. + +**Fully MIT Licensed** - v0.9.1 removes all GPL code, making the entire library consistently MIT licensed for use in commercial and proprietary projects. + +**Key Highlights:** +- **Zero-copy architecture** - minimal memory usage, handles arbitrarily large documents +- **YAML 1.2 & JSON** - full spec compliance, passes complete YAML test suite +- **Dual API** - event-based streaming (like libyaml) or document tree manipulation +- **No artificial limits** - unlike libyaml's 1024-char implicit key restriction +- **High performance** - up to 24x faster on large files with streaming mode, thread-safe lockless allocators +- **Rich manipulation** - path expressions, scanf/printf-style APIs, programmatic document building +- **Production-ready** - memory-safe, valgrind-clean, extensive test coverage + +--- + +## Quick Start + +Parse, query, and modify YAML documents with ease: + +```c +#include + +// Parse YAML from string or file +struct fy_document *fyd = fy_document_build_from_file(NULL, "config.yaml"); + +// Extract values using path-based scanf +unsigned int port; +char hostname[256]; +fy_document_scanf(fyd, "/server/port %u /server/host %255s", &port, hostname); + +// Modify document using path-based insertion +fy_document_insert_at(fyd, "/server", FY_NT, fy_node_buildf(fyd, "timeout: 30")); + +// Emit as YAML or JSON +fy_emit_document_to_file(fyd, FYECF_SORT_KEYS, "config.yaml"); +fy_document_destroy(fyd); +``` + +--- + +## What's New in v0.9.1 + +**Major Updates:** + +- **GPL Code Removed**: All GPL-licensed code has been replaced with MIT-licensed implementations. The entire library is now consistently MIT licensed. +- **Performance**: Up to 24x faster on large files with new streaming mode default in fy-tool +- **Thread Safety**: New thread-safe lockless allocators for multi-threaded applications +- **Stability**: Fixed 12 critical bugs including segmentation faults and memory corruption +- **New APIs**: Parser checkpointing, composer interface, document builder, YPath set operators +- **Build System**: CMake now at feature parity with autotools, cross-compilation support + +See [CHANGELOG.md](CHANGELOG.md) for complete release notes. + +--- + +## Why libfyaml? + +### Zero-Copy Efficiency +Content is never duplicated internally. libfyaml uses "atoms" that reference source data directly, keeping memory usage low and enabling handling of gigabyte-sized documents without performance degradation. + +### No Artificial Limits +Unlike libyaml's 1024-character limit on implicit keys, libfyaml has no hardcoded restrictions. Parse real-world YAML documents of any complexity without worrying about hitting parser limits. + +### Dual API Design +Choose the right API for your use case: +- **Event-based** (streaming): For memory-efficient processing of large streams +- **Document tree**: For convenient manipulation, queries, and modifications + +Both APIs coexist and can be mixed as needed. + +### Powerful Manipulation +- **Path expressions** (YPATH): XPath-like queries (`/foo/bar`, `/items/[0]`) +- **scanf/printf style**: Extract and build data with format strings +- **Programmatic building**: Construct documents from scratch with simple APIs +- **Rich emitter**: Output as YAML (block/flow), JSON, with colors and formatting options + +### Production Quality +- **Memory safe**: Valgrind-clean, no leaks under normal operation +- **Accurate errors**: Compiler-format diagnostics that integrate with IDEs +- **Fully tested**: Passes complete YAML 1.2 test suite + +--- + +## Features + +### YAML & JSON Support +- Full YAML 1.2 specification compliance +- YAML 1.3 compatibility (preparing for upcoming spec) +- JSON parsing and emission +- Comment preservation (experimental) +- Complete test suite coverage + +### Parsing Modes +- Event-based streaming parser (like libyaml) +- Document tree builder with full manipulation API +- Configurable resolution of anchors and merge keys +- Multiple input sources: files, strings, file pointers, streams + +### Document Manipulation +- Path-based queries and modifications (YPATH) +- scanf/printf-style data extraction and insertion +- Programmatic document and node creation +- Clone, copy, and merge operations +- Anchor and alias management +- Mapping key sorting + +### Output & Emission +- Multiple output modes: original, block, flow, JSON +- ANSI color output support +- Visible whitespace mode for debugging +- Configurable indentation and line width +- Streaming or buffered emission + +### Performance & Scalability +- Zero-copy operation on mmap'd files and constant strings +- No artificial limits on key lengths, nesting depth, or document size +- Thread-safe lockless allocators (linear, mremap, dedup, auto) for multi-threaded applications +- Streaming mode with up to 24x performance improvement on large files +- Configurable memory allocators for different usage patterns +- Efficient handling of arbitrarily large documents + +### Developer Experience +- Single header inclusion: `` +- Opaque pointers - stable ABI across versions +- Descriptive error messages in compiler format +- Extensive API documentation +- Multiple detailed examples + +--- + +## Installation + +### Using CMake (Recommended) + +Add libfyaml to your CMake project: + +```cmake +find_package(libfyaml 0.9.1 REQUIRED) +target_link_libraries(your_app PRIVATE libfyaml::libfyaml) +``` + +Check for optional features: +```cmake +if(libfyaml_HAS_LIBCLANG) + message(STATUS "Reflection support available") +endif() +``` + +### Using pkg-config + +```bash +# Compile flags +pkg-config --cflags libfyaml + +# Linker flags +pkg-config --libs libfyaml +``` + +In your Makefile: +```make +CFLAGS += $(shell pkg-config --cflags libfyaml) +LDFLAGS += $(shell pkg-config --libs libfyaml) +``` + +### Building from Source + +**Using CMake:** +```bash +mkdir build && cd build +cmake .. +cmake --build . +ctest # Run tests +cmake --install . # Install +``` + +CMake configuration options: +```bash +cmake -DBUILD_SHARED_LIBS=ON \ # Build shared libraries (default: ON) + -DENABLE_ASAN=OFF \ # AddressSanitizer for debugging (default: OFF) + -DENABLE_STATIC_TOOLS=OFF \ # Static tool executables (default: OFF) + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + .. +``` + +**Using Autotools:** +```bash +./bootstrap.sh +./configure +make +make check # Run tests +sudo make install +``` + +### Prerequisites + +**Debian/Ubuntu:** +```bash +sudo apt install gcc autoconf automake libtool git make libltdl-dev pkg-config check +``` + +**Optional dependencies:** +- `libyaml-dev` - for comparison testing +- `llvm-dev libclang-dev` - for reflection support (experimental) + +--- + +## Usage Examples + +### Level 1: Basic Parsing + +Parse and extract data from a YAML file: + +```c +#include +#include +#include + +int main(void) { + struct fy_document *fyd = fy_document_build_from_file(NULL, "config.yaml"); + if (!fyd) { + fprintf(stderr, "Failed to parse YAML\n"); + return EXIT_FAILURE; + } + + // Get root node and emit + fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stdout); + + fy_document_destroy(fyd); + return EXIT_SUCCESS; +} +``` + +### Level 2: Path-Based Queries + +Extract specific values using YPATH expressions: + +```c +#include + +int main(void) { + const char *yaml = + "server:\n" + " host: localhost\n" + " port: 8080\n" + " ssl: true\n"; + + struct fy_document *fyd = fy_document_build_from_string(NULL, yaml, FY_NT); + + // Extract multiple values at once + char host[256]; + unsigned int port; + fy_document_scanf(fyd, + "/server/host %255s " + "/server/port %u", + host, &port); + + printf("Server: %s:%u\n", host, port); + + // Query single node by path + struct fy_node *ssl_node = fy_node_by_path( + fy_document_root(fyd), "/server/ssl", FY_NT, FYNWF_DONT_FOLLOW); + if (ssl_node && fy_node_compare_string(ssl_node, "true", FY_NT) == 0) { + printf("SSL is enabled\n"); + } + + fy_document_destroy(fyd); + return 0; +} +``` + +### Level 3: Document Manipulation + +Modify existing documents programmatically: + +```c +#include + +int main(void) { + struct fy_document *fyd = fy_document_build_from_file(NULL, "invoice.yaml"); + + // Read current invoice number + unsigned int invoice_nr; + char given_name[256]; + int count = fy_document_scanf(fyd, + "/invoice %u " + "/bill-to/given %255s", + &invoice_nr, given_name); + + if (count == 2) { + printf("Processing invoice #%u for %s\n", invoice_nr, given_name); + + // Update invoice number + fy_document_insert_at(fyd, "/invoice", FY_NT, + fy_node_buildf(fyd, "%u", invoice_nr + 1)); + + // Add new fields + fy_document_insert_at(fyd, "/bill-to", FY_NT, + fy_node_buildf(fyd, "spouse: %s", "Jane")); + + fy_document_insert_at(fyd, "/bill-to", FY_NT, + fy_node_buildf(fyd, + "delivery-address:\n" + " street: 123 Main St\n" + " city: Springfield\n")); + + // Save with sorted keys + fy_emit_document_to_file(fyd, + FYECF_DEFAULT | FYECF_SORT_KEYS, + "invoice_updated.yaml"); + } + + fy_document_destroy(fyd); + return 0; +} +``` + +### Level 4: Event-Based Streaming + +Process large YAML files with minimal memory: + +```c +#include + +int main(void) { + struct fy_parser *fyp = fy_parser_create(NULL); + if (!fyp) + return EXIT_FAILURE; + + // Set input + fy_parser_set_input_file(fyp, "large_file.yaml"); + + // Process events + struct fy_event *fye; + while ((fye = fy_parser_parse(fyp)) != NULL) { + enum fy_event_type type = fye->type; + + switch (type) { + case FYET_SCALAR: { + const char *value = fy_token_get_text0(fy_event_get_token(fye)); + printf("Scalar: %s\n", value); + break; + } + case FYET_MAPPING_START: + printf("Mapping start\n"); + break; + case FYET_SEQUENCE_START: + printf("Sequence start\n"); + break; + default: + break; + } + + fy_parser_event_free(fyp, fye); + } + + fy_parser_destroy(fyp); + return 0; +} +``` + +### Level 5: Building Documents Programmatically + +Create YAML documents from scratch: + +```c +#include + +int main(void) { + // Create empty document + struct fy_document *fyd = fy_document_create(NULL); + + // Build root mapping using printf-style formatting + struct fy_node *root = fy_node_buildf(fyd, + "application: MyApp\n" + "version: %d.%d.%d\n" + "settings:\n" + " debug: %s\n" + " max_connections: %d\n" + " allowed_hosts:\n" + " - localhost\n" + " - 127.0.0.1\n", + 1, 2, 3, + "true", + 100); + + fy_document_set_root(fyd, root); + + // Add more fields programmatically + fy_document_insert_at(fyd, "/settings", FY_NT, + fy_node_buildf(fyd, "log_level: info")); + + // Output as JSON + printf("As JSON:\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_JSON, stdout); + + printf("\nAs YAML:\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_BLOCK | FYECF_SORT_KEYS, stdout); + + fy_document_destroy(fyd); + return 0; +} +``` + +--- + +## API Overview + +libfyaml provides a comprehensive API organized into functional categories: + +| Category | Purpose | Key Functions | +|----------|---------|---------------| +| **Parser** | Event-based streaming YAML parsing | `fy_parser_create()`, `fy_parser_parse()`, `fy_parser_set_input_*()` | +| **Parser Checkpointing** | Look-ahead and peek operations | `fy_parser_parse_peek()`, `fy_parser_skip()`, `fy_parser_count_*()` | +| **Composer** | Event-based document composition | `fy_composer_create()`, `fy_composer_process_event()` | +| **Document** | High-level document tree manipulation | `fy_document_build_from_*()`, `fy_document_scanf()`, `fy_document_insert_at()` | +| **Document Builder** | Programmatic document construction | `fy_document_builder_create()`, `fy_document_builder_*()` | +| **Node** | Individual YAML node operations | `fy_node_create_*()`, `fy_node_by_path()`, `fy_node_buildf()`, `fy_node_delete()` | +| **Sequence** | Array/list operations | `fy_node_sequence_iterate()`, `fy_node_sequence_append()` | +| **Mapping** | Key-value dictionary operations | `fy_node_mapping_lookup_*()`, `fy_node_mapping_append()` | +| **Emitter** | YAML/JSON output generation | `fy_emitter_create()`, `fy_emit_document_to_*()`, `fy_emit_body_node()` | +| **Path** | XPath-like YAML queries (YPATH) | `fy_node_by_path()`, `fy_path_exec_execute()` | +| **Anchor** | Anchor and alias management | `fy_document_lookup_anchor()`, `fy_node_create_alias()` | +| **Allocator** | Memory allocation control | `fy_allocator_create()`, `fy_allocator_contains()` | + +**See the [complete API documentation](https://pantoniou.github.io/libfyaml/) for detailed reference.** + +--- + +## Tools + +libfyaml includes `fy-tool`, a multi-function YAML manipulation utility: + +### fy-dump - Parse and Pretty-Print + +**Note**: As of v0.9.1, fy-dump uses streaming mode by default for up to 24x better performance on large files. Use `--no-streaming` if you need forward anchor references or strict duplicate key checking. + +```bash +# Convert YAML to JSON (streaming mode) +fy-dump -m json config.yaml + +# Use document mode for forward anchors +fy-dump --no-streaming -m json config.yaml + +# Format with visible whitespace +fy-dump -V -m block messy.yaml + +# Pretty-print with sorted keys +fy-dump -s -m block config.yaml > formatted.yaml +``` + +### fy-filter - Extract Document Parts + +```bash +# Extract specific paths +fy-filter --file config.yaml /database/host /database/port + +# Filter with complex keys +echo "{ foo: bar }: baz" | fy-filter "/{foo: bar}/" +# Output: baz +``` + +### fy-join - Merge YAML Documents + +```bash +# Merge two configuration files +fy-join base-config.yaml override-config.yaml + +# Merge inline YAML +fy-join ">foo: bar" ">baz: qux" +# Output: +# foo: bar +# baz: qux +``` + +### fy-testsuite - Test Suite Event Format + +```bash +# Generate test-suite event stream +echo "foo: bar" | fy-testsuite - +# Output: +# +STR +# +DOC +# +MAP +# =VAL :foo +# =VAL :bar +# -MAP +# -DOC +# -STR +``` + +**All tools support:** +- Syntax coloring (`--color`) +- Whitespace visualization (`--visible`) +- Anchor resolution (`--resolve`) +- Multiple output modes (`--mode`) +- Flexible input from files, stdin, or inline strings (`">yaml here"`) + +--- + +## Performance & Architecture + +### Zero-Copy Design + +libfyaml's core design principle is **zero content duplication**: + +- **Atoms**: Internal references to source data without copying +- **Memory efficiency**: Only metadata is allocated, content stays in source +- **Scalability**: Handles multi-gigabyte documents with minimal RAM +- **Speed**: No time wasted on redundant memory operations + +### Memory Usage + +For a typical YAML document: +- **libyaml**: Copies all content into internal structures +- **libfyaml**: References original data, allocates only ~1-5% overhead for structure + +### No Artificial Limits + +Common parser limitations that libfyaml **does not have**: +- No 1024-char limit on implicit keys (unlike libyaml) +- No hardcoded nesting depth limits +- No document size restrictions +- No key count limits per mapping + +Configurable limits exist for safety but can be adjusted or removed entirely. + +--- + +## Documentation + +- **[API Reference](https://pantoniou.github.io/libfyaml/)** - Complete API documentation +- **[Examples](examples/)** - Sample code and use cases +- **[CMake Integration](cmake/README-package.md)** - Using libfyaml in CMake projects +- **[Changelog](CHANGELOG.md)** - Version history and release notes + +### Building Documentation Locally + +```bash +# HTML documentation +make doc-html +# Output: doc/_build/html/index.html + +# PDF documentation +make doc-latexpdf +# Output: doc/_build/latex/libfyaml.pdf +``` + +--- + +## Platform Support + +**Primary Platform**: Linux (Debian/Ubuntu, Fedora, Arch, etc.) +**Also Supported**: macOS (via Homebrew) +**Not Yet Supported**: Windows (contributions welcome!) + +**Architectures**: x86, x86_64, ARM, ARM64, and others +**Compilers**: GCC, Clang, ICC + +--- + +## Contributing + +We welcome contributions! Here's how to get involved: + +1. **Report Issues**: [GitHub Issues](https://github.com/pantoniou/libfyaml/issues) +2. **Submit Pull Requests**: Fork, modify, and submit PRs +3. **Improve Documentation**: Help make libfyaml more accessible +4. **Add Examples**: Share your use cases with the community + +### Development Setup + +```bash +# Clone repository +git clone https://github.com/pantoniou/libfyaml.git +cd libfyaml + +# Build with CMake +mkdir build && cd build +cmake -DENABLE_ASAN=ON -DCMAKE_BUILD_TYPE=Debug .. +cmake --build . +ctest + +# Or build with Autotools +./bootstrap.sh +./configure --enable-asan +make check +``` + +### Running Tests + +```bash +# All tests +make check + +# Specific test suites +./test/libfyaml.test +./test/testemitter.test + +# With valgrind +./scripts/run-valgrind.sh +``` + +For CMake ctest is supported +```bash +cd build +ctest --progress -j`nproc` +``` + +--- + +## Missing Features + +Current limitations: +- **Windows support**: Not yet implemented (contributions welcome) +- **Unicode**: UTF-8 only, no wide character input support + +--- + +## License + +libfyaml is released under the **MIT License**. + +**As of version 0.9.1, all GPL-licensed code has been removed.** The entire library is now consistently MIT licensed, making it suitable for use in commercial and proprietary projects without any GPL licensing concerns. + +``` +Copyright (c) 2019-2025 Pantelis Antoniou + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +``` + +See [LICENSE](LICENSE) for complete terms. + +--- + +## Acknowledgments + +- **YAML Test Suite**: https://github.com/yaml/yaml-test-suite +- **Author**: Pantelis Antoniou [@pantoniou](https://github.com/pantoniou) +- **Contributors**: Thank you to all who have contributed code, bug reports, and feedback! + +--- + +**[Back to Top](#libfyaml)** diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 0000000..fc00c0d --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# potato autoconf +touch build-aux/tap-driver.sh +autoreconf -fvi diff --git a/build-aux/git-version-gen b/build-aux/git-version-gen new file mode 100755 index 0000000..b287cb7 --- /dev/null +++ b/build-aux/git-version-gen @@ -0,0 +1,160 @@ +#!/bin/sh +# Print a version string. +scriptversion=2011-02-19.19; # UTC + +# Copyright (C) 2007-2011 Free Software Foundation, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script is derived from GIT-VERSION-GEN from GIT: http://git.or.cz/. +# It may be run two ways: +# - from a git repository in which the "git describe" command below +# produces useful output (thus requiring at least one signed tag) +# - from a non-git-repo directory containing a .tarball-version file, which +# presumes this script is invoked like "./git-version-gen .tarball-version". + +# In order to use intra-version strings in your project, you will need two +# separate generated version string files: +# +# .tarball-version - present only in a distribution tarball, and not in +# a checked-out repository. Created with contents that were learned at +# the last time autoconf was run, and used by git-version-gen. Must not +# be present in either $(srcdir) or $(builddir) for git-version-gen to +# give accurate answers during normal development with a checked out tree, +# but must be present in a tarball when there is no version control system. +# Therefore, it cannot be used in any dependencies. GNUmakefile has +# hooks to force a reconfigure at distribution time to get the value +# correct, without penalizing normal development with extra reconfigures. +# +# .version - present in a checked-out repository and in a distribution +# tarball. Usable in dependencies, particularly for files that don't +# want to depend on config.h but do want to track version changes. +# Delete this file prior to any autoconf run where you want to rebuild +# files to pick up a version string change; and leave it stale to +# minimize rebuild time after unrelated changes to configure sources. +# +# It is probably wise to add these two files to .gitignore, so that you +# don't accidentally commit either generated file. +# +# Use the following line in your configure.ac, so that $(VERSION) will +# automatically be up-to-date each time configure is run (and note that +# since configure.ac no longer includes a version string, Makefile rules +# should not depend on configure.ac for version updates). +# +# AC_INIT([GNU project], +# m4_esyscmd([build-aux/git-version-gen .tarball-version]), +# [bug-project@example]) +# +# Then use the following lines in your Makefile.am, so that .version +# will be present for dependencies, and so that .tarball-version will +# exist in distribution tarballs. +# +# BUILT_SOURCES = $(top_srcdir)/.version +# $(top_srcdir)/.version: +# echo $(VERSION) > $@-t && mv $@-t $@ +# dist-hook: +# echo $(VERSION) > $(distdir)/.tarball-version + +case $# in + 1|2) ;; + *) echo 1>&2 "Usage: $0 \$srcdir/.tarball-version" \ + '[TAG-NORMALIZATION-SED-SCRIPT]' + exit 1;; +esac + +tarball_version_file=$1 +tag_sed_script="${2:-s/x/x/}" +nl=' +' + +# Avoid meddling by environment variable of the same name. +v= +v_from_git= + +# First see if there is a tarball-only version file. +# then try "git describe", then default. +if test -f $tarball_version_file +then + v=`cat $tarball_version_file` || v= + case $v in + *$nl*) v= ;; # reject multi-line output + [0-9]*) ;; + *) v= ;; + esac + test -z "$v" \ + && echo "$0: WARNING: $tarball_version_file is missing or damaged" 1>&2 +fi + +if test -n "$v" +then + : # use $v +# Otherwise, if there is at least one git commit involving the working +# directory, and "git describe" output looks sensible, use that to +# derive a version string. +elif test "`git log -1 --pretty=format:x . 2>&1`" = x \ + && v=`git describe --abbrev=4 --match='v*' HEAD 2>/dev/null \ + || git describe --abbrev=4 HEAD 2>/dev/null` \ + && v=`printf '%s\n' "$v" | sed "$tag_sed_script"` \ + && case $v in + v[0-9]*) ;; + *) (exit 1) ;; + esac +then + + # Remove the "g" in git describe's output string, to save a byte. + v=`echo "$v" | sed 's/-g/-/'`; + + case $v in + *-rc*) + ;; + *) + # Change the first '-' to a '.', so version-comparing tools work properly. + v=`echo "$v" | sed 's/-/./'`; + ;; + esac + v_from_git=1 +else + v=UNKNOWN +fi + +v=`echo "$v" |sed 's/^v//'` + +# Test whether to append the "-dirty" suffix only if the version +# string we're using came from git. I.e., skip the test if it's "UNKNOWN" +# or if it came from .tarball-version. +if test -n "$v_from_git"; then + # Don't declare a version "dirty" merely because a time stamp has changed. + git update-index --refresh > /dev/null 2>&1 + + dirty=`exec 2>/dev/null;git diff-index --name-only HEAD` || dirty= + case "$dirty" in + '') ;; + *) # Append the suffix only if there isn't one already. + case $v in + *-dirty) ;; + *) v="$v-dirty" ;; + esac ;; + esac +fi + +# Omit the trailing newline, so that m4_esyscmd can use the result directly. +echo "$v" | tr -d "$nl" + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/build-aux/shave-libtool.in b/build-aux/shave-libtool.in new file mode 100644 index 0000000..0f7fb12 --- /dev/null +++ b/build-aux/shave-libtool.in @@ -0,0 +1,109 @@ +#!/bin/sh +# +# Copyright (c) 2009, Damien Lespiau +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +# we need sed +SED=@SED@ +if test -z "$SED" ; then +SED=sed +fi + +lt_unmangle () +{ + last_result=`echo $1 | $SED -e 's#.libs/##' -e 's#[0-9a-zA-Z_\-\.]*_la-##'` +} + +# the real libtool to use +LIBTOOL="$1" +shift + +# if 1, don't print anything, the underlaying wrapper will do it +pass_though=0 + +# scan the arguments, keep the right ones for libtool, and discover the mode +preserved_args= + +# have we seen the --tag option of libtool in the command line ? +tag_seen=0 + +while test "$#" -gt 0; do + opt="$1" + shift + + case $opt in + --mode=*) + mode=`echo $opt | $SED -e 's/[-_a-zA-Z0-9]*=//'` + preserved_args="$preserved_args $opt" + ;; + -o) + lt_output="$1" + preserved_args="$preserved_args $opt" + ;; + --tag=*) + tag_seen=1 + preserved_args="$preserved_args $opt" + ;; + *) + preserved_args="$preserved_args '$opt'" + ;; + esac +done + +case "$mode" in +compile) + # shave will be called and print the actual CC/CXX/LINK line + preserved_args="$preserved_args --shave-mode=$mode" + pass_though=1 + ;; +link) + preserved_args="$preserved_args --shave-mode=$mode" + Q=" LINK " + ;; +*) + # let's u + # echo "*** libtool: Unimplemented mode: $mode, fill a bug report" + ;; +esac + +lt_unmangle "$lt_output" +output=$last_result + +# automake does not add a --tag switch to its libtool invocation when +# assembling a .s file and rely on libtool to infer the right action based +# on the compiler name. As shave is using CC to hook a wrapper, libtool gets +# confused. Let's detect these cases and add a --tag=CC option. +tag="" +if test $tag_seen -eq 0 -a x"$mode" = xcompile; then + tag="--tag=CC" +fi + +if test -z $V; then + if test $pass_though -eq 0; then + echo "$Q$output" + fi + eval "$LIBTOOL --silent $tag $preserved_args" +else + echo $LIBTOOL $tag $preserved_args + eval "$LIBTOOL $tag $preserved_args" +fi diff --git a/build-aux/shave.in b/build-aux/shave.in new file mode 100644 index 0000000..689badc --- /dev/null +++ b/build-aux/shave.in @@ -0,0 +1,130 @@ +#!/bin/sh +# +# Copyright (c) 2009, Damien Lespiau +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +# we need sed +SED=@SED@ +if test -z "$SED" ; then +SED=sed +fi + +lt_unmangle () +{ + last_result=`echo $1 | $SED -e 's#.libs/##' -e 's#[0-9a-zA-Z_\-\.]*_la-##'` +} + +# the tool to wrap (cc, cxx, ar, ranlib, ..) +tool="$1" +shift + +# the reel tool (to call) +REEL_TOOL="$1" +shift + +pass_through=0 +preserved_args= +while test "$#" -gt 0; do + opt="$1" + shift + + case $opt in + --shave-mode=*) + mode=`echo $opt | $SED -e 's/[-_a-zA-Z0-9]*=//'` + ;; + -o) + lt_output="$1" + preserved_args="$preserved_args $opt" + ;; + -out:*|/out:*) + lt_output="${opt#-out:}" + preserved_args="$preserved_args $opt" + ;; + *.l) + if [ "$tool" = "lex" ]; then + lt_output="$opt" + fi + preserved_args="$preserved_args $opt" + ;; + *.y) + if [ "$tool" = "yacc" ]; then + lt_output="$opt" + fi + preserved_args="$preserved_args $opt" + ;; + *) + preserved_args="$preserved_args '$opt'" + ;; + esac +done + +# mode=link is handled in the libtool wrapper +case "$mode,$tool" in +link,*) + pass_through=1 + ;; +*,cxx) + Q=" CXX " + ;; +*,ccas) + Q=" AS " + ;; +*,cc) + Q=" CC " + ;; +*,fc) + Q=" FC " + ;; +*,f77) + Q=" F77 " + ;; +*,objc) + Q=" OBJC " + ;; +*,mcs) + Q=" MCS " + ;; +*,lex) + Q=" LEX " + ;; +*,yacc) + Q=" YACC " + ;; +*,*) + # should not happen + Q=" CC " + ;; +esac + +lt_unmangle "$lt_output" +output=$last_result + +if test -z $V; then + if test $pass_through -eq 0; then + echo "$Q$output" + fi + eval "$REEL_TOOL $preserved_args" +else + echo $REEL_TOOL $preserved_args + eval "$REEL_TOOL $preserved_args" +fi diff --git a/cmake/CheckClangBlocks.cmake b/cmake/CheckClangBlocks.cmake new file mode 100644 index 0000000..a3b67b6 --- /dev/null +++ b/cmake/CheckClangBlocks.cmake @@ -0,0 +1,53 @@ +# --- CheckClangBlocks.cmake --- +# Detect support for Clang Blocks (Clang or AppleClang) +# Defines: HAVE_CLANG_BLOCKS (0 or 1) +# Defines: CLANG_BLOCKS_LIB (empty or "BlocksRuntime") + +# Default +set(HAVE_CLANG_BLOCKS 0) +set(CLANG_BLOCKS_LIB "") + +if (CMAKE_C_COMPILER_ID MATCHES "Clang") + + # AppleClang: Blocks are built-in + if (CMAKE_C_COMPILER_ID STREQUAL "AppleClang") + message(STATUS "AppleClang detected — Blocks support is built in") + + set(HAVE_CLANG_BLOCKS 1) + set(CLANG_BLOCKS_LIB "") + + else() + # LLVM Clang: must test for -fblocks and libBlocksRuntime + message(STATUS "LLVM Clang detected — checking for BlocksRuntime") + + include(CheckCSourceCompiles) + + set(CMAKE_REQUIRED_FLAGS "-fblocks") + set(CMAKE_REQUIRED_LIBRARIES "BlocksRuntime") + + check_c_source_compiles(" + #include + int main(void) { + void (^b)(void) = ^{ }; + b(); + return 0; + } + " HAVE_WORKING_CLANG_BLOCKS) + + unset(CMAKE_REQUIRED_FLAGS) + unset(CMAKE_REQUIRED_LIBRARIES) + + if (HAVE_WORKING_CLANG_BLOCKS) + message(STATUS "Clang Blocks support ENABLED") + set(HAVE_CLANG_BLOCKS 1) + set(CLANG_BLOCKS_LIB "BlocksRuntime") + else() + message(WARNING + "libBlocksRuntime not found — Clang Blocks disabled") + set(HAVE_CLANG_BLOCKS 0) + set(CLANG_BLOCKS_LIB "") + endif() + + endif() + +endif() diff --git a/cmake/CheckLibClang.cmake b/cmake/CheckLibClang.cmake new file mode 100644 index 0000000..65460b7 --- /dev/null +++ b/cmake/CheckLibClang.cmake @@ -0,0 +1,77 @@ +# CheckLibClang.cmake - Helper function to test if libclang works +# +# This module provides a function to test if libclang can be compiled and linked +# with the given compiler flags and libraries. +# +# Usage: +# check_libclang_works( +# CFLAGS +# INCLUDES +# LINK_DIRS +# LIBRARIES ) +# +# Arguments: +# result_var - Variable name to store the result (TRUE/FALSE) +# CFLAGS - Compiler flags to use (can be a list or string) +# INCLUDES - Include directories (list) +# LINK_DIRS - Link directories (list) +# LIBRARIES - Libraries to link (list or string) +# +# Example: +# check_libclang_works(LIBCLANG_WORKS +# CFLAGS "${LLVM_DEFINITIONS}" +# INCLUDES "${LLVM_INCLUDE_DIRS};${CLANG_INCLUDE_DIRS}" +# LINK_DIRS "${LLVM_LIBRARY_DIRS}" +# LIBRARIES "${LIBCLANG_LIBS}") + +include(CheckCSourceCompiles) + +function(check_libclang_works result_var) + # Parse arguments + set(options "") + set(oneValueArgs "") + set(multiValueArgs CFLAGS INCLUDES LINK_DIRS LIBRARIES) + cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + # Save current CMAKE_REQUIRED_* variables + set(_SAVED_CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + set(_SAVED_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") + set(_SAVED_CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES}") + set(_SAVED_CMAKE_REQUIRED_LINK_DIRECTORIES "${CMAKE_REQUIRED_LINK_DIRECTORIES}") + + # Set up test environment + if(ARG_CFLAGS) + set(CMAKE_REQUIRED_FLAGS "${ARG_CFLAGS}") + endif() + + if(ARG_INCLUDES) + set(CMAKE_REQUIRED_INCLUDES "${ARG_INCLUDES}") + endif() + + if(ARG_LINK_DIRS) + set(CMAKE_REQUIRED_LINK_DIRECTORIES "${ARG_LINK_DIRS}") + endif() + + if(ARG_LIBRARIES) + set(CMAKE_REQUIRED_LIBRARIES "${ARG_LIBRARIES}") + endif() + + # Test if we can compile and link with libclang + check_c_source_compiles(" + #include + int main() { + CXIndex idx = clang_createIndex(0, 0); + clang_disposeIndex(idx); + return 0; + } + " _LIBCLANG_COMPILE_TEST) + + # Restore CMAKE_REQUIRED_* variables + set(CMAKE_REQUIRED_LIBRARIES "${_SAVED_CMAKE_REQUIRED_LIBRARIES}" PARENT_SCOPE) + set(CMAKE_REQUIRED_FLAGS "${_SAVED_CMAKE_REQUIRED_FLAGS}" PARENT_SCOPE) + set(CMAKE_REQUIRED_INCLUDES "${_SAVED_CMAKE_REQUIRED_INCLUDES}" PARENT_SCOPE) + set(CMAKE_REQUIRED_LINK_DIRECTORIES "${_SAVED_CMAKE_REQUIRED_LINK_DIRECTORIES}" PARENT_SCOPE) + + # Return result + set(${result_var} ${_LIBCLANG_COMPILE_TEST} PARENT_SCOPE) +endfunction() diff --git a/cmake/add-tap-subtests.cmake b/cmake/add-tap-subtests.cmake new file mode 100644 index 0000000..638ee2c --- /dev/null +++ b/cmake/add-tap-subtests.cmake @@ -0,0 +1,690 @@ +# Helper function to add individual TAP subtests from test scripts +# This mimics the behavior of autotools tap-driver.sh by registering each subtest individually + +# Function to run a single test from testerrors.test +function(add_testerrors_tests) + file(GLOB error_dirs "${CMAKE_CURRENT_SOURCE_DIR}/test/test-errors/[0-9][0-9][0-9][0-9]") + + foreach(dir ${error_dirs}) + get_filename_component(test_id "${dir}" NAME) + + # Read description from === file + set(desc_file "${dir}/===") + if(EXISTS "${desc_file}") + file(READ "${desc_file}" test_desc) + string(STRIP "${test_desc}" test_desc) + else() + set(test_desc "") + endif() + + # Build test name with description if available + if(test_desc) + set(test_name_full "testerrors/${test_id} - ${test_desc}") + else() + set(test_name_full "testerrors/${test_id}") + endif() + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + testerrors "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set properties + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test" + LABELS "testerrors" + ) + endforeach() +endfunction() + +# Function to add testemitter tests +function(add_testemitter_tests test_name extra_args) + file(GLOB yaml_files "${CMAKE_CURRENT_SOURCE_DIR}/test/emitter-examples/*.yaml") + + foreach(yaml_file ${yaml_files}) + get_filename_component(test_id "${yaml_file}" NAME) + + add_test( + NAME "${test_name}/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + "${test_name}" "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("${test_name}/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;EXTRA_DUMP_ARGS=${extra_args}" + LABELS "${test_name}" + ) + endforeach() +endfunction() + +# Function to add testsuite tests (yaml test suite) +# Can work either by scanning the test-suite-data directory (if it exists) +# or by using a pre-generated list of tests from testsuite-tests.cmake +function(add_testsuite_tests test_name test_script) + set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/test/test-suite-data") + set(use_pregenerated_list FALSE) + + # Define skip/xfail lists based on test suite variant + # These match the lists in test/*.test scripts + if(test_name STREQUAL "testsuite-json") + set(skip_list "UGM3") + set(xfail_list "C4HZ") + elseif(test_name STREQUAL "testsuite-resolution") + set(skip_list "2JQS" "X38W") + set(xfail_list "") + elseif(test_name STREQUAL "testsuite-evstream") + set(skip_list "2JQS") + set(xfail_list "") + else() + set(skip_list "") + set(xfail_list "") + endif() + + # Check if we should use the pre-generated list + if(NOT EXISTS "${test_dir}") + # Try to use pre-generated list if available + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/testsuite-tests.cmake") + message(STATUS "Using pre-generated test list from cmake/testsuite-tests.cmake for ${test_name}") + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/testsuite-tests.cmake") + set(use_pregenerated_list TRUE) + else() + # Neither data nor pre-generated list exists, register monolithic fallback + message(STATUS "Test suite data not found - registering monolithic ${test_name} test") + message(STATUS "Run 'ctest -R yaml-test-suite-setup' then re-run cmake to register individual subtests") + + add_test(NAME ${test_name} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test/${test_script} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + set_tests_properties(${test_name} PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + FIXTURES_REQUIRED yaml-test-suite-data + ) + return() + endif() + else() + message(STATUS "Registering individual ${test_name} subtests from filesystem") + endif() + + if(use_pregenerated_list) + # Use pre-generated TESTSUITE_ALL_TESTS list + foreach(test_id ${TESTSUITE_ALL_TESTS}) + # test_id is either "BASE" or "BASE/SUBTEST" + string(REPLACE "/" ";" test_parts "${test_id}") + list(LENGTH test_parts parts_count) + + add_test( + NAME "${test_name}/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + "${test_name}" "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("${test_name}/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "${test_name}" + FIXTURES_REQUIRED "yaml-test-suite-data" + ) + endforeach() + else() + # Scan the filesystem (original behavior) + file(GLOB base_tests "${test_dir}/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]") + + foreach(base_test ${base_tests}) + get_filename_component(test_id "${base_test}" NAME) + + # Check for main test + if(EXISTS "${base_test}/===") + # Read description from === file + file(READ "${base_test}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "${test_name}/${test_id} - ${test_desc}") + else() + set(test_name_full "${test_name}/${test_id}") + endif() + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + "${test_name}" "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "${test_name}") + set(test_disabled FALSE) + + # Check if test should be skipped + if(test_id IN_LIST skip_list) + set(test_disabled TRUE) + set(test_labels "${test_name};skipped") + endif() + + # Check if test is expected to fail (xfail) + # Note: TAP xfail (TODO) tests still pass (exit 0), they just mark the result as TODO + # So we don't use WILL_FAIL here, just add a label for filtering + if(test_id IN_LIST xfail_list) + set(test_labels "${test_name};xfail") + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "${test_labels}" + FIXTURES_REQUIRED "yaml-test-suite-data" + DISABLED ${test_disabled} + ) + endif() + + # Check for 2-digit subtests + file(GLOB subtests_2d "${base_test}/[0-9][0-9]") + foreach(subtest ${subtests_2d}) + if(EXISTS "${subtest}/===") + get_filename_component(subtest_id "${subtest}" NAME) + set(full_test_id "${test_id}/${subtest_id}") + + # Read description from === file + file(READ "${subtest}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "${test_name}/${full_test_id} - ${test_desc}") + else() + set(test_name_full "${test_name}/${full_test_id}") + endif() + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + "${test_name}" "${full_test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "${test_name}") + set(test_disabled FALSE) + + # Check if test should be skipped + if(full_test_id IN_LIST skip_list) + set(test_disabled TRUE) + set(test_labels "${test_name};skipped") + endif() + + # Check if test is expected to fail (xfail) + # Note: TAP xfail (TODO) tests still pass (exit 0), they just mark the result as TODO + # So we don't use WILL_FAIL here, just add a label for filtering + if(full_test_id IN_LIST xfail_list) + set(test_labels "${test_name};xfail") + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "${test_labels}" + FIXTURES_REQUIRED "yaml-test-suite-data" + DISABLED ${test_disabled} + ) + endif() + endforeach() + + # Check for 3-digit subtests + file(GLOB subtests_3d "${base_test}/[0-9][0-9][0-9]") + foreach(subtest ${subtests_3d}) + if(EXISTS "${subtest}/===") + get_filename_component(subtest_id "${subtest}" NAME) + set(full_test_id "${test_id}/${subtest_id}") + + # Read description from === file + file(READ "${subtest}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "${test_name}/${full_test_id} - ${test_desc}") + else() + set(test_name_full "${test_name}/${full_test_id}") + endif() + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + "${test_name}" "${full_test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "${test_name}") + set(test_disabled FALSE) + + # Check if test should be skipped + if(full_test_id IN_LIST skip_list) + set(test_disabled TRUE) + set(test_labels "${test_name};skipped") + endif() + + # Check if test is expected to fail (xfail) + # Note: TAP xfail (TODO) tests still pass (exit 0), they just mark the result as TODO + # So we don't use WILL_FAIL here, just add a label for filtering + if(full_test_id IN_LIST xfail_list) + set(test_labels "${test_name};xfail") + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "${test_labels}" + FIXTURES_REQUIRED "yaml-test-suite-data" + DISABLED ${test_disabled} + ) + endif() + endforeach() + endforeach() + endif() +endfunction() + +# Function to add jsontestsuite tests +# Can work either by scanning the json-test-suite-data directory (if it exists) +# or by using a pre-generated list of tests from jsontestsuite-tests.cmake +function(add_jsontestsuite_tests) + set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/test/json-test-suite-data/test_parsing") + set(use_pregenerated_list FALSE) + + # Check if we should use the pre-generated list + if(NOT EXISTS "${test_dir}") + # Try to use pre-generated list if available + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/jsontestsuite-tests.cmake") + message(STATUS "Using pre-generated test list from cmake/jsontestsuite-tests.cmake") + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/jsontestsuite-tests.cmake") + set(use_pregenerated_list TRUE) + else() + # Neither data nor pre-generated list exists, register monolithic fallback + message(STATUS "JSON test suite data not found - registering monolithic jsontestsuite test") + message(STATUS "Run 'ctest -R json-test-suite-setup' then re-run cmake to register individual subtests") + + add_test(NAME jsontestsuite + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test/jsontestsuite.test + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + set_tests_properties(jsontestsuite PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + FIXTURES_REQUIRED json-test-suite-data + ) + return() + endif() + else() + message(STATUS "Registering individual jsontestsuite subtests from filesystem") + endif() + + if(use_pregenerated_list) + # Use pre-generated lists + # Expected to pass (y_*.json) + foreach(test_id ${JSONTESTSUITE_PASS_TESTS}) + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-pass" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + + # Expected to fail (n_*.json) + foreach(test_id ${JSONTESTSUITE_FAIL_TESTS}) + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-fail" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + + # Implementation defined (i_*.json) + foreach(test_id ${JSONTESTSUITE_IMPL_TESTS}) + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-impl" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + else() + # Scan the filesystem (original behavior) + # Expected to pass (y_*.json) + file(GLOB pass_tests "${test_dir}/y_*.json") + foreach(test_file ${pass_tests}) + get_filename_component(test_id "${test_file}" NAME) + + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-pass" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + + # Expected to fail (n_*.json) + file(GLOB fail_tests "${test_dir}/n_*.json") + foreach(test_file ${fail_tests}) + get_filename_component(test_id "${test_file}" NAME) + + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-fail" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + + # Implementation defined (i_*.json) + file(GLOB impl_tests "${test_dir}/i_*.json") + foreach(test_file ${impl_tests}) + get_filename_component(test_id "${test_file}" NAME) + + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-impl" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + endif() +endfunction() + +# Function to add reflection tests +function(add_testreflection_tests) + set(test_dir "${CMAKE_CURRENT_SOURCE_DIR}/test/reflection-data") + + # Skip/xfail lists from testreflection.test + # xfaillist="025PB08/00 025UXE6/00 025VL2J/00" + set(xfail_list "025PB08/00" "025UXE6/00" "025VL2J/00") + + # Probe system characteristics (matching testreflection.test logic) + include(CheckTypeSize) + check_type_size("unsigned int" SIZEOF_UINT) + check_type_size("unsigned short" SIZEOF_USHORT) + check_type_size("unsigned long" SIZEOF_ULONG) + check_type_size("unsigned char" SIZEOF_UCHAR) + check_type_size("long long" SIZEOF_LONGLONG) + + # Calculate bit sizes + math(EXPR intbits "${SIZEOF_UINT} * 8") + math(EXPR shortbits "${SIZEOF_USHORT} * 8") + math(EXPR longbits "${SIZEOF_ULONG} * 8") + math(EXPR charbits "${SIZEOF_UCHAR} * 8") + set(longlongbits 64) # Always 64 bits as per testreflection.test + + # Check if char is signed + include(CheckCSourceRuns) + check_c_source_runs(" + #include + int main() { return (CHAR_MIN < 0) ? 0 : 1; } + " CHAR_IS_SIGNED) + if(CHAR_IS_SIGNED) + set(charsigned "y") + else() + set(charsigned "n") + endif() + + message(STATUS "Reflection test system characteristics:") + message(STATUS " intbits=${intbits} shortbits=${shortbits} longbits=${longbits}") + message(STATUS " charbits=${charbits} longlongbits=${longlongbits} charsigned=${charsigned}") + + # Helper function to check if test should be skipped based on env file + macro(should_skip_test test_path result_var) + set(${result_var} FALSE) + set(env_file "${test_path}/env") + if(EXISTS "${env_file}") + file(READ "${env_file}" env_contents) + string(STRIP "${env_contents}" env_contents) + + # Parse env file for requirements + if(env_contents MATCHES "intbits=([0-9]+)") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${intbits}") + set(${result_var} TRUE) + endif() + endif() + if(env_contents MATCHES "shortbits=([0-9]+)") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${shortbits}") + set(${result_var} TRUE) + endif() + endif() + if(env_contents MATCHES "longbits=([0-9]+)") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${longbits}") + set(${result_var} TRUE) + endif() + endif() + if(env_contents MATCHES "charbits=([0-9]+)") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${charbits}") + set(${result_var} TRUE) + endif() + endif() + if(env_contents MATCHES "longlongbits=([0-9]+)") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${longlongbits}") + set(${result_var} TRUE) + endif() + endif() + if(env_contents MATCHES "charsigned=([yn])") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${charsigned}") + set(${result_var} TRUE) + endif() + endif() + endif() + endmacro() + + # Scan reflection-data directory for tests + # Note: CMake GLOB doesn't support [A-Z0-9] ranges like shell globs do + # We need to use a wildcard and filter in the loop + file(GLOB base_tests "${test_dir}/*") + + # Filter to only include directories matching the pattern [0-9][0-9][0-9][A-Z][A-Z0-9][A-Z0-9][A-Z0-9] + set(filtered_base_tests) + foreach(test_candidate ${base_tests}) + if(IS_DIRECTORY "${test_candidate}") + get_filename_component(test_name "${test_candidate}" NAME) + # Check if the name matches the pattern: 3 digits followed by 4 alphanumeric chars (starting with letter) + if(test_name MATCHES "^[0-9][0-9][0-9][A-Z][A-Z0-9][A-Z0-9][A-Z0-9]$") + list(APPEND filtered_base_tests "${test_candidate}") + endif() + endif() + endforeach() + set(base_tests ${filtered_base_tests}) + + foreach(base_test ${base_tests}) + get_filename_component(test_id "${base_test}" NAME) + + # Check for main test (base directory with === file) + if(EXISTS "${base_test}/===") + # Read description from === file + file(READ "${base_test}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "testreflection/${test_id} - ${test_desc}") + else() + set(test_name_full "testreflection/${test_id}") + endif() + + # Check if test should be skipped based on environment + should_skip_test("${base_test}" test_should_skip) + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + testreflection "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "testreflection") + set(test_disabled FALSE) + set(test_will_fail FALSE) + + # Check if test should be skipped + if(test_should_skip) + set(test_disabled TRUE) + set(test_labels "testreflection;skipped") + endif() + + # Check if test is expected to fail (xfail) + if(test_id IN_LIST xfail_list) + set(test_labels "testreflection;xfail") + set(test_will_fail TRUE) + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test" + LABELS "${test_labels}" + DISABLED ${test_disabled} + WILL_FAIL ${test_will_fail} + ) + endif() + + # Check for 2-digit subtests + file(GLOB subtests_2d "${base_test}/[0-9][0-9]") + foreach(subtest ${subtests_2d}) + if(EXISTS "${subtest}/===") + get_filename_component(subtest_id "${subtest}" NAME) + set(full_test_id "${test_id}/${subtest_id}") + + # Read description from === file + file(READ "${subtest}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "testreflection/${full_test_id} - ${test_desc}") + else() + set(test_name_full "testreflection/${full_test_id}") + endif() + + # Check if test should be skipped based on environment + should_skip_test("${subtest}" test_should_skip) + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + testreflection "${full_test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "testreflection") + set(test_disabled FALSE) + set(test_will_fail FALSE) + + # Check if test should be skipped + if(test_should_skip) + set(test_disabled TRUE) + set(test_labels "testreflection;skipped") + endif() + + # Check if test is expected to fail (xfail) + if(full_test_id IN_LIST xfail_list) + set(test_labels "testreflection;xfail") + set(test_will_fail TRUE) + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test" + LABELS "${test_labels}" + DISABLED ${test_disabled} + WILL_FAIL ${test_will_fail} + ) + endif() + endforeach() + + # Check for 3-digit subtests + file(GLOB subtests_3d "${base_test}/[0-9][0-9][0-9]") + foreach(subtest ${subtests_3d}) + if(EXISTS "${subtest}/===") + get_filename_component(subtest_id "${subtest}" NAME) + set(full_test_id "${test_id}/${subtest_id}") + + # Read description from === file + file(READ "${subtest}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "testreflection/${full_test_id} - ${test_desc}") + else() + set(test_name_full "testreflection/${full_test_id}") + endif() + + # Check if test should be skipped based on environment + should_skip_test("${subtest}" test_should_skip) + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + testreflection "${full_test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "testreflection") + set(test_disabled FALSE) + set(test_will_fail FALSE) + + # Check if test should be skipped + if(test_should_skip) + set(test_disabled TRUE) + set(test_labels "testreflection;skipped") + endif() + + # Check if test is expected to fail (xfail) + if(full_test_id IN_LIST xfail_list) + set(test_labels "testreflection;xfail") + set(test_will_fail TRUE) + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test" + LABELS "${test_labels}" + DISABLED ${test_disabled} + WILL_FAIL ${test_will_fail} + ) + endif() + endforeach() + endforeach() +endfunction() diff --git a/cmake/config.h.in b/cmake/config.h.in new file mode 100644 index 0000000..5f21f9d --- /dev/null +++ b/cmake/config.h.in @@ -0,0 +1,87 @@ +/* config.h - Generated by CMake */ + +#ifndef CONFIG_H +#define CONFIG_H + +/* Define to 1 if you have alloca.h */ +#cmakedefine HAVE_ALLOCA_H + +/* Define to 1 if you have byteswap.h */ +#cmakedefine HAVE_BYTESWAP_H + +/* Define to 1 if you have __builtin_bswap16 */ +#cmakedefine HAVE___BUILTIN_BSWAP16 + +/* Define to 1 if you have __builtin_bswap32 */ +#cmakedefine HAVE___BUILTIN_BSWAP32 + +/* Define to 1 if you have __builtin_bswap64 */ +#cmakedefine HAVE___BUILTIN_BSWAP64 + +/* Define to 1 if you have qsort_r available */ +#cmakedefine01 HAVE_QSORT_R + +/* Define to 1 if you have mremap available */ +#cmakedefine01 HAVE_MREMAP + +/* Define to 1 if you have environ declared */ +#cmakedefine HAVE_DECL_ENVIRON + +/* Define to 1 if you have libyaml available */ +#cmakedefine HAVE_LIBYAML + +/* Define to 1 if you have check available */ +#cmakedefine HAVE_CHECK + +/* Define to 1 if you have a compatible version of check available */ +#cmakedefine HAVE_COMPATIBLE_CHECK + +/* Define to 1 if static linking is available */ +#cmakedefine HAVE_STATIC + +/* Define to 1 if tools are static */ +#cmakedefine HAVE_STATIC_TOOLS + +/* Define to 1 if ASAN is enabled */ +#cmakedefine HAVE_ASAN + +/* Define to 1 if developer mode is enabled */ +#cmakedefine HAVE_DEVMODE + +/* Define to 1 if PORTABLE_TARGET is enabled */ +#cmakedefine HAVE_PORTABLE_TARGET + +/* SSE2 target support */ +#cmakedefine01 TARGET_HAS_SSE2 + +/* SSE41 target support */ +#cmakedefine01 TARGET_HAS_SSE41 + +/* AVX2 target support */ +#cmakedefine01 TARGET_HAS_AVX2 + +/* AVX512 target support */ +#cmakedefine01 TARGET_HAS_AVX512 + +/* NEON target support */ +#cmakedefine01 TARGET_HAS_NEON + +/* Define to 1 if you have git available */ +#cmakedefine HAVE_GIT + +/* Define to 1 if you have jq available */ +#cmakedefine HAVE_JQ + +/* Define to 1 if you have libclang available */ +#cmakedefine01 HAVE_LIBCLANG + +/* Define to 1 if Clang (or AppleClang) Blocks are available */ +#cmakedefine HAVE_CLANG_BLOCKS @HAVE_CLANG_BLOCKS@ + +/* Define to 1 if GCC heap trampolines are enabled */ +#cmakedefine01 HAVE_HEAP_TRAMPOLINES + +/* Version string */ +#define VERSION "@PROJECT_VERSION@" + +#endif /* CONFIG_H */ diff --git a/cmake/fyaml-config.cmake.in b/cmake/fyaml-config.cmake.in new file mode 100644 index 0000000..065660d --- /dev/null +++ b/cmake/fyaml-config.cmake.in @@ -0,0 +1,6 @@ +@PACKAGE_INIT@ + +# Any dependency? +#find_package(...) + +include(${CMAKE_CURRENT_LIST_DIR}/fyaml-targets.cmake) diff --git a/cmake/generate-jsontestsuite-list.sh b/cmake/generate-jsontestsuite-list.sh new file mode 100755 index 0000000..d700f0b --- /dev/null +++ b/cmake/generate-jsontestsuite-list.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash + +# Script to generate a CMake fragment listing all tests in the JSON test suite +# This allows registering tests even before the test data is downloaded + +set -e + +# Default values +JSONTESTSUITEURL="${JSONTESTSUITEURL:-https://github.com/nst/JSONTestSuite}" +JSONTESTSUITECHECKOUT="${JSONTESTSUITECHECKOUT:-d64aefb55228d9584d3e5b2433f720ea8fd00c82}" +OUTPUT_FILE="${1:-jsontestsuite-tests.cmake}" + +usage() { + cat << EOF +Usage: $0 [OUTPUT_FILE] + +Generate a CMake fragment listing all tests in the JSON test suite. + +Arguments: + OUTPUT_FILE Output file path (default: jsontestsuite-tests.cmake) + +Environment variables: + JSONTESTSUITEURL URL of the test suite repository + JSONTESTSUITECHECKOUT Git commit/tag to use + +Example: + $0 cmake/jsontestsuite-tests.cmake +EOF + exit 1 +} + +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage +fi + +echo "Generating jsontestsuite list from ${JSONTESTSUITEURL} @ ${JSONTESTSUITECHECKOUT}" + +# Create a temporary directory +TMPDIR=$(mktemp -d) +trap "rm -rf $TMPDIR" EXIT + +# Check if we have a local json-test-suite-data directory to use +LOCAL_DATA="" +for dir in "build/test/json-test-suite-data/test_parsing" "json-test-suite-data/test_parsing" "../build/test/json-test-suite-data/test_parsing"; do + if [ -d "$dir" ]; then + LOCAL_DATA="$dir" + echo "Using local test data from: $LOCAL_DATA" + break + fi +done + +if [ -n "$LOCAL_DATA" ]; then + # Use local data + find "$LOCAL_DATA" -maxdepth 1 -type f -name 'y_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/pass_tests.txt" + find "$LOCAL_DATA" -maxdepth 1 -type f -name 'n_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/fail_tests.txt" + find "$LOCAL_DATA" -maxdepth 1 -type f -name 'i_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/impl_tests.txt" +else + # Clone from git + echo "No local data found, cloning from git repository..." + cd "$TMPDIR" + git clone --depth 1 --filter=blob:none --sparse "${JSONTESTSUITEURL}" repo + cd repo + git checkout "${JSONTESTSUITECHECKOUT}" + git sparse-checkout set test_parsing + git sparse-checkout disable + + # List test files + find test_parsing -maxdepth 1 -type f -name 'y_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/pass_tests.txt" + find test_parsing -maxdepth 1 -type f -name 'n_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/fail_tests.txt" + find test_parsing -maxdepth 1 -type f -name 'i_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/impl_tests.txt" +fi + +# Read test counts +PASS_COUNT=$(wc -l < "$TMPDIR/pass_tests.txt") +FAIL_COUNT=$(wc -l < "$TMPDIR/fail_tests.txt") +IMPL_COUNT=$(wc -l < "$TMPDIR/impl_tests.txt") +TOTAL_COUNT=$((PASS_COUNT + FAIL_COUNT + IMPL_COUNT)) + +echo "Found $PASS_COUNT tests expected to pass (y_*.json)" +echo "Found $FAIL_COUNT tests expected to fail (n_*.json)" +echo "Found $IMPL_COUNT implementation-defined tests (i_*.json)" +echo "Total: $TOTAL_COUNT tests" + +# Generate CMake fragment +cat > "$OUTPUT_FILE" << 'EOF' +# Auto-generated file - do not edit manually +# Generated by cmake/generate-jsontestsuite-list.sh +# +# This file contains lists of tests from the JSON test suite +# It allows registering individual tests without requiring the test data +# to be downloaded first. + +EOF + +echo "# Test suite checkout: ${JSONTESTSUITECHECKOUT}" >> "$OUTPUT_FILE" +echo "# Generated on: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> "$OUTPUT_FILE" +echo "# Pass tests: ${PASS_COUNT}" >> "$OUTPUT_FILE" +echo "# Fail tests: ${FAIL_COUNT}" >> "$OUTPUT_FILE" +echo "# Impl tests: ${IMPL_COUNT}" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +# Write pass tests +echo "# Tests expected to pass (y_*.json)" >> "$OUTPUT_FILE" +echo "set(JSONTESTSUITE_PASS_TESTS" >> "$OUTPUT_FILE" +while IFS= read -r test; do + echo " \"$test\"" >> "$OUTPUT_FILE" +done < "$TMPDIR/pass_tests.txt" +echo ")" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +# Write fail tests +echo "# Tests expected to fail (n_*.json)" >> "$OUTPUT_FILE" +echo "set(JSONTESTSUITE_FAIL_TESTS" >> "$OUTPUT_FILE" +while IFS= read -r test; do + echo " \"$test\"" >> "$OUTPUT_FILE" +done < "$TMPDIR/fail_tests.txt" +echo ")" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +# Write impl tests +echo "# Implementation-defined tests (i_*.json)" >> "$OUTPUT_FILE" +echo "set(JSONTESTSUITE_IMPL_TESTS" >> "$OUTPUT_FILE" +while IFS= read -r test; do + echo " \"$test\"" >> "$OUTPUT_FILE" +done < "$TMPDIR/impl_tests.txt" +echo ")" >> "$OUTPUT_FILE" + +echo "Generated $OUTPUT_FILE successfully!" +echo "Total tests: $TOTAL_COUNT" diff --git a/cmake/generate-testsuite-list.sh b/cmake/generate-testsuite-list.sh new file mode 100755 index 0000000..91eed21 --- /dev/null +++ b/cmake/generate-testsuite-list.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash + +# Script to generate a CMake fragment listing all tests in the YAML test suite +# This allows registering tests even before the test data is downloaded + +set -e + +# Default values +TESTSUITEURL="${TESTSUITEURL:-https://github.com/yaml/yaml-test-suite}" +TESTSUITECHECKOUT="${TESTSUITECHECKOUT:-6e6c296ae9c9d2d5c4134b4b64d01b29ac19ff6f}" +OUTPUT_FILE="${1:-testsuite-tests.cmake}" + +usage() { + cat << EOF +Usage: $0 [OUTPUT_FILE] + +Generate a CMake fragment listing all tests in the YAML test suite. + +Arguments: + OUTPUT_FILE Output file path (default: testsuite-tests.cmake) + +Environment variables: + TESTSUITEURL URL of the test suite repository + TESTSUITECHECKOUT Git commit/tag to use + +Example: + $0 cmake/testsuite-tests.cmake +EOF + exit 1 +} + +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage +fi + +echo "Generating testsuite list from ${TESTSUITEURL} @ ${TESTSUITECHECKOUT}" + +# Create a temporary directory +TMPDIR=$(mktemp -d) +trap "rm -rf $TMPDIR" EXIT + +# Check if we have a local test-suite-data directory to use +LOCAL_DATA="" +for dir in "build/test/test-suite-data" "test-suite-data" "../build/test/test-suite-data"; do + if [ -d "$dir" ]; then + LOCAL_DATA="$dir" + echo "Using local test data from: $LOCAL_DATA" + break + fi +done + +if [ -n "$LOCAL_DATA" ]; then + # Use local data + find "$LOCAL_DATA" -maxdepth 1 -type d -name '[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]' | \ + xargs -n1 basename | sort > "$TMPDIR/base_tests.txt" + + # For each base test, check for subtests + > "$TMPDIR/all_tests.txt" + while IFS= read -r base; do + # Check for base test + if [ -f "$LOCAL_DATA/$base/===" ]; then + echo "$base" >> "$TMPDIR/all_tests.txt" + fi + + # Check for 2-digit subtests + for subtest in "$LOCAL_DATA/$base"/[0-9][0-9]; do + if [ -d "$subtest" ] && [ -f "$subtest/===" ]; then + echo "$base/$(basename $subtest)" >> "$TMPDIR/all_tests.txt" + fi + done + + # Check for 3-digit subtests + for subtest in "$LOCAL_DATA/$base"/[0-9][0-9][0-9]; do + if [ -d "$subtest" ] && [ -f "$subtest/===" ]; then + echo "$base/$(basename $subtest)" >> "$TMPDIR/all_tests.txt" + fi + done + done < "$TMPDIR/base_tests.txt" +else + # Clone from git + echo "No local data found, cloning from git repository..." + cd "$TMPDIR" + git clone --depth 1 --filter=blob:none --sparse "${TESTSUITEURL}" repo + cd repo + git checkout "${TESTSUITECHECKOUT}" + git sparse-checkout disable + + # List all base test directories + find . -maxdepth 1 -type d -name '[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]' | \ + sed 's|^\./||' | sort > "$TMPDIR/base_tests.txt" + + # For each base test, check for subtests + > "$TMPDIR/all_tests.txt" + while IFS= read -r base; do + # Check for base test + if [ -f "$base/===" ]; then + echo "$base" >> "$TMPDIR/all_tests.txt" + fi + + # Check for 2-digit subtests + for subtest in "$base"/[0-9][0-9]; do + if [ -d "$subtest" ] && [ -f "$subtest/===" ]; then + echo "$base/$(basename $subtest)" >> "$TMPDIR/all_tests.txt" + fi + done + + # Check for 3-digit subtests + for subtest in "$base"/[0-9][0-9][0-9]; do + if [ -d "$subtest" ] && [ -f "$subtest/===" ]; then + echo "$base/$(basename $subtest)" >> "$TMPDIR/all_tests.txt" + fi + done + done < "$TMPDIR/base_tests.txt" +fi + +# Read test counts +BASE_COUNT=$(wc -l < "$TMPDIR/base_tests.txt") +if [ -f "$TMPDIR/all_tests.txt" ]; then + ALL_COUNT=$(wc -l < "$TMPDIR/all_tests.txt") +else + ALL_COUNT=0 +fi + +echo "Found $BASE_COUNT base tests" +echo "Found $ALL_COUNT total tests (including subtests)" + +# Generate CMake fragment +cat > "$OUTPUT_FILE" << 'EOF' +# Auto-generated file - do not edit manually +# Generated by cmake/generate-testsuite-list.sh +# +# This file contains lists of tests from the YAML test suite +# It allows registering individual tests without requiring the test data +# to be downloaded first. + +EOF + +echo "# Test suite checkout: ${TESTSUITECHECKOUT}" >> "$OUTPUT_FILE" +echo "# Generated on: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> "$OUTPUT_FILE" +echo "# Base tests: ${BASE_COUNT}" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +# Write base test list +echo "# List of base test IDs" >> "$OUTPUT_FILE" +echo "set(TESTSUITE_BASE_TESTS" >> "$OUTPUT_FILE" +while IFS= read -r test; do + echo " $test" >> "$OUTPUT_FILE" +done < "$TMPDIR/base_tests.txt" +echo ")" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +if [ -f "$TMPDIR/all_tests.txt" ]; then + echo "# List of all tests (including subtests)" >> "$OUTPUT_FILE" + echo "# Format: BASE or BASE/SUBTEST" >> "$OUTPUT_FILE" + echo "set(TESTSUITE_ALL_TESTS" >> "$OUTPUT_FILE" + while IFS= read -r test; do + echo " \"$test\"" >> "$OUTPUT_FILE" + done < "$TMPDIR/all_tests.txt" + echo ")" >> "$OUTPUT_FILE" +fi + +echo "Generated $OUTPUT_FILE successfully!" +echo "Total base tests: $BASE_COUNT" +[ $ALL_COUNT -gt 0 ] && echo "Total tests (with subtests): $ALL_COUNT" diff --git a/cmake/jsontestsuite-tests.cmake b/cmake/jsontestsuite-tests.cmake new file mode 100644 index 0000000..3737900 --- /dev/null +++ b/cmake/jsontestsuite-tests.cmake @@ -0,0 +1,342 @@ +# Auto-generated file - do not edit manually +# Generated by cmake/generate-jsontestsuite-list.sh +# +# This file contains lists of tests from the JSON test suite +# It allows registering individual tests without requiring the test data +# to be downloaded first. + +# Test suite checkout: d64aefb55228d9584d3e5b2433f720ea8fd00c82 +# Generated on: 2025-10-24 08:05:59 UTC +# Pass tests: 95 +# Fail tests: 188 +# Impl tests: 35 + +# Tests expected to pass (y_*.json) +set(JSONTESTSUITE_PASS_TESTS + "y_array_arraysWithSpaces.json" + "y_array_empty.json" + "y_array_empty-string.json" + "y_array_ending_with_newline.json" + "y_array_false.json" + "y_array_heterogeneous.json" + "y_array_null.json" + "y_array_with_1_and_newline.json" + "y_array_with_leading_space.json" + "y_array_with_several_null.json" + "y_array_with_trailing_space.json" + "y_number_0e+1.json" + "y_number_0e1.json" + "y_number_after_space.json" + "y_number_double_close_to_zero.json" + "y_number_int_with_exp.json" + "y_number.json" + "y_number_minus_zero.json" + "y_number_negative_int.json" + "y_number_negative_one.json" + "y_number_negative_zero.json" + "y_number_real_capital_e.json" + "y_number_real_capital_e_neg_exp.json" + "y_number_real_capital_e_pos_exp.json" + "y_number_real_exponent.json" + "y_number_real_fraction_exponent.json" + "y_number_real_neg_exp.json" + "y_number_real_pos_exponent.json" + "y_number_simple_int.json" + "y_number_simple_real.json" + "y_object_basic.json" + "y_object_duplicated_key_and_value.json" + "y_object_duplicated_key.json" + "y_object_empty.json" + "y_object_empty_key.json" + "y_object_escaped_null_in_key.json" + "y_object_extreme_numbers.json" + "y_object.json" + "y_object_long_strings.json" + "y_object_simple.json" + "y_object_string_unicode.json" + "y_object_with_newlines.json" + "y_string_1_2_3_bytes_UTF-8_sequences.json" + "y_string_accepted_surrogate_pair.json" + "y_string_accepted_surrogate_pairs.json" + "y_string_allowed_escapes.json" + "y_string_backslash_and_u_escaped_zero.json" + "y_string_backslash_doublequotes.json" + "y_string_comments.json" + "y_string_double_escape_a.json" + "y_string_double_escape_n.json" + "y_string_escaped_control_character.json" + "y_string_escaped_noncharacter.json" + "y_string_in_array.json" + "y_string_in_array_with_leading_space.json" + "y_string_last_surrogates_1_and_2.json" + "y_string_nbsp_uescaped.json" + "y_string_nonCharacterInUTF-8_U+10FFFF.json" + "y_string_nonCharacterInUTF-8_U+FFFF.json" + "y_string_null_escape.json" + "y_string_one-byte-utf-8.json" + "y_string_pi.json" + "y_string_reservedCharacterInUTF-8_U+1BFFF.json" + "y_string_simple_ascii.json" + "y_string_space.json" + "y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json" + "y_string_three-byte-utf-8.json" + "y_string_two-byte-utf-8.json" + "y_string_u+2028_line_sep.json" + "y_string_u+2029_par_sep.json" + "y_string_uescaped_newline.json" + "y_string_uEscape.json" + "y_string_unescaped_char_delete.json" + "y_string_unicode_2.json" + "y_string_unicodeEscapedBackslash.json" + "y_string_unicode_escaped_double_quote.json" + "y_string_unicode.json" + "y_string_unicode_U+10FFFE_nonchar.json" + "y_string_unicode_U+1FFFE_nonchar.json" + "y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json" + "y_string_unicode_U+2064_invisible_plus.json" + "y_string_unicode_U+FDD0_nonchar.json" + "y_string_unicode_U+FFFE_nonchar.json" + "y_string_utf8.json" + "y_string_with_del_character.json" + "y_structure_lonely_false.json" + "y_structure_lonely_int.json" + "y_structure_lonely_negative_real.json" + "y_structure_lonely_null.json" + "y_structure_lonely_string.json" + "y_structure_lonely_true.json" + "y_structure_string_empty.json" + "y_structure_trailing_newline.json" + "y_structure_true_in_array.json" + "y_structure_whitespace_array.json" +) + +# Tests expected to fail (n_*.json) +set(JSONTESTSUITE_FAIL_TESTS + "n_array_1_true_without_comma.json" + "n_array_a_invalid_utf8.json" + "n_array_colon_instead_of_comma.json" + "n_array_comma_after_close.json" + "n_array_comma_and_number.json" + "n_array_double_comma.json" + "n_array_double_extra_comma.json" + "n_array_extra_close.json" + "n_array_extra_comma.json" + "n_array_incomplete_invalid_value.json" + "n_array_incomplete.json" + "n_array_inner_array_no_comma.json" + "n_array_invalid_utf8.json" + "n_array_items_separated_by_semicolon.json" + "n_array_just_comma.json" + "n_array_just_minus.json" + "n_array_missing_value.json" + "n_array_newlines_unclosed.json" + "n_array_number_and_comma.json" + "n_array_number_and_several_commas.json" + "n_array_spaces_vertical_tab_formfeed.json" + "n_array_star_inside.json" + "n_array_unclosed.json" + "n_array_unclosed_trailing_comma.json" + "n_array_unclosed_with_new_lines.json" + "n_array_unclosed_with_object_inside.json" + "n_incomplete_false.json" + "n_incomplete_null.json" + "n_incomplete_true.json" + "n_multidigit_number_then_00.json" + "n_number_0.1.2.json" + "n_number_-01.json" + "n_number_0.3e+.json" + "n_number_0.3e.json" + "n_number_0_capital_E+.json" + "n_number_0_capital_E.json" + "n_number_0.e1.json" + "n_number_0e+.json" + "n_number_0e.json" + "n_number_1_000.json" + "n_number_1.0e+.json" + "n_number_1.0e-.json" + "n_number_1.0e.json" + "n_number_-1.0..json" + "n_number_1eE2.json" + "n_number_+1.json" + "n_number_.-1.json" + "n_number_2.e+3.json" + "n_number_2.e-3.json" + "n_number_2.e3.json" + "n_number_.2e-3.json" + "n_number_-2..json" + "n_number_9.e+.json" + "n_number_expression.json" + "n_number_hex_1_digit.json" + "n_number_hex_2_digits.json" + "n_number_infinity.json" + "n_number_+Inf.json" + "n_number_Inf.json" + "n_number_invalid+-.json" + "n_number_invalid-negative-real.json" + "n_number_invalid-utf-8-in-bigger-int.json" + "n_number_invalid-utf-8-in-exponent.json" + "n_number_invalid-utf-8-in-int.json" + "n_number_++.json" + "n_number_minus_infinity.json" + "n_number_minus_sign_with_trailing_garbage.json" + "n_number_minus_space_1.json" + "n_number_-NaN.json" + "n_number_NaN.json" + "n_number_neg_int_starting_with_zero.json" + "n_number_neg_real_without_int_part.json" + "n_number_neg_with_garbage_at_end.json" + "n_number_real_garbage_after_e.json" + "n_number_real_with_invalid_utf8_after_e.json" + "n_number_real_without_fractional_part.json" + "n_number_starting_with_dot.json" + "n_number_U+FF11_fullwidth_digit_one.json" + "n_number_with_alpha_char.json" + "n_number_with_alpha.json" + "n_number_with_leading_zero.json" + "n_object_bad_value.json" + "n_object_bracket_key.json" + "n_object_comma_instead_of_colon.json" + "n_object_double_colon.json" + "n_object_emoji.json" + "n_object_garbage_at_end.json" + "n_object_key_with_single_quotes.json" + "n_object_lone_continuation_byte_in_key_and_trailing_comma.json" + "n_object_missing_colon.json" + "n_object_missing_key.json" + "n_object_missing_semicolon.json" + "n_object_missing_value.json" + "n_object_no-colon.json" + "n_object_non_string_key_but_huge_number_instead.json" + "n_object_non_string_key.json" + "n_object_repeated_null_null.json" + "n_object_several_trailing_commas.json" + "n_object_single_quote.json" + "n_object_trailing_comma.json" + "n_object_trailing_comment.json" + "n_object_trailing_comment_open.json" + "n_object_trailing_comment_slash_open_incomplete.json" + "n_object_trailing_comment_slash_open.json" + "n_object_two_commas_in_a_row.json" + "n_object_unquoted_key.json" + "n_object_unterminated-value.json" + "n_object_with_single_string.json" + "n_object_with_trailing_garbage.json" + "n_single_space.json" + "n_string_1_surrogate_then_escape.json" + "n_string_1_surrogate_then_escape_u1.json" + "n_string_1_surrogate_then_escape_u1x.json" + "n_string_1_surrogate_then_escape_u.json" + "n_string_accentuated_char_no_quotes.json" + "n_string_backslash_00.json" + "n_string_escaped_backslash_bad.json" + "n_string_escaped_ctrl_char_tab.json" + "n_string_escaped_emoji.json" + "n_string_escape_x.json" + "n_string_incomplete_escaped_character.json" + "n_string_incomplete_escape.json" + "n_string_incomplete_surrogate_escape_invalid.json" + "n_string_incomplete_surrogate.json" + "n_string_invalid_backslash_esc.json" + "n_string_invalid_unicode_escape.json" + "n_string_invalid_utf8_after_escape.json" + "n_string_invalid-utf-8-in-escape.json" + "n_string_leading_uescaped_thinspace.json" + "n_string_no_quotes_with_bad_escape.json" + "n_string_single_doublequote.json" + "n_string_single_quote.json" + "n_string_single_string_no_double_quotes.json" + "n_string_start_escape_unclosed.json" + "n_string_unescaped_ctrl_char.json" + "n_string_unescaped_newline.json" + "n_string_unescaped_tab.json" + "n_string_unicode_CapitalU.json" + "n_string_with_trailing_garbage.json" + "n_structure_100000_opening_arrays.json" + "n_structure_angle_bracket_..json" + "n_structure_angle_bracket_null.json" + "n_structure_array_trailing_garbage.json" + "n_structure_array_with_extra_array_close.json" + "n_structure_array_with_unclosed_string.json" + "n_structure_ascii-unicode-identifier.json" + "n_structure_capitalized_True.json" + "n_structure_close_unopened_array.json" + "n_structure_comma_instead_of_closing_brace.json" + "n_structure_double_array.json" + "n_structure_end_array.json" + "n_structure_incomplete_UTF8_BOM.json" + "n_structure_lone-invalid-utf-8.json" + "n_structure_lone-open-bracket.json" + "n_structure_no_data.json" + "n_structure_null-byte-outside-string.json" + "n_structure_number_with_trailing_garbage.json" + "n_structure_object_followed_by_closing_object.json" + "n_structure_object_unclosed_no_value.json" + "n_structure_object_with_comment.json" + "n_structure_object_with_trailing_garbage.json" + "n_structure_open_array_apostrophe.json" + "n_structure_open_array_comma.json" + "n_structure_open_array_object.json" + "n_structure_open_array_open_object.json" + "n_structure_open_array_open_string.json" + "n_structure_open_array_string.json" + "n_structure_open_object_close_array.json" + "n_structure_open_object_comma.json" + "n_structure_open_object.json" + "n_structure_open_object_open_array.json" + "n_structure_open_object_open_string.json" + "n_structure_open_object_string_with_apostrophes.json" + "n_structure_open_open.json" + "n_structure_single_eacute.json" + "n_structure_single_star.json" + "n_structure_trailing_#.json" + "n_structure_U+2060_word_joined.json" + "n_structure_uescaped_LF_before_string.json" + "n_structure_unclosed_array.json" + "n_structure_unclosed_array_partial_null.json" + "n_structure_unclosed_array_unfinished_false.json" + "n_structure_unclosed_array_unfinished_true.json" + "n_structure_unclosed_object.json" + "n_structure_unicode-identifier.json" + "n_structure_UTF8_BOM_no_data.json" + "n_structure_whitespace_formfeed.json" + "n_structure_whitespace_U+2060_word_joiner.json" +) + +# Implementation-defined tests (i_*.json) +set(JSONTESTSUITE_IMPL_TESTS + "i_number_double_huge_neg_exp.json" + "i_number_huge_exp.json" + "i_number_neg_int_huge_exp.json" + "i_number_pos_double_huge_exp.json" + "i_number_real_neg_overflow.json" + "i_number_real_pos_overflow.json" + "i_number_real_underflow.json" + "i_number_too_big_neg_int.json" + "i_number_too_big_pos_int.json" + "i_number_very_big_negative_int.json" + "i_object_key_lone_2nd_surrogate.json" + "i_string_1st_surrogate_but_2nd_missing.json" + "i_string_1st_valid_surrogate_2nd_invalid.json" + "i_string_incomplete_surrogate_and_escape_valid.json" + "i_string_incomplete_surrogate_pair.json" + "i_string_incomplete_surrogates_escape_valid.json" + "i_string_invalid_lonely_surrogate.json" + "i_string_invalid_surrogate.json" + "i_string_invalid_utf-8.json" + "i_string_inverted_surrogates_U+1D11E.json" + "i_string_iso_latin_1.json" + "i_string_lone_second_surrogate.json" + "i_string_lone_utf8_continuation_byte.json" + "i_string_not_in_unicode_range.json" + "i_string_overlong_sequence_2_bytes.json" + "i_string_overlong_sequence_6_bytes.json" + "i_string_overlong_sequence_6_bytes_null.json" + "i_string_truncated-utf-8.json" + "i_string_utf16BE_no_BOM.json" + "i_string_utf16LE_no_BOM.json" + "i_string_UTF-16LE_with_BOM.json" + "i_string_UTF-8_invalid_sequence.json" + "i_string_UTF8_surrogate_U+D800.json" + "i_structure_500_nested_arrays.json" + "i_structure_UTF-8_BOM_empty_object.json" +) diff --git a/cmake/libfyaml-config.cmake.in b/cmake/libfyaml-config.cmake.in new file mode 100644 index 0000000..d67173a --- /dev/null +++ b/cmake/libfyaml-config.cmake.in @@ -0,0 +1,54 @@ +@PACKAGE_INIT@ + +# libfyaml CMake package configuration file +# This file provides libfyaml imported targets and dependencies + +include(CMakeFindDependencyMacro) + +# Provide version information +set(libfyaml_VERSION "@PROJECT_VERSION@") +set(libfyaml_VERSION_MAJOR "@VERSION_MAJOR@") +set(libfyaml_VERSION_MINOR "@VERSION_MINOR@") +set(libfyaml_VERSION_PATCH "@VERSION_PATCH@") + +# Provide feature flags (what was enabled when libfyaml was built) +set(libfyaml_HAS_LIBCLANG "@HAVE_LIBCLANG@") +set(libfyaml_HAS_LIBYAML "@HAVE_LIBYAML@") + +# Required dependency: Threads +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_dependency(Threads REQUIRED) + +# Optional dependency: libclang (only if libfyaml was built with reflection support) +if(libfyaml_HAS_LIBCLANG) + # Try CMake config first + find_dependency(LLVM QUIET CONFIG) + if(LLVM_FOUND) + # Help find Clang in the same LLVM installation + set(Clang_DIR "${LLVM_INSTALL_PREFIX}/lib/cmake/clang") + endif() + find_dependency(Clang QUIET CONFIG) + + # If CMake config not found, try pkg-config as fallback + if(NOT LLVM_FOUND OR NOT Clang_FOUND) + # User must provide libclang manually if needed + message(STATUS "libfyaml was built with libclang support, but libclang not found via CMake config") + message(STATUS " Set CMAKE_PREFIX_PATH or LLVM_DIR to locate LLVM/Clang") + endif() +endif() + +# Import libfyaml targets (defines libfyaml::fyaml) +include(${CMAKE_CURRENT_LIST_DIR}/libfyaml-targets.cmake) + +# Provide consistent naming: libfyaml::libfyaml as main target +if(NOT TARGET libfyaml::libfyaml AND TARGET libfyaml::fyaml) + add_library(libfyaml::libfyaml ALIAS libfyaml::fyaml) +endif() + +# Backward compatibility: provide non-namespaced target +if(NOT TARGET fyaml AND TARGET libfyaml::fyaml) + add_library(fyaml ALIAS libfyaml::fyaml) +endif() + +check_required_components(libfyaml) diff --git a/cmake/libfyaml.pc.in b/cmake/libfyaml.pc.in new file mode 100644 index 0000000..e57044c --- /dev/null +++ b/cmake/libfyaml.pc.in @@ -0,0 +1,10 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@ +includedir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@ + +Name: libfyaml +Description: Fancy YAML 1.3 parser library +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -lfyaml -lpthread +Cflags: -I${includedir} diff --git a/cmake/run-single-tap-test.sh b/cmake/run-single-tap-test.sh new file mode 100755 index 0000000..5382f56 --- /dev/null +++ b/cmake/run-single-tap-test.sh @@ -0,0 +1,314 @@ +#!/usr/bin/env bash + +# Wrapper script to run a single TAP subtest +# Usage: run-single-tap-test.sh + +test_suite="$1" +test_id="$2" + +case "$test_suite" in + testerrors) + dir="${SRCDIR}/test-errors/${test_id}" + desctxt=$(cat 2>/dev/null "$dir/===") + + t=$(mktemp) + res="not ok" + + pass_yaml=0 + ${TOP_BUILDDIR}/src/fy-tool --dump -r --no-streaming "$dir/in.yaml" >"$t" 2>&1 + if [ $? -eq 0 ]; then + pass_yaml=1 + fi + + errmsg=$(cat "$t" | head -n1 | sed -e 's/^[^:]*//') + echo "errmsg: $errmsg" >&2 + + # Replace with error message + echo "$errmsg" >"$t" + + # All error tests are expected to fail + if [ "$pass_yaml" == "0" ]; then + res="ok" + + # diff is pointless under valgrind + if [ "x$USE_VALGRIND" == "x" ]; then + diff_err=0 + diff -u "$dir/test.error" "$t" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + fi + else + res="not ok" + fi + + rm -f "$t" + + echo "$res 1 $test_id - $desctxt" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + testemitter) + f="${SRCDIR}/emitter-examples/${test_id}" + + t1=$(mktemp) + t2=$(mktemp) + + res="not ok" + + pass_parse=0 + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers "$f" >"$t1" + if [ $? -eq 0 ]; then + ${TOP_BUILDDIR}/src/fy-tool --dump "$f" | \ + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t2" + if [ $? -eq 0 ]; then + pass_parse=1 + fi + fi + + if [ "$pass_parse" == "1" ]; then + diff -u "$t1" "$t2" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + else + res="not ok" + fi + + rm -f "$t1" "$t2" + + echo "$res 1 - $test_id" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + testemitter-streaming) + f="${SRCDIR}/emitter-examples/${test_id}" + + t1=$(mktemp) + t2=$(mktemp) + + res="not ok" + + pass_parse=0 + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers "$f" >"$t1" + if [ $? -eq 0 ]; then + ${TOP_BUILDDIR}/src/fy-tool --dump --streaming "$f" | \ + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t2" + if [ $? -eq 0 ]; then + pass_parse=1 + fi + fi + + if [ "$pass_parse" == "1" ]; then + diff -u "$t1" "$t2" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + else + res="not ok" + fi + + rm -f "$t1" "$t2" + + echo "$res 1 - $test_id" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + testemitter-restreaming) + f="${SRCDIR}/emitter-examples/${test_id}" + + t1=$(mktemp) + t2=$(mktemp) + + res="not ok" + + pass_parse=0 + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers "$f" >"$t1" + if [ $? -eq 0 ]; then + ${TOP_BUILDDIR}/src/fy-tool --dump --streaming --recreating "$f" | \ + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t2" + if [ $? -eq 0 ]; then + pass_parse=1 + fi + fi + + if [ "$pass_parse" == "1" ]; then + diff -u "$t1" "$t2" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + else + res="not ok" + fi + + rm -f "$t1" "$t2" + + echo "$res 1 - $test_id" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + testsuite|testsuite-json|testsuite-resolution) + tst="test-suite-data/${test_id}" + desctxt=$(cat 2>/dev/null "$tst/===") + + t=$(mktemp) + + res="not ok" + + pass_yaml=0 + ${TOP_BUILDDIR}/src/fy-tool --testsuite "$tst/in.yaml" >"$t" + if [ $? -eq 0 ]; then + pass_yaml=1 + fi + + if [ -e "$tst/error" ]; then + # Test is expected to fail + if [ $pass_yaml == "0" ]; then + res="ok" + else + res="not ok" + fi + else + # Test is expected to pass + if [ $pass_yaml == "1" ]; then + diff -u "$tst/test.event" "$t" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + else + res="not ok" + fi + fi + + rm -f "$t" + + echo "$res 1 $test_id - $desctxt" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + testsuite-evstream) + tst="test-suite-data/${test_id}" + desctxt=$(cat 2>/dev/null "$tst/===") + + # Skip tests that are expected to fail + if [ -e "$tst/error" ]; then + # echo "ok 1 $test_id - $desctxt # SKIP expected to fail" + echo "skip 1 $test_id - $desctxt # SKIP expected to fail" + exit 0 + fi + + # xfails + case "${test_id}" in + 2JQS) + echo "skip 1 $test_id - $desctxt # SKIP expected to fail" + exit 0 + ;; + esac + + t=$(mktemp) + + res="not ok" + + # run the test using document-event-stream + ${TOP_BUILDDIR}/src/fy-tool --testsuite --document-event-stream "$tst/in.yaml" >"$t" 2>/dev/null + if [ $? -eq 0 ]; then + diff -u "$tst/test.event" "$t" + if [ $? -eq 0 ]; then + res="ok" + fi + fi + + rm -f "$t" + + echo "$res 1 $test_id - $desctxt" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + jsontestsuite) + tf="$test_id" + f="json-test-suite-data/test_parsing/${tf}" + + # Determine expected result based on prefix + case "$tf" in + y_*) + # Expected to pass + ${TOP_BUILDDIR}/src/fy-tool --testsuite --streaming "$f" >/dev/null 2>&1 + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + ;; + n_*) + # Expected to fail + ${TOP_BUILDDIR}/src/fy-tool --testsuite --streaming "$f" >/dev/null 2>&1 + if [ $? -eq 0 ]; then + res="not ok" + else + res="ok" + fi + ;; + i_*) + # Implementation defined - we'll accept either result as ok + ${TOP_BUILDDIR}/src/fy-tool --testsuite --streaming "$f" >/dev/null 2>&1 + res="ok" + ;; + *) + res="not ok" + ;; + esac + + echo "$res 1 - $tf" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + *) + echo "Unknown test suite: $test_suite" >&2 + exit 1 + ;; +esac diff --git a/cmake/setup-yaml-test-suite.cmake b/cmake/setup-yaml-test-suite.cmake new file mode 100644 index 0000000..c8c7f2b --- /dev/null +++ b/cmake/setup-yaml-test-suite.cmake @@ -0,0 +1,42 @@ +# Script to set up test suite data in an idempotent way + +if(NOT DEFINED REPO_NAME) + set(REPO_NAME "test-suite-data") +endif() + +set(REPO_DIR "${TEST_DIR}/${REPO_NAME}") + +# Check if the repository already exists +if(EXISTS "${REPO_DIR}/.git") + message(STATUS "Repository ${REPO_NAME} already exists, updating...") + execute_process( + COMMAND ${GIT_EXECUTABLE} fetch -q origin + WORKING_DIRECTORY ${REPO_DIR} + RESULT_VARIABLE GIT_FETCH_RESULT + ) + if(NOT GIT_FETCH_RESULT EQUAL 0) + message(WARNING "Failed to fetch updates for ${REPO_NAME}") + endif() +else() + message(STATUS "Cloning ${REPO_NAME}...") + execute_process( + COMMAND ${GIT_EXECUTABLE} clone -q "${TESTSUITEURL}" ${REPO_NAME} + WORKING_DIRECTORY ${TEST_DIR} + RESULT_VARIABLE GIT_CLONE_RESULT + ) + if(NOT GIT_CLONE_RESULT EQUAL 0) + message(FATAL_ERROR "Failed to clone ${REPO_NAME}") + endif() +endif() + +# Checkout the specific commit +execute_process( + COMMAND ${GIT_EXECUTABLE} checkout -q --detach ${TESTSUITECHECKOUT} + WORKING_DIRECTORY ${REPO_DIR} + RESULT_VARIABLE GIT_CHECKOUT_RESULT +) +if(NOT GIT_CHECKOUT_RESULT EQUAL 0) + message(FATAL_ERROR "Failed to checkout ${TESTSUITECHECKOUT} in ${REPO_NAME}") +endif() + +message(STATUS "Test suite ${REPO_NAME} set up successfully") diff --git a/cmake/tags.cmake b/cmake/tags.cmake new file mode 100644 index 0000000..88ee25f --- /dev/null +++ b/cmake/tags.cmake @@ -0,0 +1,21 @@ +find_program(CTAGS ctags) + +if(CTAGS) + # Name of the build directory (for exclusion) + get_filename_component(_build_dir_name "${CMAKE_BINARY_DIR}" NAME) + + add_custom_target(ctags + COMMAND ${CTAGS} + -R + --exclude=${_build_dir_name} + --exclude=.git + --exclude=.cache + --extra=+q + --c-kinds=+lpx + --fields=afikmsSzt + -f "${CMAKE_BINARY_DIR}/tags" + "${CMAKE_SOURCE_DIR}" + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + VERBATIM + ) +endif() diff --git a/cmake/testsuite-tests.cmake b/cmake/testsuite-tests.cmake new file mode 100644 index 0000000..284aead --- /dev/null +++ b/cmake/testsuite-tests.cmake @@ -0,0 +1,771 @@ +# Auto-generated file - do not edit manually +# Generated by cmake/generate-testsuite-list.sh +# +# This file contains lists of tests from the YAML test suite +# It allows registering individual tests without requiring the test data +# to be downloaded first. + +# Test suite checkout: 6e6c296ae9c9d2d5c4134b4b64d01b29ac19ff6f +# Generated on: 2025-10-24 08:04:38 UTC +# Base tests: 350 + +# List of base test IDs +set(TESTSUITE_BASE_TESTS + 229Q + 236B + 26DV + 27NA + 2AUY + 2CMS + 2EBW + 2G84 + 2JQS + 2LFX + 2SXE + 2XXW + 33X3 + 35KP + 36F6 + 3ALJ + 3GZX + 3HFZ + 3MYT + 3R3P + 3RLN + 3UYS + 4ABK + 4CQQ + 4EJS + 4FJ6 + 4GC6 + 4H7K + 4HVU + 4JVG + 4MUZ + 4Q9F + 4QFQ + 4RWC + 4UYU + 4V8U + 4WA9 + 4ZYM + 52DL + 54T7 + 55WF + 565N + 57H4 + 58MP + 5BVJ + 5C5M + 5GBF + 5KJE + 5LLU + 5MUD + 5NYZ + 5T43 + 5TRB + 5TYM + 5U3A + 5WE3 + 62EZ + 652Z + 65WH + 6BCT + 6BFJ + 6CA3 + 6CK3 + 6FWR + 6H3V + 6HB6 + 6JQW + 6JTT + 6JWB + 6KGN + 6LVF + 6M2F + 6PBE + 6S55 + 6SLA + 6VJK + 6WLZ + 6WPF + 6XDY + 6ZKB + 735Y + 74H7 + 753E + 7A4E + 7BMT + 7BUB + 7FWL + 7LBH + 7MNF + 7T8X + 7TMG + 7W2P + 7Z25 + 7ZZ5 + 82AN + 87E4 + 8CWC + 8G76 + 8KB6 + 8MK2 + 8QBE + 8UDB + 8XDJ + 8XYN + 93JH + 93WF + 96L6 + 96NN + 98YD + 9BXH + 9C9N + 9CWY + 9DXL + 9FMG + 9HCY + 9J7A + 9JBA + 9KAX + 9KBC + 9MAG + 9MMA + 9MMW + 9MQT + 9SA2 + 9SHH + 9TFX + 9U5K + 9WXW + 9YRD + A2M4 + A6F9 + A984 + AB8U + AVM7 + AZ63 + AZW3 + B3HG + B63P + BD7L + BEC7 + BF9H + BS4K + BU8L + C2DT + C2SP + C4HZ + CC74 + CFD4 + CML9 + CN3R + CPZ3 + CQ3W + CT4Q + CTN5 + CUP7 + CVW2 + CXX2 + D49Q + D83L + D88J + D9TU + DBG4 + DC7X + DE56 + DFF7 + DHP8 + DK3J + DK4H + DK95 + DMG6 + DWX9 + E76Z + EB22 + EHF6 + EW3V + EX5H + EXG3 + F2C7 + F3CP + F6MC + F8F9 + FBC9 + FH7J + FP8R + FQ7F + FRK4 + FTA2 + FUP4 + G4RS + G5U8 + G7JE + G992 + G9HC + GDY7 + GH63 + GT5M + H2RW + H3Z8 + H7J7 + H7TQ + HM87 + HMK4 + HMQ5 + HRE5 + HS5T + HU3P + HWV9 + J3BT + J5UC + J7PZ + J7VC + J9HZ + JEF9 + JHB9 + JKF3 + JQ4R + JR7V + JS2J + JTV5 + JY7Z + K3WX + K4SU + K527 + K54U + K858 + KH5V + KK5P + KMK3 + KS4U + KSS4 + L24T + L383 + L94M + L9U5 + LE5A + LHL4 + LP6E + LQZ7 + LX3P + M29M + M2N8 + M5C3 + M5DY + M6YH + M7A3 + M7NX + M9B4 + MJS9 + MUS6 + MXS3 + MYW6 + MZX3 + N4JP + N782 + NAT4 + NB6Z + NHX8 + NJ66 + NKF9 + NP9H + P2AD + P2EQ + P76L + P94K + PBJ2 + PRH3 + PUW8 + PW8X + Q4CL + Q5MG + Q88A + Q8AD + Q9WF + QB6E + QF4Y + QLJ7 + QT73 + R4YG + R52L + RHX7 + RLU9 + RR7F + RTP8 + RXY3 + RZP5 + RZT7 + S3PD + S4GJ + S4JQ + S4T7 + S7BG + S98Z + S9E8 + SBG9 + SF5V + SKE5 + SM9W + SR86 + SSW6 + SU5Z + SU74 + SY6V + SYW4 + T26H + T4YY + T5N4 + T833 + TD5N + TE2A + TL85 + TS54 + U3C3 + U3XV + U44R + U99R + U9NS + UDM2 + UDR7 + UGM3 + UKK6 + UT92 + UV7Q + V55R + V9D5 + VJP3 + W42U + W4TN + W5VH + W9L4 + WZ62 + X38W + X4QW + X8DW + XLQ9 + XV9V + XW4D + Y2GN + Y79Y + YD5X + YJV2 + Z67P + Z9M4 + ZCZ6 + ZF4X + ZH7C + ZK9H + ZL4Z + ZVH3 + ZWK4 + ZXT5 +) + +# List of all tests (including subtests) +# Format: BASE or BASE/SUBTEST +set(TESTSUITE_ALL_TESTS + "229Q" + "236B" + "26DV" + "27NA" + "2AUY" + "2CMS" + "2EBW" + "2G84/00" + "2G84/01" + "2G84/02" + "2G84/03" + "2JQS" + "2LFX" + "2SXE" + "2XXW" + "33X3" + "35KP" + "36F6" + "3ALJ" + "3GZX" + "3HFZ" + "3MYT" + "3R3P" + "3RLN/00" + "3RLN/01" + "3RLN/02" + "3RLN/03" + "3RLN/04" + "3RLN/05" + "3UYS" + "4ABK" + "4CQQ" + "4EJS" + "4FJ6" + "4GC6" + "4H7K" + "4HVU" + "4JVG" + "4MUZ/00" + "4MUZ/01" + "4MUZ/02" + "4Q9F" + "4QFQ" + "4RWC" + "4UYU" + "4V8U" + "4WA9" + "4ZYM" + "52DL" + "54T7" + "55WF" + "565N" + "57H4" + "58MP" + "5BVJ" + "5C5M" + "5GBF" + "5KJE" + "5LLU" + "5MUD" + "5NYZ" + "5T43" + "5TRB" + "5TYM" + "5U3A" + "5WE3" + "62EZ" + "652Z" + "65WH" + "6BCT" + "6BFJ" + "6CA3" + "6CK3" + "6FWR" + "6H3V" + "6HB6" + "6JQW" + "6JTT" + "6JWB" + "6KGN" + "6LVF" + "6M2F" + "6PBE" + "6S55" + "6SLA" + "6VJK" + "6WLZ" + "6WPF" + "6XDY" + "6ZKB" + "735Y" + "74H7" + "753E" + "7A4E" + "7BMT" + "7BUB" + "7FWL" + "7LBH" + "7MNF" + "7T8X" + "7TMG" + "7W2P" + "7Z25" + "7ZZ5" + "82AN" + "87E4" + "8CWC" + "8G76" + "8KB6" + "8MK2" + "8QBE" + "8UDB" + "8XDJ" + "8XYN" + "93JH" + "93WF" + "96L6" + "96NN/00" + "96NN/01" + "98YD" + "9BXH" + "9C9N" + "9CWY" + "9DXL" + "9FMG" + "9HCY" + "9J7A" + "9JBA" + "9KAX" + "9KBC" + "9MAG" + "9MMA" + "9MMW" + "9MQT/00" + "9MQT/01" + "9SA2" + "9SHH" + "9TFX" + "9U5K" + "9WXW" + "9YRD" + "A2M4" + "A6F9" + "A984" + "AB8U" + "AVM7" + "AZ63" + "AZW3" + "B3HG" + "B63P" + "BD7L" + "BEC7" + "BF9H" + "BS4K" + "BU8L" + "C2DT" + "C2SP" + "C4HZ" + "CC74" + "CFD4" + "CML9" + "CN3R" + "CPZ3" + "CQ3W" + "CT4Q" + "CTN5" + "CUP7" + "CVW2" + "CXX2" + "D49Q" + "D83L" + "D88J" + "D9TU" + "DBG4" + "DC7X" + "DE56/00" + "DE56/01" + "DE56/02" + "DE56/03" + "DE56/04" + "DE56/05" + "DFF7" + "DHP8" + "DK3J" + "DK4H" + "DK95/00" + "DK95/01" + "DK95/02" + "DK95/03" + "DK95/04" + "DK95/05" + "DK95/06" + "DK95/07" + "DK95/08" + "DMG6" + "DWX9" + "E76Z" + "EB22" + "EHF6" + "EW3V" + "EX5H" + "EXG3" + "F2C7" + "F3CP" + "F6MC" + "F8F9" + "FBC9" + "FH7J" + "FP8R" + "FQ7F" + "FRK4" + "FTA2" + "FUP4" + "G4RS" + "G5U8" + "G7JE" + "G992" + "G9HC" + "GDY7" + "GH63" + "GT5M" + "H2RW" + "H3Z8" + "H7J7" + "H7TQ" + "HM87/00" + "HM87/01" + "HMK4" + "HMQ5" + "HRE5" + "HS5T" + "HU3P" + "HWV9" + "J3BT" + "J5UC" + "J7PZ" + "J7VC" + "J9HZ" + "JEF9/00" + "JEF9/01" + "JEF9/02" + "JHB9" + "JKF3" + "JQ4R" + "JR7V" + "JS2J" + "JTV5" + "JY7Z" + "K3WX" + "K4SU" + "K527" + "K54U" + "K858" + "KH5V/00" + "KH5V/01" + "KH5V/02" + "KK5P" + "KMK3" + "KS4U" + "KSS4" + "L24T/00" + "L24T/01" + "L383" + "L94M" + "L9U5" + "LE5A" + "LHL4" + "LP6E" + "LQZ7" + "LX3P" + "M29M" + "M2N8/00" + "M2N8/01" + "M5C3" + "M5DY" + "M6YH" + "M7A3" + "M7NX" + "M9B4" + "MJS9" + "MUS6/00" + "MUS6/01" + "MUS6/02" + "MUS6/03" + "MUS6/04" + "MUS6/05" + "MUS6/06" + "MXS3" + "MYW6" + "MZX3" + "N4JP" + "N782" + "NAT4" + "NB6Z" + "NHX8" + "NJ66" + "NKF9" + "NP9H" + "P2AD" + "P2EQ" + "P76L" + "P94K" + "PBJ2" + "PRH3" + "PUW8" + "PW8X" + "Q4CL" + "Q5MG" + "Q88A" + "Q8AD" + "Q9WF" + "QB6E" + "QF4Y" + "QLJ7" + "QT73" + "R4YG" + "R52L" + "RHX7" + "RLU9" + "RR7F" + "RTP8" + "RXY3" + "RZP5" + "RZT7" + "S3PD" + "S4GJ" + "S4JQ" + "S4T7" + "S7BG" + "S98Z" + "S9E8" + "SBG9" + "SF5V" + "SKE5" + "SM9W/00" + "SM9W/01" + "SR86" + "SSW6" + "SU5Z" + "SU74" + "SY6V" + "SYW4" + "T26H" + "T4YY" + "T5N4" + "T833" + "TD5N" + "TE2A" + "TL85" + "TS54" + "U3C3" + "U3XV" + "U44R" + "U99R" + "U9NS" + "UDM2" + "UDR7" + "UGM3" + "UKK6/00" + "UKK6/01" + "UKK6/02" + "UT92" + "UV7Q" + "V55R" + "V9D5" + "VJP3/00" + "VJP3/01" + "W42U" + "W4TN" + "W5VH" + "W9L4" + "WZ62" + "X38W" + "X4QW" + "X8DW" + "XLQ9" + "XV9V" + "XW4D" + "Y2GN" + "Y79Y/000" + "Y79Y/001" + "Y79Y/002" + "Y79Y/003" + "Y79Y/004" + "Y79Y/005" + "Y79Y/006" + "Y79Y/007" + "Y79Y/008" + "Y79Y/009" + "Y79Y/010" + "YD5X" + "YJV2" + "Z67P" + "Z9M4" + "ZCZ6" + "ZF4X" + "ZH7C" + "ZK9H" + "ZL4Z" + "ZVH3" + "ZWK4" + "ZXT5" +) diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..2854bd7 --- /dev/null +++ b/configure.ac @@ -0,0 +1,708 @@ +AC_PREREQ(2.69) + +AC_INIT([libfyaml], + m4_esyscmd([build-aux/git-version-gen .tarball-version]), + [pantelis.antoniou@konsulko.com]) + +AC_CONFIG_AUX_DIR([build-aux]) + +AC_PATH_PROG([M4], [m4 gm4], [no]) +if test "x$M4" = xno ; then + AC_MSG_ERROR([m4 missing]) +fi + +AC_SUBST(ACLOCAL_AMFLAGS, "-I m4") + +AC_CANONICAL_TARGET +AC_CANONICAL_HOST +AC_CANONICAL_BUILD + +AC_CONFIG_SRCDIR([src/lib/fy-parse.c]) +AC_CONFIG_HEADERS([config.h]) +AC_CONFIG_MACRO_DIR([m4]) +AM_INIT_AUTOMAKE([foreign 1.8.5 subdir-objects -Wno-portability]) + +m4_pattern_allow([^(AM_EXTRA_RECURSIVE_TARGETS|AM_PROG_AR)$])dnl + +# supoort older versions of automake +# only define the recursive targets when it's defined +# note that the top level makefile rules will not +# include them +m4_ifdef([AM_EXTRA_RECURSIVE_TARGETS], + [AM_EXTRA_RECURSIVE_TARGETS([doc-help doc-html doc-latexpdf doc-man doc-clean doc-markdown])], + [AC_MSG_WARN([Old automake version without AM_EXTRA_RECURSIVE_TARGETS, doc-* rules won't be available at top level])]) + +m4_define(fyaml_major, `echo $VERSION | cut -d. -f1 | cut -d- -f1`) +m4_define(fyaml_minor, `echo $VERSION | cut -d. -f2 | cut -d- -f1`) +m4_define(fyaml_patch, `echo $VERSION | cut -d. -f3- | cut -d- -f1`) +m4_define(fyaml_extra, `echo $VERSION | cut -d- -f2- -s`) + +AC_SUBST(MAJOR, fyaml_major) +AC_SUBST(MINOR, fyaml_minor) +AC_SUBST(PATCH, fyaml_patch) +AC_SUBST(EXTRA, fyaml_extra) + +# libtool version is of the following format :: +# and it is the library ABI version +# +# - Increase current when an interface has been added removed or changed +# - Increase revision every time a release is made +# - Increase age when changes are backwards compatible +# +# examples of semantic versioning progression mapping to libtool abi versions +# +# v0.7.4 -> v0.7.5 - patch number changes, backwards compatible (1) +# increase revision, increase age (age must be <= current), current unchanged +# +# v0.7.5 -> v0.8.0 - minor number changes, backwards compatible (same as (1)) (2) +# increase revision, increase age (age must be <= current), current unchanged +# +# v0.8.0 -> v0.9.0 - minor number changes, but breaks backwards compatibility - should not happen (3) +# with semantic versioning for major version number >= 1, allowed for major == 0 +# increase revision, increase current, age reset to 0 +# +# v0.9.0 -> v1.0.0 - major number changes, first public release (4) +# increase current, set revision and age to 0 +# +# v1.0.0 -> v1.0.1 - patch number changes, _must_ be backwards compatible (same as (1)) (5) +# increase revision, increase age (age must be <= current), current unchanged +# +# v1.0.1 -> v1.1.0 - minor number changes, backwards compatible (same as (5)) (6) +# increase revision, increase age (age must be <= current), current unchanged +# +# v1.1.0 -> v1.2.0 - minor number changes, breaking backwards compatibility (7) +# XXX illegal in semantic versioning and should not happen +# +# v1.1.0 -> v2.0.0 - major number changes, breaking backwards compatibility (same as (4)) (8) +# increase current, set revision and age to 0 + +m4_define(fyaml_libtool_version, m4_normalize(m4_include([.libtool-version]))) +AC_SUBST(LIBTOOL_VERSION, fyaml_libtool_version) + +dnl AX_* requires 2.64 +m4_version_prereq(2.64, [AX_CHECK_ENABLE_DEBUG()], [true]) + +AC_PROG_MKDIR_P +AC_PROG_CC +AM_PROG_CC_C_O +AM_PROG_AS + +AC_PROG_GCC_TRADITIONAL +AC_USE_SYSTEM_EXTENSIONS +m4_ifdef([AM_PROG_AR], [AM_PROG_AR], + [AC_MSG_WARN([Old automake version without AM_PROG_AR, library versioning will not be available])]) + +AC_PROG_CXX +AC_PROG_AWK +AC_PROG_LN_S + +AX_PTHREAD([], [AC_MSG_ERROR("Missing required pthread support")]) + +# in some cases PTHREAD_LIBS is empty - force -lpthread */ +if test "x$PTHREAD_LIBS" = "x"; then + PTHREAD_LIBS="-lpthread" +fi + +CFLAGS="$CFLAGS $PTHREAD_CFLAGS" +LIBS="$LIBS $PTHREAD_LIBS" + +# pkg-config +PKG_PROG_PKG_CONFIG + +AC_LTDL_ENABLE_INSTALL +LT_INIT([dlopen win32-dll]) +AC_SUBST(LTDLINCL) +AC_SUBST(LIBLTDL) +AC_CONFIG_SUBDIRS(libltdl) + +AC_C_CONST +AC_C_BIGENDIAN +AC_TYPE_PID_T +AC_TYPE_SIZE_T +AC_CHECK_TYPES(ssize_t, , [AC_DEFINE([ssize_t], [signed long], + [Define ssize_t if it is not done by the standard libs.])]) +AC_TYPE_OFF_T +AC_TYPE_UID_T +AC_CHECK_DECLS(environ) + +AC_CHECK_HEADERS([alloca.h byteswap.h]) +AC_CHECK_FUNCS([__builtin_bswap16 __builtin_bswap32 __builtin_bswap64]) + +dnl for old autoconf version AX_APPEND_COMPILE_FLAGS does not work +m4_version_prereq(2.64, + [AX_APPEND_COMPILE_FLAGS([-Wall -Wsign-compare -Wno-stringop-overflow \ + -fvisibility=hidden -Wno-unused-function \ + -Wno-tautological-constant-out-of-range-compare \ + -std=c2x], + [CFLAGS], [-pedantic -Werror])], + [CFLAGS="$CFLAGS -Wall -Wsign-compare"]) + +dnl -O2 is universal no need for AX_APPEND_COMPILE_FLAGS +if test "x$ax_enable_debug" != "xyes" ; then + CFLAGS="$CFLAGS -O2" +fi + +# need that as conditional because some internal tools compile as static +# to have access to internal libfyaml APIs +if test "x$enable_static" = "xyes" ; then + HAVE_STATIC=1 +else + HAVE_STATIC=0 +fi +AC_SUBST(HAVE_STATIC) +AC_DEFINE_UNQUOTED([HAVE_STATIC], [$HAVE_STATIC], [Define to 1 if static linking is available]) +AM_CONDITIONAL([HAVE_STATIC], [ test x$HAVE_STATIC = x1 ]) + +dnl check whether warning flags are supported +AX_CHECK_COMPILE_FLAG([-Wstringop-overread], , , [-Werror]) +AS_IF([test "x$ax_cv_check_cflags__Werror__Wstringop_overread" = x"yes"], + [HAVE_WSTRINGOP_OVERREAD=1], [HAVE_WSTRINGOP_OVERREAD=0]) +AC_DEFINE_UNQUOTED([HAVE_WSTRINGOP_OVERREAD], [$HAVE_WSTRINGOP_OVERREAD], [Define to 1 if -Wstringop-overread is supported]) + +AX_CHECK_COMPILE_FLAG([-Wstrict-aliasing], , , [-Werror]) +AS_IF([test "x$ax_cv_check_cflags__Werror__Wstrict_aliasing" = x"yes"], + [HAVE_WSTRICT_ALIASING=1], [HAVE_WSTRICT_ALIASING=0]) +AC_DEFINE_UNQUOTED([HAVE_WSTRICT_ALIASING], [$HAVE_WSTRICT_ALIASING], [Define to 1 if -Wstrict-aliasing is supported]) + +dnl Per target optimizations +AC_ARG_ENABLE([portable-target], + AS_HELP_STRING([--enable-portable-target], + [Enable portable mode (disable per-target optimizations)])) +HAVE_PORTABLE_TARGET=0 +if test "x$enable_portable_target" = "xyes"; then + HAVE_PORTABLE_TARGET=1 +fi +AC_SUBST(HAVE_PORTABLE_TARGET) +AC_DEFINE_UNQUOTED([HAVE_PORTABLE_TARGET], [$HAVE_PORTABLE_TARGET], [Define to 1 if PORTABLE_TARGET is enabled]) +AM_CONDITIONAL([HAVE_PORTABLE_TARGET], [ test x$HAVE_PORTABLE_TARGET = x1 ]) + +dnl Detect cross-compilation +if test "x$build" != "x$host" ; then + CROSS_COMPILING=1 +else + CROSS_COMPILING=0 +fi +AC_SUBST(CROSS_COMPILING) +AC_DEFINE_UNQUOTED([CROSS_COMPILING], [$CROSS_COMPILING], [Define to 1 if cross compiling]) +AM_CONDITIONAL([CROSS_COMPILING], [ test x$CROSS_COMPILING = x1 ]) + +AM_CONDITIONAL([TARGET_CPU_X86], [ test x$target_cpu = xx86 ]) +AM_CONDITIONAL([TARGET_CPU_X86_64], [ test x$target_cpu = xx86_64 ]) +AM_CONDITIONAL([TARGET_CPU_ANY_X86], [ test x$target_cpu = xx86 -o x$target_cpu = xx86_64 ]) +AM_CONDITIONAL([TARGET_CPU_ARM], [ test x$target_cpu = xarm ]) +AM_CONDITIONAL([TARGET_CPU_ARM64], [ test x$target_cpu = xaarch64 ]) +AM_CONDITIONAL([TARGET_CPU_ANY_ARM], [ test x$target_cpu = xarm -o x$target_cpu = xaarch64 ]) + +AM_COND_IF([TARGET_CPU_ANY_X86], [m4_version_prereq(2.64, [ + AX_CHECK_COMPILE_FLAG([-msse2], , , [-Werror]) + AX_CHECK_COMPILE_FLAG([-msse4.1], , , [-Werror]) + AX_CHECK_COMPILE_FLAG([-mavx2], , , [-Werror]) + AX_CHECK_COMPILE_FLAG([-mavx512f -mavx512vl], , , [-Werror]) + ], [true]) +]) + +AM_COND_IF([TARGET_CPU_ARM], [m4_version_prereq(2.64, [ + AX_CHECK_COMPILE_FLAG([-mfpu=neon], , , [-Werror]) + ], [true]) +]) + +AM_CONDITIONAL([TARGET_HAS_SSE2], [ test "x$ax_cv_check_cflags__Werror__msse2" = x"yes" -a "x$enable_portable_target" != "xyes" ]) +AM_CONDITIONAL([TARGET_HAS_SSE41], [ test "x$ax_cv_check_cflags__Werror__msse4_1" = x"yes" -a "x$enable_portable_target" != "xyes" ]) +AM_CONDITIONAL([TARGET_HAS_AVX2], [ test "x$ax_cv_check_cflags__Werror__mavx2" = x"yes" -a "x$enable_portable_target" != "xyes" ]) +AM_CONDITIONAL([TARGET_HAS_AVX512], [ test "x$ax_cv_check_cflags__Werror__mavx512f__mavx512vl" = x"yes" -a "x$enable_portable_target" != "xyes" ]) + +AM_CONDITIONAL([TARGET_HAS_NEON], [ test \( x$target_cpu = xaarch64 -o "x$ax_cv_check_cflags__Werror__mcpu_neon" = x"yes" \) -a "x$enable_portable_target" != "xyes" ]) + +AM_COND_IF([TARGET_HAS_SSE2], AC_DEFINE([TARGET_HAS_SSE2], [1], [SSE2 target support])) +AM_COND_IF([TARGET_HAS_SSE41], AC_DEFINE([TARGET_HAS_SSE41], [1], [SSE41 target support])) +AM_COND_IF([TARGET_HAS_AVX2], AC_DEFINE([TARGET_HAS_AVX2], [1], [AVX2 target support])) +AM_COND_IF([TARGET_HAS_AVX512], AC_DEFINE([TARGET_HAS_AVX512], [1], [AVX512 target support])) + +AM_COND_IF([TARGET_HAS_NEON], AC_DEFINE([TARGET_HAS_NEON], [1], [NEON target support])) + +dnl ASAN enable switch +AC_ARG_ENABLE([asan], + AS_HELP_STRING([--enable-asan], + [Enable ASAN support])) + +HAVE_ASAN=0 +ASAN_CFLAGS="" +ASAN_LIBS="" +if test "x$enable_asan" = "xyes" ; then + AC_MSG_CHECKING([location of ASAN library]) + ASANLIB1=`${CC} -print-file-name=libasan.so` + ASANLIB=`readlink -f "${ASANLIB1}"` + if test -f "$ASANLIB" ; then + HAVE_ASAN=1 + ASAN_CFLAGS="-fsanitize=address -fno-omit-frame-pointer" + ASAN_LIBS="-fsanitize=address" + AC_MSG_RESULT([$ASANLIB]) + m4_version_prereq(2.64, + AX_APPEND_COMPILE_FLAGS([-fsanitize=address -fno-omit-frame-pointer], [CFLAGS]), + [CFLAGS="$CFLAGS -fsanitize=address -fno-omit-frame-pointer"]) + else + AC_MSG_RESULT([Not found; disabling ASAN]) + fi +fi +AC_SUBST(HAVE_ASAN) +AC_SUBST(ASAN_CFLAGS) +AC_SUBST(ASAN_LIBS) +AM_CONDITIONAL([HAVE_ASAN], + [ test x$HAVE_ASAN = x1 ]) +AC_DEFINE_UNQUOTED([HAVE_ASAN], [$HAVE_ASAN], [Define to 1 if ASAN is enabled]) + +# check if there's a qsort_r available (musl does not have it) +AC_CHECK_FUNC([qsort_r], + HAVE_QSORT_R=1, + HAVE_QSORT_R=0) +AC_SUBST(HAVE_QSORT_R) +AC_DEFINE_UNQUOTED([HAVE_QSORT_R], [$HAVE_QSORT_R], [Define to 1 if you have qsort_r available]) +AM_CONDITIONAL([HAVE_QSORT_R], [ test x$HAVE_QSORT_R = x1 ]) + +# check if there's a mremap available (macos or BSD's don't have it) +AC_CHECK_FUNC([mremap], + HAVE_MREMAP=1, + HAVE_MREMAP=0) +AC_SUBST(HAVE_MREMAP) +AC_DEFINE_UNQUOTED([HAVE_MREMAP], [$HAVE_MREMAP], [Define to 1 if you have mremap available]) +AM_CONDITIONAL([HAVE_MREMAP], [ test x$HAVE_MREMAP = x1 ]) + +PKG_CHECK_MODULES(LIBYAML, [ yaml-0.1 ], HAVE_LIBYAML=1, HAVE_LIBYAML=0) + +# update with pkg-config's flags +if test "x$HAVE_LIBYAML" != "x1" ; then + AC_MSG_WARN([failed to find libyaml; compatibility disabled]) +fi + +AC_SUBST(HAVE_LIBYAML) +AC_SUBST(LIBYAML_CFLAGS) +AC_SUBST(LIBYAML_LIBS) +AC_DEFINE_UNQUOTED([HAVE_LIBYAML], [$HAVE_LIBYAML], [Define to 1 if you have libyaml available]) +AM_CONDITIONAL([HAVE_LIBYAML], [ test x$HAVE_LIBYAML = x1 ]) + +PKG_CHECK_MODULES(CHECK, [ check ], HAVE_CHECK=1, HAVE_CHECK=0) + +AC_SUBST(HAVE_CHECK) +AC_SUBST(CHECK_CFLAGS) +AC_SUBST(CHECK_LDFLAGS) +AC_SUBST(CHECK_LIBS) +AC_DEFINE_UNQUOTED([HAVE_CHECK], [$HAVE_CHECK], [Define to 1 if you have check available]) +AM_CONDITIONAL([HAVE_CHECK], [ test x$HAVE_CHECK = x1 ]) + +HAVE_COMPATIBLE_CHECK=0 +if test "x$HAVE_CHECK" = "x1" ; then + save_LIBS="$LIBS" + save_CFLAGS="$CFLAGS" + LIBS="$LIBS $CHECK_LIBS" + CFLAGS="$CFLAGS $CHECK_CFLAGS" + + # check if libcheck has srunner_set_tap (jessie has outdated libcheck) + AC_CHECK_FUNC([srunner_set_tap], + HAVE_COMPATIBLE_CHECK=1, + HAVE_COMPATIBLE_CHECK=0) + + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" +fi + +AC_SUBST(HAVE_COMPATIBLE_CHECK) +AC_DEFINE_UNQUOTED([HAVE_COMPATIBLE_CHECK], [$HAVE_COMPATIBLE_CHECK], [Define to 1 if you have a compatible version of check available]) +AM_CONDITIONAL([HAVE_COMPATIBLE_CHECK], [ test x$HAVE_COMPATIBLE_CHECK = x1 ]) + +dnl enable internet available when checking +AC_ARG_ENABLE([network], + AS_HELP_STRING([--disable-network], + [Disable tests requiring network access])) + +if test "x$enable_network" != "xno" ; then + HAVE_NETWORK=1 +else + HAVE_NETWORK=0 +fi +AC_SUBST(HAVE_NETWORK) +AM_CONDITIONAL([HAVE_NETWORK], + [ test x$HAVE_NETWORK = x1 ]) + +dnl enable internet available when checking +AC_ARG_ENABLE([devmode], + AS_HELP_STRING([--enable-devmode], + [Enable development mode only debugging])) + +if test "x$enable_devmode" != "xno" ; then + HAVE_DEVMODE=1 +else + HAVE_DEVMODE=0 +fi +AC_SUBST(HAVE_DEVMODE) +AM_CONDITIONAL([HAVE_DEVMODE], + [ test x$HAVE_DEVMODE = x1 ]) +AC_DEFINE_UNQUOTED([HAVE_DEVMODE], [$HAVE_DEVMODE], [Define to 1 if developer mode is enabled]) + +# check for sphinx +AC_PATH_PROG([SPHINX], [sphinx-build]) + +HAVE_SPHINX=0 +if test "x$SPHINX" != "x" ; then + AC_PATH_PROG([PIP3], [pip3]) + + # if both pip3 and sphinx-build are available check for versions + if test "x$PIP3" != "x" ; then + AC_MSG_CHECKING([sphinx version]) + SPHINX_VERSION=`pip3 2>/dev/null show sphinx | grep -i Version: | cut -d: -f2 | sed -e 's/ //g'` + if test "x$SPHINX_VERSION" != "x" ; then + AC_MSG_RESULT([$SPHINX_VERSION]) + else + AC_MSG_RESULT([N/A]) + fi + + AC_MSG_CHECKING([sphinx RTD theme version]) + SPHINX_RTD_THEME_VERSION=`pip3 2>/dev/null show sphinx_rtd_theme | grep -i Version: | cut -d: -f2 | sed -e 's/ //g'` + if test "x$SPHINX_VERSION" != "x" ; then + AC_MSG_RESULT([$SPHINX_RTD_THEME_VERSION]) + else + AC_MSG_RESULT([N/A]) + fi + + AC_MSG_CHECKING([sphinx markdown builder version]) + SPHINX_MARKDOWN_BUILDER_VERSION=`pip3 2>/dev/null show sphinx-markdown-builder | grep -i Version: | cut -d: -f2 | sed -e 's/ //g'` + if test "x$SPHINX_MARKDOWN_BUILDER_VERSION" != "x" ; then + AC_MSG_RESULT([$SPHINX_MARKDOWN_BUILDER_VERSION]) + else + AC_MSG_RESULT([N/A]) + fi + + AC_MSG_CHECKING([sphinx linuxdoc version]) + SPHINX_LINUXDOC_VERSION=`pip3 2>/dev/null show linuxdoc | grep -i Version: | cut -d: -f2 | sed -e 's/ //g'` + if test "x$SPHINX_LINUXDOC_VERSION" != "x" ; then + AC_MSG_RESULT([$SPHINX_LINUXDOC_VERSION]) + else + AC_MSG_RESULT([N/A]) + fi + + if test "x$SPHINX_VERSION" != "x" -a "x$SPHINX_RTD_THEME_VERSION" != "x" -a "x$SPHINX_MARKDOWN_BUILDER_VERSION" != "x" -a "x$SPHINX_LINUXDOC_VERSION" != "x" ; then + HAVE_SPHINX=1 + fi + fi + + AC_MSG_CHECKING([whether sphinx installation works]) + if test "x$HAVE_SPHINX" = "x1" ; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + fi +fi + +AC_DEFINE_UNQUOTED([HAVE_SPHINX], [$HAVE_SPHINX], [Define to 1 if you have sphinx (and all required packages) available]) +AM_CONDITIONAL([HAVE_SPHINX], [ test x$HAVE_SPHINX = x1 ]) + +# check for git +AC_PATH_PROG([GIT], [git]) +if test "x$GIT" != "x" ; then + HAVE_GIT=1 +else + HAVE_GIT=0 +fi +AC_DEFINE_UNQUOTED([HAVE_GIT], [$HAVE_GIT], [Define to 1 if you have git available]) +AM_CONDITIONAL([HAVE_GIT], [ test x$HAVE_GIT = x1 ]) + +AC_ARG_VAR(TESTSUITEURL, [Testsuite git repo URL (default: https://github.com/yaml/yaml-test-suite)]) + +if test "x$TESTSUITEURL" = "x" ; then + TESTSUITEURL="https://github.com/yaml/yaml-test-suite" +fi + +AC_ARG_VAR(TESTSUITECHECKOUT, [Testsuite checkout (default: 6e6c296ae9c9d2d5c4134b4b64d01b29ac19ff6f)]) + +if test "x$TESTSUITECHECKOUT" = "x" ; then + TESTSUITECHECKOUT="6e6c296ae9c9d2d5c4134b4b64d01b29ac19ff6f" +fi + +AC_ARG_VAR(JSONTESTSUITEURL, [JSON Testsuite git repo URL (default: https://github.com/nst/JSONTestSuite)]) + +if test "x$JSONTESTSUITEURL" = "x" ; then + JSONTESTSUITEURL="https://github.com/nst/JSONTestSuite" +fi + +AC_ARG_VAR(JSONTESTSUITECHECKOUT, [JSON Testsuite checkout (default: d64aefb55228d9584d3e5b2433f720ea8fd00c82)]) + +if test "x$JSONTESTSUITECHECKOUT" = "x" ; then + JSONTESTSUITECHECKOUT="d64aefb55228d9584d3e5b2433f720ea8fd00c82" +fi + +# check for docker +AC_PATH_PROG([DOCKER], [docker]) +if test "x$DOCKER" != "x" ; then + HAVE_DOCKER=1 +else + HAVE_DOCKER=0 +fi +AM_CONDITIONAL([HAVE_DOCKER], [ test x$HAVE_DOCKER = x1 ]) + +# check for jq +AC_PATH_PROG([JQ], [jq]) +if test "x$JQ" != "x" ; then + HAVE_JQ=1 +else + HAVE_JQ=0 +fi +AC_DEFINE_UNQUOTED([HAVE_JQ], [$HAVE_JQ], [Define to 1 if you have jq available]) +AM_CONDITIONAL([HAVE_JQ], [ test x$HAVE_JQ = x1 ]) +AC_SUBST(JQ, "$JQ") + + dnl enable internet available when checking +AC_ARG_ENABLE([network], + AS_HELP_STRING([--disable-network], + [Disable tests requiring network access])) + +# check for libclang +AC_ARG_WITH([libclang], + [AS_HELP_STRING([--with-libclang=[]], + [Enable support for libclang @<:@default=auto@:>@])], + [with_libclang="$withval"], + [with_libclang=auto]) + +HAVE_LIBCLANG=0 +LIBCLANG_CFLAGS="" +LIBCLANG_CPPFLAGS="" +LIBCLANG_LDFLAGS="" +LIBCLANG_LIBS="" +if test "x$with_libclang" != "xno"; then + if test "x$CROSS_COMPILING" = "x1" -a "x$with_libclang" = "xauto" ; then + AC_MSG_WARN([Cross-compiling detected: disabling libclang auto-detection.]) + AC_MSG_WARN([Use --with-libclang= to force libclang if needed (advanced use only).]) + with_libclang=no + fi +fi +if test "x$with_libclang" != "xno"; then + if test "x$with_libclang" = "xauto"; then + # try to auto detect (llvm 20 down to 10) + check="llvm-config llvm-config-20 llvm-config-19 llvm-config-18 llvm-config-17 llvm-config-16 \ + llvm-config-15 llvm-config-14 llvm-config-13 llvm-config-12 llvm-config-11 llvm-config-10" + else + if test "x$with_libclang" = "xlatest"; then + check="llvm-config" + else + check="llvm-config-$with_libclang" + fi + fi + orig_CFLAGS="$CFLAGS" + orig_CPPFLAGS="$CPPFLAGS" + orig_LDFLAGS="$LDFLAGS" + orig_LIBS="$LIBS" + for cfg in $check ; do + # no caching + $as_unset ac_cv_path_LLVM_CONFIG + AC_PATH_PROG([LLVM_CONFIG], [$cfg]) + if test "x$LLVM_CONFIG" != "x" ; then + LIBCLANG_CFLAGS=`$LLVM_CONFIG --cflags` + LIBCLANG_CPPFLAGS=`$LLVM_CONFIG --cppflags` + LIBCLANG_LDFLAGS=`$LLVM_CONFIG --ldflags` + LIBCLANG_LIBS="-lclang `$LLVM_CONFIG --libs` `$LLVM_CONFIG --system-libs`" + CPPFLAGS="$CPPFLAGS $LIBCLANG_CPPFLAGS" + CFLAGS="$CFLAGS $LIBCLANG_CFLAGS" + LDFLAGS="$LDFLAGS $LIBCLANG_LDFLAGS" + LIBS="$LIBS $LIBCLANG_LIBS" + + if test "x$CROSS_COMPILING" = "x1" ; then + AC_MSG_WARN([Cross-compiling: skipping libclang link test, checking headers only]) + HAVE_LIBCLANG_LIB=1 + else + AC_CHECK_LIB([clang], [clang_createIndex], + [HAVE_LIBCLANG_LIB=1], [HAVE_LIBCLANG_LIB=0]) + fi + AC_CHECK_HEADER([clang-c/Index.h], + [HAVE_LIBCLANG_HDR=1], [HAVE_LIBCLANG_HDR=0]) + # if we found it, break out + if test "x$HAVE_LIBCLANG_LIB" = "x1" -a "x$HAVE_LIBCLANG_HDR" = "x1" ; then + HAVE_LIBCLANG=1 + break + fi + $as_unset ac_cv_lib_clang_clang_createIndex + $as_unset ac_cv_header_clang_c_Index_h + $as_unset LIBCLANG_CFLAGS + $as_unset LIBCLANG_CPPFLAGS + $as_unset LIBCLANG_LIBS + $as_unset HAVE_LIBCLANG_LIB + $as_unset HAVE_LIBCLANG_HDR + fi + CFLAGS=$orig_CFLAGS + LDFLAGS=$orig_LDFLAGS + LIBS=$orig_LIBS + if test "x$HAVE_LIBCLANG" = "x1"; then + break + fi + $as_unset LLVM_CONFIG + done + CFLAGS=$orig_CFLAGS + CPPFLAGS=$orig_CPPFLAGS + LDFLAGS=$orig_LDFLAGS + LIBS=$orig_LIBS +fi + +AC_SUBST(HAVE_LIBCLANG) +AC_SUBST(LIBCLANG_CFLAGS) +AC_SUBST(LIBCLANG_CPPFLAGS) +AC_SUBST(LIBCLANG_LDFLAGS) +AC_SUBST(LIBCLANG_LIBS) +AC_DEFINE_UNQUOTED([HAVE_LIBCLANG], [$HAVE_LIBCLANG], [Define to 1 if libclang is available]) +AM_CONDITIONAL([HAVE_LIBCLANG], [ test x$HAVE_LIBCLANG = x1 ]) + +dnl Check for Clang Blocks support (Clang or AppleClang) +dnl Blocks provide closure support for C, useful for callback-heavy APIs +HAVE_CLANG_BLOCKS=0 +CLANG_BLOCKS_CFLAGS="" +CLANG_BLOCKS_LIBS="" + +AC_MSG_CHECKING([for Clang compiler]) +case "$CC" in + *clang*) + IS_CLANG=yes + ;; + *) + IS_CLANG=no + ;; +esac +AC_MSG_RESULT([$IS_CLANG]) + +if test "x$IS_CLANG" = "xyes"; then + dnl Check if this is AppleClang (Blocks built-in) or LLVM Clang (needs BlocksRuntime) + AC_MSG_CHECKING([for AppleClang]) + if $CC --version 2>&1 | grep -q "Apple" ; then + IS_APPLECLANG=yes + else + IS_APPLECLANG=no + fi + AC_MSG_RESULT([$IS_APPLECLANG]) + + if test "x$IS_APPLECLANG" = "xyes"; then + dnl AppleClang has Blocks built-in + AC_MSG_CHECKING([for AppleClang Blocks support]) + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS -fblocks" + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[]], [[ + void (^b)(void) = ^{ }; + b(); + ]])], + [HAVE_CLANG_BLOCKS=1 + CLANG_BLOCKS_CFLAGS="-fblocks" + AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no])]) + CFLAGS="$save_CFLAGS" + else + dnl LLVM Clang needs libBlocksRuntime + AC_MSG_CHECKING([for Clang Blocks with libBlocksRuntime]) + save_CFLAGS="$CFLAGS" + save_LIBS="$LIBS" + CFLAGS="$CFLAGS -fblocks" + LIBS="$LIBS -lBlocksRuntime" + AC_LINK_IFELSE( + [AC_LANG_PROGRAM([[#include ]], [[ + void (^b)(void) = ^{ }; + b(); + ]])], + [HAVE_CLANG_BLOCKS=1 + CLANG_BLOCKS_CFLAGS="-fblocks" + CLANG_BLOCKS_LIBS="-lBlocksRuntime" + AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no - libBlocksRuntime not found])]) + CFLAGS="$save_CFLAGS" + LIBS="$save_LIBS" + fi +fi + +AC_SUBST(HAVE_CLANG_BLOCKS) +AC_SUBST(CLANG_BLOCKS_CFLAGS) +AC_SUBST(CLANG_BLOCKS_LIBS) +AC_DEFINE_UNQUOTED([HAVE_CLANG_BLOCKS], [$HAVE_CLANG_BLOCKS], [Define to 1 if Clang Blocks are available]) +AM_CONDITIONAL([HAVE_CLANG_BLOCKS], [ test x$HAVE_CLANG_BLOCKS = x1 ]) + +dnl GCC heap trampolines support (for nested functions) +dnl Heap trampolines are more secure than executable stack trampolines +dnl Available in GCC 14+ with -ftrampoline-impl=heap +dnl Supported targets: x86_64/i386/aarch64 Linux, x86_64/i386 Darwin +HAVE_HEAP_TRAMPOLINES=0 + +AC_MSG_CHECKING([for GCC compiler]) +case "$CC" in + *gcc*) + IS_GCC=yes + ;; + *) + IS_GCC=no + ;; +esac +AC_MSG_RESULT([$IS_GCC]) + +if test "x$IS_GCC" = "xyes"; then + AX_CHECK_COMPILE_FLAG([-ftrampoline-impl=heap], + [HAVE_HEAP_TRAMPOLINES=1 + CFLAGS="$CFLAGS -ftrampoline-impl=heap" + AC_MSG_NOTICE([GCC heap trampolines enabled (-ftrampoline-impl=heap)])], + [AC_MSG_NOTICE([GCC heap trampolines not available (GCC 14+ required), using stack trampolines])], + [-Werror]) +fi + +AC_SUBST(HAVE_HEAP_TRAMPOLINES) +AC_DEFINE_UNQUOTED([HAVE_HEAP_TRAMPOLINES], [$HAVE_HEAP_TRAMPOLINES], [Define to 1 if GCC heap trampolines are enabled]) +AM_CONDITIONAL([HAVE_HEAP_TRAMPOLINES], [ test x$HAVE_HEAP_TRAMPOLINES = x1 ]) + +dnl trim cflags, cppflags, ldflags +CFLAGS=`echo $CFLAGS | sed -e 's/^[ ]*//'` +CPPFLAGS=`echo $CPPFLAGS | sed -e 's/^[ ]*//'` +LDFLAGS=`echo $LDFLAGS | sed -e 's/^[ ]*//'` + +# Shave by default on supported autoconf versions +dnl m4_version_prereq(2.64, SHAVE_INIT([build-aux], [enable]), [true]) +m4_ifdef([SHAVE_INIT], [SHAVE_INIT([build-aux], [enable])]) + +AC_CONFIG_FILES([ + build-aux/shave + build-aux/shave-libtool + Makefile + src/Makefile + test/Makefile + doc/Makefile + libfyaml.pc +]) + +AC_REQUIRE_AUX_FILE([tap-driver.sh]) + +AC_OUTPUT + +echo " + ---{ $PACKAGE_NAME $VERSION }--- + + VERSION: ${VERSION} + MAJOR.MINOR: ${MAJOR}.${MINOR} + PATCH: ${PATCH} + EXTRA: ${EXTRA} + LIBTOOL_VERSION: ${LIBTOOL_VERSION} + prefix: ${prefix} + Build system: ${build} + Host system: ${host} + Target system: ${target} + Cross compiling: ${CROSS_COMPILING} + C compiler: ${CC} + Linker: ${LD} + CFLAGS: ${CFLAGS} + LDFLAGS: ${LDFLAGS} + LIBS: ${LIBS} + HAVE_ASAN: ${HAVE_ASAN} + HAVE_CHECK: ${HAVE_CHECK} + HAVE_COMPATIBLE_CHECK: ${HAVE_COMPATIBLE_CHECK} + HAVE_NETWORK: ${HAVE_NETWORK} + HAVE_DEVMODE: ${HAVE_DEVMODE} + HAVE_SPHINX: ${HAVE_SPHINX} + GIT: ${GIT} + DOCKER: ${DOCKER} + HAVE_LIBCLANG: ${HAVE_LIBCLANG} + LIBCLANG_LIBS: ${LIBCLANG_LIBS} + HAVE_CLANG_BLOCKS: ${HAVE_CLANG_BLOCKS} + HAVE_HEAP_TRAMPOLINES: ${HAVE_HEAP_TRAMPOLINES} + TESTSUITEURL: $TESTSUITEURL + TESTSUITECHECKOUT: $TESTSUITECHECKOUT + JSONTESTSUITEURL: $JSONTESTSUITEURL + JSONTESTSUITECHECKOUT: $JSONTESTSUITECHECKOUT + +" diff --git a/doc/Makefile.am b/doc/Makefile.am new file mode 100644 index 0000000..9902908 --- /dev/null +++ b/doc/Makefile.am @@ -0,0 +1,51 @@ +BUILDDIR=_build + +if HAVE_SPHINX + +MANPAGES1 = $(BUILDDIR)/man/fy-tool.1 + +doc-%: + srctree=@top_srcdir@ @SPHINX@ -M `echo $@ | sed -s 's/^doc-//g'` "@srcdir@" "$(BUILDDIR)" + +# if we have sphinx generate the manpages +$(MANPAGES1): doc-man + +else + +# otherwise, install the canned ones +MANPAGES1 = $(srcdir)/canned-man/fy-tool.1 + +endif + +install-data-hook: $(MANPAGES1) + $(MKDIR_P) "$(DESTDIR)$(mandir)/man1" + @for i in "$(MANPAGES1)"; do \ + $(INSTALL_DATA) $$i "$(DESTDIR)$(mandir)/man1"; \ + done + (cd "$(DESTDIR)$(mandir)/man1" && $(LN_S) -f fy-tool.1 fy-dump.1) + (cd "$(DESTDIR)$(mandir)/man1" && $(LN_S) -f fy-tool.1 fy-filter.1) + (cd "$(DESTDIR)$(mandir)/man1" && $(LN_S) -f fy-tool.1 fy-testsuite.1) + (cd "$(DESTDIR)$(mandir)/man1" && $(LN_S) -f fy-tool.1 fy-join.1) + (cd "$(DESTDIR)$(mandir)/man1" && $(LN_S) -f fy-tool.1 fy-ypath.1) + (cd "$(DESTDIR)$(mandir)/man1" && $(LN_S) -f fy-tool.1 fy-compose.1) + +uninstall-hook: + @for i in "$(MANPAGES1)"; do \ + rm -f "$(DESTDIR)$(mandir)/man1/`basename $$i`"; \ + done + (cd "$(DESTDIR)$(mandir)/man1" && \ + rm -f fy-dump.1 fy-filter.1 fy-testsuite.1 fy-join.1 fy-ypath.1 fy-compose.1) + +clean-local: + @rm -rf "$(BUILDDIR)" + +maintainer-clean-local: + @rm -rf Makefile.in + +EXTRA_DIST = \ + conf.py \ + index.rst \ + intro.rst \ + libfyaml.rst \ + man/fy-tool.rst \ + canned-man/fy-tool.1 diff --git a/doc/canned-man/fy-tool.1 b/doc/canned-man/fy-tool.1 new file mode 100644 index 0000000..d9d4b56 --- /dev/null +++ b/doc/canned-man/fy-tool.1 @@ -0,0 +1,1005 @@ +.\" Man page generated from reStructuredText. +. +.TH "FY-TOOL" "1" "Jan 14, 2022" "" "libfyaml" +.SH NAME +fy-tool \- fy-tool documentation +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.sp +\fBfy\-tool\fP [\fIOPTIONS\fP] [<\fIfile\fP> ...] +.sp +\fBfy\-dump\fP [\fIOPTIONS\fP] [<\fIfile\fP> ...] +.sp +\fBfy\-testsuite\fP [\fIOPTIONS\fP] <\fIfile\fP> +.sp +\fBfy\-filter\fP [\fIOPTIONS\fP] [\-f*FILE*] [<\fIpath\fP> ...] +.sp +\fBfy\-join\fP [\fIOPTIONS\fP] [\-T*PATH*] [\-F*PATH*] [\-t*PATH*] [<\fIfile\fP> ...] +.sp +\fBfy\-ypath\fP [\fIOPTIONS\fP] <\fIypath\-expression> [<*file\fP> ...] +.sp +\fBfy\-compose\fP [\fIOPTIONS\fP] [<\fIfile\fP> ...] +.SH DESCRIPTION +.sp +\fBfy\-tool\fP is a general YAML/JSON manipulation tool using \fIlibfyaml\fP\&. +Its operation is different depending on how it\(aqs called, either via aliases +named \fIfy\-dump\fP, \fIfy\-testsuite\fP, \fIfy\-filter\fP, \fIfy\-join\fP, or via the +\fI\-\-dump\fP, \fI\-\-testsuite\fP, \fI\-\-filter\fP, \fI\-\-join\fP, \fI\-\-ypath\fP and \fI\-\-compose\fP options. +.INDENT 0.0 +.IP \(bu 2 +In \fIdump\fP mode it will parse YAML/JSON input files and output YAML/JSON +according to the output format options. +.IP \(bu 2 +In \fItestsuite\fP mode it will parse a single YAML input file and output +yaml testsuite reference output. +.IP \(bu 2 +In \fIfilter\fP mode it will parse YAML/JSON files and output YAML/JSON +output filtered according to the given option. +.IP \(bu 2 +In \fIjoin\fP mode it will parse YAML/JSON files and join them into a single +YAML/JSON document according to the given command line options. +.IP \(bu 2 +In \fIypath\fP mode it will parse YAML/JSON files and execute a ypath query +which will output a document stream according to the results. This is +an experimental mode under development, where the syntax is not yet +decided completely. +.IP \(bu 2 +In \fIcompose\fP mode, it operates similarly to dump, but the document tree +is created using the built\-in composer API. +.UNINDENT +.SH OPTIONS +.sp +A number of options are common for all the different modes of operation +and they are as follows: +Common options.INDENT 0.0 +.TP +.B \-q, \-\-quiet +Quiet operation, does not output informational messages at all. +.UNINDENT +.INDENT 0.0 +.TP +.B \-h, \-\-help +Display usage information. +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-version +Display version information. +.UNINDENT +.INDENT 0.0 +.TP +.B \-I DIR, \-\-include=DIR +Add the \fIDIR\fP directory to the search path which will be used to locate a YAML/JSON file. +The default path is set to "" +.UNINDENT +.INDENT 0.0 +.TP +.B \-d LEVEL, \-\-debug\-level=LEVEL +Set the minimum numeric debug level value of the library to \fILEVEL\fP\&. +The numeric value must be in the range of 0 to 4 and their meaning is as follows: +.INDENT 7.0 +.IP \(bu 2 +\fB0\fP (\fIDEBUG\fP) +.sp +Internal library debugging messages. No output is produced when the +library was compiled with \fI\-\-disable\-debug\fP +.IP \(bu 2 +\fB1\fP (\fIINFO\fP) +.sp +Informational messages about the internal operation of the library. +.IP \(bu 2 +\fB2\fP (\fINOTICE\fP) +.sp +Messsages that could require attention. +.IP \(bu 2 +\fB3\fP (\fIWARNING\fP) +.sp +A warning message, something is wrong, but operation can continue. +This is the default value. +.IP \(bu 2 +\fB4\fP (\fIERROR\fP) +.sp +A fatal error occured, it is impossible to continue. +.UNINDENT +.sp +The default level is 3 (\fIWARNING\fP), which means that messages +with level 3 and higher will be displayed. +.UNINDENT +Parser Options.INDENT 0.0 +.TP +.B \-j JSONOPT, \-\-json=JSONOPT +Marks the input files as JSON or YAML accordingly to: +.INDENT 7.0 +.IP \(bu 2 +\fBno\fP +.sp +The input files are always in YAML mode. +.IP \(bu 2 +\fBforce\fP +.sp +The input files are always set to JSON mode. +.IP \(bu 2 +\fBauto\fP +.sp +The input files are set to JSON mode automatically when the +file\(aqs extension is \fI\&.json\fP\&. This is the default. +.UNINDENT +.sp +JSON support is complete so all valid JSON files are parsed according +to JSON rules, even where those differ with YAML rules. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-yaml\-1.1 +Force YAML 1.1 rules, when input does not specify a version via a directive. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-yaml\-1.2 +Force YAML 1.2 rules, when input does not specify a version via a directive. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-yaml\-1.3 +Force YAML 1.3 rules, when input does not specify a version via a directive. +This option is experimental since the 1.3 spec is not yet released. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-disable\-accel +Disable use of acceleration features; use less memory but potentially more CPU. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-disable\-buffering +Disable use stdio bufferring, reads will be performed via unix fd reads. This +may reduce latency when reading from a network file descriptor, or similar. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-disable\-depth\-limit +Disable the object depth limit, which is usually set to a value near 60. +Using this option is is possible to process even pathological inputs when +using the default non\-recursive build mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-prefer\-recursive +Prefer recursive build methods, instead of iterative. This field is merely here +for evaluation purposes and will be removed in a future version. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-sloppy\-flow\-indentation +Use sloppy flow indentation, where indentation is not taken into account in flow +mode, even when the input is invalid YAML according to the spec. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-ypath\-aliases +Process aliases using ypaths. Experimental option. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-streaming +Only valid when in \fBdump\fP mode, enables streaming mode. This means +that no in\-memory graph tree is constructed, so indefinite and arbitrary +large YAML input streams can be processed. +.sp +Note that in streaming mode: +.INDENT 7.0 +.IP \(bu 2 +Key duplication checks are disabled. +.IP \(bu 2 +No reording of key order is possible when emitting (i.e. \fI\-\-sort\fP is not available). +.IP \(bu 2 +Alias resolution is not available (i.e. \fI\-\-resolve\fP). +.UNINDENT +.UNINDENT +Resolver Options.INDENT 0.0 +.TP +.B \-r, \-\-resolve +Perform anchor and merge key resolution. By default this option is disabled. +.UNINDENT +.INDENT 0.0 +.TP +.B \-l, \-\-follow +Follow aliases when performing path traversal. By default this option is disabled. +.UNINDENT +Testsuite Options.INDENT 0.0 +.TP +.B \-\-disable\-flow\-markers +Do not output flow\-markers for the testsuite output. +.UNINDENT +Emitter Options.INDENT 0.0 +.TP +.B \-i INDENT, \-\-indent=INDENT +Sets the emitter indent (in spaces). Default is \fB2\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B \-w WIDTH, \-\-width=WIDTH +Sets the preferred output width of the emitter. It is generally impossible +to strictly adhere to this limit so this is treated as a hint at best. +It not valid in any oneline output modes (i.e. \fIflow\-oneline\fP or \fIjson\-oneline\fP). +Default value is 80. +.UNINDENT +.INDENT 0.0 +.TP +.B \-m MODE, \-\-mode=MODE +Sets the output mode of the YAML emitted. Possible values are: +.INDENT 7.0 +.IP \(bu 2 +\fBoriginal\fP +.sp +The original formatting used in the input. This is default mode. +.IP \(bu 2 +\fBblock\fP +.sp +The output is forced to be in block mode. All flow constructs will +be converted to block mode. +.IP \(bu 2 +\fBflow\fP +.sp +The output is forced to be in flow mode. All block constructs will +be converted to flow mode. +.IP \(bu 2 +\fBflow\-oneline\fP +.sp +The output is forced to be in flow mode, but no newlines will be +emitted; the output is going to be a (potentially very) long line. +.IP \(bu 2 +\fBjson\fP +.sp +The output is forced to be in JSON mode. Note that it is impossible +to output an arbitrary YAML file as JSON, so this may fail. +.IP \(bu 2 +\fBjson\-oneline\fP +.sp +The output is forced to be in JSON mode and in a single line. +.IP \(bu 2 +\fBdejson\fP +.sp +Output is in block YAML mode but with special care to convert +JSON quoted strings in as non\-idiomatic YAML as possible. +For example \fI{ foo: "this is a test" }\fP will be emitted as +\fIfoo: this is a test\fP\&. YAML can handle scalars without using +excessive quoting. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B \-C MODE, \-\-color=MODE +It is possible to colorize output using ANSI color escape sequences, +and the mode can be one of: +.INDENT 7.0 +.IP \(bu 2 +\fBoff\fP +.sp +Never colorize output. +.IP \(bu 2 +\fBon\fP +.sp +Always colorize output. +.IP \(bu 2 +\fBauto\fP +.sp +Automatically colorize output when the output is a terminal. +This is the default. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B \-V, \-\-visible +Make all whitespace (spaces, unicode spaces and linebreaks) visible. +Note that this is performed using UTF8 characters so it will not work +on non\-UTF8 terminals, or a non\-UTF8 complete font. +.UNINDENT +.INDENT 0.0 +.TP +.B \-s, \-\-sort +Sort keys on output. This option is disabled by default. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c, \-\-comment +Experimental output comments option. Enabled output while preserving comments. +Disabled by default. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-strip\-labels +Strip labels on output. Disabled by default. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-strip\-tags +Strip tags on output. Disabled by default. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-strip\-doc +Strip document indicators on output. Disabled by default. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-null\-output +Do not generate any output, useful for profiling the parser. +.UNINDENT +YPATH options.INDENT 0.0 +.TP +.B \-\-dump\-pathexpr +Dump the produced path expression for debugging. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-exec +Do not execute the expression. Useful when used with \fI\-\-dump\-pathexpr\fP +.UNINDENT +Compose options.INDENT 0.0 +.TP +.B \-\-dump\-path +Dump the path while composing. +.UNINDENT +Tool mode select options.INDENT 0.0 +.TP +.B \-\-dump +Select \fIdump\fP mode of operation. This is the default. +This mode is also enabled when the called binary is aliased to +\fIfy\-dump\fP\&. +.sp +In this mode, all files provided in the command line will be dumped +in one continuous stream, to the standard output, using document start +indicators to mark the start of end new file. +.sp +If the file provided is \fI\-\fP then the input is the standard input. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-testsuite +Select \fItestsuite\fP mode of operation. +This mode is also enabled when the called binary is aliased to \fIfy\-testsuite\fP\&. +.sp +In this mode, a single YAML file is read and an event stream is +generated which is the format used for \fIyaml\-testsuite\fP compliance. +.sp +If the file provided is \fI\-\fP then the input is the standard input. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-filter +Select \fIfilter\fP mode of operation. +This mode is also enabled when the called binary is aliased to \fIfy\-filter\fP\&. +.sp +In this mode, a single YAML file is read from the standard input for each path +that is provided in the command line a document will be produced to the +standard output. +To use file instead of standard input use the \fI\-f/\-\-file\fP option. +.sp +If the file provided is \fI\-\fP then the input is the standard input. +.INDENT 7.0 +.TP +.B \-f FILE, \-\-file=FILE +Use the given file as input instead of standard input. +.sp +If first character of \fIFILE\fP is \fB>\fP the the input is the content of the option +that follows. +For example \-\-file ">foo: bar" is as \-\-file file.yaml with file.yaml "foo: bar" +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-join +Select \fIjoin\fP mode of operation. +This mode is also enabled when the called binary is aliased to \fIfy\-join\fP\&. +.sp +In this mode, multiple YAML files are joined into a single document, emitted +to the standard output. +.sp +If the file provided is \fI\-\fP then the input is the standard input. +.INDENT 7.0 +.TP +.B \-T PATH, \-\-to=PATH +The target path of the join. By default this is the root \fB/\fP\&. +.sp +If first character of \fIFILE\fP is \fB>\fP the the input is the content of the option +that follows. +.UNINDENT +.INDENT 7.0 +.TP +.B \-F PATH, \-\-from=PATH +The origin path of the join (for each input). By default this is the root \fB/\fP\&. +.sp +If first character of \fIFILE\fP is \fB>\fP the the input is the content of the option +that follows. +.UNINDENT +.INDENT 7.0 +.TP +.B \-t PATH, \-\-trim=PATH +Trim path of the output of the join. By default this is the root \fB/\fP\&. +.sp +If first character of \fIFILE\fP is \fB>\fP the the input is the content of the option +that follows. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-ypath +Process files and output query results using ypath. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-compose +Use the composer API to build the document instead of direct events. +.UNINDENT +.SH EXAMPLES +Example input files +.sp +We\(aqre going to be using a couple of YAML files in our examples. +.sp +invoice.yaml +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# invoice.yaml +invoice: 34843 +date : !!str 2001\-01\-23 +bill\-to: &id001 +given : Chris +family : Dumars +address: + lines: | + 458 Walkman Dr. + Suite #292 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +simple\-anchors.yaml +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# simple\-anchors.yaml +foo: &label { bar: frooz } +baz: *label +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +mergekeyspec.yaml +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +\-\-\- +\- &CENTER { x: 1, y: 2 } +\- &LEFT { x: 0, y: 2 } +\- &BIG { r: 10 } +\- &SMALL { r: 1 } + +# All the following maps are equal: + +\- # Explicit keys + x: 1 + y: 2 + r: 10 + label: center/big + +\- # Merge one map + << : *CENTER + r: 10 + label: center/big + +\- # Merge multiple maps + << : [ *CENTER, *BIG ] + label: center/big + +\- # Override + << : [ *BIG, *LEFT, *SMALL ] + x: 1 + label: center/big +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +bomb.yaml +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"] +b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a] +c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b] +d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c] +e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d] +f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e] +g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f] +.ft P +.fi +.UNINDENT +.UNINDENT +fy\-dump examples. +.sp +Parse and dump generated YAML document tree in the original YAML form +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-dump invoice.yaml +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +invoice: 34843 +date: !!str 2001\-01\-23 +bill\-to: &id001 +given: Chris +family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Parse and dump generated YAML document tree in flow YAML form +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-dump \-mflow invoice.yaml +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +{ + invoice: 34843, + date: !!str 2001\-01\-23, + bill\-to: &id001 { + given: Chris, + family: Dumars, + address: { + lines: "458 Walkman Dr.\enSuite #292\en" + } + } +} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Parse and dump generated YAML document from the input string +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-dump \-mjson ">foo: bar" +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +{ + "foo": "bar" +} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Using the resolve option on the \fIsimple\-anchors.yaml\fP +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-dump \-r simple\-anchor.yaml +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +foo: &label { + bar: frooz + } +baz: { + bar: frooz + } +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Stripping the labels too: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-dump \-r \-\-strip\-label simple\-anchor.yaml +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +foo: { + bar: frooz + } +baz: { + bar: frooz + } +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Merge key support: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-dump \-r \-\-strip\-label mergekeyspec.yaml +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +\-\-\- +\- { + x: 1, + y: 2 + } +\- { + x: 0, + y: 2 + } +\- { + r: 10 + } +\- { + r: 1 + } +\- x: 1 + y: 2 + r: 10 + label: center/big +\- y: 2 + x: 1 + r: 10 + label: center/big +\- r: 10 + y: 2 + x: 1 + label: center/big +\- y: 2 + r: 10 + x: 1 + label: center/big +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Sorting option: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-dump \-s invoice.yaml +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +bill\-to: &id001 + address: + lines: | + 458 Walkman Dr. + Suite #292 + family: Dumars + given: Chris +date: !!str 2001\-01\-23 +invoice: 34843 +.ft P +.fi +.UNINDENT +.UNINDENT +fy\-testsuite example. +.sp +An example using the testsuite mode generates the following +event stream from \fIinvoice.yaml\fP +.sp +Parse and dump test\-suite event format +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-testsuite invoice.yaml +.ft P +.fi +.UNINDENT +.UNINDENT +fy\-filter examples. +.sp +Filter out from the \fI/bill\-to\fP path of \fIinvoice.yaml\fP +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ cat invoice.yaml | fy\-filter /bill\-to +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +&id001 +given: Chris +family: Dumars +address: + lines: | + 458 Walkman Dr. + Suite #292 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Filter example with arrays (and use the \-\-file option) +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-filter \-\-file=mergekeyspec.yaml /5 +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +\-\-\- +<<: *CENTER +r: 10 +label: center/big +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Follow anchors example +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-filter \-\-file=simple\-anchors.yaml /baz/bar +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +frooz +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Handle YAML bombs (if you can spare the memory and cpu time) +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-filter \-\-file=bomb.yaml \-r / | wc \-l +6726047 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +You don\(aqt have to, you can just follow links to retrieve data. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-filter \-\-file=stuff/bomb.yaml \-l \-\-strip\-label /g/0/1/2/3/4/5/6 +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +"lol" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Following links works with merge keys too: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-filter \-\-file=mergekeyspec.yaml \-l \-\-strip\-label /5/x +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +\-\-\- 1 +.ft P +.fi +.UNINDENT +.UNINDENT +fy\-join examples. +.sp +Joining two YAML files that have root mappings. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-join simple\-anchors.yaml invoice.yaml +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +foo: &label { + bar: frooz + } +baz: *label +invoice: 34843 +date: !!str 2001\-01\-23 +bill\-to: &id001 + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Join two files with sequences at root: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ fy\-join \-mblock ">[ foo, bar ]" ">[ baz ]" +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +\- foo +\- bar +\- baz +.ft P +.fi +.UNINDENT +.UNINDENT +.SH AUTHOR +.sp +Pantelis Antoniou <\fI\%pantelis.antoniou@konsulko.com\fP> +.SH BUGS +.INDENT 0.0 +.IP \(bu 2 +The only supported input and output character encoding is UTF8. +.IP \(bu 2 +Sorting does not respect language settings. +.IP \(bu 2 +There is no way for the user to specific a different coloring scheme. +.UNINDENT +.SH SEE ALSO +.sp +\fBlibfyaml(1)\fP +.SH COPYRIGHT +2019, Pantelis Antoniou +.\" Generated by docutils manpage writer. +. diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..9e46afa --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,69 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'libfyaml' +copyright = '2019, Pantelis Antoniou' +author = 'Pantelis Antoniou' + +# The full version, including alpha/beta/rc tags +release = '0.1' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'linuxdoc.rstFlatTable' # Implementation of the 'flat-table' reST-directive. + , 'linuxdoc.rstKernelDoc' # Implementation of the 'kernel-doc' reST-directive. + , 'linuxdoc.kernel_include' # Implementation of the 'kernel-include' reST-directive. + , 'linuxdoc.manKernelDoc' # Implementation of the 'kernel-doc-man' builder + , 'linuxdoc.cdomain' # Replacement for the sphinx c-domain. + , 'linuxdoc.kfigure' # Sphinx extension which implements scalable image handling. + , 'sphinx_markdown_builder' # Sphinx markdown builder +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +master_doc = 'index' + +man_pages = [ + ('man/fy-tool', 'fy-tool', 'fy-tool documentation ', '', 1), +] diff --git a/doc/include b/doc/include new file mode 120000 index 0000000..3a1af68 --- /dev/null +++ b/doc/include @@ -0,0 +1 @@ +../include/ \ No newline at end of file diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000..a6b88a1 --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,23 @@ +.. libfyaml documentation master file, created by + sphinx-quickstart on Sat May 25 20:55:33 2019. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to libfyaml's documentation! +==================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + intro + man/fy-tool + libfyaml + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/intro.rst b/doc/intro.rst new file mode 100644 index 0000000..1bf5d59 --- /dev/null +++ b/doc/intro.rst @@ -0,0 +1,4 @@ +Introduction +============ + +This is the documentation for libfyaml, a fancy 1.2 YAML and JSON parser/writer. diff --git a/doc/libfyaml.rst b/doc/libfyaml.rst new file mode 100644 index 0000000..4585332 --- /dev/null +++ b/doc/libfyaml.rst @@ -0,0 +1,5 @@ +Public API +========== + + .. kernel-doc:: include/libfyaml.h + :internal: diff --git a/doc/man/fy-tool.rst b/doc/man/fy-tool.rst new file mode 100644 index 0000000..0d482f1 --- /dev/null +++ b/doc/man/fy-tool.rst @@ -0,0 +1,765 @@ +fy-tool +======= + +Synopsis +-------- + +**fy-tool** [*OPTIONS*] [<*file*> ...] + +**fy-dump** [*OPTIONS*] [<*file*> ...] + +**fy-testsuite** [*OPTIONS*] <*file*> + +**fy-filter** [*OPTIONS*] [-f*FILE*] [<*path*> ...] + +**fy-join** [*OPTIONS*] [-T*PATH*] [-F*PATH*] [-t*PATH*] [<*file*> ...] + +**fy-ypath** [*OPTIONS*] <*ypath-expression> [<*file*> ...] + +**fy-compose** [*OPTIONS*] [<*file*> ...] + +Description +----------- + +:program:`fy-tool` is a general YAML/JSON manipulation tool using `libfyaml`. +Its operation is different depending on how it's called, either via aliases +named `fy-dump`, `fy-testsuite`, `fy-filter`, `fy-join`, or via the +`--dump`, `--testsuite`, `--filter`, `--join`, `--ypath` and `--compose` options. + +* In `dump` mode it will parse YAML/JSON input files and output YAML/JSON + according to the output format options. + +* In `testsuite` mode it will parse a single YAML input file and output + yaml testsuite reference output. + +* In `filter` mode it will parse YAML/JSON files and output YAML/JSON + output filtered according to the given option. + +* In `join` mode it will parse YAML/JSON files and join them into a single + YAML/JSON document according to the given command line options. + +* In `ypath` mode it will parse YAML/JSON files and execute a ypath query + which will output a document stream according to the results. This is + an experimental mode under development, where the syntax is not yet + decided completely. + +* In `compose` mode, it operates similarly to dump, but the document tree + is created using the built-in composer API. + +Options +------- + +.. program:: fy-tool + +A number of options are common for all the different modes of operation +and they are as follows: + +.. rubric:: Common options + +.. option:: -q, --quiet + + Quiet operation, does not output informational messages at all. + +.. option:: -h, --help + + Display usage information. + +.. option:: -v, --version + + Display version information. + +.. option:: -I DIR, --include=DIR + + Add the `DIR` directory to the search path which will be used to locate a YAML/JSON file. + The default path is set to "" + +.. option:: -d LEVEL, --debug-level=LEVEL + + Set the minimum numeric debug level value of the library to `LEVEL`. + The numeric value must be in the range of 0 to 4 and their meaning is as follows: + + - **0** (`DEBUG`) + + Internal library debugging messages. No output is produced when the + library was compiled with `--disable-debug` + + - **1** (`INFO`) + + Informational messages about the internal operation of the library. + + - **2** (`NOTICE`) + + Messsages that could require attention. + + - **3** (`WARNING`) + + A warning message, something is wrong, but operation can continue. + This is the default value. + + - **4** (`ERROR`) + + A fatal error occured, it is impossible to continue. + + The default level is 3 (`WARNING`), which means that messages + with level 3 and higher will be displayed. + +.. rubric:: Parser Options + +.. option:: -j JSONOPT, --json=JSONOPT + + Marks the input files as JSON or YAML accordingly to: + + - **no** + + The input files are always in YAML mode. + + - **force** + + The input files are always set to JSON mode. + + - **auto** + + The input files are set to JSON mode automatically when the + file's extension is `.json`. This is the default. + + JSON support is complete so all valid JSON files are parsed according + to JSON rules, even where those differ with YAML rules. + +.. option:: --yaml-1.1 + + Force YAML 1.1 rules, when input does not specify a version via a directive. + +.. option:: --yaml-1.2 + + Force YAML 1.2 rules, when input does not specify a version via a directive. + +.. option:: --yaml-1.3 + + Force YAML 1.3 rules, when input does not specify a version via a directive. + This option is experimental since the 1.3 spec is not yet released. + +.. option:: --disable-accel + + Disable use of acceleration features; use less memory but potentially more CPU. + +.. option:: --disable-buffering + + Disable use stdio bufferring, reads will be performed via unix fd reads. This + may reduce latency when reading from a network file descriptor, or similar. + +.. option:: --disable-depth-limit + + Disable the object depth limit, which is usually set to a value near 60. + Using this option is is possible to process even pathological inputs when + using the default non-recursive build mode. + +.. option:: --prefer-recursive + + Prefer recursive build methods, instead of iterative. This field is merely here + for evaluation purposes and will be removed in a future version. + +.. option:: --sloppy-flow-indentation + + Use sloppy flow indentation, where indentation is not taken into account in flow + mode, even when the input is invalid YAML according to the spec. + +.. option:: --ypath-aliases + + Process aliases using ypaths. Experimental option. + +.. option:: --streaming + + Only valid when in **dump** mode, enables streaming mode. This means + that no in-memory graph tree is constructed, so indefinite and arbitrary + large YAML input streams can be processed. + + Note that in streaming mode: + + - Key duplication checks are disabled. + - No reording of key order is possible when emitting (i.e. `--sort` is not available). + - Alias resolution is not available (i.e. `--resolve`). + +.. rubric:: Resolver Options + +.. option:: -r, --resolve + + Perform anchor and merge key resolution. By default this option is disabled. + +.. option:: -l, --follow + + Follow aliases when performing path traversal. By default this option is disabled. + +.. rubric:: Testsuite Options + +.. option:: --disable-flow-markers + + Do not output flow-markers for the testsuite output. + +.. rubric:: Emitter Options + +.. option:: -i INDENT, --indent=INDENT + + Sets the emitter indent (in spaces). Default is **2**. + +.. option:: -w WIDTH, --width=WIDTH + + Sets the preferred output width of the emitter. It is generally impossible + to strictly adhere to this limit so this is treated as a hint at best. + It not valid in any oneline output modes (i.e. `flow-oneline` or `json-oneline`). + Default value is 80. + +.. option:: -m MODE, --mode=MODE + + Sets the output mode of the YAML emitted. Possible values are: + + - **original** + + The original formatting used in the input. This is default mode. + + - **block** + + The output is forced to be in block mode. All flow constructs will + be converted to block mode. + + - **flow** + + The output is forced to be in flow mode. All block constructs will + be converted to flow mode. + + - **flow-oneline** + + The output is forced to be in flow mode, but no newlines will be + emitted; the output is going to be a (potentially very) long line. + + - **json** + + The output is forced to be in JSON mode. Note that it is impossible + to output an arbitrary YAML file as JSON, so this may fail. + + - **json-oneline** + + The output is forced to be in JSON mode and in a single line. + + - **dejson** + + Output is in block YAML mode but with special care to convert + JSON quoted strings in as non-idiomatic YAML as possible. + For example `{ foo: "this is a test" }` will be emitted as + `foo: this is a test`. YAML can handle scalars without using + excessive quoting. + +.. option:: -C MODE, --color=MODE + + It is possible to colorize output using ANSI color escape sequences, + and the mode can be one of: + + - **off** + + Never colorize output. + + - **on** + + Always colorize output. + + - **auto** + + Automatically colorize output when the output is a terminal. + This is the default. + +.. option:: -V, --visible + + Make all whitespace (spaces, unicode spaces and linebreaks) visible. + Note that this is performed using UTF8 characters so it will not work + on non-UTF8 terminals, or a non-UTF8 complete font. + +.. option:: -s, --sort + + Sort keys on output. This option is disabled by default. + +.. option:: -c, --comment + + Experimental output comments option. Enabled output while preserving comments. + Disabled by default. + +.. option:: --strip-labels + + Strip labels on output. Disabled by default. + +.. option:: --strip-tags + + Strip tags on output. Disabled by default. + +.. option:: --strip-doc + + Strip document indicators on output. Disabled by default. + +.. option:: --null-output + + Do not generate any output, useful for profiling the parser. + +.. rubric:: YPATH options + +.. option:: --dump-pathexpr + + Dump the produced path expression for debugging. + +.. option:: --no-exec + + Do not execute the expression. Useful when used with `--dump-pathexpr` + +.. rubric:: Compose options + +.. option:: --dump-path + + Dump the path while composing. + +.. rubric:: Tool mode select options + +.. option:: --dump + + Select `dump` mode of operation. This is the default. + This mode is also enabled when the called binary is aliased to + *fy-dump*. + + In this mode, all files provided in the command line will be dumped + in one continuous stream, to the standard output, using document start + indicators to mark the start of end new file. + + If the file provided is `-` then the input is the standard input. + +.. option:: --testsuite + + Select `testsuite` mode of operation. + This mode is also enabled when the called binary is aliased to *fy-testsuite*. + + In this mode, a single YAML file is read and an event stream is + generated which is the format used for *yaml-testsuite* compliance. + + If the file provided is `-` then the input is the standard input. + +.. option:: --filter + + Select `filter` mode of operation. + This mode is also enabled when the called binary is aliased to *fy-filter*. + + In this mode, a single YAML file is read from the standard input for each path + that is provided in the command line a document will be produced to the + standard output. + To use file instead of standard input use the `-f/--file` option. + + If the file provided is `-` then the input is the standard input. + + .. option:: -f FILE, --file=FILE + + Use the given file as input instead of standard input. + + If first character of `FILE` is **>** the the input is the content of the option + that follows. + For example --file ">foo: bar" is as --file file.yaml with file.yaml "foo: bar" + +.. option:: --join + + Select `join` mode of operation. + This mode is also enabled when the called binary is aliased to *fy-join*. + + In this mode, multiple YAML files are joined into a single document, emitted + to the standard output. + + If the file provided is `-` then the input is the standard input. + + .. option:: -T PATH, --to=PATH + + The target path of the join. By default this is the root **/**. + + If first character of `FILE` is **>** the the input is the content of the option + that follows. + + .. option:: -F PATH, --from=PATH + + The origin path of the join (for each input). By default this is the root **/**. + + If first character of `FILE` is **>** the the input is the content of the option + that follows. + + .. option:: -t PATH, --trim=PATH + + Trim path of the output of the join. By default this is the root **/**. + + If first character of `FILE` is **>** the the input is the content of the option + that follows. + +.. option:: --ypath + + Process files and output query results using ypath. + +.. option:: --compose + + Use the composer API to build the document instead of direct events. + +Examples +-------- + +.. rubric:: Example input files + +We're going to be using a couple of YAML files in our examples. + +.. code-block:: yaml + :caption: invoice.yaml + + # invoice.yaml + invoice: 34843 + date : !!str 2001-01-23 + bill-to: &id001 + given : Chris + family : Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + +.. code-block:: yaml + :caption: simple-anchors.yaml + + # simple-anchors.yaml + foo: &label { bar: frooz } + baz: *label + +.. code-block:: yaml + :caption: mergekeyspec.yaml + + --- + - &CENTER { x: 1, y: 2 } + - &LEFT { x: 0, y: 2 } + - &BIG { r: 10 } + - &SMALL { r: 1 } + + # All the following maps are equal: + + - # Explicit keys + x: 1 + y: 2 + r: 10 + label: center/big + + - # Merge one map + << : *CENTER + r: 10 + label: center/big + + - # Merge multiple maps + << : [ *CENTER, *BIG ] + label: center/big + + - # Override + << : [ *BIG, *LEFT, *SMALL ] + x: 1 + label: center/big + + +.. code-block:: yaml + :caption: bomb.yaml + + a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"] + b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a] + c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b] + d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c] + e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d] + f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e] + g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f] + +.. rubric:: fy-dump examples. + +Parse and dump generated YAML document tree in the original YAML form + +.. code-block:: bash + + $ fy-dump invoice.yaml + +.. code-block:: yaml + + invoice: 34843 + date: !!str 2001-01-23 + bill-to: &id001 + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + +Parse and dump generated YAML document tree in flow YAML form + +.. code-block:: bash + + $ fy-dump -mflow invoice.yaml + +.. code-block:: yaml + + { + invoice: 34843, + date: !!str 2001-01-23, + bill-to: &id001 { + given: Chris, + family: Dumars, + address: { + lines: "458 Walkman Dr.\nSuite #292\n" + } + } + } + +Parse and dump generated YAML document from the input string + +.. code-block:: bash + + $ fy-dump -mjson ">foo: bar" + +.. code-block:: json + + { + "foo": "bar" + } + +Using the resolve option on the `simple-anchors.yaml` + +.. code-block:: bash + + $ fy-dump -r simple-anchor.yaml + +.. code-block:: yaml + + foo: &label { + bar: frooz + } + baz: { + bar: frooz + } + +Stripping the labels too: + +.. code-block:: bash + + $ fy-dump -r --strip-label simple-anchor.yaml + +.. code-block:: yaml + + foo: { + bar: frooz + } + baz: { + bar: frooz + } + +Merge key support: + +.. code-block:: bash + + $ fy-dump -r --strip-label mergekeyspec.yaml + +.. code-block:: yaml + + --- + - { + x: 1, + y: 2 + } + - { + x: 0, + y: 2 + } + - { + r: 10 + } + - { + r: 1 + } + - x: 1 + y: 2 + r: 10 + label: center/big + - y: 2 + x: 1 + r: 10 + label: center/big + - r: 10 + y: 2 + x: 1 + label: center/big + - y: 2 + r: 10 + x: 1 + label: center/big + +Sorting option: + +.. code-block:: bash + + $ fy-dump -s invoice.yaml + +.. code-block:: yaml + + bill-to: &id001 + address: + lines: | + 458 Walkman Dr. + Suite #292 + family: Dumars + given: Chris + date: !!str 2001-01-23 + invoice: 34843 + +.. rubric:: fy-testsuite example. + +An example using the testsuite mode generates the following +event stream from `invoice.yaml` + +Parse and dump test-suite event format + +.. code-block:: bash + + $ fy-testsuite invoice.yaml + +.. code-block:: + + +STR + +DOC + +MAP + =VAL :invoice + =VAL :34843 + =VAL :date + =VAL :2001-01-23 + =VAL :bill-to + +MAP &id001 + =VAL :given + =VAL :Chris + =VAL :family + =VAL :Dumars + =VAL :address + +MAP + =VAL :lines + =VAL |458 Walkman Dr.\nSuite #292\n + -MAP + -MAP + -MAP + -DOC + -STR + +.. rubric:: fy-filter examples. + +Filter out from the `/bill-to` path of `invoice.yaml` + +.. code-block:: bash + + $ cat invoice.yaml | fy-filter /bill-to + +.. code-block:: yaml + + &id001 + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + +Filter example with arrays (and use the --file option) + +.. code-block:: bash + + $ fy-filter --file=mergekeyspec.yaml /5 + +.. code-block:: yaml + + --- + <<: *CENTER + r: 10 + label: center/big + +Follow anchors example + +.. code-block:: bash + + $ fy-filter --file=simple-anchors.yaml /baz/bar + +.. code-block:: yaml + + frooz + +Handle YAML bombs (if you can spare the memory and cpu time) + +.. code-block:: bash + + $ fy-filter --file=bomb.yaml -r / | wc -l + 6726047 + +You don\'t have to, you can just follow links to retrieve data. + +.. code-block:: bash + + $ fy-filter --file=stuff/bomb.yaml -l --strip-label /g/0/1/2/3/4/5/6 + +.. code-block:: yaml + + "lol" + +Following links works with merge keys too: + +.. code-block:: bash + + $ fy-filter --file=mergekeyspec.yaml -l --strip-label /5/x + +.. code-block:: yaml + + --- 1 + +.. rubric:: fy-join examples. + +Joining two YAML files that have root mappings. + +.. code-block:: bash + + $ fy-join simple-anchors.yaml invoice.yaml + +.. code-block:: yaml + + foo: &label { + bar: frooz + } + baz: *label + invoice: 34843 + date: !!str 2001-01-23 + bill-to: &id001 + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + +Join two files with sequences at root: + +.. code-block:: bash + + $ fy-join -mblock ">[ foo, bar ]" ">[ baz ]" + +.. code-block:: yaml + + - foo + - bar + - baz + +Author +------ + +Pantelis Antoniou + +Bugs +---- + +* The only supported input and output character encoding is UTF8. +* Sorting does not respect language settings. +* There is no way for the user to specific a different coloring scheme. + +See also +-------- + +:manpage:`libfyaml(1)` diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 0000000..1400b77 --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,70 @@ +set(CMAKE_PREFIX_PATH "/home/panto/work/libfyaml/build") +cmake_minimum_required(VERSION 3.12) +project(libfyaml-examples C) + +# Find libfyaml +find_package(libfyaml 0.9 QUIET) + +if(NOT libfyaml_FOUND) + # Fall back to pkg-config + find_package(PkgConfig REQUIRED) + pkg_check_modules(LIBFYAML REQUIRED libfyaml) + + # Create imported target for consistency + add_library(libfyaml::libfyaml INTERFACE IMPORTED) + set_target_properties(libfyaml::libfyaml PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${LIBFYAML_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES "${LIBFYAML_LIBRARIES}" + INTERFACE_LINK_DIRECTORIES "${LIBFYAML_LIBRARY_DIRS}" + INTERFACE_COMPILE_OPTIONS "${LIBFYAML_CFLAGS_OTHER}" + ) +endif() + +# List of all examples +set(EXAMPLES + quick-start + basic-parsing + path-queries + document-manipulation + event-streaming + build-from-scratch +) + +# Build each example +foreach(example ${EXAMPLES}) + add_executable(${example} ${example}.c) + target_link_libraries(${example} PRIVATE libfyaml::libfyaml) + + # Set C standard + set_property(TARGET ${example} PROPERTY C_STANDARD 99) + + # Enable warnings + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${example} PRIVATE -Wall -Wextra) + endif() +endforeach() + +# Installation (optional) +install(TARGETS ${EXAMPLES} + RUNTIME DESTINATION bin/libfyaml-examples +) + +# Install sample files +install(FILES + README.md + config.yaml + invoice.yaml + DESTINATION share/doc/libfyaml/examples +) + +# Show configuration info +message(STATUS "libfyaml examples configuration:") +message(STATUS " Examples to build: ${EXAMPLES}") +if(libfyaml_FOUND) + message(STATUS " Found libfyaml via CMake config") + if(libfyaml_HAS_LIBCLANG) + message(STATUS " libfyaml has reflection support") + endif() +else() + message(STATUS " Found libfyaml via pkg-config") +endif() diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..f67e682 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,281 @@ +# libfyaml Examples + +This directory contains practical examples demonstrating various features of libfyaml. + +## Building the Examples + +### Using CMake (Recommended) + +```bash +cd examples +mkdir build && cd build +cmake .. +cmake --build . +``` + +The compiled examples will be in the `build/` directory. + +### Using Make + +```bash +cd examples +make +``` + +The compiled examples will be in the current directory. + +### Manual Compilation + +```bash +gcc -o quick-start quick-start.c $(pkg-config --cflags --libs libfyaml) +gcc -o basic-parsing basic-parsing.c $(pkg-config --cflags --libs libfyaml) +# ... and so on for other examples +``` + +## Examples Overview + +### 1. quick-start.c + +Example showing: +- Parsing YAML from a file +- Extracting values using `fy_document_scanf()` (path-based queries) +- Modifying the document with `fy_document_insert_at()` +- Emitting the updated YAML + +**Usage**: +```bash +./quick-start [config.yaml] +``` + +**Required YAML structure** (for config.yaml): +```yaml +server: + host: localhost + port: 8080 +``` + +--- + +### 2. basic-parsing.c + +Basic YAML parsing and different output formats: +- Parse YAML from file +- Emit as compact flow format +- Emit as standard block YAML +- Emit as JSON + +**Usage**: +```bash +./basic-parsing [file.yaml] +``` + +Works with any valid YAML file. + +--- + +### 3. path-queries.c + +Query YAML documents using path expressions: +- Extract multiple values with `fy_document_scanf()` +- Query individual nodes with `fy_node_by_path()` +- Compare node values with strings +- Handle missing paths gracefully + +**Usage**: +```bash +./path-queries +``` + +This example uses inline YAML, so no external file is needed. + +--- + +### 4. document-manipulation.c + +Example of document manipulation: +- Read existing values from YAML +- Update values (replace invoice number) +- Add new fields (spouse, delivery address) +- Emit with sorted keys + +**Usage**: +```bash +./document-manipulation [invoice.yaml] +``` + +**Expected YAML structure** (for invoice.yaml): +```yaml +invoice: 34843 +date: 2001-01-23 +bill-to: + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 +``` + +If no file is provided, the example uses built-in default data. + +--- + +### 5. event-streaming.c + +Demonstrates the low-level event-based API: +- Create parser and set input +- Process events one by one +- Handle different event types (scalars, mappings, sequences) +- Memory-efficient parsing for large files + +**Usage**: +```bash +./event-streaming [file.yaml] +``` + +This is the most memory-efficient way to parse YAML, suitable for very large files. + +--- + +### 6. build-from-scratch.c + +Create YAML documents from scratch: +- Create empty document +- Build complex structures using `fy_node_buildf()` with printf-style formatting +- Add fields programmatically +- Emit as both JSON and YAML + +**Usage**: +```bash +./build-from-scratch +``` + +No input file needed - generates a complete configuration document. + +--- + +## Sample YAML Files + +Sample YAML files are provided for testing: + +- **config.yaml** - Simple server configuration +- **invoice.yaml** - Invoice document for manipulation example + +Create these files in the examples directory, or the programs will use sensible defaults. + +### Sample config.yaml + +```yaml +server: + host: localhost + port: 8080 + ssl: true + max_connections: 100 + +database: + host: db.example.com + port: 5432 + name: production_db + +logging: + level: info + file: /var/log/app.log +``` + +### Sample invoice.yaml + +```yaml +invoice: 34843 +date: 2001-01-23 +bill-to: + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + city: Royal Oak + state: MI + postal: 48046 +ship-to: + given: Chris + family: Dumars +product: + - sku: BL394D + quantity: 4 + description: Basketball + price: 450.00 + - sku: BL4438H + quantity: 1 + description: Super Hoop + price: 2392.00 +tax: 251.42 +total: 4443.52 +comments: > + Late afternoon is best. + Backup contact is Nancy + Billsmer @ 338-4338. +``` + +## API Reference + +For complete API documentation, visit: https://pantoniou.github.io/libfyaml/ + +## Common Patterns + +### Error Handling + +All examples demonstrate proper error handling: +```c +struct fy_document *fyd = fy_document_build_from_file(NULL, "file.yaml"); +if (!fyd) { + fprintf(stderr, "Failed to parse YAML\n"); + return EXIT_FAILURE; +} + +// ... use document ... + +fy_document_destroy(fyd); // Always clean up +``` + +### Path-Based Queries + +Extract multiple values efficiently: +```c +int count = fy_document_scanf(fyd, + "/path1 %s " + "/path2 %d", + string_var, &int_var); + +if (count != 2) { + // Handle error +} +``` + +### Document Modification + +Add or update fields: +```c +fy_document_insert_at(fyd, "/parent/path", FY_NT, + fy_node_buildf(fyd, "key: value")); +``` + +### Output Formatting + +Choose output mode based on needs: +```c +// Compact JSON +fy_emit_document_to_fp(fyd, FYECF_MODE_JSON, stdout); + +// Pretty YAML with sorted keys +fy_emit_document_to_fp(fyd, FYECF_MODE_BLOCK | FYECF_SORT_KEYS, stdout); + +// One-line flow format +fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stdout); +``` + +## License + +All examples are released under the MIT License, same as libfyaml. + +Copyright (c) 2019-2025 Pantelis Antoniou diff --git a/examples/basic-parsing.c b/examples/basic-parsing.c new file mode 100644 index 0000000..06c60e2 --- /dev/null +++ b/examples/basic-parsing.c @@ -0,0 +1,50 @@ +/* + * basic-parsing.c - Basic YAML parsing example + * + * Demonstrates: + * - Parsing YAML from a file + * - Basic error handling + * - Emitting in different output modes + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct fy_document *fyd = NULL; + const char *input_file = (argc > 1) ? argv[1] : "config.yaml"; + + // Parse YAML from file + fyd = fy_document_build_from_file(NULL, input_file); + if (!fyd) { + fprintf(stderr, "Failed to parse YAML from %s\n", input_file); + return EXIT_FAILURE; + } + + printf("Successfully parsed: %s\n\n", input_file); + + // Emit as flow-oneline (compact format) + printf("Compact format:\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stdout); + printf("\n\n"); + + // Emit as block (standard YAML) + printf("Block format:\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_BLOCK, stdout); + printf("\n"); + + // Emit as JSON + printf("JSON format:\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_JSON, stdout); + printf("\n"); + + fy_document_destroy(fyd); + + return EXIT_SUCCESS; +} diff --git a/examples/build-from-scratch.c b/examples/build-from-scratch.c new file mode 100644 index 0000000..e8f0a5b --- /dev/null +++ b/examples/build-from-scratch.c @@ -0,0 +1,111 @@ +/* + * build-from-scratch.c - Building YAML documents programmatically + * + * Demonstrates: + * - Creating empty documents + * - Building nodes using printf-style formatting + * - Setting document root + * - Adding fields programmatically + * - Emitting as both JSON and YAML + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include + +int main(void) +{ + struct fy_document *fyd = NULL; + struct fy_node *root = NULL; + time_t now = time(NULL); + struct tm *tm_info = localtime(&now); + char timestamp[32]; + int ret = EXIT_FAILURE; + + strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", tm_info); + + // Create empty document + fyd = fy_document_create(NULL); + if (!fyd) { + fprintf(stderr, "Failed to create document\n"); + return EXIT_FAILURE; + } + + // Build root mapping using printf-style formatting + root = fy_node_buildf(fyd, + "application: MyApp\n" + "version: %d.%d.%d\n" + "build_date: %s\n" + "settings:\n" + " debug: %s\n" + " max_connections: %d\n" + " allowed_hosts:\n" + " - localhost\n" + " - 127.0.0.1\n", + 1, 2, 3, + timestamp, + "true", + 100); + + if (!root) { + fprintf(stderr, "Failed to build root node\n"); + goto cleanup; + } + + // Set as document root + fy_document_set_root(fyd, root); + + // Add more fields programmatically + if (fy_document_insert_at(fyd, "/settings", FY_NT, + fy_node_buildf(fyd, "log_level: info"))) { + fprintf(stderr, "Failed to add log_level\n"); + goto cleanup; + } + + if (fy_document_insert_at(fyd, "/settings", FY_NT, + fy_node_buildf(fyd, + "database:\n" + " host: localhost\n" + " port: 5432\n" + " name: myapp_db\n"))) { + fprintf(stderr, "Failed to add database config\n"); + goto cleanup; + } + + // Add feature flags + if (fy_document_insert_at(fyd, "/", FY_NT, + fy_node_buildf(fyd, + "features:\n" + " authentication: enabled\n" + " api_v2: disabled\n" + " metrics: enabled\n"))) { + fprintf(stderr, "Failed to add features\n"); + goto cleanup; + } + + // Output as JSON + printf("=== JSON Output ===\n"); + if (fy_emit_document_to_fp(fyd, FYECF_MODE_JSON, stdout)) { + fprintf(stderr, "Failed to emit as JSON\n"); + goto cleanup; + } + printf("\n\n"); + + // Output as YAML with sorted keys + printf("=== YAML Output (sorted) ===\n"); + if (fy_emit_document_to_fp(fyd, FYECF_MODE_BLOCK | FYECF_SORT_KEYS, stdout)) { + fprintf(stderr, "Failed to emit as YAML\n"); + goto cleanup; + } + + ret = EXIT_SUCCESS; + +cleanup: + fy_document_destroy(fyd); + return ret; +} diff --git a/examples/config.yaml b/examples/config.yaml new file mode 100644 index 0000000..73353c3 --- /dev/null +++ b/examples/config.yaml @@ -0,0 +1,40 @@ +# Sample configuration file for libfyaml examples +# This file demonstrates a typical application configuration + +server: + host: localhost + port: 8080 + ssl: true + max_connections: 100 + timeout: 30 + +database: + host: db.example.com + port: 5432 + name: production_db + username: dbuser + pool_size: 10 + +logging: + level: info + file: /var/log/app.log + rotate: daily + max_size: 100M + +cache: + enabled: true + type: redis + host: cache.example.com + port: 6379 + ttl: 3600 + +features: + authentication: enabled + api_v2: disabled + metrics: enabled + debug_mode: false + +allowed_origins: + - https://example.com + - https://www.example.com + - https://app.example.com diff --git a/examples/document-manipulation.c b/examples/document-manipulation.c new file mode 100644 index 0000000..8e5770d --- /dev/null +++ b/examples/document-manipulation.c @@ -0,0 +1,111 @@ +/* + * document-manipulation.c - Document manipulation example + * + * Demonstrates: + * - Reading values from a YAML document + * - Modifying existing values + * - Adding new fields to the document + * - Emitting with sorted keys + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct fy_document *fyd = NULL; + const char *input_file = (argc > 1) ? argv[1] : "invoice.yaml"; + unsigned int invoice_nr; + char given_name[256]; + int ret = EXIT_FAILURE; + + // Create a default invoice if file doesn't exist + const char *default_invoice = + "invoice: 34843\n" + "date: 2001-01-23\n" + "bill-to:\n" + " given: Chris\n" + " family: Dumars\n" + " address:\n" + " lines: |\n" + " 458 Walkman Dr.\n" + " Suite #292\n" + "product:\n" + " - sku: BL394D\n" + " quantity: 4\n" + " description: Basketball\n" + " price: 450.00\n"; + + // Try to load from file, fall back to default + fyd = fy_document_build_from_file(NULL, input_file); + if (!fyd) { + fprintf(stderr, "Note: Creating document from default data\n"); + fprintf(stderr, " (Create invoice.yaml or pass file as argument)\n\n"); + fyd = fy_document_build_from_string(NULL, default_invoice, FY_NT); + if (!fyd) { + fprintf(stderr, "Failed to create document\n"); + goto cleanup; + } + } + + // Read current invoice number and customer name + int count = fy_document_scanf(fyd, + "/invoice %u " + "/bill-to/given %255s", + &invoice_nr, given_name); + + if (count != 2) { + fprintf(stderr, "Failed to extract invoice data\n"); + goto cleanup; + } + + printf("Processing invoice #%u for %s\n\n", invoice_nr, given_name); + + // Update invoice number (replaces existing value) + if (fy_document_insert_at(fyd, "/invoice", FY_NT, + fy_node_buildf(fyd, "%u", invoice_nr + 1))) { + fprintf(stderr, "Failed to update invoice number\n"); + goto cleanup; + } + printf("Updated invoice number to %u\n", invoice_nr + 1); + + // Add spouse field to bill-to + if (fy_document_insert_at(fyd, "/bill-to", FY_NT, + fy_node_buildf(fyd, "spouse: Jane"))) { + fprintf(stderr, "Failed to add spouse\n"); + goto cleanup; + } + printf("Added spouse information\n"); + + // Add delivery address + if (fy_document_insert_at(fyd, "/bill-to", FY_NT, + fy_node_buildf(fyd, + "delivery-address:\n" + " street: 123 Main St\n" + " city: Springfield\n" + " state: IL\n" + " zip: 62701\n"))) { + fprintf(stderr, "Failed to add delivery address\n"); + goto cleanup; + } + printf("Added delivery address\n"); + + // Emit updated document with sorted keys + printf("\n--- Updated Invoice ---\n"); + if (fy_emit_document_to_fp(fyd, FYECF_DEFAULT | FYECF_SORT_KEYS, stdout)) { + fprintf(stderr, "Failed to emit document\n"); + goto cleanup; + } + + ret = EXIT_SUCCESS; + +cleanup: + fy_document_destroy(fyd); + return ret; +} diff --git a/examples/event-streaming.c b/examples/event-streaming.c new file mode 100644 index 0000000..5f2c441 --- /dev/null +++ b/examples/event-streaming.c @@ -0,0 +1,117 @@ +/* + * event-streaming.c - Event-based streaming parser example + * + * Demonstrates: + * - Event-based (streaming) YAML parsing + * - Processing different event types + * - Memory-efficient parsing of large files + * - Getting values from scalar events + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct fy_parser *fyp = NULL; + struct fy_event *fye = NULL; + const char *input_file = (argc > 1) ? argv[1] : "config.yaml"; + int indent = 0; + int ret = EXIT_FAILURE; + + // Create parser + fyp = fy_parser_create(NULL); + if (!fyp) { + fprintf(stderr, "Failed to create parser\n"); + return EXIT_FAILURE; + } + + // Set input file + if (fy_parser_set_input_file(fyp, input_file)) { + fprintf(stderr, "Failed to set input file: %s\n", input_file); + fprintf(stderr, "Note: Create config.yaml or pass file as argument\n"); + goto cleanup; + } + + printf("Parsing events from: %s\n\n", input_file); + + // Process events + while ((fye = fy_parser_parse(fyp)) != NULL) { + enum fy_event_type type = fye->type; + + // Print indentation + for (int i = 0; i < indent; i++) + printf(" "); + + switch (type) { + case FYET_STREAM_START: + printf("STREAM-START\n"); + break; + + case FYET_STREAM_END: + printf("STREAM-END\n"); + break; + + case FYET_DOCUMENT_START: + printf("DOCUMENT-START\n"); + indent++; + break; + + case FYET_DOCUMENT_END: + indent--; + printf("DOCUMENT-END\n"); + break; + + case FYET_MAPPING_START: + printf("MAPPING-START\n"); + indent++; + break; + + case FYET_MAPPING_END: + indent--; + printf("MAPPING-END\n"); + break; + + case FYET_SEQUENCE_START: + printf("SEQUENCE-START\n"); + indent++; + break; + + case FYET_SEQUENCE_END: + indent--; + printf("SEQUENCE-END\n"); + break; + + case FYET_SCALAR: { + // Get the scalar value + const char *value = fy_token_get_text0(fy_event_get_token(fye)); + printf("SCALAR: \"%s\"\n", value ? value : "(null)"); + break; + } + + case FYET_ALIAS: { + const char *anchor = fy_token_get_text0(fy_event_get_token(fye)); + printf("ALIAS: *%s\n", anchor ? anchor : "(null)"); + break; + } + + default: + printf("OTHER-EVENT (type=%d)\n", type); + break; + } + + fy_parser_event_free(fyp, fye); + } + + printf("\nParsing completed successfully\n"); + ret = EXIT_SUCCESS; + +cleanup: + fy_parser_destroy(fyp); + return ret; +} diff --git a/examples/invoice.yaml b/examples/invoice.yaml new file mode 100644 index 0000000..6de053b --- /dev/null +++ b/examples/invoice.yaml @@ -0,0 +1,42 @@ +# Sample invoice YAML for document manipulation example +# Based on the YAML specification examples + +invoice: 34843 +date: 2001-01-23 +bill-to: + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + city: Royal Oak + state: MI + postal: 48046 + country: USA +ship-to: + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + city: Royal Oak + state: MI + postal: 48046 + country: USA +product: + - sku: BL394D + quantity: 4 + description: Basketball + price: 450.00 + - sku: BL4438H + quantity: 1 + description: Super Hoop + price: 2392.00 +tax: 251.42 +total: 4443.52 +comments: > + Late afternoon is best. + Backup contact is Nancy + Billsmer @ 338-4338. diff --git a/examples/path-queries.c b/examples/path-queries.c new file mode 100644 index 0000000..19a97fb --- /dev/null +++ b/examples/path-queries.c @@ -0,0 +1,80 @@ +/* + * path-queries.c - Path-based YAML queries example + * + * Demonstrates: + * - Using YPATH expressions to query YAML + * - scanf-style value extraction + * - Single node lookup by path + * - String comparison with nodes + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +int main(void) +{ + const char *yaml = + "server:\n" + " host: localhost\n" + " port: 8080\n" + " ssl: true\n" + " max_connections: 100\n"; + + struct fy_document *fyd = fy_document_build_from_string(NULL, yaml, FY_NT); + if (!fyd) { + fprintf(stderr, "Failed to parse YAML\n"); + return EXIT_FAILURE; + } + + // Extract multiple values at once using scanf-style API + char host[256]; + unsigned int port; + int count = fy_document_scanf(fyd, + "/server/host %255s " + "/server/port %u", + host, &port); + + if (count == 2) { + printf("Server configuration:\n"); + printf(" Host: %s\n", host); + printf(" Port: %u\n", port); + } else { + fprintf(stderr, "Failed to extract server configuration (got %d/2)\n", count); + } + + // Query single node by path + struct fy_node *ssl_node = fy_node_by_path( + fy_document_root(fyd), "/server/ssl", FY_NT, FYNWF_DONT_FOLLOW); + + if (ssl_node) { + // Compare node value with string + if (fy_node_compare_string(ssl_node, "true", FY_NT) == 0) { + printf(" SSL: enabled\n"); + } else { + printf(" SSL: disabled\n"); + } + } + + // Extract another value + unsigned int max_conn; + count = fy_document_scanf(fyd, "/server/max_connections %u", &max_conn); + if (count == 1) { + printf(" Max connections: %u\n", max_conn); + } + + // Try to query a non-existent path + struct fy_node *missing = fy_node_by_path( + fy_document_root(fyd), "/server/timeout", FY_NT, FYNWF_DONT_FOLLOW); + + if (!missing) { + printf(" Timeout: not configured (using default)\n"); + } + + fy_document_destroy(fyd); + return EXIT_SUCCESS; +} diff --git a/examples/quick-start.c b/examples/quick-start.c new file mode 100644 index 0000000..b2cc515 --- /dev/null +++ b/examples/quick-start.c @@ -0,0 +1,70 @@ +/* + * quick-start.c - Quick start example for libfyaml + * + * Demonstrates: + * - Parsing YAML from a file + * - Extracting values using path-based scanf + * - Modifying the document + * - Emitting as YAML + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct fy_document *fyd = NULL; + unsigned int port; + char hostname[256]; + int ret = EXIT_FAILURE; + const char *input_file = (argc > 1) ? argv[1] : "config.yaml"; + + // Parse YAML from string or file + fyd = fy_document_build_from_file(NULL, input_file); + if (!fyd) { + fprintf(stderr, "Failed to parse %s\n", input_file); + fprintf(stderr, "Note: You can create config.yaml or pass a file as argument\n"); + goto cleanup; + } + + // Extract values using path-based scanf + int count = fy_document_scanf(fyd, + "/server/port %u " + "/server/host %255s", + &port, hostname); + + if (count != 2) { + fprintf(stderr, "Failed to extract server configuration\n"); + fprintf(stderr, "Expected /server/port and /server/host in YAML\n"); + goto cleanup; + } + + printf("Current configuration: %s:%u\n", hostname, port); + + // Modify document using path-based insertion + if (fy_document_insert_at(fyd, "/server", FY_NT, + fy_node_buildf(fyd, "timeout: 30"))) { + fprintf(stderr, "Failed to insert timeout setting\n"); + goto cleanup; + } + + printf("Added timeout setting\n"); + + // Emit as YAML + printf("\nUpdated configuration:\n"); + if (fy_emit_document_to_fp(fyd, FYECF_SORT_KEYS, stdout)) { + fprintf(stderr, "Failed to emit document\n"); + goto cleanup; + } + + ret = EXIT_SUCCESS; + +cleanup: + fy_document_destroy(fyd); + return ret; +} diff --git a/include/libfyaml.h b/include/libfyaml.h new file mode 100644 index 0000000..9a0acee --- /dev/null +++ b/include/libfyaml.h @@ -0,0 +1,9912 @@ +/* + * libfyaml.h - Main header file of the public interface + * + * Copyright (c) 2019-2021 Pantelis Antoniou + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * SPDX-License-Identifier: MIT + */ + +#ifndef LIBFYAML_H +#define LIBFYAML_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) +#include +#endif + +#include + +/* opaque types for the user */ +struct fy_token; +struct fy_document_state; +struct fy_parser; +struct fy_emitter; +struct fy_document; +struct fy_node; +struct fy_node_pair; +struct fy_anchor; +struct fy_node_mapping_sort_ctx; +struct fy_token_iter; +struct fy_diag; +struct fy_path_parser; +struct fy_path_expr; +struct fy_path_exec; +struct fy_path_component; +struct fy_path; +struct fy_composer; +struct fy_document_iterator; +struct fy_document_builder; + +#ifndef FY_BIT +#define FY_BIT(x) (1U << (x)) +#endif + +/* NULL terminated string length specifier */ +#define FY_NT ((size_t)-1) + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define FY_EXPORT __attribute__ ((visibility ("default"))) +#define FY_DEPRECATED __attribute__ ((deprecated)) +#define FY_FORMAT(_t, _x, _y) __attribute__ ((format(_t, _x, _y))) +#else +#define FY_EXPORT /* nothing */ +#define FY_DEPRECATED /* nothing */ +#define FY_FORMAT(_t, x, y) /* nothing */ +#endif + +/* make a copy of an allocated string and return it on stack + * note that this is a convenience, and should not be called + * in a loop. The string is always '\0' terminated. + * If the _str pointer is NULL, then NULL will be returned + */ +#ifndef FY_ALLOCA_COPY_FREE +#define FY_ALLOCA_COPY_FREE(_str, _len) \ + ({ \ + char *__str = (_str), *__stra = NULL; \ + size_t __len = (size_t)(_len); \ + \ + if (__str) { \ + if (__len == FY_NT) \ + __len = strlen(__str); \ + __stra = alloca(__len + 1); \ + memcpy(__stra, __str, __len); \ + __stra[__len] = '\0'; \ + free(__str); \ + } \ + (const char *)__stra; \ + }) +#endif + +/* same as above but when _str == NULL return "" */ +#ifndef FY_ALLOCA_COPY_FREE_NO_NULL +#define FY_ALLOCA_COPY_FREE_NO_NULL(_str, _len) \ + ({ \ + const char *__strb; \ + \ + __strb = FY_ALLOCA_COPY_FREE(_str, _len); \ + if (!__strb) \ + __strb = ""; \ + __strb; \ + }) +#endif + +/** + * DOC: libfyaml public API + * + */ + +/** + * struct fy_version - The YAML version + * + * @major: Major version number + * @minor: Major version number + * + * The parser fills it according to the \%YAML directive + * found in the document. + */ +struct fy_version { + int major; + int minor; +}; + +/* Build a fy_version * from the given major and minor */ +#define fy_version_make(_maj, _min) (&(struct fy_version){ (_maj), (_min) }) + +/** + * fy_version_compare() - Compare two yaml versions + * + * Compare the versions + * + * @va: The first version, if NULL use default version + * @vb: The second version, if NULL use default version + * + * Returns: + * 0 if versions are equal, > 0 if version va is higher than vb + * < 0 if version va is lower than vb + */ +int +fy_version_compare(const struct fy_version *va, const struct fy_version *vb) + FY_EXPORT; + +/** + * fy_version_default() - Get the default version of the library + * + * Return the default version of the library, i.e. the highest + * stable version that is supported. + * + * Returns: + * The default YAML version of this library + */ +const struct fy_version * +fy_version_default(void) + FY_EXPORT; + +/** + * fy_version_is_supported() - Check if supported version + * + * Check if the given YAML version is supported. + * + * @vers: The version to check, NULL means default version. + * + * Returns: + * true if version supported, false otherwise. + */ +bool +fy_version_is_supported(const struct fy_version *vers) + FY_EXPORT; + +/** + * fy_version_supported_iterate() - Iterate over the supported YAML versions + * + * This method iterates over the supported YAML versions of this ibrary. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * @prevp: The previous version iterator + * + * Returns: + * The next node in sequence or NULL at the end of the sequence. + */ +const struct fy_version * +fy_version_supported_iterate(void **prevp) + FY_EXPORT; + +/** + * fy_shutdown() - Final cleanup before exit + * + * Some libraries *cough*libclang** need explicit cleanup calls + * at the end of program execution, even if you've never called + * any of their functions. + * + * This method will make sure it calls their cleanup functions + * so that no memory leaks are reported in valgrind etc. + */ +void +fy_shutdown(void) + FY_EXPORT; + +/** + * struct fy_tag - The YAML tag structure. + * + * @handle: Handle of the tag (i.e. `"!!"`) + * @prefix: The prefix of the tag (i.e. `"tag:yaml.org,2002:"`) + * The parser fills it according to the \%TAG directives + * encountered during parsing. + */ +struct fy_tag { + const char *handle; + const char *prefix; +}; + +/** + * struct fy_mark - marker holding information about a location + * in a &struct fy_input + * + * @input_pos: Position of the mark (from the start of the current input) + * @line: Line position (0 index based) + * @column: Column position (0 index based) + */ +struct fy_mark { + size_t input_pos; + int line; + int column; +}; + +/** + * enum fy_error_type - The supported diagnostic/error types + * + * @FYET_DEBUG: Debug level (disabled if library is not compiled in debug mode) + * @FYET_INFO: Informational level + * @FYET_NOTICE: Notice level + * @FYET_WARNING: Warning level + * @FYET_ERROR: Error level - error reporting is using this level + * @FYET_MAX: Non inclusive maximum fy_error_type value + * + */ +enum fy_error_type { + FYET_DEBUG, + FYET_INFO, + FYET_NOTICE, + FYET_WARNING, + FYET_ERROR, + FYET_MAX, +}; + +/** + * enum fy_error_module - Module which generated the diagnostic/error + * + * @FYEM_UNKNOWN: Unknown, default if not specific + * @FYEM_ATOM: Atom module, used by atom chunking + * @FYEM_SCAN: Scanner module + * @FYEM_PARSE: Parser module + * @FYEM_DOC: Document module + * @FYEM_BUILD: Build document module (after tree is constructed) + * @FYEM_INTERNAL: Internal error/diagnostic module + * @FYEM_SYSTEM: System error/diagnostic module + * @FYEM_EMIT: Emitter module + * @FYEM_TYPESET: Prepare types module (C reflection) + * @FYEM_DECODE: Decode, serialization -> internal form module + * @FYEM_ENCODE: Encode, internal form -> serialized form module + * @FYEM_MAX: Non inclusive maximum fy_error_module value + */ +enum fy_error_module { + FYEM_UNKNOWN, + FYEM_ATOM, + FYEM_SCAN, + FYEM_PARSE, + FYEM_DOC, + FYEM_BUILD, + FYEM_INTERNAL, + FYEM_SYSTEM, + FYEM_EMIT, + FYEM_TYPESET, + FYEM_DECODE, + FYEM_ENCODE, + FYEM_MAX, +}; + +/* Shift amount of the default version */ +#define FYPCF_DEFAULT_VERSION_SHIFT 9 +/* Mask of the default version */ +#define FYPCF_DEFAULT_VERSION_MASK ((1U << 5) - 1) +/* Build a default version */ +#define FYPCF_DEFAULT_VERSION(x) (((unsigned int)(x) & FYPCF_DEFAULT_VERSION_MASK) << FYPCF_DEFAULT_VERSION_SHIFT) + +/* Shift amount of the JSON input mode */ +#define FYPCF_JSON_SHIFT 16 +/* Mask of the JSON input mode */ +#define FYPCF_JSON_MASK ((1U << 2) - 1) +/* Build a JSON input mode option */ +#define FYPCF_JSON(x) (((unsigned int)(x) & FYPCF_JSON_MASK) << FYPCF_JSON_SHIFT) + +/* guaranteed minimum depth limit for generated document */ +/* the actual limit is larger depending on the platform */ +#define FYPCF_GUARANTEED_MINIMUM_DEPTH_LIMIT 64 + +/** + * enum fy_parse_cfg_flags - Parse configuration flags + * + * These flags control the operation of the parse and the debugging + * output/error reporting via filling in the &fy_parse_cfg->flags member. + * + * @FYPCF_QUIET: Quiet, do not output any information messages + * @FYPCF_COLLECT_DIAG: Collect diagnostic/error messages + * @FYPCF_RESOLVE_DOCUMENT: When producing documents, automatically resolve them + * @FYPCF_DISABLE_MMAP_OPT: Disable mmap optimization + * @FYPCF_DISABLE_RECYCLING: Disable recycling optimization + * @FYPCF_PARSE_COMMENTS: Enable parsing of comments (experimental) + * @FYPCF_DISABLE_DEPTH_LIMIT: Disable depth limit check, use with enlarged stack + * @FYPCF_DISABLE_ACCELERATORS: Disable use of access accelerators (saves memory) + * @FYPCF_DISABLE_BUFFERING: Disable use of buffering where possible + * @FYPCF_DEFAULT_VERSION_AUTO: Automatically use the most recent version the library supports + * @FYPCF_DEFAULT_VERSION_1_1: Default version is YAML 1.1 + * @FYPCF_DEFAULT_VERSION_1_2: Default version is YAML 1.2 + * @FYPCF_DEFAULT_VERSION_1_3: Default version is YAML 1.3 (experimental) + * @FYPCF_SLOPPY_FLOW_INDENTATION: Allow sloppy indentation in flow mode + * @FYPCF_PREFER_RECURSIVE: Prefer recursive algorighms instead of iterative whenever possible + * @FYPCF_JSON_AUTO: Automatically enable JSON mode (when extension is .json) + * @FYPCF_JSON_NONE: Never enable JSON input mode + * @FYPCF_JSON_FORCE: Force JSON mode always + * @FYPCF_YPATH_ALIASES: Enable YPATH aliases mode + * @FYPCF_ALLOW_DUPLICATE_KEYS: Allow duplicate keys on mappings + */ +enum fy_parse_cfg_flags { + FYPCF_QUIET = FY_BIT(0), + FYPCF_COLLECT_DIAG = FY_BIT(1), + FYPCF_RESOLVE_DOCUMENT = FY_BIT(2), + FYPCF_DISABLE_MMAP_OPT = FY_BIT(3), + FYPCF_DISABLE_RECYCLING = FY_BIT(4), + FYPCF_PARSE_COMMENTS = FY_BIT(5), + FYPCF_DISABLE_DEPTH_LIMIT = FY_BIT(6), + FYPCF_DISABLE_ACCELERATORS = FY_BIT(7), + FYPCF_DISABLE_BUFFERING = FY_BIT(8), + FYPCF_DEFAULT_VERSION_AUTO = FYPCF_DEFAULT_VERSION(0), + FYPCF_DEFAULT_VERSION_1_1 = FYPCF_DEFAULT_VERSION(1), + FYPCF_DEFAULT_VERSION_1_2 = FYPCF_DEFAULT_VERSION(2), + FYPCF_DEFAULT_VERSION_1_3 = FYPCF_DEFAULT_VERSION(3), + FYPCF_SLOPPY_FLOW_INDENTATION = FY_BIT(14), + FYPCF_PREFER_RECURSIVE = FY_BIT(15), + FYPCF_JSON_AUTO = FYPCF_JSON(0), + FYPCF_JSON_NONE = FYPCF_JSON(1), + FYPCF_JSON_FORCE = FYPCF_JSON(2), + FYPCF_YPATH_ALIASES = FY_BIT(18), + FYPCF_ALLOW_DUPLICATE_KEYS = FY_BIT(19), +}; + +#define FYPCF_DEFAULT_PARSE (0) + +#define FYPCF_DEFAULT_DOC (FYPCF_QUIET | FYPCF_DEFAULT_PARSE) + +/* + * The FYPCF_DEBUG and FYPCF_COLOR flags have been removed, however + * to help with backwards compatibility we will define them as 0 + * so that code can continue to compile. + * + * You will need to eventualy modify the code if you actually depended + * on the old behaviour. + */ +#define FYPCF_MODULE_SHIFT 0 +#define FYPCF_MODULE_MASK 0 +#define FYPCF_DEBUG_LEVEL_SHIFT 0 +#define FYPCF_DEBUG_LEVEL_MASK 0 +#define FYPCF_DEBUG_LEVEL(x) 0 +#define FYPCF_DEBUG_DIAG_SHIFT 0 +#define FYPCF_DEBUG_DIAG_MASK 0 +#define FYPCF_DEBUG_DIAG_ALL 0 +#define FYPCF_DEBUG_DIAG_DEFAULT 0 +#define FYPCF_DEBUG_UNKNOWN 0 +#define FYPCF_DEBUG_ATOM 0 +#define FYPCF_DEBUG_SCAN 0 +#define FYPCF_DEBUG_PARSE 0 +#define FYPCF_DEBUG_DOC 0 +#define FYPCF_DEBUG_BUILD 0 +#define FYPCF_DEBUG_INTERNAL 0 +#define FYPCF_DEBUG_SYSTEM 0 +#define FYPCF_DEBUG_LEVEL_DEBUG 0 +#define FYPCF_DEBUG_LEVEL_INFO 0 +#define FYPCF_DEBUG_LEVEL_NOTICE 0 +#define FYPCF_DEBUG_LEVEL_WARNING 0 +#define FYPCF_DEBUG_LEVEL_ERROR 0 +#define FYPCF_DEBUG_DIAG_SOURCE 0 +#define FYPCF_DEBUG_DIAG_POSITION 0 +#define FYPCF_DEBUG_DIAG_TYPE 0 +#define FYPCF_DEBUG_DIAG_MODULE 0 +#define FYPCF_DEBUG_ALL 0 +#define FYPCF_DEBUG_DEFAULT 0 +#define FYPCF_COLOR_SHIFT 0 +#define FYPCF_COLOR_MASK 0 +#define FYPCF_COLOR(x) 0 +#define FYPCF_COLOR_AUTO 0 +#define FYPCF_COLOR_NONE 0 +#define FYPCF_COLOR_FORCE 0 + +/** + * struct fy_parse_cfg - parser configuration structure. + * + * Argument to the fy_parser_create() method which + * perform parsing of YAML files. + * + * @search_path: Search path when accessing files, seperate with ':' + * @flags: Configuration flags + * @userdata: Opaque user data pointer + * @diag: Optional diagnostic interface to use + */ +struct fy_parse_cfg { + const char *search_path; + enum fy_parse_cfg_flags flags; + void *userdata; + struct fy_diag *diag; +}; + +/** + * enum fy_event_type - Event types + * + * @FYET_NONE: No event + * @FYET_STREAM_START: Stream start event + * @FYET_STREAM_END: Stream end event + * @FYET_DOCUMENT_START: Document start event + * @FYET_DOCUMENT_END: Document end event + * @FYET_MAPPING_START: YAML mapping start event + * @FYET_MAPPING_END: YAML mapping end event + * @FYET_SEQUENCE_START: YAML sequence start event + * @FYET_SEQUENCE_END: YAML sequence end event + * @FYET_SCALAR: YAML scalar event + * @FYET_ALIAS: YAML alias event + */ +enum fy_event_type { + FYET_NONE, + FYET_STREAM_START, + FYET_STREAM_END, + FYET_DOCUMENT_START, + FYET_DOCUMENT_END, + FYET_MAPPING_START, + FYET_MAPPING_END, + FYET_SEQUENCE_START, + FYET_SEQUENCE_END, + FYET_SCALAR, + FYET_ALIAS, +}; + +/** + * fy_event_type_get_text() - Return text of an event type + * + * @type: The event type to get text from + * + * Returns: + * A pointer to a text, i.e for FYET_SCALAR "=VAL". + */ +const char * +fy_event_type_get_text(enum fy_event_type type) + FY_EXPORT; + +/** + * enum fy_scalar_style - Scalar styles supported by the parser/emitter + * + * @FYSS_ANY: Any scalar style, not generated by the parser. + * Lets the emitter to choose + * @FYSS_PLAIN: Plain scalar style + * @FYSS_SINGLE_QUOTED: Single quoted style + * @FYSS_DOUBLE_QUOTED: Double quoted style + * @FYSS_LITERAL: YAML literal block style + * @FYSS_FOLDED: YAML folded block style + * @FYSS_MAX: marks end of scalar styles + */ +enum fy_scalar_style { + FYSS_ANY = -1, + FYSS_PLAIN, + FYSS_SINGLE_QUOTED, + FYSS_DOUBLE_QUOTED, + FYSS_LITERAL, + FYSS_FOLDED, + FYSS_MAX, +}; + +/** + * struct fy_event_stream_start_data - stream start event data + * + * @stream_start: The token that started the stream + */ +struct fy_event_stream_start_data { + struct fy_token *stream_start; +}; + +/** + * struct fy_event_stream_end_data - stream end event data + * + * @stream_end: The token that ended the stream + */ +struct fy_event_stream_end_data { + struct fy_token *stream_end; +}; + +/** + * struct fy_event_document_start_data - doument start event data + * + * @document_start: The token that started the document, or NULL if + * the document was implicitly started. + * @document_state: The state of the document (i.e. information about + * the YAML version and configured tags) + * @implicit: True if the document started implicitly + */ +struct fy_event_document_start_data { + struct fy_token *document_start; + struct fy_document_state *document_state; + bool implicit; +}; + +/** + * struct fy_event_document_end_data - doument end event data + * + * @document_end: The token that ended the document, or NULL if the + * document was implicitly ended + * @implicit: True if the document ended implicitly + */ +struct fy_event_document_end_data { + struct fy_token *document_end; + bool implicit; +}; + +/** + * struct fy_event_alias_data - alias event data + * + * @anchor: The anchor token definining this alias. + */ +struct fy_event_alias_data { + struct fy_token *anchor; +}; + +/** + * struct fy_event_scalar_data - scalar event data + * + * @anchor: anchor token or NULL + * @tag: tag token or NULL + * @value: scalar value token (cannot be NULL) + * @tag_implicit: true if the tag was implicit or explicit + */ +struct fy_event_scalar_data { + struct fy_token *anchor; + struct fy_token *tag; + struct fy_token *value; + bool tag_implicit; +}; + +/** + * struct fy_event_sequence_start_data - sequence start event data + * + * @anchor: anchor token or NULL + * @tag: tag token or NULL + * @sequence_start: sequence start value token or NULL if the sequence + * was started implicitly + */ +struct fy_event_sequence_start_data { + struct fy_token *anchor; + struct fy_token *tag; + struct fy_token *sequence_start; +}; + +/** + * struct fy_event_sequence_end_data - sequence end event data + * + * @sequence_end: The token that ended the sequence, or NULL if + * the sequence was implicitly ended + */ +struct fy_event_sequence_end_data { + struct fy_token *sequence_end; +}; + +/** + * struct fy_event_mapping_start_data - mapping start event data + * + * @anchor: anchor token or NULL + * @tag: tag token or NULL + * @mapping_start: mapping start value token or NULL if the mapping + * was started implicitly + */ +struct fy_event_mapping_start_data { + struct fy_token *anchor; + struct fy_token *tag; + struct fy_token *mapping_start; +}; + +/** + * struct fy_event_mapping_end_data - mapping end event data + * + * @mapping_end: The token that ended the mapping, or NULL if + * the mapping was implicitly ended + */ +struct fy_event_mapping_end_data { + struct fy_token *mapping_end; +}; + +/** + * struct fy_event - Event generated by the parser + * + * This structure is generated by the parser by each call + * to fy_parser_parse() and release by fy_parser_event_free() + * + * @type: Type of the event, see &enum fy_event_type + * + * @stream_start: Stream start information, it is valid when + * &fy_event->type is &enum FYET_STREAM_START + * @stream_end: Stream end information, it is valid when + * &fy_event->type is &enum FYET_STREAM_END + * @document_start: Document start information, it is valid when + * &fy_event->type is &enum FYET_DOCUMENT_START + * @document_end: Document end information, it is valid when + * &fy_event->type is &enum FYET_DOCUMENT_END + * @alias: Alias information, it is valid when + * &fy_event->type is &enum FYET_ALIAS + * @scalar: Scalar information, it is valid when + * &fy_event->type is &enum FYET_SCALAR + * @sequence_start: Sequence start information, it is valid when + * &fy_event->type is &enum FYET_SEQUENCE_START + * @sequence_end: Sequence end information, it is valid when + * &fy_event->type is &enum FYET_SEQUENCE_END + * @mapping_start: Mapping start information, it is valid when + * &fy_event->type is &enum FYET_MAPPING_START + * @mapping_end: Mapping end information, it is valid when + * &fy_event->type is &enum FYET_MAPPING_END + */ +struct fy_event { + enum fy_event_type type; + /* anonymous union */ + union { + struct fy_event_stream_start_data stream_start; + struct fy_event_stream_end_data stream_end; + struct fy_event_document_start_data document_start; + struct fy_event_document_end_data document_end; + struct fy_event_alias_data alias; + struct fy_event_scalar_data scalar; + struct fy_event_sequence_start_data sequence_start; + struct fy_event_sequence_end_data sequence_end; + struct fy_event_mapping_start_data mapping_start; + struct fy_event_mapping_end_data mapping_end; + }; +}; + +/** + * enum fy_event_part - Select part of the event + * + * @FYEP_VALUE: The value of the event (the main token) + * @FYEP_TAG: The tag of the event + * @FYEP_ANCHOR: The anchor of the event + */ +enum fy_event_part { + FYEP_VALUE, + FYEP_TAG, + FYEP_ANCHOR, +}; + +/** + * fy_event_get_type() - Get the event's type + * + * Return the type of the event + * + * @fye: The event + * + * Returns: + * The event type, or FYET_NONE when the event is invalid + */ +static inline enum fy_event_type +fy_event_get_type(const struct fy_event *fye) +{ + return fye ? fye->type : FYET_NONE; +} + +/** + * fy_event_data() - Get a pointer to the event data + * + * Some languages *cough*golang*cough* really don't like + * unions, and anonymous unions in particular. + * + * You should not have to use this in other language bindings. + * + * @fye: The event + * + * Returns: + * A pointer to the event data structure, or NULL if the + * event is invalid + */ +static inline void * +fy_event_data(struct fy_event *fye) +{ + if (!fye) + return NULL; + + /* note that the unions should all be laid out + * at the same address, but play it straight and + * hope the optimizer will figure this is all + * the same... + */ + switch (fye->type) { + case FYET_STREAM_START: + return &fye->stream_start; + case FYET_STREAM_END: + return &fye->stream_end; + case FYET_DOCUMENT_START: + return &fye->document_start; + case FYET_DOCUMENT_END: + return &fye->document_end; + case FYET_ALIAS: + return &fye->alias; + case FYET_SCALAR: + return &fye->scalar; + case FYET_SEQUENCE_START: + return &fye->sequence_start; + case FYET_SEQUENCE_END: + return &fye->sequence_end; + case FYET_MAPPING_START: + return &fye->mapping_start; + case FYET_MAPPING_END: + return &fye->mapping_end; + default: + break; + } + + return NULL; +} + +/** + * fy_library_version() - Return the library version string + * + * Returns: + * A pointer to a version string of the form + * .[[.][-EXTRA-VERSION-INFO]] + */ +const char * +fy_library_version(void) + FY_EXPORT; + +/** + * fy_string_to_error_type() - Return the error type from a string + * + * @str: The string to convert to an error type + * + * Returns: + * The error type if greater or equal to zero, FYET_MAX otherwise + */ +enum fy_error_type fy_string_to_error_type(const char *str) + FY_EXPORT; + +/** + * fy_error_type_to_string() - Convert an error type to string + * + * @type: The error type to convert + * + * Returns: + * The string value of the error type or the empty string "" on error + */ +const char *fy_error_type_to_string(enum fy_error_type type) + FY_EXPORT; + +/** + * fy_string_to_error_module() - Return the error module from a string + * + * @str: The string to convert to an error module + * + * Returns: + * The error type if greater or equal to zero, FYEM_MAX otherwise + */ +enum fy_error_module fy_string_to_error_module(const char *str) + FY_EXPORT; + +/** + * fy_error_module_to_string() - Convert an error module to string + * + * @module: The error module to convert + * + * Returns: + * The string value of the error module or the empty string "" on error + */ +const char *fy_error_module_to_string(enum fy_error_module module) + FY_EXPORT; + +/** + * fy_event_is_implicit() - Check whether the given event is an implicit one + * + * @fye: A pointer to a &struct fy_event to check. + * + * Returns: + * true if the event is an implicit one. + */ +bool +fy_event_is_implicit(struct fy_event *fye) + FY_EXPORT; + +/** + * fy_document_event_is_implicit() - Check whether the given document event is an implicit one + * + * @fye: A pointer to a &struct fy_event to check. + * It must be either a document start or document end event. + * + * Returns: + * true if the event is an implicit one. + */ +bool +fy_document_event_is_implicit(const struct fy_event *fye) + FY_EXPORT; + +/** + * fy_parser_create() - Create a parser. + * + * Creates a parser with its configuration @cfg + * The parser may be destroyed by a corresponding call to + * fy_parser_destroy(). + * + * @cfg: The configuration for the parser + * + * Returns: + * A pointer to the parser or NULL in case of an error. + */ +struct fy_parser * +fy_parser_create(const struct fy_parse_cfg *cfg) + FY_EXPORT; + +/** + * fy_parser_destroy() - Destroy the given parser + * + * Destroy a parser created earlier via fy_parser_create(). + * + * @fyp: The parser to destroy + */ +void +fy_parser_destroy(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_get_cfg() - Get the configuration of a parser + * + * @fyp: The parser + * + * Returns: + * The configuration of the parser + */ +const struct fy_parse_cfg * +fy_parser_get_cfg(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_get_diag() - Get the diagnostic object of a parser + * + * Return a pointer to the diagnostic object of a parser object. + * Note that the returned diag object has a reference taken so + * you should fy_diag_unref() it when you're done with it. + * + * @fyp: The parser to get the diagnostic object + * + * Returns: + * A pointer to a ref'ed diagnostic object or NULL in case of an + * error. + */ +struct fy_diag * +fy_parser_get_diag(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_set_diag() - Set the diagnostic object of a parser + * + * Replace the parser's current diagnostic object with the one + * given as an argument. The previous diagnostic object will be + * unref'ed (and freed if its reference gets to 0). + * Also note that the diag argument shall take a reference too. + * + * @fyp: The parser to replace the diagnostic object + * @diag: The parser's new diagnostic object, NULL for default + * + * Returns: + * 0 if everything OK, -1 otherwise + */ +int +fy_parser_set_diag(struct fy_parser *fyp, struct fy_diag *diag) + FY_EXPORT; + +/** + * fy_parser_reset() - Reset a parser completely + * + * Completely reset a parser, including after an error + * that caused a parser error to be emitted. + * + * @fyp: The parser to reset + * + * Returns: + * 0 if the reset was successful, -1 otherwise + */ +int +fy_parser_reset(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_set_input_file() - Set the parser to process the given file + * + * Point the parser to the given @file for processing. The file + * is located by honoring the search path of the configuration set + * by the earlier call to fy_parser_create(). + * While the parser is in use the file will must be available. + * + * @fyp: The parser + * @file: The file to parse. + * + * Returns: + * zero on success, -1 on error + */ +int +fy_parser_set_input_file(struct fy_parser *fyp, const char *file) + FY_EXPORT; + +/** + * fy_parser_set_string() - Set the parser to process the given string. + * + * Point the parser to the given (NULL terminated) string. Note that + * while the parser is active the string must not go out of scope. + * + * @fyp: The parser + * @str: The YAML string to parse. + * @len: The length of the string (or -1 if '\0' terminated) + * + * Returns: + * zero on success, -1 on error + */ +int +fy_parser_set_string(struct fy_parser *fyp, const char *str, size_t len) + FY_EXPORT; + +/** + * fy_parser_set_malloc_string() - Set the parser to process the given malloced string. + * + * Point the parser to the given (possible NULL terminated) string. Note that + * the string is expected to be allocated via malloc(3) and ownership is transferred + * to the created input. When the input is free'ed the memory will be automatically + * freed. + * + * In case of an error the string is not freed. + * + * @fyp: The parser + * @str: The YAML string to parse (allocated). + * @len: The length of the string (or -1 if '\0' terminated) + * + * Returns: + * zero on success, -1 on error + */ +int +fy_parser_set_malloc_string(struct fy_parser *fyp, char *str, size_t len) + FY_EXPORT; + +/** + * fy_parser_set_input_fp() - Set the parser to process the given file + * + * Point the parser to use @fp for processing. + * + * @fyp: The parser + * @name: The label of the stream + * @fp: The FILE pointer, it must be open in read mode. + * + * Returns: + * zero on success, -1 on error + */ +int +fy_parser_set_input_fp(struct fy_parser *fyp, const char *name, FILE *fp) + FY_EXPORT; + +/** + * fy_parser_set_input_callback() - Set the parser to process via a callback + * + * Point the parser to use a callback for input. + * + * @fyp: The parser + * @user: The user data pointer + * @callback: The callback method to request data with + * + * Returns: + * zero on success, -1 on error + */ +int fy_parser_set_input_callback(struct fy_parser *fyp, void *user, + ssize_t (*callback)(void *user, void *buf, size_t count)) + FY_EXPORT; + +/** + * fy_parser_set_input_fd() - Set the parser to process the given file descriptor + * + * Point the parser to use @fd for processing. + * + * @fyp: The parser + * @fd: The file descriptor to use + * + * Returns: + * zero on success, -1 on error + */ +int +fy_parser_set_input_fd(struct fy_parser *fyp, int fd) + FY_EXPORT; + +/** + * fy_parser_parse() - Parse and return the next event. + * + * Each call to fy_parser_parse() returns the next event from + * the configured input of the parser, or NULL at the end of + * the stream. The returned event must be released via + * a matching call to fy_parser_event_free(). + * + * @fyp: The parser + * + * Returns: + * The next event in the stream or NULL. + */ +struct fy_event * +fy_parser_parse(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_parse_peek() - Parse and peek at the next event. + * + * It will return the next event that a call to fy_parser_parse + * would generate, but as read-only. + * + * You must not free this event. + * + * @fyp: The parser + * + * Returns: + * A peek at the next event in the stream or NULL. + */ +const struct fy_event * +fy_parser_parse_peek(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_skip() - Skip the current scalar/collection + * + * Skips the current scalar or collection. + * + * @fyp: The parser + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_parser_skip(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_count_sequence_items() - Count the sequence items + * + * Counts the number of sequence items. Parser must already + * be in a sequence state. + * Note that this uses backtracking so it might not be very + * efficient. + * + * @fyp: The parser + * + * Returns: + * The number of sequence items on success, -1 on error + * Note if the number of items exceeds INT_MAX, INT_MAX will + * be returned. + */ +int +fy_parser_count_sequence_items(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_count_mapping_items() - Count the mapping items + * + * Counts the number of mapping items. Parser must already + * be in a mapping state. + * Note that this uses backtracking so it might not be very + * efficient. + * + * @fyp: The parser + * + * Returns: + * The number of mapping items on success, -1 on error + * Note if the number of items exceeds INT_MAX, INT_MAX will + * be returned. + */ +int +fy_parser_count_mapping_items(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_event_free() - Free an event + * + * Free a previously returned event from fy_parser_parse(). + * + * @fyp: The parser + * @fye: The event to free (may be NULL) + */ +void +fy_parser_event_free(struct fy_parser *fyp, struct fy_event *fye) + FY_EXPORT; + +/** + * fy_parser_get_stream_error() - Check the parser for stream errors + * + * @fyp: The parser + * + * Returns: + * true in case of a stream error, false otherwise. + */ +bool +fy_parser_get_stream_error(struct fy_parser *fyp) + FY_EXPORT; + +enum fy_parser_mode { + fypm_invalid = -1, + fypm_none, /* scanning for mode, not committed yet */ + fypm_yaml_1_1, + fypm_yaml_1_2, + fypm_yaml_1_3, /* experimental */ + fypm_json, +}; + +static inline bool +fy_parser_mode_is_yaml(enum fy_parser_mode mode) +{ + return mode >= fypm_yaml_1_1 && mode <= fypm_yaml_1_3; +} + +/** + * fy_parser_get_mode() - Get the parser mode + * + * Retrieve the current mode of the parser, which indicates the YAML version + * or format being parsed (YAML 1.1, 1.2, 1.3, or JSON). + * + * @fyp: The parser + * + * Returns: + * The parser mode + */ +enum fy_parser_mode +fy_parser_get_mode(struct fy_parser *fyp) + FY_EXPORT; + +static inline bool +fy_parser_is_in_yaml_mode(struct fy_parser *fyp) +{ + return fy_parser_mode_is_yaml(fy_parser_get_mode(fyp)); +} + +static inline bool +fy_parser_is_in_json_mode(struct fy_parser *fyp) +{ + return fy_parser_get_mode(fyp) == fypm_json; +} + +/** + * fy_parser_vlog() - Log using the parsers diagnostics printf style (va_arg) + * + * Output a log on the parser diagnostic output + * + * @fyp: The parser + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_parser_vlog(struct fy_parser *fyp, enum fy_error_type type, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_parser_log() - Log using the parsers diagnostics printf style + * + * Output a report on the parser's diagnostics + * + * @fyp: The parser + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments + */ +void +fy_parser_log(struct fy_parser *fyp, enum fy_error_type type, const char *fmt, ...) + FY_FORMAT(printf, 3, 4) + FY_EXPORT; + +#ifndef NDEBUG + +#define fy_parser_debug(_fyp, _fmt, ...) \ + fy_parser_log((_fyp), FYET_DEBUG, (_fmt) , ## __VA_ARGS__) +#else + +#define fy_parser_debug(_fyp, _fmt, ...) \ + do { } while(0) + +#endif + +#define fy_parser_info(_fyp, _fmt, ...) \ + fy_parser_log((_fyp), FYET_INFO, (_fmt) , ## __VA_ARGS__) +#define fy_parser_notice(_fyp, _fmt, ...) \ + fy_parser_log((_fyp), FYET_NOTICE, (_fmt) , ## __VA_ARGS__) +#define fy_parser_warning(_fyp, _fmt, ...) \ + fy_parser_log((_fyp), FYET_WARNING, (_fmt) , ## __VA_ARGS__) +#define fy_parser_error(_fyp, _fmt, ...) \ + fy_parser_log((_fyp), FYET_ERROR, (_fmt) , ## __VA_ARGS__) + +/** + * fy_parser_vreport() - Report using the parsers diagnostics printf style and mark token (va_arg) + * + * Output a report on the parser and indicate the token's position + * + * @fyp: The parser + * @type: The error type + * @fyt: The token + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_parser_vreport(struct fy_parser *fyp, enum fy_error_type type, struct fy_token *fyt, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_parser_report() - Report using the parsers diagnostics printf style and mark token + * + * Output a report on the parser and indicate the token's position + * + * @fyp: The parser + * @type: The error type + * @fyt: The token + * @fmt: The printf format string + * @...: The extra arguments + */ +void +fy_parser_report(struct fy_parser *fyp, enum fy_error_type type, struct fy_token *fyt, + const char *fmt, ...) + FY_FORMAT(printf, 4, 5) + FY_EXPORT; + +#ifndef NDEBUG + +#define fy_parser_report_debug(_fyp, _fyt, _fmt, ...) \ + fy_parser_report((_fyp), FYET_DEBUG, (_fyt), (_fmt) , ## __VA_ARGS__) +#else + +#define fy_parser_report_debug(_fyp, _fyt, _fmt, ...) \ + do { } while(0) + +#endif + +#define fy_parser_report_info(_fyp, _fyt, _fmt, ...) \ + fy_parser_report((_fyp), FYET_INFO, (_fyt), (_fmt) , ## __VA_ARGS__) +#define fy_parser_report_notice(_fyp, _fyt, _fmt, ...) \ + fy_parser_report((_fyp), FYET_NOTICE, (_fyt), (_fmt) , ## __VA_ARGS__) +#define fy_parser_report_warning(_fyp, _fyt, _fmt, ...) \ + fy_parser_report((_fyp), FYET_WARNING, (_fyt), (_fmt) , ## __VA_ARGS__) +#define fy_parser_report_error(_fyp, _fyt, _fmt, ...) \ + fy_parser_report((_fyp), FYET_ERROR, (_fyt), (_fmt) , ## __VA_ARGS__) + +/** + * fy_token_scalar_style() - Get the style of a scalar token + * + * @fyt: The scalar token to get it's style. Note that a NULL + * token is a &enum FYSS_PLAIN. + * + * Returns: + * The scalar style of the token, or FYSS_PLAIN on each other case + */ +enum fy_scalar_style +fy_token_scalar_style(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_token_scalar_is_null() - Test whether the scalar is null (content) + * + * @fyt: The scalar token to check for NULLity. + * + * Note that this is different than null of the YAML type system. + * It is null as in null content. It is also different than an + * empty scalar. + * + * Returns: + * true if is a null scalar, false otherwise + */ +bool +fy_token_scalar_is_null(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_token_get_text() - Get text (and length of it) of a token + * + * This method will return a pointer to the text of a token + * along with the length of it. Note that this text is *not* + * NULL terminated. If you need a NULL terminated pointer + * use fy_token_get_text0(). + * + * In case that the token is 'simple' enough (i.e. a plain scalar) + * or something similar the returned pointer is a direct pointer + * to the space of the input that contains the token. + * + * That means that the pointer is *not* guaranteed to be valid + * after the parser is destroyed. + * + * If the token is 'complex' enough, then space shall be allocated, + * filled and returned. + * + * Note that the concept of 'simple' and 'complex' is vague, and + * that's on purpose. + * + * @fyt: The token out of which the text pointer will be returned. + * @lenp: Pointer to a variable that will hold the returned length + * + * Returns: + * A pointer to the text representation of the token, while + * @lenp will be assigned the character length of said representation. + * NULL in case of an error. + */ +const char * +fy_token_get_text(struct fy_token *fyt, size_t *lenp) + FY_EXPORT; + +/** + * fy_token_get_text0() - Get zero terminated text of a token + * + * This method will return a pointer to the text of a token + * which is zero terminated. It will allocate memory to hold + * it in the token structure so try to use fy_token_get_text() + * instead if possible. + * + * @fyt: The token out of which the text pointer will be returned. + * + * Returns: + * A pointer to a zero terminated text representation of the token. + * NULL in case of an error. + */ +const char * +fy_token_get_text0(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_token_get_text_length() - Get length of the text of a token + * + * This method will return the length of the text representation + * of a token. + * + * @fyt: The token + * + * Returns: + * The size of the text representation of a token, -1 in case of an error. + * Note that the NULL token will return a length of zero. + */ +size_t +fy_token_get_text_length(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_token_get_utf8_length() - Get length of the text of a token + * + * This method will return the length of the text representation + * of a token as a utf8 string. + * + * @fyt: The token + * + * Returns: + * The size of the utf8 text representation of a token, -1 in case of an error. + * Note that the NULL token will return a length of zero. + */ +size_t +fy_token_get_utf8_length(struct fy_token *fyt) + FY_EXPORT; + +/** + * enum fy_comment_placement - Comment placement relative to token + * + * @fycp_top: Comment on top of token + * @fycp_right: Comment to the right of the token + * @fycp_bottom: Comment to the bottom of the token + */ +enum fy_comment_placement { + fycp_top, + fycp_right, + fycp_bottom +}; +#define fycp_max (fycp_bottom + 1) + +/** + * fy_token_get_comment() - Get zero terminated comment of a token + * + * @fyt: The token out of which the comment text will be returned. + * @buf: The buffer to be filled with the contents of the token + * @maxsz: The maximum size of the comment buffer + * @which: The comment placement + * + * Returns: + * A pointer to a zero terminated text representation of the token comment. + * NULL in case of an error or if the token has no comment. + */ +const char * +fy_token_get_comment(struct fy_token *fyt, char *buf, size_t maxsz, + enum fy_comment_placement which) + FY_EXPORT; + +/** + * struct fy_iter_chunk - An iteration chunk + * + * @str: Pointer to the start of the chunk + * @len: The size of the chunk + * + * The iterator produces a stream of chunks which + * cover the whole object. + */ +struct fy_iter_chunk { + const char *str; + size_t len; +}; + +/** + * fy_token_iter_create() - Create a token iterator + * + * Create an iterator for operating on the given token, or + * a generic iterator for use with fy_token_iter_start(). + * The iterator must be destroyed with a matching call to + * fy_token_iter_destroy(). + * + * @fyt: The token to iterate, or NULL. + * + * Returns: + * A pointer to the newly created iterator, or NULL in case of + * an error. + */ +struct fy_token_iter * +fy_token_iter_create(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_token_iter_destroy() - Destroy the iterator + * + * Destroy the iterator created by fy_token_iter_create(). + * + * @iter: The iterator to destroy. + */ +void +fy_token_iter_destroy(struct fy_token_iter *iter) + FY_EXPORT; + +/** + * fy_token_iter_start() - Start iterating over the contents of a token + * + * Prepare an iterator for operating on the given token. + * The iterator must be created via a previous call to fy_token_iter_create() + * for user level API access. + * + * @fyt: The token to iterate over + * @iter: The iterator to prepare. + */ +void +fy_token_iter_start(struct fy_token *fyt, struct fy_token_iter *iter) + FY_EXPORT; + +/** + * fy_token_iter_finish() - Stop iterating over the contents of a token + * + * Stop the iteration operation. + * + * @iter: The iterator. + */ +void +fy_token_iter_finish(struct fy_token_iter *iter) + FY_EXPORT; + +/** + * fy_token_iter_peek_chunk() - Peek at the next iterator chunk + * + * Peek at the next iterator chunk + * + * @iter: The iterator. + * + * Returns: + * A pointer to the next iterator chunk, or NULL in case there's + * no other. + */ +const struct fy_iter_chunk * +fy_token_iter_peek_chunk(struct fy_token_iter *iter) + FY_EXPORT; + +/** + * fy_token_iter_chunk_next() - Get next iterator chunk + * + * Get the next iterator chunk in sequence, + * + * @iter: The iterator. + * @curr: The current chunk, or NULL for the first one. + * @errp: Pointer to an error return value or NULL + * + * Returns: + * A pointer to the next iterator chunk, or NULL in case there's + * no other. When the return value is NULL, the errp variable + * will be filled with 0 for normal end, or -1 in case of an error. + */ +const struct fy_iter_chunk * +fy_token_iter_chunk_next(struct fy_token_iter *iter, + const struct fy_iter_chunk *curr, int *errp) + FY_EXPORT; + +/** + * fy_token_iter_advance() - Advance the iterator position + * + * Advance the read pointer of the iterator. + * Note that mixing calls of this with any call of fy_token_iter_ungetc() / + * fy_token_iter_utf8_unget() in a single iterator sequence leads + * to undefined behavior. + * + * @iter: The iterator. + * @len: Number of bytes to advance the iterator position + */ +void +fy_token_iter_advance(struct fy_token_iter *iter, size_t len) + FY_EXPORT; + +/** + * fy_token_iter_read() - Read a block from an iterator + * + * Read a block from an iterator. Note than mixing calls of this + * and any of the ungetc methods leads to undefined behavior. + * + * @iter: The iterator. + * @buf: Pointer to a block of memory to receive the data. Must be at + * least count bytes long. + * @count: Amount of bytes to read. + * + * Returns: + * The amount of data read, or -1 in case of an error. + */ +ssize_t +fy_token_iter_read(struct fy_token_iter *iter, void *buf, size_t count) + FY_EXPORT; + +/** + * fy_token_iter_getc() - Get a single character from an iterator + * + * Reads a single character from an iterator. If the iterator is + * finished, it will return -1. If any calls to ungetc have pushed + * a character in the iterator it shall return that. + * + * @iter: The iterator. + * + * Returns: + * The next character in the iterator, or -1 in case of an error, or + * end of stream. + */ +int +fy_token_iter_getc(struct fy_token_iter *iter) + FY_EXPORT; + +/** + * fy_token_iter_ungetc() - Ungets a single character from an iterator + * + * Pushes back a single character to an iterator stream. It will be + * returned in subsequent calls of fy_token_iter_getc(). Currently + * only a single character is allowed to be pushed back, and any + * further calls to ungetc will return an error. + * + * @iter: The iterator. + * @c: The character to push back, or -1 to reset the pushback buffer. + * + * Returns: + * The pushed back character given as argument, or -1 in case of an error. + * If the pushed back character was -1, then 0 will be returned. + */ +int +fy_token_iter_ungetc(struct fy_token_iter *iter, int c) + FY_EXPORT; + +/** + * fy_token_iter_peekc() - Peeks at single character from an iterator + * + * Peeks at the next character to get from an iterator. If the iterator is + * finished, it will return -1. If any calls to ungetc have pushed + * a character in the iterator it shall return that. The character is not + * removed from the iterator stream. + * + * @iter: The iterator. + * + * Returns: + * The next character in the iterator, or -1 in case of an error, or end + * of stream. + */ +int +fy_token_iter_peekc(struct fy_token_iter *iter) + FY_EXPORT; + +/** + * fy_token_iter_utf8_get() - Get a single utf8 character from an iterator + * + * Reads a single utf8 character from an iterator. If the iterator is + * finished, it will return -1. If any calls to ungetc have pushed + * a character in the iterator it shall return that. + * + * @iter: The iterator. + * + * Returns: + * The next utf8 character in the iterator, or -1 in case of an error, or end + * of stream. + */ +int +fy_token_iter_utf8_get(struct fy_token_iter *iter) + FY_EXPORT; + +/** + * fy_token_iter_utf8_unget() - Ungets a single utf8 character from an iterator + * + * Pushes back a single utf8 character to an iterator stream. It will be + * returned in subsequent calls of fy_token_iter_utf8_getc(). Currently + * only a single character is allowed to be pushed back, and any + * further calls to ungetc will return an error. + * + * @iter: The iterator. + * @c: The character to push back, or -1 to reset the pushback buffer. + * + * Returns: + * The pushed back utf8 character given as argument, or -1 in case of an error. + * If the pushed back utf8 character was -1, then 0 will be returned. + */ +int +fy_token_iter_utf8_unget(struct fy_token_iter *iter, int c) + FY_EXPORT; + +/** + * fy_token_iter_utf8_peek() - Peeks at single utf8 character from an iterator + * + * Peeks at the next utf8 character to get from an iterator. If the iterator is + * finished, it will return -1. If any calls to ungetc have pushed + * a character in the iterator it shall return that. The character is not + * removed from the iterator stream. + * + * @iter: The iterator. + * + * Returns: + * The next utf8 character in the iterator, or -1 in case of an error, or end + * of stream. + */ +int +fy_token_iter_utf8_peek(struct fy_token_iter *iter) + FY_EXPORT; + +/** + * fy_parse_load_document() - Parse the next document from the parser stream + * + * This method performs parsing on a parser stream and returns the next + * document. This means that for a compound document with multiple + * documents, each call will return the next document. + * + * @fyp: The parser + * + * Returns: + * The next document from the parser stream. + */ +struct fy_document * +fy_parse_load_document(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parse_document_destroy() - Destroy a document created by fy_parse_load_document() + * + * @fyp: The parser + * @fyd: The document to destroy + */ +void +fy_parse_document_destroy(struct fy_parser *fyp, struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_resolve() - Resolve anchors and merge keys + * + * This method performs resolution of the given document, + * by replacing references to anchors with their contents + * and handling merge keys (<<) + * + * @fyd: The document to resolve + * + * Returns: + * zero on success, -1 on error + */ +int +fy_document_resolve(struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_has_directives() - Document directive check + * + * Checks whether the given document has any directives, i.e. + * %TAG or %VERSION. + * + * @fyd: The document to check for directives existence + * + * Returns: + * true if directives exist, false if not + */ +bool +fy_document_has_directives(const struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_has_explicit_document_start() - Explicit document start check + * + * Checks whether the given document has an explicit document start marker, + * i.e. --- + * + * @fyd: The document to check for explicit start marker + * + * Returns: + * true if document has an explicit document start marker, false if not + */ +bool +fy_document_has_explicit_document_start(const struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_has_explicit_document_end() - Explicit document end check + * + * Checks whether the given document has an explicit document end marker, + * i.e. ... + * + * @fyd: The document to check for explicit end marker + * + * Returns: + * true if document has an explicit document end marker, false if not + */ +bool +fy_document_has_explicit_document_end(const struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_node_document() - Retreive the document the node belong to + * + * Returns the document of the node; note that while the node may not + * be reachable via a path expression, it may still be member of a + * document. + * + * @fyn: The node to retreive it's document + * + * Returns: + * The document of the node, or NULL in case of an error, or + * when the node has no document attached. + */ +struct fy_document * +fy_node_document(struct fy_node *fyn) + FY_EXPORT; + +/** + * enum fy_emitter_write_type - Type of the emitted output + * + * Describes the kind of emitted output, which makes it + * possible to colorize the output, or do some other content related + * filtering. + * + * @fyewt_document_indicator: Output chunk is a document indicator + * @fyewt_tag_directive: Output chunk is a tag directive + * @fyewt_version_directive: Output chunk is a version directive + * @fyewt_indent: Output chunk is a document indicator + * @fyewt_indicator: Output chunk is an indicator + * @fyewt_whitespace: Output chunk is white space + * @fyewt_plain_scalar: Output chunk is a plain scalar + * @fyewt_single_quoted_scalar: Output chunk is a single quoted scalar + * @fyewt_double_quoted_scalar: Output chunk is a double quoted scalar + * @fyewt_literal_scalar: Output chunk is a literal block scalar + * @fyewt_folded_scalar: Output chunk is a folded block scalar + * @fyewt_anchor: Output chunk is an anchor + * @fyewt_tag: Output chunk is a tag + * @fyewt_linebreak: Output chunk is a linebreak + * @fyewt_alias: Output chunk is an alias + * @fyewt_terminating_zero: Output chunk is a terminating zero + * @fyewt_plain_scalar_key: Output chunk is an plain scalar key + * @fyewt_single_quoted_scalar_key: Output chunk is an single quoted scalar key + * @fyewt_double_quoted_scalar_key: Output chunk is an double quoted scalar key + * @fyewt_comment: Output chunk is a comment + * @fyewt_indicator_question_mark: ? indicator + * @fyewt_indicator_colon: : indicator + * @fyewt_indicator_dash: - indicator + * @fyewt_indicator_left_bracket: [ indicator + * @fyewt_indicator_right_bracket: ] indicator + * @fyewt_indicator_left_brace: { indicator + * @fyewt_indicator_right_brace: } indicator + * @fyewt_indicator_comma: , indicator + * @fyewt_indicator_bar: | indicator + * @fyewt_indicator_greater: > indicator + * @fyewt_indicator_single_quote_start: ' indicator + * @fyewt_indicator_single_quote_end: ' indicator + * @fyewt_indicator_double_quote_start: " indicator + * @fyewt_indicator_double_quote_end: " indicator + * @fyewt_indicator_ambersand: & indicator + * @fyewt_indicator_star: * indicator + * @fyewt_indicator_chomp: chomp indicator (- or +) + * @fyewt_indicator_explicit_indent: explicit indent indicator (0-9) + */ +enum fy_emitter_write_type { + fyewt_document_indicator, + fyewt_tag_directive, + fyewt_version_directive, + fyewt_indent, + fyewt_indicator, + fyewt_whitespace, + fyewt_plain_scalar, + fyewt_single_quoted_scalar, + fyewt_double_quoted_scalar, + fyewt_literal_scalar, + fyewt_folded_scalar, + fyewt_anchor, + fyewt_tag, + fyewt_linebreak, + fyewt_alias, + fyewt_terminating_zero, + fyewt_plain_scalar_key, + fyewt_single_quoted_scalar_key, + fyewt_double_quoted_scalar_key, + fyewt_comment, + /* extended indicators */ + fyewt_indicator_question_mark, + fyewt_indicator_colon, + fyewt_indicator_dash, + fyewt_indicator_left_bracket, + fyewt_indicator_right_bracket, + fyewt_indicator_left_brace, + fyewt_indicator_right_brace, + fyewt_indicator_comma, + fyewt_indicator_bar, + fyewt_indicator_greater, + fyewt_indicator_single_quote_start, + fyewt_indicator_single_quote_end, + fyewt_indicator_double_quote_start, + fyewt_indicator_double_quote_end, + fyewt_indicator_ambersand, + fyewt_indicator_star, + fyewt_indicator_chomp, + fyewt_indicator_explicit_indent, +}; +#define FYEWT_EXTENDED_INDICATORS_FIRST fyewt_indicator_question_mark +#define FYEWT_EXTENDED_INDICATORS_LAST fyewt_indicator_explicit_indent + +#define FYEWT_EXTENDED_START fyewt_indicator_question_mark +#define FYEWT_COUNT (fyewt_indicator_explicit_indent + 1) + +#define FYECF_INDENT_SHIFT 8 +#define FYECF_INDENT_MASK 0xf +#define FYECF_INDENT(x) (((x) & FYECF_INDENT_MASK) << FYECF_INDENT_SHIFT) + +#define FYECF_WIDTH_SHIFT 12 +#define FYECF_WIDTH_MASK 0xff +#define FYECF_WIDTH(x) (((x) & FYECF_WIDTH_MASK) << FYECF_WIDTH_SHIFT) + +#define FYECF_MODE_SHIFT 20 +#define FYECF_MODE_MASK 0xf +#define FYECF_MODE(x) (((x) & FYECF_MODE_MASK) << FYECF_MODE_SHIFT) + +#define FYECF_DOC_START_MARK_SHIFT 24 +#define FYECF_DOC_START_MARK_MASK 0x3 +#define FYECF_DOC_START_MARK(x) (((x) & FYECF_DOC_START_MARK_MASK) << FYECF_DOC_START_MARK_SHIFT) + +#define FYECF_DOC_END_MARK_SHIFT 26 +#define FYECF_DOC_END_MARK_MASK 0x3 +#define FYECF_DOC_END_MARK(x) (((x) & FYECF_DOC_END_MARK_MASK) << FYECF_DOC_END_MARK_SHIFT) + +#define FYECF_VERSION_DIR_SHIFT 28 +#define FYECF_VERSION_DIR_MASK 0x3 +#define FYECF_VERSION_DIR(x) (((x) & FYECF_VERSION_DIR_MASK) << FYECF_VERSION_DIR_SHIFT) + +#define FYECF_TAG_DIR_SHIFT 30 +#define FYECF_TAG_DIR_MASK 0x3 +#define FYECF_TAG_DIR(x) (((unsigned int)(x) & FYECF_TAG_DIR_MASK) << FYECF_TAG_DIR_SHIFT) + +/** + * enum fy_emitter_cfg_flags - Emitter configuration flags + * + * These flags control the operation of the emitter + * + * @FYECF_SORT_KEYS: Sort key when emitting + * @FYECF_OUTPUT_COMMENTS: Output comments (experimental) + * @FYECF_STRIP_LABELS: Strip labels when emitting + * @FYECF_STRIP_TAGS: Strip tags when emitting + * @FYECF_STRIP_DOC: Strip document tags and markers when emitting + * @FYECF_NO_ENDING_NEWLINE: Do not output ending new line (useful for single line mode) + * @FYECF_STRIP_EMPTY_KV: Remove all keys with empty values from the output (not available in streaming mode) + * @FYECF_EXTENDED_CFG: Extend configuration, this config structure is embedded in a fy_emitter_xcfg. + * @FYECF_INDENT_DEFAULT: Default emit output indent + * @FYECF_INDENT_1: Output indent is 1 + * @FYECF_INDENT_2: Output indent is 2 + * @FYECF_INDENT_3: Output indent is 3 + * @FYECF_INDENT_4: Output indent is 4 + * @FYECF_INDENT_5: Output indent is 5 + * @FYECF_INDENT_6: Output indent is 6 + * @FYECF_INDENT_7: Output indent is 7 + * @FYECF_INDENT_8: Output indent is 8 + * @FYECF_INDENT_9: Output indent is 9 + * @FYECF_WIDTH_DEFAULT: Default emit output width + * @FYECF_WIDTH_80: Output width is 80 + * @FYECF_WIDTH_132: Output width is 132 + * @FYECF_WIDTH_INF: Output width is infinite + * @FYECF_MODE_ORIGINAL: Emit using the same flow mode as the original + * @FYECF_MODE_BLOCK: Emit using only the block mode + * @FYECF_MODE_FLOW: Emit using only the flow mode + * @FYECF_MODE_FLOW_ONELINE: Emit using only the flow mode (in one line) + * @FYECF_MODE_JSON: Emit using JSON mode (non type preserving) + * @FYECF_MODE_JSON_TP: Emit using JSON mode (type preserving) + * @FYECF_MODE_JSON_ONELINE: Emit using JSON mode (non type preserving, one line) + * @FYECF_MODE_DEJSON: Emit YAML trying to pretify JSON + * @FYECF_MODE_PRETTY: Emit YAML that tries to look good + * @FYECF_MODE_MANUAL: Emit YAML respecting all manual style hints (reformats if needed) + * @FYECF_MODE_FLOW_COMPACT: Emit using only the flow mode in as much as possible compact form + * @FYECF_MODE_JSON_COMPACT: Emit using JSON compact form + * @FYECF_DOC_START_MARK_AUTO: Automatically generate document start markers if required + * @FYECF_DOC_START_MARK_OFF: Do not generate document start markers + * @FYECF_DOC_START_MARK_ON: Always generate document start markers + * @FYECF_DOC_END_MARK_AUTO: Automatically generate document end markers if required + * @FYECF_DOC_END_MARK_OFF: Do not generate document end markers + * @FYECF_DOC_END_MARK_ON: Always generate document end markers + * @FYECF_VERSION_DIR_AUTO: Automatically generate version directive + * @FYECF_VERSION_DIR_OFF: Never generate version directive + * @FYECF_VERSION_DIR_ON: Always generate version directive + * @FYECF_TAG_DIR_AUTO: Automatically generate tag directives + * @FYECF_TAG_DIR_OFF: Never generate tag directives + * @FYECF_TAG_DIR_ON: Always generate tag directives + * @FYECF_DEFAULT: The default emitter configuration + */ +enum fy_emitter_cfg_flags { + FYECF_SORT_KEYS = FY_BIT(0), + FYECF_OUTPUT_COMMENTS = FY_BIT(1), + FYECF_STRIP_LABELS = FY_BIT(2), + FYECF_STRIP_TAGS = FY_BIT(3), + FYECF_STRIP_DOC = FY_BIT(4), + FYECF_NO_ENDING_NEWLINE = FY_BIT(5), + FYECF_STRIP_EMPTY_KV = FY_BIT(6), + FYECF_EXTENDED_CFG = FY_BIT(7), + FYECF_INDENT_DEFAULT = FYECF_INDENT(0), + FYECF_INDENT_1 = FYECF_INDENT(1), + FYECF_INDENT_2 = FYECF_INDENT(2), + FYECF_INDENT_3 = FYECF_INDENT(3), + FYECF_INDENT_4 = FYECF_INDENT(4), + FYECF_INDENT_5 = FYECF_INDENT(5), + FYECF_INDENT_6 = FYECF_INDENT(6), + FYECF_INDENT_7 = FYECF_INDENT(7), + FYECF_INDENT_8 = FYECF_INDENT(8), + FYECF_INDENT_9 = FYECF_INDENT(9), + FYECF_WIDTH_DEFAULT = FYECF_WIDTH(80), + FYECF_WIDTH_80 = FYECF_WIDTH(80), + FYECF_WIDTH_132 = FYECF_WIDTH(132), + FYECF_WIDTH_INF = FYECF_WIDTH(255), + FYECF_MODE_ORIGINAL = FYECF_MODE(0), + FYECF_MODE_BLOCK = FYECF_MODE(1), + FYECF_MODE_FLOW = FYECF_MODE(2), + FYECF_MODE_FLOW_ONELINE = FYECF_MODE(3), + FYECF_MODE_JSON = FYECF_MODE(4), + FYECF_MODE_JSON_TP = FYECF_MODE(5), + FYECF_MODE_JSON_ONELINE = FYECF_MODE(6), + FYECF_MODE_DEJSON = FYECF_MODE(7), + FYECF_MODE_PRETTY = FYECF_MODE(8), + FYECF_MODE_MANUAL = FYECF_MODE(9), + FYECF_MODE_FLOW_COMPACT = FYECF_MODE(10), + FYECF_MODE_JSON_COMPACT = FYECF_MODE(11), + FYECF_DOC_START_MARK_AUTO = FYECF_DOC_START_MARK(0), + FYECF_DOC_START_MARK_OFF = FYECF_DOC_START_MARK(1), + FYECF_DOC_START_MARK_ON = FYECF_DOC_START_MARK(2), + FYECF_DOC_END_MARK_AUTO = FYECF_DOC_END_MARK(0), + FYECF_DOC_END_MARK_OFF = FYECF_DOC_END_MARK(1), + FYECF_DOC_END_MARK_ON = FYECF_DOC_END_MARK(2), + FYECF_VERSION_DIR_AUTO = FYECF_VERSION_DIR(0), + FYECF_VERSION_DIR_OFF = FYECF_VERSION_DIR(1), + FYECF_VERSION_DIR_ON = FYECF_VERSION_DIR(2), + FYECF_TAG_DIR_AUTO = FYECF_TAG_DIR(0), + FYECF_TAG_DIR_OFF = FYECF_TAG_DIR(1), + FYECF_TAG_DIR_ON = FYECF_TAG_DIR(2), + + FYECF_DEFAULT = FYECF_WIDTH_INF | + FYECF_MODE_ORIGINAL | + FYECF_INDENT_DEFAULT, +}; + +/** + * struct fy_emitter_cfg - emitter configuration structure. + * + * Argument to the fy_emitter_create() method which + * is the way to convert a runtime document structure back to YAML. + * + * @flags: Configuration flags + * @output: Pointer to the method that will perform output. + * @userdata: Opaque user data pointer + * @diag: Diagnostic interface + */ +struct fy_emitter_cfg { + enum fy_emitter_cfg_flags flags; + int (*output)(struct fy_emitter *emit, enum fy_emitter_write_type type, + const char *str, int len, void *userdata); + void *userdata; + struct fy_diag *diag; +}; + +/* Shift amount of the color settings */ +#define FYEXCF_COLOR_SHIFT 0 +/* Mask of the color */ +#define FYEXCF_COLOR_MASK ((1U << 2) - 1) +/* Build a color version */ +#define FYEXCF_COLOR(x) (((unsigned int)(x) & FYEXCF_COLOR_MASK) << FYEXCF_COLOR_SHIFT) + +/* Shift amount of the output settings */ +#define FYEXCF_OUTPUT_SHIFT 2 +/* Mask of the output */ +#define FYEXCF_OUTPUT_MASK ((1U << 3) - 1) +/* Build an output */ +#define FYEXCF_OUTPUT(x) (((unsigned int)(x) & FYEXCF_OUTPUT_MASK) << FYEXCF_OUTPUT_SHIFT) + +/** + * enum fy_emitter_xcfg_flags - Emitter extended configuration flags + * + * These flags control the operation of the emitter. + * If no extended configuration mode is enabled all the flags are assumed 0. + * + * @FYEXCF_COLOR_AUTO: Automatically colorize if on a terminal + * @FYEXCF_COLOR_NONE: Do not colorize output + * @FYEXCF_COLOR_FORCE: Force color generation + * @FYEXCF_VISIBLE_WS: Make white space visible + * @FYEXCF_EXTENDED_INDICATORS: Extend the indicators generated + * @FYEXCF_OUTPUT_STDOUT: Output to stdout (default) + * @FYEXCF_OUTPUT_STDERR: Output to stderr, instead of stdout + * @FYEXCF_OUTPUT_FILE: Output to the fp FILE pointer + * @FYEXCF_OUTPUT_FD: Output to the fd file descriptor + * @FYEXCF_NULL_OUTPUT: No output + * @FYEXCF_OUTPUT_FILENAME: Output to the given filename + */ +enum fy_emitter_xcfg_flags { + FYEXCF_COLOR_AUTO = FYEXCF_COLOR(0), + FYEXCF_COLOR_NONE = FYEXCF_COLOR(1), + FYEXCF_COLOR_FORCE = FYEXCF_COLOR(2), + FYEXCF_OUTPUT_STDOUT = FYEXCF_OUTPUT(0), + FYEXCF_OUTPUT_STDERR = FYEXCF_OUTPUT(1), + FYEXCF_OUTPUT_FILE = FYEXCF_OUTPUT(2), + FYEXCF_OUTPUT_FD = FYEXCF_OUTPUT(2), + FYEXCF_NULL_OUTPUT = FYEXCF_OUTPUT(3), + FYEXCF_OUTPUT_FILENAME = FYEXCF_OUTPUT(4), + FYEXCF_VISIBLE_WS = FY_BIT(5), + FYEXCF_EXTENDED_INDICATORS = FY_BIT(6), +}; + +/** + * struct fy_emitter_xcfg - emitter extended configuration structure. + * + * Argument to the fy_emitter create when FYECF_EXTENDED_CFG bit is + * set. + * + * @cfg: The standard emitter configuration + * @xflags: Extra configuration flags + * @colors: ANSI color overrides for the default output method + * @output_fp: The output FILE \*, FYEXCF\_FILE\_OUTPUT must be set + * @output_fd: The output file descriptor, FYEXCF\_FD\_OUTPUT must be set + * @output_filename: The output filename, FYEXCF\_FILENAME\_OUTPUT must be set + */ +struct fy_emitter_xcfg { + struct fy_emitter_cfg cfg; + enum fy_emitter_xcfg_flags xflags; + const char *colors[FYEWT_COUNT]; + union { + FILE *output_fp; + int output_fd; + const char *output_filename; + }; +}; + +/** + * fy_emitter_create() - Create an emitter + * + * Creates an emitter using the supplied configuration + * + * @cfg: The emitter configuration + * + * Returns: + * The newly created emitter or NULL on error. + */ +struct fy_emitter * +fy_emitter_create(const struct fy_emitter_cfg *cfg) + FY_EXPORT; + +/** + * fy_emitter_destroy() - Destroy an emitter + * + * Destroy an emitter previously created by fy_emitter_create() + * + * @emit: The emitter to destroy + */ +void +fy_emitter_destroy(struct fy_emitter *emit) + FY_EXPORT; + +/** + * fy_emitter_get_cfg() - Get the configuration of an emitter + * + * @emit: The emitter + * + * Returns: + * The configuration of the emitter + */ +const struct fy_emitter_cfg * +fy_emitter_get_cfg(struct fy_emitter *emit) + FY_EXPORT; + +/** + * fy_emitter_get_diag() - Get the diagnostic object of an emitter + * + * Return a pointer to the diagnostic object of an emitter object. + * Note that the returned diag object has a reference taken so + * you should fy_diag_unref() it when you're done with it. + * + * @emit: The emitter to get the diagnostic object + * + * Returns: + * A pointer to a ref'ed diagnostic object or NULL in case of an + * error. + */ +struct fy_diag * +fy_emitter_get_diag(struct fy_emitter *emit) + FY_EXPORT; + +/** + * fy_emitter_set_diag() - Set the diagnostic object of an emitter + * + * Replace the emitters's current diagnostic object with the one + * given as an argument. The previous diagnostic object will be + * unref'ed (and freed if its reference gets to 0). + * Also note that the diag argument shall take a reference too. + * + * @emit: The emitter to replace the diagnostic object + * @diag: The emitter's new diagnostic object, NULL for default + * + * Returns: + * 0 if everything OK, -1 otherwise + */ +int +fy_emitter_set_diag(struct fy_emitter *emit, struct fy_diag *diag) + FY_EXPORT; + +/** + * fy_emitter_set_finalizer() - Set emitter finalizer + * + * Set a method callback to be called when the emitter + * is disposed of. If finalizer is NULL, then the method + * is removed. + * + * @emit: The emitter to replace the diagnostic object + * @finalizer: The finalizer callback + */ +void +fy_emitter_set_finalizer(struct fy_emitter *emit, + void (*finalizer)(struct fy_emitter *emit)) + FY_EXPORT; + +/** + * struct fy_emitter_default_output_data - emitter default output configuration + * + * This is the argument to the default output method of the emitter. + * + * @fp: File where the output is directed to + * @colorize: Use ANSI color sequences to colorize the output + * @visible: Make whitespace visible (requires a UTF8 capable terminal) + */ +struct fy_emitter_default_output_data { + FILE *fp; + bool colorize; + bool visible; +}; + +/** + * fy_emitter_default_output() - The default colorizing output method + * + * This is the default colorizing output method. + * Will be used when the output field of the emitter configuration is NULL. + * + * @fye: The emitter + * @type: Type of the emitted output + * @str: Pointer to the string to output + * @len: Length of the string + * @userdata: Must point to a fy_emitter_default_output_data structure + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emitter_default_output(struct fy_emitter *fye, enum fy_emitter_write_type type, + const char *str, int len, void *userdata) + FY_EXPORT; + +/** + * fy_document_default_emit_to_fp() - Emit a document to a file, using defaults + * + * Simple one shot emitter to a file, using the default emitter output. + * The output will be colorized if the the file points to a tty. + * + * @fyd: The document to emit + * @fp: The file where the output is sent + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_document_default_emit_to_fp(struct fy_document *fyd, FILE *fp) + FY_EXPORT; + +/** + * fy_emit_event() - Queue (and possibly emit) an event + * + * Queue and output using the emitter. This is the streaming + * output method which does not require creating a document. + * + * @emit: The emitter to use + * @fye: The event to queue for emission + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_event(struct fy_emitter *emit, struct fy_event *fye) + FY_EXPORT; + +/** + * fy_emit_vevent() - Queue (and possibly emit) an event using varargs + * + * Queue and output using the emitter. The format is identical to + * the one used in fy_emit_event_vcreate(). + * + * @emit: The emitter to use + * @type: The event type to create + * @ap: The variable argument list pointer. + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_vevent(struct fy_emitter *emit, enum fy_event_type type, va_list ap) + FY_EXPORT; + +/** + * fy_emit_eventf() - Queue (and possibly emit) an event using variable args + * + * Queue and output using the emitter. The format is identical to + * the one used in fy_emit_event_create(). + * + * @emit: The emitter to use + * @type: The event type to create + * @...: The optional extra arguments + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_eventf(struct fy_emitter *emit, enum fy_event_type type, ...) + FY_EXPORT; + +/** + * fy_emit_scalar_write() - Emit a scalar with write semantics + * + * Queue and output using the emitter a scalar using a standard + * write interface. + * + * @fye: The emitter to use + * @style: The scalar style to use + * @anchor: The anchor or NULL + * @tag: The tag or NULL + * @buf: Pointer to the buffer + * @count: The number of bytes to write + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_scalar_write(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *buf, size_t count) + FY_EXPORT; + +/** + * fy_emit_scalar_vprintf() - Emit a scalar with vprintf semantics + * + * Queue and output using the emitter a scalar using a standard + * vprintf interface. + * + * @fye: The emitter to use + * @style: The scalar style to use + * @anchor: The anchor or NULL + * @tag: The tag or NULL + * @fmt: The printf like format string + * @ap: The argument list + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_scalar_vprintf(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_emit_scalar_printf() - Emit a scalar with printf semantics + * + * Queue and output using the emitter a scalar using a standard + * printf interface. + * + * @fye: The emitter to use + * @style: The scalar style to use + * @anchor: The anchor or NULL + * @tag: The tag or NULL + * @fmt: The printf like format string + * @...: The extra arguments + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_scalar_printf(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *fmt, ...) + FY_FORMAT(printf, 5, 6) + FY_EXPORT; + +/** + * fy_emit_event_from_parser() - Queue (and possibly emit) an event + * generated by the parser. + * + * Queue and output using the emitter. This is the streaming + * output method which does not require creating a document. + * Similar to fy_emit_event() but it is more efficient. + * + * @emit: The emitter to use + * @fyp: The parser that generated the event + * @fye: The event to queue for emission + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_event_from_parser(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_event *fye) + FY_EXPORT; + +/** + * fy_emit_document() - Emit the document using the emitter + * + * Emits a document in YAML format using the emitter. + * + * @emit: The emitter + * @fyd: The document to emit + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_document(struct fy_emitter *emit, struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_emit_document_start() - Emit document start using the emitter + * + * Emits a document start using the emitter. This is used in case + * you need finer control over the emitting output. + * + * @emit: The emitter + * @fyd: The document to use for emitting it's start + * @fyn: The root (or NULL for using the document's root) + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_document_start(struct fy_emitter *emit, struct fy_document *fyd, + struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_emit_document_end() - Emit document end using the emitter + * + * Emits a document end using the emitter. This is used in case + * you need finer control over the emitting output. + * + * @emit: The emitter + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_document_end(struct fy_emitter *emit) + FY_EXPORT; + +/** + * fy_emit_node() - Emit a single node using the emitter + * + * Emits a single node using the emitter. This is used in case + * you need finer control over the emitting output. + * + * @emit: The emitter + * @fyn: The node to emit + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_node(struct fy_emitter *emit, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_emit_root_node() - Emit a single root node using the emitter + * + * Emits a single root node using the emitter. This is used in case + * you need finer control over the emitting output. + * + * @emit: The emitter + * @fyn: The root node to emit + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_root_node(struct fy_emitter *emit, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_emit_body_node() - Emit a single body node using the emitter + * + * Emits a single body node using the emitter. This is used in case + * you need finer control over the emitting output. + * No stream and document events will be generated at all. + * + * @emit: The emitter + * @fyn: The body node to emit + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_body_node(struct fy_emitter *emit, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_emit_explicit_document_end() - Emit an explicit document end + * + * Emits an explicit document end, i.e. ... . Use this if you + * you need finer control over the emitting output. + * + * @emit: The emitter + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_explicit_document_end(struct fy_emitter *emit) + FY_EXPORT; + +/** + * fy_emit_document_to_fp() - Emit a document to an file pointer + * + * Emits a document from the root to the given file pointer. + * + * @fyd: The document to emit + * @flags: The emitter flags to use + * @fp: The file pointer to output to + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_document_to_fp(struct fy_document *fyd, + enum fy_emitter_cfg_flags flags, FILE *fp) + FY_EXPORT; + +/** + * fy_emit_document_to_file() - Emit a document to file + * + * Emits a document from the root to the given file. + * The file will be fopen'ed using a "wa" mode. + * + * @fyd: The document to emit + * @flags: The emitter flags to use + * @filename: The filename to output to, or NULL for stdout + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_document_to_file(struct fy_document *fyd, + enum fy_emitter_cfg_flags flags, + const char *filename) + FY_EXPORT; + +/** + * fy_emit_document_to_fd() - Emit a document to a file descriptor + * + * Emits a document from the root to the given file descriptor + * + * @fyd: The document to emit + * @flags: The emitter flags to use + * @fd: The file descriptor to output to + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_document_to_fd(struct fy_document *fyd, + enum fy_emitter_cfg_flags flags, int fd) + FY_EXPORT; + +/** + * fy_emit_document_to_buffer() - Emit a document to a buffer + * + * Emits an document from the root to the given buffer. + * If the document does not fit, an error will be returned. + * + * @fyd: The document to emit + * @flags: The emitter flags to use + * @buf: Pointer to the buffer area to fill + * @size: Size of the buffer + * + * Returns: + * A positive number, which is the size of the emitted document + * on the buffer on success, -1 on error + */ +int +fy_emit_document_to_buffer(struct fy_document *fyd, + enum fy_emitter_cfg_flags flags, + char *buf, size_t size) + FY_EXPORT; + +/** + * fy_emit_document_to_string() - Emit a document to an allocated string + * + * Emits an document from the root to a string which will be dynamically + * allocated. + * + * @fyd: The document to emit + * @flags: The emitter flags to use + * + * Returns: + * A pointer to the allocated string, or NULL in case of an error + */ +char * +fy_emit_document_to_string(struct fy_document *fyd, + enum fy_emitter_cfg_flags flags) + FY_EXPORT; + +#define fy_emit_document_to_string_alloca(_fyd, _flags) \ + FY_ALLOCA_COPY_FREE(fy_emit_document_to_string((_fyd), (_flags)), FY_NT) + +/** + * fy_emit_node_to_buffer() - Emit a node (recursively) to a buffer + * + * Emits a node recursively to the given buffer. + * If the document does not fit, an error will be returned. + * + * @fyn: The node to emit + * @flags: The emitter flags to use + * @buf: Pointer to the buffer area to fill + * @size: Size of the buffer + * + * Returns: + * A positive number, which is the size of the emitted node + * on the buffer on success, -1 on error + */ +int +fy_emit_node_to_buffer(struct fy_node *fyn, enum fy_emitter_cfg_flags flags, + char *buf, size_t size) + FY_EXPORT; + +/** + * fy_emit_node_to_string() - Emit a node to an allocated string + * + * Emits a node recursively to a string which will be dynamically + * allocated. + * + * @fyn: The node to emit + * @flags: The emitter flags to use + * + * Returns: + * A pointer to the allocated string, or NULL in case of an error + */ +char * +fy_emit_node_to_string(struct fy_node *fyn, enum fy_emitter_cfg_flags flags) + FY_EXPORT; + +#define fy_emit_node_to_string_alloca(_fyn, _flags) \ + FY_ALLOCA_COPY_FREE(fy_emit_node_to_string((_fyn), (_flags)), FY_NT) + +/** + * fy_emit_to_buffer() - Create an emitter for buffer output. + * + * Creates a special purpose emitter for buffer output. + * Calls to fy_emit_event() populate the buffer. + * The contents are retreived by a call to fy_emit_to_buffer_collect() + * + * @flags: The emitter flags to use + * @buf: Pointer to the buffer area to fill + * @size: Size of the buffer + * + * Returns: + * The newly created emitter or NULL on error. + */ +struct fy_emitter * +fy_emit_to_buffer(enum fy_emitter_cfg_flags flags, char *buf, size_t size) + FY_EXPORT; + +/** + * fy_emit_to_buffer_collect() - Collect the buffer emitter output + * + * Collects the output of the emitter which was created by + * fy_emit_to_buffer() and populated using fy_emit_event() calls. + * The NULL terminated returned buffer is the same as the one used in the + * fy_emit_to_buffer() call and the sizep argument will be filled with + * the size of the buffer. + * + * @emit: The emitter + * @sizep: Pointer to the size to be filled + * + * Returns: + * The buffer or NULL in case of an error. + */ +char * +fy_emit_to_buffer_collect(struct fy_emitter *emit, size_t *sizep) + FY_EXPORT; + +/** + * fy_emit_to_string() - Create an emitter to create a dynamically + * allocated string. + * + * Creates a special purpose emitter for output to a dynamically + * allocated string. + * Calls to fy_emit_event() populate the buffer. + * The contents are retreived by a call to fy_emit_to_string_collect() + * + * @flags: The emitter flags to use + * + * Returns: + * The newly created emitter or NULL on error. + */ +struct fy_emitter * +fy_emit_to_string(enum fy_emitter_cfg_flags flags) + FY_EXPORT; + +/** + * fy_emit_to_string_collect() - Collect the string emitter output + * + * Collects the output of the emitter which was created by + * fy_emit_to_string() and populated using fy_emit_event() calls. + * The NULL terminated returned buffer is dynamically allocated + * and must be freed via a call to free(). + * The sizep argument will be filled with the size of the buffer. + * + * @emit: The emitter + * @sizep: Pointer to the size to be filled + * + * Returns: + * The dynamically allocated string or NULL in case of an error. + */ +char * +fy_emit_to_string_collect(struct fy_emitter *emit, size_t *sizep) + FY_EXPORT; + +/** + * fy_emitter_vlog() - Log using the emitters diagnostics printf style (va_arg) + * + * Output a log on the emitter diagnostic output + * + * @emit: The emitter + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_emitter_vlog(struct fy_emitter *emit, enum fy_error_type type, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_emitter_log() - Log using the emitters diagnostics printf style + * + * Output a report on the emitter's diagnostics + * + * @emit: The emitter + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments + */ +void +fy_emitter_log(struct fy_emitter *emit, enum fy_error_type type, const char *fmt, ...) + FY_FORMAT(printf, 3, 4) + FY_EXPORT; + +#ifndef NDEBUG + +#define fy_emitter_debug(_emit, _fmt, ...) \ + fy_emitter_log((_emit), FYET_DEBUG, (_fmt) , ## __VA_ARGS__) +#else + +#define fy_emitter_debug(_emit, _fmt, ...) \ + do { } while(0) + +#endif + +#define fy_emitter_info(_emit, _fmt, ...) \ + fy_emitter_log((_emit), FYET_INFO, (_fmt) , ## __VA_ARGS__) +#define fy_emitter_notice(_emit, _fmt, ...) \ + fy_emitter_log((_emit), FYET_NOTICE, (_fmt) , ## __VA_ARGS__) +#define fy_emitter_warning(_emit, _fmt, ...) \ + fy_emitter_log((_emit), FYET_WARNING, (_fmt) , ## __VA_ARGS__) +#define fy_emitter_error(_emit, _fmt, ...) \ + fy_emitter_log((_emit), FYET_ERROR, (_fmt) , ## __VA_ARGS__) + +/** + * fy_node_copy() - Copy a node, associating the new node with the given document + * + * Make a deep copy of a node, associating the copy with the given document. + * Note that no content copying takes place as the contents of the nodes + * are reference counted. This means that the operation is relatively inexpensive. + * + * Note that the copy includes all anchors contained in the subtree of the + * source, so this call will register them with the document. + * + * @fyd: The document which the resulting node will be associated with + * @fyn_from: The source node to recursively copy + * + * Returns: + * The copied node on success, NULL on error + */ +struct fy_node * +fy_node_copy(struct fy_document *fyd, struct fy_node *fyn_from) + FY_EXPORT; + +/** + * fy_document_clone() - Clones a document + * + * Clone a document, by making a deep copy of the source. + * Note that no content copying takes place as the contents of the nodes + * are reference counted. This means that the operation is relatively inexpensive. + * + * @fydsrc: The source document to clone + * + * Returns: + * The newly created clone document, or NULL in case of an error + */ +struct fy_document * +fy_document_clone(struct fy_document *fydsrc) + FY_EXPORT; + +/** + * fy_node_insert() - Insert a node to the given node + * + * Insert a node to another node. If @fyn_from is NULL then this + * operation will delete the @fyn_to node. + * + * The operation varies according to the types of the arguments: + * + * from: scalar + * + * to: another-scalar -> scalar + * to: { key: value } -> scalar + * to: [ seq0, seq1 ] -> scalar + * + * from: [ seq2 ] + * to: scalar -> [ seq2 ] + * to: { key: value } -> [ seq2 ] + * to: [ seq0, seq1 ] -> [ seq0, seq1, sec2 ] + * + * from: { another-key: another-value } + * to: scalar -> { another-key: another-value } + * to: { key: value } -> { key: value, another-key: another-value } + * to: [ seq0, seq1 ] -> { another-key: another-value } + * + * from: { key: another-value } + * to: scalar -> { key: another-value } + * to: { key: value } -> { key: another-value } + * to: [ seq0, seq1 ] -> { key: another-value } + * + * The rules are: + * - If target node changes type, source ovewrites target. + * - If source or target node are scalars, source it overwrites target. + * - If target and source are sequences the items of the source sequence + * are appended to the target sequence. + * - If target and source are maps the key, value pairs of the source + * are appended to the target map. If the target map contains a + * key-value pair that is present in the source map, it is overwriten + * by it. + * + * @fyn_to: The target node + * @fyn_from: The source node + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_insert(struct fy_node *fyn_to, struct fy_node *fyn_from) + FY_EXPORT; + +/** + * fy_node_delete() - Delete a node from a document + * + * Delete the given node. + * If it's part of a sequence it will be removed from it. + * If it's the value of a node key value pair, it will + * be removed from the mapping. + * + * This is an alias for fy_document_insert_at(fyn, NULL) + * + * Note that it is expected this node is attached to a document. + * Do not call this to free a node, because if it's part of + * a collection it will not be properly removed. + * + * @fyn: The node to delete. + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_delete(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_document_insert_at() - Insert a node to the given path in the document + * + * Insert a node to a given point in the document. If @fyn is NULL then this + * operation will delete the target node. + * + * Please see fy_node_insert for details of operation. + * + * Note that in any case the fyn node will be unref'ed. + * So if the operation fails, and the reference is 0 + * the node will be freed. If you want it to stick around + * take a reference before. + * + * @fyd: The document + * @path: The path where the insert operation will target + * @pathlen: The length of the path (or -1 if '\0' terminated) + * @fyn: The source node + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_document_insert_at(struct fy_document *fyd, + const char *path, size_t pathlen, + struct fy_node *fyn) + FY_EXPORT; + +/** + * enum fy_node_type - Node type + * + * Each node may be one of the following types + * + * @FYNT_SCALAR: Node is a scalar + * @FYNT_SEQUENCE: Node is a sequence + * @FYNT_MAPPING: Node is a mapping + */ +enum fy_node_type { + FYNT_SCALAR, + FYNT_SEQUENCE, + FYNT_MAPPING, +}; + +/** + * enum fy_node_style - Node style + * + * A node may contain a hint of how it should be + * rendered, encoded as a style. + * + * @FYNS_ANY: No hint, let the emitter decide + * @FYNS_FLOW: Prefer flow style (for sequence/mappings) + * @FYNS_BLOCK: Prefer block style (for sequence/mappings) + * @FYNS_PLAIN: Plain style preferred + * @FYNS_SINGLE_QUOTED: Single quoted style preferred + * @FYNS_DOUBLE_QUOTED: Double quoted style preferred + * @FYNS_LITERAL: Literal style preferred (valid in block context) + * @FYNS_FOLDED: Folded style preferred (valid in block context) + * @FYNS_ALIAS: It's an alias + */ +enum fy_node_style { + FYNS_ANY = -1, + FYNS_FLOW, + FYNS_BLOCK, + FYNS_PLAIN, + FYNS_SINGLE_QUOTED, + FYNS_DOUBLE_QUOTED, + FYNS_LITERAL, + FYNS_FOLDED, + FYNS_ALIAS, +}; + +/* maximum depth is 256 */ +#define FYNWF_MAXDEPTH_SHIFT 4 +#define FYNWF_MAXDEPTH_MASK 0xff +#define FYNWF_MAXDEPTH(x) (((x) & FYNWF_MAXDEPTH_MASK) << FYNWF_MAXDEPTH_SHIFT) +#define FYNWF_MARKER_SHIFT 12 +#define FYNWF_MARKER_MASK 0x1f +#define FYNWF_MARKER(x) (((x) & FYNWF_MARKER_MASK) << FYNWF_MARKER_SHIFT) +#define FYNWF_PTR_SHIFT 16 +#define FYNWF_PTR_MASK 0x03 +#define FYNWF_PTR(x) (((x) & FYNWF_PTR_MASK) << FYNWF_PTR_SHIFT) + +/** + * enum fy_node_walk_flags - Node walk flags + * + * @FYNWF_DONT_FOLLOW: Don't follow aliases during pathwalk + * @FYNWF_FOLLOW: Follow aliases during pathwalk + * @FYNWF_PTR_YAML: YAML pointer path walks + * @FYNWF_PTR_JSON: JSON pointer path walks + * @FYNWF_PTR_RELJSON: Relative JSON pointer path walks + * @FYNWF_PTR_YPATH: YPATH pointer path walks + * @FYNWF_URI_ENCODED: The path is URI encoded + * @FYNWF_MAXDEPTH_DEFAULT: Max follow depth is automatically determined + * @FYNWF_MARKER_DEFAULT: Default marker to use when scanning + * @FYNWF_PTR_DEFAULT: Default path type + */ +enum fy_node_walk_flags { + FYNWF_DONT_FOLLOW = 0, + FYNWF_FOLLOW = FY_BIT(0), + FYNWF_PTR_YAML = FYNWF_PTR(0), + FYNWF_PTR_JSON = FYNWF_PTR(1), + FYNWF_PTR_RELJSON = FYNWF_PTR(2), + FYNWF_PTR_YPATH = FYNWF_PTR(3), + FYNWF_URI_ENCODED = FY_BIT(18), + FYNWF_MAXDEPTH_DEFAULT = FYNWF_MAXDEPTH(0), + FYNWF_MARKER_DEFAULT = FYNWF_MARKER(0), + FYNWF_PTR_DEFAULT = FYNWF_PTR(0), +}; + +/* the maximum user marker */ +#define FYNWF_MAX_USER_MARKER 24 + +/** + * fy_node_style_from_scalar_style() - Convert from scalar to node style + * + * Convert a scalar style to a node style. + * + * @sstyle: The input scalar style + * + * Returns: + * The matching node style + */ +static inline enum fy_node_style +fy_node_style_from_scalar_style(enum fy_scalar_style sstyle) +{ + if (sstyle == FYSS_ANY) + return FYNS_ANY; + return (enum fy_node_style)(FYNS_PLAIN + (sstyle - FYSS_PLAIN)); +} + +/** + * typedef fy_node_mapping_sort_fn - Mapping sorting method function + * + * @fynp_a: The first node_pair used in the comparison + * @fynp_b: The second node_pair used in the comparison + * @arg: The opaque user provided pointer to the sort operation + * + * Returns: + * <0 if @fynp_a is less than @fynp_b + * 0 if @fynp_a is equal to fynp_b + * >0 if @fynp_a is greater than @fynp_b + */ +typedef int (*fy_node_mapping_sort_fn)(const struct fy_node_pair *fynp_a, + const struct fy_node_pair *fynp_b, + void *arg); + +/** + * typedef fy_node_scalar_compare_fn - Node compare method function for scalars + * + * @fyn_a: The first scalar node used in the comparison + * @fyn_b: The second scalar node used in the comparison + * @arg: The opaque user provided pointer to the compare operation + * + * Returns: + * <0 if @fyn_a is less than @fyn_b + * 0 if @fyn_a is equal to fyn_b + * >0 if @fyn_a is greater than @fyn_b + */ +typedef int (*fy_node_scalar_compare_fn)(struct fy_node *fyn_a, + struct fy_node *fyn_b, + void *arg); + +/** + * fy_node_compare() - Compare two nodes for equality + * + * Compare two nodes for equality. + * The comparison is 'deep', i.e. it recurses in subnodes, + * and orders the keys of maps using default libc strcmp + * ordering. For scalar the comparison is performed after + * any escaping so it's a true content comparison. + * + * @fyn1: The first node to use in the comparison + * @fyn2: The second node to use in the comparison + * + * Returns: + * true if the nodes contain the same content, false otherwise + */ +bool +fy_node_compare(struct fy_node *fyn1, struct fy_node *fyn2) + FY_EXPORT; + +/** + * fy_node_compare_user() - Compare two nodes for equality using + * user supplied sort and scalar compare methods + * + * Compare two nodes for equality using user supplied sot and scalar + * compare methods. + * The comparison is 'deep', i.e. it recurses in subnodes, + * and orders the keys of maps using the supplied mapping sort method for + * ordering. For scalars the comparison is performed using the supplied + * scalar node compare methods. + * + * @fyn1: The first node to use in the comparison + * @fyn2: The second node to use in the comparison + * @sort_fn: The method to use for sorting maps, or NULL for the default + * @sort_fn_arg: The extra user supplied argument to the @sort_fn + * @cmp_fn: The method to use for comparing scalars, or NULL for the default + * @cmp_fn_arg: The extra user supplied argument to the @cmp_fn + * + * Returns: + * true if the nodes contain the same content, false otherwise + */ +bool +fy_node_compare_user(struct fy_node *fyn1, struct fy_node *fyn2, + fy_node_mapping_sort_fn sort_fn, void *sort_fn_arg, + fy_node_scalar_compare_fn cmp_fn, void *cmp_fn_arg) + FY_EXPORT; + +/** + * fy_node_compare_string() - Compare a node for equality with a YAML string + * + * Compare a node for equality with a YAML string. + * The comparison is performed using fy_node_compare() with the + * first node supplied as an argument and the second being generated + * by calling fy_document_build_from_string with the YAML string. + * + * @fyn: The node to use in the comparison + * @str: The YAML string to compare against + * @len: The length of the string (or -1 if '\0' terminated) + * + * Returns: + * true if the node and the string are equal. + */ +bool +fy_node_compare_string(struct fy_node *fyn, const char *str, size_t len) + FY_EXPORT; + +/** + * fy_node_compare_token() - Compare a node for equality against a token + * + * Compare a node for equality with a token. + * Both the node and the tokens must be a scalars. + * + * @fyn: The node to use in the comparison + * @fyt: The scalar token + * + * Returns: + * true if the node and the token are equal. + */ +bool +fy_node_compare_token(struct fy_node *fyn, struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_node_compare_text() - Compare a node for equality with a raw C text + * + * Compare a node for equality with a raw C string. + * The node must be a scalar. + * + * @fyn: The node to use in the comparison + * @text: The raw C text to compare against + * @len: The length of the text (or -1 if '\0' terminated) + * + * Returns: + * true if the node and the text are equal. + */ +bool +fy_node_compare_text(struct fy_node *fyn, const char *text, size_t len) + FY_EXPORT; + +/** + * fy_document_create() - Create an empty document + * + * Create an empty document using the provided parser configuration. + * If NULL use the default parse configuration. + * + * @cfg: The parse configuration to use or NULL for the default. + * + * Returns: + * The created empty document, or NULL on error. + */ +struct fy_document * +fy_document_create(const struct fy_parse_cfg *cfg) + FY_EXPORT; + +/** + * fy_document_destroy() - Destroy a document previously created via + * fy_document_create() + * + * Destroy a document (along with all children documents) + * + * @fyd: The document to destroy + * + */ +void +fy_document_destroy(struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_get_cfg() - Get the configuration of a document + * + * @fyd: The document + * + * Returns: + * The configuration of the document + */ +const struct fy_parse_cfg * +fy_document_get_cfg(struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_get_diag() - Get the diagnostic object of a document + * + * Return a pointer to the diagnostic object of a document object. + * Note that the returned diag object has a reference taken so + * you should fy_diag_unref() it when you're done with it. + * + * @fyd: The document to get the diagnostic object + * + * Returns: + * A pointer to a ref'ed diagnostic object or NULL in case of an + * error. + */ +struct fy_diag * +fy_document_get_diag(struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_set_diag() - Set the diagnostic object of a document + * + * Replace the documents's current diagnostic object with the one + * given as an argument. The previous diagnostic object will be + * unref'ed (and freed if its reference gets to 0). + * Also note that the diag argument shall take a reference too. + * + * @fyd: The document to replace the diagnostic object + * @diag: The document's new diagnostic object, NULL for default + * + * Returns: + * 0 if everything OK, -1 otherwise + */ +int +fy_document_set_diag(struct fy_document *fyd, struct fy_diag *diag) + FY_EXPORT; + +/** + * fy_document_set_parent() - Make a document a child of another + * + * Set the parent of @fyd_child document to be @fyd + * + * @fyd: The parent document + * @fyd_child: The child document + * + * Returns: + * 0 if all OK, -1 on error. + */ +int +fy_document_set_parent(struct fy_document *fyd, struct fy_document *fyd_child) + FY_EXPORT; + +/** + * fy_document_build_from_string() - Create a document using the provided YAML source. + * + * Create a document parsing the provided string as a YAML source. + * + * @cfg: The parse configuration to use or NULL for the default. + * @str: The YAML source to use. + * @len: The length of the string (or -1 if '\0' terminated) + * + * Returns: + * The created document, or NULL on error. + */ +struct fy_document * +fy_document_build_from_string(const struct fy_parse_cfg *cfg, + const char *str, size_t len) + FY_EXPORT; + +/** + * fy_document_build_from_malloc_string() - Create a document using the provided YAML source which was malloced. + * + * Create a document parsing the provided string as a YAML source. The string is expected to have been + * allocated by malloc(3) and when the document is destroyed it will be automatically freed. + * + * @cfg: The parse configuration to use or NULL for the default. + * @str: The YAML source to use. + * @len: The length of the string (or -1 if '\0' terminated) + * + * Returns: + * The created document, or NULL on error. + */ +struct fy_document * +fy_document_build_from_malloc_string(const struct fy_parse_cfg *cfg, + char *str, size_t len) + FY_EXPORT; +/** + * fy_document_build_from_file() - Create a document parsing the given file + * + * Create a document parsing the provided file as a YAML source. + * + * @cfg: The parse configuration to use or NULL for the default. + * @file: The name of the file to parse + * + * Returns: + * The created document, or NULL on error. + */ +struct fy_document * +fy_document_build_from_file(const struct fy_parse_cfg *cfg, const char *file) + FY_EXPORT; + +/** + * fy_document_build_from_fp() - Create a document parsing the given file pointer + * + * Create a document parsing the provided file pointer as a YAML source. + * + * @cfg: The parse configuration to use or NULL for the default. + * @fp: The file pointer + * + * Returns: + * The created document, or NULL on error. + */ +struct fy_document * +fy_document_build_from_fp(const struct fy_parse_cfg *cfg, FILE *fp) + FY_EXPORT; + +/** + * fy_document_vbuildf() - Create a document using the provided YAML via vprintf formatting + * + * Create a document parsing the provided string as a YAML source. The string + * is created by applying vprintf formatting. + * + * @cfg: The parse configuration to use or NULL for the default. + * @fmt: The format string creating the YAML source to use. + * @ap: The va_list argument pointer + * + * Returns: + * The created document, or NULL on error. + */ +struct fy_document * +fy_document_vbuildf(const struct fy_parse_cfg *cfg, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_document_buildf() - Create a document using the provided YAML source via printf formatting + * + * Create a document parsing the provided string as a YAML source. The string + * is created by applying printf formatting. + * + * @cfg: The parse configuration to use or NULL for the default. + * @fmt: The format string creating the YAML source to use. + * @...: The printf arguments + * + * Returns: + * The created document, or NULL on error. + */ +struct fy_document * +fy_document_buildf(const struct fy_parse_cfg *cfg, const char *fmt, ...) + FY_FORMAT(printf, 2, 3) + FY_EXPORT; + +/** + * fy_flow_document_build_from_string() - Create a document using the provided YAML source. + * + * Create a document parsing the provided string as a YAML source. + * + * The document is a flow document, i.e. does not contain any block content + * and is usually laid out in a single line. + * + * Example of flow documents: + * + * plain-scalar + * "double-quoted-scalar" + * 'single-quoted-scalar' + * { foo: bar } + * [ 0, 1, 2 ] + * + * A flow document is important because parsing stops at the end + * of it, and so can be placed in other non-yaml content. + * + * @cfg: The parse configuration to use or NULL for the default. + * @str: The YAML source to use. + * @len: The length of the string (or -1 if '\0' terminated) + * @consumed: A pointer to the consumed amount + * + * Returns: + * The created document, or NULL on error. + */ +struct fy_document * +fy_flow_document_build_from_string(const struct fy_parse_cfg *cfg, + const char *str, size_t len, size_t *consumed) + FY_EXPORT; + +/** + * fy_block_document_build_from_string() - Create a document using the provided YAML source. + * + * Create a document parsing the provided string as a YAML source. + * + * The document is a block document, and it terminates when indentation + * appears to do so. + * + * Example of block documents:: + * + * this-is-yaml + * foo: bar <- starts here + * baz: [1, 2] + * this-is-yaml-no-more + * + * @cfg: The parse configuration to use or NULL for the default. + * @str: The YAML source to use. + * @len: The length of the string (or -1 if '\0' terminated) + * @consumed: A pointer to the consumed amount + * + * Returns: + * The created document, or NULL on error. + */ +struct fy_document * +fy_block_document_build_from_string(const struct fy_parse_cfg *cfg, + const char *str, size_t len, size_t *consumed) + FY_EXPORT; + +/** + * fy_document_root() - Return the root node of the document + * + * Returns the root of the document. If the document is empty + * NULL will be returned instead. + * + * @fyd: The document + * + * Returns: + * The root of the document, or NULL if the document is empty. + */ +struct fy_node * +fy_document_root(struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_set_root() - Set the root of the document + * + * Set the root of a document. If the document was not empty + * the old root will be freed. If @fyn is NULL then the + * document is set to empty. + * + * @fyd: The document + * @fyn: The new root of the document. + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_document_set_root(struct fy_document *fyd, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_type() - Get the node type + * + * Retrieve the node type. It is one of FYNT_SCALAR, FYNT_SEQUENCE + * or FYNT_MAPPING. A NULL node argument is a FYNT_SCALAR. + * + * @fyn: The node + * + * Returns: + * The node type + */ +enum fy_node_type +fy_node_get_type(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_style() - Get the node style + * + * Retrieve the node rendering style. + * If the node is NULL then the style is FYNS_PLAIN. + * + * @fyn: The node + * + * Returns: + * The node style + */ +enum fy_node_style +fy_node_get_style(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_start_token() - Get the node start token + * + * Retrieve the node start token. + * + * For scalars, this is the same as the end token. + * + * @fyn: The node + * + * Returns: + * The node start token + */ +struct fy_token * fy_node_get_start_token(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_end_token() - Get the node end token + * + * Retrieve the node end token. + * + * For scalars, this is the same as the start token. + * + * @fyn: The node + * + * Returns: + * The node end token + */ + struct fy_token * fy_node_get_end_token(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_is_scalar() - Check whether the node is a scalar + * + * Convenience method for checking whether a node is a scalar. + * + * @fyn: The node + * + * Returns: + * true if the node is a scalar, false otherwise + */ +static inline bool +fy_node_is_scalar(struct fy_node *fyn) +{ + return fy_node_get_type(fyn) == FYNT_SCALAR; +} + +/** + * fy_node_is_sequence() - Check whether the node is a sequence + * + * Convenience method for checking whether a node is a sequence. + * + * @fyn: The node + * + * Returns: + * true if the node is a sequence, false otherwise + */ +static inline bool +fy_node_is_sequence(struct fy_node *fyn) +{ + return fy_node_get_type(fyn) == FYNT_SEQUENCE; +} + +/** + * fy_node_is_mapping() - Check whether the node is a mapping + * + * Convenience method for checking whether a node is a mapping. + * + * @fyn: The node + * + * Returns: + * true if the node is a mapping, false otherwise + */ +static inline bool +fy_node_is_mapping(struct fy_node *fyn) +{ + return fy_node_get_type(fyn) == FYNT_MAPPING; +} + +/** + * fy_node_is_alias() - Check whether the node is an alias + * + * Convenience method for checking whether a node is an alias. + * + * @fyn: The node + * + * Returns: + * true if the node is an alias, false otherwise + */ +static inline bool +fy_node_is_alias(struct fy_node *fyn) +{ + return fy_node_get_type(fyn) == FYNT_SCALAR && + fy_node_get_style(fyn) == FYNS_ALIAS; +} + +/** + * fy_node_is_null() - Check whether the node is a NULL + * + * Convenience method for checking whether a node is a NULL scalar.. + * Note that a NULL node argument returns true... + * + * @fyn: The node + * + * Returns: + * true if the node is a NULL scalar, false otherwise + */ +bool +fy_node_is_null(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_is_attached() - Check whether the node is attached + * + * Checks whether a node is attached in a document structure. + * An attached node may not be freed, before being detached. + * Note that there is no method that explicitly detaches + * a node, since this is handled internaly by the sequence + * and mapping removal methods. + * + * @fyn: The node + * + * Returns: + * true if the node is attached, false otherwise + */ +bool +fy_node_is_attached(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_tag_token() - Gets the tag token of a node (if it exists) + * + * Gets the tag token of a node, if it exists + * + * @fyn: The node which has the tag token to be returned + * + * Returns: + * The tag token of the given node, or NULL if the tag does not + * exist. + */ +struct fy_token * +fy_node_get_tag_token(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_scalar_token() - Gets the scalar token of a node (if it exists) + * + * Gets the scalar token of a node, if it exists and the node is a valid scalar + * node. Note that aliases are scalars, so if this call is issued on an alias + * node the return shall be of an alias token. + * + * @fyn: The node which has the scalar token to be returned + * + * Returns: + * The scalar token of the given node, or NULL if the node is not a scalar. + */ +struct fy_token * +fy_node_get_scalar_token(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_resolve_alias() - Resolve an alias node + * + * Resolve an alias node, following any subsequent aliases until + * a non alias node has been found. This call performs cycle detection + * and excessive redirections checks so it's safe to call in any + * context. + * + * @fyn: The alias node to be resolved + * + * Returns: + * The resolved alias node, or NULL if either fyn is not an alias, or + * resolution fails due to a graph cycle. + */ +struct fy_node * +fy_node_resolve_alias(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_dereference() - Dereference a single alias node + * + * Dereference an alias node. This is different than resolution + * in that will only perform a single alias follow call and + * it will fail if the input is not an alias. + * This call performs cycle detection + * and excessive redirections checks so it's safe to call in any + * context. + * + * @fyn: The alias node to be dereferenced + * + * Returns: + * The dereferenced alias node, or NULL if either fyn is not an alias, or + * resolution fails due to a graph cycle. + */ +struct fy_node * +fy_node_dereference(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_free() - Free a node + * + * Recursively frees the given node releasing the memory it uses, removing + * any anchors on the document it contains, and releasing references + * on the tokens it contains. + * + * This method will return an error if the node is attached, or + * if not NULL it is not a member of a document. + * + * @fyn: The node to free + * + * Returns: + * 0 on success, -1 on error. + */ +int +fy_node_free(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_build_from_string() - Create a node using the provided YAML source. + * + * Create a node parsing the provided string as a YAML source. The + * node created will be associated with the provided document. + * + * @fyd: The document + * @str: The YAML source to use. + * @len: The length of the string (or -1 if '\0' terminated) + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_build_from_string(struct fy_document *fyd, + const char *str, size_t len) + FY_EXPORT; + +/** + * fy_node_build_from_malloc_string() - Create a node using the provided YAML source which was malloced. + * + * Create a node parsing the provided string as a YAML source. The + * node created will be associated with the provided document. The string is expected to have been + * allocated by malloc(3) and when the document is destroyed it will be automatically freed. + * + * @fyd: The document + * @str: The YAML source to use. + * @len: The length of the string (or -1 if '\0' terminated) + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_build_from_malloc_string(struct fy_document *fyd, + char *str, size_t len) + FY_EXPORT; + + +/** + * fy_node_build_from_file() - Create a node using the provided YAML file. + * + * Create a node parsing the provided file as a YAML source. The + * node created will be associated with the provided document. + * + * @fyd: The document + * @file: The name of the file to parse + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_build_from_file(struct fy_document *fyd, const char *file) + FY_EXPORT; + +/** + * fy_node_build_from_fp() - Create a node using the provided file pointer. + * + * Create a node parsing the provided file pointer as a YAML source. The + * node created will be associated with the provided document. + * + * @fyd: The document + * @fp: The file pointer + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_build_from_fp(struct fy_document *fyd, FILE *fp) + FY_EXPORT; + +/** + * fy_node_vbuildf() - Create a node using the provided YAML source via vprintf formatting + * + * Create a node parsing the resulting string as a YAML source. The string + * is created by applying vprintf formatting. + * + * @fyd: The document + * @fmt: The format string creating the YAML source to use. + * @ap: The va_list argument pointer + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_vbuildf(struct fy_document *fyd, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_node_buildf() - Create a node using the provided YAML source via printf formatting + * + * Create a node parsing the resulting string as a YAML source. The string + * is created by applying printf formatting. + * + * @fyd: The document + * @fmt: The format string creating the YAML source to use. + * @...: The printf arguments + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_buildf(struct fy_document *fyd, const char *fmt, ...) + FY_FORMAT(printf, 2, 3) + FY_EXPORT; + +/** + * fy_node_by_path() - Retrieve a node using the provided path spec. + * + * This method will retrieve a node relative to the given node using + * the provided path spec. + * + * Path specs are comprised of keys seperated by slashes '/'. + * Keys are either regular YAML expressions in flow format for traversing + * mappings, or indexes in brackets for traversing sequences. + * Path specs may start with '/' which is silently ignored. + * + * A few examples will make this clear:: + * + * fyn = { foo: bar } - fy_node_by_path(fyn, "/foo") -> bar + * fyn = [ foo, bar ] - fy_node_by_path(fyn, "1") -> bar + * fyn = { { foo: bar }: baz } - fy_node_by_path(fyn, "{foo: bar}") -> baz + * fyn = [ foo, { bar: baz } } - fy_node_by_path(fyn, "1/bar") -> baz + * + * Note that the special characters /{}[] are not escaped in plain style, + * so you will not be able to use them as path traversal keys. + * In that case you can easily use either the single, or double quoted forms:: + * + * fyn = { foo/bar: baz } - fy_node_by_path(fyn, "'foo/bar'") -> baz + * + * @fyn: The node to use as start of the traversal operation + * @path: The path spec to use in the traversal operation + * @len: The length of the path (or -1 if '\0' terminated) + * @flags: The extra path walk flags + * + * Returns: + * The retrieved node, or NULL if not possible to be found. + */ +struct fy_node * +fy_node_by_path(struct fy_node *fyn, const char *path, size_t len, + enum fy_node_walk_flags flags) + FY_EXPORT; + +/** + * fy_node_get_path() - Get the path of this node + * + * Retrieve the given node's path address relative to the document root. + * The address is dynamically allocated and should be freed when + * you're done with it. + * + * @fyn: The node + * + * Returns: + * The node's address, or NULL if fyn is the root. + */ +char * +fy_node_get_path(struct fy_node *fyn) + FY_EXPORT; + +#define fy_node_get_path_alloca(_fyn) \ + FY_ALLOCA_COPY_FREE_NO_NULL(fy_node_get_path((_fyn)), FY_NT) + +/** + * fy_node_get_parent() - Get the parent node of a node + * + * Get the parent node of a node. The parent of a document's root + * is NULL, and so is the parent of the root of a key node's of a mapping. + * This is because the nodes of a key may not be addressed using a + * path expression. + * + * @fyn: The node + * + * Returns: + * The node's parent, or NULL if fyn is the root, or the root of a key mapping. + */ +struct fy_node * +fy_node_get_parent(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_document_parent() - Get the document parent node of a node + * + * Get the document parent node of a node. The document parent differs + * than the regular parent in that a key's root node of a mapping is not + * NULL, rather it points to the actual node parent. + * + * @fyn: The node + * + * Returns: + * The node's document parent, or NULL if fyn is the root + */ +struct fy_node * +fy_node_get_document_parent(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_parent_address() - Get the path address of this node's parent + * + * Retrieve the given node's parent path address + * The address is dynamically allocated and should be freed when + * you're done with it. + * + * @fyn: The node + * + * Returns: + * The parent's address, or NULL if fyn is the root. + */ +char * +fy_node_get_parent_address(struct fy_node *fyn) + FY_EXPORT; + +#define fy_node_get_parent_address_alloca(_fyn) \ + FY_ALLOCA_COPY_FREE_NO_NULL(fy_node_get_parent_address((_fyn)), FY_NT) + +/** + * fy_node_get_path_relative_to() - Get a path address of a node relative + * to one of it's parents + * + * Retrieve the given node's path address relative to an arbitrary + * parent in the tree. + * The address is dynamically allocated and should be freed when + * you're done with it. + * + * @fyn_parent: The node parent/grandparent... + * @fyn: The node + * + * Returns: + * The relative address from the parent to the node + */ +char * +fy_node_get_path_relative_to(struct fy_node *fyn_parent, struct fy_node *fyn) + FY_EXPORT; + +#define fy_node_get_path_relative_to_alloca(_fynp, _fyn) \ + FY_ALLOCA_COPY_FREE_NO_NULL(fy_node_get_path_relative_to((_fynp), (_fyn)), FY_NT) + +/** + * fy_node_get_short_path() - Get a path address of a node in the shortest path possible + * + * Retrieve the given nodes short path address relative to the + * closest anchor (either on this node, or its parent). + * If no such parent is found then returns the absolute path + * from the start of the document. + * + * Example:: + * + * --- &foo + * foo: &bar + * bar + * baz + * + * - The short path of /foo is \*foo + * - The short path of /foo/bar is \*bar + * - The short path of /baz is \*foo/baz + * + * The address is dynamically allocated and should be freed when + * you're done with it. + * + * @fyn: The node + * + * Returns: + * The shortest path describing the node + */ +char * +fy_node_get_short_path(struct fy_node *fyn) + FY_EXPORT; + +#define fy_node_get_short_path_alloca(_fyn) \ + FY_ALLOCA_COPY_FREE_NO_NULL(fy_node_get_short_path((_fyn)), FY_NT) + +/** + * fy_node_get_reference() - Get a textual reference to a node + * + * Retrieve the given node's textual reference. If the node + * contains an anchor the expression that references the anchor + * will be returned, otherwise an absolute path reference relative + * to the root of the document will be returned. + * + * The address is dynamically allocated and should be freed when + * you're done with it. + * + * @fyn: The node + * + * Returns: + * The node's text reference. + */ +char * +fy_node_get_reference(struct fy_node *fyn) + FY_EXPORT; + +#define fy_node_get_reference_alloca(_fyn) \ + FY_ALLOCA_COPY_FREE_NO_NULL(fy_node_get_reference((_fyn)), FY_NT) + +/** + * fy_node_create_reference() - Create an alias reference node + * + * Create an alias node reference. If the node + * contains an anchor the expression that references the alias will + * use the anchor, otherwise an absolute path reference relative + * to the root of the document will be created. + * + * @fyn: The node + * + * Returns: + * An alias node referencing the argument node + */ +struct fy_node * +fy_node_create_reference(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_relative_reference() - Get a textual reference to a node + * relative to a base node. + * + * Retrieve the given node's textual reference as generated using + * another parent (or grand parent) as a base. + * If the node contains an anchor the expression that references the anchor + * will be returned. + * If the base node contains an anchor the reference will be relative to it + * otherwise an absolute path reference will be returned. + * + * The address is dynamically allocated and should be freed when + * you're done with it. + * + * @fyn_base: The base node + * @fyn: The node + * + * Returns: + * The node's text reference. + */ +char * +fy_node_get_relative_reference(struct fy_node *fyn_base, struct fy_node *fyn) + FY_EXPORT; + +#define fy_node_get_relative_reference_alloca(_fynb, _fyn) \ + FY_ALLOCA_COPY_FREE_NO_NULL(fy_node_get_relative_reference((_fynb), (_fyn)), FY_NT) + +/** + * fy_node_create_relative_reference() - Create an alias reference node + * + * Create a relative alias node reference using + * another parent (or grand parent) as a base. + * If the node contains an anchor the alias will reference the anchor. + * If the base node contains an anchor the alias will be relative to it + * otherwise an absolute path reference will be created. + * + * @fyn_base: The base node + * @fyn: The node + * + * Returns: + * An alias node referencing the argument node relative to the base + */ +struct fy_node * +fy_node_create_relative_reference(struct fy_node *fyn_base, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_create_scalar() - Create a scalar node. + * + * Create a scalar node using the provided memory area as input. + * The input is expected to be regular utf8 encoded. It may contain + * escaped characters in which case the style of the scalar will be + * set to double quoted. + * + * Note that the data are not copied, merely a reference is taken, so + * it must be available while the node is in use. + * + * @fyd: The document which the resulting node will be associated with + * @data: Pointer to the data area + * @size: Size of the data area, or (size_t)-1 for '\0' terminated data. + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_create_scalar(struct fy_document *fyd, + const char *data, size_t size) + FY_EXPORT; + +/** + * fy_node_create_scalar_copy() - Create a scalar node copying the data. + * + * Create a scalar node using the provided memory area as input. + * The input is expected to be regular utf8 encoded. It may contain + * escaped characters in which case the style of the scalar will be + * set to double quoted. + * + * A copy of the data will be made, so it is safe to free the data + * after the call. + * + * @fyd: The document which the resulting node will be associated with + * @data: Pointer to the data area + * @size: Size of the data area, or (size_t)-1 for '\0' terminated data. + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_create_scalar_copy(struct fy_document *fyd, + const char *data, size_t size) + FY_EXPORT; + +/** + * fy_node_create_vscalarf() - vprintf interface for creating scalars + * + * Create a scalar node using a vprintf interface. + * The input is expected to be regular utf8 encoded. It may contain + * escaped characters in which case the style of the scalar will be + * set to double quoted. + * + * @fyd: The document which the resulting node will be associated with + * @fmt: The printf based format string + * @ap: The va_list containing the arguments + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_create_vscalarf(struct fy_document *fyd, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_node_create_scalarf() - printf interface for creating scalars + * + * Create a scalar node using a printf interface. + * The input is expected to be regular utf8 encoded. It may contain + * escaped characters in which case the style of the scalar will be + * set to double quoted. + * + * @fyd: The document which the resulting node will be associated with + * @fmt: The printf based format string + * @...: The arguments + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_create_scalarf(struct fy_document *fyd, const char *fmt, ...) + FY_FORMAT(printf, 2, 3) + FY_EXPORT; + +/** + * fy_node_create_sequence() - Create an empty sequence node. + * + * Create an empty sequence node associated with the given document. + * + * @fyd: The document which the resulting node will be associated with + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_create_sequence(struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_node_create_mapping() - Create an empty mapping node. + * + * Create an empty mapping node associated with the given document. + * + * @fyd: The document which the resulting node will be associated with + * + * Returns: + * The created node, or NULL on error. + */ +struct fy_node * +fy_node_create_mapping(struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_node_set_tag() - Set the tag of node + * + * Manually set the tag of a node. The tag must be a valid one for + * the document the node belongs to. + * + * Note that the data are not copied, merely a reference is taken, so + * it must be available while the node is in use. + * + * If the node already contains a tag it will be overwriten. + * + * @fyn: The node to set it's tag. + * @data: Pointer to the tag data. + * @len: Size of the tag data, or (size_t)-1 for '\0' terminated. + * + * Returns: + * 0 on success, -1 on error. + */ +int +fy_node_set_tag(struct fy_node *fyn, const char *data, size_t len) + FY_EXPORT; + +/** + * fy_node_remove_tag() - Remove the tag of node + * + * Remove the tag of a node. + * + * @fyn: The node to remove it's tag. + * + * Returns: + * 0 on success, -1 on error. + */ +int +fy_node_remove_tag(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_tag() - Get the tag of the node + * + * This method will return a pointer to the text of a tag + * along with the length of it. Note that this text is *not* + * NULL terminated. + * + * @fyn: The node + * @lenp: Pointer to a variable that will hold the returned length + * + * Returns: + * A pointer to the tag of the node, while @lenp will be assigned the + * length of said tag. + * A NULL will be returned in case of an error. + */ +const char * +fy_node_get_tag(struct fy_node *fyn, size_t *lenp) + FY_EXPORT; + +/** + * fy_node_get_tag0() - Get the tag of the node + * + * This method will return a pointer to the text of a tag, + * which will be NULL terminated. + * + * @fyn: The node + * + * Returns: + * A pointer to the null terminated tag of the node. + * A NULL will be returned in case of an error. + */ +const char * +fy_node_get_tag0(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_tag_length() - Get the length of the tag of the node + * + * This method will return the size of the tag of the node. + * If the node is not tagged it will return 0. + * + * @fyn: The tagged node + * + * Returns: + * The size of the tag, or 0 if node is not tagged. + */ +size_t +fy_node_get_tag_length(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_scalar() - Get the scalar content of the node + * + * This method will return a pointer to the text of the scalar + * content of a node along with the length of it. + * Note that this pointer is *not* NULL terminated. + * + * @fyn: The scalar node + * @lenp: Pointer to a variable that will hold the returned length + * + * Returns: + * A pointer to the scalar content of the node, while @lenp will be assigned the + * length of said content. + * A NULL will be returned in case of an error, i.e. the node is not + * a scalar. + */ +const char * +fy_node_get_scalar(struct fy_node *fyn, size_t *lenp) + FY_EXPORT; + +/** + * fy_node_get_scalar0() - Get the scalar content of the node + * + * This method will return a pointer to the text of the scalar + * content of a node as a null terminated string. + * Note that this call will allocate memory to hold the null terminated + * string so if possible use fy_node_get_scalar() + * + * @fyn: The scalar node + * + * Returns: + * A pointer to the scalar content of the node or NULL in returned in case of an error. + */ +const char * +fy_node_get_scalar0(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_scalar_length() - Get the length of the scalar content + * + * This method will return the size of the scalar content of the node. + * If the node is not a scalar it will return 0. + * + * @fyn: The scalar node + * + * Returns: + * The size of the scalar content, or 0 if node is not scalar. + */ +size_t +fy_node_get_scalar_length(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_scalar_utf8_length() - Get the length of the scalar content + * in utf8 characters + * + * This method will return the size of the scalar content of the node in + * utf8 characters. + * If the node is not a scalar it will return 0. + * + * @fyn: The scalar node + * + * Returns: + * The size of the scalar content in utf8 characters, or 0 if node is not scalar. + */ +size_t +fy_node_get_scalar_utf8_length(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_sequence_iterate() - Iterate over a sequence node + * + * This method iterates over all the item nodes in the sequence node. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * @fyn: The sequence node + * @prevp: The previous sequence iterator + * + * Returns: + * The next node in sequence or NULL at the end of the sequence. + */ +struct fy_node * +fy_node_sequence_iterate(struct fy_node *fyn, void **prevp) + FY_EXPORT; + +/** + * fy_node_sequence_reverse_iterate() - Iterate over a sequence node in reverse + * + * This method iterates in reverse over all the item nodes in the sequence node. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * @fyn: The sequence node + * @prevp: The previous sequence iterator + * + * Returns: + * The next node in reverse sequence or NULL at the end of the sequence. + */ +struct fy_node * +fy_node_sequence_reverse_iterate(struct fy_node *fyn, void **prevp) + FY_EXPORT; + +/** + * fy_node_sequence_item_count() - Return the item count of the sequence + * + * Get the item count of the sequence. + * + * @fyn: The sequence node + * + * Returns: + * The count of items in the sequence or -1 in case of an error. + */ +int +fy_node_sequence_item_count(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_sequence_is_empty() - Check whether the sequence is empty + * + * Check whether the sequence contains items. + * + * @fyn: The sequence node + * + * Returns: + * true if the node is a sequence containing items, false otherwise + */ +bool +fy_node_sequence_is_empty(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_sequence_get_by_index() - Return a sequence item by index + * + * Retrieve a node in the sequence using it's index. If index + * is positive or zero the count is from the start of the sequence, + * while if negative from the end. I.e. -1 returns the last item + * in the sequence. + * + * @fyn: The sequence node + * @index: The index of the node to retrieve. + * - >= 0 counting from the start + * - < 0 counting from the end + * + * Returns: + * The node at the specified index or NULL if no such item exist. + */ +struct fy_node * +fy_node_sequence_get_by_index(struct fy_node *fyn, int index) + FY_EXPORT; + +/** + * fy_node_sequence_append() - Append a node item to a sequence + * + * Append a node item to a sequence. + * + * @fyn_seq: The sequence node + * @fyn: The node item to append + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_sequence_append(struct fy_node *fyn_seq, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_sequence_prepend() - Append a node item to a sequence + * + * Prepend a node item to a sequence. + * + * @fyn_seq: The sequence node + * @fyn: The node item to prepend + * + * Returns: + * 0 on success, -1 on error + */ +int fy_node_sequence_prepend(struct fy_node *fyn_seq, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_sequence_insert_before() - Insert a node item before another + * + * Insert a node item before another in the sequence. + * + * @fyn_seq: The sequence node + * @fyn_mark: The node item which the node will be inserted before. + * @fyn: The node item to insert in the sequence. + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_sequence_insert_before(struct fy_node *fyn_seq, + struct fy_node *fyn_mark, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_sequence_insert_after() - Insert a node item after another + * + * Insert a node item after another in the sequence. + * + * @fyn_seq: The sequence node + * @fyn_mark: The node item which the node will be inserted after. + * @fyn: The node item to insert in the sequence. + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_sequence_insert_after(struct fy_node *fyn_seq, + struct fy_node *fyn_mark, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_sequence_remove() - Remove an item from a sequence + * + * Remove a node item from a sequence and return it. + * + * @fyn_seq: The sequence node + * @fyn: The node item to remove from the sequence. + * + * Returns: + * The removed node item fyn, or NULL in case of an error. + */ +struct fy_node * +fy_node_sequence_remove(struct fy_node *fyn_seq, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_mapping_iterate() - Iterate over a mapping node + * + * This method iterates over all the node pairs in the mapping node. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * Note that while a mapping is an unordered collection of key/values + * the order of which they are created is important for presentation + * purposes. + * + * @fyn: The mapping node + * @prevp: The previous sequence iterator + * + * Returns: + * The next node pair in the mapping or NULL at the end of the mapping. + */ +struct fy_node_pair * +fy_node_mapping_iterate(struct fy_node *fyn, void **prevp) + FY_EXPORT; + +/** + * fy_node_mapping_reverse_iterate() - Iterate over a mapping node in reverse + * + * This method iterates in reverse over all the node pairs in the mapping node. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * Note that while a mapping is an unordered collection of key/values + * the order of which they are created is important for presentation + * purposes. + * + * @fyn: The mapping node + * @prevp: The previous sequence iterator + * + * Returns: + * The next node pair in reverse sequence in the mapping or NULL at the end of the mapping. + */ +struct fy_node_pair * +fy_node_mapping_reverse_iterate(struct fy_node *fyn, void **prevp) + FY_EXPORT; + +/** + * fy_node_mapping_item_count() - Return the node pair count of the mapping + * + * Get the count of the node pairs in the mapping. + * + * @fyn: The mapping node + * + * Returns: + * The count of node pairs in the mapping or -1 in case of an error. + */ +int +fy_node_mapping_item_count(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_mapping_is_empty() - Check whether the mapping is empty + * + * Check whether the mapping contains any node pairs. + * + * @fyn: The mapping node + * + * Returns: + * true if the node is a mapping containing node pairs, false otherwise + */ +bool +fy_node_mapping_is_empty(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_mapping_get_by_index() - Return a node pair by index + * + * Retrieve a node pair in the mapping using its index. If index + * is positive or zero the count is from the start of the sequence, + * while if negative from the end. I.e. -1 returns the last node pair + * in the mapping. + * + * @fyn: The mapping node + * @index: The index of the node pair to retrieve. + * - >= 0 counting from the start + * - < 0 counting from the end + * + * Returns: + * The node pair at the specified index or NULL if no such item exist. + */ +struct fy_node_pair * +fy_node_mapping_get_by_index(struct fy_node *fyn, int index) + FY_EXPORT; + +/** + * fy_node_mapping_lookup_pair_by_string() - Lookup a node pair in mapping by string + * + * This method will return the node pair that contains the same key + * from the YAML node created from the @key argument. The comparison of the + * node is using fy_node_compare() + * + * @fyn: The mapping node + * @key: The YAML source to use as key + * @len: The length of the key (or -1 if '\0' terminated) + * + * Returns: + * The value matching the given key, or NULL if not found. + */ +struct fy_node_pair * +fy_node_mapping_lookup_pair_by_string(struct fy_node *fyn, + const char *key, size_t len) + FY_EXPORT; + +/** + * fy_node_mapping_lookup_by_string() - Lookup a node value in mapping by string + * + * This method will return the value of node pair that contains the same key + * from the YAML node created from the @key argument. The comparison of the + * node is using fy_node_compare() + * + * @fyn: The mapping node + * @key: The YAML source to use as key + * @len: The length of the key (or -1 if '\0' terminated) + * + * Returns: + * The value matching the given key, or NULL if not found. + */ +struct fy_node * +fy_node_mapping_lookup_by_string(struct fy_node *fyn, + const char *key, size_t len) + FY_EXPORT; + +/** + * fy_node_mapping_lookup_value_by_string() - Lookup a node value in mapping by string + * + * This method will return the value of node pair that contains the same key + * from the YAML node created from the @key argument. The comparison of the + * node is using fy_node_compare() + * + * It is synonymous with fy_node_mapping_lookup_by_string(). + * + * @fyn: The mapping node + * @key: The YAML source to use as key + * @len: The length of the key (or -1 if '\0' terminated) + * + * Returns: + * The value matching the given key, or NULL if not found. + */ +struct fy_node * +fy_node_mapping_lookup_value_by_string(struct fy_node *fyn, + const char *key, size_t len) + FY_EXPORT; + +/** + * fy_node_mapping_lookup_key_by_string() - Lookup a node key in mapping by string + * + * This method will return the key of node pair that contains the same key + * from the YAML node created from the @key argument. The comparison of the + * node is using fy_node_compare() + * + * @fyn: The mapping node + * @key: The YAML source to use as key + * @len: The length of the key (or -1 if '\0' terminated) + * + * Returns: + * The key matching the given key, or NULL if not found. + */ +struct fy_node * +fy_node_mapping_lookup_key_by_string(struct fy_node *fyn, + const char *key, size_t len) + FY_EXPORT; + + +/** + * fy_node_mapping_lookup_pair_by_simple_key() - Lookup a node pair in mapping by simple string + * + * This method will return the node pair that contains the same key + * from the YAML node created from the @key argument. The comparison of the + * node is using by comparing the strings for identity. + * + * @fyn: The mapping node + * @key: The string to use as key + * @len: The length of the key (or -1 if '\0' terminated) + * + * Returns: + * The node pair matching the given key, or NULL if not found. + */ +struct fy_node_pair * +fy_node_mapping_lookup_pair_by_simple_key(struct fy_node *fyn, + const char *key, size_t len) + FY_EXPORT; +/** + * fy_node_mapping_lookup_value_by_simple_key() - Lookup a node value in mapping by simple string + * + * This method will return the value of node pair that contains the same key + * from the YAML node created from the @key argument. The comparison of the + * node is using by comparing the strings for identity. + * + * @fyn: The mapping node + * @key: The string to use as key + * @len: The length of the key (or -1 if '\0' terminated) + * + * Returns: + * The value matching the given key, or NULL if not found. + */ +struct fy_node * +fy_node_mapping_lookup_value_by_simple_key(struct fy_node *fyn, + const char *key, size_t len) + FY_EXPORT; + +/** + * fy_node_mapping_lookup_pair_by_null_key() - Lookup a node pair in mapping that has a NULL key + * + * This method will return the node pair that has a NULL key. + * Note this method is not using the mapping accelerator + * and arguably NULL keys should not exist. Alas... + * + * @fyn: The mapping node + * + * Returns: + * The node pair with a NULL key, NULL otherwise + */ +struct fy_node_pair * +fy_node_mapping_lookup_pair_by_null_key(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_mapping_lookup_value_by_null_key() - Lookup a node value with a NULL key. + * + * Return the value of a node pair that has a NULL key. + * + * @fyn: The mapping node + * + * Returns: + * The value matching the null key, NULL otherwise. + * Note that the value may be NULL too, but for that pathological case + * use the node pair method instead. + */ +struct fy_node * +fy_node_mapping_lookup_value_by_null_key(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_mapping_lookup_scalar_by_simple_key() - Lookup a scalar in mapping by simple string + * + * This method will return the scalar contents that contains the same key + * from the YAML node created from the @key argument. The comparison of the + * node is using by comparing the strings for identity. + * + * @fyn: The mapping node + * @lenp: Pointer to a variable that will hold the returned length + * @key: The string to use as key + * @keylen: The length of the key (or -1 if '\0' terminated) + * + * Returns: + * The scalar contents matching the given key, or NULL if not found. + */ +const char * +fy_node_mapping_lookup_scalar_by_simple_key(struct fy_node *fyn, size_t *lenp, + const char *key, size_t keylen) + FY_EXPORT; + +/** + * fy_node_mapping_lookup_scalar0_by_simple_key() - Lookup a scalar in mapping by simple string + * returning a '\0' terminated scalar + * + * This method will return the NUL terminated scalar contents that contains the same key + * from the YAML node created from the @key argument. The comparison of the + * node is using by comparing the strings for identity. + * + * @fyn: The mapping node + * @key: The string to use as key + * @keylen: The length of the key (or -1 if '\0' terminated) + * + * Returns: + * The NUL terminated scalar contents matching the given key, or NULL if not found. + */ +const char * +fy_node_mapping_lookup_scalar0_by_simple_key(struct fy_node *fyn, + const char *key, size_t keylen) + FY_EXPORT; + +/** + * fy_node_mapping_lookup_pair() - Lookup a node pair matching the provided key + * + * This method will return the node pair that matches the provided @fyn_key + * + * @fyn: The mapping node + * @fyn_key: The node to use as key + * + * Returns: + * The node pair matching the given key, or NULL if not found. + */ +struct fy_node_pair * +fy_node_mapping_lookup_pair(struct fy_node *fyn, struct fy_node *fyn_key) + FY_EXPORT; + + +/** + * fy_node_mapping_lookup_value_by_key() - Lookup a node pair's value matching the provided key + * + * This method will return the node pair that matches the provided @fyn_key + * The key may be collection and a content match check is performed recursively + * in order to find the right key. + * + * @fyn: The mapping node + * @fyn_key: The node to use as key + * + * Returns: + * The node value matching the given key, or NULL if not found. + */ +struct fy_node * +fy_node_mapping_lookup_value_by_key(struct fy_node *fyn, struct fy_node *fyn_key); + +/** + * fy_node_mapping_lookup_key_by_key() - Lookup a node pair's key matching the provided key + * + * This method will return the node pair that matches the provided @fyn_key + * The key may be collection and a content match check is performed recursively + * in order to find the right key. + * + * @fyn: The mapping node + * @fyn_key: The node to use as key + * + * Returns: + * The node key matching the given key, or NULL if not found. + */ +struct fy_node * +fy_node_mapping_lookup_key_by_key(struct fy_node *fyn, struct fy_node *fyn_key); + +/** + * fy_node_mapping_get_pair_index() - Return the node pair index in the mapping + * + * This method will return the node pair index in the mapping of the given + * node pair argument. + * + * @fyn: The mapping node + * @fynp: The node pair + * + * Returns: + * The index of the node pair in the mapping or -1 in case of an error. + */ +int +fy_node_mapping_get_pair_index(struct fy_node *fyn, + const struct fy_node_pair *fynp) + FY_EXPORT; + +/** + * fy_node_pair_key() - Return the key of a node pair + * + * This method will return the node pair's key. + * Note that this may be NULL, which is returned also in case + * the node pair argument is NULL, so you should protect against + * such a case. + * + * @fynp: The node pair + * + * Returns: + * The node pair key + */ +struct fy_node * +fy_node_pair_key(struct fy_node_pair *fynp) + FY_EXPORT; + +/** + * fy_node_pair_value() - Return the value of a node pair + * + * This method will return the node pair's value. + * Note that this may be NULL, which is returned also in case + * the node pair argument is NULL, so you should protect against + * such a case. + * + * @fynp: The node pair + * + * Returns: + * The node pair value + */ +struct fy_node * +fy_node_pair_value(struct fy_node_pair *fynp) + FY_EXPORT; + +/** + * fy_node_pair_set_key() - Sets the key of a node pair + * + * This method will set the key part of the node pair. + * It will ovewrite any previous key. + * + * Note that no checks for duplicate keys are going to be + * performed. + * + * @fynp: The node pair + * @fyn: The key node + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_pair_set_key(struct fy_node_pair *fynp, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_pair_set_value() - Sets the value of a node pair + * + * This method will set the value part of the node pair. + * It will ovewrite any previous value. + * + * @fynp: The node pair + * @fyn: The value node + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_pair_set_value(struct fy_node_pair *fynp, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_mapping_append() - Append a node item to a mapping + * + * Append a node pair to a mapping. + * + * @fyn_map: The mapping node + * @fyn_key: The node pair's key + * @fyn_value: The node pair's value + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_mapping_append(struct fy_node *fyn_map, + struct fy_node *fyn_key, struct fy_node *fyn_value) + FY_EXPORT; + +/** + * fy_node_mapping_prepend() - Prepend a node item to a mapping + * + * Prepend a node pair to a mapping. + * + * @fyn_map: The mapping node + * @fyn_key: The node pair's key + * @fyn_value: The node pair's value + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_mapping_prepend(struct fy_node *fyn_map, + struct fy_node *fyn_key, struct fy_node *fyn_value) + FY_EXPORT; + +/** + * fy_node_mapping_remove() - Remove a node pair from a mapping + * + * Remove node pair from a mapping. + * + * @fyn_map: The mapping node + * @fynp: The node pair to remove + * + * Returns: + * 0 on success, -1 on failure. + */ +int +fy_node_mapping_remove(struct fy_node *fyn_map, struct fy_node_pair *fynp) + FY_EXPORT; + +/** + * fy_node_mapping_remove_by_key() - Remove a node pair from a mapping returning the value + * + * Remove node pair from a mapping using the supplied key. + * + * @fyn_map: The mapping node + * @fyn_key: The node pair's key + * + * Returns: + * The value part of removed node pair, or NULL in case of an error. + */ +struct fy_node * +fy_node_mapping_remove_by_key(struct fy_node *fyn_map, struct fy_node *fyn_key) + FY_EXPORT; + +/** + * fy_node_sort() - Recursively sort node + * + * Recursively sort all mappings of the given node, using the given + * comparison method (if NULL use the default one). + * + * @fyn: The node to sort + * @key_cmp: The comparison method + * @arg: An opaque user pointer for the comparison method + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_sort(struct fy_node *fyn, fy_node_mapping_sort_fn key_cmp, void *arg) + FY_EXPORT; + +/** + * fy_node_vscanf() - Retrieve data via vscanf + * + * This method easily retrieves data using a familiar vscanf interface. + * The format string is a regular scanf format string with the following format. + * + * "pathspec %opt pathspec %opt..." + * + * Each pathspec is separated with space from the scanf option + * + * For example: + * fyn = { foo: 3 } -> fy_node_scanf(fyn, "/foo %d", &var) -> var = 3 + * + * + * @fyn: The node to use as a pathspec root + * @fmt: The scanf based format string + * @ap: The va_list containing the arguments + * + * Returns: + * The number of scanned arguments, or -1 on error. + */ +int fy_node_vscanf(struct fy_node *fyn, const char *fmt, va_list ap); + +/** + * fy_node_scanf() - Retrieve data via scanf + * + * This method easily retrieves data using a familiar vscanf interface. + * The format string is a regular scanf format string with the following format. + * + * "pathspec %opt pathspec %opt..." + * + * Each pathspec is separated with space from the scanf option + * + * For example: + * fyn = { foo: 3 } -> fy_node_scanf(fyn, "/foo %d", &var) -> var = 3 + * + * + * @fyn: The node to use as a pathspec root + * @fmt: The scanf based format string + * @...: The arguments + * + * Returns: + * The number of scanned arguments, or -1 on error. + */ +int +fy_node_scanf(struct fy_node *fyn, const char *fmt, ...) + FY_FORMAT(scanf, 2, 3) + FY_EXPORT; + +/** + * fy_document_vscanf() - Retrieve data via vscanf relative to document root + * + * This method easily retrieves data using a familiar vscanf interface. + * The format string is a regular scanf format string with the following format. + * + * "pathspec %opt pathspec %opt..." + * + * Each pathspec is separated with space from the scanf option + * + * For example: + * fyd = { foo: 3 } -> fy_document_scanf(fyd, "/foo %d", &var) -> var = 3 + * + * + * @fyd: The document + * @fmt: The scanf based format string + * @ap: The va_list containing the arguments + * + * Returns: + * The number of scanned arguments, or -1 on error. + */ +int +fy_document_vscanf(struct fy_document *fyd, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_document_scanf() - Retrieve data via scanf relative to document root + * + * This method easily retrieves data using a familiar vscanf interface. + * The format string is a regular scanf format string with the following format. + * + * "pathspec %opt pathspec %opt..." + * + * Each pathspec is separated with space from the scanf option + * + * For example: + * fyn = { foo: 3 } -> fy_node_scanf(fyd, "/foo %d", &var) -> var = 3 + * + * + * @fyd: The document + * @fmt: The scanf based format string + * @...: The arguments + * + * Returns: + * The number of scanned arguments, or -1 on error. + */ +int +fy_document_scanf(struct fy_document *fyd, const char *fmt, ...) + FY_FORMAT(scanf, 2, 3) + FY_EXPORT; + +/** + * fy_document_tag_directive_iterate() - Iterate over a document's tag directives + * + * This method iterates over all the documents tag directives. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * @fyd: The document + * @prevp: The previous state of the iterator + * + * Returns: + * The next tag directive token in the document or NULL at the end of them. + */ +struct fy_token * +fy_document_tag_directive_iterate(struct fy_document *fyd, void **prevp) + FY_EXPORT; + +/** + * fy_document_tag_directive_lookup() - Retreive a document's tag directive + * + * Retreives the matching tag directive token of the document matching the handle. + * + * @fyd: The document + * @handle: The handle to look for + * + * Returns: + * The tag directive token with the given handle or NULL if not found + */ +struct fy_token * +fy_document_tag_directive_lookup(struct fy_document *fyd, const char *handle) + FY_EXPORT; + +/** + * fy_tag_directive_token_handle() - Get a tag directive handle + * + * Retreives the tag directives token handle value. Only valid on + * tag directive tokens. + * + * @fyt: The tag directive token + * @lenp: Pointer to a variable that will hold the returned length + * + * Returns: + * A pointer to the tag directive's handle, while @lenp will be assigned the + * length of said handle. + * A NULL will be returned in case of an error. + */ +const char * +fy_tag_directive_token_handle(struct fy_token *fyt, size_t *lenp) + FY_EXPORT; + +/** + * fy_tag_directive_token_prefix() - Get a tag directive prefix + * + * Retreives the tag directives token prefix value. Only valid on + * tag directive tokens. + * + * @fyt: The tag directive token + * @lenp: Pointer to a variable that will hold the returned length + * + * Returns: + * A pointer to the tag directive's prefix, while @lenp will be assigned the + * length of said prefix. + * A NULL will be returned in case of an error. + */ +const char * +fy_tag_directive_token_prefix(struct fy_token *fyt, size_t *lenp) + FY_EXPORT; + +/** + * fy_document_tag_directive_add() - Add a tag directive to a document + * + * Add tag directive to the document. + * + * @fyd: The document + * @handle: The handle of the tag directive + * @prefix: The prefix of the tag directive + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_document_tag_directive_add(struct fy_document *fyd, + const char *handle, const char *prefix) + FY_EXPORT; + +/** + * fy_document_tag_directive_remove() - Remove a tag directive + * + * Remove a tag directive from a document. + * Note that removal is prohibited if any node is still using this tag directive. + * + * @fyd: The document + * @handle: The handle of the tag directive to remove. + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_document_tag_directive_remove(struct fy_document *fyd, const char *handle) + FY_EXPORT; + +/** + * fy_document_lookup_anchor() - Lookup an anchor + * + * Lookup for an anchor having the name provided + * + * @fyd: The document + * @anchor: The anchor to look for + * @len: The length of the anchor (or -1 if '\0' terminated) + * + * Returns: + * The anchor if found, NULL otherwise + */ +struct fy_anchor * +fy_document_lookup_anchor(struct fy_document *fyd, + const char *anchor, size_t len) + FY_EXPORT; + +/** + * fy_document_lookup_anchor_by_token() - Lookup an anchor by token text + * + * Lookup for an anchor having the name provided from the text of the token + * + * @fyd: The document + * @anchor: The token contains the anchor text to look for + * + * Returns: + * The anchor if found, NULL otherwise + */ +struct fy_anchor * +fy_document_lookup_anchor_by_token(struct fy_document *fyd, + struct fy_token *anchor) + FY_EXPORT; + +/** + * fy_document_lookup_anchor_by_node() - Lookup an anchor by node + * + * Lookup for an anchor located in the given node + * + * @fyd: The document + * @fyn: The node + * + * Returns: + * The anchor if found, NULL otherwise + */ +struct fy_anchor * +fy_document_lookup_anchor_by_node(struct fy_document *fyd, + struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_anchor_get_text() - Get the text of an anchor + * + * This method will return a pointer to the text of an anchor + * along with the length of it. Note that this text is *not* + * NULL terminated. + * + * @fya: The anchor + * @lenp: Pointer to a variable that will hold the returned length + * + * Returns: + * A pointer to the text of the anchor, while @lenp will be assigned the + * length of said anchor. + * A NULL will be returned in case of an error. + */ +const char * +fy_anchor_get_text(struct fy_anchor *fya, size_t *lenp) + FY_EXPORT; + +/** + * fy_anchor_node() - Get the node of an anchor + * + * This method returns the node associated with the anchor. + * + * @fya: The anchor + * + * Returns: + * The node of the anchor, or NULL in case of an error. + */ +struct fy_node * +fy_anchor_node(struct fy_anchor *fya) + FY_EXPORT; + +/** + * fy_document_anchor_iterate() - Iterate over a document's anchors + * + * This method iterates over all the documents anchors. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * @fyd: The document + * @prevp: The previous state of the iterator + * + * Returns: + * The next anchor in the document or NULL at the end of them. + */ +struct fy_anchor * +fy_document_anchor_iterate(struct fy_document *fyd, void **prevp) + FY_EXPORT; + +/** + * fy_document_set_anchor() - Place an anchor + * + * Places an anchor to the node with the give text name. + * + * Note that the data are not copied, merely a reference is taken, so + * it must be available while the node is in use. + * + * Also not that this method is deprecated; use fy_node_set_anchor() + * instead. + * + * @fyd: The document + * @fyn: The node to set the anchor to + * @text: Pointer to the anchor text + * @len: Size of the anchor text, or (size_t)-1 for '\0' terminated. + * + * Returns: + * 0 on success, -1 on error. + */ +int +fy_document_set_anchor(struct fy_document *fyd, struct fy_node *fyn, + const char *text, size_t len) + FY_EXPORT FY_DEPRECATED; + +/** + * fy_node_set_anchor() - Place an anchor to the node + * + * Places an anchor to the node with the give text name. + * + * Note that the data are not copied, merely a reference is taken, so + * it must be available while the node is in use. + * + * This is similar to fy_document_set_anchor() with the document set + * to the document of the @fyn node. + * + * @fyn: The node to set the anchor to + * @text: Pointer to the anchor text + * @len: Size of the anchor text, or (size_t)-1 for '\0' terminated. + * + * Returns: + * 0 on success, -1 on error. + */ +int +fy_node_set_anchor(struct fy_node *fyn, const char *text, size_t len) + FY_EXPORT; + +/** + * fy_node_set_anchor_copy() - Place an anchor to the node copying the text + * + * Places an anchor to the node with the give text name, which + * will be copied, so it's safe to dispose the text after the call. + * + * @fyn: The node to set the anchor to + * @text: Pointer to the anchor text + * @len: Size of the anchor text, or (size_t)-1 for '\0' terminated. + * + * Returns: + * 0 on success, -1 on error. + */ +int +fy_node_set_anchor_copy(struct fy_node *fyn, const char *text, size_t len) + FY_EXPORT; + +/** + * fy_node_set_vanchorf() - Place an anchor to the node using a vprintf interface. + * + * Places an anchor to the node with the give text name as created + * via vprintf'ing the arguments. + * + * @fyn: The node to set the anchor to + * @fmt: Pointer to the anchor format string + * @ap: The argument list. + * + * Returns: + * 0 on success, -1 on error. + */ +int +fy_node_set_vanchorf(struct fy_node *fyn, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_node_set_anchorf() - Place an anchor to the node using a printf interface. + * + * Places an anchor to the node with the give text name as created + * via printf'ing the arguments. + * + * @fyn: The node to set the anchor to + * @fmt: Pointer to the anchor format string + * @...: The extra arguments. + * + * Returns: + * 0 on success, -1 on error. + */ +int +fy_node_set_anchorf(struct fy_node *fyn, const char *fmt, ...) + FY_FORMAT(printf, 2, 3) + FY_EXPORT; + +/** + * fy_node_remove_anchor() - Remove an anchor + * + * Remove an anchor for the given node (if it exists) + * + * @fyn: The node to remove anchors from + * + * Returns: + * 0 on success, -1 on error. + */ +int +fy_node_remove_anchor(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_anchor() - Get the anchor of a node + * + * Retrieve the anchor of the given node (if it exists) + * + * @fyn: The node + * + * Returns: + * The anchor if there's one at the node, or NULL otherwise + */ +struct fy_anchor * +fy_node_get_anchor(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_nearest_anchor() - Get the nearest anchor of the node + * + * Retrieve the anchor of the nearest parent of the given node or + * the given node if it has one. + * + * @fyn: The node + * + * Returns: + * The nearest anchor if there's one, or NULL otherwise + */ +struct fy_anchor * +fy_node_get_nearest_anchor(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_nearest_child_of() - Get the nearest node which is a + * child of the base + * + * Retrieve the nearest node which is a child of fyn_base starting + * at fyn and working upwards. + * + * @fyn_base: The base node + * @fyn: The node to start from + * + * Returns: + * The nearest child of the base if there's one, or NULL otherwise + */ +struct fy_node * +fy_node_get_nearest_child_of(struct fy_node *fyn_base, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_create_alias() - Create an alias node + * + * Create an alias on the given document + * + * Note that the data are not copied, merely a reference is taken, so + * it must be available while the node is in use. + * + * @fyd: The document + * @alias: The alias text + * @len: The length of the alias (or -1 if '\0' terminated) + * + * Returns: + * The created alias node, or NULL in case of an error + */ +struct fy_node * +fy_node_create_alias(struct fy_document *fyd, + const char *alias, size_t len) + FY_EXPORT; + +/** + * fy_node_create_alias_copy() - Create an alias node copying the data + * + * Create an alias on the given document + * + * A copy of the data will be made, so it is safe to free the data + * after the call. + * + * @fyd: The document + * @alias: The alias text + * @len: The length of the alias (or -1 if '\0' terminated) + * + * Returns: + * The created alias node, or NULL in case of an error + */ +struct fy_node * +fy_node_create_alias_copy(struct fy_document *fyd, + const char *alias, size_t len) + FY_EXPORT; + +/** + * fy_node_get_meta() - Get the meta pointer of a node + * + * Return the meta pointer of a node. + * + * @fyn: The node to get meta data from + * + * Returns: + * The stored meta data pointer + */ +void * +fy_node_get_meta(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_set_meta() - Set the meta pointer of a node + * + * Set the meta pointer of a node. If @meta is NULL + * then clear the meta data. + * + * @fyn: The node to set meta data + * @meta: The meta data pointer + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_set_meta(struct fy_node *fyn, void *meta) + FY_EXPORT; + +/** + * fy_node_clear_meta() - Clear the meta data of a node + * + * Clears the meta data of a node. + * + * @fyn: The node to clear meta data + */ +void +fy_node_clear_meta(struct fy_node *fyn) + FY_EXPORT; + +/** + * typedef fy_node_meta_clear_fn - Meta data clear method + * + * This is the callback called when meta data are cleared. + * + * @fyn: The node which the meta data is being cleared + * @meta: The meta data of the node assigned via fy_node_set_meta() + * @user: The user pointer of fy_document_register_meta() + * + */ +typedef void (*fy_node_meta_clear_fn)(struct fy_node *fyn, void *meta, void *user); + +/** + * fy_document_register_meta() - Register a meta cleanup hook + * + * Register a meta data cleanup hook, to be called when + * the node is freed via a final call to fy_node_free(). + * The hook is active for all nodes belonging to the document. + * + * @fyd: The document which the hook is registered to + * @clear_fn: The clear hook method + * @user: Opaque user provided pointer to the clear method + * + * Returns: + * 0 on success, -1 if another hook is already registered. + */ +int +fy_document_register_meta(struct fy_document *fyd, + fy_node_meta_clear_fn clear_fn, void *user) + FY_EXPORT; + +/** + * fy_document_unregister_meta() - Unregister a meta cleanup hook + * + * Unregister the currently active meta cleanup hook. + * The previous cleanup hook will be called for every node in + * the document. + * + * @fyd: The document to unregister it's meta cleanup hook. + */ +void +fy_document_unregister_meta(struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_node_set_marker() - Set a marker of a node + * + * Sets the marker of the given node, while returning + * the previous state of the marker. Note that the + * markers use the same space as the node follow markers. + * + * @fyn: The node + * @marker: The marker to set + * + * Returns: + * The previous value of the marker + */ +bool +fy_node_set_marker(struct fy_node *fyn, unsigned int marker) + FY_EXPORT; + +/** + * fy_node_clear_marker() - Clear a marker of a node + * + * Clears the marker of the given node, while returning + * the previous state of the marker. Note that the + * markers use the same space as the node follow markers. + * + * @fyn: The node + * @marker: The marker to clear + * + * Returns: + * The previous value of the marker + */ +bool +fy_node_clear_marker(struct fy_node *fyn, unsigned int marker) + FY_EXPORT; + +/** + * fy_node_is_marker_set() - Checks whether a marker is set + * + * Check the state of the given marker. + * + * @fyn: The node + * @marker: The marker index (must be less that FYNWF_MAX_USER_MARKER) + * + * Returns: + * The value of the marker (invalid markers return false) + */ +bool +fy_node_is_marker_set(struct fy_node *fyn, unsigned int marker) + FY_EXPORT; + +/** + * fy_node_vreport() - Report about a node vprintf style + * + * Output a report about the given node via the specific + * error type, and using the reporting configuration of the node's + * document. + * + * @fyn: The node + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_node_vreport(struct fy_node *fyn, enum fy_error_type type, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_node_report() - Report about a node printf style + * + * Output a report about the given node via the specific + * error type, and using the reporting configuration of the node's + * document. + * + * @fyn: The node + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_node_report(struct fy_node *fyn, enum fy_error_type type, + const char *fmt, ...) + FY_FORMAT(printf, 3, 4) + FY_EXPORT; + +/** + * fy_node_override_vreport() - Report about a node vprintf style, + * overriding file, line and column info + * + * Output a report about the given node via the specific + * error type, and using the reporting configuration of the node's + * document. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @fyn: The node + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_node_override_vreport(struct fy_node *fyn, enum fy_error_type type, + const char *file, int line, int column, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_node_override_report() - Report about a node printf style, + * overriding file, line and column info + * + * Output a report about the given node via the specific + * error type, and using the reporting configuration of the node's + * document. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @fyn: The node + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_node_override_report(struct fy_node *fyn, enum fy_error_type type, + const char *file, int line, int column, + const char *fmt, ...) + FY_FORMAT(printf, 6, 7) + FY_EXPORT; + +/** + * fy_event_vreport() - Report about an event vprintf style + * + * Output a report about the given event via the specific + * error type, focusing at the given event part. + * + * @fyp: The parser of which the event was generated from + * @fye: The event + * @fyep: The event part which the report is about + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_event_vreport(struct fy_parser *fyp, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_event_report() - Report about an event printf style + * + * Output a report about the given event via the specific + * error type, focusing at the given event part. + * + * @fyp: The parser of which the event was generated from + * @fye: The event + * @fyep: The event part which the report is about + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_event_report(struct fy_parser *fyp, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, ...) + FY_FORMAT(printf, 5, 6) + FY_EXPORT; + +typedef void (*fy_diag_output_fn)(struct fy_diag *diag, void *user, + const char *buf, size_t len); + +/** + * struct fy_diag_cfg - The diagnostics configuration + * + * @fp: File descriptor of the error output + * @output_fn: Callback to use when fp is NULL + * @user: User pointer to pass to the output_fn + * @level: The minimum debugging level + * @module_mask: A bitmask of the enabled modules + * @colorize: true if output should be colorized using ANSI sequences + * @show_source: true if source location should be displayed + * @show_position: true if position should be displayed + * @show_type: true if the type should be displayed + * @show_module: true if the module should be displayed + * @source_width: Width of the source field + * @position_width: Width of the position field + * @type_width: Width of the type field + * @module_width: Width of the module field + * + * This structure contains the configuration of the + * diagnostic object. + */ +struct fy_diag_cfg { + FILE *fp; + fy_diag_output_fn output_fn; + void *user; + enum fy_error_type level; + unsigned int module_mask; + bool colorize : 1; + bool show_source : 1; + bool show_position : 1; + bool show_type : 1; + bool show_module : 1; + int source_width; + int position_width; + int type_width; + int module_width; +}; + +/** + * struct fy_diag_error - A collected diagnostic error + * + * @type: Error type + * @module: The module + * @fyt: The token (may be NULL) + * @msg: The message to be output alongside + * @file: The file which contained the input + * @line: The line at the error + * @column: The column at the error + * + * This structure contains information about an error + * being collected by the diagnostic object. + */ +struct fy_diag_error { + enum fy_error_type type; + enum fy_error_module module; + struct fy_token *fyt; + const char *msg; + const char *file; + int line; + int column; +}; + +/** + * fy_diag_create() - Create a diagnostic object + * + * Creates a diagnostic object using the provided configuration. + * + * @cfg: The configuration for the diagnostic object (NULL is default) + * + * Returns: + * A pointer to the diagnostic object or NULL in case of an error. + */ +struct fy_diag * +fy_diag_create(const struct fy_diag_cfg *cfg) + FY_EXPORT; + +/** + * fy_diag_destroy() - Destroy a diagnostic object + * + * Destroy a diagnostic object; note that the actual + * destruction only occurs when no more references to the + * object are present. However no output will be generated + * after this call. + * + * @diag: The diagnostic object to destroy + */ +void +fy_diag_destroy(struct fy_diag *diag) + FY_EXPORT; + +/** + * fy_diag_get_cfg() - Get a diagnostic object's configuration + * + * Return the current configuration of a diagnostic object + * + * @diag: The diagnostic object + * + * Returns: + * A const pointer to the diagnostic object configuration, or NULL + * in case where diag is NULL + */ +const struct fy_diag_cfg * +fy_diag_get_cfg(struct fy_diag *diag) + FY_EXPORT; + +/** + * fy_diag_set_cfg() - Set a diagnostic object's configuration + * + * Replace the current diagnostic configuration with the given + * configuration passed as an argument. + * + * @diag: The diagnostic object + * @cfg: The diagnostic configuration + */ +void +fy_diag_set_cfg(struct fy_diag *diag, const struct fy_diag_cfg *cfg) + FY_EXPORT; + +/** + * fy_diag_set_level() - Set a diagnostic object's debug error level + * + * @diag: The diagnostic object + * @level: The debugging level to set + */ +void +fy_diag_set_level(struct fy_diag *diag, enum fy_error_type level); + +/** + * fy_diag_set_colorize() - Set a diagnostic object's colorize option + * + * @diag: The diagnostic object + * @colorize: The colorize option + */ +void +fy_diag_set_colorize(struct fy_diag *diag, bool colorize); + +/** + * fy_diag_ref() - Increment that reference counter of a diagnostic object + * + * Increment the reference. + * + * @diag: The diagnostic object to reference + * + * Returns: + * Always returns the @diag argument + */ +struct fy_diag * +fy_diag_ref(struct fy_diag *diag) + FY_EXPORT; + +/** + * fy_diag_unref() - Take away a ref from a diagnostic object + * + * Take away a reference, if it gets to 0, the diagnostic object + * is freed. + * + * @diag: The diagnostic object to unref + */ +void +fy_diag_unref(struct fy_diag *diag) + FY_EXPORT; + +/** + * fy_diag_got_error() - Test whether an error level diagnostic + * has been produced + * + * Tests whether an error diagnostic has been produced. + * + * @diag: The diagnostic object + * + * Returns: + * true if an error has been produced, false otherwise + */ +bool +fy_diag_got_error(struct fy_diag *diag) + FY_EXPORT; + +/** + * fy_diag_set_error() - Sets the error produced state + * + * Sets the error produced state + * + * @diag: The diagnostic object + * @on_error: The set error state + */ +void +fy_diag_set_error(struct fy_diag *diag, bool on_error) + FY_EXPORT; + +/** + * fy_diag_reset_error() - Reset the error flag of + * the diagnostic object + * + * Clears the error flag which was set by an output + * of an error level diagnostic + * + * @diag: The diagnostic object + */ +void +fy_diag_reset_error(struct fy_diag *diag) + FY_EXPORT; + +/** + * fy_diag_set_collect_errors() - Collect errors instead of outputting + * + * Set the collect errors mode. When true instead of outputting to + * the diagnostic interface, errors are collected for later + * retrieval. + * + * @diag: The diagnostic object + * @collect_errors: The collect errors mode + */ +void +fy_diag_set_collect_errors(struct fy_diag *diag, bool collect_errors) + FY_EXPORT; + +/** + * fy_diag_cfg_default() - Fill in the configuration structure + * with defaults + * + * Fills the configuration structure with defaults. The default + * always associates the file descriptor to stderr. + * + * @cfg: The diagnostic configuration structure + */ +void +fy_diag_cfg_default(struct fy_diag_cfg *cfg) + FY_EXPORT; + +/** + * fy_diag_cfg_from_parser_flags() - Fill partial diagnostic config + * structure from parser config flags + * + * Fills in part of the configuration structure using parser flags. + * Since the parser flags do not contain debugging flags anymore this + * method is deprecated. + * + * @cfg: The diagnostic configuration structure + * @pflags: The parser flags + */ +void +fy_diag_cfg_from_parser_flags(struct fy_diag_cfg *cfg, + enum fy_parse_cfg_flags pflags) + FY_EXPORT FY_DEPRECATED; + +/** + * fy_diag_vprintf() - vprintf raw interface to diagnostics + * + * Raw output to the diagnostic object using a standard + * vprintf interface. Note that this is the lowest level + * interface, and as such is not recommended for use, since + * no formatting or coloring will take place. + * + * @diag: The diagnostic object to use + * @fmt: The vprintf format string + * @ap: The arguments + * + * Returns: + * The number of characters output, or -1 in case of an error + * Note that 0 shall be returned if the diagnostic object has + * been destroyed but not yet freed. + */ +int +fy_diag_vprintf(struct fy_diag *diag, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_diag_printf() - printf raw interface to diagnostics + * + * Raw output to the diagnostic object using a standard + * printf interface. Note that this is the lowest level + * interface, and as such is not recommended for use, since + * no formatting or coloring will take place. + * + * @diag: The diagnostic object to use + * @fmt: The printf format string + * @...: The arguments + * + * Returns: + * The number of characters output, or -1 in case of an error + * Note that 0 shall be returned if the diagnostic object has + * been destroyed but not yet freed. + */ +int +fy_diag_printf(struct fy_diag *diag, const char *fmt, ...) + FY_FORMAT(printf, 2, 3) + FY_EXPORT; + +/** + * struct fy_diag_ctx - The diagnostics context + * + * @level: The level of the diagnostic + * @module: The module of the diagnostic + * @source_func: The source function + * @source_file: The source file + * @source_line: The source line + * @file: The file that caused the error + * @line: The line where the diagnostic occured + * @column: The column where the diagnostic occured + * + * This structure contains the diagnostic context + */ +struct fy_diag_ctx { + enum fy_error_type level; + enum fy_error_module module; + const char *source_func; + const char *source_file; + int source_line; + const char *file; + int line; + int column; +}; + +/** + * fy_vdiag() - context aware diagnostic output like vprintf + * + * Context aware output to the diagnostic object using a standard + * vprintf interface. + * + * @diag: The diagnostic object to use + * @fydc: The diagnostic context + * @fmt: The vprintf format string + * @ap: The arguments + * + * Returns: + * The number of characters output, or -1 in case of an error + * Note that 0 shall be returned if the diagnostic object has + * been destroyed but not yet freed. + */ +int +fy_vdiag(struct fy_diag *diag, const struct fy_diag_ctx *fydc, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_diagf() - context aware diagnostic output like printf + * + * Context aware output to the diagnostic object using a standard + * printf interface. + * + * @diag: The diagnostic object to use + * @fydc: The diagnostic context + * @fmt: The vprintf format string + * + * Returns: + * The number of characters output, or -1 in case of an error + * Note that 0 shall be returned if the diagnostic object has + * been destroyed but not yet freed. + */ +int +fy_diagf(struct fy_diag *diag, const struct fy_diag_ctx *fydc, + const char *fmt, ...) + FY_FORMAT(printf, 3, 4) + FY_EXPORT; + +#define fy_diag_diag(_diag, _level, _fmt, ...) \ + ({ \ + struct fy_diag_ctx _ctx = { \ + .level = (_level), \ + .module = FYEM_UNKNOWN, \ + .source_func = __func__, \ + .source_file = __FILE__, \ + .source_line = __LINE__, \ + .file = NULL, \ + .line = 0, \ + .column = 0, \ + }; \ + fy_diagf((_diag), &_ctx, (_fmt) , ## __VA_ARGS__); \ + }) + +#ifndef NDEBUG + +#define fy_debug(_diag, _fmt, ...) \ + fy_diag_diag((_diag), FYET_DEBUG, (_fmt) , ## __VA_ARGS__) +#else + +#define fy_debug(_diag, _fmt, ...) \ + do { } while(0) + +#endif + +#define fy_info(_diag, _fmt, ...) \ + fy_diag_diag((_diag), FYET_INFO, (_fmt) , ## __VA_ARGS__) +#define fy_notice(_diag, _fmt, ...) \ + fy_diag_diag((_diag), FYET_NOTICE, (_fmt) , ## __VA_ARGS__) +#define fy_warning(_diag, _fmt, ...) \ + fy_diag_diag((_diag), FYET_WARNING, (_fmt) , ## __VA_ARGS__) +#define fy_error(_diag, _fmt, ...) \ + fy_diag_diag((_diag), FYET_ERROR, (_fmt) , ## __VA_ARGS__) + +/** + * fy_diag_token_vreport() - Report about a token vprintf style using + * the given diagnostic object + * + * Output a report about the given token via the specific + * error type, and using the reporting configuration of the token's + * document. + * + * @diag: The diag object + * @fyt: The token + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_diag_token_vreport(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_diag_token_report() - Report about a token printf style using + * the given diagnostic object + * + * Output a report about the given token via the specific + * error type, and using the reporting configuration of the token's + * document. + * + * @diag: The diag object + * @fye: The token + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_diag_token_report(struct fy_diag *diag, struct fy_token *fye, + enum fy_error_type type, const char *fmt, ...) + FY_FORMAT(printf, 4, 5) + FY_EXPORT; + +/** + * fy_diag_token_override_vreport() - Report about a token vprintf style, + * overriding file, line and column info using + * the given diagnostic object + * + * Output a report about the given token via the specific + * error type, and using the reporting configuration of the token's + * document. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @diag: The diag object + * @fyt: The token + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_diag_token_override_vreport(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *file, + int line, int column, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_diag_token_override_report() - Report about a token printf style, + * overriding file, line and column info using + * the given diagnostic object + * + * Output a report about the given token via the specific + * error type, and using the reporting configuration of the token's + * document. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @diag: The diag object + * @fyt: The token + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_diag_token_override_report(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *file, + int line, int column, const char *fmt, ...) + FY_FORMAT(printf, 7, 8) + FY_EXPORT; + +/** + * fy_diag_node_vreport() - Report about a node vprintf style using + * the given diagnostic object + * + * Output a report about the given node via the specific + * error type, and using the reporting configuration of the node's + * document. + * + * @diag: The diag object + * @fyn: The node + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_diag_node_vreport(struct fy_diag *diag, struct fy_node *fyn, + enum fy_error_type type, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_diag_node_report() - Report about a node printf style using + * the given diagnostic object + * + * Output a report about the given node via the specific + * error type, and using the reporting configuration of the node's + * document. + * + * @diag: The diag object + * @fyn: The node + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_diag_node_report(struct fy_diag *diag, struct fy_node *fyn, + enum fy_error_type type, const char *fmt, ...) + FY_FORMAT(printf, 4, 5) + FY_EXPORT; + +/** + * fy_diag_node_override_vreport() - Report about a node vprintf style, + * overriding file, line and column info using + * the given diagnostic object + * + * Output a report about the given node via the specific + * error type, and using the reporting configuration of the node's + * document. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @diag: The diag object + * @fyn: The node + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_diag_node_override_vreport(struct fy_diag *diag, struct fy_node *fyn, + enum fy_error_type type, const char *file, + int line, int column, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_diag_node_override_report() - Report about a node printf style, + * overriding file, line and column info using + * the given diagnostic object + * + * Output a report about the given node via the specific + * error type, and using the reporting configuration of the node's + * document. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @diag: The diag object + * @fyn: The node + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_diag_node_override_report(struct fy_diag *diag, struct fy_node *fyn, + enum fy_error_type type, const char *file, + int line, int column, const char *fmt, ...) + FY_FORMAT(printf, 7, 8) + FY_EXPORT; + +/** + * fy_diag_event_vreport() - Report about an event vprintf style using + * the given diagnostic object + * + * Output a report about the given event part via the specific + * error type. + * + * @diag: The diag object + * @fye: The event + * @fyep: The event part + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_diag_event_vreport(struct fy_diag *diag, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_diag_event_report() - Report about a event printf style using + * the given diagnostic object + * + * Output a report about the given event part via the specific + * error type. + * + * @diag: The diag object + * @fye: The event + * @fyep: The event part + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_diag_event_report(struct fy_diag *diag, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, ...) + FY_FORMAT(printf, 5, 6) + FY_EXPORT; + +/** + * fy_diag_event_override_vreport() - Report about a token vprintf style, + * overriding file, line and column info using + * the given diagnostic object + * + * Output a report about the given event part via the specific + * error type. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @diag: The diag object + * @fye: The event + * @fyep: The event part + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_diag_event_override_vreport(struct fy_diag *diag, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *file, int line, int column, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_diag_event_override_report() - Report about a token printf style, + * overriding file, line and column info using + * the given diagnostic object + * + * Output a report about the given event part via the specific + * error type. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @diag: The diag object + * @fyt: The event + * @fyep: The event part + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_diag_event_override_report(struct fy_diag *diag, struct fy_token *fyt, + enum fy_event_part fyep, enum fy_error_type type, + const char *file, int line, int column, + const char *fmt, ...) + FY_FORMAT(printf, 8, 9) + FY_EXPORT; + +/** + * fy_diag_errors_iterate() - Iterate over the errors of a diagnostic object + * + * This method iterates over all the errors collected on the diagnostic object. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * @diag: The diagnostic object + * @prevp: The previous result iterator + * + * Returns: + * The next errors or NULL when there are not any more. + */ +struct fy_diag_error * +fy_diag_errors_iterate(struct fy_diag *diag, void **prevp) + FY_EXPORT; + +/** + * enum fy_path_parse_cfg_flags - Path parse configuration flags + * + * These flags control the operation of the path parse + * + * @FYPPCF_QUIET: Quiet, do not output any information messages + * @FYPPCF_DISABLE_RECYCLING: Disable recycling optimization + * @FYPPCF_DISABLE_ACCELERATORS: Disable use of access accelerators (saves memory) + */ +enum fy_path_parse_cfg_flags { + FYPPCF_QUIET = FY_BIT(0), + FYPPCF_DISABLE_RECYCLING = FY_BIT(1), + FYPPCF_DISABLE_ACCELERATORS = FY_BIT(2), +}; + +/** + * struct fy_path_parse_cfg - path parser configuration structure. + * + * Argument to the fy_path_parser_create() method which + * performs parsing of a ypath expression + * + * @flags: Configuration flags + * @userdata: Opaque user data pointer + * @diag: Optional diagnostic interface to use + */ +struct fy_path_parse_cfg { + enum fy_path_parse_cfg_flags flags; + void *userdata; + struct fy_diag *diag; +}; + +/** + * fy_path_parser_create() - Create a ypath parser. + * + * Creates a path parser with its configuration @cfg + * The path parser may be destroyed by a corresponding call to + * fy_path_parser_destroy(). + * If @cfg is NULL a default yaml parser is created. + * + * @cfg: The configuration for the path parser + * + * Returns: + * A pointer to the path parser or NULL in case of an error. + */ +struct fy_path_parser * +fy_path_parser_create(const struct fy_path_parse_cfg *cfg) + FY_EXPORT; + +/** + * fy_path_parser_destroy() - Destroy the given path parser + * + * Destroy a path parser created earlier via fy_path_parser_create(). + * + * @fypp: The path parser to destroy + */ +void +fy_path_parser_destroy(struct fy_path_parser *fypp) + FY_EXPORT; + +/** + * fy_path_parser_reset() - Reset a path parser completely + * + * Completely reset a path parser, including after an error + * that caused a parser error to be emitted. + * + * @fypp: The path parser to reset + * + * Returns: + * 0 if the reset was successful, -1 otherwise + */ +int +fy_path_parser_reset(struct fy_path_parser *fypp) + FY_EXPORT; + +/** + * fy_path_parse_expr_from_string() - Parse an expression from a given string + * + * Create a path expression from a string using the provided path parser. + * + * @fypp: The path parser to use + * @str: The ypath source to use. + * @len: The length of the string (or -1 if '\0' terminated) + * + * Returns: + * The created path expression or NULL on error. + */ +struct fy_path_expr * +fy_path_parse_expr_from_string(struct fy_path_parser *fypp, + const char *str, size_t len) + FY_EXPORT; + +/** + * fy_path_expr_build_from_string() - Parse an expression from a given string + * + * Create a path expression from a string using the provided path parser + * configuration. + * + * @pcfg: The path parser configuration to use, or NULL for default + * @str: The ypath source to use. + * @len: The length of the string (or -1 if '\0' terminated) + * + * Returns: + * The created path expression or NULL on error. + */ +struct fy_path_expr * +fy_path_expr_build_from_string(const struct fy_path_parse_cfg *pcfg, + const char *str, size_t len) + FY_EXPORT; + +/** + * fy_path_expr_free() - Free a path expression + * + * Free a previously returned expression from any of the path parser + * methods like fy_path_expr_build_from_string() + * + * @expr: The expression to free (may be NULL) + */ +void +fy_path_expr_free(struct fy_path_expr *expr) + FY_EXPORT; + +/** + * fy_path_expr_dump() - Dump the contents of a path expression to + * the diagnostic object + * + * Dumps the expression using the provided error level. + * + * @expr: The expression to dump + * @diag: The diagnostic object to use + * @errlevel: The error level which the diagnostic will use + * @level: The nest level; should be set to 0 + * @banner: The banner to display on level 0 + */ +void +fy_path_expr_dump(struct fy_path_expr *expr, struct fy_diag *diag, + enum fy_error_type errlevel, int level, const char *banner) + FY_EXPORT; + +/** + * fy_path_expr_to_document() - Converts the path expression to a YAML document + * + * Converts the expression to a YAML document which is useful for + * understanding what the expression evaluates to. + * + * @expr: The expression to convert to a document + * + * Returns: + * The document of the expression or NULL on error. + */ +struct fy_document * +fy_path_expr_to_document(struct fy_path_expr *expr) + FY_EXPORT; + +/** + * enum fy_path_exec_cfg_flags - Path executor configuration flags + * + * These flags control the operation of the path expression executor + * + * @FYPXCF_QUIET: Quiet, do not output any information messages + * @FYPXCF_DISABLE_RECYCLING: Disable recycling optimization + * @FYPXCF_DISABLE_ACCELERATORS: Disable use of access accelerators (saves memory) + */ +enum fy_path_exec_cfg_flags { + FYPXCF_QUIET = FY_BIT(0), + FYPXCF_DISABLE_RECYCLING = FY_BIT(1), + FYPXCF_DISABLE_ACCELERATORS = FY_BIT(2), +}; + +/** + * struct fy_path_exec_cfg - path expression executor configuration structure. + * + * Argument to the fy_path_exec_create() method which + * performs execution of a ypath expression + * + * @flags: Configuration flags + * @userdata: Opaque user data pointer + * @diag: Optional diagnostic interface to use + */ +struct fy_path_exec_cfg { + enum fy_path_exec_cfg_flags flags; + void *userdata; + struct fy_diag *diag; +}; + +/** + * fy_path_exec_create() - Create a ypath expression executor. + * + * Creates a ypath expression parser with its configuration @cfg + * The executor may be destroyed by a corresponding call to + * fy_path_exec_destroy(). + * + * @xcfg: The configuration for the executor + * + * Returns: + * A pointer to the executor or NULL in case of an error. + */ +struct fy_path_exec * +fy_path_exec_create(const struct fy_path_exec_cfg *xcfg) + FY_EXPORT; + +/** + * fy_path_exec_destroy() - Destroy the given path expression executor + * + * Destroy ane executor created earlier via fy_path_exec_create(). + * + * @fypx: The path parser to destroy + */ +void +fy_path_exec_destroy(struct fy_path_exec *fypx) + FY_EXPORT; + +/** + * fy_path_exec_reset() - Reset an executor + * + * Completely reset an executor without releasing it. + * + * @fypx: The executor to reset + * + * Returns: + * 0 if the reset was successful, -1 otherwise + */ +int +fy_path_exec_reset(struct fy_path_exec *fypx) + FY_EXPORT; + +/** + * fy_path_exec_execute() - Execute a path expression starting at + * the given start node + * + * Execute the expression starting at fyn_start. If execution + * is successful the results are available via fy_path_exec_results_iterate() + * + * Note that it is illegal to modify the state of the document that the + * results reside between this call and the results collection. + * + * @fypx: The executor to use + * @expr: The expression to use + * @fyn_start: The node on which the expression will begin. + * + * Returns: + * 0 if the execution was successful, -1 otherwise + * + * Note that the execution may be successful but no results were + * produced, in which case the iterator will return NULL. + */ +int +fy_path_exec_execute(struct fy_path_exec *fypx, struct fy_path_expr *expr, + struct fy_node *fyn_start) + FY_EXPORT; + +/** + * fy_path_exec_results_iterate() - Iterate over the results of execution + * + * This method iterates over all the results in the executor. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * @fypx: The executor + * @prevp: The previous result iterator + * + * Returns: + * The next node in the result set or NULL at the end of the results. + */ +struct fy_node * +fy_path_exec_results_iterate(struct fy_path_exec *fypx, void **prevp) + FY_EXPORT; + +/* + * Helper methods for binding implementers + * Note that users of the library do not need to know these details. + * However bindings that were developed against libyaml expect these + * to be exported, so provide a shim here + */ + +/** + * enum fy_token_type - Token types + * + * @FYTT_NONE: No token + * @FYTT_STREAM_START: Stream start token + * @FYTT_STREAM_END: Stream end token + * @FYTT_VERSION_DIRECTIVE: Version directive token + * @FYTT_TAG_DIRECTIVE: Tag directive token + * @FYTT_DOCUMENT_START: Document start token + * @FYTT_DOCUMENT_END: Document end token + * @FYTT_BLOCK_SEQUENCE_START: Block sequence start token + * @FYTT_BLOCK_MAPPING_START: Block mapping start token + * @FYTT_BLOCK_END: Block end token + * @FYTT_FLOW_SEQUENCE_START: Flow sequence start token + * @FYTT_FLOW_SEQUENCE_END: Flow sequence end token + * @FYTT_FLOW_MAPPING_START: Flow mapping start token + * @FYTT_FLOW_MAPPING_END: Flow mapping end token + * @FYTT_BLOCK_ENTRY: Block entry token + * @FYTT_FLOW_ENTRY: Flow entry token + * @FYTT_KEY: Key token + * @FYTT_VALUE: Value token + * @FYTT_ALIAS: Alias token + * @FYTT_ANCHOR: Anchor token + * @FYTT_TAG: Tag token + * @FYTT_SCALAR: Scalar token + * @FYTT_INPUT_MARKER: Input marker token + * @FYTT_PE_SLASH: Path expression slash token + * @FYTT_PE_ROOT: Path expression root token + * @FYTT_PE_THIS: Path expression this token + * @FYTT_PE_PARENT: Path expression parent token + * @FYTT_PE_MAP_KEY: Path expression map key token + * @FYTT_PE_SEQ_INDEX: Path expression sequence index token + * @FYTT_PE_SEQ_SLICE: Path expression sequence slice token + * @FYTT_PE_SCALAR_FILTER: Path expression scalar filter token + * @FYTT_PE_COLLECTION_FILTER: Path expression collection filter token + * @FYTT_PE_SEQ_FILTER: Path expression sequence filter token + * @FYTT_PE_MAP_FILTER: Path expression map filter token + * @FYTT_PE_UNIQUE_FILTER: Path expression unique filter token + * @FYTT_PE_EVERY_CHILD: Path expression every child token + * @FYTT_PE_EVERY_CHILD_R: Path expression every child recursive token + * @FYTT_PE_ALIAS: Path expression alias token + * @FYTT_PE_SIBLING: Path expression sibling token + * @FYTT_PE_COMMA: Path expression comma token + * @FYTT_PE_BARBAR: Path expression || token + * @FYTT_PE_AMPAMP: Path expression && token + * @FYTT_PE_LPAREN: Path expression ( token + * @FYTT_PE_RPAREN: Path expression ) token + * @FYTT_PE_EQEQ: Path expression == token + * @FYTT_PE_NOTEQ: Path expression != token + * @FYTT_PE_LT: Path expression < token + * @FYTT_PE_GT: Path expression > token + * @FYTT_PE_LTE: Path expression <= token + * @FYTT_PE_GTE: Path expression >= token + * @FYTT_SE_PLUS: Scalar expression + token + * @FYTT_SE_MINUS: Scalar expression - token + * @FYTT_SE_MULT: Scalar expression \* token + * @FYTT_SE_DIV: Scalar expression / token + * @FYTT_PE_METHOD: Path expression method token + * @FYTT_SE_METHOD: Scalar expression method token + * @FYTT_PE_BANG: Path expression ! token + * @FYTT_PE_AT: Path expression \@ token + */ +enum fy_token_type { + /* non-content token types */ + FYTT_NONE, + FYTT_STREAM_START, + FYTT_STREAM_END, + FYTT_VERSION_DIRECTIVE, + FYTT_TAG_DIRECTIVE, + FYTT_DOCUMENT_START, + FYTT_DOCUMENT_END, + /* content token types */ + FYTT_BLOCK_SEQUENCE_START, + FYTT_BLOCK_MAPPING_START, + FYTT_BLOCK_END, + FYTT_FLOW_SEQUENCE_START, + FYTT_FLOW_SEQUENCE_END, + FYTT_FLOW_MAPPING_START, + FYTT_FLOW_MAPPING_END, + FYTT_BLOCK_ENTRY, + FYTT_FLOW_ENTRY, + FYTT_KEY, + FYTT_VALUE, + FYTT_ALIAS, + FYTT_ANCHOR, + FYTT_TAG, + FYTT_SCALAR, + + /* special error reporting */ + FYTT_INPUT_MARKER, + + /* path expression tokens */ + FYTT_PE_SLASH, + FYTT_PE_ROOT, + FYTT_PE_THIS, + FYTT_PE_PARENT, + FYTT_PE_MAP_KEY, + FYTT_PE_SEQ_INDEX, + FYTT_PE_SEQ_SLICE, + FYTT_PE_SCALAR_FILTER, + FYTT_PE_COLLECTION_FILTER, + FYTT_PE_SEQ_FILTER, + FYTT_PE_MAP_FILTER, + FYTT_PE_UNIQUE_FILTER, + FYTT_PE_EVERY_CHILD, + FYTT_PE_EVERY_CHILD_R, + FYTT_PE_ALIAS, + FYTT_PE_SIBLING, + FYTT_PE_COMMA, + FYTT_PE_BARBAR, + FYTT_PE_AMPAMP, + FYTT_PE_LPAREN, + FYTT_PE_RPAREN, + + /* comparison operators */ + FYTT_PE_EQEQ, + FYTT_PE_NOTEQ, + FYTT_PE_LT, + FYTT_PE_GT, + FYTT_PE_LTE, + FYTT_PE_GTE, + + /* scalar expression tokens */ + FYTT_SE_PLUS, + FYTT_SE_MINUS, + FYTT_SE_MULT, + FYTT_SE_DIV, + + FYTT_PE_METHOD, + FYTT_SE_METHOD, + + FYTT_PE_BANG, + FYTT_PE_AT, +}; + +/* The number of token types available */ +#define FYTT_COUNT (FYTT_PE_AT+1) + +/** + * fy_token_type_is_valid() - Check token type validity + * + * Check if argument token type is a valid one. + * + * @type: The token type + * + * Returns: + * true if the token type is valid, false otherwise + */ +static inline bool +fy_token_type_is_valid(enum fy_token_type type) +{ + return type >= FYTT_NONE && type < FYTT_COUNT; +} + +/** + * fy_token_type_is_yaml() - Check if token type is valid for YAML + * + * Check if argument token type is a valid YAML one. + * + * @type: The token type + * + * Returns: + * true if the token type is a valid YAML one, false otherwise + */ +static inline bool +fy_token_type_is_yaml(enum fy_token_type type) +{ + return type >= FYTT_STREAM_START && type <= FYTT_SCALAR; +} + +/** + * fy_token_type_is_content() - Check if token type is + * valid for YAML content + * + * Check if argument token type is a valid YAML content one. + * + * @type: The token type + * + * Returns: + * true if the token type is a valid YAML content one, false otherwise + */ +static inline bool +fy_token_type_is_content(enum fy_token_type type) +{ + return type >= FYTT_BLOCK_SEQUENCE_START && type <= FYTT_SCALAR; +} + +/** + * fy_token_type_is_path_expr() - Check if token type is + * valid for a YPATH expression + * + * Check if argument token type is a valid YPATH parse expression token + * + * @type: The token type + * + * Returns: + * true if the token type is a valid YPATH one, false otherwise + */ +static inline bool +fy_token_type_is_path_expr(enum fy_token_type type) +{ + return type >= FYTT_PE_SLASH && type <= FYTT_PE_GTE; +} + +/** + * fy_token_type_is_scalar_expr() - Check if token type is + * valid for a YPATH scalar expression + * + * Check if argument token type is a valid YPATH parse scalar expression token + * + * @type: The token type + * + * Returns: + * true if the token type is a valid YPATH scalar one, false otherwise + */ +static inline bool +fy_token_type_is_scalar_expr(enum fy_token_type type) +{ + return type >= FYTT_SE_PLUS && type <= FYTT_SE_DIV; +} + +/** + * fy_token_get_type() - Return the token's type + * + * Return the token's type; if NULL then FYTT_NONE is returned + * + * @fyt: The token + * + * Returns: + * The token's type; FYTT_NONE if not a valid token (or NULL) + */ +enum fy_token_type +fy_token_get_type(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_token_start_mark() - Get token's start marker + * + * Return the token's start marker if it exists. Note + * it is permissable for some token types to have no + * start marker because they are without content. + * + * @fyt: The token to get its start marker + * + * Returns: + * The token's start marker, NULL if not available. + */ +const struct fy_mark * +fy_token_start_mark(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_token_end_mark() - Get token's end marker + * + * Return the token's end marker if it exists. Note + * it is permissable for some token types to have no + * end marker because they are without content. + * + * @fyt: The token to get its end marker + * + * Returns: + * The token's end marker, NULL if not available. + */ +const struct fy_mark * +fy_token_end_mark(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_scan() - Low level access to the scanner + * + * Return the next scanner token. Note this is a very + * low level interface, intended for users that want/need + * to implement their own YAML parser. The returned + * token is expected to be disposed using fy_scan_token_free() + * + * @fyp: The parser to get the next token from. + * + * Returns: + * The next token, or NULL if no more tokens are available. + */ +struct fy_token * +fy_scan(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_scan_token_free() - Free the token returned by fy_scan() + * + * Free the token returned by fy_scan(). + * + * @fyp: The parser of which the token was returned by fy_scan() + * @fyt: The token to free + */ +void +fy_scan_token_free(struct fy_parser *fyp, struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_tag_directive_token_prefix0() - Get the prefix contained in the + * tag directive token as zero terminated + * string + * + * Retrieve the tag directive's prefix contents as a zero terminated string. + * It is similar to fy_tag_directive_token_prefix(), with the difference + * that the returned string is zero terminated and memory may be allocated + * to hold it associated with the token. + * + * @fyt: The tag directive token out of which the prefix pointer + * will be returned. + * + * Returns: + * A pointer to the zero terminated text representation of the prefix token. + * NULL in case of an error. + */ +const char * +fy_tag_directive_token_prefix0(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_tag_directive_token_handle0() - Get the handle contained in the + * tag directive token as zero terminated + * string + * + * Retrieve the tag directive's handle contents as a zero terminated string. + * It is similar to fy_tag_directive_token_handle(), with the difference + * that the returned string is zero terminated and memory may be allocated + * to hold it associated with the token. + * + * @fyt: The tag directive token out of which the handle pointer + * will be returned. + * + * Returns: + * A pointer to the zero terminated text representation of the handle token. + * NULL in case of an error. + */ +const char * +fy_tag_directive_token_handle0(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_tag_token_handle() - Get the handle contained in the + * tag token + * + * Retrieve the tag handle contents. Will fail if + * token is not a tag token, or if a memory error happens. + * + * @fyt: The tag token out of which the handle pointer + * will be returned. + * @lenp: Pointer to a variable that will hold the returned length + * + * Returns: + * A pointer to the text representation of the handle token, while + * @lenp will be assigned the character length of said representation. + * NULL in case of an error. + */ +const char * +fy_tag_token_handle(struct fy_token *fyt, size_t *lenp) + FY_EXPORT; + +/** + * fy_tag_token_suffix() - Get the suffix contained in the + * tag token + * + * Retrieve the tag suffix contents. Will fail if + * token is not a tag token, or if a memory error happens. + * + * @fyt: The tag token out of which the suffix pointer + * will be returned. + * @lenp: Pointer to a variable that will hold the returned length + * + * Returns: + * A pointer to the text representation of the suffix token, while + * @lenp will be assigned the character length of said representation. + * NULL in case of an error. + */ +const char * +fy_tag_token_suffix(struct fy_token *fyt, size_t *lenp) + FY_EXPORT; + +/** + * fy_tag_token_short() - Get the short tag of the tag token + * + * Retrieve the short tag contents. The short tag is the same + * one that will need to be emitted. + * Will fail if token is not a tag token, or if a memory error happens. + * + * @fyt: The tag token out of which the short pointer + * will be returned. + * @lenp: Pointer to a variable that will hold the returned length + * + * Returns: + * A pointer to the text representation of the short tag, while + * @lenp will be assigned the character length of said representation. + * NULL in case of an error. + */ +const char * +fy_tag_token_short(struct fy_token *fyt, size_t *lenp) + FY_EXPORT; + +/** + * fy_tag_token_handle0() - Get the handle contained in the + * tag token as zero terminated string + * + * Retrieve the tag handle contents as a zero terminated string. + * It is similar to fy_tag_token_handle(), with the difference + * that the returned string is zero terminated and memory may be allocated + * to hold it associated with the token. + * + * @fyt: The tag token out of which the handle pointer will be returned. + * + * Returns: + * A pointer to the zero terminated text representation of the handle token. + * NULL in case of an error. + */ +const char * +fy_tag_token_handle0(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_tag_token_short0() - Get the short tag of the tag token as zero + * terminated string. + * + * Retrieve the short tag contents. The short tag is the same + * one that will need to be emitted. + * Will fail if token is not a tag token, or if a memory error happens. + * + * @fyt: The tag token out of which the short pointer will be returned. + * + * Returns: + * A pointer to the null terminated text representation of the short tag. + * NULL in case of an error. + */ +const char * +fy_tag_token_short0(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_tag_token_suffix0() - Get the suffix contained in the + * tag token as zero terminated string + * + * Retrieve the tag suffix contents as a zero terminated string. + * It is similar to fy_tag_token_suffix(), with the difference + * that the returned string is zero terminated and memory may be allocated + * to hold it associated with the token. + * + * @fyt: The tag token out of which the suffix pointer will be returned. + * + * Returns: + * A pointer to the zero terminated text representation of the suffix token. + * NULL in case of an error. + */ +const char * +fy_tag_token_suffix0(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_version_directive_token_version() - Return the version of a version + * directive token + * + * Retrieve the version stored in a version directive token. + * + * @fyt: The version directive token + * + * Returns: + * A pointer to the version stored in the version directive token, or + * NULL in case of an error, or the token not being a version directive token. + */ +const struct fy_version * +fy_version_directive_token_version(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_scalar_token_get_style() - Return the style of a scalar token + * + * Retrieve the style of a scalar token. + * + * @fyt: The scalar token + * + * Returns: + * The scalar style of the token, or FYSS_ANY for an error + */ +enum fy_scalar_style +fy_scalar_token_get_style(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_tag_token_tag() - Get tag of a tag token + * + * Retrieve the tag of a tag token. + * + * @fyt: The tag token + * + * Returns: + * A pointer to the tag or NULL in case of an error + */ +const struct fy_tag * +fy_tag_token_tag(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_tag_directive_token_tag() - Get tag of a tag directive token + * + * Retrieve the tag of a tag directive token. + * + * @fyt: The tag directive token + * + * Returns: + * A pointer to the tag or NULL in case of an error + */ +const struct fy_tag * +fy_tag_directive_token_tag(struct fy_token *fyt) + FY_EXPORT; + +/** + * fy_event_get_token() - Return the main token of an event + * + * Retrieve the main token (i.e. not the tag or the anchor) of + * an event. It may be NULL in case of an implicit event. + * + * @fye: The event to get its main token + * + * Returns: + * The main token if it exists, NULL otherwise or in case of an error + */ +struct fy_token * +fy_event_get_token(struct fy_event *fye) + FY_EXPORT; + +/** + * fy_event_get_anchor_token() - Return the anchor token of an event + * + * Retrieve the anchor token if it exists. Only valid for + * mapping/sequence start and scalar events. + * + * @fye: The event to get its anchor token + * + * Returns: + * The anchor token if it exists, NULL otherwise or in case of an error + */ +struct fy_token * +fy_event_get_anchor_token(struct fy_event *fye) + FY_EXPORT; + +/** + * fy_event_get_tag_token() - Return the tag token of an event + * + * Retrieve the tag token if it exists. Only valid for + * mapping/sequence start and scalar events. + * + * @fye: The event to get its tag token + * + * Returns: + * The tag token if it exists, NULL otherwise or in case of an error + */ +struct fy_token * +fy_event_get_tag_token(struct fy_event *fye) + FY_EXPORT; + +/** + * fy_event_start_mark() - Get event's start marker + * + * Return the event's start marker if it exists. The + * start marker is the one of the event's main token. + * + * @fye: The event to get its start marker + * + * Returns: + * The event's start marker, NULL if not available. + */ +const struct fy_mark * +fy_event_start_mark(struct fy_event *fye) + FY_EXPORT; + +/** + * fy_event_end_mark() - Get event's end marker + * + * Return the event's end marker if it exists. The + * end marker is the one of the event's main token. + * + * @fye: The event to get its end marker + * + * Returns: + * The event's end marker, NULL if not available. + */ +const struct fy_mark * +fy_event_end_mark(struct fy_event *fye) + FY_EXPORT; + +/** + * fy_event_get_node_style() - Get the node style of an event + * + * Return the node style (FYNS_*) of an event. May return + * FYNS_ANY if the event is implicit. + * For collection start events the only possible values is + * FYNS_ANY, FYNS_FLOW, FYNS_BLOCK. + * A scalar event may return any. + * + * @fye: The event to get it's node style + * + * Returns: + * The event's end marker, NULL if not available. + */ +enum fy_node_style +fy_event_get_node_style(struct fy_event *fye) + FY_EXPORT; + +/** + * fy_document_start_event_version() - Return the version of a document + * start event + * + * Retrieve the version stored in a document start event + * + * @fye: The document start event + * + * Returns: + * A pointer to the version, or NULL in case of an error, or the event + * not being a document start event. + */ +const struct fy_version * +fy_document_start_event_version(struct fy_event *fye) + FY_EXPORT; + +/** + * fy_document_state_version() - Return the version of a document state + * object + * + * Retrieve the version stored in a document state object + * + * @fyds: The document state object + * + * Returns: + * A pointer to the version, or NULL in case of an error + */ +const struct fy_version * +fy_document_state_version(struct fy_document_state *fyds) + FY_EXPORT; + +/** + * fy_document_state_start_mark() - Get document state's start mark + * + * Return the document state's start mark (if it exists). + * Note that purely synthetic documents do not contain one + * + * @fyds: The document state object + * + * Returns: + * The document's start marker, NULL if not available. + */ +const struct fy_mark * +fy_document_state_start_mark(struct fy_document_state *fyds) + FY_EXPORT; + +/** + * fy_document_state_end_mark() - Get document state's end mark + * + * Return the document state's end mark (if it exists). + * Note that purely synthetic documents do not contain one + * + * @fyds: The document state object + * + * Returns: + * The document's end marker, NULL if not available. + */ +const struct fy_mark * +fy_document_state_end_mark(struct fy_document_state *fyds) + FY_EXPORT; + +/** + * fy_document_state_version_explicit() - Version explicit? + * + * Find out if a document state object's version was explicitly + * set in the document. + * Note that for synthetic documents this method returns false. + * + * @fyds: The document state object + * + * Returns: + * true if version was set explicitly, false otherwise + */ +bool +fy_document_state_version_explicit(struct fy_document_state *fyds) + FY_EXPORT; + +/** + * fy_document_state_tags_explicit() - Tags explicit? + * + * Find out if a document state object's tags were explicitly + * set in the document. + * Note that for synthetic documents this method returns false. + * + * @fyds: The document state object + * + * Returns: + * true if document had tags set explicitly, false otherwise + */ +bool +fy_document_state_tags_explicit(struct fy_document_state *fyds) + FY_EXPORT; + +/** + * fy_document_state_start_implicit() - Started implicitly? + * + * Find out if a document state object's document was + * started implicitly. + * Note that for synthetic documents this method returns false. + * + * @fyds: The document state object + * + * Returns: + * true if document was started implicitly, false otherwise + */ +bool +fy_document_state_start_implicit(struct fy_document_state *fyds) + FY_EXPORT; + +/** + * fy_document_state_end_implicit() - Started implicitly? + * + * Find out if a document state object's document was + * ended implicitly. + * Note that for synthetic documents this method returns false. + * + * @fyds: The document state object + * + * Returns: + * true if document was ended implicitly, false otherwise + */ +bool +fy_document_state_end_implicit(struct fy_document_state *fyds) + FY_EXPORT; + +/** + * fy_document_state_json_mode() - Input was JSON? + * + * Find out if a document state object's document was + * created by a JSON input. + * Note that for synthetic documents this method returns false. + * + * @fyds: The document state object + * + * Returns: + * true if document was created in JSON mode, false otherwise + */ +bool +fy_document_state_json_mode(struct fy_document_state *fyds) + FY_EXPORT; + +/** + * fy_document_state_tag_directive_iterate() - Iterate over the tag + * directives of a document state + * object + * + * This method iterates over all the tag directives nodes in the document state + * object. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * @fyds: The document state + * @prevp: The previous iterator + * + * Returns: + * The next tag or NULL at the end of the iteration sequence. + */ +const struct fy_tag * +fy_document_state_tag_directive_iterate(struct fy_document_state *fyds, void **prevp) + FY_EXPORT; + +/** + * fy_document_state_tag_directives() - Get all the tag directives in a + * malloc'ed array + * + * Return all the tag directives in a dynamically allocated area. + * Must be free()'d when not in use. + * + * @fyds: The document state + * + * Returns: + * An array of fy_tag pointer structures, terminated with a NULL pointer + * NULL on error + */ +struct fy_tag ** +fy_document_state_tag_directives(struct fy_document_state *fyds) + FY_EXPORT; + +/** + * fy_document_state_tag_is_default() - Test whether the given tag is a default one + * + * Test whether a tag is a default (i.e. impliciticly set) + * + * @fyds: The document state + * @tag: The tag to check + * + * Returns: + * true in case that the tag is a default one, false otherwise + */ +bool +fy_document_state_tag_is_default(struct fy_document_state *fyds, const struct fy_tag *tag) + FY_EXPORT; + +/** + * fy_parser_get_document_state() - Get the document state of a parser object + * + * Retrieve the document state object of a parser. Note that this is only + * valid during parsing. + * + * @fyp: The parser + * + * Returns: + * The active document state object of the parser, NULL otherwise + */ +struct fy_document_state * +fy_parser_get_document_state(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_document_get_document_state() - Get the document state of a document + * + * Retrieve the document state object of a document. + * + * @fyd: The document + * + * Returns: + * The document state object of the document, NULL otherwise + */ +struct fy_document_state * +fy_document_get_document_state(struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_set_document_state() - Set the document state of a document + * + * Set the document state of a document + * + * @fyd: The document + * @fyds: The document state to use, or NULL for default + * + * Returns: + * 0 if set operation was successful, -1 otherwise + */ +int +fy_document_set_document_state(struct fy_document *fyd, struct fy_document_state *fyds) + FY_EXPORT; + +/** + * fy_document_create_from_event() - Create an empty document using the event + * + * Create an empty document using the FYET_DOCUMENT_START event generated + * by the parser. + * + * @fyp: The parser + * @fye: The event + * + * Returns: + * The created empty document, or NULL on error. + */ +struct fy_document * +fy_document_create_from_event(struct fy_parser *fyp, struct fy_event *fye) + FY_EXPORT; + +/** + * fy_document_update_from_event() - Update the document with the event + * + * Update the document using the FYET_DOCUMENT_END event generated + * by the parser. + * + * @fyd: The document + * @fyp: The parser + * @fye: The event + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_document_update_from_event(struct fy_document *fyd, struct fy_parser *fyp, struct fy_event *fye) + FY_EXPORT; + +/** + * fy_node_create_from_event() - Create a node using the event + * + * Create a new node using the supplied event. + * Allowed events are FYET_SCALAR, FYET_ALIAS, FYET_MAPPING_START & FYET_SEQUENCE_START + * + * @fyd: The document + * @fyp: The parser + * @fye: The event + * + * Returns: + * The newly created node, or NULL on error + */ +struct fy_node * +fy_node_create_from_event(struct fy_document *fyd, struct fy_parser *fyp, struct fy_event *fye) + FY_EXPORT; + +/** + * fy_node_update_from_event() - Update a node using the event + * + * Update information of node created using fy_node_create_from_event() + * Allowed events are FYET_MAPPING_END & FYET_SEQUENCE_END + * + * @fyn: The node + * @fyp: The parser + * @fye: The event + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_update_from_event(struct fy_node *fyn, struct fy_parser *fyp, struct fy_event *fye) + FY_EXPORT; + +/** + * fy_node_pair_create_with_key() - Create a new node pair and set it's key + * + * Create a new node pair using the supplied fyn_parent mapping and fyn node as + * a key. Note that the nodes _must_ have been created by fy_node_create_from_event + * and they are not interchangeable with other node pair methods. + * + * The node pair will be added to the fyn_parent mapping with a subsequent call + * to fy_node_pair_update_with_value(). + * + * @fyd: The document + * @fyn_parent: The mapping + * @fyn: The node pair key + * + * Returns: + * The newly created node pair, or NULL on error + */ +struct fy_node_pair * +fy_node_pair_create_with_key(struct fy_document *fyd, struct fy_node *fyn_parent, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_pair_update_with_value() - Update a node pair with a value and add it to the parent mapping + * + * Update the node pair with the given value and add it to the parent mapping. + * Note that the fyn node _must_ have been created by fy_node_create_from_event + * and the node pair created by fy_node_pair_create_with_key(). + * Do not intermix other node pair manipulation methods. + * + * @fynp: The node pair + * @fyn: The node pair value + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_pair_update_with_value(struct fy_node_pair *fynp, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_sequence_add_item() - Add an item node to a sequence node + * + * Add an item to the end of the sequence node fyn_parent. + * Note that the fyn_parent and fyn nodes _must_ have been created by + * fy_node_create_from_event. + * Do not intermix other sequence node manipulation methods. + * + * @fyn_parent: The parent sequence node + * @fyn: The node pair item + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_sequence_add_item(struct fy_node *fyn_parent, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_emitter_get_document_state() - Get the document state of an emitter object + * + * Retrieve the document state object of an emitter. Note that this is only + * valid during emitting. + * + * @emit: The emitter + * + * Returns: + * The active document state object of the emitter, NULL otherwise + */ +struct fy_document_state * +fy_emitter_get_document_state(struct fy_emitter *emit) + FY_EXPORT; + +/** + * fy_emit_event_create() - Create an emit event. + * + * Create an emit event to pass to fy_emit_event() + * The extra arguments differ according to the event to be created + * + * FYET_STREAM_START: + * - None + * + * FYET_STREAM_END: + * - None + * + * FYET_DOCUMENT_START: + * - int implicit + * true if document start should be marked implicit + * false if document start should not be marked implicit + * - const struct fy_version \*vers + * Pointer to version to use for the document, or NULL for default + * - const struct fy_tag \* const \*tags + * Pointer to a NULL terminated array of tag pointers (like argv) + * NULL if no extra tags are to be used + * + * FYET_DOCUMENT_END: + * - int implicit + * true if document end should be marked implicit + * false if document end should not be marked implicit + * + * FYET_MAPPING_START: + * - enum fy_node_style style + * Style of the mapping (one of FYNS_ANY, FYNS_BLOCK or FYNS_FLOW) + * - const char \*anchor + * Anchor to place at the mapping, or NULL for none + * - const char \*tag + * Tag to place at the mapping, or NULL for none + * + * FYET_MAPPING_END: + * - None + * + * FYET_SEQUENCE_START: + * - enum fy_node_style style + * Style of the sequence (one of FYNS_ANY, FYNS_BLOCK or FYNS_FLOW) + * - const char \*anchor + * Anchor to place at the sequence, or NULL for none + * - const char \*tag + * Tag to place at the sequence, or NULL for none + * + * FYET_SEQUENCE_END: + * - None + * + * FYET_SCALAR: + * - enum fy_scalar_style style + * Style of the scalar, any valid FYSS_* value + * Note that certain styles may not be used according to the + * contents of the data + * - const char \*value + * Pointer to the scalar contents. + * - size_t len + * Length of the scalar contents, of FY_NT (-1) for strlen(value) + * - const char \*anchor + * Anchor to place at the scalar, or NULL for none + * - const char \*tag + * Tag to place at the scalar, or NULL for none + * + * FYET_ALIAS: + * - const char \*value + * Pointer to the alias. + * + * @emit: The emitter + * @type: The event type to create + * @...: The optional extra arguments + * + * Returns: + * The created event or NULL in case of an error + */ +struct fy_event * +fy_emit_event_create(struct fy_emitter *emit, enum fy_event_type type, ...) + FY_EXPORT; + +/** + * fy_emit_event_vcreate() - Create an emit event using varargs. + * + * Create an emit event to pass to fy_emit_event() + * The varargs analogous to fy_emit_event_create(). + * + * @emit: The emitter + * @type: The event type to create + * @ap: The variable argument list pointer. + * + * Returns: + * The created event or NULL in case of an error + */ +struct fy_event * +fy_emit_event_vcreate(struct fy_emitter *emit, enum fy_event_type type, va_list ap) + FY_EXPORT; + +/** + * fy_emit_event_free() - Free an event created via fy_emit_event_create() + * + * Free an event previously created via fy_emit_event_create(). Note + * that usually you don't have to call this method since if you pass + * the event to fy_emit_event() it shall be disposed properly. + * Only use is error recovery and cleanup. + * + * @emit: The emitter + * @fye: The event to free + */ +void +fy_emit_event_free(struct fy_emitter *emit, struct fy_event *fye) + FY_EXPORT; + +/** + * fy_parse_event_create() - Create an emit event. + * + * See fy_emit_event_create()... + * + * @fyp: The parser + * @type: The event type to create + * + * Returns: + * The created event or NULL in case of an error + */ +struct fy_event * +fy_parse_event_create(struct fy_parser *fyp, enum fy_event_type type, ...) + FY_EXPORT; + +/** + * fy_parse_event_vcreate() - Create an emit event using varargs. + * + * Create an emit event to pass to fy_emit_event() + * The varargs analogous to fy_parse_event_create(). + * + * @fyp: The parser + * @type: The event type to create + * @ap: The variable argument list pointer. + * + * Returns: + * The created event or NULL in case of an error + */ +struct fy_event * +fy_parse_event_vcreate(struct fy_parser *fyp, enum fy_event_type type, va_list ap) + FY_EXPORT; + +/** + * enum fy_composer_return - The returns of the composer callback + * + * @FYCR_OK_CONTINUE: continue processing, event processed + * @FYCR_OK_STOP: stop processing, event processed + * @FYCR_OK_START_SKIP: start skip object(s), event processed + * @FYCR_OK_STOP_SKIP: stop skipping of objects, event processed + * @FYCR_ERROR: error, stop processing + */ +enum fy_composer_return { + FYCR_OK_CONTINUE = 0, + FYCR_OK_STOP = 1, + FYCR_OK_START_SKIP = 2, + FYCR_OK_STOP_SKIP = 3, + FYCR_ERROR = -1, +}; + +/** + * fy_composer_return_is_ok() - Check if the return code is OK + * + * Convenience method for checking if it's OK to continue + * + * @ret: the composer return to check + * + * Returns: + * true if non error or skip condition + */ +static inline bool +fy_composer_return_is_ok(enum fy_composer_return ret) +{ + return ret == FYCR_OK_CONTINUE || ret == FYCR_OK_STOP; +} + +/** + * typedef fy_parse_composer_cb - composer callback method + * + * This method is called by the fy_parse_compose() method + * when an event must be processed. + * + * @fyp: The parser + * @fye: The event + * @path: The path that the parser is processing + * @userdata: The user data of the fy_parse_compose() method + * + * Returns: + * fy_composer_return code telling the parser what to do + */ +typedef enum fy_composer_return +(*fy_parse_composer_cb)(struct fy_parser *fyp, struct fy_event *fye, + struct fy_path *path, void *userdata); + +/** + * fy_parse_compose() - Parse using a compose callback + * + * Alternative parsing method using a composer callback. + * + * The parser will construct a path argument that is used + * by the callback to make intelligent decisions about + * creating a document and/or DOM. + * + * @fyp: The parser + * @cb: The callback that will be called + * @userdata: user pointer to pass to the callback + * + * Returns: + * 0 if no error occured + * -1 on error + */ +int +fy_parse_compose(struct fy_parser *fyp, fy_parse_composer_cb cb, + void *userdata) + FY_EXPORT; + +/** + * fy_path_component_is_mapping() - Check if the component is a mapping + * + * @fypc: The path component to check + * + * Returns: + * true if the path component is a mapping, false otherwise + */ +bool +fy_path_component_is_mapping(struct fy_path_component *fypc) + FY_EXPORT; + +/** + * fy_path_component_is_sequence() - Check if the component is a sequence + * + * @fypc: The path component to check + * + * Returns: + * true if the path component is a sequence, false otherwise + */ +bool +fy_path_component_is_sequence(struct fy_path_component *fypc) + FY_EXPORT; + +/** + * fy_path_component_sequence_get_index() - Get the index of sequence path component + * + * @fypc: The sequence path component to return it's index value + * + * Returns: + * >= 0 the sequence index + * -1 if the component is either not in the proper mode, or not a sequence + */ +int +fy_path_component_sequence_get_index(struct fy_path_component *fypc) + FY_EXPORT; + +/** + * fy_path_component_mapping_get_scalar_key() - Get the scalar key of a mapping + * + * @fypc: The mapping path component to return it's scalar key + * + * Returns: + * a non NULL scalar or alias token if the mapping contains a scalar key + * NULL in case of an error, or if the component has a complex key + */ +struct fy_token * +fy_path_component_mapping_get_scalar_key(struct fy_path_component *fypc) + FY_EXPORT; + +/** + * fy_path_component_mapping_get_scalar_key_tag() - Get the scalar key's tag of a mapping + * + * @fypc: The mapping path component to return it's scalar key's tag + * + * Returns: + * a non NULL tag token if the mapping contains a scalar key with a tag or + * NULL in case of an error, or if the component has a complex key + */ +struct fy_token * +fy_path_component_mapping_get_scalar_key_tag(struct fy_path_component *fypc) + FY_EXPORT; + +/** + * fy_path_component_mapping_get_complex_key() - Get the complex key of a mapping + * + * @fypc: The mapping path component to return it's complex key + * + * Returns: + * a non NULL document if the mapping contains a complex key + * NULL in case of an error, or if the component has a simple key + */ +struct fy_document * +fy_path_component_mapping_get_complex_key(struct fy_path_component *fypc) + FY_EXPORT; + +/** + * fy_path_component_get_text() - Get the textual representation of a path component + * + * Given a path component, return a malloc'ed string which contains + * the textual representation of it. + * + * Note that this method will only return fully completed components and not + * ones that are in the building process. + * + * @fypc: The path component to get it's textual representation + * + * Returns: + * The textual representation of the path component, NULL on error, or + * if the component has not been completely built during the composition + * of a complex key. + * The string must be free'ed using free. + */ +char * +fy_path_component_get_text(struct fy_path_component *fypc) + FY_EXPORT; + +#define fy_path_component_get_text_alloca(_fypc) \ + FY_ALLOCA_COPY_FREE(fy_path_component_get_text((_fypc)), FY_NT) +/** + * fy_path_depth() - Get the depth of a path + * + * @fypp: The path to query + * + * Returns: + * The depth of the path, or -1 on error + */ +int +fy_path_depth(struct fy_path *fypp) + FY_EXPORT; + +/** + * fy_path_parent() - Get the parent of a path + * + * Paths may contain parents when traversing complex keys. + * This method returns the immediate parent. + * + * @fypp: The path to return it's parent + * + * Returns: + * The path parent or NULL on error, if it doesn't exist + */ +struct fy_path * +fy_path_parent(struct fy_path *fypp) + FY_EXPORT; + +/** + * fy_path_get_text() - Get the textual representation of a path + * + * Given a path, return a malloc'ed string which contains + * the textual representation of it. + * + * Note that during processing, complex key paths are simply + * indicative and not to be used for addressing. + * + * @fypp: The path to get it's textual representation + * + * Returns: + * The textual representation of the path, NULL on error. + * The string must be free'ed using free. + */ +char * +fy_path_get_text(struct fy_path *fypp) + FY_EXPORT; + +#define fy_path_get_text_alloca(_fypp) \ + FY_ALLOCA_COPY_FREE(fy_path_get_text((_fypp)), FY_NT) + +/** + * fy_path_in_root() - Check if the path is in the root of the document + * + * @fypp: The path + * + * Returns: + * true if the path is located within the root of the document + */ +bool +fy_path_in_root(struct fy_path *fypp) + FY_EXPORT; + +/** + * fy_path_in_mapping() - Check if the path is in a mapping + * + * @fypp: The path + * + * Returns: + * true if the path is located within a mapping + */ +bool +fy_path_in_mapping(struct fy_path *fypp) + FY_EXPORT; + +/** + * fy_path_in_sequence() - Check if the path is in a sequence + * + * @fypp: The path + * + * Returns: + * true if the path is located within a sequence + */ +bool +fy_path_in_sequence(struct fy_path *fypp) + FY_EXPORT; + +/** + * fy_path_in_mapping_key() - Check if the path is in a mapping key state + * + * @fypp: The path + * + * Returns: + * true if the path is located within a mapping key state + */ +bool +fy_path_in_mapping_key(struct fy_path *fypp) + FY_EXPORT; + +/** + * fy_path_in_mapping_value() - Check if the path is in a mapping value state + * + * @fypp: The path + * + * Returns: + * true if the path is located within a mapping value state + */ +bool +fy_path_in_mapping_value(struct fy_path *fypp) + FY_EXPORT; + +/** + * fy_path_in_collection_root() - Check if the path is in a collection root + * + * A collection root state is when the path points to a sequence or mapping + * but the state does not allow setting keys, values or adding items. + * + * This occurs on MAPPING/SEQUENCE START/END events. + * + * @fypp: The path + * + * Returns: + * true if the path is located within a collectin root state + */ +bool +fy_path_in_collection_root(struct fy_path *fypp) + FY_EXPORT; + +/** + * fy_path_get_root_user_data() - Return the userdata associated with the path root + * + * @fypp: The path + * + * Returns: + * The user data associated with the root of the path, or NULL if no path + */ +void * +fy_path_get_root_user_data(struct fy_path *fypp) + FY_EXPORT; + +/** + * fy_path_set_root_user_data() - Set the user data associated with the root + * + * Note, no error condition if not a path + * + * @fypp: The path + * @data: The data to set as root data + */ +void +fy_path_set_root_user_data(struct fy_path *fypp, void *data) + FY_EXPORT; + +/** + * fy_path_component_get_mapping_user_data() - Return the userdata associated with the mapping + * + * @fypc: The path component + * + * Returns: + * The user data associated with the mapping, or NULL if not a mapping or the user data are NULL + */ +void * +fy_path_component_get_mapping_user_data(struct fy_path_component *fypc) + FY_EXPORT; + +/** + * fy_path_component_get_mapping_key_user_data() - Return the userdata associated with the mapping key + * + * @fypc: The path component + * + * Returns: + * The user data associated with the mapping key, or NULL if not a mapping or the user data are NULL + */ +void * +fy_path_component_get_mapping_key_user_data(struct fy_path_component *fypc) + FY_EXPORT; + +/** + * fy_path_component_get_sequence_user_data() - Return the userdata associated with the sequence + * + * @fypc: The path component + * + * Returns: + * The user data associated with the sequence, or NULL if not a sequence or the user data are NULL + */ +void * +fy_path_component_get_sequence_user_data(struct fy_path_component *fypc) + FY_EXPORT; + +/** + * fy_path_component_set_mapping_user_data() - Set the user data associated with a mapping + * + * Note, no error condition if not a mapping + * + * @fypc: The path component + * @data: The data to set as mapping data + */ +void +fy_path_component_set_mapping_user_data(struct fy_path_component *fypc, void *data) + FY_EXPORT; + +/** + * fy_path_component_set_mapping_key_user_data() - Set the user data associated with a mapping key + * + * Note, no error condition if not in a mapping key state + * + * @fypc: The path component + * @data: The data to set as mapping key data + */ +void +fy_path_component_set_mapping_key_user_data(struct fy_path_component *fypc, void *data) + FY_EXPORT; + +/** + * fy_path_component_set_sequence_user_data() - Set the user data associated with a sequence + * + * Note, no error condition if not a sequence + * + * @fypc: The path component + * @data: The data to set as sequence data + */ +void +fy_path_component_set_sequence_user_data(struct fy_path_component *fypc, void *data) + FY_EXPORT; + +/** + * fy_path_get_parent_user_data() - Return the userdata of the parent collection + * + * @path: The path + * + * Returns: + * The user data associated with the parent collection of the path, or NULL if no path + */ +void * +fy_path_get_parent_user_data(struct fy_path *path) + FY_EXPORT; + +/** + * fy_path_set_parent_user_data() - Set the user data associated with the parent collection + * + * Note, no error condition if not a path + * + * @path: The path + * @data: The data to set as parent collection data + */ +void +fy_path_set_parent_user_data(struct fy_path *path, void *data) + FY_EXPORT; + +/** + * fy_path_get_last_user_data() - Return the userdata of the last collection + * + * @path: The path + * + * Returns: + * The user data associated with the last collection of the path, or NULL if no path + */ +void * +fy_path_get_last_user_data(struct fy_path *path) + FY_EXPORT; + +/** + * fy_path_set_last_user_data() - Set the user data associated with the last collection + * + * Note, no error condition if not a path + * + * @path: The path + * @data: The data to set as last collection data + */ +void +fy_path_set_last_user_data(struct fy_path *path, void *data) + FY_EXPORT; + +/** + * fy_path_last_component() - Get the very last component of a path + * + * Returns the last component of a path. + * + * @fypp: The path + * + * Returns: + * The last path component (which may be a collection root component), or NULL + * if it does not exist + */ +struct fy_path_component * +fy_path_last_component(struct fy_path *fypp) + FY_EXPORT; + +/** + * fy_path_last_not_collection_root_component() - Get the last non collection root component of a path + * + * Returns the last non collection root component of a path. This may not be the + * last component that is returned by fy_path_last_component(). + * + * The difference is present on MAPPING/SEQUENCE START/END events where the + * last component is present but not usuable as a object parent. + * + * @fypp: The path + * + * Returns: + * The last non collection root component, or NULL if it does not exist + */ +struct fy_path_component * +fy_path_last_not_collection_root_component(struct fy_path *fypp) + FY_EXPORT; + +/** + * struct fy_composer_ops - Composer operation callbacks + * + * Callbacks used by the composer to process events and create document builders. + * + * @process_event: Callback for processing a single YAML event with path context + * @create_document_builder: Callback for creating a document builder instance + */ +struct fy_composer_ops { + /* single process event callback */ + enum fy_composer_return (*process_event)(struct fy_composer *fyc, struct fy_path *path, struct fy_event *fye); + struct fy_document_builder *(*create_document_builder)(struct fy_composer *fyc); +}; + +/** + * struct fy_composer_cfg - Composer configuration structure + * + * Configuration structure for creating a composer instance. + * + * @ops: Pointer to composer operation callbacks + * @userdata: Opaque user data pointer passed to callbacks + * @diag: Optional diagnostic interface to use, NULL for default + */ +struct fy_composer_cfg { + const struct fy_composer_ops *ops; + void *userdata; + struct fy_diag *diag; +}; + +/** + * fy_composer_create() - Create a composer + * + * Creates a composer with the given configuration. The composer processes + * YAML events using callback methods and maintains path information for + * intelligent document composition. The composer may be destroyed by a + * corresponding call to fy_composer_destroy(). + * + * @cfg: The configuration for the composer + * + * Returns: + * A pointer to the composer or NULL in case of an error. + */ +struct fy_composer * +fy_composer_create(struct fy_composer_cfg *cfg) + FY_EXPORT; + +/** + * fy_composer_destroy() - Destroy the given composer + * + * Destroy a composer created earlier via fy_composer_create(). + * + * @fyc: The composer to destroy + */ +void fy_composer_destroy(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_process_event() - Process a YAML event through the composer + * + * Process a YAML event by calling the configured process_event callback + * with path context. The composer maintains the current path and provides + * it to the callback for intelligent processing decisions. + * + * @fyc: The composer + * @fye: The event to process + * + * Returns: + * A fy_composer_return code indicating how to proceed (continue, stop, skip, or error) + */ +enum fy_composer_return +fy_composer_process_event(struct fy_composer *fyc, struct fy_event *fye) + FY_EXPORT; + +/** + * fy_composer_get_cfg() - Get the configuration of a composer + * + * @fyc: The composer + * + * Returns: + * The configuration of the composer + */ +struct fy_composer_cfg * +fy_composer_get_cfg(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_get_cfg_userdata() - Get the userdata from composer configuration + * + * Retrieve the opaque userdata pointer from the composer's configuration. + * + * @fyc: The composer + * + * Returns: + * The userdata pointer from the configuration + */ +void * +fy_composer_get_cfg_userdata(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_get_diag() - Get the diagnostic object of a composer + * + * Return a pointer to the diagnostic object of a composer object. + * Note that the returned diag object has a reference taken so + * you should fy_diag_unref() it when you're done with it. + * + * @fyc: The composer to get the diagnostic object + * + * Returns: + * A pointer to a ref'ed diagnostic object or NULL in case of an error. + */ +struct fy_diag * +fy_composer_get_diag(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_get_path() - Get the current path of the composer + * + * Retrieve the current path being processed by the composer. + * The path represents the location in the YAML document structure + * where the composer is currently positioned. + * + * @fyc: The composer + * + * Returns: + * The current path, or NULL if no path is active + */ +struct fy_path * +fy_composer_get_path(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_get_root_path() - Get the root path of the composer + * + * Retrieve the root path of the composer's path hierarchy. + * + * @fyc: The composer + * + * Returns: + * The root path, or NULL if no root exists + */ +struct fy_path * +fy_composer_get_root_path(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_get_next_path() - Get the next path in the composer's path list + * + * Iterate through the composer's path list. Pass NULL to get the first path, + * or pass the previous path to get the next one. + * + * @fyc: The composer + * @fypp: The previous path, or NULL to get the first path + * + * Returns: + * The next path in the list, or NULL if no more paths exist + */ +struct fy_path * +fy_composer_get_next_path(struct fy_composer *fyc, struct fy_path *fypp) + FY_EXPORT; + +/* Shift amount of the want mode */ +#define FYDICF_WANT_SHIFT 0 +/* Mask of the WANT mode */ +#define FYDICF_WANT_MASK ((1U << 2) - 1) +/* Build a WANT mode option */ +#define FYDICF_WANT(x) (((unsigned int)(x) & FYDICF_WANT_MASK) << FYDICF_WANT_SHIFT) + +/** + * enum fy_document_iterator_cfg_flags - Document iterator configuration flags + * + * These flags control the operation of the document iterator + * + * @FYDICF_WANT_BODY_EVENTS: Generate body events + * @FYDICF_WANT_DOCUMENT_BODY_EVENTS: Generate document and body events + * @FYDICF_WANT_STREAM_DOCUMENT_BODY_EVENTS: Generate stream, document and body events + */ +enum fy_document_iterator_cfg_flags { + FYDICF_WANT_BODY_EVENTS = FYDICF_WANT(0), + FYDICF_WANT_DOCUMENT_BODY_EVENTS = FYDICF_WANT(1), + FYDICF_WANT_STREAM_DOCUMENT_BODY_EVENTS = FYDICF_WANT(2), +}; + +/** + * struct fy_document_iterator_cfg - document iterator configuration structure. + * + * Argument to the fy_document_iterator_create_cfg() method. + * + * @flags: The document iterator flags + * @fyd: The document to iterate on (or NULL if iterate_root is set) + * @iterate_root: The root of iteration (NULL when fyd is not NULL) + */ +struct fy_document_iterator_cfg { + enum fy_document_iterator_cfg_flags flags; + struct fy_document *fyd; + struct fy_node *iterate_root; +}; + +/** + * fy_document_iterator_create() - Create a document iterator + * + * Creates a document iterator, that can trawl through a document + * without using recursion. + * + * Returns: + * The newly created document iterator or NULL on error + */ +struct fy_document_iterator * +fy_document_iterator_create(void) + FY_EXPORT; + +/** + * fy_document_iterator_create_cfg() - Create a document iterator using config + * + * Creates a document iterator, that can trawl through a document + * without using recursion. The iterator will generate all the events + * that created the given document starting at iterator root. + * + * @cfg: The document iterator to destroy + * + * Returns: + * The newly created document iterator or NULL on error + */ +struct fy_document_iterator * +fy_document_iterator_create_cfg(const struct fy_document_iterator_cfg *cfg) + FY_EXPORT; + +/** + * fy_document_iterator_create_on_document() - Create a document iterator on document + * + * Creates a document iterator, starting at the root of the given document. + * + * @fyd: The document to iterate on + * + * Returns: + * The newly created document iterator or NULL on error + */ +struct fy_document_iterator * +fy_document_iterator_create_on_document(struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_iterator_create_on_node() - Create a document iterator on node + * + * Creates a document iterator, starting at the given node + * + * @fyn: The node to iterate on + * + * Returns: + * The newly created document iterator or NULL on error + */ +struct fy_document_iterator * +fy_document_iterator_create_on_node(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_document_iterator_destroy() - Destroy the given document iterator + * + * Destroy a document iterator created earlier via fy_document_iterator_create(). + * + * @fydi: The document iterator to destroy + */ +void +fy_document_iterator_destroy(struct fy_document_iterator *fydi) + FY_EXPORT; + +/** + * fy_document_iterator_event_free() - Free an event that was created by a document iterator + * + * Free (possibly recycling) an event that was created by a document iterator. + * + * @fydi: The document iterator that created the event + * @fye: The event + */ +void +fy_document_iterator_event_free(struct fy_document_iterator *fydi, struct fy_event *fye) + FY_EXPORT; + +/** + * fy_document_iterator_stream_start() - Create a stream start event using the iterator + * + * Creates a stream start event on the document iterator and advances the internal state + * of it accordingly. + * + * @fydi: The document iterator to create the event + * + * Returns: + * The newly created stream start event, or NULL on error. + */ +struct fy_event * +fy_document_iterator_stream_start(struct fy_document_iterator *fydi) + FY_EXPORT; + +/** + * fy_document_iterator_stream_end() - Create a stream end event using the iterator + * + * Creates a stream end event on the document iterator and advances the internal state + * of it accordingly. + * + * @fydi: The document iterator to create the event + * + * Returns: + * The newly created stream end event, or NULL on error. + */ +struct fy_event * +fy_document_iterator_stream_end(struct fy_document_iterator *fydi) + FY_EXPORT; + +/** + * fy_document_iterator_document_start() - Create a document start event using the iterator + * + * Creates a document start event on the document iterator and advances the internal state + * of it accordingly. The document must not be released until an error, cleanup or a call + * to fy_document_iterator_document_end(). + * + * @fydi: The document iterator to create the event + * @fyd: The document containing the document state that is used in the event + * + * Returns: + * The newly created document start event, or NULL on error. + */ +struct fy_event * +fy_document_iterator_document_start(struct fy_document_iterator *fydi, struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_iterator_document_end() - Create a document end event using the iterator + * + * Creates a document end event on the document iterator and advances the internal state + * of it accordingly. The document that was used earlier in the call of + * fy_document_iterator_document_start() can now be released. + * + * @fydi: The document iterator to create the event + * + * Returns: + * The newly created document end event, or NULL on error. + */ +struct fy_event * +fy_document_iterator_document_end(struct fy_document_iterator *fydi) + FY_EXPORT; + +/** + * fy_document_iterator_body_next() - Create document body events until the end + * + * Creates the next document body, depth first until the end of the document. + * The events are created depth first and are in same exact sequence that the + * original events that created the document. + * + * That means that the finite event stream that generated the document is losslesly + * preserved in such a way that the document tree representation is functionally + * equivalent. + * + * Repeated calls to this function will generate a stream of SCALAR, ALIAS, SEQUENCE + * START, SEQUENCE END, MAPPING START and MAPPING END events, returning NULL at the + * end of the body event stream. + * + * @fydi: The document iterator to create the event + * + * Returns: + * The newly created document body event or NULL at an error, or an end of the + * event stream. Use fy_document_iterator_get_error() to check if an error occured. + */ +struct fy_event * +fy_document_iterator_body_next(struct fy_document_iterator *fydi) + FY_EXPORT; + +/** + * fy_document_iterator_node_start() - Start a document node iteration run using a starting point + * + * Starts an iteration run starting at the given node. + * + * @fydi: The document iterator to run with + * @fyn: The iterator root for the iteration + */ +void +fy_document_iterator_node_start(struct fy_document_iterator *fydi, struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_document_iterator_node_next() - Return the next node in the iteration sequence + * + * Returns a pointer to the next node iterating using as a start the node given + * at fy_document_iterator_node_start(). The first node returned will be that, + * followed by all the remaing nodes in the subtree. + * + * @fydi: The document iterator to use for the iteration + * + * Returns: + * The next node in the iteration sequence or NULL at the end, or if an error occured. + */ +struct fy_node * +fy_document_iterator_node_next(struct fy_document_iterator *fydi) + FY_EXPORT; + +/** + * fy_document_iterator_generate_next() - Create events from document iterator + * + * This is a method that will handle the complex state of generating + * stream, document and body events on the given iterator. + * + * When generation is complete a NULL event will be generated. + * + * @fydi: The document iterator to create the event + * + * Returns: + * The newly created event or NULL at an error, or an end of the + * event stream. Use fy_document_iterator_get_error() to check if an error occured. + */ +struct fy_event * +fy_document_iterator_generate_next(struct fy_document_iterator *fydi) + FY_EXPORT; + +/** + * fy_document_iterator_get_error() - Get the error state of the document iterator + * + * Returns the error state of the iterator. If it's in error state, return true + * and reset the iterator to the state just after creation. + * + * @fydi: The document iterator to use for checking it's error state. + * + * Returns: + * true if it was in an error state, false otherwise. + */ +bool +fy_document_iterator_get_error(struct fy_document_iterator *fydi) + FY_EXPORT; + +/** + * struct fy_document_builder_cfg - document builder configuration structure. + * + * Argument to the fy_document_builder_create() method + * + * @parse_cfg: Parser configuration + * @userdata: Opaque user data pointer + * @diag: Optional diagnostic interface to use + */ +struct fy_document_builder_cfg { + struct fy_parse_cfg parse_cfg; + void *userdata; + struct fy_diag *diag; +}; + +/** + * fy_document_builder_create() - Create a document builder + * + * Creates a document builder with its configuration @cfg + * The document builder may be destroyed by a corresponding call to + * fy_document_builder_destroy(). + * + * @cfg: The configuration for the document builder + * + * Returns: + * A pointer to the document builder or NULL in case of an error. + */ +struct fy_document_builder * +fy_document_builder_create(const struct fy_document_builder_cfg *cfg) + FY_EXPORT; + +/** + * fy_document_builder_create_on_parser() - Create a document builder + * pulling state from the parser + * + * Creates a document builder pulling state from the given parser + * + * @fyp: The parser to associate with + * + * Returns: + * A pointer to the document builder or NULL in case of an error. + */ +struct fy_document_builder * +fy_document_builder_create_on_parser(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_document_builder_reset() - Reset a document builder + * + * Resets a document builder without destroying it + * + * @fydb: The document builder + */ +void +fy_document_builder_reset(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_destroy() - Destroy a document builder + * + * Destroy a document builder + * + * @fydb: The document builder + */ +void +fy_document_builder_destroy(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_get_document() - Get the document of a builder + * + * Retrieve the document of a document builder. This document + * may be incomplete. If you need to take ownership use + * fy_document_builder_take_document(). + * + * @fydb: The document builder + * + * Returns: + * The document that the builder built, or NULL in case of an error + */ +struct fy_document * +fy_document_builder_get_document(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_is_in_stream() - Test document builder in stream + * + * Find out if the document builder is in 'stream' state, + * i.e. after stream start but before stream end events are generated. + * + * @fydb: The document builder + * + * Returns: + * true if in stream, false otherwise + */ +bool +fy_document_builder_is_in_stream(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_is_in_document() - Test document builder in document + * + * Find out if the document builder is in 'document' state, + * i.e. after document start but before document end events are generated. + * + * @fydb: The document builder + * + * Returns: + * true if in document, false otherwise + */ +bool +fy_document_builder_is_in_document(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_is_document_complete() - Test document builder complete + * + * Find out if the document of the builder is complete. + * If it is complete then a call to fy_document_builder_take_document() will + * transfer ownership of the document to the caller. + * + * @fydb: The document builder + * + * Returns: + * true if document complete, false otherwise + */ +bool +fy_document_builder_is_document_complete(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_take_document() - Take ownership the document of a builder + * + * Take ownership of the document of a document builder. + * The document builder's document must be complete. + * + * @fydb: The document builder + * + * Returns: + * The document that the builder built, or NULL in case of an error + */ +struct fy_document * +fy_document_builder_take_document(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_peek_document() - Peek at the document of a builder + * + * Peek at the document of a document builder. + * Ownership still remains with the builder. + * + * @fydb: The document builder + * + * Returns: + * A peek to the document that the builder built, or NULL in case of an error + */ +struct fy_document * +fy_document_builder_peek_document(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_set_in_stream() - Set the builders state in 'stream' + * + * Set the document builders state to in 'stream' + * + * @fydb: The document builder + */ +void +fy_document_builder_set_in_stream(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_set_in_document() - Set the builders state in 'document' + * + * Set the document builders state to in 'document' + * + * @fydb: The document builder + * @fyds: The document state + * @single: Single document mode + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_document_builder_set_in_document(struct fy_document_builder *fydb, struct fy_document_state *fyds, bool single) + FY_EXPORT; + +/** + * fy_document_builder_load_document() - Create a document from parser events + * + * Load a document by pumping the parser for events and then processing them + * with the builder. + * + * @fydb: The document builder + * @fyp: The parser + * + * Returns: + * The document that results from the parser, or NULL in case of an error (or EOF) + */ +struct fy_document * +fy_document_builder_load_document(struct fy_document_builder *fydb, struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parse_load_document_with_builder() - Parse a document via built-in builder + * + * Load a document by pumping the parser for events and then processing them + * with the in-parser builder. + * + * @fyp: The parser + * + * Returns: + * The document that results from the parser, or NULL in case of an error (or EOF) + */ +struct fy_document * +fy_parse_load_document_with_builder(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_document_builder_process_event() - Process an event with a builder + * + * Pump an event to a document builder for processing. + * + * @fydb: The document builder + * @fye: The event + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_document_builder_process_event(struct fy_document_builder *fydb, struct fy_event *fye) + FY_EXPORT; + +/** + * enum fy_parser_event_generator_flags - The parser event generator flags + * + * @FYPEGF_GENERATE_DOCUMENT_EVENTS: generate document events + * @FYPEGF_GENERATE_STREAM_EVENTS: generate stream events + * @FYPEGF_GENERATE_ALL_EVENTS: generate all events + */ +enum fy_parser_event_generator_flags { + FYPEGF_GENERATE_DOCUMENT_EVENTS = FY_BIT(0), + FYPEGF_GENERATE_STREAM_EVENTS = FY_BIT(1), + FYPEGF_GENERATE_ALL_EVENTS = FYPEGF_GENERATE_STREAM_EVENTS | FYPEGF_GENERATE_DOCUMENT_EVENTS, +}; + +/** + * fy_parser_set_document_iterator() - Associate a parser with a document iterator + * + * Associate a parser with a document iterator, that is instead of parsing the events + * will be generated by the document iterator. + * + * @fyp: The parser + * @flags: The event generation flags + * @fydi: The document iterator to associate + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_parser_set_document_iterator(struct fy_parser *fyp, enum fy_parser_event_generator_flags flags, + struct fy_document_iterator *fydi) + FY_EXPORT; + +/* + * The libfyaml's push-pull thread implementation + * + */ + +/* opaque types for the user */ +struct fy_thread_pool; +struct fy_thread; +struct fy_work_pool; + +/** + * typedef fy_work_exec_fn - Work exec function + * + * The callback executed on work submission + * + * @arg: The argument to the method + * + */ +typedef void (*fy_work_exec_fn)(void *arg); + +/** + * typedef fy_work_check_fn - Work check function + * + * Work checker function to decide if it's worth to + * offload to a thread. + * + * @arg: The argument to the method + * + * Returns: + * true if it should offload to thread, false otherwise + * + */ +typedef bool (*fy_work_check_fn)(const void *arg); + +/** + * struct fy_thread_work - Work submitted to a thread for execution + * + * @fn: The execution function for this work + * @arg: The argument to the fn + * @wp: Used internally, must be set to NULL on entry + * + * This is the structure describing the work submitted + * to a thread for execution. + */ +struct fy_thread_work { + fy_work_exec_fn fn; + void *arg; + struct fy_work_pool *wp; +}; + +/** + * enum fy_thread_pool_cfg_flags - Thread pool configuration flags + * + * These flags control the operation of the thread pool. + * For now only the steal mode flag is defined. + * + * @FYTPCF_STEAL_MODE: Enable steal mode for the thread pool + */ +enum fy_thread_pool_cfg_flags { + FYTPCF_STEAL_MODE = FY_BIT(0), +}; + +/** + * struct fy_thread_pool_cfg - thread pool configuration structure. + * + * Argument to the fy_thread_pool_create() method. + * + * @flags: Thread pool configuration flags + * @num_threads: Number of threads, if 0 == online CPUs + * @userdata: A userdata pointer + */ +struct fy_thread_pool_cfg { + enum fy_thread_pool_cfg_flags flags; + unsigned int num_threads; + void *userdata; +}; + +/** + * fy_thread_pool_create() - Create a thread pool + * + * Creates a thread pool with its configuration @cfg + * The thread pool may be destroyed by a corresponding call to + * fy_thread_pool_destroy(). + * + * @cfg: The configuration for the thread pool + * + * Returns: + * A pointer to the thread pool or NULL in case of an error. + */ +struct fy_thread_pool * +fy_thread_pool_create(const struct fy_thread_pool_cfg *cfg) + FY_EXPORT; + +/** + * fy_thread_pool_destroy() - Destroy the given thread pool + * + * Destroy a thread pool created earlier via fy_thread_pool_create(). + * Note that this function will block until all threads + * of the pool are destroyed. + * + * @tp: The thread pool to destroy + */ +void +fy_thread_pool_destroy(struct fy_thread_pool *tp) + FY_EXPORT; + +/** + * fy_thread_pool_get_num_threads() - Get the number of threads + * + * Returns the actual number of created threads. + * + * @tp: The thread pool + * + * Returns: + * > 0 for the number of actual threads created, + * -1 on error + */ +int +fy_thread_pool_get_num_threads(struct fy_thread_pool *tp) + FY_EXPORT; + +/** + * fy_thread_pool_get_cfg() - Get the configuration of a thread pool + * + * @tp: The thread pool + * + * Returns: + * The configuration of the thread pool + */ +const struct fy_thread_pool_cfg * +fy_thread_pool_get_cfg(struct fy_thread_pool *tp) + FY_EXPORT; + +/** + * fy_thread_reserve() - Reserve a thread from the pool. + * + * Reserve a thread from the pool and return it. + * Note this is only valid for a non-work stealing thread pool. + * You release the thread again via a call to fy_thread_unreserve. + * + * @tp: The thread pool + * + * Returns: + * A reserved thread if not NULL, NULL if no threads are available. + */ +struct fy_thread * +fy_thread_reserve(struct fy_thread_pool *tp) + FY_EXPORT; + +/** + * fy_thread_unreserve() - Unreserve a previously reserved thread + * + * Unreserve a thread previously reserved via a call to fy_thread_reserve() + * Note this is only valid for a non-work stealing thread pool. + * + * @t: The thread + */ +void +fy_thread_unreserve(struct fy_thread *t) + FY_EXPORT; + +/** + * fy_thread_submit_work() - Submit work for execution + * + * Submit work for execution. If successful the thread + * will start executing the work in parallel with the + * calling thread. You can wait for the thread to + * terminate via a call to fy_thread_wait_work(). + * The thread must have been reserved earlier via fy_thread_reserve() + * + * Note this is only valid for a non-work stealing thread pool. + * + * @t: The thread + * @work: The work + * + * Returns: + * 0 if work has been submitted, -1 otherwise. + */ +int +fy_thread_submit_work(struct fy_thread *t, struct fy_thread_work *work) + FY_EXPORT; + +/** + * fy_thread_wait_work() - Wait for completion of submitted work + * + * Wait until submitted work to the thread has finished. + * Note this is only valid for a non-work stealing thread pool. + * + * @t: The thread + * + * Returns: + * 0 if work finished, -1 on error. + */ +int +fy_thread_wait_work(struct fy_thread *t) + FY_EXPORT; + +/** + * fy_thread_work_join() - Submit works for execution and wait + * + * Submit works for possible parallel execution. If no offloading + * is possible at the time execute in the current context. + * It is possible to use in both stealing and non-stealing mode + * with the difference being that stealing mode is about 30% faster. + * + * @tp: The thread pool + * @works: Pointer to an array of works sized @work_count + * @work_count: The size of the @works array + * @check_fn: Pointer to a check function, or NULL for no checks + */ +void +fy_thread_work_join(struct fy_thread_pool *tp, + struct fy_thread_work *works, size_t work_count, + fy_work_check_fn check_fn) + FY_EXPORT; + +/** + * fy_thread_args_join() - Execute function in parallel using arguments as pointers + * + * Execute @fn possibly in parallel using the threads in the thread pool. + * The arguments of the function are provided by the args array. + * + * @tp: The thread pool + * @fn: The function to execute in parallel + * @check_fn: Pointer to a check function, or NULL for no checks + * @args: An args array sized @count of argument pointers + * @count: The count of the args array items + */ +void +fy_thread_args_join(struct fy_thread_pool *tp, + fy_work_exec_fn fn, fy_work_check_fn check_fn, + void **args, size_t count) + FY_EXPORT; + +/** + * fy_thread_arg_array_join() - Execute function in parallel using argument array + * + * Execute @fn possibly in parallel using the threads in the thread pool. + * The arguments of the function are provided by the args array. + * + * @tp: The thread pool + * @fn: The function to execute in parallel + * @check_fn: Pointer to a check function, or NULL for no checks + * @args: An args array of @argsize items + * @argsize: The size of each argument array item + * @count: The count of the args array items + */ +void +fy_thread_arg_array_join(struct fy_thread_pool *tp, + fy_work_exec_fn fn, fy_work_check_fn check_fn, + void *args, size_t argsize, size_t count) + FY_EXPORT; + +/** + * fy_thread_arg_join() - Execute function in parallel with the same argument + * + * Execute @fn possibly in parallel using the threads in the thread pool. + * The argument of the functions is the same. + * + * @tp: The thread pool + * @fn: The function to execute in parallel + * @check_fn: Pointer to a check function, or NULL for no checks + * @arg: The common argument + * @count: The count of executions + */ +void +fy_thread_arg_join(struct fy_thread_pool *tp, + fy_work_exec_fn fn, fy_work_check_fn check_fn, + void *arg, size_t count) + FY_EXPORT; + +/* + * Minimal exposing of internal BLAKE3 implementation + * + */ + +/* BLAKE3 key length */ +#define FY_BLAKE3_KEY_LEN 32 +/* BLAKE3 output length */ +#define FY_BLAKE3_OUT_LEN 32 + +/* opaque BLAKE3 hasher type for the user*/ +struct fy_blake3_hasher; + +/** + * fy_blake3_backend_iterate() - Iterate over the supported BLAKE3 backends + * + * This method iterates over the supported BLAKE3 backends. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * The default backend is always the last in sequence, so for + * example if the order is [ "portable", "sse2", NULL ] the + * default is "sse2". + * + * @prevp: The previous backend pointer, or NULL at start + * + * Returns: + * The next backend or NULL at the end. + */ +const char * +fy_blake3_backend_iterate(const char **prevp) + FY_EXPORT; + +/** + * struct fy_blake3_hasher_cfg - BLAKE3 hasher configuration + * + * Argument to the fy_blake3_hasher_create() method which + * is the fyaml's user facing BLAKE3 API. + * It is very minimal, on purpose, since it's meant to be + * exposing a full blown BLAKE3 API. + * + * @backend: NULL for default, or a specific backend name + * @file_buffer: Use this amount of buffer for buffering, zero for default + * @mmap_min_chunk: Minimum chunk size for mmap case + * @mmap_max_chunk: Maximum chunk size for mmap case + * @no_mmap: Disable mmap for file access + * @key: pointer to a FY_BLAKE3_KEY_LEN area when in keyed mode. + * NULL otherwise. + * @context: pointer to a context when in key derivation mode. + * NULL otherwise. + * @context_len: The size of the context when in key derivation mode. + * 0 otherwise. + * @tp: The thread pool to use, if NULL, create a private one + * @num_threads: Number of threads to use + * - 0 means default: NUM_CPUS * 3 / 2 + * - > 0 specific number of threads + * - -1 disable threading entirely + */ +struct fy_blake3_hasher_cfg { + const char *backend; + size_t file_buffer; + size_t mmap_min_chunk; + size_t mmap_max_chunk; + bool no_mmap; + const uint8_t *key; + const void *context; + size_t context_len; + struct fy_thread_pool *tp; + int num_threads; +}; + +/** + * fy_blake3_hasher_create() - Create a BLAKE3 hasher object. + * + * Creates a BLAKE3 hasher with its configuration @cfg + * The hasher may be destroyed by a corresponding call to + * fy_blake3_hasher_destroy(). + * + * @cfg: The configuration for the BLAKE3 hasher + * + * Returns: + * A pointer to the BLAKE3 hasher or NULL in case of an error. + */ +struct fy_blake3_hasher * +fy_blake3_hasher_create(const struct fy_blake3_hasher_cfg *cfg) + FY_EXPORT; + +/** + * fy_blake3_hasher_destroy() - Destroy the given BLAKE3 hasher + * + * Destroy a BLAKE3 hasher created earlier via fy_blake3_hasher_create(). + * + * @fyh: The BLAKE3 hasher to destroy + */ +void +fy_blake3_hasher_destroy(struct fy_blake3_hasher *fyh) + FY_EXPORT; + +/** + * fy_blake3_hasher_update() - Update the BLAKE3 hasher state with the given input + * + * Updates the BLAKE3 hasher state by hashing the given input. + * + * @fyh: The BLAKE3 hasher + * @input: Pointer to the input + * @input_len: Size of the input in bytes + */ +void +fy_blake3_hasher_update(struct fy_blake3_hasher *fyh, const void *input, size_t input_len) + FY_EXPORT; + +/** + * fy_blake3_hasher_finalize() - Finalize the hash and get output + * + * Finalizes the BLAKE3 hasher and returns the output + * + * @fyh: The BLAKE3 hasher + * + * Returns: + * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL + * in case of an error. + */ +const uint8_t * +fy_blake3_hasher_finalize(struct fy_blake3_hasher *fyh) + FY_EXPORT; + +/** + * fy_blake3_hasher_reset() - Resets the hasher + * + * Resets the hasher for re-use + * + * @fyh: The BLAKE3 hasher + */ +void +fy_blake3_hasher_reset(struct fy_blake3_hasher *fyh) + FY_EXPORT; + +/** + * fy_blake3_hash() - BLAKE3 hash a memory area + * + * Hash a memory area and return the BLAKE3 output. + * + * @fyh: The BLAKE3 hasher + * @mem: Pointer to the memory to use + * @size: The size of the memory in bytes + * + * Returns: + * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL + * in case of an error. + */ +const uint8_t * +fy_blake3_hash(struct fy_blake3_hasher *fyh, const void *mem, size_t size) + FY_EXPORT; + +/** + * fy_blake3_hash_file() - BLAKE3 hash a file. + * + * Hash the given file (possibly using mmap) + * + * @fyh: The BLAKE3 hasher + * @filename: The filename + * + * Returns: + * A pointer to the BLAKE3 output (sized FY_BLAKE3_OUT_LEN), or NULL + * in case of an error. + */ +const uint8_t * +fy_blake3_hash_file(struct fy_blake3_hasher *fyh, const char *filename) + FY_EXPORT; + +/* forward decl of allocator interfaces */ +struct fy_allocator; + +/* A tag that represents the default tag */ +#define FY_ALLOC_TAG_DEFAULT 0 +/* A tag that denotes error */ +#define FY_ALLOC_TAG_ERROR -1 +/* A tag that represents 'none' */ +#define FY_ALLOC_TAG_NONE -2 + +/** + * fy_allocator_iterate() - Iterate over available allocator names + * + * This method iterates over all the available allocator names. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * @prevp: The previous allocator iterator pointer + * + * Returns: + * The next allocator name in sequence or NULL at the end. + */ +const char * +fy_allocator_iterate(const char **prevp) + FY_EXPORT; + +/** + * fy_allocator_is_available() - Check if an allocator is available + * + * Check if the named allocator is available. + * + * @name: The name of the allocator to check + * + * Returns: + * true if the allocator is available, false otherwise + */ +bool +fy_allocator_is_available(const char *name) + FY_EXPORT; + +/** + * fy_allocator_create() - Create an allocator. + * + * Creates an allocator of the given type, using the configuration + * argument provided. + * The allocator may be destroyed by a corresponding call to + * fy_allocator_destroy(). + * + * You can retrieve the names of available allocators + * with the fy_allocator_get_names() method. + * + * @name: The name of the allocator + * @cfg: The type specific configuration for the allocator, or NULL + * for the default. + * + * Returns: + * A pointer to the allocator or NULL in case of an error. + */ +struct fy_allocator * +fy_allocator_create(const char *name, const void *cfg) + FY_EXPORT; + +/** + * fy_allocator_destroy() - Destroy the given allocator + * + * Destroy an allocator created earlier via fy_allocator_create(). + * Tracking allocators will release all memory allocated using them. + * + * @a: The allocator to destroy + */ +void +fy_allocator_destroy(struct fy_allocator *a) + FY_EXPORT; + +/** The minimum amount of memory for an inplace linear allocator */ +#define FY_LINEAR_ALLOCATOR_IN_PLACE_MIN_SIZE 256 + +/** + * fy_linear_allocator_create_in_place() - Create a linear allocator in place + * + * Creates a linear allocator in place, using the buffer provided. + * No memory allocations will be performed, so it's safe to embed. + * There is no need to call fy_allocator_destroy for this allocator. + * + * @buffer: The memory buffer to use for both storage and the allocator + * @size: The size of the memory buffer + * + * Returns: + * A pointer to the allocator, or NULL if there is no space + */ +struct fy_allocator * +fy_linear_allocator_create_in_place(void *buffer, size_t size) + FY_EXPORT; + +/** The minimum amount of memory for an inplace dedup allocator */ +#define FY_DEDUP_ALLOCATOR_IN_PLACE_MIN_SIZE 4096 + +/** + * fy_dedup_allocator_create_in_place() - Create a dedp allocator in place + * + * Creates a dedup allocator in place, using the buffer provided. + * No memory allocations will be performed, so it's safe to embed. + * There is no need to call fy_allocator_destroy for this allocator. + * The parent allocator of this will be a linear allocator. + * + * @buffer: The memory buffer to use for both storage and the allocator + * @size: The size of the memory buffer + * + * Returns: + * A pointer to the allocator, or NULL if there is no space + */ +struct fy_allocator * +fy_dedup_allocator_create_in_place(void *buffer, size_t size) + FY_EXPORT; + +/** + * fy_allocator_get_tag() - Get a tag from an allocator + * + * The allocator interface requires all allocation to belong + * to a tag. This call creates a tag and returns its value, + * or an error if not available. + * + * If an allocator only provides a single tag (like the linear + * allocator for instance), the same tag number, usually 0, is + * returned. + * + * @a: The allocator + * + * Returns: + * The created tag or -1 in case of an error. + */ +int +fy_allocator_get_tag(struct fy_allocator *a) + FY_EXPORT; + +/** + * fy_allocator_release_tag() - Release a tag from an allocator + * + * Releases a tag from an allocator and frees all memory it + * allocated (if such an operation is provided by the allocator). + * + * @a: The allocator + * @tag: The tag to release + */ +void +fy_allocator_release_tag(struct fy_allocator *a, int tag) + FY_EXPORT; + +/** + * fy_allocator_get_tag_count() - Get the maximum number of tags a + * allocator supports + * + * Get the maximum amount of tags an allocator supports. + * + * If an allocator only provides a single tag (like the linear + * allocator for instance), 1 will be returned. + * + * @a: The allocator + * + * Returns: + * The number of tags, or -1 on error + */ +int +fy_allocator_get_tag_count(struct fy_allocator *a) + FY_EXPORT; + +/** + * fy_allocator_set_tag_count() - Set the maximum number of tags a + * allocator supports + * + * Sets the maximum amount of tags an allocator supports. + * If the set allocator tag count is less than the current + * the additional tags will be released. + * + * @a: The allocator + * @count: The amount of tags the allocator should support + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_allocator_set_tag_count(struct fy_allocator *a, unsigned int count) + FY_EXPORT; + +/** + * fy_allocator_trim_tag() - Trim a tag + * + * Trim a tag, that is free any excess memory it allocator, fitting + * it to the size of the content it carries. + * Allocators that cannot perform this operation treat it as a NOP. + * + * @a: The allocator + * @tag: The tag to trim + */ +void +fy_allocator_trim_tag(struct fy_allocator *a, int tag) + FY_EXPORT; + +/** + * fy_allocator_reset_tag() - Reset a tag + * + * Reset a tag, that is free any content it carries, but do not + * release the tag. + * + * @a: The allocator + * @tag: The tag to reset + */ +void +fy_allocator_reset_tag(struct fy_allocator *a, int tag) + FY_EXPORT; + +/** + * fy_allocator_alloc() - Allocate memory from an allocator + * + * Allocate memory from the given allocator tag, satisfying the + * size and align restrictions. + * + * @a: The allocator + * @tag: The tag to allocate from + * @size: The size of the memory to allocate + * @align: The alignment of the object + * + * Returns: + * A pointer to the allocated memory or NULL + */ +void * +fy_allocator_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) + FY_EXPORT; + +/** + * fy_allocator_free() - Free memory allocated from an allocator + * + * Attempt to free the memory allocated previously by fy_allocator_alloc() + * Note that non per object tracking allocators treat this as a NOP + * + * @a: The allocator + * @tag: The tag used to allocate the memory + * @ptr: The pointer to the memory to free + */ +void +fy_allocator_free(struct fy_allocator *a, int tag, void *ptr) + FY_EXPORT; + +/** + * fy_allocator_store() - Store an object to an allocator + * + * Store an object to an allocator and return a pointer to the location + * it was stored. When using a deduplicating allocator no new allocation + * will take place and a pointer to the object already stored will be + * returned. + * + * The return pointer must not be modified, the objects stored are idempotent. + * + * @a: The allocator + * @tag: The tag used to allocate the memory + * @data: The pointer to object to store + * @size: The size of the object + * @align: The alignment restriction of the object + * + * Returns: + * A constant pointer to the object stored, or NULL in case of an error + */ +const void * +fy_allocator_store(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) + FY_EXPORT; + +/** + * fy_allocator_storev() - Store an object to an allocator (scatter gather) + * + * Store an object to an allocator and return a pointer to the location + * it was stored. When using a deduplicating allocator no new allocation + * will take place and a pointer to the object already stored will be + * returned. + * + * The object is created linearly from the scatter gather io vector provided. + * + * The return pointer must not be modified, the objects stored are immutable. + * + * @a: The allocator + * @tag: The tag used to allocate the memory from + * @iov: The I/O scatter gather vector + * @iovcnt: The number of vectors + * @align: The alignment restriction of the object + * + * Returns: + * A constant pointer to the object stored, or NULL in case of an error + */ +const void * +fy_allocator_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align) + FY_EXPORT; + +/** + * fy_allocator_lookup() - Lookup for object in an allocator. + * + * Lookup for the exact contents of an object stored in an allocator + * and return a pointer to the location it was stored. + * The allocator must have the FYACF_CAN_LOOKUP capability. + * + * @a: The allocator + * @tag: The tag used to locate the memory + * @data: The pointer to object to store + * @size: The size of the object + * @align: The alignment restriction of the object + * + * Returns: + * A constant pointer to the object stored, or NULL if the object does not exist + */ +const void * +fy_allocator_lookup(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) + FY_EXPORT; + +/** + * fy_allocator_lookupv() - Lookup for object in an allocator (scatter gather) + * + * Lookup for the exact contents of an object stored in an allocator + * and return a pointer to the location it was stored. + * The allocator must have the FYACF_CAN_LOOKUP capability. + * + * The scatter gather vector is used to recreate the object. + * + * @a: The allocator + * @tag: The tag used to search into + * @iov: The I/O scatter gather vector + * @iovcnt: The number of vectors + * @align: The alignment restriction of the object + * + * Returns: + * A constant pointer to the object stored, or NULL in case the object does not exist + */ +const void * +fy_allocator_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align) + FY_EXPORT; + +/** + * fy_allocator_dump() - Dump internal allocator state + * + * @a: The allocator + */ +void +fy_allocator_dump(struct fy_allocator *a) + FY_EXPORT; + +/** + * enum fy_allocator_cap_flags - Allocator capability flags + * + * @FYACF_CAN_FREE_INDIVIDUAL: Allocator supports freeing individual allocations + * @FYACF_CAN_FREE_TAG: Allocator supports releasing entire tags + * @FYACF_CAN_DEDUP: Allocator supports deduplication + * @FYACF_HAS_CONTAINS: Allocator can report if it contains a pointer (even if inefficiently) + * @FYACF_HAS_EFFICIENT_CONTAINS: Allocator can report if it contains a pointer (efficiently) + * @FYACF_HAS_TAGS: Allocator has individual tags or not + * @FYACF_CAN_LOOKUP: Allocator supports lookup for content + * + * These flags describe what operations an allocator supports. + */ +enum fy_allocator_cap_flags { + FYACF_CAN_FREE_INDIVIDUAL = FY_BIT(0), + FYACF_CAN_FREE_TAG = FY_BIT(1), + FYACF_CAN_DEDUP = FY_BIT(2), + FYACF_HAS_CONTAINS = FY_BIT(3), + FYACF_HAS_EFFICIENT_CONTAINS = FY_BIT(4), + FYACF_HAS_TAGS = FY_BIT(5), + FYACF_CAN_LOOKUP = FY_BIT(6), +}; + +/** + * fy_allocator_get_caps() - Get allocator capabilities + * + * Retrieve the capabilities of an allocator. + * + * @a: The allocator + * + * Returns: + * The capabilities of the allocator + */ +enum fy_allocator_cap_flags +fy_allocator_get_caps(struct fy_allocator *a) + FY_EXPORT; + +/** + * fy_allocator_contains() - Check if a allocator contains a pointer + * + * Report if an allocator contains the pointer + * + * @a: The allocator + * @tag: Tag to search in, -1 for all + * @ptr: The object pointer + * + * Returns: + * true if the pointer ptr is contained in the allocator, false otherwise + */ +bool +fy_allocator_contains(struct fy_allocator *a, int tag, const void *ptr) + FY_EXPORT; + +/** + * fy_allocator_get_tag_linear_size() - Get the linear size of an allocator tag + * + * Retrieve the linear size of the content of a tag. + * That is the size of a buffer if one was to copy the content of the tag in + * that buffer in a linear manner. + * + * @a: The allocator + * @tag: The tag + * + * Returns: + * The linear size of the content stored in the tag or -1 in case of an error. + */ +ssize_t +fy_allocator_get_tag_linear_size(struct fy_allocator *a, int tag) + FY_EXPORT; + +/** + * fy_allocator_get_tag_single_linear() - Get the linear extend of a tag + * + * If a tag stores it's content in a single linear buffer, retrieve it + * directly. This is possible only under careful arrangement of allocator + * configuration, but it is an important optimization case. + * + * @a: The allocator + * @tag: The tag + * @sizep: Pointer to a variable that will be filled with the size. + * + * Returns: + * A pointer to the linear content of the tag, or NULL if othersize. + */ +const void * +fy_allocator_get_tag_single_linear(struct fy_allocator *a, int tag, size_t *sizep) + FY_EXPORT; + +/** + * struct fy_linear_allocator_cfg - linear allocator configuration + * + * @buf: A pointer to a buffer that will be used, or NULL in order to allocate + * @size: Size of the buffer in bytes + */ +struct fy_linear_allocator_cfg { + void *buf; + size_t size; +}; + +/** + * enum fy_mremap_arena_type - The mremap allocator arena types + * + * @FYMRAT_DEFAULT: Use what's optimal for this platform + * @FYMRAT_MALLOC: Use malloc/realloc arena type (not recommended) + * @FYMRAT_MMAP: Use mmap/mremap arena type + * + */ +enum fy_mremap_arena_type { + FYMRAT_DEFAULT, + FYMRAT_MALLOC, + FYMRAT_MMAP, +}; + +/** + * struct fy_mremap_allocator_cfg - mremap allocator configuration + * + * If any of the fields is zero, then the system will provide (somewhat) + * reasonable defaults. + * + * @big_alloc_threshold: Threshold for immediately creating a new arena. + * @empty_threshold: The threshold under which an arena is moved to the full list. + * @minimum_arena_size: The minimum (and starting size) of an arena. + * @grow_ratio: The ratio which an arena will try to grow if full (>1.0) + * @balloon_ratio: The multiplier for the vm area first allocation + * @arena_type: The arena type + */ +struct fy_mremap_allocator_cfg { + size_t big_alloc_threshold; + size_t empty_threshold; + size_t minimum_arena_size; + float grow_ratio; + float balloon_ratio; + enum fy_mremap_arena_type arena_type; +}; + +/* malloc allocator has no configuration data, pass NULL */ + +/** + * struct fy_dedup_allocator_cfg - dedup allocator configuration + * + * @parent_allocator: The parent allocator (required) + * @bloom_filter_bits: Number of bits of the bloom filter (or 0 for default) + * @bucket_count_bits: Number of bits for the bucket count (or 0 for default) + * @dedup_threshold: Number of bytes over which dedup takes place (default 0=always) + * @chain_length_grow_trigger: Chain length of a bucket over which a grow takes place (or 0 for auto) + * @estimated_content_size: Estimated content size (or 0 for don't know) + * @minimum_bucket_occupancy: The minimum amount that a tag bucket must be full before + * growth is allowed (default 50%, or 0.0) + */ +struct fy_dedup_allocator_cfg { + struct fy_allocator *parent_allocator; + unsigned int bloom_filter_bits; + unsigned int bucket_count_bits; + size_t dedup_threshold; + unsigned int chain_length_grow_trigger; + size_t estimated_content_size; + float minimum_bucket_occupancy; +}; + +/** + * enum fy_auto_allocator_scenario_type - auto allocator scenario type + * + * @FYAST_PER_TAG_FREE: only per tag freeing, no individual obj free + * @FYAST_PER_TAG_FREE_DEDUP: per tag freeing, dedup obj store + * @FYAST_PER_OBJ_FREE: object freeing allowed, tag freeing still works + * @FYAST_PER_OBJ_FREE_DEDUP: per obj freeing, dedup obj store + * @FYAST_SINGLE_LINEAR_RANGE: just a single linear range, no frees at all + * @FYAST_SINGLE_LINEAR_RANGE_DEDUP: single linear range, with dedup + */ +enum fy_auto_allocator_scenario_type { + FYAST_PER_TAG_FREE, + FYAST_PER_TAG_FREE_DEDUP, + FYAST_PER_OBJ_FREE, + FYAST_PER_OBJ_FREE_DEDUP, + FYAST_SINGLE_LINEAR_RANGE, + FYAST_SINGLE_LINEAR_RANGE_DEDUP, +}; + +/** + * struct fy_auto_allocator_cfg - auto allocator configuration + * + * @scenario: Auto allocator scenario + * @estimated_max_size: Estimated max content size (or 0 for don't know) + */ +struct fy_auto_allocator_cfg { + enum fy_auto_allocator_scenario_type scenario; + size_t estimated_max_size; +}; + +struct fy_parser_checkpoint; + +/** + * fy_parser_checkpoint_create() - Create a parser checkpoint + * + * Create a checkpoint of the parser's current state. This allows you + * to save the parser state and potentially roll back to it later using + * fy_parser_rollback(). The checkpoint must be destroyed via + * fy_parser_checkpoint_destroy() when no longer needed. + * + * @fyp: The parser + * + * Returns: + * A pointer to the checkpoint, or NULL in case of an error + */ +struct fy_parser_checkpoint * +fy_parser_checkpoint_create(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_checkpoint_destroy() - Destroy a parser checkpoint + * + * Destroy a checkpoint created earlier via fy_parser_checkpoint_create(). + * + * @fypchk: The checkpoint to destroy + */ +void +fy_parser_checkpoint_destroy(struct fy_parser_checkpoint *fypchk) + FY_EXPORT; + +/** + * fy_parser_rollback() - Roll back the parser to a checkpoint + * + * Roll back the parser state to a previously created checkpoint. This allows + * you to revert the parser to an earlier state and re-parse from that point. + * The checkpoint remains valid after rollback and can be used again or + * destroyed via fy_parser_checkpoint_destroy(). + * + * @fyp: The parser + * @fypc: The checkpoint to roll back to + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_parser_rollback(struct fy_parser *fyp, struct fy_parser_checkpoint *fypc) + FY_EXPORT; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/libfyaml.pc.in b/libfyaml.pc.in new file mode 100644 index 0000000..ef73e76 --- /dev/null +++ b/libfyaml.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: libfyaml +Description: Fancy YAML 1.3 parser library +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} @ASAN_LIBS@ @PTHREAD_LIBS@ @LIBCLANG_LDFLAGS@ @LIBCLANG_LIBS@ -lfyaml +Cflags: -I${includedir} @ASAN_CFLAGS@ @PTHREAD_CFLAGS@ diff --git a/m4/ax_check_compile_flag.m4 b/m4/ax_check_compile_flag.m4 new file mode 100644 index 0000000..ca36397 --- /dev/null +++ b/m4/ax_check_compile_flag.m4 @@ -0,0 +1,74 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# INPUT gives an alternative input source to AC_COMPILE_IFELSE. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# Copyright (c) 2011 Maarten Bosmans +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 4 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_VAR_IF(CACHEVAR,yes, + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS diff --git a/m4/ax_check_define.m4 b/m4/ax_check_define.m4 new file mode 100644 index 0000000..4bc6948 --- /dev/null +++ b/m4/ax_check_define.m4 @@ -0,0 +1,92 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_define.html +# =========================================================================== +# +# SYNOPSIS +# +# AC_CHECK_DEFINE([symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# AX_CHECK_DEFINE([includes],[symbol], [ACTION-IF-FOUND], [ACTION-IF-NOT]) +# +# DESCRIPTION +# +# Complements AC_CHECK_FUNC but it does not check for a function but for a +# define to exist. Consider a usage like: +# +# AC_CHECK_DEFINE(__STRICT_ANSI__, CFLAGS="$CFLAGS -D_XOPEN_SOURCE=500") +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 8 + +AU_ALIAS([AC_CHECK_DEFINED], [AC_CHECK_DEFINE]) +AC_DEFUN([AC_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$1])dnl +AC_CACHE_CHECK([for $1 defined], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ + #ifdef $1 + int ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$2], [$3])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AU_ALIAS([AX_CHECK_DEFINED], [AX_CHECK_DEFINE]) +AC_DEFUN([AX_CHECK_DEFINE],[ +AS_VAR_PUSHDEF([ac_var],[ac_cv_defined_$2_$1])dnl +AC_CACHE_CHECK([for $2 defined in $1], ac_var, +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <$1>]], [[ + #ifdef $2 + int ok; + #else + choke me + #endif +]])],[AS_VAR_SET(ac_var, yes)],[AS_VAR_SET(ac_var, no)])) +AS_IF([test AS_VAR_GET(ac_var) != "no"], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +]) + +AC_DEFUN([AX_CHECK_FUNC], +[AS_VAR_PUSHDEF([ac_var], [ac_cv_func_$2])dnl +AC_CACHE_CHECK([for $2], ac_var, +dnl AC_LANG_FUNC_LINK_TRY +[AC_LINK_IFELSE([AC_LANG_PROGRAM([$1 + #undef $2 + char $2 ();],[ + char (*f) () = $2; + return f != $2; ])], + [AS_VAR_SET(ac_var, yes)], + [AS_VAR_SET(ac_var, no)])]) +AS_IF([test AS_VAR_GET(ac_var) = yes], [$3], [$4])dnl +AS_VAR_POPDEF([ac_var])dnl +])# AC_CHECK_FUNC diff --git a/m4/ax_check_enable_debug.m4 b/m4/ax_check_enable_debug.m4 new file mode 100644 index 0000000..7bc7710 --- /dev/null +++ b/m4/ax_check_enable_debug.m4 @@ -0,0 +1,124 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_enable_debug.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_ENABLE_DEBUG([enable by default=yes/info/profile/no], [ENABLE DEBUG VARIABLES ...], [DISABLE DEBUG VARIABLES NDEBUG ...], [IS-RELEASE]) +# +# DESCRIPTION +# +# Check for the presence of an --enable-debug option to configure, with +# the specified default value used when the option is not present. Return +# the value in the variable $ax_enable_debug. +# +# Specifying 'yes' adds '-g -O0' to the compilation flags for all +# languages. Specifying 'info' adds '-g' to the compilation flags. +# Specifying 'profile' adds '-g -pg' to the compilation flags and '-pg' to +# the linking flags. Otherwise, nothing is added. +# +# Define the variables listed in the second argument if debug is enabled, +# defaulting to no variables. Defines the variables listed in the third +# argument if debug is disabled, defaulting to NDEBUG. All lists of +# variables should be space-separated. +# +# If debug is not enabled, ensure AC_PROG_* will not add debugging flags. +# Should be invoked prior to any AC_PROG_* compiler checks. +# +# IS-RELEASE can be used to change the default to 'no' when making a +# release. Set IS-RELEASE to 'yes' or 'no' as appropriate. By default, it +# uses the value of $ax_is_release, so if you are using the AX_IS_RELEASE +# macro, there is no need to pass this parameter. +# +# AX_IS_RELEASE([git-directory]) +# AX_CHECK_ENABLE_DEBUG() +# +# LICENSE +# +# Copyright (c) 2011 Rhys Ulerich +# Copyright (c) 2014, 2015 Philip Withnall +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. + +#serial 9 + +AC_DEFUN([AX_CHECK_ENABLE_DEBUG],[ + AC_BEFORE([$0],[AC_PROG_CC])dnl + AC_BEFORE([$0],[AC_PROG_CXX])dnl + AC_BEFORE([$0],[AC_PROG_F77])dnl + AC_BEFORE([$0],[AC_PROG_FC])dnl + + AC_MSG_CHECKING(whether to enable debugging) + + ax_enable_debug_default=m4_tolower(m4_normalize(ifelse([$1],,[no],[$1]))) + ax_enable_debug_is_release=m4_tolower(m4_normalize(ifelse([$4],, + [$ax_is_release], + [$4]))) + + # If this is a release, override the default. + AS_IF([test "$ax_enable_debug_is_release" = "yes"], + [ax_enable_debug_default="no"]) + + m4_define(ax_enable_debug_vars,[m4_normalize(ifelse([$2],,,[$2]))]) + m4_define(ax_disable_debug_vars,[m4_normalize(ifelse([$3],,[NDEBUG],[$3]))]) + + AC_ARG_ENABLE(debug, + [AS_HELP_STRING([--enable-debug=]@<:@yes/info/profile/no@:>@,[compile with debugging])], + [],enable_debug=$ax_enable_debug_default) + + # empty mean debug yes + AS_IF([test "x$enable_debug" = "x"], + [enable_debug="yes"]) + + # case of debug + AS_CASE([$enable_debug], + [yes],[ + AC_MSG_RESULT(yes) + CFLAGS="${CFLAGS} -g -O0" + CXXFLAGS="${CXXFLAGS} -g -O0" + FFLAGS="${FFLAGS} -g -O0" + FCFLAGS="${FCFLAGS} -g -O0" + OBJCFLAGS="${OBJCFLAGS} -g -O0" + ], + [info],[ + AC_MSG_RESULT(info) + CFLAGS="${CFLAGS} -g" + CXXFLAGS="${CXXFLAGS} -g" + FFLAGS="${FFLAGS} -g" + FCFLAGS="${FCFLAGS} -g" + OBJCFLAGS="${OBJCFLAGS} -g" + ], + [profile],[ + AC_MSG_RESULT(profile) + CFLAGS="${CFLAGS} -g -pg" + CXXFLAGS="${CXXFLAGS} -g -pg" + FFLAGS="${FFLAGS} -g -pg" + FCFLAGS="${FCFLAGS} -g -pg" + OBJCFLAGS="${OBJCFLAGS} -g -pg" + LDFLAGS="${LDFLAGS} -pg" + ], + [ + AC_MSG_RESULT(no) + dnl Ensure AC_PROG_CC/CXX/F77/FC/OBJC will not enable debug flags + dnl by setting any unset environment flag variables + AS_IF([test "x${CFLAGS+set}" != "xset"], + [CFLAGS=""]) + AS_IF([test "x${CXXFLAGS+set}" != "xset"], + [CXXFLAGS=""]) + AS_IF([test "x${FFLAGS+set}" != "xset"], + [FFLAGS=""]) + AS_IF([test "x${FCFLAGS+set}" != "xset"], + [FCFLAGS=""]) + AS_IF([test "x${OBJCFLAGS+set}" != "xset"], + [OBJCFLAGS=""]) + ]) + + dnl Define various variables if debugging is disabled. + dnl assert.h is a NOP if NDEBUG is defined, so define it by default. + AS_IF([test "x$enable_debug" = "xyes"], + [m4_map_args_w(ax_enable_debug_vars, [AC_DEFINE(], [,[1],[Define if debugging is enabled])])], + [m4_map_args_w(ax_disable_debug_vars, [AC_DEFINE(], [,[1],[Define if debugging is disabled])])]) + ax_enable_debug=$enable_debug +]) diff --git a/m4/ax_check_flag.m4 b/m4/ax_check_flag.m4 new file mode 100644 index 0000000..52405fd --- /dev/null +++ b/m4/ax_check_flag.m4 @@ -0,0 +1,147 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_PREPROC_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) +# AX_CHECK_LINK_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) +# AX_APPEND_FLAG(FLAG, [FLAGS-VARIABLE]) +# AX_APPEND_COMPILE_FLAGS([FLAG1 FLAG2 ...], [FLAGS-VARIABLE], [EXTRA-FLAGS]) +# AX_APPEND_LINK_FLAGS([FLAG1 FLAG2 ...], [FLAGS-VARIABLE], [EXTRA-FLAGS]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's +# preprocessor/compiler/linker, or whether they give an error. (Warnings, +# however, are ignored.) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check us thus made +# with the following flags: "CFLAGS EXTRA-FLAGS FLAG". EXTRA-FLAGS can +# for example be used to force the compiler to issue an error when a bad +# flag is given. +# +# AX_APPEND_FLAG appends the FLAG to the FLAG-VARIABLE shell variable or +# the current language's flags if not specified. FLAG is not added to +# FLAG-VARIABLE if it is already in the shell variable. +# +# AX_APPEND_COMPILE_FLAGS checks for each FLAG1, FLAG2, etc. using +# AX_CHECK_COMPILE_FLAG and if the check is successful the flag is added +# to the appropriate FLAGS variable with AX_APPEND_FLAG. The +# FLAGS-VARIABLE and EXTRA-FLAGS arguments are the same as in the other +# macros. AX_APPEND_LINK_FLAGS does the same for linker flags. +# +# NOTE: Based on AX_CHECK_COMPILER_FLAGS and AX_CFLAGS_GCC_OPTION. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# Copyright (c) 2009 Steven G. Johnson +# Copyright (c) 2009 Matteo Frigo +# Copyright (c) 2011 Maarten Bosmans +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 2 + +AC_DEFUN([AX_CHECK_PREPROC_FLAG], +[AC_PREREQ(2.59) dnl for _AC_LANG_PREFIX +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]cppflags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG preprocessor accepts $1], CACHEVAR, [ + ax_check_save_flags=$CPPFLAGS + CPPFLAGS="$CPPFLAGS $4 $1" + AC_PREPROC_IFELSE([AC_LANG_PROGRAM()], + [AS_VAR_SET([CACHEVAR],[yes])], + [AS_VAR_SET([CACHEVAR],[no])]) + CPPFLAGS=$ax_check_save_flags]) +AS_VAR_IF([CACHEVAR], "yes", + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_PREPROC_FLAGS + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.59) dnl for _AC_LANG_PREFIX +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], + [AS_VAR_SET([CACHEVAR],[yes])], + [AS_VAR_SET([CACHEVAR],[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_VAR_IF([CACHEVAR], "yes", + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS + +AC_DEFUN([AX_CHECK_LINK_FLAG], +[AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_ldflags_$4_$1])dnl +AC_CACHE_CHECK([whether the linker accepts $1], CACHEVAR, [ + ax_check_save_flags=$LDFLAGS + LDFLAGS="$LDFLAGS $4 $1" + AC_LINK_IFELSE([AC_LANG_PROGRAM()], + [AS_VAR_SET([CACHEVAR],[yes])], + [AS_VAR_SET([CACHEVAR],[no])]) + LDFLAGS=$ax_check_save_flags]) +AS_VAR_IF([CACHEVAR], "yes", + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_LINK_FLAGS + + +AC_DEFUN([AX_APPEND_FLAG], +[AC_PREREQ(2.59) dnl for _AC_LANG_PREFIX +AC_REQUIRE([AC_PROG_GREP]) +AS_VAR_PUSHDEF([FLAGS], [m4_default($2,_AC_LANG_PREFIX[]FLAGS)])dnl +AS_VAR_SET_IF([FLAGS], + [AS_IF([AS_ECHO(" $[]FLAGS ") | $GREP " $1 " 2>&1 >/dev/null], + [AC_RUN_LOG([: FLAGS already contains $1])], + [AC_RUN_LOG([: FLAGS="$FLAGS $1"]) + AS_VAR_APPEND([FLAGS], [" $1"])])], + [AS_VAR_SET([FLAGS],[$1])]) +AS_VAR_POPDEF([FLAGS])dnl +])dnl AX_APPEND_FLAG + +AC_DEFUN([AX_APPEND_COMPILE_FLAGS], +[for flag in $1; do + AX_CHECK_COMPILE_FLAG([$flag], [AX_APPEND_FLAG([$flag], [$2])], [], [$3]) +done +])dnl AX_APPEND_COMPILE_FLAGS + +AC_DEFUN([AX_APPEND_LINK_FLAGS], +[for flag in $1; do + AX_CHECK_LINK_FLAG([$flag], [AX_APPEND_FLAG([$flag], [m4_default([$2], [LDFLAGS])])], [], [$3]) +done +])dnl AX_APPEND_LINK_FLAGS diff --git a/m4/ax_cxx_compile_stdcxx.m4 b/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 0000000..079e17d --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,558 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXXFLAGS to +# enable support. VERSION may be '11' (for the C++11 standard) or '14' +# (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik +# Copyright (c) 2012 Zack Weinberg +# Copyright (c) 2013 Roy Stogner +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov +# Copyright (c) 2015 Paul Norman +# Copyright (c) 2015 Moritz Klammler +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 1 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [], + [$1], [14], [], + [$1], [17], [m4_fatal([support for C++17 not yet implemented in AX_CXX_COMPILE_STDCXX])], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, + ax_cv_cxx_compile_cxx$1, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [ax_cv_cxx_compile_cxx$1=yes], + [ax_cv_cxx_compile_cxx$1=no])]) + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then + ac_success=yes + fi + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for switch in -std=gnu++$1 -std=gnu++0x; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXXFLAGS="$ac_save_CXXFLAGS"]) + if eval test x\$$cachevar = xyes; then + CXXFLAGS="$CXXFLAGS $switch" + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for switch in -std=c++$1 -std=c++0x +std=c++$1 "-h std=c++$1"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXXFLAGS="$ac_save_CXXFLAGS"]) + if eval test x\$$cachevar = xyes; then + CXXFLAGS="$CXXFLAGS $switch" + ac_success=yes + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + else + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + + AC_SUBST(HAVE_CXX$1) + fi +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual void f() {} + }; + + struct Derived : public Base + { + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check single_type; + typedef check> double_type; + typedef check>> triple_type; + typedef check>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template + struct sum; + + template + struct sum + { + static constexpr auto value = N0 + sum::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template + using member = typename T::member_type; + + template + void func(...) {} + + template + void func(member*) {} + + void test(); + + void test() { func(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_seperators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same::value, ""); + static_assert(is_same::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) diff --git a/m4/ax_cxx_compile_stdcxx_11.m4 b/m4/ax_cxx_compile_stdcxx_11.m4 new file mode 100644 index 0000000..09db383 --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx_11.m4 @@ -0,0 +1,39 @@ +# ============================================================================ +# http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_11.html +# ============================================================================ +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX_11([ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the C++11 +# standard; if necessary, add switches to CXXFLAGS to enable support. +# +# This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX +# macro with the version set to C++11. The two optional arguments are +# forwarded literally as the second and third argument respectively. +# Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for +# more information. If you want to use this macro, you also need to +# download the ax_cxx_compile_stdcxx.m4 file. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik +# Copyright (c) 2012 Zack Weinberg +# Copyright (c) 2013 Roy Stogner +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov +# Copyright (c) 2015 Paul Norman +# Copyright (c) 2015 Moritz Klammler +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 14 + +include([ax_cxx_compile_stdcxx.m4]) + +AC_DEFUN([AX_CXX_COMPILE_STDCXX_11], [AX_CXX_COMPILE_STDCXX([11], [$1], [$2])]) diff --git a/m4/ax_define_dir.m4 b/m4/ax_define_dir.m4 new file mode 100644 index 0000000..b74d155 --- /dev/null +++ b/m4/ax_define_dir.m4 @@ -0,0 +1,49 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_define_dir.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_DEFINE_DIR(VARNAME, DIR [, DESCRIPTION]) +# +# DESCRIPTION +# +# This macro sets VARNAME to the expansion of the DIR variable, taking +# care of fixing up ${prefix} and such. +# +# VARNAME is then offered as both an output variable and a C preprocessor +# symbol. +# +# Example: +# +# AX_DEFINE_DIR([DATADIR], [datadir], [Where data are placed to.]) +# +# LICENSE +# +# Copyright (c) 2008 Stepan Kasal +# Copyright (c) 2008 Andreas Schwab +# Copyright (c) 2008 Guido U. Draheim +# Copyright (c) 2008 Alexandre Oliva +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 6 + +AU_ALIAS([AC_DEFINE_DIR], [AX_DEFINE_DIR]) +AC_DEFUN([AX_DEFINE_DIR], [ + prefix_NONE= + exec_prefix_NONE= + test "x$prefix" = xNONE && prefix_NONE=yes && prefix=$ac_default_prefix + test "x$exec_prefix" = xNONE && exec_prefix_NONE=yes && exec_prefix=$prefix +dnl In Autoconf 2.60, ${datadir} refers to ${datarootdir}, which in turn +dnl refers to ${prefix}. Thus we have to use `eval' twice. + eval ax_define_dir="\"[$]$2\"" + eval ax_define_dir="\"$ax_define_dir\"" + AC_SUBST($1, "$ax_define_dir") + AC_DEFINE_UNQUOTED($1, "$ax_define_dir", [$3]) + test "$prefix_NONE" && prefix=NONE + test "$exec_prefix_NONE" && exec_prefix=NONE +]) diff --git a/m4/ax_is_release.m4 b/m4/ax_is_release.m4 new file mode 100644 index 0000000..3a62609 --- /dev/null +++ b/m4/ax_is_release.m4 @@ -0,0 +1,81 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_is_release.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_IS_RELEASE(POLICY) +# +# DESCRIPTION +# +# Determine whether the code is being configured as a release, or from +# git. Set the ax_is_release variable to 'yes' or 'no'. +# +# If building a release version, it is recommended that the configure +# script disable compiler errors and debug features, by conditionalising +# them on the ax_is_release variable. If building from git, these +# features should be enabled. +# +# The POLICY parameter specifies how ax_is_release is determined. It can +# take the following values: +# +# * git-directory: ax_is_release will be 'no' if a '.git' +# directory or git worktree exists +# * minor-version: ax_is_release will be 'no' if the minor version number +# in $PACKAGE_VERSION is odd; this assumes +# $PACKAGE_VERSION follows the 'major.minor.micro' scheme +# * micro-version: ax_is_release will be 'no' if the micro version number +# in $PACKAGE_VERSION is odd; this assumes +# $PACKAGE_VERSION follows the 'major.minor.micro' scheme +# * dash-version: ax_is_release will be 'no' if there is a dash '-' +# in $PACKAGE_VERSION, for example 1.2-pre3, 1.2.42-a8b9 +# or 2.0-dirty (in particular this is suitable for use +# with git-version-gen) +# * always: ax_is_release will always be 'yes' +# * never: ax_is_release will always be 'no' +# +# Other policies may be added in future. +# +# LICENSE +# +# Copyright (c) 2015 Philip Withnall +# Copyright (c) 2016 Collabora Ltd. +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. + +#serial 8 + +AC_DEFUN([AX_IS_RELEASE],[ + AC_BEFORE([AC_INIT],[$0]) + + m4_case([$1], + [git-directory],[ + # $is_release = (.git directory does not exist) + AS_IF([test -d ${srcdir}/.git || (test -f ${srcdir}/.git && grep \.git/worktrees ${srcdir}/.git)],[ax_is_release=no],[ax_is_release=yes]) + ], + [minor-version],[ + # $is_release = ($minor_version is even) + minor_version=`echo "$PACKAGE_VERSION" | sed 's/[[^.]][[^.]]*.\([[^.]][[^.]]*\).*/\1/'` + AS_IF([test "$(( $minor_version % 2 ))" -ne 0], + [ax_is_release=no],[ax_is_release=yes]) + ], + [micro-version],[ + # $is_release = ($micro_version is even) + micro_version=`echo "$PACKAGE_VERSION" | sed 's/[[^.]]*\.[[^.]]*\.\([[^.]]*\).*/\1/'` + AS_IF([test "$(( $micro_version % 2 ))" -ne 0], + [ax_is_release=no],[ax_is_release=yes]) + ], + [dash-version],[ + # $is_release = ($PACKAGE_VERSION has a dash) + AS_CASE([$PACKAGE_VERSION], + [*-*], [ax_is_release=no], + [*], [ax_is_release=yes]) + ], + [always],[ax_is_release=yes], + [never],[ax_is_release=no], + [ + AC_MSG_ERROR([Invalid policy. Valid policies: git-directory, minor-version, micro-version, dash-version, always, never.]) + ]) +]) diff --git a/m4/ax_pthread.m4 b/m4/ax_pthread.m4 new file mode 100644 index 0000000..6d400ed --- /dev/null +++ b/m4/ax_pthread.m4 @@ -0,0 +1,317 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_pthread.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_PTHREAD([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]]) +# +# DESCRIPTION +# +# This macro figures out how to build C programs using POSIX threads. It +# sets the PTHREAD_LIBS output variable to the threads library and linker +# flags, and the PTHREAD_CFLAGS output variable to any special C compiler +# flags that are needed. (The user can also force certain compiler +# flags/libs to be tested by setting these environment variables.) +# +# Also sets PTHREAD_CC to any special C compiler that is needed for +# multi-threaded programs (defaults to the value of CC otherwise). (This +# is necessary on AIX to use the special cc_r compiler alias.) +# +# NOTE: You are assumed to not only compile your program with these flags, +# but also link it with them as well. e.g. you should link with +# $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS $LIBS +# +# If you are only building threads programs, you may wish to use these +# variables in your default LIBS, CFLAGS, and CC: +# +# LIBS="$PTHREAD_LIBS $LIBS" +# CFLAGS="$CFLAGS $PTHREAD_CFLAGS" +# CC="$PTHREAD_CC" +# +# In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute constant +# has a nonstandard name, defines PTHREAD_CREATE_JOINABLE to that name +# (e.g. PTHREAD_CREATE_UNDETACHED on AIX). +# +# Also HAVE_PTHREAD_PRIO_INHERIT is defined if pthread is found and the +# PTHREAD_PRIO_INHERIT symbol is defined when compiling with +# PTHREAD_CFLAGS. +# +# ACTION-IF-FOUND is a list of shell commands to run if a threads library +# is found, and ACTION-IF-NOT-FOUND is a list of commands to run it if it +# is not found. If ACTION-IF-FOUND is not specified, the default action +# will define HAVE_PTHREAD. +# +# Please let the authors know if this macro fails on any platform, or if +# you have any other suggestions or comments. This macro was based on work +# by SGJ on autoconf scripts for FFTW (http://www.fftw.org/) (with help +# from M. Frigo), as well as ac_pthread and hb_pthread macros posted by +# Alejandro Forero Cuervo to the autoconf macro repository. We are also +# grateful for the helpful feedback of numerous users. +# +# Updated for Autoconf 2.68 by Daniel Richard G. +# +# LICENSE +# +# Copyright (c) 2008 Steven G. Johnson +# Copyright (c) 2011 Daniel Richard G. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 20 + +AU_ALIAS([ACX_PTHREAD], [AX_PTHREAD]) +AC_DEFUN([AX_PTHREAD], [ +AC_REQUIRE([AC_CANONICAL_HOST]) +AC_LANG_PUSH([C]) +ax_pthread_ok=no + +# We used to check for pthread.h first, but this fails if pthread.h +# requires special compiler flags (e.g. on True64 or Sequent). +# It gets checked for in the link test anyway. + +# First of all, check if the user has set any of the PTHREAD_LIBS, +# etcetera environment variables, and if threads linking works using +# them: +if test x"$PTHREAD_LIBS$PTHREAD_CFLAGS" != x; then + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + save_LIBS="$LIBS" + LIBS="$PTHREAD_LIBS $LIBS" + AC_MSG_CHECKING([for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS]) + AC_TRY_LINK_FUNC(pthread_join, ax_pthread_ok=yes) + AC_MSG_RESULT($ax_pthread_ok) + if test x"$ax_pthread_ok" = xno; then + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" + fi + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" +fi + +# We must check for the threads library under a number of different +# names; the ordering is very important because some systems +# (e.g. DEC) have both -lpthread and -lpthreads, where one of the +# libraries is broken (non-POSIX). + +# Create a list of thread flags to try. Items starting with a "-" are +# C compiler flags, and other items are library names, except for "none" +# which indicates that we try without any flags at all, and "pthread-config" +# which is a program returning the flags for the Pth emulation library. + +ax_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config" + +# The ordering *is* (sometimes) important. Some notes on the +# individual items follow: + +# pthreads: AIX (must check this before -lpthread) +# none: in case threads are in libc; should be tried before -Kthread and +# other compiler flags to prevent continual compiler warnings +# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h) +# -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able) +# lthread: LinuxThreads port on FreeBSD (also preferred to -pthread) +# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads) +# -pthreads: Solaris/gcc +# -mthreads: Mingw32/gcc, Lynx/gcc +# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it +# doesn't hurt to check since this sometimes defines pthreads too; +# also defines -D_REENTRANT) +# ... -mt is also the pthreads flag for HP/aCC +# pthread: Linux, etcetera +# --thread-safe: KAI C++ +# pthread-config: use pthread-config program (for GNU Pth library) + +case ${host_os} in + solaris*) + + # On Solaris (at least, for some versions), libc contains stubbed + # (non-functional) versions of the pthreads routines, so link-based + # tests will erroneously succeed. (We need to link with -pthreads/-mt/ + # -lpthread.) (The stubs are missing pthread_cleanup_push, or rather + # a function called by this macro, so we could check for that, but + # who knows whether they'll stub that too in a future libc.) So, + # we'll just look for -pthreads and -lpthread first: + + ax_pthread_flags="-pthreads pthread -mt -pthread $ax_pthread_flags" + ;; + + darwin*) + ax_pthread_flags="-pthread $ax_pthread_flags" + ;; +esac + +if test x"$ax_pthread_ok" = xno; then +for flag in $ax_pthread_flags; do + + case $flag in + none) + AC_MSG_CHECKING([whether pthreads work without any flags]) + ;; + + -*) + AC_MSG_CHECKING([whether pthreads work with $flag]) + PTHREAD_CFLAGS="$flag" + ;; + + pthread-config) + AC_CHECK_PROG(ax_pthread_config, pthread-config, yes, no) + if test x"$ax_pthread_config" = xno; then continue; fi + PTHREAD_CFLAGS="`pthread-config --cflags`" + PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" + ;; + + *) + AC_MSG_CHECKING([for the pthreads library -l$flag]) + PTHREAD_LIBS="-l$flag" + ;; + esac + + save_LIBS="$LIBS" + save_CFLAGS="$CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + + # Check for various functions. We must include pthread.h, + # since some functions may be macros. (On the Sequent, we + # need a special flag -Kthread to make this header compile.) + # We check for pthread_join because it is in -lpthread on IRIX + # while pthread_create is in libc. We check for pthread_attr_init + # due to DEC craziness with -lpthreads. We check for + # pthread_cleanup_push because it is one of the few pthread + # functions on Solaris that doesn't have a non-functional libc stub. + # We try pthread_create on general principles. + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include + static void routine(void *a) { a = 0; } + static void *start_routine(void *a) { return a; }], + [pthread_t th; pthread_attr_t attr; + pthread_create(&th, 0, start_routine, 0); + pthread_join(th, 0); + pthread_attr_init(&attr); + pthread_cleanup_push(routine, 0); + pthread_cleanup_pop(0) /* ; */])], + [ax_pthread_ok=yes], + []) + + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" + + AC_MSG_RESULT($ax_pthread_ok) + if test "x$ax_pthread_ok" = xyes; then + break; + fi + + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" +done +fi + +# Various other checks: +if test "x$ax_pthread_ok" = xyes; then + save_LIBS="$LIBS" + LIBS="$PTHREAD_LIBS $LIBS" + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + + # Detect AIX lossage: JOINABLE attribute is called UNDETACHED. + AC_MSG_CHECKING([for joinable pthread attribute]) + attr_name=unknown + for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], + [int attr = $attr; return attr /* ; */])], + [attr_name=$attr; break], + []) + done + AC_MSG_RESULT($attr_name) + if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then + AC_DEFINE_UNQUOTED(PTHREAD_CREATE_JOINABLE, $attr_name, + [Define to necessary symbol if this constant + uses a non-standard name on your system.]) + fi + + AC_MSG_CHECKING([if more special flags are required for pthreads]) + flag=no + case ${host_os} in + aix* | freebsd* | darwin*) flag="-D_THREAD_SAFE";; + osf* | hpux*) flag="-D_REENTRANT";; + solaris*) + if test "$GCC" = "yes"; then + flag="-D_REENTRANT" + else + flag="-mt -D_REENTRANT" + fi + ;; + esac + AC_MSG_RESULT(${flag}) + if test "x$flag" != xno; then + PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS" + fi + + AC_CACHE_CHECK([for PTHREAD_PRIO_INHERIT], + ax_cv_PTHREAD_PRIO_INHERIT, [ + AC_LINK_IFELSE([ + AC_LANG_PROGRAM([[#include ]], [[int i = PTHREAD_PRIO_INHERIT;]])], + [ax_cv_PTHREAD_PRIO_INHERIT=yes], + [ax_cv_PTHREAD_PRIO_INHERIT=no]) + ]) + AS_IF([test "x$ax_cv_PTHREAD_PRIO_INHERIT" = "xyes"], + AC_DEFINE([HAVE_PTHREAD_PRIO_INHERIT], 1, [Have PTHREAD_PRIO_INHERIT.])) + + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" + + # More AIX lossage: compile with *_r variant + if test "x$GCC" != xyes; then + case $host_os in + aix*) + AS_CASE(["x/$CC"], + [x*/c89|x*/c89_128|x*/c99|x*/c99_128|x*/cc|x*/cc128|x*/xlc|x*/xlc_v6|x*/xlc128|x*/xlc128_v6], + [#handle absolute path differently from PATH based program lookup + AS_CASE(["x$CC"], + [x/*], + [AS_IF([AS_EXECUTABLE_P([${CC}_r])],[PTHREAD_CC="${CC}_r"])], + [AC_CHECK_PROGS([PTHREAD_CC],[${CC}_r],[$CC])])]) + ;; + esac + fi +fi + +test -n "$PTHREAD_CC" || PTHREAD_CC="$CC" + +AC_SUBST(PTHREAD_LIBS) +AC_SUBST(PTHREAD_CFLAGS) +AC_SUBST(PTHREAD_CC) + +# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: +if test x"$ax_pthread_ok" = xyes; then + ifelse([$1],,AC_DEFINE(HAVE_PTHREAD,1,[Define if you have POSIX threads libraries and header files.]),[$1]) + : +else + ax_pthread_ok=no + $2 +fi +AC_LANG_POP +])dnl AX_PTHREAD diff --git a/m4/ax_tls.m4 b/m4/ax_tls.m4 new file mode 100644 index 0000000..033e3b1 --- /dev/null +++ b/m4/ax_tls.m4 @@ -0,0 +1,76 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_tls.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_TLS([action-if-found], [action-if-not-found]) +# +# DESCRIPTION +# +# Provides a test for the compiler support of thread local storage (TLS) +# extensions. Defines TLS if it is found. Currently knows about GCC/ICC +# and MSVC. I think SunPro uses the same as GCC, and Borland apparently +# supports either. +# +# LICENSE +# +# Copyright (c) 2008 Alan Woodland +# Copyright (c) 2010 Diego Elio Petteno` +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 10 + +AC_DEFUN([AX_TLS], [ + AC_MSG_CHECKING(for thread local storage (TLS) class) + AC_CACHE_VAL(ac_cv_tls, [ + ax_tls_keywords="__thread __declspec(thread) none" + for ax_tls_keyword in $ax_tls_keywords; do + AS_CASE([$ax_tls_keyword], + [none], [ac_cv_tls=none ; break], + [AC_TRY_COMPILE( + [#include + static void + foo(void) { + static ] $ax_tls_keyword [ int bar; + exit(1); + }], + [], + [ac_cv_tls=$ax_tls_keyword ; break], + ac_cv_tls=none + )]) + done + ]) + AC_MSG_RESULT($ac_cv_tls) + + AS_IF([test "$ac_cv_tls" != "none"], + AC_DEFINE_UNQUOTED([TLS], $ac_cv_tls, [If the compiler supports a TLS storage class define it to that here]) + m4_ifnblank([$1], [$1]), + m4_ifnblank([$2], [$2]) + ) +]) diff --git a/m4/shave.m4 b/m4/shave.m4 new file mode 100644 index 0000000..f2da7d8 --- /dev/null +++ b/m4/shave.m4 @@ -0,0 +1,111 @@ +dnl Make automake/libtool output more friendly to humans +dnl +dnl Copyright (c) 2009, Damien Lespiau +dnl +dnl Permission is hereby granted, free of charge, to any person +dnl obtaining a copy of this software and associated documentation +dnl files (the "Software"), to deal in the Software without +dnl restriction, including without limitation the rights to use, +dnl copy, modify, merge, publish, distribute, sublicense, and/or sell +dnl copies of the Software, and to permit persons to whom the +dnl Software is furnished to do so, subject to the following +dnl conditions: +dnl +dnl The above copyright notice and this permission notice shall be +dnl included in all copies or substantial portions of the Software. +dnl +dnl THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +dnl EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +dnl OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +dnl NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +dnl HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +dnl WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +dnl FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +dnl OTHER DEALINGS IN THE SOFTWARE. +dnl +dnl SHAVE_INIT([shavedir],[default_mode]) +dnl +dnl shavedir: the directory where the shave scripts are, it defaults to +dnl $(top_builddir) +dnl default_mode: (enable|disable) default shave mode. This parameter +dnl controls shave's behaviour when no option has been +dnl given to configure. It defaults to disable. +dnl +dnl * SHAVE_INIT should be called late in your configure.(ac|in) file (just +dnl before AC_CONFIG_FILE/AC_OUTPUT is perfect. This macro rewrites CC and +dnl LIBTOOL, you don't want the configure tests to have these variables +dnl re-defined. +dnl * This macro requires GNU make's -s option. + +AC_DEFUN([_SHAVE_ARG_ENABLE], +[ + AC_ARG_ENABLE([shave], + AS_HELP_STRING( + [--enable-shave], + [use shave to make the build pretty [[default=$1]]]),, + [enable_shave=$1] + ) +]) + +AC_DEFUN([SHAVE_INIT], +[ + dnl you can tweak the default value of enable_shave + m4_if([$2], [enable], [_SHAVE_ARG_ENABLE(yes)], [_SHAVE_ARG_ENABLE(no)]) + + if test x"$enable_shave" = xyes; then + dnl where can we find the shave scripts? + m4_if([$1],, + [shavedir="$ac_pwd"], + [shavedir="$ac_pwd/$1"]) + AC_SUBST(shavedir) + + dnl make is now quiet + AC_SUBST([MAKEFLAGS], [-s]) + AC_SUBST([AM_MAKEFLAGS], ['`test -z $V && echo -s`']) + + dnl we need sed + AC_CHECK_PROG(SED,sed,sed,false) + + dnl substitute libtool + SHAVE_SAVED_LIBTOOL=$LIBTOOL + LIBTOOL="${SHELL} ${shavedir}/shave-libtool '${SHAVE_SAVED_LIBTOOL}'" + AC_SUBST(LIBTOOL) + + dnl substitute cc/cxx + SHAVE_SAVED_CCAS=$CCAS + SHAVE_SAVED_CC=$CC + SHAVE_SAVED_CXX=$CXX + SHAVE_SAVED_FC=$FC + SHAVE_SAVED_F77=$F77 + SHAVE_SAVED_OBJC=$OBJC + SHAVE_SAVED_MCS=$MCS + SHAVE_SAVED_LEX=$LEX + SHAVE_SAVED_YACC=$YACC + CCAS="${SHELL} ${shavedir}/shave ccas ${SHAVE_SAVED_CCAS}" + CC="${SHELL} ${shavedir}/shave cc ${SHAVE_SAVED_CC}" + CXX="${SHELL} ${shavedir}/shave cxx ${SHAVE_SAVED_CXX}" + FC="${SHELL} ${shavedir}/shave fc ${SHAVE_SAVED_FC}" + F77="${SHELL} ${shavedir}/shave f77 ${SHAVE_SAVED_F77}" + OBJC="${SHELL} ${shavedir}/shave objc ${SHAVE_SAVED_OBJC}" + MCS="${SHELL} ${shavedir}/shave mcs ${SHAVE_SAVED_MCS}" + LEX="${SHELL} ${shavedir}/shave lex ${SHAVE_SAVED_LEX}" + YACC="${SHELL} ${shavedir}/shave yacc ${SHAVE_SAVED_YACC}" + AC_SUBST(CCAS) + AC_SUBST(CC) + AC_SUBST(CXX) + AC_SUBST(FC) + AC_SUBST(F77) + AC_SUBST(OBJC) + AC_SUBST(MCS) + AC_SUBST(LEX) + AC_SUBST(YACC) + + V=@ + else + V=1 + fi + Q='$(V:1=)' + AC_SUBST(V) + AC_SUBST(Q) +]) + diff --git a/scripts/create-virtual-env b/scripts/create-virtual-env new file mode 100644 index 0000000..0d3595d --- /dev/null +++ b/scripts/create-virtual-env @@ -0,0 +1,6 @@ +virtualenv -p python3 sphinx +. sphinx/bin/activate +pip3 install sphinx +pip3 install git+http://github.com/return42/linuxdoc.git +pip3 install sphinx_rtd_theme +pip3 install sphinx-markdown-builder diff --git a/scripts/get-gpg-key-id-by-email.sh b/scripts/get-gpg-key-id-by-email.sh new file mode 100755 index 0000000..6c551f4 --- /dev/null +++ b/scripts/get-gpg-key-id-by-email.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# either use the supplied email address, or the current git user's email +if [ "x$1" != "x" ] ; then + email="$1" +else + email=`git config user.email` +fi +last="" +id="" +gpg --list-secret-keys | \ + while read line; do + if [[ $line =~ ^uid\ .* ]] ; then + temail=`echo $line | cut -d'<' -f2 | cut -d'>' -f1` + if [[ "$email" == "$temail" && $last =~ ^sec\ .* ]]; then + echo $last | cut -d/ -f 2 | cut -d' ' -f1 + fi + fi + last="$line" + done diff --git a/scripts/install-linuxdoc.sh b/scripts/install-linuxdoc.sh new file mode 100755 index 0000000..7a64900 --- /dev/null +++ b/scripts/install-linuxdoc.sh @@ -0,0 +1,2 @@ +#!/bin/sh +pip3 install --user git+http://github.com/return42/linuxdoc.git diff --git a/scripts/run-check-errors.sh b/scripts/run-check-errors.sh new file mode 100755 index 0000000..72a0e95 --- /dev/null +++ b/scripts/run-check-errors.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +ERROR=1 +PARSER="./src/libfyaml-parser -mtestsuite" +FILE= + +while true; do + case "$1" in + -- ) shift; break ;; + + --errors | -e ) + ERROR=1 + PASS=0 + shift ;; + + --file | -f ) + shift + FILE=$1 + shift ;; + + * ) break ;; + esac +done + +declare -a rf + +# check tty mode (whether we can colorize) +if [ -t 1 ]; then + GRN="\e[32m" + RED="\e[31m" + GRY="\e[37m" + YLW="\e[33m" + CYN="\e[36m" + NRM="\e[0m" +else + GRN="" + RED="" + GRY="" + YLW="" + CYN="" + NRM="" +fi + +rf=() +i=0 + +if [ "x$FILE" = "x" ]; then + for prefix in $*; do + echo "Collecting tests from $prefix" + while read full ; do + + # full-path:prefix:path-without-prefix:path-without-prefix-filename + relative=`echo $full | sed -e "s#^$prefix##g"` + file=`echo $relative | sed -e "s#/in.yaml##g" | tr '/' '-'` + rf+=( "$full:$prefix:$relative:$file" ) + # echo ${rf[$i]} + i=$(($i + 1)) + done < <(find "$prefix" -name "in.yaml" -print | sort) + done +else + echo "reading tests from $FILE" + DIR=`echo $1 | sed -e 's#/$##g'` + for dir in `cat $FILE`; do + prefix="$DIR/$dir" + + # only if directory exists + if [ ! -d "$prefix" ]; then + continue + fi + + while read full ; do + + # full-path:prefix:path-without-prefix:path-without-prefix-filename + relative=`echo $full | sed -e "s#^$DIR##g"` + file=`echo $relative | sed -e "s#/in.yaml##g" | sed -e "s#^/##g"` + rf+=( "$full:$1:$relative:$file" ) + # echo ${rf[$i]} + i=$(($i + 1)) + done < <(find "$prefix" -name "in.yaml" -print | sort) + done +fi + +count=${#rf[@]} + +for (( i=0; i < $count; i++)); do + v="${rf[$i]}" + full=`echo $v | cut -d: -f1` + prefix=`echo $v | cut -d: -f2` + relative=`echo $v | cut -d: -f3` + file=`echo $v | cut -d: -f4` + # echo "$i: full=$full prefix=$prefix relative=$relative file=$file" + + has_yaml=1 + + desc=`echo $full | sed -e 's#in.yaml#===#'` + desctxt="" + if [ -e "$desc" ]; then + descf=`realpath "$desc"` + desctxt=`cat 2>/dev/null "$descf"` + has_desc=1 + else + has_desc=0 + fi + + errf=`echo $full | sed -e 's#in.yaml#error#'` + if [ -e "$errf" ] ; then + expected_error="1" + has_error=1 + else + expected_error="0" + has_error=0 + continue + fi + + if [ $expected_error == 0 ]; then + col="$GRN" + else + col="$RED" + fi + echo -e "$col$file$NRM: $desctxt" + + ${PARSER} >/dev/null "$full" + + echo +done diff --git a/scripts/run-compare-dump.sh b/scripts/run-compare-dump.sh new file mode 100755 index 0000000..0de6567 --- /dev/null +++ b/scripts/run-compare-dump.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# VALGRIND="" +VALGRIND="valgrind" +BN=`basename $1` +ON="output/$BN" +mkdir -p output +a="$ON.libyaml.dump" +b="$ON.libfyaml.dump" +./src/libfyaml-parser -mlibyaml-dump "$1" >"$a" +./src/libfyaml-parser -mdump "$1" >"$b" +diff -au "$a" "$b" | tee "$ON.dump.diff" +./src/libfyaml-parser -mscan -d0 "$1" >"$ON.dump.log" 2>&1 diff --git a/scripts/run-compare-examples.sh b/scripts/run-compare-examples.sh new file mode 100755 index 0000000..dafbbc5 --- /dev/null +++ b/scripts/run-compare-examples.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +for i in $*; do + echo "checking: $i" + ./run-compare-scan.sh $i + ./run-compare-parse.sh $i + echo +done diff --git a/scripts/run-compare-parse.sh b/scripts/run-compare-parse.sh new file mode 100755 index 0000000..0410ed9 --- /dev/null +++ b/scripts/run-compare-parse.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# VALGRIND="" +VALGRIND="valgrind" +BN=`basename $1` +ON="output/$BN" +mkdir -p output +a="$ON.libyaml.parse" +b="$ON.libfyaml.parse" +./src/libfyaml-parser -mlibyaml-parse "$1" >"$a" +./src/libfyaml-parser -mparse "$1" >"$b" +diff -u "$a" "$b" | tee "$ON.parse.diff" +./src/libfyaml-parser -mparse -d0 "$1" >"$ON.parse.log" 2>&1 diff --git a/scripts/run-compare-scan.sh b/scripts/run-compare-scan.sh new file mode 100755 index 0000000..3509178 --- /dev/null +++ b/scripts/run-compare-scan.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# VALGRIND="" +VALGRIND="valgrind" +BN=`basename $1` +ON="output/$BN" +mkdir -p output +a="$ON.libyaml.scan" +b="$ON.libfyaml.scan" +./src/libfyaml-parser -mlibyaml-scan "$1" >"$a" +./src/libfyaml-parser -mscan "$1" >"$b" +diff -u "$a" "$b" | tee "$ON.scan.diff" +./src/libfyaml-parser -mscan -d0 "$1" >"$ON.scan.log" 2>&1 diff --git a/scripts/run-compare-testsuite.sh b/scripts/run-compare-testsuite.sh new file mode 100755 index 0000000..bb9520d --- /dev/null +++ b/scripts/run-compare-testsuite.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +declare -a rf + +# check tty mode (whether we can colorize) +if [ -t 1 ]; then + GRN="\e[32m" + RED="\e[31m" + GRY="\e[37m" + YLW="\e[33m" + CYN="\e[36m" + NRM="\e[0m" +else + GRN="" + RED="" + GRY="" + YLW="" + CYN="" + NRM="" +fi + +rf=() +i=0 +for prefix in $*; do + echo "Collecting tests from $prefix" + while read full ; do + + # full-path:prefix:path-without-prefix:path-without-prefix-filename + relative=`echo $full | sed -e "s#^$prefix##g"` + file=`echo $relative | sed -e "s#/in.yaml##g" | tr '/' '-'` + rf+=( "$full:$prefix:$relative:$file" ) + # echo ${rf[$i]} + i=$(($i + 1)) + done < <(find "$prefix" -name "in.yaml" -print | sort) +done + +count=${#rf[@]} + +mkdir -p output + +pass=0 +fail=0 +unknown=0 +experr=0 +for (( i=0; i < $count; i++)); do + v="${rf[$i]}" + full=`echo $v | cut -d: -f1` + prefix=`echo $v | cut -d: -f2` + relative=`echo $v | cut -d: -f3` + file=`echo $v | cut -d: -f4` + # echo "$i: full=$full prefix=$prefix relative=$relative file=$file" + + f="output/$file.yaml" + rm -f "$f" "output/$file.pass" "output/$file.fail" \ + "output/$file.unknown" "output/$file.desc" \ + "output/$file.event" + ln -s `realpath $full` "$f" + + desc=`echo $full | sed -e 's#in.yaml#===#'` + desctxt="" + if [ -e "$desc" ]; then + descf=`realpath $desc` + ln -s "$descf" "output/$file.desc" + desctxt=`cat 2>/dev/null "$descf"` + fi + + event=`echo $full | sed -e 's#in.yaml#test.event#'` + if [ -e "$event" ]; then + eventf=`realpath $event` + ln -s "$eventf" "output/$file.event" + fi + + errf=`echo $full | sed -e 's#in.yaml#error#'` + if [ -e "$errf" ] ; then + expected_error="1" + exp="!" + else + expected_error="0" + exp="-" + fi + + this_fail=0 + this_unknown=0 + + a0="output/$file.libyaml" + a1="output/$file.libfyaml" + + pass0=0 + pass1=0 + diff=0 + + # execute both libyaml & libfyaml + ./src/libfyaml-parser -mlibyaml-parse "$f" > "$a0" 2>/dev/null + if [ $? -eq 0 ]; then pass0=1; fi + + ./src/libfyaml-parser -mparse "$f" > "$a1" 2>/dev/null + if [ $? -eq 0 ]; then pass1=1; fi + + # compare output anyway + diff -u "$a0" "$a1" > "output/$file.diff" + if [ $? -eq 0 ]; then diff=1; fi + + if [ $expected_error -eq 0 ]; then + # it fails if any fail + this_fail=$(($pass0 == 0 || $pass1 == 0 || $diff == 0)) + this_unknown=$(($pass0 == 0)) + else + # test expected to fail (both must fail) + this_fail=$(($pass0 == 1 || $pass1 == 1)) + this_unknown=$(($pass0 == 1)) + fi + + if [ $this_unknown -ne 0 ]; then + touch "output/$file.unknown" + if [[ $pass1 == $pass0 ]]; then + COL="${GRY}" + else + COL="${CYN}" + fi + res="${COL}UNKN${NRM}" + unknown=$(($unknown + 1)) + elif [ $this_fail -ne 0 ]; then + touch "output/$file.fail" + res="${RED}FAIL${NRM}" + fail=$(($fail + 1)) + else + touch "output/$file.pass" + res="${GRN}PASS${NRM}" + pass=$(($pass + 1)) + fi + + echo -e "${exp}${pass0}${pass1} ${res} $file: $desctxt" +done + +echo +echo -e " TOTAL: ${count}" +echo -e " PASS: ${GRN}${pass}${NRM}" +echo -e " FAIL: ${RED}${fail}${NRM}" +echo -e "UNKNOWN: ${GRY}${unknown}${NRM}" diff --git a/scripts/run-emit-check.sh b/scripts/run-emit-check.sh new file mode 100755 index 0000000..6f6efc9 --- /dev/null +++ b/scripts/run-emit-check.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +BN=`basename $1` +ON="output/$BN" +mkdir -p output +a="$ON.1.dump" +b="$ON.2.dump" +./src/fy-tool --testsuite "$1" >"$a" +./src/fy-tool --dump "$1" | ./src/fy-tool --testsuite - >"$b" +diff -au "$a" "$b" | tee "$ON.12.diff" +if [ ! -s "${ON}.12.diff" ] ; then + rm -f "${ON}.1.dump" "${ON}.2.dump" "${ON}.12.diff" +fi diff --git a/scripts/run-kcachegrind.sh b/scripts/run-kcachegrind.sh new file mode 100755 index 0000000..002129b --- /dev/null +++ b/scripts/run-kcachegrind.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# LD_LIBRARY_PATH=${PWD}/src/.libs/ valgrind --leak-check=full ./src/.libs/libfyaml-parser -mscan "$1" 2>&1 | tee valgrind.log +LD_LIBRARY_PATH=${PWD}/src/.libs/ valgrind --tool=callgrind --dump-instr=yes --simulate-cache=yes --collect-jumps=yes \ + ./src/.libs/libfyaml-parser $* 2>&1 | tee valgrind.log diff --git a/scripts/run-list-testsuite.sh b/scripts/run-list-testsuite.sh new file mode 100755 index 0000000..08a84d6 --- /dev/null +++ b/scripts/run-list-testsuite.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash + +ERROR=1 +PASS=1 + +while true; do + case "$1" in + -- ) shift; break ;; + + --pass | -p) + ERROR=0 + PASS=1 + shift ;; + + --errors | -e ) + ERROR=1 + PASS=0 + shift ;; + + * ) break ;; + esac +done + +declare -a rf + +# check tty mode (whether we can colorize) +if [ -t 1 ]; then + GRN="\e[32m" + RED="\e[31m" + GRY="\e[37m" + YLW="\e[33m" + CYN="\e[36m" + NRM="\e[0m" +else + GRN="" + RED="" + GRY="" + YLW="" + CYN="" + NRM="" +fi + +rf=() +i=0 +for prefix in $*; do + echo "Collecting tests from $prefix" + while read full ; do + + # full-path:prefix:path-without-prefix:path-without-prefix-filename + relative=`echo $full | sed -e "s#^$prefix##g"` + file=`echo $relative | sed -e "s#/in.yaml##g" | tr '/' '-'` + rf+=( "$full:$prefix:$relative:$file" ) + # echo ${rf[$i]} + i=$(($i + 1)) + done < <(find "$prefix" -name "in.yaml" -print | sort) +done + +count=${#rf[@]} + +for (( i=0; i < $count; i++)); do + v="${rf[$i]}" + full=`echo $v | cut -d: -f1` + prefix=`echo $v | cut -d: -f2` + relative=`echo $v | cut -d: -f3` + file=`echo $v | cut -d: -f4` + # echo "$i: full=$full prefix=$prefix relative=$relative file=$file" + + f="output/$file.yaml" + + has_yaml=1 + + desc=`echo $full | sed -e 's#in.yaml#===#'` + desctxt="" + if [ -e "$desc" ]; then + descf=`realpath "$desc"` + desctxt=`cat 2>/dev/null "$descf"` + has_desc=1 + else + has_desc=0 + fi + + errf=`echo $full | sed -e 's#in.yaml#error#'` + if [ -e "$errf" ] ; then + expected_error="1" + has_error=1 + else + expected_error="0" + has_error=0 + fi + + if [ $expected_error == 0 -a $PASS != 1 ]; then + continue + fi + + if [ $expected_error == 1 -a $ERROR != 1 ]; then + continue + fi + + if [ $expected_error == 0 ]; then + col="$GRN" + else + col="$RED" + fi + echo -e "$col$file$NRM: $desctxt" +done diff --git a/scripts/run-massif.sh b/scripts/run-massif.sh new file mode 100755 index 0000000..6078345 --- /dev/null +++ b/scripts/run-massif.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# LD_LIBRARY_PATH=${PWD}/src/.libs/ valgrind --leak-check=full ./src/.libs/libfyaml-parser -mscan "$1" 2>&1 | tee valgrind.log +LD_LIBRARY_PATH=${PWD}/src/.libs/ valgrind --tool=massif ./src/.libs/libfyaml-parser $* 2>&1 | tee massif.log diff --git a/scripts/run-test.sh b/scripts/run-test.sh new file mode 100755 index 0000000..fcb0e59 --- /dev/null +++ b/scripts/run-test.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash + +PREFIX="./src" +PARSER="libfyaml-parser -d0 -mtestsuite" +# PARSER="fy-public-parser -d0" +KEEP=0 +VALGRIND=0 +STDIN=0 + +while true; do + case "$1" in + -- ) shift; break ;; + + --private | -P ) + PARSER="libfyaml-parser -d0 -mtestsuite" + shift ;; + + --public | -p ) + PARSER="fy-public-parser -d0" + shift ;; + + --libyaml | -l ) + PARSER="libfyaml-parser -d0 -mlibyaml-testsuite" + shift ;; + + --valgrind | -V) + VALGRIND=1 + shift ;; + + --keep | -k) + KEEP=1 + shift ;; + + --stdin | -s ) + STDIN=1 + shift ;; + + * ) break ;; + esac +done + +if [[ $VALGRIND != 0 ]]; then + export LD_LIBRARY_PATH="${PWD}/src/.libs" + PARSER="valgrind --track-origins=yes --leak-check=full $PREFIX/.libs/$PARSER" +else + PARSER="$PREFIX/$PARSER" +fi + +echo " PARSER: ${PARSER}" + +declare -a rf + +# check tty mode (whether we can colorize) +if [ -t 1 ]; then + GRN="\e[32m" + RED="\e[31m" + GRY="\e[37m" + YLW="\e[33m" + CYN="\e[36m" + NRM="\e[0m" +else + GRN="" + RED="" + GRY="" + YLW="" + CYN="" + NRM="" +fi + +rf=() +i=0 +for prefix in $*; do + echo "Collecting tests from $prefix" + while read full ; do + + # full-path:prefix:path-without-prefix:path-without-prefix-filename + relative=`echo $full | sed -e "s#^$prefix##g"` + file=`echo $relative | sed -e "s#/in.yaml##g" | tr '/' '-'` + rf+=( "$full:$prefix:$relative:$file" ) + # echo ${rf[$i]} + i=$(($i + 1)) + done < <(find "$prefix" -name "in.yaml" -print | sort) +done + +count=${#rf[@]} + +mkdir -p output + +pass=0 +fail=0 +unknown=0 +experr=0 + +pass_count_yaml=0 +fail_count_yaml=0 + +pass_count_json=0 +fail_count_json=0 + +for (( i=0; i < $count; i++)); do + v="${rf[$i]}" + full=`echo $v | cut -d: -f1` + prefix=`echo $v | cut -d: -f2` + relative=`echo $v | cut -d: -f3` + file=`echo $v | cut -d: -f4` + # echo "$i: full=$full prefix=$prefix relative=$relative file=$file" + + f="output/$file.yaml" + rm -f "$f" "output/$file.pass" "output/$file.fail" \ + "output/$file.unknown" "output/$file.desc" \ + "output/$file.event" "output/$file.log" \ + "output/$file.gm-event" \ + "output/$file.libyaml-event" \ + "output/$file.json" "output/$file.json.diff" \ + "output/$file.json.event" + ln -s `realpath $full` "$f" + has_yaml=1 + + desc=`echo $full | sed -e 's#in.yaml#===#'` + desctxt="" + if [ -e "$desc" ]; then + descf=`realpath "$desc"` + ln -s "$descf" "output/$file.desc" + desctxt=`cat 2>/dev/null "$descf"` + has_desc=1 + else + has_desc=0 + fi + + event=`echo $full | sed -e 's#in.yaml#test.event#'` + if [ -e "$event" ]; then + eventf=`realpath "$event"` + ln -s "$eventf" "output/$file.gm-event" + has_event=1 + else + has_event=0 + fi + + errf=`echo $full | sed -e 's#in.yaml#error#'` + if [ -e "$errf" ] ; then + expected_error="1" + has_error=1 + else + expected_error="0" + has_error=0 + fi + + jsonf=`echo $full | sed -e 's#in.yaml#in.json#'` + if [ -e "$jsonf" ]; then + ln -s `realpath "$jsonf"` "output/$file.json" + has_json=1 + # verify that the json file is a valid one + jsonlint-php -q "$jsonf" >/dev/null 2>&1 + if [ $? -eq 0 ]; then + has_good_json=1 + else + has_good_json=0 + fi + else + has_json=0 + has_good_json=1 + fi + + this_fail=0 + + a0="output/$file.event" + + # execute in testsuite event mode + pass_yaml=0 + if [[ $STDIN == 0 ]]; then + ${PARSER} "$f" > "$a0" 2> "output/$file.log" + else + cat "$f" | ${PARSER} > "$a0" 2> "output/$file.log" + fi + if [ $? -eq 0 ]; then pass_yaml=1; fi + + # compare output + diff_yaml=0 + diff -u "$a0" "$event" > "output/$file.diff" + if [ $? -eq 0 ]; then diff_yaml=1; fi + + if [ $expected_error -eq 0 ]; then + eexp="${GRN}e${NRM}" + else + eexp="${RED}E${NRM}" + fi + + if [[ $has_json == 1 && $has_good_json == 1 ]]; then + a1="output/$file.json.event" + + pass_json=0 + if [[ $STDIN == 0 ]]; then + ${PARSER} "$jsonf" > "$a1" 2> "output/$file.log" + else + cat "$jsonf" | ${PARSER} > "$a1" 2> "output/$file.log" + fi + if [ $? -eq 0 ]; then pass_json=1; fi + + # it is not possible to diff test.event again json + # because the json output does not contain anchor + # and tag information + else + pass_json=$(( ! ${expected_error})) + fi + + if [ $expected_error -eq 0 ]; then + # it fails if any fail + this_fail=$(($pass_yaml == 0 || $diff_yaml == 0 || $pass_json == 0)) + eexp="${GRN}e${NRM}" + else + # test expected to fail + this_fail=$(($pass_yaml == 1 || $pass_json == 1)) + eexp="${RED}E${NRM}" + fi + + if [[ $has_json == 1 && $has_good_json == 1 ]]; then + hjexp="${GRN}+${NRM}"; + elif [[ $has_json == 1 && $has_good_json == 0 ]]; then + hjexp="${CYN}!${NRM}"; + else + hjexp="-"; + fi + + if [[ $pass_yaml == 1 ]]; then + yexp="${GRN}Y${NRM}"; + else + yexp="${RED}y${NRM}"; + fi + + if [[ $diff_yaml == 1 ]]; then + dyexp="${GRN}D${NRM}"; + else + dyexp="${RED}d${NRM}"; + fi + + if [[ $has_json == 1 && $has_good_json == 1 ]]; then + if [[ $pass_json == 1 ]]; then + jexp="${GRN}J${NRM}"; + else + jexp="${RED}j${NRM}"; + fi + else + jexp="-" + fi + + if [ $this_fail -ne 0 ]; then + touch "output/$file.fail" + res="${RED}FAIL${NRM}" + fail=$(($fail + 1)) + else + touch "output/$file.pass" + res="${GRN}PASS${NRM}" + pass=$(($pass + 1)) + + if [[ $KEEP == 0 ]]; then + # remove intermediate files + rm -f "$f" "output/$file.pass" "output/$file.fail" \ + "output/$file.unknown" "output/$file.desc" \ + "output/$file.event" "output/$file.log" \ + "output/$file.gm-event" "output/$file.diff" \ + "output/$file.libyaml-event" \ + "output/$file.json" "output/$file.json.diff" \ + "output/$file.json.event" + fi + fi + + echo -e "${eexp}${yexp}${dyexp}${hjexp}${jexp} ${res} $file: $desctxt" +done + +echo +echo " PARSER: ${PARSER}" +echo -e " TOTAL: ${count}" +echo -e " PASS: ${GRN}${pass}${NRM}" +echo -e " FAIL: ${RED}${fail}${NRM}" +echo -e "UNKNOWN: ${GRY}${unknown}${NRM}" diff --git a/scripts/run-valgrind.sh b/scripts/run-valgrind.sh new file mode 100755 index 0000000..451c69d --- /dev/null +++ b/scripts/run-valgrind.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# LD_LIBRARY_PATH=${PWD}/src/.libs/ valgrind --leak-check=full ./src/.libs/libfyaml-parser -mscan "$1" 2>&1 | tee valgrind.log +LD_LIBRARY_PATH=${PWD}/src/.libs/ valgrind --track-origins=yes --leak-check=full --error-exitcode=5 ./src/.libs/libfyaml-parser $* 2>&1 | tee valgrind.log diff --git a/scripts/show-desc.sh b/scripts/show-desc.sh new file mode 100755 index 0000000..5087dd7 --- /dev/null +++ b/scripts/show-desc.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +for i in output/*.desc; do echo -n "$i: "; cat $i; done diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..3d79cd5 --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,246 @@ +bin_PROGRAMS = +noinst_PROGRAMS = +noinst_LIBRARIES = +lib_LTLIBRARIES = +noinst_LTLIBRARIES = + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src \ + -I$(top_srcdir)/include +AM_CFLAGS = + +INTERNAL_INCLUDES = \ + -I$(top_srcdir)/src/valgrind \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/xxhash \ + -I$(top_srcdir)/src/util \ + -I$(top_srcdir)/src/thread \ + -I$(top_srcdir)/src/allocator + +lib_LTLIBRARIES += libfyaml.la +libfyaml_la_SOURCES = \ + lib/fy-parse.c lib/fy-parse.h \ + lib/fy-types.c lib/fy-types.h \ + lib/fy-diag.c lib/fy-diag.h \ + lib/fy-dump.c lib/fy-dump.h \ + lib/fy-atom.c lib/fy-atom.h \ + lib/fy-token.c lib/fy-token.h \ + lib/fy-input.c lib/fy-input.h \ + lib/fy-docstate.c lib/fy-docstate.h \ + lib/fy-doc.c lib/fy-doc.h \ + lib/fy-docbuilder.c lib/fy-docbuilder.h \ + lib/fy-emit.c lib/fy-emit.h lib/fy-emit-accum.h \ + lib/fy-event.h lib/fy-event.c \ + lib/fy-accel.c lib/fy-accel.h \ + lib/fy-walk.c lib/fy-walk.h \ + lib/fy-path.c lib/fy-path.h \ + lib/fy-composer.c lib/fy-composer.h \ + lib/fy-parse-diag.c lib/fy-input-diag.c lib/fy-doc-diag.c \ + lib/fy-docbuilder-diag.c lib/fy-composer-diag.c + +# util +libfyaml_la_SOURCES += \ + util/fy-list.h \ + util/fy-typelist.h \ + util/fy-ctype.c util/fy-ctype.h \ + util/fy-utf8.c util/fy-utf8.h \ + util/fy-utils.c util/fy-utils.h \ + util/fy-endian.h \ + util/fy-blob.c util/fy-blob.h \ + util/fy-id.h \ + util/fy-align.h \ + util/fy-bit64.h \ + util/fy-vlsize.h \ + util/fy-atomics.h + +# xxhash +libfyaml_la_SOURCES += \ + xxhash/xxhash.c xxhash/xxhash.h + +# thread +libfyaml_la_SOURCES += \ + thread/fy-thread.h thread/fy-thread.c + +# allocator +libfyaml_la_SOURCES += \ + allocator/fy-allocator.c allocator/fy-allocator.h \ + allocator/fy-allocator-linear.c allocator/fy-allocator-linear.h \ + allocator/fy-allocator-malloc.c allocator/fy-allocator-malloc.h \ + allocator/fy-allocator-mremap.c allocator/fy-allocator-mremap.h \ + allocator/fy-allocator-dedup.c allocator/fy-allocator-dedup.h \ + allocator/fy-allocator-auto.c allocator/fy-allocator-auto.h + +libfyaml_la_CPPFLAGS = $(AM_CPPFLAGS) $(INTERNAL_INCLUDES) +libfyaml_la_CFLAGS = $(AM_CFLAGS) +libfyaml_la_LDFLAGS = -no-undefined $(AM_LDFLAGS) $(AM_LIBLDFLAGS) \ + -version $(LIBTOOL_VERSION) +libfyaml_la_LIBADD = + +# blake3 section +libfyaml_la_CPPFLAGS += -I$(top_srcdir)/src/blake3 +libfyaml_la_SOURCES += blake3/blake3.h \ + blake3/blake3_impl.h \ + blake3/blake3_internal.h \ + blake3/blake3_host_state.c \ + blake3/blake3_backend.c \ + blake3/blake3_be_cpusimd.c \ + blake3/fy-blake3.c + +LIBB3_COMMON_CPPFLAGS=$(AM_CPPFLAGS) -I$(top_srcdir)/src/util -I$(top_srcdir)/src/thread + +# the portable implementation +noinst_LTLIBRARIES += libb3portable.la +libb3portable_la_SOURCES = blake3/blake3_portable.c blake3/blake3.c +libb3portable_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -DHASHER_SUFFIX=portable -DSIMD_DEGREE=1 +libb3portable_la_CFLAGS = $(AM_CPPFLAGS) +libfyaml_la_LIBADD += libb3portable.la + +# note that each target lib has blake3 compiled again but with a suffix +if TARGET_HAS_SSE2 +noinst_LTLIBRARIES += libb3sse2.la +libb3sse2_la_SOURCES = blake3/blake3_sse2.c blake3/blake3_sse2_x86-64_unix.S blake3/blake3.c +libb3sse2_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -msse2 -DHASHER_SUFFIX=sse2 -DSIMD_DEGREE=4 +libb3sse2_la_CFLAGS = $(AM_CPPFLAGS) -msse2 +libfyaml_la_LIBADD += libb3sse2.la +endif + +if TARGET_HAS_SSE41 +noinst_LTLIBRARIES += libb3sse41.la +libb3sse41_la_SOURCES = blake3/blake3_sse41.c blake3/blake3_sse41_x86-64_unix.S blake3/blake3.c +libb3sse41_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -msse4.1 -DHASHER_SUFFIX=sse41 -DSIMD_DEGREE=4 +libb3sse41_la_CFLAGS = $(AM_CPPFLAGS) -msse4.1 +libfyaml_la_LIBADD += libb3sse41.la +endif + +if TARGET_HAS_AVX2 +noinst_LTLIBRARIES += libb3avx2.la +libb3avx2_la_SOURCES = blake3/blake3_avx2.c blake3/blake3_avx2_x86-64_unix.S blake3/blake3.c +libb3avx2_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -mavx2 -DHASHER_SUFFIX=avx2 -DSIMD_DEGREE=8 +libb3avx2_la_CFLAGS = $(AM_CPPFLAGS) -mavx2 +libfyaml_la_LIBADD += libb3avx2.la +endif + +if TARGET_HAS_AVX512 +noinst_LTLIBRARIES += libb3avx512.la +libb3avx512_la_SOURCES = blake3/blake3_avx512.c blake3/blake3_avx512_x86-64_unix.S blake3/blake3.c +libb3avx512_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -mavx512f -mavx512vl -DHASHER_SUFFIX=avx512 -DSIMD_DEGREE=16 +libb3avx512_la_CFLAGS = $(AM_CPPFLAGS) -mavx512f -mavx512vl +libfyaml_la_LIBADD += libb3avx512.la +endif + +if TARGET_HAS_NEON +noinst_LTLIBRARIES += libb3neon.la +libb3neon_la_SOURCES = blake3/blake3_neon.c blake3/blake3.c +libb3neon_la_CPPFLAGS = $(LIBB3_COMMON_CPPFLAGS) -DHASHER_SUFFIX=neon -DSIMD_DEGREE=4 +libb3neon_la_CFLAGS = $(AM_CPPFLAGS) +if TARGET_CPU_ARM +libb3neon_la_CPPFLAGS += -mfpu=neon +libb3neon_la_CFLAGS += -mfpu=neon +endif +libfyaml_la_LIBADD += libb3neon.la +endif + +if HAVE_LIBCLANG +libfyaml_la_CPPFLAGS += $(LIBCLANG_CPPFLAGS) +libfyaml_la_CFLAGS += $(LIBCLANG_CFLAGS) +libfyaml_la_LDFLAGS += $(LIBCLANG_LDFLAGS) $(LIBCLANG_LIBS) +endif + +if HAVE_CLANG_BLOCKS +libfyaml_la_CFLAGS += $(CLANG_BLOCKS_CFLAGS) +libfyaml_la_LDFLAGS += $(CLANG_BLOCKS_LIBS) +endif + +if HAVE_STATIC + +if HAVE_LIBYAML +if !CROSS_COMPILING + +noinst_PROGRAMS += libfyaml-parser + +libfyaml_parser_SOURCES = \ + internal/libfyaml-parser.c \ + valgrind/fy-valgrind.h + +libfyaml_parser_CPPFLAGS = $(AM_CPPFLAGS) $(INTERNAL_INCLUDES) +libfyaml_parser_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la +libfyaml_parser_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS) + +libfyaml_parser_LDFLAGS = $(AM_LDFLAGS) -static +endif +endif + +if !CROSS_COMPILING + +noinst_PROGRAMS += fy-thread + +fy_thread_SOURCES = \ + internal/fy-thread.c \ + valgrind/fy-valgrind.h + +fy_thread_CPPFLAGS = $(AM_CPPFLAGS) $(INTERNAL_INCLUDES) +fy_thread_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la +fy_thread_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS) + +fy_thread_LDFLAGS = $(AM_LDFLAGS) -static +endif + +if !CROSS_COMPILING + +noinst_PROGRAMS += fy-b3sum + +fy_b3sum_SOURCES = \ + internal/fy-b3sum.c \ + valgrind/fy-valgrind.h + +fy_b3sum_CPPFLAGS = $(AM_CPPFLAGS) $(INTERNAL_INCLUDES) -I$(top_srcdir)/src/blake3 +fy_b3sum_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la +fy_b3sum_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS) + +fy_b3sum_LDFLAGS = $(AM_LDFLAGS) -static +endif + +if !CROSS_COMPILING + +noinst_PROGRAMS += fy-allocators + +fy_allocators_SOURCES = \ + internal/fy-allocators.c \ + valgrind/fy-valgrind.h + +fy_allocators_CPPFLAGS = $(AM_CPPFLAGS) $(INTERNAL_INCLUDES) +fy_allocators_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la +fy_allocators_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS) + +fy_allocators_LDFLAGS = $(AM_LDFLAGS) -static +endif + +endif # HAVE_STATIC + +bin_PROGRAMS += fy-tool + +fy_tool_SOURCES = \ + tool/fy-tool.c \ + tool/fy-tool-util.c \ + tool/fy-tool-dump.c \ + tool/fy-tool-util.h \ + valgrind/fy-valgrind.h + +fy_tool_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/valgrind +fy_tool_LDADD = $(AM_LDADD) libfyaml.la +fy_tool_CFLAGS = $(AM_CFLAGS) +fy_tool_LDFLAGS = $(AM_LDFLAGS) + +include_HEADERS = \ + $(top_srcdir)/include/libfyaml.h + +install-exec-hook: + (cd "$(DESTDIR)$(bindir)" && $(LN_S) -f fy-tool fy-dump) + (cd "$(DESTDIR)$(bindir)" && $(LN_S) -f fy-tool fy-filter) + (cd "$(DESTDIR)$(bindir)" && $(LN_S) -f fy-tool fy-testsuite) + (cd "$(DESTDIR)$(bindir)" && $(LN_S) -f fy-tool fy-join) + (cd "$(DESTDIR)$(bindir)" && $(LN_S) -f fy-tool fy-ypath) + (cd "$(DESTDIR)$(bindir)" && $(LN_S) -f fy-tool fy-compose) + +uninstall-hook: + (cd "$(DESTDIR)$(bindir)" && rm -f fy-dump fy-filter fy-testsuite fy-join fy-ypath fy-compose) diff --git a/src/allocator/fy-allocator-auto.c b/src/allocator/fy-allocator-auto.c new file mode 100644 index 0000000..c5aa7c6 --- /dev/null +++ b/src/allocator/fy-allocator-auto.c @@ -0,0 +1,399 @@ +/* + * fy-allocator-auto.c - automatic allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-utils.h" +#include "fy-allocator-mremap.h" +#include "fy-allocator-dedup.h" +#include "fy-allocator-linear.h" +#include "fy-allocator-auto.h" + +/* fixed parameters */ +#ifndef AUTO_ALLOCATOR_BIG_ALLOC_THRESHOLD +#define AUTO_ALLOCATOR_BIG_ALLOC_THRESHOLD SIZE_MAX +#endif + +#ifndef AUTO_ALLOCATOR_EMPTY_THRESHOLD +#define AUTO_ALLOCATOR_EMPTY_THRESHOLD 64 +#endif + +#ifndef AUTO_ALLOCATOR_GROW_RATIO +#define AUTO_ALLOCATOR_GROW_RATIO 1.5 +#endif + +#ifndef AUTO_ALOCATOR_BALLOON_RATIO +#define AUTO_ALOCATOR_BALLOON_RATIO 8.0 +#endif + +#ifndef AUTO_ALLOCATOR_ARENA_TYPE +#define AUTO_ALLOCATOR_ARENA_TYPE FYMRAT_MMAP +#endif + +#ifndef AUTO_ALLOCATOR_MINIMUM_ARENA_SIZE +#define AUTO_ALLOCATOR_MINIMUM_ARENA_SIZE ((size_t)16 << 20) /* 16 MB */ +#endif + +#ifndef AUTO_ALLOCATOR_DEFAULT_ESTIMATED_MAX_SIZE +#define AUTO_ALLOCATOR_DEFAULT_ESTIMATED_MAX_SIZE ((size_t)1 << 20) /* 1MB */ +#endif + +#ifndef AUTO_ALLOCATOR_DEFAULT_BLOOM_FILTER_BITS +#define AUTO_ALLOCATOR_DEFAULT_BLOOM_FILTER_BITS 0 +#endif + +#ifndef AUTO_ALLOCATOR_DEFAULT_BUCKET_COUNT_BITS +#define AUTO_ALLOCATOR_DEFAULT_BUCKET_COUNT_BITS 0 +#endif + +#ifndef AUTO_ALLOCATOR_DEFAULT_DEDUP_THRESHOLD +#define AUTO_ALLOCATOR_DEFAULT_DEDUP_THRESHOLD 0 /* dedup everything */ +#endif + +#ifndef AUTO_ALLOCATOR_DEFAULT_CHAIN_LENGTH_GROW_TRIGGER +#define AUTO_ALLOCATOR_DEFAULT_CHAIN_LENGTH_GROW_TRIGGER 0 /* let dedup decide */ +#endif + +static const struct fy_auto_allocator_cfg default_cfg = { + .scenario = FYAST_PER_TAG_FREE_DEDUP, + .estimated_max_size = AUTO_ALLOCATOR_DEFAULT_ESTIMATED_MAX_SIZE, +}; + +static void fy_auto_cleanup(struct fy_allocator *a); + +static int fy_auto_setup(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *cfg_data) +{ + size_t pagesz, size; + struct fy_auto_allocator *aa; + const struct fy_auto_allocator_cfg *cfg; + struct fy_mremap_allocator_cfg mrcfg; + struct fy_dedup_allocator_cfg dcfg; + struct fy_linear_allocator_cfg lcfg; + struct fy_allocator *mra = NULL, *da = NULL, *ma = NULL, *la = NULL; + struct fy_allocator *topa = NULL, *suba = NULL; + + cfg = cfg_data ? cfg_data : &default_cfg; + + aa = container_of(a, struct fy_auto_allocator, a); + memset(aa, 0, sizeof(*aa)); + aa->a.name = "auto"; + aa->a.ops = &fy_auto_allocator_ops; + aa->a.parent = parent; + aa->a.parent_tag = parent_tag; + aa->cfg = *cfg; + + pagesz = sysconf(_SC_PAGESIZE); + size = cfg->estimated_max_size && cfg->estimated_max_size != SIZE_MAX ? + cfg->estimated_max_size : + AUTO_ALLOCATOR_MINIMUM_ARENA_SIZE; + + /* first allocator */ + switch (cfg->scenario) { + case FYAST_PER_TAG_FREE: + case FYAST_PER_TAG_FREE_DEDUP: + memset(&mrcfg, 0, sizeof(mrcfg)); + mrcfg.big_alloc_threshold = AUTO_ALLOCATOR_BIG_ALLOC_THRESHOLD; + mrcfg.empty_threshold = AUTO_ALLOCATOR_EMPTY_THRESHOLD; + mrcfg.grow_ratio = AUTO_ALLOCATOR_GROW_RATIO; + mrcfg.balloon_ratio = AUTO_ALOCATOR_BALLOON_RATIO; + mrcfg.arena_type = AUTO_ALLOCATOR_ARENA_TYPE; + + mrcfg.minimum_arena_size = fy_size_t_align(size, pagesz); + + mra = fy_allocator_create("mremap", &mrcfg); + if (!mra) + goto err_out; + topa = mra; + + break; + + case FYAST_PER_OBJ_FREE: + case FYAST_PER_OBJ_FREE_DEDUP: + ma = fy_allocator_create("malloc", NULL); + if (!ma) + goto err_out; + topa = ma; + break; + + case FYAST_SINGLE_LINEAR_RANGE: + case FYAST_SINGLE_LINEAR_RANGE_DEDUP: + memset(&lcfg, 0, sizeof(lcfg)); + lcfg.size = fy_size_t_align(size, pagesz); + + la = fy_allocator_create("linear", &lcfg); + if (!la) + goto err_out; + topa = la; + break; + + default: + goto err_out; + } + + if (!topa) + goto err_out; + + /* stack the dedup */ + switch (cfg->scenario) { + case FYAST_PER_TAG_FREE_DEDUP: + case FYAST_PER_OBJ_FREE_DEDUP: + case FYAST_SINGLE_LINEAR_RANGE_DEDUP: + memset(&dcfg, 0, sizeof(dcfg)); + dcfg.parent_allocator = topa; + dcfg.bloom_filter_bits = AUTO_ALLOCATOR_DEFAULT_BLOOM_FILTER_BITS; + dcfg.bucket_count_bits = AUTO_ALLOCATOR_DEFAULT_BUCKET_COUNT_BITS; + dcfg.dedup_threshold = AUTO_ALLOCATOR_DEFAULT_DEDUP_THRESHOLD; + dcfg.chain_length_grow_trigger = AUTO_ALLOCATOR_DEFAULT_CHAIN_LENGTH_GROW_TRIGGER; + dcfg.estimated_content_size = size; + dcfg.minimum_bucket_occupancy = 0.5; + + da = fy_allocator_create("dedup", &dcfg); + if (!da) + goto err_out; + /* the top is sub now */ + suba = topa; + topa = da; + + break; + + /* nothing to stack for these */ + case FYAST_PER_TAG_FREE: + case FYAST_PER_OBJ_FREE: + case FYAST_SINGLE_LINEAR_RANGE: + break; + + default: + goto err_out; + } + + /* OK, assign the allocators now */ + aa->parent_allocator = topa; + aa->sub_parent_allocator = suba; + + return 0; + +err_out: + fy_allocator_destroy(suba); + fy_allocator_destroy(topa); + fy_auto_cleanup(a); + return -1; +} + +static void fy_auto_cleanup(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + + /* order is important! */ + if (aa->parent_allocator) { + fy_allocator_destroy(aa->parent_allocator); + aa->parent_allocator = NULL; + } + + if (aa->sub_parent_allocator) { + fy_allocator_destroy(aa->sub_parent_allocator); + aa->sub_parent_allocator = NULL; + } +} + +struct fy_allocator *fy_auto_create(struct fy_allocator *parent, int parent_tag, const void *cfg) +{ + struct fy_auto_allocator *aa = NULL; + int rc; + + aa = fy_early_parent_allocator_alloc(parent, parent_tag, sizeof(*aa), _Alignof(struct fy_auto_allocator)); + + rc = fy_auto_setup(&aa->a, parent, parent_tag, cfg); + if (rc) + goto err_out; + + return &aa->a; + +err_out: + fy_early_parent_allocator_free(parent, parent_tag, aa); + + return NULL; +} + +void fy_auto_destroy(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + + fy_auto_cleanup(a); + + fy_parent_allocator_free(a, aa); +} + +void fy_auto_dump(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + + fy_allocator_dump_nocheck(aa->parent_allocator); +} + +static void *fy_auto_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_alloc_nocheck(aa->parent_allocator, tag, size, align); +} + +static void fy_auto_free(struct fy_allocator *a, int tag, void *data) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + fy_allocator_free_nocheck(aa->parent_allocator, tag, data); +} + +static int fy_auto_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_update_stats_nocheck(aa->parent_allocator, tag, stats); +} + +static const void *fy_auto_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_storev_hash_nocheck(aa->parent_allocator, tag, iov, iovcnt, align, hash); +} + +static const void *fy_auto_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_lookupv_nocheck(aa->parent_allocator, tag, iov, iovcnt, align, hash); +} + +static void fy_auto_release(struct fy_allocator *a, int tag, const void *data, size_t size) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + fy_allocator_release_nocheck(aa->parent_allocator, tag, data, size); +} + +static int fy_auto_get_tag(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + /* TODO, convert tag config? */ + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_get_tag_nocheck(aa->parent_allocator); +} + +static void fy_auto_release_tag(struct fy_allocator *a, int tag) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + fy_allocator_release_tag_nocheck(aa->parent_allocator, tag); +} + +static int fy_auto_get_tag_count(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_get_tag_count_nocheck(aa->parent_allocator); +} + +static int fy_auto_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_set_tag_count_nocheck(aa->parent_allocator, count); +} + +static void fy_auto_trim_tag(struct fy_allocator *a, int tag) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + fy_allocator_trim_tag_nocheck(aa->parent_allocator, tag); +} + +static void fy_auto_reset_tag(struct fy_allocator *a, int tag) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + fy_allocator_reset_tag_nocheck(aa->parent_allocator, tag); +} + +static struct fy_allocator_info * +fy_auto_get_info(struct fy_allocator *a, int tag) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_get_info_nocheck(aa->parent_allocator, tag); +} + +static enum fy_allocator_cap_flags +fy_auto_get_caps(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_get_caps_nocheck(aa->parent_allocator); +} + +static bool fy_auto_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_contains_nocheck(aa->parent_allocator, tag, ptr); +} + +const struct fy_allocator_ops fy_auto_allocator_ops = { + .setup = fy_auto_setup, + .cleanup = fy_auto_cleanup, + .create = fy_auto_create, + .destroy = fy_auto_destroy, + .dump = fy_auto_dump, + .alloc = fy_auto_alloc, + .free = fy_auto_free, + .update_stats = fy_auto_update_stats, + .storev = fy_auto_storev, + .lookupv = fy_auto_lookupv, + .release = fy_auto_release, + .get_tag = fy_auto_get_tag, + .release_tag = fy_auto_release_tag, + .get_tag_count = fy_auto_get_tag_count, + .set_tag_count = fy_auto_set_tag_count, + .trim_tag = fy_auto_trim_tag, + .reset_tag = fy_auto_reset_tag, + .get_info = fy_auto_get_info, + .get_caps = fy_auto_get_caps, + .contains = fy_auto_contains, +}; diff --git a/src/allocator/fy-allocator-auto.h b/src/allocator/fy-allocator-auto.h new file mode 100644 index 0000000..8027820 --- /dev/null +++ b/src/allocator/fy-allocator-auto.h @@ -0,0 +1,26 @@ +/* + * fy-allocator-auto.h - the auto allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_AUTO_H +#define FY_ALLOCATOR_AUTO_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "fy-allocator.h" + +struct fy_auto_allocator { + struct fy_allocator a; + struct fy_auto_allocator_cfg cfg; + struct fy_allocator *parent_allocator; + struct fy_allocator *sub_parent_allocator; +}; + +extern const struct fy_allocator_ops fy_auto_allocator_ops; + +#endif diff --git a/src/allocator/fy-allocator-dedup.c b/src/allocator/fy-allocator-dedup.c new file mode 100644 index 0000000..e964fcf --- /dev/null +++ b/src/allocator/fy-allocator-dedup.c @@ -0,0 +1,1224 @@ +/* + * fy-allocator-dedup.c - dedup allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +/* for container_of */ +#include "fy-list.h" +#include "fy-utils.h" + +#include "fy-allocator-dedup.h" + +// #define DEBUG_GROWS + +#undef BEFORE +#define BEFORE() \ + do { \ + clock_gettime(CLOCK_MONOTONIC, &before); \ + } while(0) + +#undef AFTER +#define AFTER() \ + ({ \ + clock_gettime(CLOCK_MONOTONIC, &after); \ + (int64_t)(after.tv_sec - before.tv_sec) * (int64_t)1000000000UL + (int64_t)(after.tv_nsec - before.tv_nsec); \ + }) + +static const unsigned int bit_to_chain_length_map[] = { + [0] = 1, /* 1 */ + [1] = 1, /* 2 */ + [2] = 1, /* 4 */ + [3] = 1, /* 8 */ + [4] = 1, /* 16 */ + [5] = 1, /* 32 */ + [6] = 2, /* 64 */ + [7] = 2, /* 128 */ + [8] = 2, /* 256 */ + [9] = 2, /* 512 */ + [10] = 3, /* 1024 */ + [11] = 3, /* 2048 */ + [12] = 3, /* 2048 */ + [13] = 3, /* 2048 */ + [14] = 4, /* 4096 */ + [15] = 4, /* 8192 */ + [16] = 5, /* 16384 */ + [17] = 5, /* 32768 */ + [18] = 6, /* 65536 */ + [19] = 7, /* 65536 */ + [20] = 8, /* 131072 */ + [21] = 9, /* 262144*/ + [22] = 10, /* 524288 */ + [23] = INT_MAX /* infinite from now on */ +}; + +void fy_dedup_tag_data_destroy(struct fy_dedup_tag_data *dtd); + +static inline struct fy_dedup_tag * +fy_dedup_tag_from_tag(struct fy_dedup_allocator *da, int tag) +{ + if (!da) + return NULL; + + if ((unsigned int)tag >= da->tag_count) + return NULL; + + if (!fy_id_is_used(da->ids, da->tag_id_count, tag)) + return NULL; + + return &da->tags[tag]; +} + +static void fy_dedup_tag_data_cleanup(struct fy_dedup_tag_data *dtd) +{ + struct fy_dedup_allocator *da = dtd->cfg.da; + + fy_parent_allocator_free(&da->a, dtd->buckets); + fy_parent_allocator_free(&da->a, dtd->bloom_id); + fy_parent_allocator_free(&da->a, dtd->buckets_in_use); +} + +static int fy_dedup_tag_data_setup(struct fy_dedup_tag_data *dtd, + const struct fy_dedup_tag_data_cfg *cfg) +{ + struct fy_dedup_allocator *da; + unsigned int bloom_filter_bits, bucket_count_bits, chain_length_grow_trigger; + size_t dedup_threshold; + size_t tmpsz; + unsigned int i; + + assert(dtd); + assert(cfg); + + da = cfg->da; + + bloom_filter_bits = cfg->bloom_filter_bits; + bucket_count_bits = cfg->bucket_count_bits; + dedup_threshold = cfg->dedup_threshold; + chain_length_grow_trigger = cfg->chain_length_grow_trigger; + + memset(dtd, 0, sizeof(*dtd)); + dtd->cfg = *cfg; + + dtd->bloom_filter_bits = bloom_filter_bits; + dtd->bucket_count_bits = bucket_count_bits; + dtd->dedup_threshold = dedup_threshold; + dtd->chain_length_grow_trigger = chain_length_grow_trigger; + if (!dtd->chain_length_grow_trigger) { + if (bucket_count_bits >= ARRAY_SIZE(bit_to_chain_length_map)) + dtd->chain_length_grow_trigger = bit_to_chain_length_map[ARRAY_SIZE(bit_to_chain_length_map) - 1]; + else + dtd->chain_length_grow_trigger = bit_to_chain_length_map[bucket_count_bits]; + + if (dtd->chain_length_grow_trigger == 0) + dtd->chain_length_grow_trigger++; + } + + dtd->bloom_filter_mask = (1U << dtd->bloom_filter_bits) - 1; + dtd->bucket_count_mask = (1U << dtd->bucket_count_bits) - 1; + + dtd->bloom_id_count = ((1U << dtd->bloom_filter_bits) + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + assert(dtd->bloom_id_count > 0); + tmpsz = dtd->bloom_id_count * sizeof(*dtd->bloom_id); + dtd->bloom_id = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(fy_id_bits)); + if (!dtd->bloom_id) + goto err_out; + fy_id_reset(dtd->bloom_id, dtd->bloom_id_count); + + dtd->bucket_count = 1U << dtd->bucket_count_bits; + assert(dtd->bucket_count); + + tmpsz = sizeof(*dtd->buckets) * dtd->bucket_count; + dtd->buckets = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(struct fy_dedup_entry *)); + if (!dtd->buckets) + goto err_out; + for (i = 0; i < dtd->bucket_count; i++) + atomic_store(&dtd->buckets[i], NULL); + + dtd->bucket_id_count = ((1U << dtd->bucket_count_bits) + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + assert(dtd->bucket_id_count > 0); + tmpsz = dtd->bucket_id_count * sizeof(*dtd->buckets_in_use); + dtd->buckets_in_use = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(fy_id_bits)); + if (!dtd->buckets_in_use) + goto err_out; + fy_id_reset(dtd->buckets_in_use, dtd->bucket_id_count); + + return 0; + +err_out: + fy_dedup_tag_data_cleanup(dtd); + return -1; +} + +static void fy_dedup_tag_cleanup(struct fy_dedup_allocator *da, struct fy_dedup_tag *dt) +{ + struct fy_dedup_tag_data *dtd; + + if (!da || !dt) + return; + + while ((dtd = fy_atomic_load(&dt->tag_datas)) != NULL) { + + /* only do it if you are the winner */ + if (!fy_atomic_compare_exchange_strong(&dt->tag_datas, &dtd, dtd->next)) + continue; + +#ifdef DEBUG_GROWS + fprintf(stderr, "%s: dump of state at close\n", __func__); + fprintf(stderr, "%s: bloom count=%u used=%lu\n", + __func__, 1 << dtd->bloom_filter_bits, + fy_id_count_used(dtd->bloom_id, dtd->bloom_id_count)); + fprintf(stderr, "%s: bucket count=%u used=%lu\n", + __func__, 1 << dtd->bucket_count_bits, + fy_id_count_used(dtd->buckets_in_use, dtd->bucket_id_count)); +#endif + + fy_dedup_tag_data_destroy(dtd); + } + + /* we just release the tags, the underlying allocator should free everything */ + if (dt->content_tag != FY_ALLOC_TAG_NONE) + fy_allocator_release_tag(da->parent_allocator, dt->content_tag); + +} + +struct fy_dedup_tag_data * +fy_dedup_tag_data_create(const struct fy_dedup_tag_data_cfg *cfg) +{ + struct fy_dedup_allocator *da = cfg->da; + struct fy_dedup_tag_data *dtd; + int ret; + + dtd = fy_parent_allocator_alloc(&da->a, sizeof(*dtd), _Alignof(struct fy_dedup_tag_data)); + if (!dtd) + return NULL; + + ret = fy_dedup_tag_data_setup(dtd, cfg); + if (ret) + goto err_out; + + return dtd; + +err_out: + fy_parent_allocator_free(&da->a, dtd); + return NULL; +} + +void fy_dedup_tag_data_destroy(struct fy_dedup_tag_data *dtd) +{ + struct fy_dedup_allocator *da; + + if (!dtd) + return; + + da = dtd->cfg.da; + + fy_dedup_tag_data_cleanup(dtd); + fy_parent_allocator_free(&da->a, dtd); +} + +static int fy_dedup_tag_setup(struct fy_dedup_allocator *da, struct fy_dedup_tag *dt) +{ + struct fy_dedup_tag_data *dtd; + + assert(da); + assert(dt); + + memset(dt, 0, sizeof(*dt)); + + fy_atomic_flag_clear(&dt->growing); + if (da->parent_caps & FYACF_CAN_FREE_TAG) { + dt->content_tag = fy_allocator_get_tag(da->parent_allocator); + if (dt->content_tag == FY_ALLOC_TAG_ERROR) + goto err_out; + } else + dt->content_tag = FY_ALLOC_TAG_DEFAULT; + + fy_atomic_store(&dt->tag_datas, NULL); + + dtd = fy_dedup_tag_data_create( + &(struct fy_dedup_tag_data_cfg) { + .da = da, + .dt = dt, + .bloom_filter_bits = da->bloom_filter_bits, + .bucket_count_bits = da->bucket_count_bits, + .dedup_threshold = da->dedup_threshold, + .chain_length_grow_trigger = da->chain_length_grow_trigger + }); + if (!dtd) + goto err_out; + + /* start with this one */ + fy_atomic_store(&dt->tag_datas, dtd); + + return 0; + +err_out: + fy_dedup_tag_cleanup(da, dt); + return -1; +} + +static int +fy_dedup_tag_prepare_new(struct fy_dedup_tag_data *dtd, struct fy_dedup_tag_data *new_dtd, + int bloom_filter_adjust_bits, int bucket_adjust_bits) +{ + unsigned int bit_shift, new_bucket_count_bits, new_bloom_filter_bits; + int rc; + + bit_shift = (unsigned int)fy_id_ffs(FY_ID_BITS_BITS); + + new_bucket_count_bits = (unsigned int)((int)dtd->bucket_count_bits + bucket_adjust_bits); + if (new_bucket_count_bits > (sizeof(int) * 8 - 1)) + new_bucket_count_bits = (sizeof(int) * 8) - 1; + else if (new_bucket_count_bits < bit_shift) + new_bucket_count_bits = bit_shift; + + new_bloom_filter_bits = (unsigned int)((int)dtd->bloom_filter_bits + bloom_filter_adjust_bits); + if (new_bloom_filter_bits > (sizeof(int) * 8 - 1)) + new_bloom_filter_bits = (sizeof(int) * 8) - 1; + else if (new_bloom_filter_bits < new_bucket_count_bits) + new_bloom_filter_bits = new_bucket_count_bits; + + /* setup the new data */ + rc = fy_dedup_tag_data_setup(new_dtd, + &(struct fy_dedup_tag_data_cfg) { + .da = dtd->cfg.da, + .dt = dtd->cfg.dt, + .bloom_filter_bits = new_bloom_filter_bits, + .bucket_count_bits = new_bucket_count_bits, + .dedup_threshold = dtd->cfg.da->dedup_threshold, + .chain_length_grow_trigger = dtd->cfg.da->chain_length_grow_trigger + }); + return rc; +} + + +static int +fy_dedup_tag_adjust(struct fy_dedup_allocator *da, struct fy_dedup_tag *dt, + int bloom_filter_adjust_bits, int bucket_adjust_bits) +{ + size_t bucket_count, bucket_used; + struct fy_dedup_tag_data *dtd, *new_dtd = NULL; + float occupancy_ratio; + int rc; + + rc = -1; + + assert(da); + assert(dt); + + dtd = fy_atomic_load(&dt->tag_datas); + if (!dtd) + return -1; + + if (!fy_atomic_flag_test_and_set(&dt->growing)) { +#ifdef DEBUG_GROWS + fprintf(stderr, "grow: abort due another grow in progress\n"); +#endif + return -1; + } + + bucket_count = 1U << dtd->bucket_count_bits; + bucket_used = fy_id_count_used(dtd->buckets_in_use, dtd->bucket_id_count); + occupancy_ratio = (double)bucket_used/(double)bucket_count; + + /* do not grow until we're over 60% full */ + if (occupancy_ratio < da->cfg.minimum_bucket_occupancy) { +#ifdef DEBUG_GROWS + fprintf(stderr, "grow: abort due to less that %f%% full (is %f)\n", + 100 * da->cfg.minimum_bucket_occupancy, + 100 * occupancy_ratio); +#endif + goto out_ok; + } + + new_dtd = fy_parent_allocator_alloc(&da->a, sizeof(*new_dtd), _Alignof(struct fy_dedup_tag_data)); + if (!new_dtd) + goto err_out; + + /* prepare the new one */ + rc = fy_dedup_tag_prepare_new(dtd, new_dtd, bloom_filter_adjust_bits, bucket_adjust_bits); + if (rc) + goto err_out; + +#ifdef DEBUG_GROWS + { + size_t bloom_count, new_bloom_count, bloom_used; + size_t bucket_count, new_bucket_count, bucket_used; + + bloom_count = 1U << dtd->bloom_filter_bits; + new_bloom_count = 1U << new_dtd->bloom_filter_bits; + bloom_used = fy_id_count_used(dtd->bloom_id, dtd->bloom_id_count); + bucket_count = 1U << dtd->bucket_count_bits; + new_bucket_count = 1U << new_dtd->bucket_count_bits; + bucket_used = fy_id_count_used(dtd->buckets_in_use, dtd->bucket_id_count); + + fprintf(stderr, "grow: chain_length_grow_trigger=%u->%u bloom %zu->%zu used %zu (%2.2f%%) ", + dtd->chain_length_grow_trigger, new_dtd->chain_length_grow_trigger, + bloom_count, new_bloom_count, + bloom_used, 100.0*(double)bloom_used/(double)bloom_count); + fprintf(stderr, "bucket %zu->%zu used %zu (%2.2f%%)\n", bucket_count, new_bucket_count, + bucket_used, 100.0*(double)bucket_used/(double)bucket_count); + } +#endif + + /* try to add it, if the head changed, then drop our stuff and pretend nothing happened */ + new_dtd->next = dtd; + if (!fy_atomic_compare_exchange_strong(&dt->tag_datas, &dtd, new_dtd)) { +#ifdef DEBUG_GROWS + fprintf(stderr, "grow: Update cancelled, someone beat us to it\n"); +#endif + fy_dedup_tag_data_cleanup(new_dtd); + fy_parent_allocator_free(&da->a, new_dtd); + } + +out_ok: + rc = 0; +out: + fy_atomic_flag_clear(&dt->growing); + return rc; + +err_out: + fy_parent_allocator_free(&da->a, new_dtd); + rc = -1; + goto out; +} + +static void fy_dedup_tag_trim(struct fy_dedup_allocator *da, struct fy_dedup_tag *dt) +{ + if (!da || !dt) + return; + + /* just pass them trim down to the parent */ + if (da->parent_caps & FYACF_CAN_FREE_TAG) + fy_allocator_trim_tag(da->parent_allocator, dt->content_tag); +} + +static void fy_dedup_tag_reset(struct fy_dedup_allocator *da, struct fy_dedup_tag *dt) +{ + struct fy_dedup_tag_data *dtd, *dtd_head; + size_t i; + + if (!da || !dt) + return; + + /* just pass them reset down to the parent */ + if (da->parent_caps & FYACF_CAN_FREE_TAG) + fy_allocator_reset_tag(da->parent_allocator, dt->content_tag); + + /* pop the head which we will keep */ + + dtd_head = NULL; + while ((dtd = fy_atomic_load(&dt->tag_datas)) != NULL) { + if (!fy_atomic_compare_exchange_strong(&dt->tag_datas, &dtd, dtd->next)) + continue; + + dtd->next = NULL; + + /* keep the head as is */ + if (!dtd_head) + dtd_head = dtd; + else + fy_dedup_tag_data_destroy(dtd); + } + + if (dtd_head) { + dtd = dtd_head; + + dtd->next = NULL; + fy_id_reset(dtd->bloom_id, dtd->bloom_id_count); + for (i = 0; i < dtd->bucket_count; i++) + atomic_store(&dtd->buckets[i], NULL); + fy_id_reset(dtd->buckets_in_use, dtd->bucket_id_count); + fy_atomic_store(&dt->tag_datas, dtd); + } +} + +static void fy_dedup_cleanup(struct fy_allocator *a); + +#define BUCKET_ESTIMATE_DIV 1024 +#define BLOOM_ESTIMATE_DIV 128 + +static void fy_dedup_cleanup(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + unsigned int i; + + if (!a) + return; + + da = container_of(a, struct fy_dedup_allocator, a); + + for (i = 0; i < da->tag_count; i++) { + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + fy_dedup_tag_cleanup(da, dt); + } + + fy_parent_allocator_free(&da->a, da->ids); + fy_parent_allocator_free(&da->a, da->tags); +} + +static int fy_dedup_setup(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *cfg_data) +{ + struct fy_dedup_allocator *da = NULL; + const struct fy_dedup_allocator_cfg *cfg; + unsigned int bloom_filter_bits, bucket_count_bits; + unsigned int bit_shift, chain_length_grow_trigger; + size_t dedup_threshold, tmpsz; + bool has_estimate; + struct fy_dedup_tag *dt; + int rc; + + if (!a || !cfg_data) + return -1; + + cfg = cfg_data; + if (!cfg->parent_allocator) + return -1; + + has_estimate = cfg->estimated_content_size && cfg->estimated_content_size != SIZE_MAX; + + /* power of two so ffs = log2 */ + bit_shift = (unsigned int)fy_id_ffs(FY_ID_BITS_BITS); + + bucket_count_bits = cfg->bucket_count_bits; + if (!bucket_count_bits && has_estimate) { + bucket_count_bits = 1; + while ((1LU << bucket_count_bits) < (cfg->estimated_content_size / BUCKET_ESTIMATE_DIV)) + bucket_count_bits++; +#ifdef DEBUG_GROWS + fprintf(stderr, "bucket_count_bits %u\n", bucket_count_bits); +#endif + } + /* at least that amount */ + if (bucket_count_bits < bit_shift) + bucket_count_bits = bit_shift; + /* keep the bucket count bits in signed int range */ + if (bucket_count_bits > (sizeof(int) * 8 - 1)) + bucket_count_bits = (sizeof(int) * 8) - 1; + + bloom_filter_bits = cfg->bloom_filter_bits; + if (!bloom_filter_bits && has_estimate) { + bloom_filter_bits = 1; + while ((1LU << bloom_filter_bits) < (cfg->estimated_content_size / BLOOM_ESTIMATE_DIV)) + bloom_filter_bits++; +#ifdef DEBUG_GROWS + fprintf(stderr, "bloom_filter_bits %u\n", bloom_filter_bits); +#endif + } + /* must be more than bucket count bits */ + if (bloom_filter_bits < bucket_count_bits) + bloom_filter_bits = bucket_count_bits + 3; /* minimum fanout */ + /* keep the bloom filter bits in signed int range */ + if (bloom_filter_bits > (sizeof(int) * 8 - 1)) + bloom_filter_bits = (sizeof(int) * 8) - 1; + + dedup_threshold = cfg->dedup_threshold; + chain_length_grow_trigger = cfg->chain_length_grow_trigger; + + da = container_of(a, struct fy_dedup_allocator, a); + memset(da, 0, sizeof(*da)); + + da->a.name = "dedup"; + da->a.ops = &fy_dedup_allocator_ops; + da->a.parent = parent; + da->a.parent_tag = parent_tag; + da->cfg = *cfg; + + da->parent_allocator = cfg->parent_allocator; + da->parent_caps = fy_allocator_get_caps(da->parent_allocator); + + da->bloom_filter_bits = bloom_filter_bits; + da->bucket_count_bits = bucket_count_bits; + da->dedup_threshold = dedup_threshold; + da->chain_length_grow_trigger = chain_length_grow_trigger; + + /* we use as many tags as the parent allocator */ + rc = fy_allocator_get_tag_count(da->parent_allocator); + if (rc <= 0) + goto err_out; + da->tag_count = (unsigned int)rc; + da->tag_id_count = (da->tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + tmpsz = da->tag_id_count * sizeof(*da->ids); + da->ids = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(fy_id_bits)); + if (!da->ids) + goto err_out; + fy_id_reset(da->ids, da->tag_id_count); + + tmpsz = da->tag_count * sizeof(*da->tags); + da->tags = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(struct fy_dedup_tag)); + if (!da->tags) + goto err_out; + + /* start with tag 0 as general use */ + fy_id_set_used(da->ids, da->tag_id_count, 0); + dt = fy_dedup_tag_from_tag(da, 0); + rc = fy_dedup_tag_setup(da, dt); + if (rc) + goto err_out; + + return 0; +err_out: + fy_dedup_cleanup(a); + return -1; +} + +struct fy_allocator *fy_dedup_create(struct fy_allocator *parent, int parent_tag, const void *cfg) +{ + struct fy_dedup_allocator *da = NULL; + int rc; + + da = fy_early_parent_allocator_alloc(parent, parent_tag, sizeof(*da), _Alignof(struct fy_dedup_allocator)); + if (!da) + goto err_out; + + rc = fy_dedup_setup(&da->a, parent, parent_tag, cfg); + if (rc) + goto err_out; + + return &da->a; + +err_out: + fy_early_parent_allocator_free(parent, parent_tag, da); + return NULL; +} + +void fy_dedup_destroy(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + + da = container_of(a, struct fy_dedup_allocator, a); + fy_dedup_cleanup(a); + + fy_parent_allocator_free(a, da); +} + +void fy_dedup_dump(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + unsigned int i; + + da = container_of(a, struct fy_dedup_allocator, a); + + fprintf(stderr, "dedup: "); + for (i = 0; i < da->tag_count; i++) { + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + fprintf(stderr, "%c", fy_id_is_free(da->ids, da->tag_id_count, i) ? '.' : 'x'); + } + fprintf(stderr, "\n"); + + for (i = 0; i < da->tag_count; i++) { + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + fprintf(stderr, " %d: tags: content=%d\n", i, dt->content_tag); + } + + fprintf(stderr, "dedup: dumping parent allocator\n"); + fy_allocator_dump(da->parent_allocator); +} + +static void *fy_dedup_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + void *p; + + da = container_of(a, struct fy_dedup_allocator, a); + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + goto err_out; + + /* just pass to the parent allocator using the content tag */ + p = fy_allocator_alloc_nocheck(da->parent_allocator, dt->content_tag, size, align); + if (!p) + goto err_out; + + return p; + +err_out: + return NULL; +} + +static void fy_dedup_free(struct fy_allocator *a, int tag, void *data) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + + da = container_of(a, struct fy_dedup_allocator, a); + if (!(da->parent_caps & FYACF_CAN_FREE_INDIVIDUAL)) + return; + + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return; + + /* just pass to the parent allocator */ + fy_allocator_free(da->parent_allocator, dt->content_tag, data); +} + +static int fy_dedup_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + int r; + + da = container_of(a, struct fy_dedup_allocator, a); + + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return -1; + + r = fy_allocator_update_stats(da->parent_allocator, dt->content_tag, stats); + if (r) + return -1; + + stats->unique_stores += fy_atomic_get_and_clear_counter(&dt->unique_stores); + stats->dup_stores += fy_atomic_get_and_clear_counter(&dt->dup_stores); + stats->collisions += fy_atomic_get_and_clear_counter(&dt->collisions); + + return 0; +} + +/* we can lookup! */ +static const void *fy_dedup_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + struct fy_dedup_tag_data *dtd; + struct fy_dedup_entry *de; + int bloom_pos, bucket_pos; + bool bloom_hit; + size_t total_size; + + da = container_of(a, struct fy_dedup_allocator, a); + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return NULL; + + /* calculate data total size */ + total_size = fy_iovec_size(iov, iovcnt); + if (total_size == SIZE_MAX) + return NULL; + + /* if it's under the dedup threshold just allocate and copy */ + if (total_size < da->cfg.dedup_threshold) + return NULL; + + /* calculate hash if not given */ + if (!hash) + hash = fy_iovec_xxhash64(iov, iovcnt); + + for (dtd = fy_atomic_load(&dt->tag_datas); dtd; dtd = dtd->next) { + bloom_pos = (int)(hash & (uint64_t)dtd->bloom_filter_mask); + bloom_hit = fy_id_is_used(dtd->bloom_id, dtd->bloom_id_count, bloom_pos); + if (bloom_hit) { + bucket_pos = (int)(hash & (uint64_t)dtd->bucket_count_mask); + for (de = fy_atomic_load(&dtd->buckets[bucket_pos]); de; de = de->next) { + if (de->hash == hash && total_size == de->size && + !fy_iovec_cmp(iov, iovcnt, de->mem) && + ((uintptr_t)de->mem & (align - 1)) == 0) + return de->mem; + } + } + } + return NULL; +} + +static const void *fy_dedup_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + struct fy_dedup_tag_data *dtd, *dtd_best; + struct fy_dedup_entry *de, *de_head; + int bloom_pos, bucket_pos; + bool bloom_hit; + unsigned int chain_length; + void *mem = NULL; + size_t total_size, de_offset, max_align; + bool at_head; + + da = container_of(a, struct fy_dedup_allocator, a); + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return NULL; + + /* calculate data total size */ + total_size = fy_iovec_size(iov, iovcnt); + if (total_size == SIZE_MAX) + return NULL; + + /* if it's under the dedup threshold just allocate and copy */ + if (total_size < da->cfg.dedup_threshold) { + + /* just pass to the parent allocator using the content tag */ + void *p = fy_allocator_alloc_nocheck(da->parent_allocator, dt->content_tag, total_size, align); + if (!p) + return NULL; + + fy_iovec_copy_from(iov, iovcnt, p); + return p; + } + + /* calculate hash if not given */ + if (!hash) + hash = fy_iovec_xxhash64(iov, iovcnt); + +again: + de = NULL; + chain_length = 0; + + at_head = true; + dtd_best = NULL; + for (dtd = fy_atomic_load(&dt->tag_datas); dtd; dtd = dtd->next) { + + /* first check in the bloom filter */ + bloom_pos = (int)(hash & (uint64_t)dtd->bloom_filter_mask); + bucket_pos = (int)(hash & (uint64_t)dtd->bucket_count_mask); + + bloom_hit = fy_id_is_used(dtd->bloom_id, dtd->bloom_id_count, bloom_pos); + if (bloom_hit) { + for (de = fy_atomic_load(&dtd->buckets[bucket_pos]); de; de = de->next) { + if (de->hash == hash) { + if (total_size == de->size && !fy_iovec_cmp(iov, iovcnt, de->mem) && + ((uintptr_t)de->mem & (align - 1)) == 0) { + + /* only update stats if someone asked for it */ + if (a->flags & FYAF_KEEP_STATS) + fy_atomic_fetch_add(&dt->dup_stores, 1); + + if (a->flags & FYAF_TRACE) { + int i; + size_t j; + printf("%s: %p: dup-store %p 0x%016"PRIx64" %zx:", __func__, + a, de->mem, hash, total_size); + for (i = 0; i < iovcnt; i++) { + for (j = 0; j < iov[i].iov_len; j++) { + printf("%02x", (int)(((uint8_t *)iov[i].iov_base)[j]) & 0xff); + } + } + printf("\n"); + } + + return de->mem; + } + + /* mark that we had a collision here */ + if (a->flags & FYAF_KEEP_STATS) + fy_atomic_fetch_add(&dt->collisions, 1); + } + + if (at_head) + chain_length++; + } + } else { + if (!dtd_best) + dtd_best = dtd; /* keep whenever we had a empty bloom slot */ + } + at_head = false; + } + + /* use the one that had space, otherwise the top */ + if (dtd_best) + dtd = dtd_best; + else + dtd = fy_atomic_load(&dt->tag_datas); + + /* recalc positions for the delected dtd */ + bloom_pos = (int)(hash & (uint64_t)dtd->bloom_filter_mask); + bucket_pos = (int)(hash & (uint64_t)dtd->bucket_count_mask); + + /* we might be retrying; don't allocate and copy again */ + if (!mem) { + /* place the dedup entry at the aligned offset after the data */ + de_offset = fy_size_t_align(total_size, _Alignof(struct fy_dedup_entry)); + max_align = align > _Alignof(struct fy_dedup_entry) ? align : _Alignof(struct fy_dedup_entry); + mem = fy_allocator_alloc_nocheck(da->parent_allocator, dt->content_tag, de_offset + sizeof(*de), max_align); + if (!mem) + return NULL; + + /* verify it's aligned correctly */ + assert(((uintptr_t)mem & (align - 1)) == 0); + + de = mem + de_offset; + + de->hash = hash; + de->size = total_size; + de->mem = mem; + + /* and copy the data */ + fy_iovec_copy_from(iov, iovcnt, de->mem); + } + + /* set this bucket to used */ + fy_id_set_used(dtd->buckets_in_use, dtd->bucket_id_count, bucket_pos); + + /* turn the update bit for the bloom position */ + fy_id_set_used(dtd->bloom_id, dtd->bloom_id_count, bloom_pos); + + /* add to the bucket last atomically */ + de_head = fy_atomic_load(&dtd->buckets[bucket_pos]); + de->next = de_head; + if (!fy_atomic_compare_exchange_strong(&dtd->buckets[bucket_pos], &de_head, de)) + goto again; // we're leaking the entry, but that's fine + + /* adjust by one bit, if we've hit the trigger */ + if (chain_length > dtd->chain_length_grow_trigger) + fy_dedup_tag_adjust(da, dt, 1, 1); + + /* only update stats if someone asked for it */ + if (a->flags & FYAF_KEEP_STATS) + fy_atomic_fetch_add(&dt->unique_stores, 1); + + if (a->flags & FYAF_TRACE) { + int i; + size_t j; + printf("%s: %p: new-store %p 0x%016"PRIx64" %zx:", __func__, + a, de->mem, hash, total_size); + for (i = 0; i < iovcnt; i++) { + for (j = 0; j < iov[i].iov_len; j++) { + printf("%02x", (int)(((uint8_t *)iov[i].iov_base)[j]) & 0xff); + } + } + printf("\n"); + } + + return de->mem; +} + +static void fy_dedup_release(struct fy_allocator *a, int tag, const void *data, size_t size) +{ + /* we do nothing */ +} + +static int fy_dedup_get_tag(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt = NULL; + int tag; + int id, rc; + + da = container_of(a, struct fy_dedup_allocator, a); + + /* for a single tag, just return 0 */ + if (!(da->parent_caps & FYACF_CAN_FREE_TAG)) + return 0; + + /* and one from us */ + id = fy_id_alloc(da->ids, da->tag_id_count); + if (id < 0) + goto err_out; + + tag = (int)id; + + dt = fy_dedup_tag_from_tag(da, tag); + assert(dt); + + rc = fy_dedup_tag_setup(da, dt); + if (rc) + goto err_out; + + return tag; + +err_out: + fy_dedup_tag_cleanup(da, dt); + if (id >= 0) + fy_id_free(da->ids, da->tag_id_count, id); + return FY_ALLOC_TAG_ERROR; +} + +static void fy_dedup_release_tag(struct fy_allocator *a, int tag) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + + da = container_of(a, struct fy_dedup_allocator, a); + + /* for a single tag, just return */ + if (!(da->parent_caps & FYACF_CAN_FREE_TAG)) + return; + + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return; + + fy_dedup_tag_cleanup(da, dt); + + fy_id_free(da->ids, da->tag_count, tag); +} + +static int fy_dedup_get_tag_count(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + + da = container_of(a, struct fy_dedup_allocator, a); + return da->tag_count; +} + +static int fy_dedup_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag_data *dtd; + fy_id_bits *ids = NULL, *alloc_ids = NULL; + struct fy_dedup_tag *dt, *dt_new, *tags = NULL, *alloc_tags = NULL; + unsigned int i, tag_count, tag_id_count; + size_t tmpsz; + int rc; + + if (count < 1) + return -1; + + da = container_of(a, struct fy_dedup_allocator, a); + + if (count == da->tag_count) + return 0; + + tag_count = count; + tag_id_count = (tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + /* check if all content tag are within limits */ + for (i = 0; i < da->tag_count; i++) { + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + if (dt->content_tag >= (int)tag_count) + return -1; + } + + if (count > da->tag_count) { + /* we need to grow */ + tmpsz = tag_id_count * sizeof(*da->ids); + alloc_ids = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(fy_id_bits)); + if (!alloc_ids) + goto err_out; + fy_id_reset(alloc_ids, tag_id_count); + + tmpsz = tag_count * sizeof(*tags); + alloc_tags = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(struct fy_dedup_tag)); + if (!alloc_tags) + goto err_out; + } + + /* shrink?, just clip */ + if (count < da->tag_count) { + tags = da->tags; + ids = da->ids; + for (i = count; i < da->tag_count; i++) { + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + fy_dedup_tag_cleanup(da, dt); + fy_id_free(da->ids, da->tag_id_count, i); + } + } else { + tags = alloc_tags; + ids = alloc_ids; + /* copy over the old entries */ + for (i = 0; i < da->tag_count; i++) { + + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + + dt_new = &tags[i]; + + fy_dedup_tag_setup(da, dt_new); + + /* move the old entries over */ + do { + dtd = fy_atomic_load(&dt->tag_datas); + } while (fy_atomic_compare_exchange_strong(&dt->tag_datas, &dtd, NULL)); + fy_atomic_store(&dt_new->tag_datas, dtd); + dt_new->content_tag = dt->content_tag; + fy_atomic_flag_clear(&dt_new->growing); + fy_atomic_store(&dt_new->unique_stores, fy_atomic_load(&dt->unique_stores)); + fy_atomic_store(&dt_new->dup_stores, fy_atomic_load(&dt->dup_stores)); + fy_atomic_store(&dt_new->collisions, fy_atomic_load(&dt->collisions)); + + /* clean the old entry */ + fy_dedup_tag_cleanup(da, dt); + + fy_id_set_used(ids, tag_id_count, i); + } + } + + /* ok, drop the parent bits first */ + rc = fy_allocator_set_tag_count(da->parent_allocator, tag_count); + if (rc) + goto err_out; + + /* switch */ + if (da->tags != tags) { + da->tags = tags; + fy_parent_allocator_free(&da->a, da->tags); + } + + if (da->ids != ids) { + da->ids = ids; + fy_parent_allocator_free(&da->a, da->ids); + } + + da->tag_count = tag_count; + da->tag_id_count = tag_id_count; + + return 0; + +err_out: + fy_parent_allocator_free(&da->a, alloc_tags); + fy_parent_allocator_free(&da->a, alloc_ids); + return -1; +} + +static void fy_dedup_trim_tag(struct fy_allocator *a, int tag) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + + da = container_of(a, struct fy_dedup_allocator, a); + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return; + + fy_dedup_tag_trim(da, dt); +} + +static void fy_dedup_reset_tag(struct fy_allocator *a, int tag) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + + da = container_of(a, struct fy_dedup_allocator, a); + + /* if it can't free an individual tag it can't reset it */ + if (!(da->parent_caps & FYACF_CAN_FREE_TAG)) + return; + + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return; + + fy_dedup_tag_reset(da, dt); +} + +static struct fy_allocator_info * +fy_dedup_get_info(struct fy_allocator *a, int tag) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + unsigned int i; + + /* full dump not supported yet */ + if (tag == FY_ALLOC_TAG_NONE) + return NULL; + + da = container_of(a, struct fy_dedup_allocator, a); + + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return NULL; + + info = fy_allocator_get_info(da->parent_allocator, dt->content_tag); + if (!info) + return NULL; + + /* we will have to change the tag from content to this one */ + for (i = 0; i < info->num_tag_infos; i++) { + tag_info = &info->tag_infos[i]; + if (tag_info->tag == dt->content_tag) + tag_info->tag = tag; + } + + return info; +} + +static enum fy_allocator_cap_flags +fy_dedup_get_caps(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + enum fy_allocator_cap_flags flags; + + da = container_of(a, struct fy_dedup_allocator, a); + + flags = da->parent_caps | FYACF_CAN_DEDUP | FYACF_CAN_LOOKUP; + flags &= ~FYACF_CAN_FREE_INDIVIDUAL; + + return flags; +} + +static bool fy_dedup_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + int tag_start, tag_end; + + da = container_of(a, struct fy_dedup_allocator, a); + if (!(da->parent_caps & FYACF_HAS_CONTAINS)) + return false; + + if (tag >= 0) { + if (tag >= (int)da->tag_count) + return false; + tag_start = tag; + tag_end = tag + 1; + } else { + tag_start = 0; + tag_end = (int)da->tag_count; + } + + for (tag = tag_start; tag < tag_end; tag++) { + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + continue; + + if (fy_allocator_contains(da->parent_allocator, dt->content_tag, ptr)) + return true; + } + return false; +} + +const struct fy_allocator_ops fy_dedup_allocator_ops = { + .setup = fy_dedup_setup, + .cleanup = fy_dedup_cleanup, + .create = fy_dedup_create, + .destroy = fy_dedup_destroy, + .dump = fy_dedup_dump, + .alloc = fy_dedup_alloc, + .free = fy_dedup_free, + .update_stats = fy_dedup_update_stats, + .storev = fy_dedup_storev, + .lookupv = fy_dedup_lookupv, + .release = fy_dedup_release, + .get_tag = fy_dedup_get_tag, + .release_tag = fy_dedup_release_tag, + .get_tag_count = fy_dedup_get_tag_count, + .set_tag_count = fy_dedup_set_tag_count, + .trim_tag = fy_dedup_trim_tag, + .reset_tag = fy_dedup_reset_tag, + .get_info = fy_dedup_get_info, + .get_caps = fy_dedup_get_caps, + .contains = fy_dedup_contains, +}; diff --git a/src/allocator/fy-allocator-dedup.h b/src/allocator/fy-allocator-dedup.h new file mode 100644 index 0000000..967fe23 --- /dev/null +++ b/src/allocator/fy-allocator-dedup.h @@ -0,0 +1,89 @@ +/* + * fy-allocator-dedup.h - the dedup allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_DEDUP_H +#define FY_ALLOCATOR_DEDUP_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "fy-typelist.h" +#include "fy-id.h" +#include "xxhash.h" + +#include "fy-atomics.h" +#include "fy-allocator.h" + +#define FY_DEDUP_TAG_MAX 128 + +struct fy_dedup_entry { + struct fy_dedup_entry *next; + uint64_t hash; + size_t size; + void *mem; +}; + +struct fy_dedup_tag; + +struct fy_dedup_tag_data_cfg { + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + unsigned int bloom_filter_bits; + unsigned int bucket_count_bits; + size_t dedup_threshold; + unsigned int chain_length_grow_trigger; +}; + +struct fy_dedup_tag_data { + struct fy_dedup_tag_data *next; + struct fy_dedup_tag_data_cfg cfg; + unsigned int bloom_filter_bits; + unsigned int bloom_filter_mask; + size_t bloom_id_count; + fy_id_bits *bloom_id; + fy_id_bits *bloom_update_id; + unsigned int bucket_count_bits; + unsigned int bucket_count_mask; + size_t bucket_count; + FY_ATOMIC(struct fy_dedup_entry *) *buckets; + size_t bucket_id_count; + fy_id_bits *buckets_in_use; + size_t dedup_threshold; + unsigned int chain_length_grow_trigger; +}; + +struct fy_dedup_tag { + FY_ATOMIC(struct fy_dedup_tag_data *)tag_datas; + int content_tag; + fy_atomic_flag growing; + FY_ATOMIC(uint64_t) unique_stores; + FY_ATOMIC(uint64_t) dup_stores; + FY_ATOMIC(uint64_t) collisions; +}; + +struct fy_dedup_allocator { + struct fy_allocator a; + struct fy_dedup_allocator_cfg cfg; + struct fy_allocator *parent_allocator; + enum fy_allocator_cap_flags parent_caps; + int entries_tag; + unsigned int bloom_filter_bits; + unsigned int bucket_count_bits; + size_t dedup_threshold; + unsigned int chain_length_grow_trigger; + fy_id_bits *ids; + unsigned int tag_id_count; + struct fy_dedup_tag *tags; + unsigned int tag_count; +}; + +extern const struct fy_allocator_ops fy_dedup_allocator_ops; + +#define FY_DEDUP_XXHASH64_SEED FY_XXHASH64_SEED + +#endif diff --git a/src/allocator/fy-allocator-linear.c b/src/allocator/fy-allocator-linear.c new file mode 100644 index 0000000..f516f9e --- /dev/null +++ b/src/allocator/fy-allocator-linear.c @@ -0,0 +1,395 @@ +/* + * fy-allocator-linear.c - linear allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + + +#include + +#include "fy-utils.h" +#include "fy-allocator-linear.h" + +static int fy_linear_setup(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *cfg_data) +{ + struct fy_linear_allocator *la; + const struct fy_linear_allocator_cfg *cfg; + void *buf, *alloc = NULL; + bool need_zero; + + if (!a || !cfg_data) + return -1; + + cfg = cfg_data; + if (!cfg->size) + return -1; + + if (!cfg->buf) { + alloc = fy_early_parent_allocator_alloc(parent, parent_tag, cfg->size, 0); + if (!alloc) + goto err_out; + buf = alloc; + need_zero = true; + } else { + buf = cfg->buf; + need_zero = false; + } + + la = container_of(a, struct fy_linear_allocator, a); + memset(la, 0, sizeof(*la)); + la->a.name = "linear"; + la->a.ops = &fy_linear_allocator_ops; + la->a.parent = parent; + la->a.parent_tag = parent_tag; + la->cfg = *cfg; + la->alloc = alloc; + la->start = buf; + la->need_zero = need_zero; + + fy_atomic_store(&la->next, 0); + + return 0; +err_out: + fy_early_parent_allocator_free(parent, parent_tag, alloc); + return -1; +} + +static void fy_linear_cleanup(struct fy_allocator *a) +{ + struct fy_linear_allocator *la; + + if (!a) + return; + + la = container_of(a, struct fy_linear_allocator, a); + fy_parent_allocator_free(&la->a, la->alloc); + la->alloc = NULL; +} + +struct fy_allocator *fy_linear_create(struct fy_allocator *parent, int parent_tag, const void *cfg_data) +{ + struct fy_linear_allocator *la; + const struct fy_linear_allocator_cfg *cfg; + struct fy_linear_allocator_cfg newcfg; + void *s, *e, *buf, *alloc = NULL; + int rc; + + if (!cfg_data) + return NULL; + + cfg = cfg_data; + if (!cfg->size) + return NULL; + + if (!cfg->buf) { + alloc = fy_early_parent_allocator_alloc(parent, parent_tag, cfg->size, 0); + if (!alloc) + goto err_out; + buf = alloc; + } else + buf = cfg->buf; + + s = buf; + e = s + cfg->size; + + s = fy_ptr_align(s, _Alignof(struct fy_linear_allocator)); + if ((size_t)(e - s) < sizeof(*la)) + goto err_out; + + la = s; + s += sizeof(*la); + + memset(&newcfg, 0, sizeof(newcfg)); + newcfg.buf = s; + newcfg.size = (size_t)(e - s); + + rc = fy_linear_setup(&la->a, parent, parent_tag, &newcfg); + if (rc) + goto err_out; + + la->alloc = alloc; + + return &la->a; + +err_out: + fy_early_parent_allocator_free(parent, parent_tag, alloc); + return NULL; +} + +void fy_linear_destroy(struct fy_allocator *a) +{ + struct fy_linear_allocator *la; + struct fy_allocator *parent; + int parent_tag; + void *alloc; + + la = container_of(a, struct fy_linear_allocator, a); + + /* take out the allocation of create */ + alloc = la->alloc; + la->alloc = NULL; + parent = la->a.parent; + parent_tag = la->a.parent_tag; + + fy_linear_cleanup(a); + + fy_early_parent_allocator_free(parent, parent_tag, alloc); +} + +void fy_linear_dump(struct fy_allocator *a) +{ + struct fy_linear_allocator *la; + size_t next; + + la = container_of(a, struct fy_linear_allocator, a); + + next = fy_atomic_load(&la->next); + fprintf(stderr, "linear: total %zu used %zu free %zu\n", + la->cfg.size, next, la->cfg.size - la->next); +} + +static void *fy_linear_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + struct fy_linear_allocator *la; + size_t next, new_next, real_size; + void *s; + + assert(a); + + la = container_of(a, struct fy_linear_allocator, a); + + /* atomically update the pointer */ + do { + next = fy_atomic_load(&la->next); + s = fy_ptr_align(la->start + next, align); + new_next = (s + size) - la->start; + /* handle both overflow and underflow */ + if (new_next > la->cfg.size) + goto err_out; + } while (!fy_atomic_compare_exchange_strong(&la->next, &next, new_next)); + + real_size = new_next - next; + + /* only update stats if someone asked for it */ + if (a->flags & FYAF_KEEP_STATS) { + fy_atomic_fetch_add(&la->allocations, 1); + fy_atomic_fetch_add(&la->allocated, real_size); + } + + /* zero out buffer if not guaranteed */ + if (la->need_zero) + memset(s, 0, size); + + return s; + +err_out: + return NULL; +} + +static void fy_linear_free(struct fy_allocator *a, int tag, void *data) +{ + /* linear allocator does not free anything */ +} + +static int fy_linear_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + struct fy_linear_allocator *la; + + la = container_of(a, struct fy_linear_allocator, a); + + stats->allocations = fy_atomic_get_and_clear_counter(&la->allocations); + stats->allocated = fy_atomic_get_and_clear_counter(&la->allocated); + + return 0; +} + +static const void *fy_linear_storev(struct fy_allocator *a, int tag, + const struct iovec *iov, int iovcnt, size_t align, + uint64_t hash) +{ + struct fy_linear_allocator *la; + void *p; + size_t size; + + size = fy_iovec_size(iov, iovcnt); + if (size == SIZE_MAX) + return NULL; + + p = fy_linear_alloc(a, tag, size, align); + if (!p) + return NULL; + + fy_iovec_copy_from(iov, iovcnt, p); + + if (a->flags & FYAF_KEEP_STATS) { + la = container_of(a, struct fy_linear_allocator, a); + + fy_atomic_fetch_add(&la->stores, 1); + fy_atomic_fetch_add(&la->stores, size); + } + + return p; +} + +static const void *fy_linear_lookupv(struct fy_allocator *a, int tag, + const struct iovec *iov, int iovcnt, size_t align, + uint64_t hash) +{ + /* no way to lookup */ + return NULL; +} + +static void fy_linear_release(struct fy_allocator *a, int tag, const void *data, size_t size) +{ + /* nothing */ +} + +static int fy_linear_get_tag(struct fy_allocator *a) +{ + /* always return 0, we don't do tags for linear */ + return 0; +} + +static void fy_linear_release_tag(struct fy_allocator *a, int tag) +{ + /* nothing */ +} + +static int fy_linear_get_tag_count(struct fy_allocator *a) +{ + return 1; +} + +static int fy_linear_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + if (count != 1) + return -1; + return 0; +} + +static void fy_linear_trim_tag(struct fy_allocator *a, int tag) +{ + /* nothing */ +} + +static void fy_linear_reset_tag(struct fy_allocator *a, int tag) +{ + struct fy_linear_allocator *la; + + if (tag) + return; + + la = container_of(a, struct fy_linear_allocator, a); + + /* we just rewind */ + fy_atomic_store(&la->next, 0); +} + +static struct fy_allocator_info *fy_linear_get_info(struct fy_allocator *a, int tag) +{ + struct fy_linear_allocator *la; + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + struct fy_allocator_arena_info *arena_info; + size_t next, size; + + /* only single tag (or all tags with 0) */ + if (tag != 0 && tag != FY_ALLOC_TAG_NONE) + return NULL; + + la = container_of(a, struct fy_linear_allocator, a); + + /* one of each */ + size = sizeof(*info) + + sizeof(*tag_info) + + sizeof(*arena_info); + + info = malloc(size); + if (!info) + return NULL; + memset(info, 0, sizeof(*info)); + + tag_info = (void *)(info + 1); + assert(((uintptr_t)tag_info % _Alignof(struct fy_allocator_tag_info)) == 0); + arena_info = (void *)(tag_info + 1); + assert(((uintptr_t)arena_info % _Alignof(struct fy_allocator_arena_info)) == 0); + + /* fill-in the single arena */ + next = fy_atomic_load(&la->next); + + arena_info->free = la->cfg.size - next; + arena_info->used = next; + arena_info->total = la->cfg.size; + arena_info->data = la->start; + arena_info->size = arena_info->used; + + /* there's a single tag for linear */ + tag_info->tag = 0; + tag_info->free = arena_info->free; + tag_info->used = arena_info->used; + tag_info->total = arena_info->total; + tag_info->num_arena_infos = 1; + tag_info->arena_infos = arena_info; + + /* fill in the single tag */ + info->free = tag_info->free; + info->used = tag_info->used; + info->total = tag_info->total; + info->num_tag_infos = 1; + info->tag_infos = tag_info; + + return info; +} + +static enum fy_allocator_cap_flags +fy_linear_get_caps(struct fy_allocator *a) +{ + return FYACF_HAS_CONTAINS | FYACF_HAS_EFFICIENT_CONTAINS; +} + +static bool fy_linear_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + struct fy_linear_allocator *la; + + la = container_of(a, struct fy_linear_allocator, a); + return ptr >= la->start && ptr < (la->start + la->cfg.size); +} + +const struct fy_allocator_ops fy_linear_allocator_ops = { + .setup = fy_linear_setup, + .cleanup = fy_linear_cleanup, + .create = fy_linear_create, + .destroy = fy_linear_destroy, + .dump = fy_linear_dump, + .alloc = fy_linear_alloc, + .free = fy_linear_free, + .update_stats = fy_linear_update_stats, + .storev = fy_linear_storev, + .lookupv = fy_linear_lookupv, + .release = fy_linear_release, + .get_tag = fy_linear_get_tag, + .release_tag = fy_linear_release_tag, + .get_tag_count = fy_linear_get_tag_count, + .set_tag_count = fy_linear_set_tag_count, + .trim_tag = fy_linear_trim_tag, + .reset_tag = fy_linear_reset_tag, + .get_info = fy_linear_get_info, + .get_caps = fy_linear_get_caps, + .contains = fy_linear_contains, +}; diff --git a/src/allocator/fy-allocator-linear.h b/src/allocator/fy-allocator-linear.h new file mode 100644 index 0000000..a30a9e2 --- /dev/null +++ b/src/allocator/fy-allocator-linear.h @@ -0,0 +1,34 @@ +/* + * fy-allocator-linear.h - the linear allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_LINEAR_H +#define FY_ALLOCATOR_LINEAR_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "fy-atomics.h" +#include "fy-allocator.h" + +struct fy_linear_allocator { + FY_ATOMIC(size_t) next FY_CACHE_ALIGNED; // hot hot hot + struct fy_allocator a; + struct fy_linear_allocator_cfg cfg; + void *alloc; + void *start; + bool need_zero; + /* no need to keep anything else */ + FY_ATOMIC(uint64_t) allocations; + FY_ATOMIC(uint64_t) allocated; + FY_ATOMIC(uint64_t) stores; + FY_ATOMIC(uint64_t) stored; +}; + +extern const struct fy_allocator_ops fy_linear_allocator_ops; + +#endif diff --git a/src/allocator/fy-allocator-malloc.c b/src/allocator/fy-allocator-malloc.c new file mode 100644 index 0000000..7c06185 --- /dev/null +++ b/src/allocator/fy-allocator-malloc.c @@ -0,0 +1,717 @@ +/* + * fy-allocator-malloc.c - malloc allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include + +#include + +/* for container_of */ +#include "fy-list.h" +#include "fy-utils.h" + +#include "fy-allocator-malloc.h" + +static inline void +fy_malloc_tag_list_lock(struct fy_malloc_tag *mt) +{ + int loops FY_DEBUG_UNUSED; + + loops = 0; + while (!fy_atomic_flag_test_and_set(&mt->lock)) { + loops++; + assert(loops < 10000000); + fy_cpu_relax(); + } +} + +static inline void +fy_malloc_tag_list_unlock(struct fy_malloc_tag *mt) +{ + fy_atomic_flag_clear(&mt->lock); +} + +static inline struct fy_malloc_tag * +fy_malloc_tag_from_tag(struct fy_malloc_allocator *ma, int tag) +{ + if (!ma) + return NULL; + + if ((unsigned int)tag >= ma->tag_count) + return NULL; + + if (!fy_id_is_used(ma->ids, ma->tag_id_count, (int)tag)) + return NULL; + + return &ma->tags[tag]; +} + +static void fy_malloc_tag_setup(struct fy_malloc_allocator *ma, struct fy_malloc_tag *mt) +{ + fy_malloc_entry_list_init(&mt->entries); + atomic_flag_clear(&mt->lock); +} + +static void fy_malloc_tag_cleanup(struct fy_malloc_allocator *ma, struct fy_malloc_tag *mt) +{ + struct fy_malloc_entry *me; + + /* cleanup should happen from a single thread */ + while ((me = fy_malloc_entry_list_pop(&mt->entries)) != NULL) + free(me->mem); +} + +static void fy_malloc_cleanup(struct fy_allocator *a) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + unsigned int i; + + if (!a) + return; + + ma = container_of(a, struct fy_malloc_allocator, a); + + /* no need for locks, this is one thread only */ + for (i = 0; i < ma->tag_count; i++) { + mt = fy_malloc_tag_from_tag(ma, i); + if (!mt) + continue; + fy_malloc_tag_cleanup(ma, mt); + } + + fy_parent_allocator_free(&ma->a, ma->ids); + fy_parent_allocator_free(&ma->a, ma->tags); +} + +static int fy_malloc_setup(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *data) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + size_t tmpsz; + + if (!a) + return -1; + + ma = container_of(a, struct fy_malloc_allocator, a); + memset(ma, 0, sizeof(*ma)); + ma->a.name = "malloc"; + ma->a.ops = &fy_malloc_allocator_ops; + ma->a.parent = parent; + ma->a.parent_tag = parent_tag; + + ma->tag_count = FY_MALLOC_TAG_MAX; + ma->tag_id_count = (ma->tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + tmpsz = ma->tag_id_count * sizeof(*ma->ids); + ma->ids = fy_parent_allocator_alloc(&ma->a, tmpsz, _Alignof(fy_id_bits)); + if (!ma->ids) + goto err_out; + fy_id_reset(ma->ids, ma->tag_id_count); + + tmpsz = ma->tag_count * sizeof(*ma->tags); + ma->tags = fy_parent_allocator_alloc(&ma->a, tmpsz, _Alignof(struct fy_malloc_tag)); + if (!ma->tags) + goto err_out; + + /* start with tag 0 as general use */ + fy_id_set_used(ma->ids, ma->tag_id_count, 0); + mt = fy_malloc_tag_from_tag(ma, 0); + fy_malloc_tag_setup(ma, mt); + + return 0; + +err_out: + fy_malloc_cleanup(a); + return -1; +} + +struct fy_allocator *fy_malloc_create(struct fy_allocator *parent, int parent_tag, const void *cfg) +{ + struct fy_malloc_allocator *ma = NULL; + int rc; + + ma = fy_early_parent_allocator_alloc(parent, parent_tag, sizeof(*ma), _Alignof(struct fy_malloc_allocator)); + if (!ma) + goto err_out; + + rc = fy_malloc_setup(&ma->a, parent, parent_tag, cfg); + if (rc) + goto err_out; + + return &ma->a; + +err_out: + fy_early_parent_allocator_free(parent, parent_tag, ma); + return NULL; +} + +void fy_malloc_destroy(struct fy_allocator *a) +{ + struct fy_malloc_allocator *ma; + + ma = container_of(a, struct fy_malloc_allocator, a); + fy_malloc_cleanup(a); + + fy_parent_allocator_free(a, ma); +} + +void fy_malloc_dump(struct fy_allocator *a) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + struct fy_malloc_entry *me; + unsigned int i; + size_t count, total, system_total; + + ma = container_of(a, struct fy_malloc_allocator, a); + + fprintf(stderr, "malloc: "); + for (i = 0; i < ma->tag_count; i++) { + mt = fy_malloc_tag_from_tag(ma, i); + if (!mt) + continue; + fy_malloc_tag_list_lock(mt); + fprintf(stderr, "%c", fy_malloc_entry_list_empty(&mt->entries) ? '.' : 'x'); + fy_malloc_tag_list_unlock(mt); + } + fprintf(stderr, "\n"); + + for (i = 0; i < ma->tag_count; i++) { + mt = fy_malloc_tag_from_tag(ma, i); + if (!mt) + continue; + fy_malloc_tag_list_lock(mt); + count = total = system_total = 0; + if (!fy_malloc_entry_list_empty(&mt->entries)) { + for (me = fy_malloc_entry_list_head(&mt->entries); me; me = fy_malloc_entry_next(&mt->entries, me)) { + count++; + total += me->size; + system_total += sizeof(*me) + me->size; + } + } + fy_malloc_tag_list_unlock(mt); + if (!count) + continue; + fprintf(stderr, " %d: count %zu total %zu system %zu overhead %zu (%2.2f%%)\n", i, + count, total, system_total, system_total - total, + 100.0 * (double)(system_total - total) / (double)system_total); + } +} + +static void *fy_malloc_tag_alloc(struct fy_malloc_allocator *ma, struct fy_malloc_tag *mt, size_t size, size_t align) +{ + struct fy_malloc_entry *me; + size_t me_offset, max_align; + int r; + void *mem; + + me_offset = fy_size_t_align(size, _Alignof(struct fy_malloc_entry)); + max_align = align > _Alignof(struct fy_malloc_entry) ? align : _Alignof(struct fy_malloc_entry); + + r = posix_memalign(&mem, max_align, me_offset + sizeof(*me)); + if (r) + return NULL; + + me = mem + me_offset; + + me->size = size; + fy_malloc_tag_list_lock(mt); + fy_malloc_entry_list_add_tail(&mt->entries, me); + fy_malloc_tag_list_unlock(mt); + me->mem = mem; + + return mem; +} + +static void fy_malloc_tag_free(struct fy_malloc_allocator *ma, struct fy_malloc_tag *mt, void *data) +{ + struct fy_malloc_entry *me; + + me = container_of(data, struct fy_malloc_entry, mem); + + fy_malloc_tag_list_lock(mt); + fy_malloc_entry_list_del(&mt->entries, me); + fy_malloc_tag_list_unlock(mt); + + free(me->mem); +} + +static void *fy_malloc_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + void *p; + + ma = container_of(a, struct fy_malloc_allocator, a); + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + goto err_out; + + p = fy_malloc_tag_alloc(ma, mt, size, align); + if (!p) + goto err_out; + + mt->stats.allocations++; + mt->stats.allocated += size; + + return p; + +err_out: + return NULL; +} + +static void fy_malloc_free(struct fy_allocator *a, int tag, void *data) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + + ma = container_of(a, struct fy_malloc_allocator, a); + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + return; + + fy_malloc_tag_free(ma, mt, data); + + mt->stats.frees++; +} + +static int fy_malloc_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + unsigned int i; + + ma = container_of(a, struct fy_malloc_allocator, a); + + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + goto err_out; + + /* and update with this ones */ + for (i = 0; i < ARRAY_SIZE(mt->stats.counters); i++) { + stats->counters[i] += mt->stats.counters[i]; + mt->stats.counters[i] = 0; + } + + return 0; + +err_out: + return -1; +} + +static const void *fy_malloc_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + void *start; + size_t total_size; + + ma = container_of(a, struct fy_malloc_allocator, a); + + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + goto err_out; + + total_size = fy_iovec_size(iov, iovcnt); + if (total_size == SIZE_MAX) + goto err_out; + + start = fy_malloc_tag_alloc(ma, mt, total_size, align); + if (!start) + goto err_out; + + fy_iovec_copy_from(iov, iovcnt, start); + + mt->stats.stores++; + mt->stats.stored += total_size; + + return start; + +err_out: + return NULL; +} + +static const void *fy_malloc_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + /* no lookups */ + return NULL; +} + +static void fy_malloc_release(struct fy_allocator *a, int tag, const void *data, size_t size) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + + ma = container_of(a, struct fy_malloc_allocator, a); + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + return; + + /* the malloc's release is just a free */ + fy_malloc_tag_free(ma, mt, (void *)data); + + mt->stats.releases++; + mt->stats.released += size; +} + +static void fy_malloc_release_tag(struct fy_allocator *a, int tag) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + + ma = container_of(a, struct fy_malloc_allocator, a); + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + return; + + fy_malloc_tag_cleanup(ma, mt); + + fy_id_free(ma->ids, ma->tag_id_count, tag); +} + +static int fy_malloc_get_tag(struct fy_allocator *a) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + int id; + + ma = container_of(a, struct fy_malloc_allocator, a); + + id = fy_id_alloc(ma->ids, ma->tag_id_count); + if (id < 0) + goto err_out; + + mt = fy_malloc_tag_from_tag(ma, id); + if (!mt) + goto err_out; + + fy_malloc_tag_setup(ma, mt); + + return (int)id; + +err_out: + return FY_ALLOC_TAG_ERROR; +} + +static int fy_malloc_get_tag_count(struct fy_allocator *a) +{ + struct fy_malloc_allocator *ma; + + ma = container_of(a, struct fy_malloc_allocator, a); + return ma->tag_count; +} + +static int fy_malloc_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + struct fy_malloc_allocator *ma; + fy_id_bits *ids = NULL; + struct fy_malloc_tag *mt, *mt_new, *tags = NULL; + struct fy_malloc_entry *me; + unsigned int i, tag_count, tag_id_count; + size_t tmpsz; + + if (count < 1) + return -1; + + ma = container_of(a, struct fy_malloc_allocator, a); + if (count == ma->tag_count) + return 0; + + tag_count = count; + tag_id_count = (tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + /* shrink?, just clip */ + if (count < ma->tag_count) { + for (i = count; i < ma->tag_count; i++) { + mt = fy_malloc_tag_from_tag(ma, i); + if (!mt) + continue; + fy_malloc_tag_cleanup(ma, mt); + fy_id_free(ma->ids, ma->tag_id_count, i); + } + tags = ma->tags; + ids = ma->ids; + } else { + /* we need to grow */ + tmpsz = tag_id_count * sizeof(*ma->ids); + ids = fy_parent_allocator_alloc(&ma->a, tmpsz, _Alignof(fy_id_bits)); + if (!ids) + goto err_out; + fy_id_reset(ids, tag_id_count); + + tmpsz = tag_count * sizeof(*tags); + tags = fy_parent_allocator_alloc(&ma->a, tmpsz, _Alignof(struct fy_malloc_tag)); + if (!tags) + goto err_out; + + /* copy over the old entries */ + for (i = 0; i < ma->tag_count; i++) { + + mt = fy_malloc_tag_from_tag(ma, i); + if (!mt) + continue; + + mt_new = &tags[i]; + + fy_malloc_tag_setup(ma, mt_new); + /* move the old entries over */ + fy_malloc_tag_list_lock(mt); + while ((me = fy_malloc_entry_list_pop(&mt->entries)) != NULL) + fy_malloc_entry_list_add_tail(&mt_new->entries, me); + fy_malloc_tag_list_unlock(mt); + memcpy(&mt_new->stats, &mt->stats, sizeof(mt->stats)); + + /* clean the old entry */ + fy_malloc_tag_cleanup(ma, mt); + + fy_id_set_used(ids, tag_id_count, i); + } + } + + if (ma->tags != tags) { + ma->tags = tags; + fy_parent_allocator_free(&ma->a, ma->tags); + } + if (ma->ids != ids) { + ma->ids = ids; + fy_parent_allocator_free(&ma->a, ma->ids); + } + ma->tag_count = tag_count; + ma->tag_id_count = tag_id_count; + return 0; + +err_out: + fy_parent_allocator_free(&ma->a, tags); + fy_parent_allocator_free(&ma->a, ids); + return -1; +} + +static void fy_malloc_trim_tag(struct fy_allocator *a, int tag) +{ + /* nothing */ +} + +static void fy_malloc_reset_tag(struct fy_allocator *a, int tag) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + struct fy_malloc_entry *me; + + ma = container_of(a, struct fy_malloc_allocator, a); + + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + return; + + /* no lock, reset is single thread only */ + while ((me = fy_malloc_entry_list_pop(&mt->entries)) != NULL) + free(me->mem); +} + +static struct fy_allocator_info * +fy_malloc_get_info(struct fy_allocator *a, int tag) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + struct fy_malloc_entry *me; + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + struct fy_allocator_arena_info *arena_info; + size_t size, free, used, total; + size_t tag_free, tag_used, tag_total; + size_t arena_free, arena_used, arena_total; + unsigned int num_tags, num_arenas, i; + int id; + + ma = container_of(a, struct fy_malloc_allocator, a); + + /* allocate for the worst case always */ + num_tags = 0; + num_arenas = 0; + + free = 0; + used = 0; + total = 0; + + /* two passes */ + for (i = 0; i < 2; i++) { + + if (!i) { + info = NULL; + tag_info = NULL; + arena_info = NULL; + + } else { + size = sizeof(*info) + + sizeof(*tag_info) * num_tags + + sizeof(*arena_info) * num_arenas; + + info = malloc(size); + if (!info) + return NULL; + memset(info, 0, sizeof(*info)); + + tag_info = (void *)(info + 1); + assert(((uintptr_t)tag_info % _Alignof(struct fy_allocator_tag_info)) == 0); + arena_info = (void *)(tag_info + num_tags); + assert(((uintptr_t)arena_info % _Alignof(struct fy_allocator_arena_info)) == 0); + + info->free = free; + info->used = used; + info->total = total; + + info->num_tag_infos = 0; + info->tag_infos = tag_info; + } + + free = 0; + used = 0; + total = sizeof(*ma); + + for (id = 0; id < (int)ma->tag_count; id++) { + + if (!fy_id_is_used(ma->ids, ma->tag_id_count, id)) + continue; + + mt = &ma->tags[id]; + + tag_free = 0; + tag_used = 0; + tag_total = 0; + + if (i) { + tag_info->num_arena_infos = 0; + tag_info->arena_infos = arena_info; + } + + fy_malloc_tag_list_lock(mt); + + for (me = fy_malloc_entry_list_head(&mt->entries); me; me = fy_malloc_entry_next(&mt->entries, me)) { + arena_free = 0; + arena_used = me->size; + arena_total = sizeof(*me) + me->size; + + tag_free += arena_free; + tag_used += arena_used; + tag_total += arena_total; + + if (!i) { + num_arenas++; + } else { + arena_info->free = arena_free; + arena_info->used = arena_used; + arena_info->total = arena_total; + arena_info->data = me->mem; + arena_info->size = me->size; + arena_info++; + tag_info->num_arena_infos++; + } + } + fy_malloc_tag_list_unlock(mt); + + if (!i) { + num_tags++; + } else { + + /* only store the tag if there's a match */ + if (tag == FY_ALLOC_TAG_NONE || tag == id) { + tag_info->tag = id; + tag_info->free = tag_free; + tag_info->used = tag_used; + tag_info->total = tag_total; + tag_info++; + info->num_tag_infos++; + } + } + + free += tag_free; + used += tag_used; + total += tag_total; + } + } + + return info; +} + +static enum fy_allocator_cap_flags +fy_malloc_get_caps(struct fy_allocator *a) +{ + return FYACF_CAN_FREE_INDIVIDUAL | FYACF_CAN_FREE_TAG | + FYACF_HAS_CONTAINS | FYACF_HAS_TAGS; +} + +/* malloc allocator actually tracks allocations, it's just very inefficient + * to scan through. It might be good for debugging though + */ +static bool fy_malloc_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + struct fy_malloc_entry *me; + int tag_start, tag_end; + + ma = container_of(a, struct fy_malloc_allocator, a); + if (tag >= 0) { + if (tag >= (int)ma->tag_count) + return false; + tag_start = tag; + tag_end = tag_start + 1; + } else { + tag_start = 0; + tag_end = ma->tag_count; + } + + for (tag = tag_start; tag < tag_end; tag++) { + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + continue; + + fy_malloc_tag_list_lock(mt); + for (me = fy_malloc_entry_list_head(&mt->entries); me; + me = fy_malloc_entry_next(&mt->entries, me)) { + + if (ptr >= (void *)me->mem && ptr < (void *)me->mem + me->size) { + fy_malloc_tag_list_unlock(mt); + return true; + } + } + fy_malloc_tag_list_unlock(mt); + } + + return false; +} + +const struct fy_allocator_ops fy_malloc_allocator_ops = { + .setup = fy_malloc_setup, + .cleanup = fy_malloc_cleanup, + .create = fy_malloc_create, + .destroy = fy_malloc_destroy, + .dump = fy_malloc_dump, + .alloc = fy_malloc_alloc, + .free = fy_malloc_free, + .update_stats = fy_malloc_update_stats, + .storev = fy_malloc_storev, + .lookupv = fy_malloc_lookupv, + .release = fy_malloc_release, + .get_tag = fy_malloc_get_tag, + .release_tag = fy_malloc_release_tag, + .get_tag_count = fy_malloc_get_tag_count, + .set_tag_count = fy_malloc_set_tag_count, + .trim_tag = fy_malloc_trim_tag, + .reset_tag = fy_malloc_reset_tag, + .get_info = fy_malloc_get_info, + .get_caps = fy_malloc_get_caps, + .contains = fy_malloc_contains, +}; diff --git a/src/allocator/fy-allocator-malloc.h b/src/allocator/fy-allocator-malloc.h new file mode 100644 index 0000000..745d85d --- /dev/null +++ b/src/allocator/fy-allocator-malloc.h @@ -0,0 +1,49 @@ +/* + * fy-allocator-malloc.h - the malloc allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_MALLOC_H +#define FY_ALLOCATOR_MALLOC_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "fy-typelist.h" +#include "fy-id.h" + +#include "fy-atomics.h" +#include "fy-allocator.h" + +struct fy_malloc_tag; + +FY_TYPE_FWD_DECL_LIST(malloc_entry); +struct fy_malloc_entry { + struct list_head node; + size_t size; + void *mem; +}; +FY_TYPE_DECL_LIST(malloc_entry); + +#define FY_MALLOC_TAG_MAX 32 + +struct fy_malloc_tag { + fy_atomic_flag lock; + struct fy_malloc_entry_list entries; + struct fy_allocator_stats stats; +}; + +struct fy_malloc_allocator { + struct fy_allocator a; + fy_id_bits *ids; + unsigned int tag_id_count; + struct fy_malloc_tag *tags; + unsigned int tag_count; +}; + +extern const struct fy_allocator_ops fy_malloc_allocator_ops; + +#endif diff --git a/src/allocator/fy-allocator-mremap.c b/src/allocator/fy-allocator-mremap.c new file mode 100644 index 0000000..d943221 --- /dev/null +++ b/src/allocator/fy-allocator-mremap.c @@ -0,0 +1,1186 @@ +/* + * fy-allocator-mremap.c - mremap allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + + +#include + +/* for container_of */ +#include "fy-list.h" +#include "fy-utils.h" + +#include "fy-allocator-mremap.h" + +// #define DEBUG_ARENA + +#if defined(__NetBSD__) +#define DISABLE_MREMAP +#endif + +#if HAVE_MREMAP && !defined(DISABLE_MREMAP) +#define USE_MREMAP 1 +#else +#define USE_MREMAP 0 +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_BIG_ALLOC_THRESHOLD +#define MREMAP_ALLOCATOR_DEFAULT_BIG_ALLOC_THRESHOLD SIZE_MAX +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_EMPTY_THRESHOLD +#define MREMAP_ALLOCATOR_DEFAULT_EMPTY_THRESHOLD 64 +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_MINIMUM_ARENA_SIZE +#define MREMAP_ALLOCATOR_DEFAULT_MINIMUM_ARENA_SIZE (1U << 20) /* 1M */ +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_GROW_RATIO +#define MREMAP_ALLOCATOR_DEFAULT_GROW_RATIO 2.0 +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_BALLON_RATIO +#define MREMAP_ALLOCATOR_DEFAULT_BALLON_RATIO 32.0 +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_ARENA_TYPE +#define MREMAP_ALLOCATOR_DEFAULT_ARENA_TYPE FYMRAT_MMAP +#endif + +static inline size_t fy_mremap_useable_arena_size(struct fy_mremap_allocator *mra, size_t size) +{ + return fy_size_t_align(size + FY_MREMAP_ARENA_OVERHEAD, mra->pagesz) - + FY_MREMAP_ARENA_OVERHEAD; +} + +static inline bool fy_mremap_arena_available(struct fy_mremap_arena *mran) +{ + return mran->size - fy_atomic_load(&mran->next); +} + +static inline bool fy_mremap_arena_check_fit(struct fy_mremap_arena *mran, size_t size, size_t align) +{ + size_t old_next, new_next; + + old_next = fy_atomic_load(&mran->next); + new_next = fy_size_t_align(old_next, align) + size; + return new_next <= mran->size; +} + +static struct fy_mremap_arena * +fy_mremap_arena_create(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt, size_t size) +{ + struct fy_mremap_arena *mran = NULL; + uint64_t flags; + void *mem; + size_t size_page_align, balloon_size; +#if !USE_MREMAP + int rc FY_DEBUG_UNUSED; +#endif + + if (size < mra->minimum_arena_size) + size = mra->minimum_arena_size; + + size_page_align = fy_mremap_useable_arena_size(mra, size); + switch (mra->arena_type) { + case FYMRAT_MALLOC: + mran = malloc(size_page_align); + if (!mran) + return NULL; + memset(mran, 0, sizeof(*mran)); + break; + + case FYMRAT_MMAP: + /* allocate an initial ballooned size */ + balloon_size = fy_size_t_align((size_t)(size_page_align * mra->balloon_ratio), mra->pagesz); + if (balloon_size == size_page_align) + balloon_size = size_page_align + mra->pagesz; + mem = mmap(NULL, balloon_size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + /* first allocation failed, that's ok, try again */ + mran = mmap(NULL, size_page_align, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + } else { +#if USE_MREMAP + mran = mremap(mem, balloon_size, size_page_align, 0); + /* either it fails, or it moves we handle it */ +#else + /* we don't shrink, we just unmap over the limit */ + rc = munmap(mem + size_page_align, balloon_size - size_page_align); + if (rc) { +#ifdef DEBUG_ARENA + fprintf(stderr, "%s: failed to unmap for shink\n", __func__); +#endif + size_page_align = balloon_size; /* keep the balloon size? */ + } + mran = mem; +#endif + } + if (mran == MAP_FAILED) + return NULL; + break; + + default: + FY_IMPOSSIBLE_ABORT(); + } + + mran->next_arena = NULL; + flags = 0; + if (!fy_mremap_arena_type_is_growable(mra->arena_type)) + flags |= FYMRAF_CANT_GROW; + fy_atomic_store(&mran->flags, flags); + mran->size = size_page_align; + fy_atomic_store(&mran->next, FY_MREMAP_ARENA_OVERHEAD); + +#ifdef DEBUG_ARENA + fprintf(stderr, "%s: #%zu created arena %p size=%zu (%zuMB)\n", __func__, mrt - mra->tags, mran, mran->size, mran->size >> 20); +#endif + + return mran; +} + +static void fy_mremap_arena_destroy(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt, struct fy_mremap_arena *mran) +{ + if (!mran) + return; + +#ifdef DEBUG_ARENA + fprintf(stderr, "%s: #%zu destroy arena %p size=%zu (%zuMB)\n", __func__, mrt - mra->tags, mran, mran->size, mran->size >> 20); +#endif + switch (mra->arena_type) { + case FYMRAT_MALLOC: + free(mran); + break; + + case FYMRAT_MMAP: + (void)munmap(mran, mran->size); + break; + + default: + FY_IMPOSSIBLE_ABORT(); + } +} + +static inline bool +fy_mremap_arena_should_grow(struct fy_mremap_allocator *mra, struct fy_mremap_arena *mran, + size_t size, size_t align) +{ + size_t next; + + if (!mran || !size) + return false; + + if (!fy_mremap_arena_type_is_growable(mra->arena_type)) + return false; + + /* there's no point trying to grow something this big */ + if (mra->big_alloc_threshold && size >= mra->big_alloc_threshold) + return false; + + /* if the grow needs to be larger than double, don't bother */ + next = atomic_load(&mran->next); + if (fy_size_t_align(next, align) + size > 2 * mran->size) + return false; + + /* go for it */ + return true; +} + +static int fy_mremap_arena_grow(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt, + struct fy_mremap_arena *mran, size_t size, size_t align) +{ + void *mem; + + if (!fy_mremap_arena_should_grow(mra, mran, size, align)) + return -1; + + switch (mra->arena_type) { + case FYMRAT_MALLOC: + /* can't grow malloc without moving the pointer */ + break; + + case FYMRAT_MMAP: +#if USE_MREMAP + /* double the arena */ + mem = mremap(mran, mran->size, mran->size * 2, 0); + if (mem == MAP_FAILED) + break; + + assert(mem == mran); +#else + /* do a mmap right after the one we have, if it succeeds we have grown */ + mem = mmap((void *)mran + mran->size, mran->size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (mem != (void *)mran + mran->size) { + if (mem != MAP_FAILED) + munmap(mem, mran->size); + break; + } +#endif + mran->size *= 2; + +#ifdef DEBUG_ARENA + fprintf(stderr, "%s: #%zu grew arena %p size=%zu (%zuMB)\n", __func__, mrt - mra->tags, mran, mran->size, mran->size >> 20); +#endif + return 0; + + default: + FY_IMPOSSIBLE_ABORT(); + } + + return -1; +} + +static int fy_mremap_arena_trim(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt, struct fy_mremap_arena *mran) +{ + size_t next, new_size; +#if USE_MREMAP + void *mem FY_DEBUG_UNUSED; +#else + int rc FY_DEBUG_UNUSED; +#endif + + if (!mran) + return -1; + + switch (mra->arena_type) { + case FYMRAT_MALLOC: + /* trim not possible (or is it? should be possible to probe) */ + break; + + case FYMRAT_MMAP: + /* check the page size */ + next = fy_atomic_load(&mran->next); + new_size = fy_size_t_align(next, mra->pagesz); + if (new_size >= mran->size) + break; + +#ifdef DEBUG_ARENA + fprintf(stderr, "trim: %zu -> %zu\n", mran->size, new_size); +#endif + +#if USE_MREMAP + /* failure to shrink a mapping is unthinkable, but check anyway */ + mem = mremap(mran, mran->size, new_size, 0); + if (mem == MAP_FAILED || mem != mran) + return -1; +#else + /* we don't shrink, we just unmap over the limit */ + rc = munmap((void *)mran + new_size, mran->size - new_size); + if (rc) + return -1; +#endif + mran->size = new_size; + return 0; + + default: + FY_IMPOSSIBLE_ABORT(); + } + + return -1; +} + +static inline struct fy_mremap_tag * +fy_mremap_tag_from_tag(struct fy_mremap_allocator *mra, int tag) +{ + if (!mra) + return NULL; + + if ((unsigned int)tag >= mra->tag_count) + return NULL; + + if (!fy_id_is_used(mra->ids, mra->tag_id_count, (int)tag)) + return NULL; + + return &mra->tags[tag]; +} + +static void fy_mremap_tag_cleanup(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt) +{ + struct fy_mremap_arena *mran; +#ifdef DEBUG_ARENA + size_t total_sys_alloc, total_wasted, next; +#endif + + if (!mra || !mrt) + return; + +#ifdef DEBUG_ARENA + total_sys_alloc = 0; + total_wasted = 0; + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { + total_sys_alloc += mran->size; + next = fy_atomic_load(&mran->next); + total_wasted += (mran->size - next); + } + fprintf(stderr, "total_sys_alloc=%zu total_wasted=%zu\n", total_sys_alloc, total_wasted); +#endif + +#ifdef DEBUG_ARENA + fprintf(stderr, "%s: destroying active arenas\n", __func__); +#endif + /* pop and destroy */ + while ((mran = fy_atomic_load(&mrt->arenas)) != NULL) { + if (fy_atomic_compare_exchange_strong(&mrt->arenas, &mran, mran->next_arena)) + fy_mremap_arena_destroy(mra, mrt, mran); + } + + /* nuke it all */ + memset(mrt, 0, sizeof(*mrt)); +} + +static void fy_mremap_tag_trim(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt) +{ + struct fy_mremap_arena *mran; +#ifdef DEBUG_ARENA + size_t wasted_before, wasted_after, next; +#endif + + if (!mra || !mrt) + return; + + if (!fy_mremap_arena_type_is_trimmable(mra->arena_type)) + return; + +#ifdef DEBUG_ARENA + wasted_before = 0; + wasted_after = 0; +#endif + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { +#ifdef DEBUG_ARENA + next = fy_atomic_load(&mran->next); + wasted_before += (mran->size - next); +#endif + (void)fy_mremap_arena_trim(mra, mrt, mran); +#ifdef DEBUG_ARENA + next = fy_atomic_load(&mran->next); + wasted_after += (mran->size - next); +#endif + } + +#ifdef DEBUG_ARENA + fprintf(stderr, "wasted_before=%zu wasted_after=%zu\n", wasted_before, wasted_after); +#endif +} + +static void fy_mremap_tag_reset(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt) +{ + struct fy_mremap_arena *mran; + + if (!mra || !mrt) + return; + + /* pop and destroy */ + while ((mran = fy_atomic_load(&mrt->arenas)) != NULL) { + if (fy_atomic_compare_exchange_strong(&mrt->arenas, &mran, mran->next_arena)) + fy_mremap_arena_destroy(mra, mrt, mran); + } +} + +static void fy_mremap_tag_setup(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt) +{ + assert(mra); + assert(mrt); + + memset(mrt, 0, sizeof(*mrt)); + fy_atomic_store(&mrt->arenas, NULL); + fy_atomic_store(&mrt->next_arena_sz, mra->pagesz); + fy_atomic_store(&mrt->allocations, 0); + fy_atomic_store(&mrt->allocated, 0); + fy_atomic_store(&mrt->stores, 0); + fy_atomic_store(&mrt->stored, 0); +} + +static void *fy_mremap_tag_alloc(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt, size_t size, size_t align) +{ + struct fy_mremap_arena *mran, *old_arenas; + size_t next_sz, next_arena_sz, old_next_arena_sz, old_next, new_next, data_pos; + uint64_t flags; + void *ptr; + int rc; + +again: + /* hot path, try to find an arena that fits first */ + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { + + flags = fy_atomic_load(&mran->flags); + if (flags & FYMRAF_FULL) + continue; + + if (fy_mremap_arena_check_fit(mran, size, align)) + goto do_alloc; + + /* if it cant grown and is under the threshold, mark it full */ + if ((flags & FYMRAF_CANT_GROW) && fy_mremap_arena_available(mran) < mra->empty_threshold) + fy_atomic_fetch_or(&mran->flags, FYMRAF_FULL); + } + + /* this arena type is not growable, skip growing step */ + if (!fy_mremap_arena_type_is_growable(mra->arena_type)) + goto create_arena; + + /* not found space in any arena, try to grow */ + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { + + flags = fy_atomic_load(&mran->flags); + if (flags & (FYMRAF_FULL | FYMRAF_CANT_GROW | FYMRAF_GROWING)) + continue; + + /* don't even try? */ + if (!fy_mremap_arena_should_grow(mra, mran, size, align)) + continue; + + /* try to grab the growing lock */ + if (!fy_atomic_compare_exchange_strong(&mran->flags, &flags, flags | FYMRAF_GROWING)) + continue; + + /* try to grow, note that the arena is still available */ + rc = fy_mremap_arena_grow(mra, mrt, mran, size, align); + if (rc) { +#ifdef DEBUG_ARENA + fprintf(stderr, "failed to grow %p\n", mran); +#endif + /* release the lock, and mark it as can't grow */ + fy_atomic_fetch_or(&mran->flags, FYMRAF_CANT_GROW); + fy_atomic_fetch_and(&mran->flags, ~FYMRAF_GROWING); + + continue; + } + +#ifdef DEBUG_ARENA + fprintf(stderr, "grow successful %p mran->size=%zu size=%zu\n", mran, mran->size, size); +#endif + + /* release the lock */ + fy_atomic_fetch_and(&mran->flags, ~FYMRAF_GROWING); + + if (fy_mremap_arena_check_fit(mran, size, align)) + goto do_alloc; + } + +create_arena: + /* everything failed, we have to allocate a new arena */ + + /* it's a relatively small allocation, try to resize until we fit */ + if (!mra->big_alloc_threshold || size < mra->big_alloc_threshold) { + + do { + /* increase by the ratio until we're over */ + old_next_arena_sz = fy_atomic_load(&mrt->next_arena_sz); + next_arena_sz = old_next_arena_sz; + while (fy_mremap_useable_arena_size(mra, next_arena_sz) < size) { + next_sz = (size_t)(next_arena_sz * mra->grow_ratio); + + /* very very unlikely */ + if (next_sz <= next_arena_sz) + goto err_out; + + next_arena_sz = next_sz; + } + + /* update to the next possible arena size */ + next_sz = (size_t)(next_arena_sz * mra->grow_ratio); + if (next_sz > next_arena_sz) + next_arena_sz = next_sz; + + } while (!fy_atomic_compare_exchange_strong(&mrt->next_arena_sz, + &old_next_arena_sz, next_arena_sz)); + } else + next_arena_sz = size; /* something big, just use it as is */ + + /* all failed, just new */ + mran = fy_mremap_arena_create(mra, mrt, next_arena_sz); + if (!mran) + goto err_out; + +#ifdef DEBUG_ARENA + fprintf(stderr, "allocated new %p mran->size=%zu size=%zu\n", mran, mran->size, size); +#endif + + /* atomically add it */ + do { + old_arenas = fy_atomic_load(&mrt->arenas); + mran->next_arena = old_arenas; + } while (!fy_atomic_compare_exchange_strong(&mrt->arenas, &old_arenas, mran)); + +do_alloc: + + do { + old_next = fy_atomic_load(&mran->next); + data_pos = fy_size_t_align(old_next, align); + new_next = data_pos + size; + if (new_next > mran->size) { +#ifdef DEBUG_ARENA + fprintf(stderr, "failed new %p mran->size=%zu size=%zu failed to fit!\n", mran, mran->size, size); +#endif + goto again; + } + ptr = (void *)mran + data_pos; + } while (!fy_atomic_compare_exchange_strong(&mran->next, &old_next, new_next)); + + /* malloc arenas need to zero out */ + if (mra->arena_type == FYMRAT_MALLOC) + memset(ptr, 0, size); + +#ifdef DEBUG_ARENA + fprintf(stderr, "allocated OK %p mran->size=%zu ptr=%p size=%zu align=%zu\n", mran, mran->size, ptr, size, align); +#endif + return ptr; + +err_out: + return NULL; +} + +static const struct fy_mremap_allocator_cfg default_cfg = { + .big_alloc_threshold = MREMAP_ALLOCATOR_DEFAULT_BIG_ALLOC_THRESHOLD, + .empty_threshold = MREMAP_ALLOCATOR_DEFAULT_EMPTY_THRESHOLD, + .minimum_arena_size = MREMAP_ALLOCATOR_DEFAULT_MINIMUM_ARENA_SIZE, + .grow_ratio = MREMAP_ALLOCATOR_DEFAULT_GROW_RATIO, + .balloon_ratio = MREMAP_ALLOCATOR_DEFAULT_BALLON_RATIO, + .arena_type = MREMAP_ALLOCATOR_DEFAULT_ARENA_TYPE, +}; + +static void fy_mremap_cleanup(struct fy_allocator *a) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + unsigned int i; + + if (!a) + return; + + mra = container_of(a, struct fy_mremap_allocator, a); + + for (i = 0; i < mra->tag_count; i++) { + mrt = fy_mremap_tag_from_tag(mra, i); + if (!mrt) + continue; + fy_mremap_tag_cleanup(mra, mrt); + } + + fy_parent_allocator_free(&mra->a, mra->ids); + fy_parent_allocator_free(&mra->a, mra->tags); +} + +static int fy_mremap_setup(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *cfg_data) +{ + const struct fy_mremap_allocator_cfg *cfg; + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + size_t tmpsz; + + if (!a) + return -1; + + cfg = cfg_data ? cfg_data : &default_cfg; + + mra = container_of(a, struct fy_mremap_allocator, a); + memset(mra, 0, sizeof(*mra)); + mra->a.name = "mremap"; + mra->a.ops = &fy_mremap_allocator_ops; + mra->a.parent = parent; + mra->a.parent_tag = parent_tag; + mra->cfg = *cfg; + + mra->pagesz = sysconf(_SC_PAGESIZE); + /* pagesz is size of 2 find the first set bit */ + mra->pageshift = fy_bit64_ffs(mra->pagesz); + + mra->big_alloc_threshold = cfg->big_alloc_threshold; + if (!mra->big_alloc_threshold) + mra->big_alloc_threshold = MREMAP_ALLOCATOR_DEFAULT_BIG_ALLOC_THRESHOLD; + + mra->empty_threshold = cfg->empty_threshold; + if (!mra->empty_threshold) + mra->empty_threshold = MREMAP_ALLOCATOR_DEFAULT_EMPTY_THRESHOLD; + + mra->minimum_arena_size = cfg->minimum_arena_size; + if (!mra->minimum_arena_size) + mra->minimum_arena_size = MREMAP_ALLOCATOR_DEFAULT_MINIMUM_ARENA_SIZE; + + mra->grow_ratio = cfg->grow_ratio; + if (mra->grow_ratio <= 1) + mra->grow_ratio = MREMAP_ALLOCATOR_DEFAULT_GROW_RATIO; + + mra->balloon_ratio = cfg->balloon_ratio; + if (mra->balloon_ratio <= 1) + mra->balloon_ratio = MREMAP_ALLOCATOR_DEFAULT_BALLON_RATIO; + + mra->arena_type = cfg->arena_type; + if (mra->arena_type == FYMRAT_DEFAULT) + mra->arena_type = MREMAP_ALLOCATOR_DEFAULT_ARENA_TYPE; + + mra->tag_count = FY_MREMAP_TAG_MAX; + mra->tag_id_count = (mra->tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + tmpsz = mra->tag_id_count * sizeof(*mra->ids); + mra->ids = fy_parent_allocator_alloc(&mra->a, tmpsz, _Alignof(fy_id_bits)); + if (!mra->ids) + goto err_out; + fy_id_reset(mra->ids, mra->tag_id_count); + + tmpsz = mra->tag_count * sizeof(*mra->tags); + mra->tags = fy_parent_allocator_alloc(&mra->a, tmpsz, _Alignof(struct fy_mremap_tag)); + if (!mra->tags) + goto err_out; + + /* start with tag 0 as general use */ + fy_id_set_used(mra->ids, mra->tag_id_count, 0); + mrt = fy_mremap_tag_from_tag(mra, 0); + fy_mremap_tag_setup(mra, mrt); + + return 0; + +err_out: + fy_mremap_cleanup(a); + return -1; +} + +struct fy_allocator *fy_mremap_create(struct fy_allocator *parent, int parent_tag, const void *cfg) +{ + struct fy_mremap_allocator *mra = NULL; + int rc; + + mra = fy_early_parent_allocator_alloc(parent, parent_tag, sizeof(*mra), _Alignof(struct fy_mremap_allocator)); + if (!mra) + goto err_out; + + rc = fy_mremap_setup(&mra->a, parent, parent_tag, cfg); + if (rc) + goto err_out; + + return &mra->a; + +err_out: + fy_early_parent_allocator_free(parent, parent_tag, mra); + return NULL; +} + +void fy_mremap_destroy(struct fy_allocator *a) +{ + struct fy_mremap_allocator *mra; + + mra = container_of(a, struct fy_mremap_allocator, a); + fy_mremap_cleanup(a); + fy_parent_allocator_free(a, mra); +} + +void fy_mremap_dump(struct fy_allocator *a) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + struct fy_mremap_arena *mran; + size_t count, active_count, full_count, total, system_total; + unsigned int i; + + mra = container_of(a, struct fy_mremap_allocator, a); + + fprintf(stderr, "mremap: "); + for (i = 0; i < mra->tag_count; i++) { + mrt = fy_mremap_tag_from_tag(mra, i); + if (!mrt) + continue; + fprintf(stderr, "%c", fy_id_is_free(mra->ids, mra->tag_id_count, i) ? '.' : 'x'); + } + fprintf(stderr, "\n"); + + for (i = 0; i < mra->tag_count; i++) { + mrt = fy_mremap_tag_from_tag(mra, i); + if (!mrt) + continue; + + count = full_count = active_count = total = system_total = 0; + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { + total += fy_atomic_load(&mran->next); + system_total += mran->size; + count++; + active_count++; + /* XXX full count */ + } + + fprintf(stderr, " %d: count %zu (a=%zu/f=%zu) total %zu system %zu overhead %zu (%2.2f%%)\n", i, + count, active_count, full_count, + total, system_total, system_total - total, + 100.0 * (double)(system_total - total) / (double)system_total); + } +} + +static void *fy_mremap_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + void *ptr; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + goto err_out; + + ptr = fy_mremap_tag_alloc(mra, mrt, size, align); + if (!ptr) + goto err_out; + + if (a->flags & FYAF_KEEP_STATS) { + fy_atomic_fetch_add(&mrt->allocations, 1); + fy_atomic_fetch_add(&mrt->allocated, size); + } + + return ptr; + +err_out: + return NULL; +} + +static void fy_mremap_free(struct fy_allocator *a, int tag, void *data) +{ + /* no frees */ +} + +static int fy_mremap_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + goto err_out; + + stats->allocations += fy_atomic_get_and_clear_counter(&mrt->allocations); + stats->allocated += fy_atomic_get_and_clear_counter(&mrt->allocated); + stats->stores += fy_atomic_get_and_clear_counter(&mrt->stores); + stats->stored += fy_atomic_get_and_clear_counter(&mrt->stored); + + return 0; + +err_out: + return -1; +} + +static const void *fy_mremap_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + void *start; + size_t total_size; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + goto err_out; + + total_size = fy_iovec_size(iov, iovcnt); + if (total_size == SIZE_MAX) + goto err_out; + + start = fy_mremap_tag_alloc(mra, mrt, total_size, align); + if (!start) + goto err_out; + + fy_iovec_copy_from(iov, iovcnt, start); + + if (a->flags & FYAF_KEEP_STATS) { + fy_atomic_fetch_add(&mrt->stores, 1); + fy_atomic_fetch_add(&mrt->stored, total_size); + } + + return start; + +err_out: + return NULL; +} + +static const void *fy_mremap_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + /* no lookups */ + return NULL; +} + +static void fy_mremap_release(struct fy_allocator *a, int tag, const void *data, size_t size) +{ + /* no releases */ +} + +static void fy_mremap_release_tag(struct fy_allocator *a, int tag) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + return; + + fy_mremap_tag_cleanup(mra, mrt); + + /* must be last */ + fy_id_free(mra->ids, mra->tag_id_count, tag); +} + +static int fy_mremap_get_tag(struct fy_allocator *a) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + int id = -1; + + mra = container_of(a, struct fy_mremap_allocator, a); + + /* this is atomic, so safe for multiple threads */ + id = fy_id_alloc(mra->ids, mra->tag_id_count); + if (id < 0) + goto err_out; + + mrt = fy_mremap_tag_from_tag(mra, id); + if (!mrt) + goto err_out; + + fy_mremap_tag_setup(mra, mrt); + + return (int)id; + +err_out: + if (id >= 0) + fy_id_free(mra->ids, mra->tag_id_count, id); + return FY_ALLOC_TAG_ERROR; +} + +static int fy_mremap_get_tag_count(struct fy_allocator *a) +{ + struct fy_mremap_allocator *mra; + + mra = container_of(a, struct fy_mremap_allocator, a); + return mra->tag_count; +} + +static int fy_mremap_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_arena *mran; + fy_id_bits *ids = NULL; + struct fy_mremap_tag *mrt, *mrt_new, *tags = NULL; + unsigned int i, tag_count, tag_id_count; + size_t tmpsz; + + if (count < 1) + return -1; + + mra = container_of(a, struct fy_mremap_allocator, a); + + if (count == mra->tag_count) + return 0; + + tag_count = count; + tag_id_count = (tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + /* shrink?, just clip */ + if (count < mra->tag_count) { + for (i = count; i < mra->tag_count; i++) { + mrt = fy_mremap_tag_from_tag(mra, i); + if (!mrt) + continue; + fy_mremap_tag_cleanup(mra, mrt); + fy_id_free(mra->ids, mra->tag_id_count, i); + } + tags = mra->tags; + ids = mra->ids; + } else { + /* we need to grow */ + tmpsz = tag_id_count * sizeof(*mra->ids); + ids = fy_parent_allocator_alloc(&mra->a, tmpsz, _Alignof(fy_id_bits)); + if (!ids) + goto err_out; + fy_id_reset(ids, tag_id_count); + + tmpsz = tag_count * sizeof(*tags); + tags = fy_parent_allocator_alloc(&mra->a, tmpsz, _Alignof(struct fy_mremap_tag)); + if (!tags) + goto err_out; + + /* copy over the old entries */ + for (i = 0; i < mra->tag_count; i++) { + + mrt = fy_mremap_tag_from_tag(mra, i); + if (!mrt) + continue; + + mrt_new = &tags[i]; + + fy_mremap_tag_setup(mra, mrt_new); + do { + mran = fy_atomic_load(&mrt->arenas); + } while (fy_atomic_compare_exchange_strong(&mrt->arenas, &mran, NULL)); + fy_atomic_store(&mrt_new->arenas, mran); + fy_atomic_store(&mrt_new->next_arena_sz, fy_atomic_load(&mrt->next_arena_sz)); + fy_atomic_store(&mrt_new->allocations, fy_atomic_load(&mrt->allocations)); + fy_atomic_store(&mrt_new->allocated, fy_atomic_load(&mrt->allocated)); + fy_atomic_store(&mrt_new->stores, fy_atomic_load(&mrt->stores)); + fy_atomic_store(&mrt_new->stored, fy_atomic_load(&mrt->stored)); + + /* clean the old entry */ + fy_mremap_tag_cleanup(mra, mrt); + + fy_id_set_used(ids, tag_id_count, i); + } + } + + if (mra->tags != tags) { + mra->tags = tags; + fy_parent_allocator_free(&mra->a, mra->tags); + } + if (mra->ids != ids) { + mra->ids = ids; + fy_parent_allocator_free(&mra->a, mra->ids); + } + mra->tag_count = tag_count; + mra->tag_id_count = tag_id_count; + return 0; + +err_out: + fy_parent_allocator_free(&mra->a, tags); + fy_parent_allocator_free(&mra->a, ids); + return -1; +} + +static void fy_mremap_trim_tag(struct fy_allocator *a, int tag) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + return; + + fy_mremap_tag_trim(mra, mrt); +} + +static void fy_mremap_reset_tag(struct fy_allocator *a, int tag) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + return; + + fy_mremap_tag_reset(mra, mrt); +} + +static struct fy_allocator_info * +fy_mremap_get_info(struct fy_allocator *a, int tag) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + struct fy_mremap_arena *mran; + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + struct fy_allocator_arena_info *arena_info; + size_t size, free, used, total; + size_t tag_free, tag_used, tag_total; + size_t arena_free, arena_used, arena_total; + unsigned int num_tags, num_arenas, i; + int id; + + mra = container_of(a, struct fy_mremap_allocator, a); + + /* allocate for the worst case always */ + num_tags = 0; + num_arenas = 0; + + free = 0; + used = 0; + total = 0; + + /* two passes */ + for (i = 0; i < 2; i++) { + + if (!i) { + tag_info = NULL; + arena_info = NULL; + + } else { + size = sizeof(*info) + + sizeof(*tag_info) * num_tags + + sizeof(*arena_info) * num_arenas; + + info = malloc(size); + if (!info) + return NULL; + memset(info, 0, sizeof(*info)); + + tag_info = (void *)(info + 1); + assert(((uintptr_t)tag_info % _Alignof(struct fy_allocator_tag_info)) == 0); + arena_info = (void *)(tag_info + num_tags); + assert(((uintptr_t)arena_info % _Alignof(struct fy_allocator_arena_info)) == 0); + + info->free = free; + info->used = used; + info->total = total; + + info->num_tag_infos = 0; + info->tag_infos = tag_info; + } + + free = 0; + used = 0; + total = sizeof(*mra); + + for (id = 0; id < (int)mra->tag_count; id++) { + + if (!fy_id_is_used(mra->ids, mra->tag_id_count, id)) + continue; + + mrt = &mra->tags[id]; + + tag_free = 0; + tag_used = 0; + tag_total = 0; + + if (i) { + tag_info->num_arena_infos = 0; + tag_info->arena_infos = arena_info; + } + + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { + arena_free = (size_t)(mran->size - mran->next); + arena_used = (size_t)(mran->next - FY_MREMAP_ARENA_OVERHEAD); + arena_total = mran->size; + + tag_free += arena_free; + tag_used += arena_used; + tag_total += arena_total; + + if (!i) { + num_arenas++; + } else { + arena_info->free = arena_free; + arena_info->used = arena_used; + arena_info->total = arena_total; + arena_info->data = (void *)mran + FY_MREMAP_ARENA_OVERHEAD; + arena_info->size = arena_info->used; + arena_info++; + tag_info->num_arena_infos++; + } + } + + if (!i) { + num_tags++; + } else { + + /* only store the tag if there's a match */ + if (tag == FY_ALLOC_TAG_NONE || tag == id) { + tag_info->tag = id; + tag_info->free = tag_free; + tag_info->used = tag_used; + tag_info->total = tag_total; + tag_info++; + info->num_tag_infos++; + } + } + + free += tag_free; + used += tag_used; + total += tag_total; + } + } + + return info; +} + +static enum fy_allocator_cap_flags +fy_mremap_get_caps(struct fy_allocator *a) +{ + return FYACF_CAN_FREE_TAG | FYACF_HAS_EFFICIENT_CONTAINS | \ + FYACF_HAS_CONTAINS | FYACF_HAS_TAGS; +} + +static bool mremap_tag_contains(struct fy_mremap_tag *mrt, const void *p) +{ + struct fy_mremap_arena *mra; + + for (mra = fy_atomic_load(&mrt->arenas); mra; mra = mra->next_arena) { + if (p >= (const void *)mra->mem && + p < (const void *)mra->mem + mra->size) + return true; + } + + return false; +} + +static bool fy_mremap_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + int tag_start, tag_end; + + mra = container_of(a, struct fy_mremap_allocator, a); + + if (tag >= 0) { + if (tag >= (int)mra->tag_count) + return false; + tag_start = tag; + tag_end = tag + 1; + } else { + tag_start = 0; + tag_end = (int)mra->tag_count; + } + + for (tag = tag_start; tag < tag_end; tag++) { + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + continue; + + if (mremap_tag_contains(mrt, ptr)) + return true; + } + + return false; +} + +const struct fy_allocator_ops fy_mremap_allocator_ops = { + .setup = fy_mremap_setup, + .cleanup = fy_mremap_cleanup, + .create = fy_mremap_create, + .destroy = fy_mremap_destroy, + .dump = fy_mremap_dump, + .alloc = fy_mremap_alloc, + .free = fy_mremap_free, + .update_stats = fy_mremap_update_stats, + .storev = fy_mremap_storev, + .lookupv = fy_mremap_lookupv, + .release = fy_mremap_release, + .get_tag = fy_mremap_get_tag, + .release_tag = fy_mremap_release_tag, + .get_tag_count = fy_mremap_get_tag_count, + .set_tag_count = fy_mremap_set_tag_count, + .trim_tag = fy_mremap_trim_tag, + .reset_tag = fy_mremap_reset_tag, + .get_info = fy_mremap_get_info, + .get_caps = fy_mremap_get_caps, + .contains = fy_mremap_contains, +}; diff --git a/src/allocator/fy-allocator-mremap.h b/src/allocator/fy-allocator-mremap.h new file mode 100644 index 0000000..7503426 --- /dev/null +++ b/src/allocator/fy-allocator-mremap.h @@ -0,0 +1,77 @@ +/* + * fy-allocator-mremap.h - the mremap allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_MREMAP_H +#define FY_ALLOCATOR_MREMAP_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "fy-typelist.h" +#include "fy-id.h" + +#include "fy-atomics.h" +#include "fy-allocator.h" + +struct fy_mremap_tag; + +#define FY_MREMAP_TAG_MAX 32 + +static inline bool fy_mremap_arena_type_is_growable(enum fy_mremap_arena_type type) +{ + return type == FYMRAT_MMAP; +} + +static inline bool fy_mremap_arena_type_is_trimmable(enum fy_mremap_arena_type type) +{ + return type == FYMRAT_MMAP; +} + +#define FYMRAF_FULL FY_BIT(0) +#define FYMRAF_GROWING FY_BIT(1) +#define FYMRAF_CANT_GROW FY_BIT(2) + +struct fy_mremap_arena { + struct fy_mremap_arena *next_arena; + size_t size; /* includes the arena header */ + FY_ATOMIC(uint64_t) flags; + FY_ATOMIC(size_t) next; + uint64_t mem[] __attribute__((aligned(16))); +}; + +#define FY_MREMAP_ARENA_OVERHEAD (offsetof(struct fy_mremap_arena, mem)) + +struct fy_mremap_tag { + FY_ATOMIC(struct fy_mremap_arena *) arenas; + FY_ATOMIC(size_t) next_arena_sz; + FY_ATOMIC(uint64_t) allocations; + FY_ATOMIC(uint64_t) allocated; + FY_ATOMIC(uint64_t) stores; + FY_ATOMIC(uint64_t) stored; +}; + +struct fy_mremap_allocator { + struct fy_allocator a; + struct fy_mremap_allocator_cfg cfg; + size_t pagesz; + size_t pageshift; + size_t big_alloc_threshold; /* bigger than that and a new allocation */ + size_t empty_threshold; /* less than that and get moved to full */ + size_t minimum_arena_size; /* the minimum arena size */ + float grow_ratio; + float balloon_ratio; + enum fy_mremap_arena_type arena_type; + fy_id_bits *ids; + unsigned int tag_id_count; + struct fy_mremap_tag *tags; + unsigned int tag_count; +}; + +extern const struct fy_allocator_ops fy_mremap_allocator_ops; + +#endif diff --git a/src/allocator/fy-allocator.c b/src/allocator/fy-allocator.c new file mode 100644 index 0000000..84fc059 --- /dev/null +++ b/src/allocator/fy-allocator.c @@ -0,0 +1,756 @@ +/* + * fy-allocator.c - allocators + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#include + +#include + +#ifndef FY_NON_LOCKING_REGISTRY +#include +#endif + +/* for container_of */ +#include "fy-list.h" +#include "fy-utils.h" + +#include "fy-allocator.h" +#include "fy-allocator-linear.h" +#include "fy-allocator-malloc.h" +#include "fy-allocator-mremap.h" +#include "fy-allocator-dedup.h" +#include "fy-allocator-auto.h" + +static struct fy_registered_allocator_entry_list allocator_registry_list; +static bool allocator_registry_initialized = false; +static bool allocator_registry_locked = false; + +#ifndef FY_NON_LOCKING_REGISTRY +static pthread_mutex_t allocator_registry_mutex = PTHREAD_MUTEX_INITIALIZER; +static inline void allocator_registry_lock(void) +{ + int rc FY_UNUSED; + + rc = pthread_mutex_lock(&allocator_registry_mutex); + assert(!rc); + + assert(!allocator_registry_locked); + allocator_registry_locked = true; +} + +static inline void allocator_registry_unlock(void) +{ + int rc FY_UNUSED; + + assert(allocator_registry_locked); + allocator_registry_locked = false; + + rc = pthread_mutex_unlock(&allocator_registry_mutex); + assert(!rc); +} +#else +static inline void allocator_registry_lock(void) +{ + /* nothing */ +} +static inline void allocator_registry_unlock(void) +{ + /* nothing */ +} +#endif + +static inline void allocator_registry_init(void) +{ + if (allocator_registry_initialized) + return; + + fy_registered_allocator_entry_list_init(&allocator_registry_list); + + allocator_registry_initialized = true; +} + +static struct fy_registered_allocator_entry * +fy_registered_allocator_entry_create(const char *name, const struct fy_allocator_ops *ops) +{ + struct fy_registered_allocator_entry *ae; + + ae = malloc(sizeof(*ae)); + if (!ae) + return NULL; + memset(ae, 0, sizeof(*ae)); + ae->name = name; + ae->ops = ops; + + return ae; +} + +static void fy_registered_allocator_entry_destroy(struct fy_registered_allocator_entry *ae) +{ + if (!ae) + return; + free(ae); +} + +static const struct { + const char *name; + const struct fy_allocator_ops *ops; +} builtin_allocators[] = { + { + .name = "linear", + .ops = &fy_linear_allocator_ops, + }, { + .name = "malloc", + .ops = &fy_malloc_allocator_ops, + }, { + .name = "mremap", + .ops = &fy_mremap_allocator_ops, + }, { + .name = "dedup", + .ops = &fy_dedup_allocator_ops, + }, { + .name = "auto", + .ops = &fy_auto_allocator_ops, + } +}; + +int fy_allocator_register(const char *name, const struct fy_allocator_ops *ops) +{ + struct fy_registered_allocator_entry *ae; + unsigned int i; + int ret; + + if (!name || !ops || + !ops->setup || + !ops->cleanup || + !ops->create || + !ops->destroy || + !ops->dump || + !ops->alloc || + !ops->free || + !ops->update_stats || + !ops->storev || + !ops->lookupv || + !ops->release | + !ops->get_tag || + !ops->release_tag || + !ops->get_tag_count || + !ops->set_tag_count || + !ops->trim_tag || + !ops->reset_tag || + !ops->get_info || + !ops->get_caps || + !ops->contains) + return -1; + + allocator_registry_lock(); + allocator_registry_init(); + + ret = -1; + + /* must not clash with the builtins */ + for (i = 0; i < ARRAY_SIZE(builtin_allocators); i++) { + if (!strcmp(builtin_allocators[i].name, name)) + goto out_unlock; + } + + /* must not clash with the other entries */ + for (ae = fy_registered_allocator_entry_list_head(&allocator_registry_list); ae; + ae = fy_registered_allocator_entry_next(&allocator_registry_list, ae)) { + if (!strcmp(ae->name, name)) + goto out_unlock; + } + + /* OK, create the entry */ + ae = fy_registered_allocator_entry_create(name, ops); + if (!ae) + goto out_unlock; + + /* and add it to the list */ + fy_registered_allocator_entry_list_add(&allocator_registry_list, ae); + + /* all clear */ + ret = 0; + +out_unlock: + allocator_registry_unlock(); + + return ret; +} + +int fy_allocator_unregister(const char *name) +{ + struct fy_registered_allocator_entry *ae; + unsigned int i; + int ret; + + ret = -1; + + allocator_registry_lock(); + allocator_registry_init(); + + /* must not try to unregister a builtin */ + for (i = 0; i < ARRAY_SIZE(builtin_allocators); i++) { + if (!strcmp(builtin_allocators[i].name, name)) + goto out_unlock; + } + + /* find the entry now */ + for (ae = fy_registered_allocator_entry_list_head(&allocator_registry_list); ae; + ae = fy_registered_allocator_entry_next(&allocator_registry_list, ae)) { + if (!strcmp(ae->name, name)) + break; + } + if (!ae) + goto out_unlock; + + fy_registered_allocator_entry_list_del(&allocator_registry_list, ae); + + /* and destroy it */ + fy_registered_allocator_entry_destroy(ae); + + ret = 0; + +out_unlock: + allocator_registry_unlock(); + + return ret; +} + +struct fy_allocator * +fy_allocator_create_internal(const char *name, struct fy_allocator *parent, int parent_tag, const void *cfg) +{ + struct fy_registered_allocator_entry *ae; + const struct fy_allocator_ops *ops = NULL; + unsigned int i; + + if (!name) + name = builtin_allocators[0].name; + + allocator_registry_lock(); + allocator_registry_init(); + + /* try the builtins first */ + for (i = 0; i < ARRAY_SIZE(builtin_allocators); i++) { + if (!strcmp(builtin_allocators[i].name, name)) { + ops = builtin_allocators[i].ops; + break; + } + } + + /* if not found there, try the registry */ + if (!ops) { + for (ae = fy_registered_allocator_entry_list_head(&allocator_registry_list); ae; + ae = fy_registered_allocator_entry_next(&allocator_registry_list, ae)) { + if (!strcmp(ae->name, name)) { + ops = ae->ops; + break; + } + } + } + allocator_registry_unlock(); + if (!ops) + return NULL; + + /* XXX just malloc for now */ + return ops->create(parent, parent_tag, cfg); +} + +struct fy_allocator *fy_allocator_create(const char *name, const void *cfg) +{ + /* for now just use internal */ + return fy_allocator_create_internal(name, NULL, FY_ALLOC_TAG_DEFAULT, cfg); +} + +/* special in place allocator */ +struct fy_allocator * +fy_linear_allocator_create_in_place(void *buffer, size_t size) +{ + const struct fy_allocator_ops *ops = &fy_linear_allocator_ops; + struct fy_linear_allocator_cfg cfg = { .buf = buffer, .size = size }; + + if (!buffer || size < FY_LINEAR_ALLOCATOR_IN_PLACE_MIN_SIZE) + return NULL; + + return ops->create(FY_PARENT_ALLOCATOR_INPLACE, FY_ALLOC_TAG_DEFAULT, &cfg); +} + +struct fy_allocator * +fy_dedup_allocator_create_in_place(void *buffer, size_t size) +{ + struct fy_allocator *pa; + struct fy_dedup_allocator_cfg dcfg; + size_t dedup_available; + + if (!buffer || size < FY_DEDUP_ALLOCATOR_IN_PLACE_MIN_SIZE) + return NULL; + + pa = fy_linear_allocator_create_in_place(buffer, size); + if (!pa) + return NULL; + + /* try to size the dedup structures to about 10% of the available space */ + dedup_available = size - FY_LINEAR_ALLOCATOR_IN_PLACE_MIN_SIZE; + + memset(&dcfg, 0, sizeof(dcfg)); + dcfg.parent_allocator = pa; + dcfg.bloom_filter_bits = 0; /* use default */ + dcfg.bucket_count_bits = 0; + dcfg.estimated_content_size = dedup_available; + dcfg.minimum_bucket_occupancy = 1.0; /* will never grow */ + + return fy_allocator_create_internal("dedup", pa, 0, &dcfg); +} + +void fy_allocator_registry_cleanup_internal(bool show_leftovers) +{ + struct fy_registered_allocator_entry *ae; + + if (!allocator_registry_initialized) + return; + + allocator_registry_lock(); + while ((ae = fy_registered_allocator_entry_list_pop(&allocator_registry_list)) != NULL) { + if (show_leftovers) + fprintf(stderr, "%s: destroying %s\n", __func__, ae->name); + fy_registered_allocator_entry_destroy(ae); + } + allocator_registry_unlock(); +} + +void fy_allocator_registry_cleanup(void) +{ + fy_allocator_registry_cleanup_internal(false); +} + +#ifdef FY_HAS_CONSTRUCTOR +static FY_CONSTRUCTOR void fy_allocator_registry_constructor(void) +{ + allocator_registry_init(); +} +#endif + +#ifdef FY_HAS_DESTRUCTOR +static FY_DESTRUCTOR void fy_allocator_registry_destructor(void) +{ + bool show_leftovers = false; + +#ifdef FY_DESTRUCTOR_SHOW_LEFTOVERS + show_leftovers = true; +#endif + + /* make sure the registry is not locked because we will hang */ + if (allocator_registry_locked) { + if (show_leftovers) + fprintf(stderr, "%s: refusing to work on locked registry\n", __func__); + return; + } + + fy_allocator_registry_cleanup_internal(show_leftovers); +} +#endif + +static const char **create_allocator_array_names(void) +{ + unsigned int i, j, count; + struct fy_registered_allocator_entry *ae; + const char **names = NULL; + + allocator_registry_lock(); + + /* two passes */ + for (j = 0; j < 2; j++) { + + count = 0; + + /* try the builtins first */ + for (i = 0; i < ARRAY_SIZE(builtin_allocators); i++) { + if (j) + names[count] = builtin_allocators[i].name; + count++; + + } + for (ae = fy_registered_allocator_entry_list_head(&allocator_registry_list); ae; + ae = fy_registered_allocator_entry_next(&allocator_registry_list, ae)) { + if (j) + names[count] = ae->name; + count++; + } + + if (!j) { + names = malloc(sizeof(*names) * (count + 1)); + if (!names) + return NULL; + memset(names, 0, sizeof(*names) * (count + 1)); + } else + names[count++] = NULL; + + } + + allocator_registry_unlock(); + + return names; +} + +const char *fy_allocator_iterate(const char **prevp) +{ + unsigned int i; + const char **names; + + if (!prevp) + return NULL; + + /* yeah, it's not fast, but should be negligible */ + names = create_allocator_array_names(); + if (!names) + return NULL; + + i = 0; + if (*prevp) { + while (names[i] && strcmp(*prevp, names[i])) + i++; + if (names[i]) + i++; + } + + *prevp = names[i]; + + free(names); + + return *prevp; +} + +bool fy_allocator_is_available(const char *allocator) +{ + const char *name, *prev; + + prev = NULL; + while ((name = fy_allocator_iterate(&prev)) != NULL) { + if (!strcmp(allocator, name)) + return true; + } + return false; +} + +char *fy_allocator_get_names(void) +{ + const char *name, *prev; + size_t size, len; + char *names, *s; + + size = 0; + prev = NULL; + while ((name = fy_allocator_iterate(&prev)) != NULL) + size += strlen(name) + 1; + + names = malloc(size + 1); + if (!names) + return NULL; + + s = names; + prev = NULL; + while ((name = fy_allocator_iterate(&prev)) != NULL) { + len = strlen(name); + assert((size_t)(s + len + 1 - names) <= size); + memcpy(s, name, len); + s += len; + *s++ = ' '; + } + if (s > names && s[-1] == ' ') + s[-1] = '\0'; + return names; +} + +ssize_t fy_allocator_get_tag_linear_size(struct fy_allocator *a, int tag) +{ + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + struct fy_allocator_arena_info *arena_info; + unsigned int i, j; + size_t size; + + if (!a || tag == FY_ALLOC_TAG_NONE) + return -1; + + info = fy_allocator_get_info(a, tag); + if (!info) + return -1; + + size = 0; + for (i = 0; i < info->num_tag_infos; i++) { + tag_info = &info->tag_infos[i]; + + for (j = 0; j < tag_info->num_arena_infos; j++) { + arena_info = &tag_info->arena_infos[j]; + + size = fy_size_t_align(size, 16); /* align at 16 always */ + size += arena_info->size; + } + } + + free(info); + + /* too large? really? */ + if ((ssize_t)size < 0) + return -1; + + return (ssize_t)size; +} + +const void *fy_allocator_get_tag_single_linear(struct fy_allocator *a, int tag, size_t *sizep) +{ + struct fy_allocator_info *info; + struct fy_allocator_arena_info *arena_info; + const void *data; + + if (!a || tag == FY_ALLOC_TAG_NONE || !sizep) + return NULL; + + *sizep = 0; + data = NULL; + + info = fy_allocator_get_info(a, tag); + if (!info) + return NULL; + + /* this is great, everything is linear */ + if (info->num_tag_infos == 1 && info->tag_infos[0].num_arena_infos == 1) { + arena_info = &info->tag_infos[0].arena_infos[0]; + data = arena_info->data; + *sizep = arena_info->size; + } + + free(info); + + return data; +} + +void fy_allocator_destroy(struct fy_allocator *a) +{ + if (!a) + return; + fy_allocator_destroy_nocheck(a); +} + +void fy_allocator_dump(struct fy_allocator *a) +{ + if (!a) + return; + fy_allocator_dump_nocheck(a); +} + +int fy_allocator_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + if (!a || !stats) + return -1; + return fy_allocator_update_stats_nocheck(a, tag, stats); +} + +void *fy_allocator_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + if (!a) + return NULL; + return fy_allocator_alloc_nocheck(a, tag, size, align); +} + +void fy_allocator_free(struct fy_allocator *a, int tag, void *ptr) +{ + if (!a || !ptr) + return; + return fy_allocator_free_nocheck(a, tag, ptr); +} + +const void *fy_allocator_storev_hash(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + if (!a || !iov) + return NULL; + return fy_allocator_storev_hash_nocheck(a, tag, iov, iovcnt, align, hash); +} + +const void *fy_allocator_store(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) +{ + struct iovec iov[1]; + + if (!a || !data) + return NULL; + + /* just call the storev */ + iov[0].iov_base = (void *)data; + iov[0].iov_len = size; + + return fy_allocator_storev_hash_nocheck(a, tag, iov, 1, align, 0); +} + +const void *fy_allocator_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align) +{ + if (!a || !iov) + return NULL; + return fy_allocator_storev_nocheck(a, tag, iov, iovcnt, align); +} + +const void *fy_allocator_lookupv_hash(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + if (!a || !iov) + return NULL; + return fy_allocator_lookupv_nocheck(a, tag, iov, iovcnt, align, hash); +} + +const void *fy_allocator_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align) +{ + if (!a || !iov) + return NULL; + return fy_allocator_lookupv_nocheck(a, tag, iov, iovcnt, align, 0); +} + +const void *fy_allocator_lookup(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) +{ + struct iovec iov[1]; + + if (!a || !data) + return NULL; + + /* just call the storev */ + iov[0].iov_base = (void *)data; + iov[0].iov_len = size; + + return fy_allocator_lookupv_nocheck(a, tag, iov, 1, align, 0); +} + +void fy_allocator_release(struct fy_allocator *a, int tag, const void *ptr, size_t size) +{ + if (!a || !ptr) + return; + fy_allocator_release(a, tag, ptr, size); +} + +int fy_allocator_get_tag(struct fy_allocator *a) +{ + if (!a) + return 0; + return fy_allocator_get_tag_nocheck(a); +} + +void fy_allocator_release_tag(struct fy_allocator *a, int tag) +{ + if (!a) + return; + fy_allocator_release_tag_nocheck(a, tag); +} + +int fy_allocator_get_tag_count(struct fy_allocator *a) +{ + if (!a) + return -1; + return fy_allocator_get_tag_count_nocheck(a); +} + +int fy_allocator_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + if (!a) + return -1; + return fy_allocator_set_tag_count_nocheck(a, count); +} + +void fy_allocator_trim_tag(struct fy_allocator *a, int tag) +{ + if (!a) + return; + fy_allocator_trim_tag_nocheck(a, tag); +} + +void fy_allocator_reset_tag(struct fy_allocator *a, int tag) +{ + if (!a) + return; + fy_allocator_reset_tag_nocheck(a, tag); +} + +struct fy_allocator_info * +fy_allocator_get_info(struct fy_allocator *a, int tag) +{ + if (!a) + return NULL; + return fy_allocator_get_info_nocheck(a, tag); +} + +enum fy_allocator_cap_flags +fy_allocator_get_caps(struct fy_allocator *a) +{ + if (!a) + return 0; + + return fy_allocator_get_caps_nocheck(a); +} + +bool +fy_allocator_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + if (!a || !ptr) + return false; + + return fy_allocator_contains_nocheck(a, tag, ptr); +} + +/* respects the parent allocator (or uses posix_memalign if NULL) */ +void *fy_early_parent_allocator_alloc(struct fy_allocator *parent, int parent_tag, size_t size, size_t align) +{ + void *ptr; + int r; + + /* yeah, not gonna work */ + if (parent == FY_PARENT_ALLOCATOR_INPLACE) + return NULL; + + if (!align) + align = _Alignof(max_align_t); + + if (parent) + return fy_allocator_alloc(parent, parent_tag, size, align); + + r = posix_memalign(&ptr, align, size); + if (r) + return NULL; + + return ptr; +} + +void fy_early_parent_allocator_free(struct fy_allocator *parent, int parent_tag, void *ptr) +{ + if (!ptr || parent == FY_PARENT_ALLOCATOR_INPLACE) + return; + + if (parent) + fy_allocator_free(parent, parent_tag, ptr); + else + free(ptr); +} + +/* respects the parent allocator (or uses posix_memalign if NULL) */ +void *fy_parent_allocator_alloc(struct fy_allocator *a, size_t size, size_t align) +{ + if (!a) + return NULL; + + return fy_early_parent_allocator_alloc(a->parent, a->parent_tag, size, align); +} + +void fy_parent_allocator_free(struct fy_allocator *a, void *ptr) +{ + if (!a || !ptr) + return; + + return fy_early_parent_allocator_free(a->parent, a->parent_tag, ptr); +} diff --git a/src/allocator/fy-allocator.h b/src/allocator/fy-allocator.h new file mode 100644 index 0000000..b53fea0 --- /dev/null +++ b/src/allocator/fy-allocator.h @@ -0,0 +1,284 @@ +/* + * fy-allocator.h - allocators + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_H +#define FY_ALLOCATOR_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include + +#include "fy-typelist.h" +#include "fy-utils.h" + +#include + +struct fy_allocator; +struct fy_allocator_stats; + +struct fy_allocator_stats; +struct fy_allocator_info; + +struct fy_allocator_ops { + int (*setup)(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *cfg); + void (*cleanup)(struct fy_allocator *a); + struct fy_allocator *(*create)(struct fy_allocator *parent, int parent_tag, const void *cfg); + void (*destroy)(struct fy_allocator *a); + void (*dump)(struct fy_allocator *a); + void *(*alloc)(struct fy_allocator *a, int tag, size_t size, size_t align); + void (*free)(struct fy_allocator *a, int tag, void *data); + int (*update_stats)(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats); + const void *(*storev)(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash); + const void *(*lookupv)(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash); + void (*release)(struct fy_allocator *a, int tag, const void *data, size_t size); + int (*get_tag)(struct fy_allocator *a); + void (*release_tag)(struct fy_allocator *a, int tag); + int (*get_tag_count)(struct fy_allocator *a); + int (*set_tag_count)(struct fy_allocator *a, unsigned int tag_count); + void (*trim_tag)(struct fy_allocator *a, int tag); + void (*reset_tag)(struct fy_allocator *a, int tag); + struct fy_allocator_info *(*get_info)(struct fy_allocator *a, int tag); + unsigned int (*get_caps)(struct fy_allocator *a); /* Get capability flags (FYACF_*) */ + bool (*contains)(struct fy_allocator *a, int tag, const void *ptr); +}; + +struct fy_allocator_stats { + union { + struct { + uint64_t allocations; + uint64_t allocated; + uint64_t frees; + uint64_t freed; + uint64_t stores; + uint64_t stored; + uint64_t releases; + uint64_t released; + uint64_t dup_stores; + uint64_t dup_saved; + uint64_t system_claimed; + uint64_t system_free; + uint64_t collisions; + uint64_t unique_stores; + }; + uint64_t counters[14]; + }; +}; + +struct fy_allocator_arena_info { + size_t free, used, total; + void *data; + size_t size; +}; + +struct fy_allocator_tag_info { + int tag; + size_t free, used, total; + unsigned int num_arena_infos; + struct fy_allocator_arena_info *arena_infos; +}; + +struct fy_allocator_info { + size_t free, used, total; + unsigned int num_tag_infos; + struct fy_allocator_tag_info *tag_infos; +}; + +enum fy_allocator_flags { + FYAF_KEEP_STATS = FY_BIT(0), + FYAF_TRACE = FY_BIT(1), +}; + +struct fy_allocator { + enum fy_allocator_flags flags; + const char *name; + const struct fy_allocator_ops *ops; + struct fy_allocator *parent; + int parent_tag; +}; + +/* these private still */ +int fy_allocator_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats); +struct fy_allocator_info *fy_allocator_get_info(struct fy_allocator *a, int tag); + +static inline void fy_allocator_set_keep_stats(struct fy_allocator *a, bool keep_flags) +{ + if (!a) + return; + + if (keep_flags) + a->flags |= FYAF_KEEP_STATS; + else + a->flags &= ~FYAF_KEEP_STATS; +} + +static inline void fy_allocator_set_trace(struct fy_allocator *a, bool trace) +{ + if (!a) + return; + + if (trace) + a->flags |= FYAF_TRACE; + else + a->flags &= ~FYAF_TRACE; +} + +FY_TYPE_FWD_DECL_LIST(registered_allocator_entry); +struct fy_registered_allocator_entry { + struct list_head node; + const char *name; + const struct fy_allocator_ops *ops; +}; +FY_TYPE_DECL_LIST(registered_allocator_entry); + +/* these are private still */ +char *fy_allocator_get_names(void); +int fy_allocator_register(const char *name, const struct fy_allocator_ops *ops); +int fy_allocator_unregister(const char *name); +void fy_allocator_release(struct fy_allocator *a, int tag, const void *ptr, size_t size); + +/* respects the parent allocator (or uses posix_memalign if NULL) */ +void *fy_early_parent_allocator_alloc(struct fy_allocator *parent, int parent_tag, size_t size, size_t align); +void fy_early_parent_allocator_free(struct fy_allocator *parent, int parent_tag, void *ptr); +void *fy_parent_allocator_alloc(struct fy_allocator *a, size_t size, size_t align); +void fy_parent_allocator_free(struct fy_allocator *a, void *ptr); + +#define FY_PARENT_ALLOCATOR_MALLOC ((struct fy_allocator *)NULL) +#define FY_PARENT_ALLOCATOR_INPLACE ((struct fy_allocator *)(uintptr_t)-1) + +static inline struct fy_allocator * +fy_allocator_get_parent(struct fy_allocator *a) +{ + if (!a || !a->parent || a->parent == FY_PARENT_ALLOCATOR_INPLACE) + return NULL; + return a->parent; +} + +struct fy_allocator * +fy_allocator_create_internal(const char *name, struct fy_allocator *parent, int parent_tag, const void *cfg); + +static inline void fy_allocator_destroy_nocheck(struct fy_allocator *a) +{ + a->ops->destroy(a); +} + +static inline void fy_allocator_dump_nocheck(struct fy_allocator *a) +{ + a->ops->dump(a); +} + +static inline int fy_allocator_update_stats_nocheck(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + return a->ops->update_stats(a, tag, stats); +} + +static inline void *fy_allocator_alloc_nocheck(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + return a->ops->alloc(a, tag, size, align); +} + +static inline void fy_allocator_free_nocheck(struct fy_allocator *a, int tag, void *ptr) +{ + a->ops->free(a, tag, ptr); +} + +static inline const void *fy_allocator_store_nocheck(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) +{ + struct iovec iov[1]; + + /* just call the storev */ + iov[0].iov_base = (void *)data; + iov[0].iov_len = size; + + return a->ops->storev(a, tag, iov, 1, align, 0); +} + +static inline const void *fy_allocator_lookup_nocheck(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) +{ + struct iovec iov[1]; + + /* just call the storev */ + iov[0].iov_base = (void *)data; + iov[0].iov_len = size; + + return a->ops->lookupv(a, tag, iov, 1, align, 0); +} +static inline const void *fy_allocator_lookupv_nocheck(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + return a->ops->lookupv(a, tag, iov, iovcnt, align, hash); +} + +static inline const void *fy_allocator_storev_hash_nocheck(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + return a->ops->storev(a, tag, iov, iovcnt, align, hash); +} + +static inline const void *fy_allocator_storev_nocheck(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align) +{ + return a->ops->storev(a, tag, iov, iovcnt, align, 0); +} + +static inline void fy_allocator_release_nocheck(struct fy_allocator *a, int tag, const void *ptr, size_t size) +{ + a->ops->release(a, tag, ptr, size); +} + +static inline int fy_allocator_get_tag_nocheck(struct fy_allocator *a) +{ + return a->ops->get_tag(a); +} + +static inline void fy_allocator_release_tag_nocheck(struct fy_allocator *a, int tag) +{ + a->ops->release_tag(a, tag); +} + +static inline int fy_allocator_get_tag_count_nocheck(struct fy_allocator *a) +{ + return a->ops->get_tag_count(a); +} + +static inline int fy_allocator_set_tag_count_nocheck(struct fy_allocator *a, unsigned int count) +{ + return a->ops->set_tag_count(a, count); +} + +static inline void fy_allocator_trim_tag_nocheck(struct fy_allocator *a, int tag) +{ + a->ops->trim_tag(a, tag); +} + +static inline void fy_allocator_reset_tag_nocheck(struct fy_allocator *a, int tag) +{ + a->ops->reset_tag(a, tag); +} + +static inline struct fy_allocator_info * +fy_allocator_get_info_nocheck(struct fy_allocator *a, int tag) +{ + return a->ops->get_info(a, tag); +} + +static inline enum fy_allocator_cap_flags +fy_allocator_get_caps_nocheck(struct fy_allocator *a) +{ + return a->ops->get_caps(a); +} + +static inline bool +fy_allocator_contains_nocheck(struct fy_allocator *a, int tag, const void *ptr) +{ + return a->ops->contains(a, tag, ptr); +} + +#endif diff --git a/src/blake3/blake3.c b/src/blake3/blake3.c new file mode 100644 index 0000000..65983a2 --- /dev/null +++ b/src/blake3/blake3.c @@ -0,0 +1,745 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + +#include +#include + +#include "blake3.h" +#include "blake3_impl.h" +#include "blake3_internal.h" + +#include "fy-thread.h" + +// GCC gets confused sometimes +#ifdef GCC_DISABLE_WSTRING_OP_OVERREAD +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-overread" +#endif + +#undef HASHER_OP +#ifdef HASHER_SUFFIX +// two levels of indirection required +#define __HASHER_OP(_name, _sfx) _name ##_ ## _sfx +#define _HASHER_OP(_name, _sfx) __HASHER_OP(_name, _sfx) +#define HASHER_OP(_name) _HASHER_OP(_name, HASHER_SUFFIX) +#else +#define HASHER_OP(_name) _name +#endif + +#ifndef SIMD_DEGREE +#define SIMD_DEGREE (hs->simd_degree) +#endif + +#ifndef SIMD_DEGREE_OR_2 +#define SIMD_DEGREE_OR_2 (SIMD_DEGREE >= 2 ? SIMD_DEGREE : 2) +#endif + +// keep the same naming +#define MAX_SIMD_DEGREE SIMD_DEGREE +#define MAX_SIMD_DEGREE_OR_2 SIMD_DEGREE_OR_2 +#define blake3_simd_degree() SIMD_DEGREE + +// rename to keep the same +#define blake3_compress_in_place hasher->hs->compress_in_place +#define blake3_compress_xof hasher->hs->compress_xof +#define blake3_hash_many self->hs->hash_many + +INLINE void chunk_state_init(blake3_chunk_state *self, const uint32_t key[8], + uint8_t flags) { + memcpy(self->cv, key, BLAKE3_KEY_LEN); + self->chunk_counter = 0; + self->buf_len = 0; + self->blocks_compressed = 0; + self->flags = flags; +} + +INLINE void chunk_state_reset(blake3_chunk_state *self, const uint32_t key[8], + uint64_t chunk_counter) { + memcpy(self->cv, key, BLAKE3_KEY_LEN); + self->chunk_counter = chunk_counter; + self->blocks_compressed = 0; + self->buf_len = 0; +} + +INLINE size_t chunk_state_len(const blake3_chunk_state *self) { + return (BLAKE3_BLOCK_LEN * (size_t)self->blocks_compressed) + + ((size_t)self->buf_len); +} + +INLINE size_t chunk_state_fill_buf(blake3_chunk_state *self, + const uint8_t *input, size_t input_len) { + size_t take = BLAKE3_BLOCK_LEN - ((size_t)self->buf_len); + if (take > input_len) { + take = input_len; + } + uint8_t *dest = self->buf + ((size_t)self->buf_len); + memcpy(dest, input, take); + self->buf_len += (uint8_t)take; + return take; +} + +INLINE uint8_t chunk_state_maybe_start_flag(const blake3_chunk_state *self) { + if (self->blocks_compressed == 0) { + return CHUNK_START; + } else { + return 0; + } +} + +typedef struct { + uint32_t input_cv[8] BLAKE3_ALIGN; + uint8_t block[BLAKE3_BLOCK_LEN] BLAKE3_ALIGN; + uint64_t counter; + uint8_t block_len; + uint8_t flags; +} output_t; + +INLINE output_t make_output(const uint32_t input_cv[BLAKE3_OUT_WORDS], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + output_t ret; + memcpy(ret.input_cv, input_cv, BLAKE3_OUT_LEN); + memcpy(ret.block, block, block_len); + ret.block_len = block_len; + ret.counter = counter; + ret.flags = flags; + return ret; +} + +// Chaining values within a given chunk (specifically the compress_in_place +// interface) are represented as words. This avoids unnecessary bytes<->words +// conversion overhead in the portable implementation. However, the hash_many +// interface handles both user input and parent node blocks, so it accepts +// bytes. For that reason, chaining values in the CV stack are represented as +// bytes. +INLINE void output_chaining_value(const blake3_hasher *hasher, output_t *self, uint8_t cv[BLAKE3_OUT_LEN]) { + uint32_t cv_words[BLAKE3_OUT_WORDS] BLAKE3_ALIGN; + memcpy(cv_words, self->input_cv, BLAKE3_OUT_LEN); + memset(self->block + self->block_len, 0, BLAKE3_BLOCK_LEN - self->block_len); + blake3_compress_in_place(cv_words, self->block, self->block_len, + self->counter, self->flags); + store_cv_words(cv, cv_words); +} + +INLINE void output_root_bytes(const blake3_hasher *hasher, output_t *self, uint64_t seek, uint8_t *out, + size_t out_len) { + uint64_t output_block_counter = seek / 64; + size_t offset_within_block = seek % 64; + uint8_t wide_buf[64] BLAKE3_ALIGN; + while (out_len > 0) { + memset(self->block + self->block_len, 0, BLAKE3_BLOCK_LEN - self->block_len); + blake3_compress_xof(self->input_cv, self->block, self->block_len, + output_block_counter, self->flags | ROOT, wide_buf); + size_t available_bytes = 64 - offset_within_block; + size_t memcpy_len; + if (out_len > available_bytes) { + memcpy_len = available_bytes; + } else { + memcpy_len = out_len; + } + memcpy(out, wide_buf + offset_within_block, memcpy_len); + out += memcpy_len; + out_len -= memcpy_len; + output_block_counter += 1; + offset_within_block = 0; + } +} + +INLINE void chunk_state_update(blake3_hasher *hasher, blake3_chunk_state *self, const uint8_t *input, + size_t input_len) { + if (self->buf_len > 0) { + size_t take = chunk_state_fill_buf(self, input, input_len); + input += take; + input_len -= take; + if (input_len > 0) { + blake3_compress_in_place( + self->cv, self->buf, BLAKE3_BLOCK_LEN, self->chunk_counter, + self->flags | chunk_state_maybe_start_flag(self)); + self->blocks_compressed += 1; + self->buf_len = 0; + } + } + + while (input_len > BLAKE3_BLOCK_LEN) { + blake3_compress_in_place(self->cv, input, BLAKE3_BLOCK_LEN, + self->chunk_counter, + self->flags | chunk_state_maybe_start_flag(self)); + self->blocks_compressed += 1; + input += BLAKE3_BLOCK_LEN; + input_len -= BLAKE3_BLOCK_LEN; + } + + size_t take = chunk_state_fill_buf(self, input, input_len); + input += take; + input_len -= take; +} + +INLINE output_t chunk_state_output(const blake3_chunk_state *self) { + uint8_t block_flags = + self->flags | chunk_state_maybe_start_flag(self) | CHUNK_END; + return make_output(self->cv, self->buf, self->buf_len, self->chunk_counter, + block_flags); +} + +INLINE output_t parent_output(const uint8_t block[BLAKE3_BLOCK_LEN], + const uint32_t key[8], uint8_t flags) { + return make_output(key, block, BLAKE3_BLOCK_LEN, 0, flags | PARENT); +} + +// Given some input larger than one chunk, return the number of bytes that +// should go in the left subtree. This is the largest power-of-2 number of +// chunks that leaves at least 1 byte for the right subtree. +INLINE size_t left_len(size_t content_len) { + // Subtract 1 to reserve at least one byte for the right side. content_len + // should always be greater than BLAKE3_CHUNK_LEN. + size_t full_chunks = (content_len - 1) / BLAKE3_CHUNK_LEN; + return round_down_to_power_of_2(full_chunks) * BLAKE3_CHUNK_LEN; +} + +// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE chunks at the same time +// on a single thread. Write out the chunk chaining values and return the +// number of chunks hashed. These chunks are never the root and never empty; +// those cases use a different codepath. +INLINE size_t compress_chunks_parallel(blake3_hasher *self, + const uint8_t *input, size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, + uint8_t *out) { +#if defined(BLAKE3_TESTING) + assert(0 < input_len); + assert(input_len <= MAX_SIMD_DEGREE * BLAKE3_CHUNK_LEN); +#endif + + const uint8_t *chunks_array[MAX_SIMD_DEGREE]; + size_t input_position = 0; + size_t chunks_array_len = 0; + while (input_len - input_position >= BLAKE3_CHUNK_LEN) { + chunks_array[chunks_array_len] = &input[input_position]; + input_position += BLAKE3_CHUNK_LEN; + chunks_array_len += 1; + } + size_t i = chunks_array_len; + while (i < MAX_SIMD_DEGREE) + chunks_array[i++] = NULL; + + blake3_hash_many(chunks_array, chunks_array_len, + BLAKE3_CHUNK_LEN / BLAKE3_BLOCK_LEN, key, chunk_counter, + true, flags, CHUNK_START, CHUNK_END, out); + + // Hash the remaining partial chunk, if there is one. Note that the empty + // chunk (meaning the empty message) is a different codepath. + if (input_len > input_position) { + uint64_t counter = chunk_counter + (uint64_t)chunks_array_len; + blake3_chunk_state chunk_state; + chunk_state_init(&chunk_state, key, flags); + chunk_state.chunk_counter = counter; + chunk_state_update(self, &chunk_state, &input[input_position], + input_len - input_position); + output_t output = chunk_state_output(&chunk_state); + output_chaining_value(self, &output, &out[chunks_array_len * BLAKE3_OUT_LEN]); + return chunks_array_len + 1; + } else { + return chunks_array_len; + } +} + +// Use SIMD parallelism to hash up to MAX_SIMD_DEGREE parents at the same time +// on a single thread. Write out the parent chaining values and return the +// number of parents hashed. (If there's an odd input chaining value left over, +// return it as an additional output.) These parents are never the root and +// never empty; those cases use a different codepath. +INLINE size_t compress_parents_parallel(blake3_hasher *self, + const uint8_t *child_chaining_values, + size_t num_chaining_values, + const uint32_t key[8], uint8_t flags, + uint8_t *out) { +#if defined(BLAKE3_TESTING) + assert(2 <= num_chaining_values); + assert(num_chaining_values <= 2 * MAX_SIMD_DEGREE_OR_2); +#endif + + const uint8_t *parents_array[MAX_SIMD_DEGREE_OR_2]; + size_t parents_array_len = 0; + while (num_chaining_values - (2 * parents_array_len) >= 2) { + parents_array[parents_array_len] = + &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN]; + parents_array_len += 1; + } + + blake3_hash_many(parents_array, parents_array_len, 1, key, + 0, // Parents always use counter 0. + false, flags | PARENT, + 0, // Parents have no start flags. + 0, // Parents have no end flags. + out); + + // If there's an odd child left over, it becomes an output. + if (num_chaining_values > 2 * parents_array_len) { + memcpy(&out[parents_array_len * BLAKE3_OUT_LEN], + &child_chaining_values[2 * parents_array_len * BLAKE3_OUT_LEN], + BLAKE3_OUT_LEN); + return parents_array_len + 1; + } else { + return parents_array_len; + } +} + +static size_t blake3_compress_subtree_wide(blake3_hasher *self, + const uint8_t *input, + size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, + uint8_t flags, uint8_t *out); + +static bool blake3_compress_subtree_wide_work_check(const void *arg) { + const blake3_compress_subtree_state *s = arg; + + /* only off-load to thread if we have enough input */ + return s->input_len >= s->self->hs->mt_degree * BLAKE3_CHUNK_LEN; +} + +static void blake3_compress_subtree_wide_thread(void *arg) { + blake3_compress_subtree_state *s = arg; + + s->n = blake3_compress_subtree_wide(s->self, + s->input, s->input_len, + s->key, + s->chunk_counter, + s->flags, + s->out); +} + +// The wide helper function returns (writes out) an array of chaining values +// and returns the length of that array. The number of chaining values returned +// is the dynamically detected SIMD degree, at most MAX_SIMD_DEGREE. Or fewer, +// if the input is shorter than that many chunks. The reason for maintaining a +// wide array of chaining values going back up the tree, is to allow the +// implementation to hash as many parents in parallel as possible. +// +// As a special case when the SIMD degree is 1, this function will still return +// at least 2 outputs. This guarantees that this function doesn't perform the +// root compression. (If it did, it would use the wrong flags, and also we +// wouldn't be able to implement extendable output.) Note that this function is +// not used when the whole input is only 1 chunk long; that's a different +// codepath. +// +// Why not just have the caller split the input on the first update(), instead +// of implementing this special rule? Because we don't want to limit SIMD or +// multi-threading parallelism for that update(). +static size_t blake3_compress_subtree_wide(blake3_hasher *self, const uint8_t *input, + size_t input_len, + const uint32_t key[8], + uint64_t chunk_counter, + uint8_t flags, uint8_t *out) { + // Note that the single chunk case does *not* bump the SIMD degree up to 2 + // when it is 1. If this implementation adds multi-threading in the future, + // this gives us the option of multi-threading even the 2-chunk case, which + // can help performance on smaller platforms. + if (input_len <= blake3_simd_degree() * BLAKE3_CHUNK_LEN) { + return compress_chunks_parallel(self, input, input_len, key, chunk_counter, flags, + out); + } + + // With more than simd_degree chunks, we need to recurse. Start by dividing + // the input into left and right subtrees. (Note that this is only optimal + // as long as the SIMD degree is a power of 2. If we ever get a SIMD degree + // of 3 or something, we'll need a more complicated strategy.) + size_t left_input_len = left_len(input_len); + size_t right_input_len = input_len - left_input_len; + const uint8_t *right_input = &input[left_input_len]; + uint64_t right_chunk_counter = + chunk_counter + (uint64_t)(left_input_len / BLAKE3_CHUNK_LEN); + + // Make space for the child outputs. Here we use MAX_SIMD_DEGREE_OR_2 to + // account for the special case of returning 2 outputs when the SIMD degree + // is 1. + uint8_t cv_array[2 * MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; + size_t degree = blake3_simd_degree(); + if (left_input_len > BLAKE3_CHUNK_LEN && degree == 1) { + // The special case: We always use a degree of at least two, to make + // sure there are two outputs. Except, as noted above, at the chunk + // level, where we allow degree=1. (Note that the 1-chunk-input case is + // a different codepath.) + degree = 2; + } + uint8_t *right_cvs = &cv_array[degree * BLAKE3_OUT_LEN]; + + // Recurse + size_t left_n, right_n; + if (self->hs->cfg.no_mthread) { + left_n = blake3_compress_subtree_wide(self, input, left_input_len, key, + chunk_counter, flags, cv_array); + right_n = blake3_compress_subtree_wide(self, right_input, right_input_len, key, + right_chunk_counter, flags, right_cvs); + } else { + + // this is the multi-threaded implementation + blake3_compress_subtree_state states[2]; + + /* common */ + states[0].self = states[1].self = self; + states[0].key = states[1].key = key; + states[0].flags = states[1].flags = flags; + states[0].n = states[1].n = 0; + + /* left */ + states[0].input = input; + states[0].input_len = left_input_len; + states[0].chunk_counter = chunk_counter; + states[0].out = cv_array; + + /* right */ + states[1].input = right_input; + states[1].input_len = right_input_len; + states[1].chunk_counter = right_chunk_counter; + states[1].out = right_cvs; + + fy_thread_arg_array_join(self->hs->tp, + blake3_compress_subtree_wide_thread, + blake3_compress_subtree_wide_work_check, + states, sizeof(states[0]), 2); + + left_n = states[0].n; + right_n = states[1].n; + } + + // The special case again. If SIMD_DEGREE=1, then we'll have left_n=1 and + // right_n=1. Rather than compressing them into a single output, return + // them directly, to make sure we always have at least two outputs. + if (left_n == 1) { + memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); + return 2; + } + + // Otherwise, do one layer of parent node compression. + size_t num_chaining_values = left_n + right_n; + return compress_parents_parallel(self, cv_array, num_chaining_values, key, flags, + out); +} + +// Hash a subtree with compress_subtree_wide(), and then condense the resulting +// list of chaining values down to a single parent node. Don't compress that +// last parent node, however. Instead, return its message bytes (the +// concatenated chaining values of its children). This is necessary when the +// first call to update() supplies a complete subtree, because the topmost +// parent node of that subtree could end up being the root. It's also necessary +// for extended output in the general case. +// +// As with compress_subtree_wide(), this function is not used on inputs of 1 +// chunk or less. That's a different codepath. +INLINE void compress_subtree_to_parent_node(blake3_hasher *self, + const uint8_t *input, size_t input_len, const uint32_t key[8], + uint64_t chunk_counter, uint8_t flags, uint8_t out[2 * BLAKE3_OUT_LEN]) { +#if defined(BLAKE3_TESTING) + assert(input_len > BLAKE3_CHUNK_LEN); +#endif + + uint8_t cv_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN]; + size_t num_cvs = blake3_compress_subtree_wide(self, input, input_len, key, + chunk_counter, flags, cv_array); + assert(num_cvs <= MAX_SIMD_DEGREE_OR_2); + + // If MAX_SIMD_DEGREE is greater than 2 and there's enough input, + // compress_subtree_wide() returns more than 2 chaining values. Condense + // them into 2 by forming parent nodes repeatedly. + uint8_t out_array[MAX_SIMD_DEGREE_OR_2 * BLAKE3_OUT_LEN / 2]; + // The second half of this loop condition is always true, and we just + // asserted it above. But GCC can't tell that it's always true, and if NDEBUG + // is set on platforms where MAX_SIMD_DEGREE_OR_2 == 2, GCC emits spurious + // warnings here. GCC 8.5 is particularly sensitive, so if you're changing + // this code, test it against that version. + while (num_cvs > 2 && num_cvs <= MAX_SIMD_DEGREE_OR_2) { + num_cvs = + compress_parents_parallel(self, cv_array, num_cvs, key, flags, out_array); + memcpy(cv_array, out_array, num_cvs * BLAKE3_OUT_LEN); + } + memcpy(out, cv_array, 2 * BLAKE3_OUT_LEN); +} + +INLINE void hasher_init_base(blake3_host_state *hs, blake3_hasher *self, const uint32_t key[8], + uint8_t flags) { + memset(self, 0, sizeof(*self)); + self->hs = hs; + memcpy(self->key, key, BLAKE3_KEY_LEN); + chunk_state_init(&self->chunk, key, flags); + self->cv_stack_len = 0; +} + +static const uint32_t IV[8] BLAKE3_ALIGN = { + B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3, + B3_IV_4, B3_IV_5, B3_IV_6, B3_IV_7, +}; + +void HASHER_OP(blake3_hasher_init) (blake3_host_state *hs, blake3_hasher *self) { hasher_init_base(hs, self, IV, 0); } + +void HASHER_OP(blake3_hasher_init_keyed) (blake3_host_state *hs, blake3_hasher *self, + const uint8_t key[BLAKE3_KEY_LEN]) { + uint32_t key_words[8] BLAKE3_ALIGN; + load_key_words(key, key_words); + hasher_init_base(hs, self, key_words, KEYED_HASH); +} + +void HASHER_OP(blake3_hasher_init_derive_key_raw) (blake3_host_state *hs, blake3_hasher *self, const void *context, + size_t context_len) { + blake3_hasher context_hasher; + hasher_init_base(hs, &context_hasher, IV, DERIVE_KEY_CONTEXT); + blake3_hasher_update(&context_hasher, context, context_len); + uint8_t context_key[BLAKE3_KEY_LEN]; + blake3_hasher_finalize(&context_hasher, context_key, BLAKE3_KEY_LEN); + uint32_t context_key_words[8]; + load_key_words(context_key, context_key_words); + hasher_init_base(hs, self, context_key_words, DERIVE_KEY_MATERIAL); +} + +void HASHER_OP(blake3_hasher_init_derive_key) (blake3_host_state *hs, blake3_hasher *self, const char *context) { + blake3_hasher_init_derive_key_raw(hs, self, context, strlen(context)); +} + +// As described in hasher_push_cv() below, we do "lazy merging", delaying +// merges until right before the next CV is about to be added. This is +// different from the reference implementation. Another difference is that we +// aren't always merging 1 chunk at a time. Instead, each CV might represent +// any power-of-two number of chunks, as long as the smaller-above-larger stack +// order is maintained. Instead of the "count the trailing 0-bits" algorithm +// described in the spec, we use a "count the total number of 1-bits" variant +// that doesn't require us to retain the subtree size of the CV on top of the +// stack. The principle is the same: each CV that should remain in the stack is +// represented by a 1-bit in the total number of chunks (or bytes) so far. +INLINE void hasher_merge_cv_stack(blake3_hasher *self, uint64_t total_len) { + size_t post_merge_stack_len = (size_t)popcnt(total_len); + while (self->cv_stack_len > post_merge_stack_len) { + uint8_t *parent_node = + &self->cv_stack[(self->cv_stack_len - 2) * BLAKE3_OUT_LEN]; + output_t output = parent_output(parent_node, self->key, self->chunk.flags); + output_chaining_value(self, &output, parent_node); + self->cv_stack_len -= 1; + } +} + +// In reference_impl.rs, we merge the new CV with existing CVs from the stack +// before pushing it. We can do that because we know more input is coming, so +// we know none of the merges are root. +// +// This setting is different. We want to feed as much input as possible to +// compress_subtree_wide(), without setting aside anything for the chunk_state. +// If the user gives us 64 KiB, we want to parallelize over all 64 KiB at once +// as a single subtree, if at all possible. +// +// This leads to two problems: +// 1) This 64 KiB input might be the only call that ever gets made to update. +// In this case, the root node of the 64 KiB subtree would be the root node +// of the whole tree, and it would need to be ROOT finalized. We can't +// compress it until we know. +// 2) This 64 KiB input might complete a larger tree, whose root node is +// similarly going to be the the root of the whole tree. For example, maybe +// we have 196 KiB (that is, 128 + 64) hashed so far. We can't compress the +// node at the root of the 256 KiB subtree until we know how to finalize it. +// +// The second problem is solved with "lazy merging". That is, when we're about +// to add a CV to the stack, we don't merge it with anything first, as the +// reference impl does. Instead we do merges using the *previous* CV that was +// added, which is sitting on top of the stack, and we put the new CV +// (unmerged) on top of the stack afterwards. This guarantees that we never +// merge the root node until finalize(). +// +// Solving the first problem requires an additional tool, +// compress_subtree_to_parent_node(). That function always returns the top +// *two* chaining values of the subtree it's compressing. We then do lazy +// merging with each of them separately, so that the second CV will always +// remain unmerged. (That also helps us support extendable output when we're +// hashing an input all-at-once.) +INLINE void hasher_push_cv(blake3_hasher *self, uint8_t new_cv[BLAKE3_OUT_LEN], + uint64_t chunk_counter) { + hasher_merge_cv_stack(self, chunk_counter); + memcpy(&self->cv_stack[self->cv_stack_len * BLAKE3_OUT_LEN], new_cv, + BLAKE3_OUT_LEN); + self->cv_stack_len += 1; +} + +void HASHER_OP(blake3_hasher_update) (blake3_hasher *self, const void *input, + size_t input_len) { + // Explicitly checking for zero avoids causing UB by passing a null pointer + // to memcpy. This comes up in practice with things like: + // std::vector v; + // blake3_hasher_update(&hasher, v.data(), v.size()); + if (input_len == 0) { + return; + } + + const uint8_t *input_bytes = (const uint8_t *)input; + + // If we have some partial chunk bytes in the internal chunk_state, we need + // to finish that chunk first. + if (chunk_state_len(&self->chunk) > 0) { + size_t take = BLAKE3_CHUNK_LEN - chunk_state_len(&self->chunk); + if (take > input_len) { + take = input_len; + } + chunk_state_update(self, &self->chunk, input_bytes, take); + input_bytes += take; + input_len -= take; + // If we've filled the current chunk and there's more coming, finalize this + // chunk and proceed. In this case we know it's not the root. + if (input_len > 0) { + output_t output = chunk_state_output(&self->chunk); + uint8_t chunk_cv[32] BLAKE3_ALIGN; + output_chaining_value(self, &output, chunk_cv); + hasher_push_cv(self, chunk_cv, self->chunk.chunk_counter); + chunk_state_reset(&self->chunk, self->key, self->chunk.chunk_counter + 1); + } else { + return; + } + } + + // Now the chunk_state is clear, and we have more input. If there's more than + // a single chunk (so, definitely not the root chunk), hash the largest whole + // subtree we can, with the full benefits of SIMD (and maybe in the future, + // multi-threading) parallelism. Two restrictions: + // - The subtree has to be a power-of-2 number of chunks. Only subtrees along + // the right edge can be incomplete, and we don't know where the right edge + // is going to be until we get to finalize(). + // - The subtree must evenly divide the total number of chunks up until this + // point (if total is not 0). If the current incomplete subtree is only + // waiting for 1 more chunk, we can't hash a subtree of 4 chunks. We have + // to complete the current subtree first. + // Because we might need to break up the input to form powers of 2, or to + // evenly divide what we already have, this part runs in a loop. + while (input_len > BLAKE3_CHUNK_LEN) { + size_t subtree_len = round_down_to_power_of_2(input_len); + uint64_t count_so_far = self->chunk.chunk_counter * BLAKE3_CHUNK_LEN; + // Shrink the subtree_len until it evenly divides the count so far. We know + // that subtree_len itself is a power of 2, so we can use a bitmasking + // trick instead of an actual remainder operation. (Note that if the caller + // consistently passes power-of-2 inputs of the same size, as is hopefully + // typical, this loop condition will always fail, and subtree_len will + // always be the full length of the input.) + // + // An aside: We don't have to shrink subtree_len quite this much. For + // example, if count_so_far is 1, we could pass 2 chunks to + // compress_subtree_to_parent_node. Since we'll get 2 CVs back, we'll still + // get the right answer in the end, and we might get to use 2-way SIMD + // parallelism. The problem with this optimization, is that it gets us + // stuck always hashing 2 chunks. The total number of chunks will remain + // odd, and we'll never graduate to higher degrees of parallelism. See + // https://github.com/BLAKE3-team/BLAKE3/issues/69. + while ((((uint64_t)(subtree_len - 1)) & count_so_far) != 0) { + subtree_len /= 2; + } + // The shrunken subtree_len might now be 1 chunk long. If so, hash that one + // chunk by itself. Otherwise, compress the subtree into a pair of CVs. + uint64_t subtree_chunks = subtree_len / BLAKE3_CHUNK_LEN; + if (subtree_len <= BLAKE3_CHUNK_LEN) { + blake3_chunk_state chunk_state; + chunk_state_init(&chunk_state, self->key, self->chunk.flags); + chunk_state.chunk_counter = self->chunk.chunk_counter; + chunk_state_update(self, &chunk_state, input_bytes, subtree_len); + output_t output = chunk_state_output(&chunk_state); + uint8_t cv[BLAKE3_OUT_LEN] BLAKE3_ALIGN; + output_chaining_value(self, &output, cv); + hasher_push_cv(self, cv, chunk_state.chunk_counter); + } else { + // This is the high-performance happy path, though getting here depends + // on the caller giving us a long enough input. + uint8_t cv_pair[2 * BLAKE3_OUT_LEN] BLAKE3_ALIGN; + compress_subtree_to_parent_node(self, input_bytes, subtree_len, self->key, + self->chunk.chunk_counter, + self->chunk.flags, cv_pair); + hasher_push_cv(self, cv_pair, self->chunk.chunk_counter); + hasher_push_cv(self, &cv_pair[BLAKE3_OUT_LEN], + self->chunk.chunk_counter + (subtree_chunks / 2)); + } + self->chunk.chunk_counter += subtree_chunks; + input_bytes += subtree_len; + input_len -= subtree_len; + } + + // If there's any remaining input less than a full chunk, add it to the chunk + // state. In that case, also do a final merge loop to make sure the subtree + // stack doesn't contain any unmerged pairs. The remaining input means we + // know these merges are non-root. This merge loop isn't strictly necessary + // here, because hasher_push_chunk_cv already does its own merge loop, but it + // simplifies blake3_hasher_finalize below. + if (input_len > 0) { + chunk_state_update(self, &self->chunk, input_bytes, input_len); + hasher_merge_cv_stack(self, self->chunk.chunk_counter); + } +} + +void HASHER_OP(blake3_hasher_finalize_seek) (const blake3_hasher *self, uint64_t seek, + uint8_t *out, size_t out_len) { + // Explicitly checking for zero avoids causing UB by passing a null pointer + // to memcpy. This comes up in practice with things like: + // std::vector v; + // blake3_hasher_finalize(&hasher, v.data(), v.size()); + if (out_len == 0) { + return; + } + + // If the subtree stack is empty, then the current chunk is the root. + if (self->cv_stack_len == 0) { + output_t output = chunk_state_output(&self->chunk); + output_root_bytes(self, &output, seek, out, out_len); + return; + } + // If there are any bytes in the chunk state, finalize that chunk and do a + // roll-up merge between that chunk hash and every subtree in the stack. In + // this case, the extra merge loop at the end of blake3_hasher_update + // guarantees that none of the subtrees in the stack need to be merged with + // each other first. Otherwise, if there are no bytes in the chunk state, + // then the top of the stack is a chunk hash, and we start the merge from + // that. + output_t output; + size_t cvs_remaining; + if (chunk_state_len(&self->chunk) > 0) { + cvs_remaining = self->cv_stack_len; + output = chunk_state_output(&self->chunk); + } else { + // There are always at least 2 CVs in the stack in this case. + cvs_remaining = self->cv_stack_len - 2; + output = parent_output(&self->cv_stack[cvs_remaining * 32], self->key, + self->chunk.flags); + } + while (cvs_remaining > 0) { + cvs_remaining -= 1; + uint8_t parent_block[BLAKE3_BLOCK_LEN] BLAKE3_ALIGN; + memcpy(parent_block, &self->cv_stack[cvs_remaining * 32], 32); + output_chaining_value(self, &output, &parent_block[32]); + output = parent_output(parent_block, self->key, self->chunk.flags); + } + output_root_bytes(self, &output, seek, out, out_len); +} + +void HASHER_OP(blake3_hasher_finalize) (const blake3_hasher *self, uint8_t *out, + size_t out_len) { + HASHER_OP(blake3_hasher_finalize_seek) (self, 0, out, out_len); +} + +void HASHER_OP(blake3_hasher_reset) (blake3_hasher *self) { + chunk_state_reset(&self->chunk, self->key, 0); + self->cv_stack_len = 0; +} +#ifdef GCC_DISABLE_WSTRING_OP_OVERREAD +#pragma GCC diagnostic pop +#endif + +const blake3_hasher_ops HASHER_OP(blake3_hasher_op) = { + .hasher_init = HASHER_OP(blake3_hasher_init), + .hasher_init_keyed = HASHER_OP(blake3_hasher_init_keyed), + .hasher_init_derive_key = HASHER_OP(blake3_hasher_init_derive_key), + .hasher_init_derive_key_raw = HASHER_OP(blake3_hasher_init_derive_key_raw), + .hasher_update = HASHER_OP(blake3_hasher_update), + .hasher_finalize = HASHER_OP(blake3_hasher_finalize), + .hasher_finalize_seek = HASHER_OP(blake3_hasher_finalize_seek), + .hasher_reset = HASHER_OP(blake3_hasher_reset), +}; diff --git a/src/blake3/blake3.h b/src/blake3/blake3.h new file mode 100644 index 0000000..e13d114 --- /dev/null +++ b/src/blake3/blake3.h @@ -0,0 +1,164 @@ +#ifndef BLAKE3_H +#define BLAKE3_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#define BLAKE3_VERSION_STRING "1.4.1" + +#define BLAKE3_KEY_LEN 32 +#define BLAKE3_KEY_WORDS (BLAKE3_KEY_LEN / 4) // words are 4 bytes +#define BLAKE3_OUT_LEN 32 +#define BLAKE3_OUT_WORDS (BLAKE3_OUT_LEN / 4) // words are 4 bytes +#define BLAKE3_BLOCK_LEN 64 +#define BLAKE3_CHUNK_LEN 1024 + +struct blake_host_state; +struct blake_hasher; + +const char *blake3_version(void); + +/* NOTE: maximum supported backends is 64 + * and backends are ordered by reverse order + * of preference. + * + * PORTABLE is always available and is the fall-back. + */ + +typedef enum { + + /* just the portable implementation */ + B3BID_PORTABLE, + + /* x86 */ + B3BID_SSE2, + B3BID_SSE2_ASM, + B3BID_SSE41, + B3BID_SSE41_ASM, + B3BID_AVX2, + B3BID_AVX2_ASM, + B3BID_AVX512, + B3BID_AVX512_ASM, + + /* arm */ + B3BID_NEON, + + /* gpu */ + B3BID_VULKAN, + B3BID_OPENCL, + B3BID_CUDA, + + /* CPU simd (experimental) */ + B3BID_CPUSIMD, + +} blake3_backend_id; + +#define B3BID_COUNT (B3BID_CPUSIMD+1) +#define B3BID_INVALID ((blake3_backend_id)-1) + +#define B3BF_BIT(_x) ((uint64_t)1 << (_x)) + +#define B3BF_PORTABLE B3BF_BIT(B3BID_PORTABLE) + +#define B3BF_SSE2 B3BF_BIT(B3BID_SSE2) +#define B3BF_SSE2_ASM B3BF_BIT(B3BID_SSE2_ASM) +#define B3BF_SSE41 B3BF_BIT(B3BID_SSE41) +#define B3BF_SSE41_ASM B3BF_BIT(B3BID_SSE41_ASM) +#define B3BF_AVX2 B3BF_BIT(B3BID_AVX2) +#define B3BF_AVX2_ASM B3BF_BIT(B3BID_AVX2_ASM) +#define B3BF_AVX512 B3BF_BIT(B3BID_AVX512) +#define B3BF_AVX512_ASM B3BF_BIT(B3BID_AVX512_ASM) + +#define B3BF_NEON B3BF_BIT(B3BID_NEON) + +#define B3BF_VULKAN B3BF_BIT(B3BID_VULKAN) +#define B3BF_OPENCL B3BF_BIT(B3BID_OPENCL) +#define B3BF_CUDA B3BF_BIT(B3BID_CUDA) + +#define B3BF_CPUSIMD B3BF_BIT(B3BID_CPUSIMD) + +uint64_t blake3_get_supported_backends(void); +uint64_t blake3_get_detected_backends(void); +uint64_t blake3_get_selectable_backends(void); + +typedef enum { + B3FID_HASH_MANY, + B3FID_COMPRESS_XOF, + B3FID_COMPRESS_IN_PLACE, +} blake3_func_id; + +#define B3FID_COUNT (B3FID_COMPRESS_IN_PLACE+1) +#define B3FID_INVALID ((blake3_func_id)-1) + +#define B3FF_BIT(_x) ((uint64_t)1 << (_x)) + +#define B3FF_HASH_MANY B3FF_BIT(B3FID_HASH_MANY) +#define B3FF_COMPRESS_XOF B3FF_BIT(B3FID_COMPRESS_XOF) +#define B3FF_COMPRESS_IN_PLACE B3FF_BIT(B3FID_COMPRESS_IN_PLACE) + +typedef struct blake3_backend_info { + blake3_backend_id id; + const char *name; + const char *description; + unsigned int simd_degree; + uint64_t funcs; +} blake3_backend_info; + +const blake3_backend_info *blake3_get_backend_info(blake3_backend_id id); + +typedef struct blake3_host_config { + bool debug; + bool no_mthread; + bool no_mmap; + unsigned int num_threads; + unsigned int mt_degree; // number of chunks to be worth spinning up a thread + const char *backend; // backend selection string, or NULL for default + size_t file_io_bufsz; // buffer size when doing file I/O + size_t mmap_min_chunk; // minimum chunk size for mmap + size_t mmap_max_chunk; // maximum chunk size for mmap + struct fy_thread_pool *tp; // use this thread pool instead of spinning one up +} blake3_host_config; + +struct blake3_host_state *blake3_host_state_create(const blake3_host_config *cfg); +void blake3_host_state_destroy(struct blake3_host_state *hs); + +struct blake3_hasher *blake3_hasher_create(struct blake3_host_state *hs, const uint8_t *key, const void *context, size_t context_len); +void blake3_hasher_destroy(struct blake3_hasher *self); + +/* simple optimized method for file hashing */ +int blake3_hash_file(struct blake3_hasher *hasher, const char *filename, uint8_t output[BLAKE3_OUT_LEN]); +void blake3_hash(struct blake3_hasher *hasher, const void *mem, size_t size, uint8_t output[BLAKE3_OUT_LEN]); + +/* experimental CPUSIMD enable */ +int blake3_backend_cpusimd_setup(unsigned int num_cpus, unsigned int mult_fact); +void blake3_backend_cpusimd_cleanup(void); + +void blake3_hasher_init(struct blake3_host_state *hs, struct blake3_hasher *self); +void blake3_hasher_init_keyed(struct blake3_host_state *hs, struct blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]); +void blake3_hasher_init_derive_key(struct blake3_host_state *hs, struct blake3_hasher *self, const char *context); +void blake3_hasher_init_derive_key_raw(struct blake3_host_state *hs, struct blake3_hasher *self, const void *context, size_t context_len); + +void blake3_hasher_update(struct blake3_hasher *self, const void *input, size_t input_len); +void blake3_hasher_finalize(const struct blake3_hasher *self, uint8_t *out, size_t out_len); +void blake3_hasher_finalize_seek(const struct blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len); +void blake3_hasher_reset(struct blake3_hasher *self); + +typedef struct blake3_hasher_ops { + void (*hasher_init)(struct blake3_host_state *hs, struct blake3_hasher *self); + void (*hasher_init_keyed)(struct blake3_host_state *hs, struct blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]); + void (*hasher_init_derive_key)(struct blake3_host_state *hs, struct blake3_hasher *self, const char *context); + void (*hasher_init_derive_key_raw)(struct blake3_host_state *hs, struct blake3_hasher *self, const void *context, size_t context_len); + + void (*hasher_update)(struct blake3_hasher *self, const void *input, size_t input_len); + void (*hasher_finalize)(const struct blake3_hasher *self, uint8_t *out, size_t out_len); + void (*hasher_finalize_seek)(const struct blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len); + void (*hasher_reset)(struct blake3_hasher *self); +} blake3_hasher_ops; + +#endif /* BLAKE3_H */ diff --git a/src/blake3/blake3_avx2.c b/src/blake3/blake3_avx2.c new file mode 100644 index 0000000..8d5b427 --- /dev/null +++ b/src/blake3/blake3_avx2.c @@ -0,0 +1,333 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" + +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +#define DEGREE 8 + +INLINE __m256i loadu(const uint8_t src[32]) { + return _mm256_loadu_si256((const __m256i *)src); +} + +INLINE void storeu(__m256i src, uint8_t dest[16]) { + _mm256_storeu_si256((__m256i *)dest, src); +} + +INLINE __m256i addv(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } + +// Note that clang-format doesn't like the name "xor" for some reason. +INLINE __m256i xorv(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } + +INLINE __m256i set1(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } + +INLINE __m256i rot16(__m256i x) { + return _mm256_shuffle_epi8( + x, _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, + 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); +} + +INLINE __m256i rot12(__m256i x) { + return _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 32 - 12)); +} + +INLINE __m256i rot8(__m256i x) { + return _mm256_shuffle_epi8( + x, _mm256_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1, + 12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); +} + +INLINE __m256i rot7(__m256i x) { + return _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 32 - 7)); +} + +INLINE void round_fn(__m256i v[16], __m256i m[16], size_t r) { + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[15] = rot16(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot12(v[4]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[15] = rot8(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot7(v[4]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot16(v[15]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[4] = rot12(v[4]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot8(v[15]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + v[4] = rot7(v[4]); +} + +INLINE void transpose_vecs(__m256i vecs[DEGREE]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high + // is 22/33/66/77. + __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); + __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); + __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); + __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); + __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); + __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); + __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); + __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); + + // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is + // 11/33. + __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); + __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); + __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); + __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); + __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); + __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); + __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); + __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); + + // Interleave 128-bit lanes. + vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); + vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); + vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); + vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); + vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); + vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); + vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); + vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); +} + +INLINE void transpose_msg_vecs(const uint8_t *const *inputs, + size_t block_offset, __m256i out[16]) { + out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m256i)]); + out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m256i)]); + out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m256i)]); + out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m256i)]); + out[4] = loadu(&inputs[4][block_offset + 0 * sizeof(__m256i)]); + out[5] = loadu(&inputs[5][block_offset + 0 * sizeof(__m256i)]); + out[6] = loadu(&inputs[6][block_offset + 0 * sizeof(__m256i)]); + out[7] = loadu(&inputs[7][block_offset + 0 * sizeof(__m256i)]); + out[8] = loadu(&inputs[0][block_offset + 1 * sizeof(__m256i)]); + out[9] = loadu(&inputs[1][block_offset + 1 * sizeof(__m256i)]); + out[10] = loadu(&inputs[2][block_offset + 1 * sizeof(__m256i)]); + out[11] = loadu(&inputs[3][block_offset + 1 * sizeof(__m256i)]); + out[12] = loadu(&inputs[4][block_offset + 1 * sizeof(__m256i)]); + out[13] = loadu(&inputs[5][block_offset + 1 * sizeof(__m256i)]); + out[14] = loadu(&inputs[6][block_offset + 1 * sizeof(__m256i)]); + out[15] = loadu(&inputs[7][block_offset + 1 * sizeof(__m256i)]); + for (size_t i = 0; i < 8; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs(&out[0]); + transpose_vecs(&out[8]); +} + +INLINE void load_counters(uint64_t counter, bool increment_counter, + __m256i *out_lo, __m256i *out_hi) { + const __m256i mask = _mm256_set1_epi32(-(int32_t)increment_counter); + const __m256i add0 = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0); + const __m256i add1 = _mm256_and_si256(mask, add0); + __m256i l = _mm256_add_epi32(_mm256_set1_epi32((int32_t)counter), add1); + __m256i carry = _mm256_cmpgt_epi32(_mm256_xor_si256(add1, _mm256_set1_epi32(0x80000000)), + _mm256_xor_si256( l, _mm256_set1_epi32(0x80000000))); + __m256i h = _mm256_sub_epi32(_mm256_set1_epi32((int32_t)(counter >> 32)), carry); + *out_lo = l; + *out_hi = h; +} + +static +void blake3_hash8_avx2(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m256i h_vecs[8] = { + set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), + set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), + }; + __m256i counter_low_vec, counter_high_vec; + load_counters(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m256i block_len_vec = set1(BLAKE3_BLOCK_LEN); + __m256i block_flags_vec = set1(block_flags); + __m256i msg_vecs[16]; + transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m256i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1(B3_IV_0), set1(B3_IV_1), set1(B3_IV_2), set1(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn(v, msg_vecs, 0); + round_fn(v, msg_vecs, 1); + round_fn(v, msg_vecs, 2); + round_fn(v, msg_vecs, 3); + round_fn(v, msg_vecs, 4); + round_fn(v, msg_vecs, 5); + round_fn(v, msg_vecs, 6); + h_vecs[0] = xorv(v[0], v[8]); + h_vecs[1] = xorv(v[1], v[9]); + h_vecs[2] = xorv(v[2], v[10]); + h_vecs[3] = xorv(v[3], v[11]); + h_vecs[4] = xorv(v[4], v[12]); + h_vecs[5] = xorv(v[5], v[13]); + h_vecs[6] = xorv(v[6], v[14]); + h_vecs[7] = xorv(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs(h_vecs); + storeu(h_vecs[0], &out[0 * sizeof(__m256i)]); + storeu(h_vecs[1], &out[1 * sizeof(__m256i)]); + storeu(h_vecs[2], &out[2 * sizeof(__m256i)]); + storeu(h_vecs[3], &out[3 * sizeof(__m256i)]); + storeu(h_vecs[4], &out[4 * sizeof(__m256i)]); + storeu(h_vecs[5], &out[5 * sizeof(__m256i)]); + storeu(h_vecs[6], &out[6 * sizeof(__m256i)]); + storeu(h_vecs[7], &out[7 * sizeof(__m256i)]); +} + +#if !defined(BLAKE3_NO_SSE41) +void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#else +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +#endif + +void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= DEGREE) { + blake3_hash8_avx2(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += DEGREE; + } + inputs += DEGREE; + num_inputs -= DEGREE; + out = &out[DEGREE * BLAKE3_OUT_LEN]; + } +#if !defined(BLAKE3_NO_SSE41) + blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, out); +#else + blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, + increment_counter, flags, flags_start, flags_end, + out); +#endif +} diff --git a/src/blake3/blake3_avx2_x86-64_unix.S b/src/blake3/blake3_avx2_x86-64_unix.S new file mode 100644 index 0000000..29729ec --- /dev/null +++ b/src/blake3/blake3_avx2_x86-64_unix.S @@ -0,0 +1,1815 @@ +#if defined(__ELF__) && defined(__linux__) +.section .note.GNU-stack,"",%progbits +#endif + +#if defined(__ELF__) && defined(__CET__) && defined(__has_include) +#if __has_include() +#include +#endif +#endif + +#if !defined(_CET_ENDBR) +#define _CET_ENDBR +#endif + +.intel_syntax noprefix +.global _blake3_hash_many_avx2_asm +.global blake3_hash_many_avx2_asm +#ifdef __APPLE__ +.text +#else +.section .text +#endif + .p2align 6 +_blake3_hash_many_avx2_asm: +blake3_hash_many_avx2_asm: + _CET_ENDBR + push r15 + push r14 + push r13 + push r12 + push rbx + push rbp + mov rbp, rsp + sub rsp, 680 + and rsp, 0xFFFFFFFFFFFFFFC0 + neg r9d + vmovd xmm0, r9d + vpbroadcastd ymm0, xmm0 + vmovdqa ymmword ptr [rsp+0x280], ymm0 + vpand ymm1, ymm0, ymmword ptr [ADD0+rip] + vpand ymm2, ymm0, ymmword ptr [ADD1+rip] + vmovdqa ymmword ptr [rsp+0x220], ymm2 + vmovd xmm2, r8d + vpbroadcastd ymm2, xmm2 + vpaddd ymm2, ymm2, ymm1 + vmovdqa ymmword ptr [rsp+0x240], ymm2 + vpxor ymm1, ymm1, ymmword ptr [CMP_MSB_MASK+rip] + vpxor ymm2, ymm2, ymmword ptr [CMP_MSB_MASK+rip] + vpcmpgtd ymm2, ymm1, ymm2 + shr r8, 32 + vmovd xmm3, r8d + vpbroadcastd ymm3, xmm3 + vpsubd ymm3, ymm3, ymm2 + vmovdqa ymmword ptr [rsp+0x260], ymm3 + shl rdx, 6 + mov qword ptr [rsp+0x2A0], rdx + cmp rsi, 8 + jc 3f +2: + vpbroadcastd ymm0, dword ptr [rcx] + vpbroadcastd ymm1, dword ptr [rcx+0x4] + vpbroadcastd ymm2, dword ptr [rcx+0x8] + vpbroadcastd ymm3, dword ptr [rcx+0xC] + vpbroadcastd ymm4, dword ptr [rcx+0x10] + vpbroadcastd ymm5, dword ptr [rcx+0x14] + vpbroadcastd ymm6, dword ptr [rcx+0x18] + vpbroadcastd ymm7, dword ptr [rcx+0x1C] + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + mov r12, qword ptr [rdi+0x20] + mov r13, qword ptr [rdi+0x28] + mov r14, qword ptr [rdi+0x30] + mov r15, qword ptr [rdi+0x38] + movzx eax, byte ptr [rbp+0x38] + movzx ebx, byte ptr [rbp+0x40] + or eax, ebx + xor edx, edx +.p2align 5 +9: + movzx ebx, byte ptr [rbp+0x48] + or ebx, eax + add rdx, 64 + cmp rdx, qword ptr [rsp+0x2A0] + cmove eax, ebx + mov dword ptr [rsp+0x200], eax + vmovups xmm8, xmmword ptr [r8+rdx-0x40] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x40], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x40] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x40], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x40] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x40], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x40] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x40], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm8, ymm12, ymm14, 136 + vmovaps ymmword ptr [rsp], ymm8 + vshufps ymm9, ymm12, ymm14, 221 + vmovaps ymmword ptr [rsp+0x20], ymm9 + vshufps ymm10, ymm13, ymm15, 136 + vmovaps ymmword ptr [rsp+0x40], ymm10 + vshufps ymm11, ymm13, ymm15, 221 + vmovaps ymmword ptr [rsp+0x60], ymm11 + vmovups xmm8, xmmword ptr [r8+rdx-0x30] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x30], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x30] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x30], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x30] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x30], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x30] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x30], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm8, ymm12, ymm14, 136 + vmovaps ymmword ptr [rsp+0x80], ymm8 + vshufps ymm9, ymm12, ymm14, 221 + vmovaps ymmword ptr [rsp+0xA0], ymm9 + vshufps ymm10, ymm13, ymm15, 136 + vmovaps ymmword ptr [rsp+0xC0], ymm10 + vshufps ymm11, ymm13, ymm15, 221 + vmovaps ymmword ptr [rsp+0xE0], ymm11 + vmovups xmm8, xmmword ptr [r8+rdx-0x20] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x20], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x20] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x20], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x20] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x20], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x20] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x20], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm8, ymm12, ymm14, 136 + vmovaps ymmword ptr [rsp+0x100], ymm8 + vshufps ymm9, ymm12, ymm14, 221 + vmovaps ymmword ptr [rsp+0x120], ymm9 + vshufps ymm10, ymm13, ymm15, 136 + vmovaps ymmword ptr [rsp+0x140], ymm10 + vshufps ymm11, ymm13, ymm15, 221 + vmovaps ymmword ptr [rsp+0x160], ymm11 + vmovups xmm8, xmmword ptr [r8+rdx-0x10] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x10], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x10] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x10], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x10] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x10], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x10] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x10], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm8, ymm12, ymm14, 136 + vmovaps ymmword ptr [rsp+0x180], ymm8 + vshufps ymm9, ymm12, ymm14, 221 + vmovaps ymmword ptr [rsp+0x1A0], ymm9 + vshufps ymm10, ymm13, ymm15, 136 + vmovaps ymmword ptr [rsp+0x1C0], ymm10 + vshufps ymm11, ymm13, ymm15, 221 + vmovaps ymmword ptr [rsp+0x1E0], ymm11 + vpbroadcastd ymm15, dword ptr [rsp+0x200] + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r12+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r13+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r14+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + prefetcht0 [r15+rdx+0x80] + vpaddd ymm0, ymm0, ymmword ptr [rsp] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x40] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x80] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm0, ymmword ptr [rsp+0x240] + vpxor ymm13, ymm1, ymmword ptr [rsp+0x260] + vpxor ymm14, ymm2, ymmword ptr [BLAKE3_BLOCK_LEN+rip] + vpxor ymm15, ymm3, ymm15 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [BLAKE3_IV_0+rip] + vpaddd ymm9, ymm13, ymmword ptr [BLAKE3_IV_1+rip] + vpaddd ymm10, ymm14, ymmword ptr [BLAKE3_IV_2+rip] + vpaddd ymm11, ymm15, ymmword ptr [BLAKE3_IV_3+rip] + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x20] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x60] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0xA0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x100] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x180] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1C0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x120] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1A0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x40] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x60] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0xE0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x80] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0xC0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140] + vpaddd ymm2, ymm2, ymmword ptr [rsp] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1A0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x20] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x120] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x160] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1C0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x60] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1A0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x80] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x40] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1C0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0xC0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x120] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x160] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0xA0] + vpaddd ymm1, ymm1, ymmword ptr [rsp] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1E0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x20] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x140] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1C0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1A0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0xE0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x120] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x60] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x80] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0xA0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x20] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x40] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x100] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x180] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x120] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x1E0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1C0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1A0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x140] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0xE0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0] + vpaddd ymm2, ymm2, ymmword ptr [rsp] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x40] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x60] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x20] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x80] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x120] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x160] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x100] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1E0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1C0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x180] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x20] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1A0] + vpaddd ymm1, ymm1, ymmword ptr [rsp] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x40] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x80] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x60] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x140] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0xC0] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x160] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0xA0] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x20] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x100] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1E0] + vpaddd ymm1, ymm1, ymmword ptr [rsp] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x120] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xC0] + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxor ymm12, ymm12, ymm0 + vpxor ymm13, ymm13, ymm1 + vpxor ymm14, ymm14, ymm2 + vpxor ymm15, ymm15, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpshufb ymm15, ymm15, ymm8 + vpaddd ymm8, ymm12, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxor ymm4, ymm4, ymm8 + vpxor ymm5, ymm5, ymm9 + vpxor ymm6, ymm6, ymm10 + vpxor ymm7, ymm7, ymm11 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x1C0] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x40] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x60] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0xE0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT16+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vmovdqa ymmword ptr [rsp+0x200], ymm8 + vpsrld ymm8, ymm5, 12 + vpslld ymm5, ymm5, 20 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 12 + vpslld ymm6, ymm6, 20 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 12 + vpslld ymm7, ymm7, 20 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 12 + vpslld ymm4, ymm4, 20 + vpor ymm4, ymm4, ymm8 + vpaddd ymm0, ymm0, ymmword ptr [rsp+0x140] + vpaddd ymm1, ymm1, ymmword ptr [rsp+0x180] + vpaddd ymm2, ymm2, ymmword ptr [rsp+0x80] + vpaddd ymm3, ymm3, ymmword ptr [rsp+0x1A0] + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxor ymm15, ymm15, ymm0 + vpxor ymm12, ymm12, ymm1 + vpxor ymm13, ymm13, ymm2 + vpxor ymm14, ymm14, ymm3 + vbroadcasti128 ymm8, xmmword ptr [ROT8+rip] + vpshufb ymm15, ymm15, ymm8 + vpshufb ymm12, ymm12, ymm8 + vpshufb ymm13, ymm13, ymm8 + vpshufb ymm14, ymm14, ymm8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm13, ymmword ptr [rsp+0x200] + vpaddd ymm9, ymm9, ymm14 + vpxor ymm5, ymm5, ymm10 + vpxor ymm6, ymm6, ymm11 + vpxor ymm7, ymm7, ymm8 + vpxor ymm4, ymm4, ymm9 + vpxor ymm0, ymm0, ymm8 + vpxor ymm1, ymm1, ymm9 + vpxor ymm2, ymm2, ymm10 + vpxor ymm3, ymm3, ymm11 + vpsrld ymm8, ymm5, 7 + vpslld ymm5, ymm5, 25 + vpor ymm5, ymm5, ymm8 + vpsrld ymm8, ymm6, 7 + vpslld ymm6, ymm6, 25 + vpor ymm6, ymm6, ymm8 + vpsrld ymm8, ymm7, 7 + vpslld ymm7, ymm7, 25 + vpor ymm7, ymm7, ymm8 + vpsrld ymm8, ymm4, 7 + vpslld ymm4, ymm4, 25 + vpor ymm4, ymm4, ymm8 + vpxor ymm4, ymm4, ymm12 + vpxor ymm5, ymm5, ymm13 + vpxor ymm6, ymm6, ymm14 + vpxor ymm7, ymm7, ymm15 + movzx eax, byte ptr [rbp+0x38] + jne 9b + mov rbx, qword ptr [rbp+0x50] + vunpcklps ymm8, ymm0, ymm1 + vunpcklps ymm9, ymm2, ymm3 + vunpckhps ymm10, ymm0, ymm1 + vunpcklps ymm11, ymm4, ymm5 + vunpcklps ymm0, ymm6, ymm7 + vshufps ymm12, ymm8, ymm9, 78 + vblendps ymm1, ymm8, ymm12, 0xCC + vshufps ymm8, ymm11, ymm0, 78 + vunpckhps ymm13, ymm2, ymm3 + vblendps ymm2, ymm11, ymm8, 0xCC + vblendps ymm3, ymm12, ymm9, 0xCC + vperm2f128 ymm12, ymm1, ymm2, 0x20 + vmovups ymmword ptr [rbx], ymm12 + vunpckhps ymm14, ymm4, ymm5 + vblendps ymm4, ymm8, ymm0, 0xCC + vunpckhps ymm15, ymm6, ymm7 + vperm2f128 ymm7, ymm3, ymm4, 0x20 + vmovups ymmword ptr [rbx+0x20], ymm7 + vshufps ymm5, ymm10, ymm13, 78 + vblendps ymm6, ymm5, ymm13, 0xCC + vshufps ymm13, ymm14, ymm15, 78 + vblendps ymm10, ymm10, ymm5, 0xCC + vblendps ymm14, ymm14, ymm13, 0xCC + vperm2f128 ymm8, ymm10, ymm14, 0x20 + vmovups ymmword ptr [rbx+0x40], ymm8 + vblendps ymm15, ymm13, ymm15, 0xCC + vperm2f128 ymm13, ymm6, ymm15, 0x20 + vmovups ymmword ptr [rbx+0x60], ymm13 + vperm2f128 ymm9, ymm1, ymm2, 0x31 + vperm2f128 ymm11, ymm3, ymm4, 0x31 + vmovups ymmword ptr [rbx+0x80], ymm9 + vperm2f128 ymm14, ymm10, ymm14, 0x31 + vperm2f128 ymm15, ymm6, ymm15, 0x31 + vmovups ymmword ptr [rbx+0xA0], ymm11 + vmovups ymmword ptr [rbx+0xC0], ymm14 + vmovups ymmword ptr [rbx+0xE0], ymm15 + vmovdqa ymm0, ymmword ptr [rsp+0x220] + vpaddd ymm1, ymm0, ymmword ptr [rsp+0x240] + vmovdqa ymmword ptr [rsp+0x240], ymm1 + vpxor ymm0, ymm0, ymmword ptr [CMP_MSB_MASK+rip] + vpxor ymm2, ymm1, ymmword ptr [CMP_MSB_MASK+rip] + vpcmpgtd ymm2, ymm0, ymm2 + vmovdqa ymm0, ymmword ptr [rsp+0x260] + vpsubd ymm2, ymm0, ymm2 + vmovdqa ymmword ptr [rsp+0x260], ymm2 + add rdi, 64 + add rbx, 256 + mov qword ptr [rbp+0x50], rbx + sub rsi, 8 + cmp rsi, 8 + jnc 2b + test rsi, rsi + jnz 3f +4: + vzeroupper + mov rsp, rbp + pop rbp + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + ret +.p2align 5 +3: + mov rbx, qword ptr [rbp+0x50] + mov r15, qword ptr [rsp+0x2A0] + movzx r13d, byte ptr [rbp+0x38] + movzx r12d, byte ptr [rbp+0x48] + test rsi, 0x4 + je 3f + vbroadcasti128 ymm0, xmmword ptr [rcx] + vbroadcasti128 ymm1, xmmword ptr [rcx+0x10] + vmovdqa ymm8, ymm0 + vmovdqa ymm9, ymm1 + vbroadcasti128 ymm12, xmmword ptr [rsp+0x240] + vbroadcasti128 ymm13, xmmword ptr [rsp+0x260] + vpunpckldq ymm14, ymm12, ymm13 + vpunpckhdq ymm15, ymm12, ymm13 + vpermq ymm14, ymm14, 0x50 + vpermq ymm15, ymm15, 0x50 + vbroadcasti128 ymm12, xmmword ptr [BLAKE3_BLOCK_LEN+rip] + vpblendd ymm14, ymm14, ymm12, 0x44 + vpblendd ymm15, ymm15, ymm12, 0x44 + vmovdqa ymmword ptr [rsp], ymm14 + vmovdqa ymmword ptr [rsp+0x20], ymm15 + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + mov dword ptr [rsp+0x200], eax + vmovups ymm2, ymmword ptr [r8+rdx-0x40] + vinsertf128 ymm2, ymm2, xmmword ptr [r9+rdx-0x40], 0x01 + vmovups ymm3, ymmword ptr [r8+rdx-0x30] + vinsertf128 ymm3, ymm3, xmmword ptr [r9+rdx-0x30], 0x01 + vshufps ymm4, ymm2, ymm3, 136 + vshufps ymm5, ymm2, ymm3, 221 + vmovups ymm2, ymmword ptr [r8+rdx-0x20] + vinsertf128 ymm2, ymm2, xmmword ptr [r9+rdx-0x20], 0x01 + vmovups ymm3, ymmword ptr [r8+rdx-0x10] + vinsertf128 ymm3, ymm3, xmmword ptr [r9+rdx-0x10], 0x01 + vshufps ymm6, ymm2, ymm3, 136 + vshufps ymm7, ymm2, ymm3, 221 + vpshufd ymm6, ymm6, 0x93 + vpshufd ymm7, ymm7, 0x93 + vmovups ymm10, ymmword ptr [r10+rdx-0x40] + vinsertf128 ymm10, ymm10, xmmword ptr [r11+rdx-0x40], 0x01 + vmovups ymm11, ymmword ptr [r10+rdx-0x30] + vinsertf128 ymm11, ymm11, xmmword ptr [r11+rdx-0x30], 0x01 + vshufps ymm12, ymm10, ymm11, 136 + vshufps ymm13, ymm10, ymm11, 221 + vmovups ymm10, ymmword ptr [r10+rdx-0x20] + vinsertf128 ymm10, ymm10, xmmword ptr [r11+rdx-0x20], 0x01 + vmovups ymm11, ymmword ptr [r10+rdx-0x10] + vinsertf128 ymm11, ymm11, xmmword ptr [r11+rdx-0x10], 0x01 + vshufps ymm14, ymm10, ymm11, 136 + vshufps ymm15, ymm10, ymm11, 221 + vpshufd ymm14, ymm14, 0x93 + vpshufd ymm15, ymm15, 0x93 + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + vpbroadcastd ymm2, dword ptr [rsp+0x200] + vmovdqa ymm3, ymmword ptr [rsp] + vmovdqa ymm11, ymmword ptr [rsp+0x20] + vpblendd ymm3, ymm3, ymm2, 0x88 + vpblendd ymm11, ymm11, ymm2, 0x88 + vbroadcasti128 ymm2, xmmword ptr [BLAKE3_IV+rip] + vmovdqa ymm10, ymm2 + mov al, 7 +9: + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm8, ymm8, ymm12 + vmovdqa ymmword ptr [rsp+0x40], ymm4 + nop + vmovdqa ymmword ptr [rsp+0x60], ymm12 + nop + vpaddd ymm0, ymm0, ymm1 + vpaddd ymm8, ymm8, ymm9 + vpxor ymm3, ymm3, ymm0 + vpxor ymm11, ymm11, ymm8 + vbroadcasti128 ymm4, xmmword ptr [ROT16+rip] + vpshufb ymm3, ymm3, ymm4 + vpshufb ymm11, ymm11, ymm4 + vpaddd ymm2, ymm2, ymm3 + vpaddd ymm10, ymm10, ymm11 + vpxor ymm1, ymm1, ymm2 + vpxor ymm9, ymm9, ymm10 + vpsrld ymm4, ymm1, 12 + vpslld ymm1, ymm1, 20 + vpor ymm1, ymm1, ymm4 + vpsrld ymm4, ymm9, 12 + vpslld ymm9, ymm9, 20 + vpor ymm9, ymm9, ymm4 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm0, ymm0, ymm1 + vpaddd ymm8, ymm8, ymm9 + vmovdqa ymmword ptr [rsp+0x80], ymm5 + vmovdqa ymmword ptr [rsp+0xA0], ymm13 + vpxor ymm3, ymm3, ymm0 + vpxor ymm11, ymm11, ymm8 + vbroadcasti128 ymm4, xmmword ptr [ROT8+rip] + vpshufb ymm3, ymm3, ymm4 + vpshufb ymm11, ymm11, ymm4 + vpaddd ymm2, ymm2, ymm3 + vpaddd ymm10, ymm10, ymm11 + vpxor ymm1, ymm1, ymm2 + vpxor ymm9, ymm9, ymm10 + vpsrld ymm4, ymm1, 7 + vpslld ymm1, ymm1, 25 + vpor ymm1, ymm1, ymm4 + vpsrld ymm4, ymm9, 7 + vpslld ymm9, ymm9, 25 + vpor ymm9, ymm9, ymm4 + vpshufd ymm0, ymm0, 0x93 + vpshufd ymm8, ymm8, 0x93 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm11, ymm11, 0x4E + vpshufd ymm2, ymm2, 0x39 + vpshufd ymm10, ymm10, 0x39 + vpaddd ymm0, ymm0, ymm6 + vpaddd ymm8, ymm8, ymm14 + vpaddd ymm0, ymm0, ymm1 + vpaddd ymm8, ymm8, ymm9 + vpxor ymm3, ymm3, ymm0 + vpxor ymm11, ymm11, ymm8 + vbroadcasti128 ymm4, xmmword ptr [ROT16+rip] + vpshufb ymm3, ymm3, ymm4 + vpshufb ymm11, ymm11, ymm4 + vpaddd ymm2, ymm2, ymm3 + vpaddd ymm10, ymm10, ymm11 + vpxor ymm1, ymm1, ymm2 + vpxor ymm9, ymm9, ymm10 + vpsrld ymm4, ymm1, 12 + vpslld ymm1, ymm1, 20 + vpor ymm1, ymm1, ymm4 + vpsrld ymm4, ymm9, 12 + vpslld ymm9, ymm9, 20 + vpor ymm9, ymm9, ymm4 + vpaddd ymm0, ymm0, ymm7 + vpaddd ymm8, ymm8, ymm15 + vpaddd ymm0, ymm0, ymm1 + vpaddd ymm8, ymm8, ymm9 + vpxor ymm3, ymm3, ymm0 + vpxor ymm11, ymm11, ymm8 + vbroadcasti128 ymm4, xmmword ptr [ROT8+rip] + vpshufb ymm3, ymm3, ymm4 + vpshufb ymm11, ymm11, ymm4 + vpaddd ymm2, ymm2, ymm3 + vpaddd ymm10, ymm10, ymm11 + vpxor ymm1, ymm1, ymm2 + vpxor ymm9, ymm9, ymm10 + vpsrld ymm4, ymm1, 7 + vpslld ymm1, ymm1, 25 + vpor ymm1, ymm1, ymm4 + vpsrld ymm4, ymm9, 7 + vpslld ymm9, ymm9, 25 + vpor ymm9, ymm9, ymm4 + vpshufd ymm0, ymm0, 0x39 + vpshufd ymm8, ymm8, 0x39 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm11, ymm11, 0x4E + vpshufd ymm2, ymm2, 0x93 + vpshufd ymm10, ymm10, 0x93 + dec al + je 9f + vmovdqa ymm4, ymmword ptr [rsp+0x40] + vmovdqa ymm5, ymmword ptr [rsp+0x80] + vshufps ymm12, ymm4, ymm5, 214 + vpshufd ymm13, ymm4, 0x0F + vpshufd ymm4, ymm12, 0x39 + vshufps ymm12, ymm6, ymm7, 250 + vpblendd ymm13, ymm13, ymm12, 0xAA + vpunpcklqdq ymm12, ymm7, ymm5 + vpblendd ymm12, ymm12, ymm6, 0x88 + vpshufd ymm12, ymm12, 0x78 + vpunpckhdq ymm5, ymm5, ymm7 + vpunpckldq ymm6, ymm6, ymm5 + vpshufd ymm7, ymm6, 0x1E + vmovdqa ymmword ptr [rsp+0x40], ymm13 + vmovdqa ymmword ptr [rsp+0x80], ymm12 + vmovdqa ymm12, ymmword ptr [rsp+0x60] + vmovdqa ymm13, ymmword ptr [rsp+0xA0] + vshufps ymm5, ymm12, ymm13, 214 + vpshufd ymm6, ymm12, 0x0F + vpshufd ymm12, ymm5, 0x39 + vshufps ymm5, ymm14, ymm15, 250 + vpblendd ymm6, ymm6, ymm5, 0xAA + vpunpcklqdq ymm5, ymm15, ymm13 + vpblendd ymm5, ymm5, ymm14, 0x88 + vpshufd ymm5, ymm5, 0x78 + vpunpckhdq ymm13, ymm13, ymm15 + vpunpckldq ymm14, ymm14, ymm13 + vpshufd ymm15, ymm14, 0x1E + vmovdqa ymm13, ymm6 + vmovdqa ymm14, ymm5 + vmovdqa ymm5, ymmword ptr [rsp+0x40] + vmovdqa ymm6, ymmword ptr [rsp+0x80] + jmp 9b +9: + vpxor ymm0, ymm0, ymm2 + vpxor ymm1, ymm1, ymm3 + vpxor ymm8, ymm8, ymm10 + vpxor ymm9, ymm9, ymm11 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01 + vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01 + vmovdqu xmmword ptr [rbx+0x40], xmm8 + vmovdqu xmmword ptr [rbx+0x50], xmm9 + vextracti128 xmmword ptr [rbx+0x60], ymm8, 0x01 + vextracti128 xmmword ptr [rbx+0x70], ymm9, 0x01 + vmovaps xmm8, xmmword ptr [rsp+0x280] + vmovaps xmm0, xmmword ptr [rsp+0x240] + vmovaps xmm1, xmmword ptr [rsp+0x250] + vmovaps xmm2, xmmword ptr [rsp+0x260] + vmovaps xmm3, xmmword ptr [rsp+0x270] + vblendvps xmm0, xmm0, xmm1, xmm8 + vblendvps xmm2, xmm2, xmm3, xmm8 + vmovaps xmmword ptr [rsp+0x240], xmm0 + vmovaps xmmword ptr [rsp+0x260], xmm2 + add rbx, 128 + add rdi, 32 + sub rsi, 4 +3: + test rsi, 0x2 + je 3f + vbroadcasti128 ymm0, xmmword ptr [rcx] + vbroadcasti128 ymm1, xmmword ptr [rcx+0x10] + vmovd xmm13, dword ptr [rsp+0x240] + vpinsrd xmm13, xmm13, dword ptr [rsp+0x260], 1 + vpinsrd xmm13, xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vmovd xmm14, dword ptr [rsp+0x244] + vpinsrd xmm14, xmm14, dword ptr [rsp+0x264], 1 + vpinsrd xmm14, xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vinserti128 ymm13, ymm13, xmm14, 0x01 + vbroadcasti128 ymm14, xmmword ptr [ROT16+rip] + vbroadcasti128 ymm15, xmmword ptr [ROT8+rip] + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + mov dword ptr [rsp+0x200], eax + vbroadcasti128 ymm2, xmmword ptr [BLAKE3_IV+rip] + vpbroadcastd ymm8, dword ptr [rsp+0x200] + vpblendd ymm3, ymm13, ymm8, 0x88 + vmovups ymm8, ymmword ptr [r8+rdx-0x40] + vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x40], 0x01 + vmovups ymm9, ymmword ptr [r8+rdx-0x30] + vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x30], 0x01 + vshufps ymm4, ymm8, ymm9, 136 + vshufps ymm5, ymm8, ymm9, 221 + vmovups ymm8, ymmword ptr [r8+rdx-0x20] + vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x20], 0x01 + vmovups ymm9, ymmword ptr [r8+rdx-0x10] + vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x10], 0x01 + vshufps ymm6, ymm8, ymm9, 136 + vshufps ymm7, ymm8, ymm9, 221 + vpshufd ymm6, ymm6, 0x93 + vpshufd ymm7, ymm7, 0x93 + mov al, 7 +9: + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm0, ymm0, ymm1 + vpxor ymm3, ymm3, ymm0 + vpshufb ymm3, ymm3, ymm14 + vpaddd ymm2, ymm2, ymm3 + vpxor ymm1, ymm1, ymm2 + vpsrld ymm8, ymm1, 12 + vpslld ymm1, ymm1, 20 + vpor ymm1, ymm1, ymm8 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm0, ymm0, ymm1 + vpxor ymm3, ymm3, ymm0 + vpshufb ymm3, ymm3, ymm15 + vpaddd ymm2, ymm2, ymm3 + vpxor ymm1, ymm1, ymm2 + vpsrld ymm8, ymm1, 7 + vpslld ymm1, ymm1, 25 + vpor ymm1, ymm1, ymm8 + vpshufd ymm0, ymm0, 0x93 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm2, ymm2, 0x39 + vpaddd ymm0, ymm0, ymm6 + vpaddd ymm0, ymm0, ymm1 + vpxor ymm3, ymm3, ymm0 + vpshufb ymm3, ymm3, ymm14 + vpaddd ymm2, ymm2, ymm3 + vpxor ymm1, ymm1, ymm2 + vpsrld ymm8, ymm1, 12 + vpslld ymm1, ymm1, 20 + vpor ymm1, ymm1, ymm8 + vpaddd ymm0, ymm0, ymm7 + vpaddd ymm0, ymm0, ymm1 + vpxor ymm3, ymm3, ymm0 + vpshufb ymm3, ymm3, ymm15 + vpaddd ymm2, ymm2, ymm3 + vpxor ymm1, ymm1, ymm2 + vpsrld ymm8, ymm1, 7 + vpslld ymm1, ymm1, 25 + vpor ymm1, ymm1, ymm8 + vpshufd ymm0, ymm0, 0x39 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm2, ymm2, 0x93 + dec al + jz 9f + vshufps ymm8, ymm4, ymm5, 214 + vpshufd ymm9, ymm4, 0x0F + vpshufd ymm4, ymm8, 0x39 + vshufps ymm8, ymm6, ymm7, 250 + vpblendd ymm9, ymm9, ymm8, 0xAA + vpunpcklqdq ymm8, ymm7, ymm5 + vpblendd ymm8, ymm8, ymm6, 0x88 + vpshufd ymm8, ymm8, 0x78 + vpunpckhdq ymm5, ymm5, ymm7 + vpunpckldq ymm6, ymm6, ymm5 + vpshufd ymm7, ymm6, 0x1E + vmovdqa ymm5, ymm9 + vmovdqa ymm6, ymm8 + jmp 9b +9: + vpxor ymm0, ymm0, ymm2 + vpxor ymm1, ymm1, ymm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01 + vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01 + vmovaps ymm8, ymmword ptr [rsp+0x280] + vmovaps ymm0, ymmword ptr [rsp+0x240] + vmovups ymm1, ymmword ptr [rsp+0x248] + vmovaps ymm2, ymmword ptr [rsp+0x260] + vmovups ymm3, ymmword ptr [rsp+0x268] + vblendvps ymm0, ymm0, ymm1, ymm8 + vblendvps ymm2, ymm2, ymm3, ymm8 + vmovaps ymmword ptr [rsp+0x240], ymm0 + vmovaps ymmword ptr [rsp+0x260], ymm2 + add rbx, 64 + add rdi, 16 + sub rsi, 2 +3: + test rsi, 0x1 + je 4b + vmovdqu xmm0, xmmword ptr [rcx] + vmovdqu xmm1, xmmword ptr [rcx+0x10] + vmovd xmm3, dword ptr [rsp+0x240] + vpinsrd xmm3, xmm3, dword ptr [rsp+0x260], 1 + vpinsrd xmm13, xmm3, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vmovdqa xmm14, xmmword ptr [ROT16+rip] + vmovdqa xmm15, xmmword ptr [ROT8+rip] + mov r8, qword ptr [rdi] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + vmovdqa xmm2, xmmword ptr [BLAKE3_IV+rip] + vmovdqa xmm3, xmm13 + vpinsrd xmm3, xmm3, eax, 3 + vmovups xmm8, xmmword ptr [r8+rdx-0x40] + vmovups xmm9, xmmword ptr [r8+rdx-0x30] + vshufps xmm4, xmm8, xmm9, 136 + vshufps xmm5, xmm8, xmm9, 221 + vmovups xmm8, xmmword ptr [r8+rdx-0x20] + vmovups xmm9, xmmword ptr [r8+rdx-0x10] + vshufps xmm6, xmm8, xmm9, 136 + vshufps xmm7, xmm8, xmm9, 221 + vpshufd xmm6, xmm6, 0x93 + vpshufd xmm7, xmm7, 0x93 + mov al, 7 +9: + vpaddd xmm0, xmm0, xmm4 + vpaddd xmm0, xmm0, xmm1 + vpxor xmm3, xmm3, xmm0 + vpshufb xmm3, xmm3, xmm14 + vpaddd xmm2, xmm2, xmm3 + vpxor xmm1, xmm1, xmm2 + vpsrld xmm8, xmm1, 12 + vpslld xmm1, xmm1, 20 + vpor xmm1, xmm1, xmm8 + vpaddd xmm0, xmm0, xmm5 + vpaddd xmm0, xmm0, xmm1 + vpxor xmm3, xmm3, xmm0 + vpshufb xmm3, xmm3, xmm15 + vpaddd xmm2, xmm2, xmm3 + vpxor xmm1, xmm1, xmm2 + vpsrld xmm8, xmm1, 7 + vpslld xmm1, xmm1, 25 + vpor xmm1, xmm1, xmm8 + vpshufd xmm0, xmm0, 0x93 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x39 + vpaddd xmm0, xmm0, xmm6 + vpaddd xmm0, xmm0, xmm1 + vpxor xmm3, xmm3, xmm0 + vpshufb xmm3, xmm3, xmm14 + vpaddd xmm2, xmm2, xmm3 + vpxor xmm1, xmm1, xmm2 + vpsrld xmm8, xmm1, 12 + vpslld xmm1, xmm1, 20 + vpor xmm1, xmm1, xmm8 + vpaddd xmm0, xmm0, xmm7 + vpaddd xmm0, xmm0, xmm1 + vpxor xmm3, xmm3, xmm0 + vpshufb xmm3, xmm3, xmm15 + vpaddd xmm2, xmm2, xmm3 + vpxor xmm1, xmm1, xmm2 + vpsrld xmm8, xmm1, 7 + vpslld xmm1, xmm1, 25 + vpor xmm1, xmm1, xmm8 + vpshufd xmm0, xmm0, 0x39 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x93 + dec al + jz 9f + vshufps xmm8, xmm4, xmm5, 214 + vpshufd xmm9, xmm4, 0x0F + vpshufd xmm4, xmm8, 0x39 + vshufps xmm8, xmm6, xmm7, 250 + vpblendd xmm9, xmm9, xmm8, 0xAA + vpunpcklqdq xmm8, xmm7, xmm5 + vpblendd xmm8, xmm8, xmm6, 0x88 + vpshufd xmm8, xmm8, 0x78 + vpunpckhdq xmm5, xmm5, xmm7 + vpunpckldq xmm6, xmm6, xmm5 + vpshufd xmm7, xmm6, 0x1E + vmovdqa xmm5, xmm9 + vmovdqa xmm6, xmm8 + jmp 9b +9: + vpxor xmm0, xmm0, xmm2 + vpxor xmm1, xmm1, xmm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + jmp 4b + + +#ifdef __APPLE__ +.static_data +#else +.section .rodata +#endif +.p2align 6 +ADD0: + .long 0, 1, 2, 3, 4, 5, 6, 7 +ADD1: + .long 8, 8, 8, 8, 8, 8, 8, 8 +BLAKE3_IV_0: + .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667 + .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667 +BLAKE3_IV_1: + .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85 + .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85 +BLAKE3_IV_2: + .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372 + .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372 +BLAKE3_IV_3: + .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A + .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A +BLAKE3_BLOCK_LEN: + .long 0x00000040, 0x00000040, 0x00000040, 0x00000040 + .long 0x00000040, 0x00000040, 0x00000040, 0x00000040 +ROT16: + .byte 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 +ROT8: + .byte 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12 +CMP_MSB_MASK: + .long 0x80000000, 0x80000000, 0x80000000, 0x80000000 + .long 0x80000000, 0x80000000, 0x80000000, 0x80000000 +BLAKE3_IV: + .long 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A + diff --git a/src/blake3/blake3_avx512.c b/src/blake3/blake3_avx512.c new file mode 100644 index 0000000..0bdc457 --- /dev/null +++ b/src/blake3/blake3_avx512.c @@ -0,0 +1,1227 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" + +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +#define _mm_shuffle_ps2(a, b, c) \ + (_mm_castps_si128( \ + _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) + +INLINE __m128i loadu_128(const uint8_t src[16]) { + return _mm_loadu_si128((const __m128i *)src); +} + +INLINE __m256i loadu_256(const uint8_t src[32]) { + return _mm256_loadu_si256((const __m256i *)src); +} + +INLINE __m512i loadu_512(const uint8_t src[64]) { + return _mm512_loadu_si512((const __m512i *)src); +} + +INLINE void storeu_128(__m128i src, uint8_t dest[16]) { + _mm_storeu_si128((__m128i *)dest, src); +} + +INLINE void storeu_256(__m256i src, uint8_t dest[16]) { + _mm256_storeu_si256((__m256i *)dest, src); +} + +INLINE __m128i add_128(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } + +INLINE __m256i add_256(__m256i a, __m256i b) { return _mm256_add_epi32(a, b); } + +INLINE __m512i add_512(__m512i a, __m512i b) { return _mm512_add_epi32(a, b); } + +INLINE __m128i xor_128(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } + +INLINE __m256i xor_256(__m256i a, __m256i b) { return _mm256_xor_si256(a, b); } + +INLINE __m512i xor_512(__m512i a, __m512i b) { return _mm512_xor_si512(a, b); } + +INLINE __m128i set1_128(uint32_t x) { return _mm_set1_epi32((int32_t)x); } + +INLINE __m256i set1_256(uint32_t x) { return _mm256_set1_epi32((int32_t)x); } + +INLINE __m512i set1_512(uint32_t x) { return _mm512_set1_epi32((int32_t)x); } + +INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); +} + +INLINE __m128i rot16_128(__m128i x) { return _mm_ror_epi32(x, 16); } + +INLINE __m256i rot16_256(__m256i x) { return _mm256_ror_epi32(x, 16); } + +INLINE __m512i rot16_512(__m512i x) { return _mm512_ror_epi32(x, 16); } + +INLINE __m128i rot12_128(__m128i x) { return _mm_ror_epi32(x, 12); } + +INLINE __m256i rot12_256(__m256i x) { return _mm256_ror_epi32(x, 12); } + +INLINE __m512i rot12_512(__m512i x) { return _mm512_ror_epi32(x, 12); } + +INLINE __m128i rot8_128(__m128i x) { return _mm_ror_epi32(x, 8); } + +INLINE __m256i rot8_256(__m256i x) { return _mm256_ror_epi32(x, 8); } + +INLINE __m512i rot8_512(__m512i x) { return _mm512_ror_epi32(x, 8); } + +INLINE __m128i rot7_128(__m128i x) { return _mm_ror_epi32(x, 7); } + +INLINE __m256i rot7_256(__m256i x) { return _mm256_ror_epi32(x, 7); } + +INLINE __m512i rot7_512(__m512i x) { return _mm512_ror_epi32(x, 7); } + +/* + * ---------------------------------------------------------------------------- + * compress_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = add_128(add_128(*row0, m), *row1); + *row3 = xor_128(*row3, *row0); + *row3 = rot16_128(*row3); + *row2 = add_128(*row2, *row3); + *row1 = xor_128(*row1, *row2); + *row1 = rot12_128(*row1); +} + +INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = add_128(add_128(*row0, m), *row1); + *row3 = xor_128(*row3, *row0); + *row3 = rot8_128(*row3); + *row2 = add_128(*row2, *row3); + *row1 = xor_128(*row1, *row2); + *row1 = rot7_128(*row1); +} + +// Note the optimization here of leaving row1 as the unrotated row, rather than +// row0. All the message loads below are adjusted to compensate for this. See +// discussion at https://github.com/sneves/blake2-avx2/pull/4 +INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); +} + +INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); +} + +INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + rows[0] = loadu_128((uint8_t *)&cv[0]); + rows[1] = loadu_128((uint8_t *)&cv[4]); + rows[2] = set4(B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3); + rows[3] = set4(counter_low(counter), counter_high(counter), + (uint32_t)block_len, (uint32_t)flags); + + __m128i m0 = loadu_128(&block[sizeof(__m128i) * 0]); + __m128i m1 = loadu_128(&block[sizeof(__m128i) * 1]); + __m128i m2 = loadu_128(&block[sizeof(__m128i) * 2]); + __m128i m3 = loadu_128(&block[sizeof(__m128i) * 3]); + + __m128i t0, t1, t2, t3, tt; + + // Round 1. The first round permutes the message words from the original + // input order, into the groups that get mixed in parallel. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 + t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 + t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 2. This round and all following rounds apply a fixed permutation + // to the message words from the round before. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 3 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 4 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 5 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 6 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 7 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); +} + +void blake3_compress_xof_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu_128(xor_128(rows[0], rows[2]), &out[0]); + storeu_128(xor_128(rows[1], rows[3]), &out[16]); + storeu_128(xor_128(rows[2], loadu_128((uint8_t *)&cv[0])), &out[32]); + storeu_128(xor_128(rows[3], loadu_128((uint8_t *)&cv[4])), &out[48]); +} + +void blake3_compress_in_place_avx512(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu_128(xor_128(rows[0], rows[2]), (uint8_t *)&cv[0]); + storeu_128(xor_128(rows[1], rows[3]), (uint8_t *)&cv[4]); +} + +/* + * ---------------------------------------------------------------------------- + * hash4_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn4(__m128i v[16], __m128i m[16], size_t r) { + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[15] = rot16_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot12_128(v[4]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[15] = rot8_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot7_128(v[4]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot16_128(v[15]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[4] = rot12_128(v[4]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot8_128(v[15]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + v[4] = rot7_128(v[4]); +} + +INLINE void transpose_vecs_128(__m128i vecs[4]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is + // 22/33. Note that this doesn't split the vector into two lanes, as the + // AVX2 counterparts do. + __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); + __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); + __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); + __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); + + // Interleave 64-bit lanes. + __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); + __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); + __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); + __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); + + vecs[0] = abcd_0; + vecs[1] = abcd_1; + vecs[2] = abcd_2; + vecs[3] = abcd_3; +} + +INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, + size_t block_offset, __m128i out[16]) { + out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(__m128i)]); + out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(__m128i)]); + out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(__m128i)]); + out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(__m128i)]); + out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(__m128i)]); + out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(__m128i)]); + out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(__m128i)]); + out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(__m128i)]); + out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(__m128i)]); + out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(__m128i)]); + out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(__m128i)]); + out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(__m128i)]); + out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(__m128i)]); + out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(__m128i)]); + out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(__m128i)]); + out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(__m128i)]); + for (size_t i = 0; i < 4; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs_128(&out[0]); + transpose_vecs_128(&out[4]); + transpose_vecs_128(&out[8]); + transpose_vecs_128(&out[12]); +} + +INLINE void load_counters4(uint64_t counter, bool increment_counter, + __m128i *out_lo, __m128i *out_hi) { + uint64_t mask = (increment_counter ? ~0 : 0); + __m256i mask_vec = _mm256_set1_epi64x(mask); + __m256i deltas = _mm256_setr_epi64x(0, 1, 2, 3); + deltas = _mm256_and_si256(mask_vec, deltas); + __m256i counters = + _mm256_add_epi64(_mm256_set1_epi64x((int64_t)counter), deltas); + *out_lo = _mm256_cvtepi64_epi32(counters); + *out_hi = _mm256_cvtepi64_epi32(_mm256_srli_epi64(counters, 32)); +} + +static +void blake3_hash4_avx512(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m128i h_vecs[8] = { + set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), + set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), + }; + __m128i counter_low_vec, counter_high_vec; + load_counters4(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m128i block_len_vec = set1_128(BLAKE3_BLOCK_LEN); + __m128i block_flags_vec = set1_128(block_flags); + __m128i msg_vecs[16]; + transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m128i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_128(B3_IV_0), set1_128(B3_IV_1), set1_128(B3_IV_2), set1_128(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn4(v, msg_vecs, 0); + round_fn4(v, msg_vecs, 1); + round_fn4(v, msg_vecs, 2); + round_fn4(v, msg_vecs, 3); + round_fn4(v, msg_vecs, 4); + round_fn4(v, msg_vecs, 5); + round_fn4(v, msg_vecs, 6); + h_vecs[0] = xor_128(v[0], v[8]); + h_vecs[1] = xor_128(v[1], v[9]); + h_vecs[2] = xor_128(v[2], v[10]); + h_vecs[3] = xor_128(v[3], v[11]); + h_vecs[4] = xor_128(v[4], v[12]); + h_vecs[5] = xor_128(v[5], v[13]); + h_vecs[6] = xor_128(v[6], v[14]); + h_vecs[7] = xor_128(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs_128(&h_vecs[0]); + transpose_vecs_128(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu_128(h_vecs[0], &out[0 * sizeof(__m128i)]); + storeu_128(h_vecs[4], &out[1 * sizeof(__m128i)]); + storeu_128(h_vecs[1], &out[2 * sizeof(__m128i)]); + storeu_128(h_vecs[5], &out[3 * sizeof(__m128i)]); + storeu_128(h_vecs[2], &out[4 * sizeof(__m128i)]); + storeu_128(h_vecs[6], &out[5 * sizeof(__m128i)]); + storeu_128(h_vecs[3], &out[6 * sizeof(__m128i)]); + storeu_128(h_vecs[7], &out[7 * sizeof(__m128i)]); +} + +/* + * ---------------------------------------------------------------------------- + * hash8_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn8(__m256i v[16], __m256i m[16], size_t r) { + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_256(v[0], v[4]); + v[1] = add_256(v[1], v[5]); + v[2] = add_256(v[2], v[6]); + v[3] = add_256(v[3], v[7]); + v[12] = xor_256(v[12], v[0]); + v[13] = xor_256(v[13], v[1]); + v[14] = xor_256(v[14], v[2]); + v[15] = xor_256(v[15], v[3]); + v[12] = rot16_256(v[12]); + v[13] = rot16_256(v[13]); + v[14] = rot16_256(v[14]); + v[15] = rot16_256(v[15]); + v[8] = add_256(v[8], v[12]); + v[9] = add_256(v[9], v[13]); + v[10] = add_256(v[10], v[14]); + v[11] = add_256(v[11], v[15]); + v[4] = xor_256(v[4], v[8]); + v[5] = xor_256(v[5], v[9]); + v[6] = xor_256(v[6], v[10]); + v[7] = xor_256(v[7], v[11]); + v[4] = rot12_256(v[4]); + v[5] = rot12_256(v[5]); + v[6] = rot12_256(v[6]); + v[7] = rot12_256(v[7]); + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_256(v[0], v[4]); + v[1] = add_256(v[1], v[5]); + v[2] = add_256(v[2], v[6]); + v[3] = add_256(v[3], v[7]); + v[12] = xor_256(v[12], v[0]); + v[13] = xor_256(v[13], v[1]); + v[14] = xor_256(v[14], v[2]); + v[15] = xor_256(v[15], v[3]); + v[12] = rot8_256(v[12]); + v[13] = rot8_256(v[13]); + v[14] = rot8_256(v[14]); + v[15] = rot8_256(v[15]); + v[8] = add_256(v[8], v[12]); + v[9] = add_256(v[9], v[13]); + v[10] = add_256(v[10], v[14]); + v[11] = add_256(v[11], v[15]); + v[4] = xor_256(v[4], v[8]); + v[5] = xor_256(v[5], v[9]); + v[6] = xor_256(v[6], v[10]); + v[7] = xor_256(v[7], v[11]); + v[4] = rot7_256(v[4]); + v[5] = rot7_256(v[5]); + v[6] = rot7_256(v[6]); + v[7] = rot7_256(v[7]); + + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_256(v[0], v[5]); + v[1] = add_256(v[1], v[6]); + v[2] = add_256(v[2], v[7]); + v[3] = add_256(v[3], v[4]); + v[15] = xor_256(v[15], v[0]); + v[12] = xor_256(v[12], v[1]); + v[13] = xor_256(v[13], v[2]); + v[14] = xor_256(v[14], v[3]); + v[15] = rot16_256(v[15]); + v[12] = rot16_256(v[12]); + v[13] = rot16_256(v[13]); + v[14] = rot16_256(v[14]); + v[10] = add_256(v[10], v[15]); + v[11] = add_256(v[11], v[12]); + v[8] = add_256(v[8], v[13]); + v[9] = add_256(v[9], v[14]); + v[5] = xor_256(v[5], v[10]); + v[6] = xor_256(v[6], v[11]); + v[7] = xor_256(v[7], v[8]); + v[4] = xor_256(v[4], v[9]); + v[5] = rot12_256(v[5]); + v[6] = rot12_256(v[6]); + v[7] = rot12_256(v[7]); + v[4] = rot12_256(v[4]); + v[0] = add_256(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_256(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_256(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_256(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_256(v[0], v[5]); + v[1] = add_256(v[1], v[6]); + v[2] = add_256(v[2], v[7]); + v[3] = add_256(v[3], v[4]); + v[15] = xor_256(v[15], v[0]); + v[12] = xor_256(v[12], v[1]); + v[13] = xor_256(v[13], v[2]); + v[14] = xor_256(v[14], v[3]); + v[15] = rot8_256(v[15]); + v[12] = rot8_256(v[12]); + v[13] = rot8_256(v[13]); + v[14] = rot8_256(v[14]); + v[10] = add_256(v[10], v[15]); + v[11] = add_256(v[11], v[12]); + v[8] = add_256(v[8], v[13]); + v[9] = add_256(v[9], v[14]); + v[5] = xor_256(v[5], v[10]); + v[6] = xor_256(v[6], v[11]); + v[7] = xor_256(v[7], v[8]); + v[4] = xor_256(v[4], v[9]); + v[5] = rot7_256(v[5]); + v[6] = rot7_256(v[6]); + v[7] = rot7_256(v[7]); + v[4] = rot7_256(v[4]); +} + +INLINE void transpose_vecs_256(__m256i vecs[8]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11/44/55, and the high + // is 22/33/66/77. + __m256i ab_0145 = _mm256_unpacklo_epi32(vecs[0], vecs[1]); + __m256i ab_2367 = _mm256_unpackhi_epi32(vecs[0], vecs[1]); + __m256i cd_0145 = _mm256_unpacklo_epi32(vecs[2], vecs[3]); + __m256i cd_2367 = _mm256_unpackhi_epi32(vecs[2], vecs[3]); + __m256i ef_0145 = _mm256_unpacklo_epi32(vecs[4], vecs[5]); + __m256i ef_2367 = _mm256_unpackhi_epi32(vecs[4], vecs[5]); + __m256i gh_0145 = _mm256_unpacklo_epi32(vecs[6], vecs[7]); + __m256i gh_2367 = _mm256_unpackhi_epi32(vecs[6], vecs[7]); + + // Interleave 64-bit lanes. The low unpack is lanes 00/22 and the high is + // 11/33. + __m256i abcd_04 = _mm256_unpacklo_epi64(ab_0145, cd_0145); + __m256i abcd_15 = _mm256_unpackhi_epi64(ab_0145, cd_0145); + __m256i abcd_26 = _mm256_unpacklo_epi64(ab_2367, cd_2367); + __m256i abcd_37 = _mm256_unpackhi_epi64(ab_2367, cd_2367); + __m256i efgh_04 = _mm256_unpacklo_epi64(ef_0145, gh_0145); + __m256i efgh_15 = _mm256_unpackhi_epi64(ef_0145, gh_0145); + __m256i efgh_26 = _mm256_unpacklo_epi64(ef_2367, gh_2367); + __m256i efgh_37 = _mm256_unpackhi_epi64(ef_2367, gh_2367); + + // Interleave 128-bit lanes. + vecs[0] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x20); + vecs[1] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x20); + vecs[2] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x20); + vecs[3] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x20); + vecs[4] = _mm256_permute2x128_si256(abcd_04, efgh_04, 0x31); + vecs[5] = _mm256_permute2x128_si256(abcd_15, efgh_15, 0x31); + vecs[6] = _mm256_permute2x128_si256(abcd_26, efgh_26, 0x31); + vecs[7] = _mm256_permute2x128_si256(abcd_37, efgh_37, 0x31); +} + +INLINE void transpose_msg_vecs8(const uint8_t *const *inputs, + size_t block_offset, __m256i out[16]) { + out[0] = loadu_256(&inputs[0][block_offset + 0 * sizeof(__m256i)]); + out[1] = loadu_256(&inputs[1][block_offset + 0 * sizeof(__m256i)]); + out[2] = loadu_256(&inputs[2][block_offset + 0 * sizeof(__m256i)]); + out[3] = loadu_256(&inputs[3][block_offset + 0 * sizeof(__m256i)]); + out[4] = loadu_256(&inputs[4][block_offset + 0 * sizeof(__m256i)]); + out[5] = loadu_256(&inputs[5][block_offset + 0 * sizeof(__m256i)]); + out[6] = loadu_256(&inputs[6][block_offset + 0 * sizeof(__m256i)]); + out[7] = loadu_256(&inputs[7][block_offset + 0 * sizeof(__m256i)]); + out[8] = loadu_256(&inputs[0][block_offset + 1 * sizeof(__m256i)]); + out[9] = loadu_256(&inputs[1][block_offset + 1 * sizeof(__m256i)]); + out[10] = loadu_256(&inputs[2][block_offset + 1 * sizeof(__m256i)]); + out[11] = loadu_256(&inputs[3][block_offset + 1 * sizeof(__m256i)]); + out[12] = loadu_256(&inputs[4][block_offset + 1 * sizeof(__m256i)]); + out[13] = loadu_256(&inputs[5][block_offset + 1 * sizeof(__m256i)]); + out[14] = loadu_256(&inputs[6][block_offset + 1 * sizeof(__m256i)]); + out[15] = loadu_256(&inputs[7][block_offset + 1 * sizeof(__m256i)]); + for (size_t i = 0; i < 8; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs_256(&out[0]); + transpose_vecs_256(&out[8]); +} + +INLINE void load_counters8(uint64_t counter, bool increment_counter, + __m256i *out_lo, __m256i *out_hi) { + uint64_t mask = (increment_counter ? ~0 : 0); + __m512i mask_vec = _mm512_set1_epi64(mask); + __m512i deltas = _mm512_setr_epi64(0, 1, 2, 3, 4, 5, 6, 7); + deltas = _mm512_and_si512(mask_vec, deltas); + __m512i counters = + _mm512_add_epi64(_mm512_set1_epi64((int64_t)counter), deltas); + *out_lo = _mm512_cvtepi64_epi32(counters); + *out_hi = _mm512_cvtepi64_epi32(_mm512_srli_epi64(counters, 32)); +} + +static +void blake3_hash8_avx512(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m256i h_vecs[8] = { + set1_256(key[0]), set1_256(key[1]), set1_256(key[2]), set1_256(key[3]), + set1_256(key[4]), set1_256(key[5]), set1_256(key[6]), set1_256(key[7]), + }; + __m256i counter_low_vec, counter_high_vec; + load_counters8(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m256i block_len_vec = set1_256(BLAKE3_BLOCK_LEN); + __m256i block_flags_vec = set1_256(block_flags); + __m256i msg_vecs[16]; + transpose_msg_vecs8(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m256i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_256(B3_IV_0), set1_256(B3_IV_1), set1_256(B3_IV_2), set1_256(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn8(v, msg_vecs, 0); + round_fn8(v, msg_vecs, 1); + round_fn8(v, msg_vecs, 2); + round_fn8(v, msg_vecs, 3); + round_fn8(v, msg_vecs, 4); + round_fn8(v, msg_vecs, 5); + round_fn8(v, msg_vecs, 6); + h_vecs[0] = xor_256(v[0], v[8]); + h_vecs[1] = xor_256(v[1], v[9]); + h_vecs[2] = xor_256(v[2], v[10]); + h_vecs[3] = xor_256(v[3], v[11]); + h_vecs[4] = xor_256(v[4], v[12]); + h_vecs[5] = xor_256(v[5], v[13]); + h_vecs[6] = xor_256(v[6], v[14]); + h_vecs[7] = xor_256(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs_256(h_vecs); + storeu_256(h_vecs[0], &out[0 * sizeof(__m256i)]); + storeu_256(h_vecs[1], &out[1 * sizeof(__m256i)]); + storeu_256(h_vecs[2], &out[2 * sizeof(__m256i)]); + storeu_256(h_vecs[3], &out[3 * sizeof(__m256i)]); + storeu_256(h_vecs[4], &out[4 * sizeof(__m256i)]); + storeu_256(h_vecs[5], &out[5 * sizeof(__m256i)]); + storeu_256(h_vecs[6], &out[6 * sizeof(__m256i)]); + storeu_256(h_vecs[7], &out[7 * sizeof(__m256i)]); +} + +/* + * ---------------------------------------------------------------------------- + * hash16_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn16(__m512i v[16], __m512i m[16], size_t r) { + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_512(v[0], v[4]); + v[1] = add_512(v[1], v[5]); + v[2] = add_512(v[2], v[6]); + v[3] = add_512(v[3], v[7]); + v[12] = xor_512(v[12], v[0]); + v[13] = xor_512(v[13], v[1]); + v[14] = xor_512(v[14], v[2]); + v[15] = xor_512(v[15], v[3]); + v[12] = rot16_512(v[12]); + v[13] = rot16_512(v[13]); + v[14] = rot16_512(v[14]); + v[15] = rot16_512(v[15]); + v[8] = add_512(v[8], v[12]); + v[9] = add_512(v[9], v[13]); + v[10] = add_512(v[10], v[14]); + v[11] = add_512(v[11], v[15]); + v[4] = xor_512(v[4], v[8]); + v[5] = xor_512(v[5], v[9]); + v[6] = xor_512(v[6], v[10]); + v[7] = xor_512(v[7], v[11]); + v[4] = rot12_512(v[4]); + v[5] = rot12_512(v[5]); + v[6] = rot12_512(v[6]); + v[7] = rot12_512(v[7]); + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_512(v[0], v[4]); + v[1] = add_512(v[1], v[5]); + v[2] = add_512(v[2], v[6]); + v[3] = add_512(v[3], v[7]); + v[12] = xor_512(v[12], v[0]); + v[13] = xor_512(v[13], v[1]); + v[14] = xor_512(v[14], v[2]); + v[15] = xor_512(v[15], v[3]); + v[12] = rot8_512(v[12]); + v[13] = rot8_512(v[13]); + v[14] = rot8_512(v[14]); + v[15] = rot8_512(v[15]); + v[8] = add_512(v[8], v[12]); + v[9] = add_512(v[9], v[13]); + v[10] = add_512(v[10], v[14]); + v[11] = add_512(v[11], v[15]); + v[4] = xor_512(v[4], v[8]); + v[5] = xor_512(v[5], v[9]); + v[6] = xor_512(v[6], v[10]); + v[7] = xor_512(v[7], v[11]); + v[4] = rot7_512(v[4]); + v[5] = rot7_512(v[5]); + v[6] = rot7_512(v[6]); + v[7] = rot7_512(v[7]); + + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_512(v[0], v[5]); + v[1] = add_512(v[1], v[6]); + v[2] = add_512(v[2], v[7]); + v[3] = add_512(v[3], v[4]); + v[15] = xor_512(v[15], v[0]); + v[12] = xor_512(v[12], v[1]); + v[13] = xor_512(v[13], v[2]); + v[14] = xor_512(v[14], v[3]); + v[15] = rot16_512(v[15]); + v[12] = rot16_512(v[12]); + v[13] = rot16_512(v[13]); + v[14] = rot16_512(v[14]); + v[10] = add_512(v[10], v[15]); + v[11] = add_512(v[11], v[12]); + v[8] = add_512(v[8], v[13]); + v[9] = add_512(v[9], v[14]); + v[5] = xor_512(v[5], v[10]); + v[6] = xor_512(v[6], v[11]); + v[7] = xor_512(v[7], v[8]); + v[4] = xor_512(v[4], v[9]); + v[5] = rot12_512(v[5]); + v[6] = rot12_512(v[6]); + v[7] = rot12_512(v[7]); + v[4] = rot12_512(v[4]); + v[0] = add_512(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_512(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_512(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_512(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_512(v[0], v[5]); + v[1] = add_512(v[1], v[6]); + v[2] = add_512(v[2], v[7]); + v[3] = add_512(v[3], v[4]); + v[15] = xor_512(v[15], v[0]); + v[12] = xor_512(v[12], v[1]); + v[13] = xor_512(v[13], v[2]); + v[14] = xor_512(v[14], v[3]); + v[15] = rot8_512(v[15]); + v[12] = rot8_512(v[12]); + v[13] = rot8_512(v[13]); + v[14] = rot8_512(v[14]); + v[10] = add_512(v[10], v[15]); + v[11] = add_512(v[11], v[12]); + v[8] = add_512(v[8], v[13]); + v[9] = add_512(v[9], v[14]); + v[5] = xor_512(v[5], v[10]); + v[6] = xor_512(v[6], v[11]); + v[7] = xor_512(v[7], v[8]); + v[4] = xor_512(v[4], v[9]); + v[5] = rot7_512(v[5]); + v[6] = rot7_512(v[6]); + v[7] = rot7_512(v[7]); + v[4] = rot7_512(v[4]); +} + +// 0b10001000, or lanes a0/a2/b0/b2 in little-endian order +#define LO_IMM8 0x88 + +INLINE __m512i unpack_lo_128(__m512i a, __m512i b) { + return _mm512_shuffle_i32x4(a, b, LO_IMM8); +} + +// 0b11011101, or lanes a1/a3/b1/b3 in little-endian order +#define HI_IMM8 0xdd + +INLINE __m512i unpack_hi_128(__m512i a, __m512i b) { + return _mm512_shuffle_i32x4(a, b, HI_IMM8); +} + +INLINE void transpose_vecs_512(__m512i vecs[16]) { + // Interleave 32-bit lanes. The _0 unpack is lanes + // 0/0/1/1/4/4/5/5/8/8/9/9/12/12/13/13, and the _2 unpack is lanes + // 2/2/3/3/6/6/7/7/10/10/11/11/14/14/15/15. + __m512i ab_0 = _mm512_unpacklo_epi32(vecs[0], vecs[1]); + __m512i ab_2 = _mm512_unpackhi_epi32(vecs[0], vecs[1]); + __m512i cd_0 = _mm512_unpacklo_epi32(vecs[2], vecs[3]); + __m512i cd_2 = _mm512_unpackhi_epi32(vecs[2], vecs[3]); + __m512i ef_0 = _mm512_unpacklo_epi32(vecs[4], vecs[5]); + __m512i ef_2 = _mm512_unpackhi_epi32(vecs[4], vecs[5]); + __m512i gh_0 = _mm512_unpacklo_epi32(vecs[6], vecs[7]); + __m512i gh_2 = _mm512_unpackhi_epi32(vecs[6], vecs[7]); + __m512i ij_0 = _mm512_unpacklo_epi32(vecs[8], vecs[9]); + __m512i ij_2 = _mm512_unpackhi_epi32(vecs[8], vecs[9]); + __m512i kl_0 = _mm512_unpacklo_epi32(vecs[10], vecs[11]); + __m512i kl_2 = _mm512_unpackhi_epi32(vecs[10], vecs[11]); + __m512i mn_0 = _mm512_unpacklo_epi32(vecs[12], vecs[13]); + __m512i mn_2 = _mm512_unpackhi_epi32(vecs[12], vecs[13]); + __m512i op_0 = _mm512_unpacklo_epi32(vecs[14], vecs[15]); + __m512i op_2 = _mm512_unpackhi_epi32(vecs[14], vecs[15]); + + // Interleave 64-bit lanes. The _0 unpack is lanes + // 0/0/0/0/4/4/4/4/8/8/8/8/12/12/12/12, the _1 unpack is lanes + // 1/1/1/1/5/5/5/5/9/9/9/9/13/13/13/13, the _2 unpack is lanes + // 2/2/2/2/6/6/6/6/10/10/10/10/14/14/14/14, and the _3 unpack is lanes + // 3/3/3/3/7/7/7/7/11/11/11/11/15/15/15/15. + __m512i abcd_0 = _mm512_unpacklo_epi64(ab_0, cd_0); + __m512i abcd_1 = _mm512_unpackhi_epi64(ab_0, cd_0); + __m512i abcd_2 = _mm512_unpacklo_epi64(ab_2, cd_2); + __m512i abcd_3 = _mm512_unpackhi_epi64(ab_2, cd_2); + __m512i efgh_0 = _mm512_unpacklo_epi64(ef_0, gh_0); + __m512i efgh_1 = _mm512_unpackhi_epi64(ef_0, gh_0); + __m512i efgh_2 = _mm512_unpacklo_epi64(ef_2, gh_2); + __m512i efgh_3 = _mm512_unpackhi_epi64(ef_2, gh_2); + __m512i ijkl_0 = _mm512_unpacklo_epi64(ij_0, kl_0); + __m512i ijkl_1 = _mm512_unpackhi_epi64(ij_0, kl_0); + __m512i ijkl_2 = _mm512_unpacklo_epi64(ij_2, kl_2); + __m512i ijkl_3 = _mm512_unpackhi_epi64(ij_2, kl_2); + __m512i mnop_0 = _mm512_unpacklo_epi64(mn_0, op_0); + __m512i mnop_1 = _mm512_unpackhi_epi64(mn_0, op_0); + __m512i mnop_2 = _mm512_unpacklo_epi64(mn_2, op_2); + __m512i mnop_3 = _mm512_unpackhi_epi64(mn_2, op_2); + + // Interleave 128-bit lanes. The _0 unpack is + // 0/0/0/0/8/8/8/8/0/0/0/0/8/8/8/8, the _1 unpack is + // 1/1/1/1/9/9/9/9/1/1/1/1/9/9/9/9, and so on. + __m512i abcdefgh_0 = unpack_lo_128(abcd_0, efgh_0); + __m512i abcdefgh_1 = unpack_lo_128(abcd_1, efgh_1); + __m512i abcdefgh_2 = unpack_lo_128(abcd_2, efgh_2); + __m512i abcdefgh_3 = unpack_lo_128(abcd_3, efgh_3); + __m512i abcdefgh_4 = unpack_hi_128(abcd_0, efgh_0); + __m512i abcdefgh_5 = unpack_hi_128(abcd_1, efgh_1); + __m512i abcdefgh_6 = unpack_hi_128(abcd_2, efgh_2); + __m512i abcdefgh_7 = unpack_hi_128(abcd_3, efgh_3); + __m512i ijklmnop_0 = unpack_lo_128(ijkl_0, mnop_0); + __m512i ijklmnop_1 = unpack_lo_128(ijkl_1, mnop_1); + __m512i ijklmnop_2 = unpack_lo_128(ijkl_2, mnop_2); + __m512i ijklmnop_3 = unpack_lo_128(ijkl_3, mnop_3); + __m512i ijklmnop_4 = unpack_hi_128(ijkl_0, mnop_0); + __m512i ijklmnop_5 = unpack_hi_128(ijkl_1, mnop_1); + __m512i ijklmnop_6 = unpack_hi_128(ijkl_2, mnop_2); + __m512i ijklmnop_7 = unpack_hi_128(ijkl_3, mnop_3); + + // Interleave 128-bit lanes again for the final outputs. + vecs[0] = unpack_lo_128(abcdefgh_0, ijklmnop_0); + vecs[1] = unpack_lo_128(abcdefgh_1, ijklmnop_1); + vecs[2] = unpack_lo_128(abcdefgh_2, ijklmnop_2); + vecs[3] = unpack_lo_128(abcdefgh_3, ijklmnop_3); + vecs[4] = unpack_lo_128(abcdefgh_4, ijklmnop_4); + vecs[5] = unpack_lo_128(abcdefgh_5, ijklmnop_5); + vecs[6] = unpack_lo_128(abcdefgh_6, ijklmnop_6); + vecs[7] = unpack_lo_128(abcdefgh_7, ijklmnop_7); + vecs[8] = unpack_hi_128(abcdefgh_0, ijklmnop_0); + vecs[9] = unpack_hi_128(abcdefgh_1, ijklmnop_1); + vecs[10] = unpack_hi_128(abcdefgh_2, ijklmnop_2); + vecs[11] = unpack_hi_128(abcdefgh_3, ijklmnop_3); + vecs[12] = unpack_hi_128(abcdefgh_4, ijklmnop_4); + vecs[13] = unpack_hi_128(abcdefgh_5, ijklmnop_5); + vecs[14] = unpack_hi_128(abcdefgh_6, ijklmnop_6); + vecs[15] = unpack_hi_128(abcdefgh_7, ijklmnop_7); +} + +INLINE void transpose_msg_vecs16(const uint8_t *const *inputs, + size_t block_offset, __m512i out[16]) { + out[0] = loadu_512(&inputs[0][block_offset]); + out[1] = loadu_512(&inputs[1][block_offset]); + out[2] = loadu_512(&inputs[2][block_offset]); + out[3] = loadu_512(&inputs[3][block_offset]); + out[4] = loadu_512(&inputs[4][block_offset]); + out[5] = loadu_512(&inputs[5][block_offset]); + out[6] = loadu_512(&inputs[6][block_offset]); + out[7] = loadu_512(&inputs[7][block_offset]); + out[8] = loadu_512(&inputs[8][block_offset]); + out[9] = loadu_512(&inputs[9][block_offset]); + out[10] = loadu_512(&inputs[10][block_offset]); + out[11] = loadu_512(&inputs[11][block_offset]); + out[12] = loadu_512(&inputs[12][block_offset]); + out[13] = loadu_512(&inputs[13][block_offset]); + out[14] = loadu_512(&inputs[14][block_offset]); + out[15] = loadu_512(&inputs[15][block_offset]); + for (size_t i = 0; i < 16; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs_512(out); +} + +INLINE void load_counters16(uint64_t counter, bool increment_counter, + __m512i *out_lo, __m512i *out_hi) { + const __m512i mask = _mm512_set1_epi32(-(int32_t)increment_counter); + const __m512i deltas = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + const __m512i masked_deltas = _mm512_and_si512(deltas, mask); + const __m512i low_words = _mm512_add_epi32( + _mm512_set1_epi32((int32_t)counter), + masked_deltas); + // The carry bit is 1 if the high bit of the word was 1 before addition and is + // 0 after. + // NOTE: It would be a bit more natural to use _mm512_cmp_epu32_mask to + // compute the carry bits here, and originally we did, but that intrinsic is + // broken under GCC 5.4. See https://github.com/BLAKE3-team/BLAKE3/issues/271. + const __m512i carries = _mm512_srli_epi32( + _mm512_andnot_si512( + low_words, // 0 after (gets inverted by andnot) + _mm512_set1_epi32((int32_t)counter)), // and 1 before + 31); + const __m512i high_words = _mm512_add_epi32( + _mm512_set1_epi32((int32_t)(counter >> 32)), + carries); + *out_lo = low_words; + *out_hi = high_words; +} + +static +void blake3_hash16_avx512(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, + uint8_t *out) { + __m512i h_vecs[8] = { + set1_512(key[0]), set1_512(key[1]), set1_512(key[2]), set1_512(key[3]), + set1_512(key[4]), set1_512(key[5]), set1_512(key[6]), set1_512(key[7]), + }; + __m512i counter_low_vec, counter_high_vec; + load_counters16(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m512i block_len_vec = set1_512(BLAKE3_BLOCK_LEN); + __m512i block_flags_vec = set1_512(block_flags); + __m512i msg_vecs[16]; + transpose_msg_vecs16(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m512i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_512(B3_IV_0), set1_512(B3_IV_1), set1_512(B3_IV_2), set1_512(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn16(v, msg_vecs, 0); + round_fn16(v, msg_vecs, 1); + round_fn16(v, msg_vecs, 2); + round_fn16(v, msg_vecs, 3); + round_fn16(v, msg_vecs, 4); + round_fn16(v, msg_vecs, 5); + round_fn16(v, msg_vecs, 6); + h_vecs[0] = xor_512(v[0], v[8]); + h_vecs[1] = xor_512(v[1], v[9]); + h_vecs[2] = xor_512(v[2], v[10]); + h_vecs[3] = xor_512(v[3], v[11]); + h_vecs[4] = xor_512(v[4], v[12]); + h_vecs[5] = xor_512(v[5], v[13]); + h_vecs[6] = xor_512(v[6], v[14]); + h_vecs[7] = xor_512(v[7], v[15]); + + block_flags = flags; + } + + // transpose_vecs_512 operates on a 16x16 matrix of words, but we only have 8 + // state vectors. Pad the matrix with zeros. After transposition, store the + // lower half of each vector. + __m512i padded[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_512(0), set1_512(0), set1_512(0), set1_512(0), + set1_512(0), set1_512(0), set1_512(0), set1_512(0), + }; + transpose_vecs_512(padded); + _mm256_mask_storeu_epi32(&out[0 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[0])); + _mm256_mask_storeu_epi32(&out[1 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[1])); + _mm256_mask_storeu_epi32(&out[2 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[2])); + _mm256_mask_storeu_epi32(&out[3 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[3])); + _mm256_mask_storeu_epi32(&out[4 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[4])); + _mm256_mask_storeu_epi32(&out[5 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[5])); + _mm256_mask_storeu_epi32(&out[6 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[6])); + _mm256_mask_storeu_epi32(&out[7 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[7])); + _mm256_mask_storeu_epi32(&out[8 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[8])); + _mm256_mask_storeu_epi32(&out[9 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[9])); + _mm256_mask_storeu_epi32(&out[10 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[10])); + _mm256_mask_storeu_epi32(&out[11 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[11])); + _mm256_mask_storeu_epi32(&out[12 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[12])); + _mm256_mask_storeu_epi32(&out[13 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[13])); + _mm256_mask_storeu_epi32(&out[14 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[14])); + _mm256_mask_storeu_epi32(&out[15 * sizeof(__m256i)], (__mmask8)-1, _mm512_castsi512_si256(padded[15])); +} + +/* + * ---------------------------------------------------------------------------- + * hash_many_avx512 + * ---------------------------------------------------------------------------- + */ + +INLINE void hash_one_avx512(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_avx512(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= 16) { + blake3_hash16_avx512(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 16; + } + inputs += 16; + num_inputs -= 16; + out = &out[16 * BLAKE3_OUT_LEN]; + } + while (num_inputs >= 8) { + blake3_hash8_avx512(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 8; + } + inputs += 8; + num_inputs -= 8; + out = &out[8 * BLAKE3_OUT_LEN]; + } + while (num_inputs >= 4) { + blake3_hash4_avx512(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 4; + } + inputs += 4; + num_inputs -= 4; + out = &out[4 * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_avx512(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/src/blake3/blake3_avx512_x86-64_unix.S b/src/blake3/blake3_avx512_x86-64_unix.S new file mode 100644 index 0000000..4286f76 --- /dev/null +++ b/src/blake3/blake3_avx512_x86-64_unix.S @@ -0,0 +1,2585 @@ +#if defined(__ELF__) && defined(__linux__) +.section .note.GNU-stack,"",%progbits +#endif + +#if defined(__ELF__) && defined(__CET__) && defined(__has_include) +#if __has_include() +#include +#endif +#endif + +#if !defined(_CET_ENDBR) +#define _CET_ENDBR +#endif + +.intel_syntax noprefix +.global _blake3_hash_many_avx512_asm +.global blake3_hash_many_avx512_asm +.global blake3_compress_in_place_avx512_asm +.global _blake3_compress_in_place_avx512_asm +.global blake3_compress_xof_avx512_asm +.global _blake3_compress_xof_avx512_asm + +#ifdef __APPLE__ +.text +#else +.section .text +#endif +.p2align 6 +_blake3_hash_many_avx512_asm: +blake3_hash_many_avx512_asm: + _CET_ENDBR + push r15 + push r14 + push r13 + push r12 + push rbx + push rbp + mov rbp, rsp + sub rsp, 144 + and rsp, 0xFFFFFFFFFFFFFFC0 + neg r9 + kmovw k1, r9d + vmovd xmm0, r8d + vpbroadcastd ymm0, xmm0 + shr r8, 32 + vmovd xmm1, r8d + vpbroadcastd ymm1, xmm1 + vmovdqa ymm4, ymm1 + vmovdqa ymm5, ymm1 + vpaddd ymm2, ymm0, ymmword ptr [ADD0+rip] + vpaddd ymm3, ymm0, ymmword ptr [ADD0+32+rip] + vpcmpltud k2, ymm2, ymm0 + vpcmpltud k3, ymm3, ymm0 + vpaddd ymm4 {k2}, ymm4, dword ptr [ADD1+rip] {1to8} + vpaddd ymm5 {k3}, ymm5, dword ptr [ADD1+rip] {1to8} + knotw k2, k1 + vmovdqa32 ymm2 {k2}, ymm0 + vmovdqa32 ymm3 {k2}, ymm0 + vmovdqa32 ymm4 {k2}, ymm1 + vmovdqa32 ymm5 {k2}, ymm1 + vmovdqa ymmword ptr [rsp], ymm2 + vmovdqa ymmword ptr [rsp+0x1*0x20], ymm3 + vmovdqa ymmword ptr [rsp+0x2*0x20], ymm4 + vmovdqa ymmword ptr [rsp+0x3*0x20], ymm5 + shl rdx, 6 + mov qword ptr [rsp+0x80], rdx + cmp rsi, 16 + jc 3f +2: + vpbroadcastd zmm0, dword ptr [rcx] + vpbroadcastd zmm1, dword ptr [rcx+0x1*0x4] + vpbroadcastd zmm2, dword ptr [rcx+0x2*0x4] + vpbroadcastd zmm3, dword ptr [rcx+0x3*0x4] + vpbroadcastd zmm4, dword ptr [rcx+0x4*0x4] + vpbroadcastd zmm5, dword ptr [rcx+0x5*0x4] + vpbroadcastd zmm6, dword ptr [rcx+0x6*0x4] + vpbroadcastd zmm7, dword ptr [rcx+0x7*0x4] + movzx eax, byte ptr [rbp+0x38] + movzx ebx, byte ptr [rbp+0x40] + or eax, ebx + xor edx, edx +.p2align 5 +9: + movzx ebx, byte ptr [rbp+0x48] + or ebx, eax + add rdx, 64 + cmp rdx, qword ptr [rsp+0x80] + cmove eax, ebx + mov dword ptr [rsp+0x88], eax + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + mov r12, qword ptr [rdi+0x40] + mov r13, qword ptr [rdi+0x48] + mov r14, qword ptr [rdi+0x50] + mov r15, qword ptr [rdi+0x58] + vmovdqu32 ymm16, ymmword ptr [rdx+r8-0x2*0x20] + vinserti64x4 zmm16, zmm16, ymmword ptr [rdx+r12-0x2*0x20], 0x01 + vmovdqu32 ymm17, ymmword ptr [rdx+r9-0x2*0x20] + vinserti64x4 zmm17, zmm17, ymmword ptr [rdx+r13-0x2*0x20], 0x01 + vpunpcklqdq zmm8, zmm16, zmm17 + vpunpckhqdq zmm9, zmm16, zmm17 + vmovdqu32 ymm18, ymmword ptr [rdx+r10-0x2*0x20] + vinserti64x4 zmm18, zmm18, ymmword ptr [rdx+r14-0x2*0x20], 0x01 + vmovdqu32 ymm19, ymmword ptr [rdx+r11-0x2*0x20] + vinserti64x4 zmm19, zmm19, ymmword ptr [rdx+r15-0x2*0x20], 0x01 + vpunpcklqdq zmm10, zmm18, zmm19 + vpunpckhqdq zmm11, zmm18, zmm19 + mov r8, qword ptr [rdi+0x20] + mov r9, qword ptr [rdi+0x28] + mov r10, qword ptr [rdi+0x30] + mov r11, qword ptr [rdi+0x38] + mov r12, qword ptr [rdi+0x60] + mov r13, qword ptr [rdi+0x68] + mov r14, qword ptr [rdi+0x70] + mov r15, qword ptr [rdi+0x78] + vmovdqu32 ymm16, ymmword ptr [rdx+r8-0x2*0x20] + vinserti64x4 zmm16, zmm16, ymmword ptr [rdx+r12-0x2*0x20], 0x01 + vmovdqu32 ymm17, ymmword ptr [rdx+r9-0x2*0x20] + vinserti64x4 zmm17, zmm17, ymmword ptr [rdx+r13-0x2*0x20], 0x01 + vpunpcklqdq zmm12, zmm16, zmm17 + vpunpckhqdq zmm13, zmm16, zmm17 + vmovdqu32 ymm18, ymmword ptr [rdx+r10-0x2*0x20] + vinserti64x4 zmm18, zmm18, ymmword ptr [rdx+r14-0x2*0x20], 0x01 + vmovdqu32 ymm19, ymmword ptr [rdx+r11-0x2*0x20] + vinserti64x4 zmm19, zmm19, ymmword ptr [rdx+r15-0x2*0x20], 0x01 + vpunpcklqdq zmm14, zmm18, zmm19 + vpunpckhqdq zmm15, zmm18, zmm19 + vmovdqa32 zmm27, zmmword ptr [INDEX0+rip] + vmovdqa32 zmm31, zmmword ptr [INDEX1+rip] + vshufps zmm16, zmm8, zmm10, 136 + vshufps zmm17, zmm12, zmm14, 136 + vmovdqa32 zmm20, zmm16 + vpermt2d zmm16, zmm27, zmm17 + vpermt2d zmm20, zmm31, zmm17 + vshufps zmm17, zmm8, zmm10, 221 + vshufps zmm30, zmm12, zmm14, 221 + vmovdqa32 zmm21, zmm17 + vpermt2d zmm17, zmm27, zmm30 + vpermt2d zmm21, zmm31, zmm30 + vshufps zmm18, zmm9, zmm11, 136 + vshufps zmm8, zmm13, zmm15, 136 + vmovdqa32 zmm22, zmm18 + vpermt2d zmm18, zmm27, zmm8 + vpermt2d zmm22, zmm31, zmm8 + vshufps zmm19, zmm9, zmm11, 221 + vshufps zmm8, zmm13, zmm15, 221 + vmovdqa32 zmm23, zmm19 + vpermt2d zmm19, zmm27, zmm8 + vpermt2d zmm23, zmm31, zmm8 + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + mov r12, qword ptr [rdi+0x40] + mov r13, qword ptr [rdi+0x48] + mov r14, qword ptr [rdi+0x50] + mov r15, qword ptr [rdi+0x58] + vmovdqu32 ymm24, ymmword ptr [r8+rdx-0x1*0x20] + vinserti64x4 zmm24, zmm24, ymmword ptr [r12+rdx-0x1*0x20], 0x01 + vmovdqu32 ymm25, ymmword ptr [r9+rdx-0x1*0x20] + vinserti64x4 zmm25, zmm25, ymmword ptr [r13+rdx-0x1*0x20], 0x01 + vpunpcklqdq zmm8, zmm24, zmm25 + vpunpckhqdq zmm9, zmm24, zmm25 + vmovdqu32 ymm24, ymmword ptr [r10+rdx-0x1*0x20] + vinserti64x4 zmm24, zmm24, ymmword ptr [r14+rdx-0x1*0x20], 0x01 + vmovdqu32 ymm25, ymmword ptr [r11+rdx-0x1*0x20] + vinserti64x4 zmm25, zmm25, ymmword ptr [r15+rdx-0x1*0x20], 0x01 + vpunpcklqdq zmm10, zmm24, zmm25 + vpunpckhqdq zmm11, zmm24, zmm25 + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r12+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r13+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r14+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + prefetcht0 [r15+rdx+0x80] + mov r8, qword ptr [rdi+0x20] + mov r9, qword ptr [rdi+0x28] + mov r10, qword ptr [rdi+0x30] + mov r11, qword ptr [rdi+0x38] + mov r12, qword ptr [rdi+0x60] + mov r13, qword ptr [rdi+0x68] + mov r14, qword ptr [rdi+0x70] + mov r15, qword ptr [rdi+0x78] + vmovdqu32 ymm24, ymmword ptr [r8+rdx-0x1*0x20] + vinserti64x4 zmm24, zmm24, ymmword ptr [r12+rdx-0x1*0x20], 0x01 + vmovdqu32 ymm25, ymmword ptr [r9+rdx-0x1*0x20] + vinserti64x4 zmm25, zmm25, ymmword ptr [r13+rdx-0x1*0x20], 0x01 + vpunpcklqdq zmm12, zmm24, zmm25 + vpunpckhqdq zmm13, zmm24, zmm25 + vmovdqu32 ymm24, ymmword ptr [r10+rdx-0x1*0x20] + vinserti64x4 zmm24, zmm24, ymmword ptr [r14+rdx-0x1*0x20], 0x01 + vmovdqu32 ymm25, ymmword ptr [r11+rdx-0x1*0x20] + vinserti64x4 zmm25, zmm25, ymmword ptr [r15+rdx-0x1*0x20], 0x01 + vpunpcklqdq zmm14, zmm24, zmm25 + vpunpckhqdq zmm15, zmm24, zmm25 + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r12+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r13+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r14+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + prefetcht0 [r15+rdx+0x80] + vshufps zmm24, zmm8, zmm10, 136 + vshufps zmm30, zmm12, zmm14, 136 + vmovdqa32 zmm28, zmm24 + vpermt2d zmm24, zmm27, zmm30 + vpermt2d zmm28, zmm31, zmm30 + vshufps zmm25, zmm8, zmm10, 221 + vshufps zmm30, zmm12, zmm14, 221 + vmovdqa32 zmm29, zmm25 + vpermt2d zmm25, zmm27, zmm30 + vpermt2d zmm29, zmm31, zmm30 + vshufps zmm26, zmm9, zmm11, 136 + vshufps zmm8, zmm13, zmm15, 136 + vmovdqa32 zmm30, zmm26 + vpermt2d zmm26, zmm27, zmm8 + vpermt2d zmm30, zmm31, zmm8 + vshufps zmm8, zmm9, zmm11, 221 + vshufps zmm10, zmm13, zmm15, 221 + vpermi2d zmm27, zmm8, zmm10 + vpermi2d zmm31, zmm8, zmm10 + vpbroadcastd zmm8, dword ptr [BLAKE3_IV_0+rip] + vpbroadcastd zmm9, dword ptr [BLAKE3_IV_1+rip] + vpbroadcastd zmm10, dword ptr [BLAKE3_IV_2+rip] + vpbroadcastd zmm11, dword ptr [BLAKE3_IV_3+rip] + vmovdqa32 zmm12, zmmword ptr [rsp] + vmovdqa32 zmm13, zmmword ptr [rsp+0x1*0x40] + vpbroadcastd zmm14, dword ptr [BLAKE3_BLOCK_LEN+rip] + vpbroadcastd zmm15, dword ptr [rsp+0x22*0x4] + vpaddd zmm0, zmm0, zmm16 + vpaddd zmm1, zmm1, zmm18 + vpaddd zmm2, zmm2, zmm20 + vpaddd zmm3, zmm3, zmm22 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm17 + vpaddd zmm1, zmm1, zmm19 + vpaddd zmm2, zmm2, zmm21 + vpaddd zmm3, zmm3, zmm23 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm24 + vpaddd zmm1, zmm1, zmm26 + vpaddd zmm2, zmm2, zmm28 + vpaddd zmm3, zmm3, zmm30 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm25 + vpaddd zmm1, zmm1, zmm27 + vpaddd zmm2, zmm2, zmm29 + vpaddd zmm3, zmm3, zmm31 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm18 + vpaddd zmm1, zmm1, zmm19 + vpaddd zmm2, zmm2, zmm23 + vpaddd zmm3, zmm3, zmm20 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm22 + vpaddd zmm1, zmm1, zmm26 + vpaddd zmm2, zmm2, zmm16 + vpaddd zmm3, zmm3, zmm29 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm17 + vpaddd zmm1, zmm1, zmm28 + vpaddd zmm2, zmm2, zmm25 + vpaddd zmm3, zmm3, zmm31 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm27 + vpaddd zmm1, zmm1, zmm21 + vpaddd zmm2, zmm2, zmm30 + vpaddd zmm3, zmm3, zmm24 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm19 + vpaddd zmm1, zmm1, zmm26 + vpaddd zmm2, zmm2, zmm29 + vpaddd zmm3, zmm3, zmm23 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm20 + vpaddd zmm1, zmm1, zmm28 + vpaddd zmm2, zmm2, zmm18 + vpaddd zmm3, zmm3, zmm30 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm22 + vpaddd zmm1, zmm1, zmm25 + vpaddd zmm2, zmm2, zmm27 + vpaddd zmm3, zmm3, zmm24 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm21 + vpaddd zmm1, zmm1, zmm16 + vpaddd zmm2, zmm2, zmm31 + vpaddd zmm3, zmm3, zmm17 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm26 + vpaddd zmm1, zmm1, zmm28 + vpaddd zmm2, zmm2, zmm30 + vpaddd zmm3, zmm3, zmm29 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm23 + vpaddd zmm1, zmm1, zmm25 + vpaddd zmm2, zmm2, zmm19 + vpaddd zmm3, zmm3, zmm31 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm20 + vpaddd zmm1, zmm1, zmm27 + vpaddd zmm2, zmm2, zmm21 + vpaddd zmm3, zmm3, zmm17 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm16 + vpaddd zmm1, zmm1, zmm18 + vpaddd zmm2, zmm2, zmm24 + vpaddd zmm3, zmm3, zmm22 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm28 + vpaddd zmm1, zmm1, zmm25 + vpaddd zmm2, zmm2, zmm31 + vpaddd zmm3, zmm3, zmm30 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm29 + vpaddd zmm1, zmm1, zmm27 + vpaddd zmm2, zmm2, zmm26 + vpaddd zmm3, zmm3, zmm24 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm23 + vpaddd zmm1, zmm1, zmm21 + vpaddd zmm2, zmm2, zmm16 + vpaddd zmm3, zmm3, zmm22 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm18 + vpaddd zmm1, zmm1, zmm19 + vpaddd zmm2, zmm2, zmm17 + vpaddd zmm3, zmm3, zmm20 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm25 + vpaddd zmm1, zmm1, zmm27 + vpaddd zmm2, zmm2, zmm24 + vpaddd zmm3, zmm3, zmm31 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm30 + vpaddd zmm1, zmm1, zmm21 + vpaddd zmm2, zmm2, zmm28 + vpaddd zmm3, zmm3, zmm17 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm29 + vpaddd zmm1, zmm1, zmm16 + vpaddd zmm2, zmm2, zmm18 + vpaddd zmm3, zmm3, zmm20 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm19 + vpaddd zmm1, zmm1, zmm26 + vpaddd zmm2, zmm2, zmm22 + vpaddd zmm3, zmm3, zmm23 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpaddd zmm0, zmm0, zmm27 + vpaddd zmm1, zmm1, zmm21 + vpaddd zmm2, zmm2, zmm17 + vpaddd zmm3, zmm3, zmm24 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vprord zmm15, zmm15, 16 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 12 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vpaddd zmm0, zmm0, zmm31 + vpaddd zmm1, zmm1, zmm16 + vpaddd zmm2, zmm2, zmm25 + vpaddd zmm3, zmm3, zmm22 + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm1, zmm1, zmm5 + vpaddd zmm2, zmm2, zmm6 + vpaddd zmm3, zmm3, zmm7 + vpxord zmm12, zmm12, zmm0 + vpxord zmm13, zmm13, zmm1 + vpxord zmm14, zmm14, zmm2 + vpxord zmm15, zmm15, zmm3 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vprord zmm15, zmm15, 8 + vpaddd zmm8, zmm8, zmm12 + vpaddd zmm9, zmm9, zmm13 + vpaddd zmm10, zmm10, zmm14 + vpaddd zmm11, zmm11, zmm15 + vpxord zmm4, zmm4, zmm8 + vpxord zmm5, zmm5, zmm9 + vpxord zmm6, zmm6, zmm10 + vpxord zmm7, zmm7, zmm11 + vprord zmm4, zmm4, 7 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vpaddd zmm0, zmm0, zmm30 + vpaddd zmm1, zmm1, zmm18 + vpaddd zmm2, zmm2, zmm19 + vpaddd zmm3, zmm3, zmm23 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 16 + vprord zmm12, zmm12, 16 + vprord zmm13, zmm13, 16 + vprord zmm14, zmm14, 16 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 12 + vprord zmm6, zmm6, 12 + vprord zmm7, zmm7, 12 + vprord zmm4, zmm4, 12 + vpaddd zmm0, zmm0, zmm26 + vpaddd zmm1, zmm1, zmm28 + vpaddd zmm2, zmm2, zmm20 + vpaddd zmm3, zmm3, zmm29 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm1, zmm1, zmm6 + vpaddd zmm2, zmm2, zmm7 + vpaddd zmm3, zmm3, zmm4 + vpxord zmm15, zmm15, zmm0 + vpxord zmm12, zmm12, zmm1 + vpxord zmm13, zmm13, zmm2 + vpxord zmm14, zmm14, zmm3 + vprord zmm15, zmm15, 8 + vprord zmm12, zmm12, 8 + vprord zmm13, zmm13, 8 + vprord zmm14, zmm14, 8 + vpaddd zmm10, zmm10, zmm15 + vpaddd zmm11, zmm11, zmm12 + vpaddd zmm8, zmm8, zmm13 + vpaddd zmm9, zmm9, zmm14 + vpxord zmm5, zmm5, zmm10 + vpxord zmm6, zmm6, zmm11 + vpxord zmm7, zmm7, zmm8 + vpxord zmm4, zmm4, zmm9 + vprord zmm5, zmm5, 7 + vprord zmm6, zmm6, 7 + vprord zmm7, zmm7, 7 + vprord zmm4, zmm4, 7 + vpxord zmm0, zmm0, zmm8 + vpxord zmm1, zmm1, zmm9 + vpxord zmm2, zmm2, zmm10 + vpxord zmm3, zmm3, zmm11 + vpxord zmm4, zmm4, zmm12 + vpxord zmm5, zmm5, zmm13 + vpxord zmm6, zmm6, zmm14 + vpxord zmm7, zmm7, zmm15 + movzx eax, byte ptr [rbp+0x38] + jne 9b + mov rbx, qword ptr [rbp+0x50] + vpunpckldq zmm16, zmm0, zmm1 + vpunpckhdq zmm17, zmm0, zmm1 + vpunpckldq zmm18, zmm2, zmm3 + vpunpckhdq zmm19, zmm2, zmm3 + vpunpckldq zmm20, zmm4, zmm5 + vpunpckhdq zmm21, zmm4, zmm5 + vpunpckldq zmm22, zmm6, zmm7 + vpunpckhdq zmm23, zmm6, zmm7 + vpunpcklqdq zmm0, zmm16, zmm18 + vpunpckhqdq zmm1, zmm16, zmm18 + vpunpcklqdq zmm2, zmm17, zmm19 + vpunpckhqdq zmm3, zmm17, zmm19 + vpunpcklqdq zmm4, zmm20, zmm22 + vpunpckhqdq zmm5, zmm20, zmm22 + vpunpcklqdq zmm6, zmm21, zmm23 + vpunpckhqdq zmm7, zmm21, zmm23 + vshufi32x4 zmm16, zmm0, zmm4, 0x88 + vshufi32x4 zmm17, zmm1, zmm5, 0x88 + vshufi32x4 zmm18, zmm2, zmm6, 0x88 + vshufi32x4 zmm19, zmm3, zmm7, 0x88 + vshufi32x4 zmm20, zmm0, zmm4, 0xDD + vshufi32x4 zmm21, zmm1, zmm5, 0xDD + vshufi32x4 zmm22, zmm2, zmm6, 0xDD + vshufi32x4 zmm23, zmm3, zmm7, 0xDD + vshufi32x4 zmm0, zmm16, zmm17, 0x88 + vshufi32x4 zmm1, zmm18, zmm19, 0x88 + vshufi32x4 zmm2, zmm20, zmm21, 0x88 + vshufi32x4 zmm3, zmm22, zmm23, 0x88 + vshufi32x4 zmm4, zmm16, zmm17, 0xDD + vshufi32x4 zmm5, zmm18, zmm19, 0xDD + vshufi32x4 zmm6, zmm20, zmm21, 0xDD + vshufi32x4 zmm7, zmm22, zmm23, 0xDD + vmovdqu32 zmmword ptr [rbx], zmm0 + vmovdqu32 zmmword ptr [rbx+0x1*0x40], zmm1 + vmovdqu32 zmmword ptr [rbx+0x2*0x40], zmm2 + vmovdqu32 zmmword ptr [rbx+0x3*0x40], zmm3 + vmovdqu32 zmmword ptr [rbx+0x4*0x40], zmm4 + vmovdqu32 zmmword ptr [rbx+0x5*0x40], zmm5 + vmovdqu32 zmmword ptr [rbx+0x6*0x40], zmm6 + vmovdqu32 zmmword ptr [rbx+0x7*0x40], zmm7 + vmovdqa32 zmm0, zmmword ptr [rsp] + vmovdqa32 zmm1, zmmword ptr [rsp+0x1*0x40] + vmovdqa32 zmm2, zmm0 + vpaddd zmm2{k1}, zmm0, dword ptr [ADD16+rip] {1to16} + vpcmpltud k2, zmm2, zmm0 + vpaddd zmm1 {k2}, zmm1, dword ptr [ADD1+rip] {1to16} + vmovdqa32 zmmword ptr [rsp], zmm2 + vmovdqa32 zmmword ptr [rsp+0x1*0x40], zmm1 + add rdi, 128 + add rbx, 512 + mov qword ptr [rbp+0x50], rbx + sub rsi, 16 + cmp rsi, 16 + jnc 2b + test rsi, rsi + jnz 3f +4: + vzeroupper + mov rsp, rbp + pop rbp + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + ret +.p2align 6 +3: + test esi, 0x8 + je 3f + vpbroadcastd ymm0, dword ptr [rcx] + vpbroadcastd ymm1, dword ptr [rcx+0x4] + vpbroadcastd ymm2, dword ptr [rcx+0x8] + vpbroadcastd ymm3, dword ptr [rcx+0xC] + vpbroadcastd ymm4, dword ptr [rcx+0x10] + vpbroadcastd ymm5, dword ptr [rcx+0x14] + vpbroadcastd ymm6, dword ptr [rcx+0x18] + vpbroadcastd ymm7, dword ptr [rcx+0x1C] + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + mov r12, qword ptr [rdi+0x20] + mov r13, qword ptr [rdi+0x28] + mov r14, qword ptr [rdi+0x30] + mov r15, qword ptr [rdi+0x38] + movzx eax, byte ptr [rbp+0x38] + movzx ebx, byte ptr [rbp+0x40] + or eax, ebx + xor edx, edx +2: + movzx ebx, byte ptr [rbp+0x48] + or ebx, eax + add rdx, 64 + cmp rdx, qword ptr [rsp+0x80] + cmove eax, ebx + mov dword ptr [rsp+0x88], eax + vmovups xmm8, xmmword ptr [r8+rdx-0x40] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x40], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x40] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x40], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x40] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x40], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x40] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x40], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm16, ymm12, ymm14, 136 + vshufps ymm17, ymm12, ymm14, 221 + vshufps ymm18, ymm13, ymm15, 136 + vshufps ymm19, ymm13, ymm15, 221 + vmovups xmm8, xmmword ptr [r8+rdx-0x30] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x30], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x30] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x30], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x30] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x30], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x30] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x30], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm20, ymm12, ymm14, 136 + vshufps ymm21, ymm12, ymm14, 221 + vshufps ymm22, ymm13, ymm15, 136 + vshufps ymm23, ymm13, ymm15, 221 + vmovups xmm8, xmmword ptr [r8+rdx-0x20] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x20], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x20] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x20], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x20] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x20], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x20] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x20], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm24, ymm12, ymm14, 136 + vshufps ymm25, ymm12, ymm14, 221 + vshufps ymm26, ymm13, ymm15, 136 + vshufps ymm27, ymm13, ymm15, 221 + vmovups xmm8, xmmword ptr [r8+rdx-0x10] + vinsertf128 ymm8, ymm8, xmmword ptr [r12+rdx-0x10], 0x01 + vmovups xmm9, xmmword ptr [r9+rdx-0x10] + vinsertf128 ymm9, ymm9, xmmword ptr [r13+rdx-0x10], 0x01 + vunpcklpd ymm12, ymm8, ymm9 + vunpckhpd ymm13, ymm8, ymm9 + vmovups xmm10, xmmword ptr [r10+rdx-0x10] + vinsertf128 ymm10, ymm10, xmmword ptr [r14+rdx-0x10], 0x01 + vmovups xmm11, xmmword ptr [r11+rdx-0x10] + vinsertf128 ymm11, ymm11, xmmword ptr [r15+rdx-0x10], 0x01 + vunpcklpd ymm14, ymm10, ymm11 + vunpckhpd ymm15, ymm10, ymm11 + vshufps ymm28, ymm12, ymm14, 136 + vshufps ymm29, ymm12, ymm14, 221 + vshufps ymm30, ymm13, ymm15, 136 + vshufps ymm31, ymm13, ymm15, 221 + vpbroadcastd ymm8, dword ptr [BLAKE3_IV_0+rip] + vpbroadcastd ymm9, dword ptr [BLAKE3_IV_1+rip] + vpbroadcastd ymm10, dword ptr [BLAKE3_IV_2+rip] + vpbroadcastd ymm11, dword ptr [BLAKE3_IV_3+rip] + vmovdqa ymm12, ymmword ptr [rsp] + vmovdqa ymm13, ymmword ptr [rsp+0x40] + vpbroadcastd ymm14, dword ptr [BLAKE3_BLOCK_LEN+rip] + vpbroadcastd ymm15, dword ptr [rsp+0x88] + vpaddd ymm0, ymm0, ymm16 + vpaddd ymm1, ymm1, ymm18 + vpaddd ymm2, ymm2, ymm20 + vpaddd ymm3, ymm3, ymm22 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm17 + vpaddd ymm1, ymm1, ymm19 + vpaddd ymm2, ymm2, ymm21 + vpaddd ymm3, ymm3, ymm23 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm24 + vpaddd ymm1, ymm1, ymm26 + vpaddd ymm2, ymm2, ymm28 + vpaddd ymm3, ymm3, ymm30 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm25 + vpaddd ymm1, ymm1, ymm27 + vpaddd ymm2, ymm2, ymm29 + vpaddd ymm3, ymm3, ymm31 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm18 + vpaddd ymm1, ymm1, ymm19 + vpaddd ymm2, ymm2, ymm23 + vpaddd ymm3, ymm3, ymm20 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm22 + vpaddd ymm1, ymm1, ymm26 + vpaddd ymm2, ymm2, ymm16 + vpaddd ymm3, ymm3, ymm29 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm17 + vpaddd ymm1, ymm1, ymm28 + vpaddd ymm2, ymm2, ymm25 + vpaddd ymm3, ymm3, ymm31 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm27 + vpaddd ymm1, ymm1, ymm21 + vpaddd ymm2, ymm2, ymm30 + vpaddd ymm3, ymm3, ymm24 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm19 + vpaddd ymm1, ymm1, ymm26 + vpaddd ymm2, ymm2, ymm29 + vpaddd ymm3, ymm3, ymm23 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm20 + vpaddd ymm1, ymm1, ymm28 + vpaddd ymm2, ymm2, ymm18 + vpaddd ymm3, ymm3, ymm30 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm22 + vpaddd ymm1, ymm1, ymm25 + vpaddd ymm2, ymm2, ymm27 + vpaddd ymm3, ymm3, ymm24 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm21 + vpaddd ymm1, ymm1, ymm16 + vpaddd ymm2, ymm2, ymm31 + vpaddd ymm3, ymm3, ymm17 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm26 + vpaddd ymm1, ymm1, ymm28 + vpaddd ymm2, ymm2, ymm30 + vpaddd ymm3, ymm3, ymm29 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm23 + vpaddd ymm1, ymm1, ymm25 + vpaddd ymm2, ymm2, ymm19 + vpaddd ymm3, ymm3, ymm31 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm20 + vpaddd ymm1, ymm1, ymm27 + vpaddd ymm2, ymm2, ymm21 + vpaddd ymm3, ymm3, ymm17 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm16 + vpaddd ymm1, ymm1, ymm18 + vpaddd ymm2, ymm2, ymm24 + vpaddd ymm3, ymm3, ymm22 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm28 + vpaddd ymm1, ymm1, ymm25 + vpaddd ymm2, ymm2, ymm31 + vpaddd ymm3, ymm3, ymm30 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm29 + vpaddd ymm1, ymm1, ymm27 + vpaddd ymm2, ymm2, ymm26 + vpaddd ymm3, ymm3, ymm24 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm23 + vpaddd ymm1, ymm1, ymm21 + vpaddd ymm2, ymm2, ymm16 + vpaddd ymm3, ymm3, ymm22 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm18 + vpaddd ymm1, ymm1, ymm19 + vpaddd ymm2, ymm2, ymm17 + vpaddd ymm3, ymm3, ymm20 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm25 + vpaddd ymm1, ymm1, ymm27 + vpaddd ymm2, ymm2, ymm24 + vpaddd ymm3, ymm3, ymm31 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm30 + vpaddd ymm1, ymm1, ymm21 + vpaddd ymm2, ymm2, ymm28 + vpaddd ymm3, ymm3, ymm17 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm29 + vpaddd ymm1, ymm1, ymm16 + vpaddd ymm2, ymm2, ymm18 + vpaddd ymm3, ymm3, ymm20 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm19 + vpaddd ymm1, ymm1, ymm26 + vpaddd ymm2, ymm2, ymm22 + vpaddd ymm3, ymm3, ymm23 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpaddd ymm0, ymm0, ymm27 + vpaddd ymm1, ymm1, ymm21 + vpaddd ymm2, ymm2, ymm17 + vpaddd ymm3, ymm3, ymm24 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vprord ymm15, ymm15, 16 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 12 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vpaddd ymm0, ymm0, ymm31 + vpaddd ymm1, ymm1, ymm16 + vpaddd ymm2, ymm2, ymm25 + vpaddd ymm3, ymm3, ymm22 + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm1, ymm1, ymm5 + vpaddd ymm2, ymm2, ymm6 + vpaddd ymm3, ymm3, ymm7 + vpxord ymm12, ymm12, ymm0 + vpxord ymm13, ymm13, ymm1 + vpxord ymm14, ymm14, ymm2 + vpxord ymm15, ymm15, ymm3 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vprord ymm15, ymm15, 8 + vpaddd ymm8, ymm8, ymm12 + vpaddd ymm9, ymm9, ymm13 + vpaddd ymm10, ymm10, ymm14 + vpaddd ymm11, ymm11, ymm15 + vpxord ymm4, ymm4, ymm8 + vpxord ymm5, ymm5, ymm9 + vpxord ymm6, ymm6, ymm10 + vpxord ymm7, ymm7, ymm11 + vprord ymm4, ymm4, 7 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vpaddd ymm0, ymm0, ymm30 + vpaddd ymm1, ymm1, ymm18 + vpaddd ymm2, ymm2, ymm19 + vpaddd ymm3, ymm3, ymm23 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 16 + vprord ymm12, ymm12, 16 + vprord ymm13, ymm13, 16 + vprord ymm14, ymm14, 16 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 12 + vprord ymm6, ymm6, 12 + vprord ymm7, ymm7, 12 + vprord ymm4, ymm4, 12 + vpaddd ymm0, ymm0, ymm26 + vpaddd ymm1, ymm1, ymm28 + vpaddd ymm2, ymm2, ymm20 + vpaddd ymm3, ymm3, ymm29 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm1, ymm1, ymm6 + vpaddd ymm2, ymm2, ymm7 + vpaddd ymm3, ymm3, ymm4 + vpxord ymm15, ymm15, ymm0 + vpxord ymm12, ymm12, ymm1 + vpxord ymm13, ymm13, ymm2 + vpxord ymm14, ymm14, ymm3 + vprord ymm15, ymm15, 8 + vprord ymm12, ymm12, 8 + vprord ymm13, ymm13, 8 + vprord ymm14, ymm14, 8 + vpaddd ymm10, ymm10, ymm15 + vpaddd ymm11, ymm11, ymm12 + vpaddd ymm8, ymm8, ymm13 + vpaddd ymm9, ymm9, ymm14 + vpxord ymm5, ymm5, ymm10 + vpxord ymm6, ymm6, ymm11 + vpxord ymm7, ymm7, ymm8 + vpxord ymm4, ymm4, ymm9 + vprord ymm5, ymm5, 7 + vprord ymm6, ymm6, 7 + vprord ymm7, ymm7, 7 + vprord ymm4, ymm4, 7 + vpxor ymm0, ymm0, ymm8 + vpxor ymm1, ymm1, ymm9 + vpxor ymm2, ymm2, ymm10 + vpxor ymm3, ymm3, ymm11 + vpxor ymm4, ymm4, ymm12 + vpxor ymm5, ymm5, ymm13 + vpxor ymm6, ymm6, ymm14 + vpxor ymm7, ymm7, ymm15 + movzx eax, byte ptr [rbp+0x38] + jne 2b + mov rbx, qword ptr [rbp+0x50] + vunpcklps ymm8, ymm0, ymm1 + vunpcklps ymm9, ymm2, ymm3 + vunpckhps ymm10, ymm0, ymm1 + vunpcklps ymm11, ymm4, ymm5 + vunpcklps ymm0, ymm6, ymm7 + vshufps ymm12, ymm8, ymm9, 78 + vblendps ymm1, ymm8, ymm12, 0xCC + vshufps ymm8, ymm11, ymm0, 78 + vunpckhps ymm13, ymm2, ymm3 + vblendps ymm2, ymm11, ymm8, 0xCC + vblendps ymm3, ymm12, ymm9, 0xCC + vperm2f128 ymm12, ymm1, ymm2, 0x20 + vmovups ymmword ptr [rbx], ymm12 + vunpckhps ymm14, ymm4, ymm5 + vblendps ymm4, ymm8, ymm0, 0xCC + vunpckhps ymm15, ymm6, ymm7 + vperm2f128 ymm7, ymm3, ymm4, 0x20 + vmovups ymmword ptr [rbx+0x20], ymm7 + vshufps ymm5, ymm10, ymm13, 78 + vblendps ymm6, ymm5, ymm13, 0xCC + vshufps ymm13, ymm14, ymm15, 78 + vblendps ymm10, ymm10, ymm5, 0xCC + vblendps ymm14, ymm14, ymm13, 0xCC + vperm2f128 ymm8, ymm10, ymm14, 0x20 + vmovups ymmword ptr [rbx+0x40], ymm8 + vblendps ymm15, ymm13, ymm15, 0xCC + vperm2f128 ymm13, ymm6, ymm15, 0x20 + vmovups ymmword ptr [rbx+0x60], ymm13 + vperm2f128 ymm9, ymm1, ymm2, 0x31 + vperm2f128 ymm11, ymm3, ymm4, 0x31 + vmovups ymmword ptr [rbx+0x80], ymm9 + vperm2f128 ymm14, ymm10, ymm14, 0x31 + vperm2f128 ymm15, ymm6, ymm15, 0x31 + vmovups ymmword ptr [rbx+0xA0], ymm11 + vmovups ymmword ptr [rbx+0xC0], ymm14 + vmovups ymmword ptr [rbx+0xE0], ymm15 + vmovdqa ymm0, ymmword ptr [rsp] + vmovdqa ymm2, ymmword ptr [rsp+0x2*0x20] + vmovdqa32 ymm0 {k1}, ymmword ptr [rsp+0x1*0x20] + vmovdqa32 ymm2 {k1}, ymmword ptr [rsp+0x3*0x20] + vmovdqa ymmword ptr [rsp], ymm0 + vmovdqa ymmword ptr [rsp+0x2*0x20], ymm2 + add rbx, 256 + mov qword ptr [rbp+0x50], rbx + add rdi, 64 + sub rsi, 8 +3: + mov rbx, qword ptr [rbp+0x50] + mov r15, qword ptr [rsp+0x80] + movzx r13, byte ptr [rbp+0x38] + movzx r12, byte ptr [rbp+0x48] + test esi, 0x4 + je 3f + vbroadcasti32x4 zmm0, xmmword ptr [rcx] + vbroadcasti32x4 zmm1, xmmword ptr [rcx+0x1*0x10] + vmovdqa xmm12, xmmword ptr [rsp] + vmovdqa xmm13, xmmword ptr [rsp+0x4*0x10] + vpunpckldq xmm14, xmm12, xmm13 + vpunpckhdq xmm15, xmm12, xmm13 + vpermq ymm14, ymm14, 0xDC + vpermq ymm15, ymm15, 0xDC + vpbroadcastd zmm12, dword ptr [BLAKE3_BLOCK_LEN+rip] + vinserti64x4 zmm13, zmm14, ymm15, 0x01 + mov eax, 17476 + kmovw k2, eax + vpblendmd zmm13 {k2}, zmm13, zmm12 + vbroadcasti32x4 zmm15, xmmword ptr [BLAKE3_IV+rip] + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + mov eax, 43690 + kmovw k3, eax + mov eax, 34952 + kmovw k4, eax + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + mov dword ptr [rsp+0x88], eax + vmovdqa32 zmm2, zmm15 + vpbroadcastd zmm8, dword ptr [rsp+0x22*0x4] + vpblendmd zmm3 {k4}, zmm13, zmm8 + vmovups zmm8, zmmword ptr [r8+rdx-0x1*0x40] + vinserti32x4 zmm8, zmm8, xmmword ptr [r9+rdx-0x4*0x10], 0x01 + vinserti32x4 zmm8, zmm8, xmmword ptr [r10+rdx-0x4*0x10], 0x02 + vinserti32x4 zmm8, zmm8, xmmword ptr [r11+rdx-0x4*0x10], 0x03 + vmovups zmm9, zmmword ptr [r8+rdx-0x30] + vinserti32x4 zmm9, zmm9, xmmword ptr [r9+rdx-0x3*0x10], 0x01 + vinserti32x4 zmm9, zmm9, xmmword ptr [r10+rdx-0x3*0x10], 0x02 + vinserti32x4 zmm9, zmm9, xmmword ptr [r11+rdx-0x3*0x10], 0x03 + vshufps zmm4, zmm8, zmm9, 136 + vshufps zmm5, zmm8, zmm9, 221 + vmovups zmm8, zmmword ptr [r8+rdx-0x20] + vinserti32x4 zmm8, zmm8, xmmword ptr [r9+rdx-0x2*0x10], 0x01 + vinserti32x4 zmm8, zmm8, xmmword ptr [r10+rdx-0x2*0x10], 0x02 + vinserti32x4 zmm8, zmm8, xmmword ptr [r11+rdx-0x2*0x10], 0x03 + vmovups zmm9, zmmword ptr [r8+rdx-0x10] + vinserti32x4 zmm9, zmm9, xmmword ptr [r9+rdx-0x1*0x10], 0x01 + vinserti32x4 zmm9, zmm9, xmmword ptr [r10+rdx-0x1*0x10], 0x02 + vinserti32x4 zmm9, zmm9, xmmword ptr [r11+rdx-0x1*0x10], 0x03 + vshufps zmm6, zmm8, zmm9, 136 + vshufps zmm7, zmm8, zmm9, 221 + vpshufd zmm6, zmm6, 0x93 + vpshufd zmm7, zmm7, 0x93 + mov al, 7 +9: + vpaddd zmm0, zmm0, zmm4 + vpaddd zmm0, zmm0, zmm1 + vpxord zmm3, zmm3, zmm0 + vprord zmm3, zmm3, 16 + vpaddd zmm2, zmm2, zmm3 + vpxord zmm1, zmm1, zmm2 + vprord zmm1, zmm1, 12 + vpaddd zmm0, zmm0, zmm5 + vpaddd zmm0, zmm0, zmm1 + vpxord zmm3, zmm3, zmm0 + vprord zmm3, zmm3, 8 + vpaddd zmm2, zmm2, zmm3 + vpxord zmm1, zmm1, zmm2 + vprord zmm1, zmm1, 7 + vpshufd zmm0, zmm0, 0x93 + vpshufd zmm3, zmm3, 0x4E + vpshufd zmm2, zmm2, 0x39 + vpaddd zmm0, zmm0, zmm6 + vpaddd zmm0, zmm0, zmm1 + vpxord zmm3, zmm3, zmm0 + vprord zmm3, zmm3, 16 + vpaddd zmm2, zmm2, zmm3 + vpxord zmm1, zmm1, zmm2 + vprord zmm1, zmm1, 12 + vpaddd zmm0, zmm0, zmm7 + vpaddd zmm0, zmm0, zmm1 + vpxord zmm3, zmm3, zmm0 + vprord zmm3, zmm3, 8 + vpaddd zmm2, zmm2, zmm3 + vpxord zmm1, zmm1, zmm2 + vprord zmm1, zmm1, 7 + vpshufd zmm0, zmm0, 0x39 + vpshufd zmm3, zmm3, 0x4E + vpshufd zmm2, zmm2, 0x93 + dec al + jz 9f + vshufps zmm8, zmm4, zmm5, 214 + vpshufd zmm9, zmm4, 0x0F + vpshufd zmm4, zmm8, 0x39 + vshufps zmm8, zmm6, zmm7, 250 + vpblendmd zmm9 {k3}, zmm9, zmm8 + vpunpcklqdq zmm8, zmm7, zmm5 + vpblendmd zmm8 {k4}, zmm8, zmm6 + vpshufd zmm8, zmm8, 0x78 + vpunpckhdq zmm5, zmm5, zmm7 + vpunpckldq zmm6, zmm6, zmm5 + vpshufd zmm7, zmm6, 0x1E + vmovdqa32 zmm5, zmm9 + vmovdqa32 zmm6, zmm8 + jmp 9b +9: + vpxord zmm0, zmm0, zmm2 + vpxord zmm1, zmm1, zmm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01 + vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01 + vextracti32x4 xmmword ptr [rbx+0x4*0x10], zmm0, 0x02 + vextracti32x4 xmmword ptr [rbx+0x5*0x10], zmm1, 0x02 + vextracti32x4 xmmword ptr [rbx+0x6*0x10], zmm0, 0x03 + vextracti32x4 xmmword ptr [rbx+0x7*0x10], zmm1, 0x03 + vmovdqa xmm0, xmmword ptr [rsp] + vmovdqa xmm2, xmmword ptr [rsp+0x40] + vmovdqa32 xmm0 {k1}, xmmword ptr [rsp+0x1*0x10] + vmovdqa32 xmm2 {k1}, xmmword ptr [rsp+0x5*0x10] + vmovdqa xmmword ptr [rsp], xmm0 + vmovdqa xmmword ptr [rsp+0x40], xmm2 + add rbx, 128 + add rdi, 32 + sub rsi, 4 +3: + test esi, 0x2 + je 3f + vbroadcasti128 ymm0, xmmword ptr [rcx] + vbroadcasti128 ymm1, xmmword ptr [rcx+0x10] + vmovd xmm13, dword ptr [rsp] + vpinsrd xmm13, xmm13, dword ptr [rsp+0x40], 1 + vpinsrd xmm13, xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vmovd xmm14, dword ptr [rsp+0x4] + vpinsrd xmm14, xmm14, dword ptr [rsp+0x44], 1 + vpinsrd xmm14, xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vinserti128 ymm13, ymm13, xmm14, 0x01 + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + mov dword ptr [rsp+0x88], eax + vbroadcasti128 ymm2, xmmword ptr [BLAKE3_IV+rip] + vpbroadcastd ymm8, dword ptr [rsp+0x88] + vpblendd ymm3, ymm13, ymm8, 0x88 + vmovups ymm8, ymmword ptr [r8+rdx-0x40] + vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x40], 0x01 + vmovups ymm9, ymmword ptr [r8+rdx-0x30] + vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x30], 0x01 + vshufps ymm4, ymm8, ymm9, 136 + vshufps ymm5, ymm8, ymm9, 221 + vmovups ymm8, ymmword ptr [r8+rdx-0x20] + vinsertf128 ymm8, ymm8, xmmword ptr [r9+rdx-0x20], 0x01 + vmovups ymm9, ymmword ptr [r8+rdx-0x10] + vinsertf128 ymm9, ymm9, xmmword ptr [r9+rdx-0x10], 0x01 + vshufps ymm6, ymm8, ymm9, 136 + vshufps ymm7, ymm8, ymm9, 221 + vpshufd ymm6, ymm6, 0x93 + vpshufd ymm7, ymm7, 0x93 + mov al, 7 +9: + vpaddd ymm0, ymm0, ymm4 + vpaddd ymm0, ymm0, ymm1 + vpxord ymm3, ymm3, ymm0 + vprord ymm3, ymm3, 16 + vpaddd ymm2, ymm2, ymm3 + vpxord ymm1, ymm1, ymm2 + vprord ymm1, ymm1, 12 + vpaddd ymm0, ymm0, ymm5 + vpaddd ymm0, ymm0, ymm1 + vpxord ymm3, ymm3, ymm0 + vprord ymm3, ymm3, 8 + vpaddd ymm2, ymm2, ymm3 + vpxord ymm1, ymm1, ymm2 + vprord ymm1, ymm1, 7 + vpshufd ymm0, ymm0, 0x93 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm2, ymm2, 0x39 + vpaddd ymm0, ymm0, ymm6 + vpaddd ymm0, ymm0, ymm1 + vpxord ymm3, ymm3, ymm0 + vprord ymm3, ymm3, 16 + vpaddd ymm2, ymm2, ymm3 + vpxord ymm1, ymm1, ymm2 + vprord ymm1, ymm1, 12 + vpaddd ymm0, ymm0, ymm7 + vpaddd ymm0, ymm0, ymm1 + vpxord ymm3, ymm3, ymm0 + vprord ymm3, ymm3, 8 + vpaddd ymm2, ymm2, ymm3 + vpxord ymm1, ymm1, ymm2 + vprord ymm1, ymm1, 7 + vpshufd ymm0, ymm0, 0x39 + vpshufd ymm3, ymm3, 0x4E + vpshufd ymm2, ymm2, 0x93 + dec al + jz 9f + vshufps ymm8, ymm4, ymm5, 214 + vpshufd ymm9, ymm4, 0x0F + vpshufd ymm4, ymm8, 0x39 + vshufps ymm8, ymm6, ymm7, 250 + vpblendd ymm9, ymm9, ymm8, 0xAA + vpunpcklqdq ymm8, ymm7, ymm5 + vpblendd ymm8, ymm8, ymm6, 0x88 + vpshufd ymm8, ymm8, 0x78 + vpunpckhdq ymm5, ymm5, ymm7 + vpunpckldq ymm6, ymm6, ymm5 + vpshufd ymm7, ymm6, 0x1E + vmovdqa ymm5, ymm9 + vmovdqa ymm6, ymm8 + jmp 9b +9: + vpxor ymm0, ymm0, ymm2 + vpxor ymm1, ymm1, ymm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + vextracti128 xmmword ptr [rbx+0x20], ymm0, 0x01 + vextracti128 xmmword ptr [rbx+0x30], ymm1, 0x01 + vmovdqa xmm0, xmmword ptr [rsp] + vmovdqa xmm2, xmmword ptr [rsp+0x4*0x10] + vmovdqu32 xmm0 {k1}, xmmword ptr [rsp+0x8] + vmovdqu32 xmm2 {k1}, xmmword ptr [rsp+0x48] + vmovdqa xmmword ptr [rsp], xmm0 + vmovdqa xmmword ptr [rsp+0x4*0x10], xmm2 + add rbx, 64 + add rdi, 16 + sub rsi, 2 +3: + test esi, 0x1 + je 4b + vmovdqu xmm0, xmmword ptr [rcx] + vmovdqu xmm1, xmmword ptr [rcx+0x10] + vmovd xmm14, dword ptr [rsp] + vpinsrd xmm14, xmm14, dword ptr [rsp+0x40], 1 + vpinsrd xmm14, xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + vmovdqa xmm15, xmmword ptr [BLAKE3_IV+rip] + mov r8, qword ptr [rdi] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +.p2align 5 +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + vpinsrd xmm3, xmm14, eax, 3 + vmovdqa xmm2, xmm15 + vmovups xmm8, xmmword ptr [r8+rdx-0x40] + vmovups xmm9, xmmword ptr [r8+rdx-0x30] + vshufps xmm4, xmm8, xmm9, 136 + vshufps xmm5, xmm8, xmm9, 221 + vmovups xmm8, xmmword ptr [r8+rdx-0x20] + vmovups xmm9, xmmword ptr [r8+rdx-0x10] + vshufps xmm6, xmm8, xmm9, 136 + vshufps xmm7, xmm8, xmm9, 221 + vpshufd xmm6, xmm6, 0x93 + vpshufd xmm7, xmm7, 0x93 + mov al, 7 +9: + vpaddd xmm0, xmm0, xmm4 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm5 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x93 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x39 + vpaddd xmm0, xmm0, xmm6 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm7 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x39 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x93 + dec al + jz 9f + vshufps xmm8, xmm4, xmm5, 214 + vpshufd xmm9, xmm4, 0x0F + vpshufd xmm4, xmm8, 0x39 + vshufps xmm8, xmm6, xmm7, 250 + vpblendd xmm9, xmm9, xmm8, 0xAA + vpunpcklqdq xmm8, xmm7, xmm5 + vpblendd xmm8, xmm8, xmm6, 0x88 + vpshufd xmm8, xmm8, 0x78 + vpunpckhdq xmm5, xmm5, xmm7 + vpunpckldq xmm6, xmm6, xmm5 + vpshufd xmm7, xmm6, 0x1E + vmovdqa xmm5, xmm9 + vmovdqa xmm6, xmm8 + jmp 9b +9: + vpxor xmm0, xmm0, xmm2 + vpxor xmm1, xmm1, xmm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + vmovdqu xmmword ptr [rbx], xmm0 + vmovdqu xmmword ptr [rbx+0x10], xmm1 + jmp 4b +.p2align 6 +_blake3_compress_in_place_avx512_asm: +blake3_compress_in_place_avx512_asm: + _CET_ENDBR + vmovdqu xmm0, xmmword ptr [rdi] + vmovdqu xmm1, xmmword ptr [rdi+0x10] + movzx eax, r8b + movzx edx, dl + shl rax, 32 + add rdx, rax + vmovq xmm3, rcx + vmovq xmm4, rdx + vpunpcklqdq xmm3, xmm3, xmm4 + vmovaps xmm2, xmmword ptr [BLAKE3_IV+rip] + vmovups xmm8, xmmword ptr [rsi] + vmovups xmm9, xmmword ptr [rsi+0x10] + vshufps xmm4, xmm8, xmm9, 136 + vshufps xmm5, xmm8, xmm9, 221 + vmovups xmm8, xmmword ptr [rsi+0x20] + vmovups xmm9, xmmword ptr [rsi+0x30] + vshufps xmm6, xmm8, xmm9, 136 + vshufps xmm7, xmm8, xmm9, 221 + vpshufd xmm6, xmm6, 0x93 + vpshufd xmm7, xmm7, 0x93 + mov al, 7 +9: + vpaddd xmm0, xmm0, xmm4 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm5 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x93 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x39 + vpaddd xmm0, xmm0, xmm6 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm7 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x39 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x93 + dec al + jz 9f + vshufps xmm8, xmm4, xmm5, 214 + vpshufd xmm9, xmm4, 0x0F + vpshufd xmm4, xmm8, 0x39 + vshufps xmm8, xmm6, xmm7, 250 + vpblendd xmm9, xmm9, xmm8, 0xAA + vpunpcklqdq xmm8, xmm7, xmm5 + vpblendd xmm8, xmm8, xmm6, 0x88 + vpshufd xmm8, xmm8, 0x78 + vpunpckhdq xmm5, xmm5, xmm7 + vpunpckldq xmm6, xmm6, xmm5 + vpshufd xmm7, xmm6, 0x1E + vmovdqa xmm5, xmm9 + vmovdqa xmm6, xmm8 + jmp 9b +9: + vpxor xmm0, xmm0, xmm2 + vpxor xmm1, xmm1, xmm3 + vmovdqu xmmword ptr [rdi], xmm0 + vmovdqu xmmword ptr [rdi+0x10], xmm1 + ret + +.p2align 6 +_blake3_compress_xof_avx512_asm: +blake3_compress_xof_avx512_asm: + _CET_ENDBR + vmovdqu xmm0, xmmword ptr [rdi] + vmovdqu xmm1, xmmword ptr [rdi+0x10] + movzx eax, r8b + movzx edx, dl + shl rax, 32 + add rdx, rax + vmovq xmm3, rcx + vmovq xmm4, rdx + vpunpcklqdq xmm3, xmm3, xmm4 + vmovaps xmm2, xmmword ptr [BLAKE3_IV+rip] + vmovups xmm8, xmmword ptr [rsi] + vmovups xmm9, xmmword ptr [rsi+0x10] + vshufps xmm4, xmm8, xmm9, 136 + vshufps xmm5, xmm8, xmm9, 221 + vmovups xmm8, xmmword ptr [rsi+0x20] + vmovups xmm9, xmmword ptr [rsi+0x30] + vshufps xmm6, xmm8, xmm9, 136 + vshufps xmm7, xmm8, xmm9, 221 + vpshufd xmm6, xmm6, 0x93 + vpshufd xmm7, xmm7, 0x93 + mov al, 7 +9: + vpaddd xmm0, xmm0, xmm4 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm5 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x93 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x39 + vpaddd xmm0, xmm0, xmm6 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 16 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 12 + vpaddd xmm0, xmm0, xmm7 + vpaddd xmm0, xmm0, xmm1 + vpxord xmm3, xmm3, xmm0 + vprord xmm3, xmm3, 8 + vpaddd xmm2, xmm2, xmm3 + vpxord xmm1, xmm1, xmm2 + vprord xmm1, xmm1, 7 + vpshufd xmm0, xmm0, 0x39 + vpshufd xmm3, xmm3, 0x4E + vpshufd xmm2, xmm2, 0x93 + dec al + jz 9f + vshufps xmm8, xmm4, xmm5, 214 + vpshufd xmm9, xmm4, 0x0F + vpshufd xmm4, xmm8, 0x39 + vshufps xmm8, xmm6, xmm7, 250 + vpblendd xmm9, xmm9, xmm8, 0xAA + vpunpcklqdq xmm8, xmm7, xmm5 + vpblendd xmm8, xmm8, xmm6, 0x88 + vpshufd xmm8, xmm8, 0x78 + vpunpckhdq xmm5, xmm5, xmm7 + vpunpckldq xmm6, xmm6, xmm5 + vpshufd xmm7, xmm6, 0x1E + vmovdqa xmm5, xmm9 + vmovdqa xmm6, xmm8 + jmp 9b +9: + vpxor xmm0, xmm0, xmm2 + vpxor xmm1, xmm1, xmm3 + vpxor xmm2, xmm2, [rdi] + vpxor xmm3, xmm3, [rdi+0x10] + vmovdqu xmmword ptr [r9], xmm0 + vmovdqu xmmword ptr [r9+0x10], xmm1 + vmovdqu xmmword ptr [r9+0x20], xmm2 + vmovdqu xmmword ptr [r9+0x30], xmm3 + ret + +#ifdef __APPLE__ +.static_data +#else +.section .rodata +#endif +.p2align 6 +INDEX0: + .long 0, 1, 2, 3, 16, 17, 18, 19 + .long 8, 9, 10, 11, 24, 25, 26, 27 +INDEX1: + .long 4, 5, 6, 7, 20, 21, 22, 23 + .long 12, 13, 14, 15, 28, 29, 30, 31 +ADD0: + .long 0, 1, 2, 3, 4, 5, 6, 7 + .long 8, 9, 10, 11, 12, 13, 14, 15 +ADD1: .long 1 + +ADD16: .long 16 +BLAKE3_BLOCK_LEN: + .long 64 +.p2align 6 +BLAKE3_IV: +BLAKE3_IV_0: + .long 0x6A09E667 +BLAKE3_IV_1: + .long 0xBB67AE85 +BLAKE3_IV_2: + .long 0x3C6EF372 +BLAKE3_IV_3: + .long 0xA54FF53A diff --git a/src/blake3/blake3_backend.c b/src/blake3/blake3_backend.c new file mode 100644 index 0000000..4f00dd6 --- /dev/null +++ b/src/blake3/blake3_backend.c @@ -0,0 +1,588 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + +#include +#include + +#include "fy-bit64.h" + +#include "blake3.h" +#include "blake3_impl.h" +#include "blake3_internal.h" + +// Declarations for implementation-specific functions. +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +/* the portable hasher (can drive the optimized ones, but not optimally) */ +extern const blake3_hasher_ops blake3_hasher_op_portable; + +#if defined(IS_X86) +void blake3_compress_in_place_sse2(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse2(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +void blake3_compress_in_place_sse2_asm(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse2_asm(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse2_asm(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +extern const blake3_hasher_ops blake3_hasher_op_sse2; + +void blake3_compress_in_place_sse41(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse41(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +void blake3_compress_in_place_sse41_asm(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_sse41_asm(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_sse41_asm(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +extern const blake3_hasher_ops blake3_hasher_op_sse41; + +void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +void blake3_hash_many_avx2_asm(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +extern const blake3_hasher_ops blake3_hasher_op_avx2; + +void blake3_compress_in_place_avx512(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_avx512(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); +void blake3_compress_in_place_avx512_asm(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); +void blake3_compress_xof_avx512_asm(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]); +void blake3_hash_many_avx512_asm(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +extern const blake3_hasher_ops blake3_hasher_op_avx512; + +#endif + +#if defined(IS_ARM) +void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out); + +extern const blake3_hasher_ops blake3_hasher_op_neon; +#endif + +static uint64_t supported_cpusimd_backend(void) +{ + const blake3_backend *be = &blake3_backends[B3BID_CPUSIMD]; + return be->user ? B3BF_CPUSIMD : 0; +} + +static uint64_t detected_cpusimd_backend(void) +{ + const blake3_backend *be = &blake3_backends[B3BID_CPUSIMD]; + return be->user ? B3BF_CPUSIMD : 0; +} + +static uint64_t supported_synthetic_backends(void) +{ + uint64_t backends = 0; + + backends |= supported_cpusimd_backend(); + return backends; +} + +static uint64_t detected_synthetic_backends(void) +{ + uint64_t backends = 0; + + backends |= detected_cpusimd_backend(); + return backends; +} + +static uint64_t supported_gpu_backends(void) +{ + return 0; +} + +static uint64_t detected_gpu_backends(void) +{ + return 0; +} + +#if defined(IS_X86) + +static uint64_t supported_backends_x86(void) +{ + uint64_t backends = 0; + +#if !defined(BLAKE3_NO_SSE2) + backends |= B3BF_SSE2 | B3BF_SSE2_ASM; +#endif +#if !defined(BLAKE3_NO_SSE41) + backends |= B3BF_SSE41 | B3BF_SSE41_ASM; +#endif +#if !defined(BLAKE3_NO_AVX) + backends |= B3BF_AVX2 | B3BF_AVX2_ASM; +#endif +#if !defined(BLAKE3_NO_AVX512) + backends |= B3BF_AVX512 | B3BF_AVX512_ASM; +#endif + return backends; +} + +static inline uint64_t xgetbv(void) +{ +#if defined(_MSC_VER) + return _xgetbv(0); +#else + uint32_t eax = 0, edx = 0; + __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0)); + return ((uint64_t)edx << 32) | eax; +#endif +} + +static inline void cpuid(uint32_t out[4], uint32_t id) +{ +#if defined(_MSC_VER) + __cpuid((int *)out, id); +#elif defined(__i386__) || defined(_M_IX86) + __asm__ __volatile__( + "movl %%ebx, %1\n" + "cpuid\n" + "xchgl %1, %%ebx\n" + : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id)); +#else + __asm__ __volatile__("cpuid\n" + : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id)); +#endif +} + +static inline void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) +{ +#if defined(_MSC_VER) + __cpuidex((int *)out, id, sid); +#elif defined(__i386__) || defined(_M_IX86) + __asm__ __volatile__( + "movl %%ebx, %1\n" + "cpuid\n" + "xchgl %1, %%ebx\n" + : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id), "c"(sid)); +#else + __asm__ __volatile__("cpuid\n" + : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) + : "a"(id), "c"(sid)); +#endif +} + +static uint64_t detected_backends_x86(void) +{ + uint32_t regs[4] = {0}; + uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3]; + int max_id; + uint64_t mask = 0; + uint64_t backends = 0; + + (void)edx; + + cpuid(regs, 0); + max_id = *eax; + cpuid(regs, 1); + +#if defined(__amd64__) || defined(_M_X64) + backends |= B3BF_SSE2 | B3BF_SSE2_ASM; +#else + if (*edx & (1UL << 26)) + backends |= B3BF_SSE2 | B3BF_SSE2_ASM; +#endif + if (*ecx & (1UL << 19)) + backends |= B3BF_SSE41 | B3BF_SSE41_ASM; + + if (*ecx & (1UL << 27)) { // OSXSAVE + mask = xgetbv(); + if ((mask & 6) == 6) { // SSE and AVX states + if (max_id >= 7) { + cpuidex(regs, 7, 0); + if (*ebx & (1UL << 5)) + backends |= B3BF_AVX2 | B3BF_AVX2_ASM; + if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm + if ((*ebx & ((1UL << 31) | (1UL << 16))) == ((1UL << 31) | (1UL << 16))) + backends |= B3BF_AVX512 | B3BF_AVX512_ASM; /* AVX412VL+AVX512F */ + } + } + } + } + + return backends; +} +#endif + +#if defined(IS_ARM) +static uint64_t supported_backends_arm(void) +{ + uint64_t backends = 0; + +#if defined(IS_AARCH64) + backends |= B3BF_NEON; +#endif + + return backends; +} + +static uint64_t detected_backends_arm(void) +{ + uint64_t backends = 0; + +#if defined(IS_AARCH64) + backends |= B3BF_NEON; +#endif + + return backends; +} +#endif + +uint64_t blake3_get_supported_backends(void) +{ + uint64_t supported_backends; + + supported_backends = B3BF_PORTABLE; + + supported_backends |= supported_gpu_backends(); + +#if defined(IS_X86) + supported_backends |= supported_backends_x86(); +#endif + +#if defined(IS_ARM) + supported_backends |= supported_backends_arm(); +#endif + + supported_backends |= supported_synthetic_backends(); + + return supported_backends; +} + +uint64_t blake3_get_detected_backends(void) +{ + uint64_t detected_backends; + + detected_backends = B3BF_PORTABLE; + + detected_backends |= detected_gpu_backends(); + +#if defined(IS_X86) + detected_backends |= detected_backends_x86(); +#endif + +#if defined(IS_ARM) + detected_backends |= detected_backends_arm(); +#endif + + detected_backends |= detected_synthetic_backends(); + + return detected_backends; +} + +uint64_t blake3_get_selectable_backends(void) +{ + return blake3_get_supported_backends() | + blake3_get_detected_backends(); +} + +blake3_backend blake3_backends[B3BID_COUNT] = { + [B3BID_PORTABLE] = { + .info = { + .id = B3BID_PORTABLE, + .name = "portable", + .description = "portable C implementation", + .simd_degree = 1, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_portable, + .hash_many = blake3_hash_many_portable, + .compress_xof = blake3_compress_xof_portable, + .compress_in_place = blake3_compress_in_place_portable, + }, +#if defined(IS_X86) + +#if defined(TARGET_HAS_SSE2) && TARGET_HAS_SSE2 + [B3BID_SSE2] = { + .info = { + .id = B3BID_SSE2, + .name = "sse2", + .description = "x86 SSE2 implementation in C", + .simd_degree = 4, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_sse2, + .hash_many = blake3_hash_many_sse2, + .compress_xof = blake3_compress_xof_sse2, + .compress_in_place = blake3_compress_in_place_sse2, + }, + [B3BID_SSE2_ASM] = { + .info = { + .id = B3BID_SSE2_ASM, + .name = "sse2-asm", + .description = "x86 SSE2 implementation in assembly", + .simd_degree = 4, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_sse2, + .hash_many = blake3_hash_many_sse2_asm, + .compress_xof = blake3_compress_xof_sse2_asm, + .compress_in_place = blake3_compress_in_place_sse2_asm, + }, +#endif + +#if defined(TARGET_HAS_SSE41) && TARGET_HAS_SSE41 + [B3BID_SSE41] = { + .info = { + .id = B3BID_SSE41, + .name = "sse41", + .description = "x86 SSE41 implementation in C", + .simd_degree = 4, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_sse41, + .hash_many = blake3_hash_many_sse41, + .compress_xof = blake3_compress_xof_sse41, + .compress_in_place = blake3_compress_in_place_sse41, + }, + [B3BID_SSE41_ASM] = { + .info = { + .id = B3BID_SSE41_ASM, + .name = "sse41-asm", + .description = "x86 SSE41 implementation in assembly", + .simd_degree = 4, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_sse41, + .hash_many = blake3_hash_many_sse41_asm, + .compress_xof = blake3_compress_xof_sse41_asm, + .compress_in_place = blake3_compress_in_place_sse41_asm, + }, +#endif + +#if defined(TARGET_HAS_AVX2) && TARGET_HAS_AVX2 + [B3BID_AVX2] = { + .info = { + .id = B3BID_AVX2, + .name = "avx2", + .description = "x86 AVX2 implementation in C", + .simd_degree = 8, + .funcs = B3FF_HASH_MANY, + }, + .hasher_ops = &blake3_hasher_op_avx2, + .hash_many = blake3_hash_many_avx2, + }, + [B3BID_AVX2_ASM] = { + .info = { + .id = B3BID_AVX2_ASM, + .name = "avx2-asm", + .description = "x86 AVX2 implementation in assembly", + .simd_degree = 8, + .funcs = B3FF_HASH_MANY, + }, + .hasher_ops = &blake3_hasher_op_avx2, + .hash_many = blake3_hash_many_avx2_asm, + }, +#endif +#if defined(TARGET_HAS_AVX512) && TARGET_HAS_AVX512 + [B3BID_AVX512] = { + .info = { + .id = B3BID_AVX512, + .name = "avx512", + .description = "x86 AVX512 VL+F implementation in C", + .simd_degree = 16, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_avx512, + .hash_many = blake3_hash_many_avx512, + .compress_xof = blake3_compress_xof_avx512, + .compress_in_place = blake3_compress_in_place_avx512, + }, + [B3BID_AVX512_ASM] = { + .info = { + .id = B3BID_AVX512_ASM, + .name = "avx512-asm", + .description = "x86 AVX512 VL+F implementation in assembly", + .simd_degree = 16, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_avx512, + .hash_many = blake3_hash_many_avx512, + .compress_xof = blake3_compress_xof_avx512_asm, + .compress_in_place = blake3_compress_in_place_avx512_asm, + }, +#endif + +#endif + +#if defined(IS_ARM) + +#if defined(TARGET_HAS_NEON) && TARGET_HAS_NEON + [B3BID_NEON] = { + .info = { + .id = B3BID_NEON, + .name = "neon", + .description = "arm NEON implementation", + .simd_degree = 4, + .funcs = B3FF_HASH_MANY | B3FF_COMPRESS_XOF | B3FF_COMPRESS_IN_PLACE, + }, + .hasher_ops = &blake3_hasher_op_neon, + .hash_many = blake3_hash_many_neon, + .compress_xof = blake3_compress_xof_portable, /* no NEON for this */ + .compress_in_place = blake3_compress_in_place_portable, + }, +#endif + +#endif +}; + +const blake3_backend *blake3_backend_select_function(uint64_t selectable_backends, blake3_func_id fid) +{ + const blake3_backend *be; + unsigned int i; + + while (selectable_backends) { + i = highest_one(selectable_backends); + selectable_backends &= ~FY_BIT64(i); + be = blake3_backends + i; + if (be->info.funcs & FY_BIT64(fid)) + return be; + } + /* should never happen because portable is always set */ + assert(false); + return NULL; +} + +const blake3_backend *blake3_get_backend_by_id(blake3_backend_id id) +{ + const blake3_backend *be; + + if ((unsigned int)id >= B3BID_COUNT) + return NULL; + + be = &blake3_backends[id]; + + /* non-enabled backend? */ + if (be->info.id != id || !be->info.name) + return NULL; + + return be; +} + +const blake3_backend *blake3_get_backend_by_name(const char *name) +{ + const blake3_backend *be; + unsigned int i; + + if (!name) + return NULL; + + for (i = 0, be = blake3_backends; i < B3BID_COUNT; i++, be++) { + if (be->info.name && !strcmp(be->info.name, name)) + return be; + } + return NULL; +} + +const blake3_backend_info *blake3_get_backend_info(blake3_backend_id id) +{ + const blake3_backend_info *bei; + + if ((unsigned int)id >= B3BID_COUNT) + return NULL; + + bei = &blake3_backends[id].info; + + /* not supported */ + if (bei->id == B3BID_INVALID || !bei->name) + return NULL; + + return bei; +} diff --git a/src/blake3/blake3_be_cpusimd.c b/src/blake3/blake3_be_cpusimd.c new file mode 100644 index 0000000..8a00951 --- /dev/null +++ b/src/blake3/blake3_be_cpusimd.c @@ -0,0 +1,243 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + +#include +#include + +#include "blake3.h" +#include "blake3_impl.h" +#include "blake3_internal.h" + +#include "fy-thread.h" + +struct cpusimd_data { + unsigned int num_cpus; + unsigned int simd_cpus; + unsigned int be_simd_degree_mult; + const blake3_backend *be_best; + char *description; + struct fy_thread_pool *tp; +}; + +static void blake3_cpusimd_hash_many_thread(void *arg) +{ + blake3_hash_many_state *s = arg; + const blake3_hash_many_common_state *c; + + c = s->common; + c->hash_many(s->inputs, s->num_inputs, c->blocks, c->key, s->counter, c->increment_counter, c->flags, c->flags_start, c->flags_end, s->out); +} + +// #define CPUSIMD_CHECK + +static void blake3_hash_many_cpusimd( + const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) +{ + struct cpusimd_data *d; + const blake3_backend *be, *be_best; + unsigned int i, stepi, be_simd_degree_mult, simd_degree, count, states_count; + blake3_hash_many_state *states, *s; + blake3_hash_many_common_state common_local, *common = &common_local; + const uint8_t *const *inputsi; + uint8_t *outi; + uint64_t counteri; +#ifdef CPUSIMD_CHECK + size_t out_size; + uint8_t *out_cmp; +#endif + + (void)simd_degree; + + be = &blake3_backends[B3BID_CPUSIMD]; + d = be->user; + assert(d); + + count = (unsigned int)num_inputs; + simd_degree = be->info.simd_degree; + + be_best = d->be_best; + be_simd_degree_mult = d->be_simd_degree_mult; + +#ifdef CPUSIMD_CHECK + /* if less than the underlyind backend degree just do it there */ + out_size = BLAKE3_OUT_LEN * num_inputs; + out_cmp = alloca(out_size); + be_best->hash_many(inputs, num_inputs, blocks, key, counter, increment_counter, flags, flags_start, flags_end, out_cmp); +#endif + + states_count = (count / be_simd_degree_mult) + !!(count % be_simd_degree_mult); + states = alloca(sizeof(*states) * states_count); + + /* the common */ + common->hash_many = be_best->hash_many; + common->blocks = blocks; + common->key = key; + common->increment_counter = increment_counter; + common->flags = flags; + common->flags_start = flags_start; + common->flags_end = flags_end; + + inputsi = inputs; + outi = out; + counteri = counter; + for (i = 0, s = states; i < count; i += stepi, s++) { + + stepi = (i + be_simd_degree_mult) <= count ? be_simd_degree_mult : (count - i); + + s->inputs = inputsi; + s->num_inputs = stepi; + s->counter = counteri; + s->out = outi; + s->common = common; + + if (increment_counter) + counteri += stepi; + inputsi += stepi; + outi += BLAKE3_OUT_LEN * stepi; + } + + fy_thread_arg_array_join(d->tp, blake3_cpusimd_hash_many_thread, NULL, states, sizeof(states[0]), states_count); + +#ifdef CPUSIMD_CHECK + for (i = 0; i < count; i += stepi) { + stepi = (i + be_simd_degree_mult) <= count ? be_simd_degree_mult : (count - i); + + if (memcmp(out + i * BLAKE3_OUT_LEN, out_cmp + i * BLAKE3_OUT_LEN, BLAKE3_OUT_LEN)) { + fprintf(stderr, "%s: differ at #%u\n", __func__, i); + } + } +#endif +} + +static void cpusimd_data_free(struct cpusimd_data *d) +{ + if (!d) + return; + + if (d->description) + free(d->description); + + if (d->tp) + fy_thread_pool_destroy(d->tp); + + free(d); +} + +int blake3_backend_cpusimd_setup(unsigned int num_cpus, unsigned int mult_fact) +{ + blake3_backend *be = &blake3_backends[B3BID_CPUSIMD]; + const blake3_backend *be_best; + struct cpusimd_data *d = NULL; + struct fy_thread_pool_cfg tp_cfg; + long scval; + uint64_t supported_backends, detected_backends, selectable_backends; + unsigned int num_simd_cpus; + ssize_t n; + + if (!num_cpus) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } + + if (num_cpus <= 1) + return 0; + + num_simd_cpus = round_down_to_power_of_2(num_cpus); + + if (!mult_fact) + mult_fact = 1; + + d = malloc(sizeof(*d)); + if (!d) + goto err_out; + + memset(d, 0, sizeof(*d)); + + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = 0; + tp_cfg.num_threads = num_simd_cpus; + tp_cfg.userdata = NULL; + + d->tp = fy_thread_pool_create(&tp_cfg); + if (!d->tp) + goto err_out; + + /* probe for available backends */ + supported_backends = blake3_get_supported_backends(); + detected_backends = blake3_get_detected_backends(); + selectable_backends = supported_backends & detected_backends; + + /* remove ourselves (and anything above) */ + selectable_backends &= B3BF_BIT(B3BID_CPUSIMD) - 1; + + /* select the best one for hash many */ + be_best = blake3_backend_select_function(selectable_backends, B3FID_HASH_MANY); + assert(be_best); + + d->num_cpus = num_cpus; + d->simd_cpus = num_simd_cpus; + d->be_best = be_best; + d->be_simd_degree_mult = be_best->info.simd_degree * mult_fact; + + be->hasher_ops = be_best->hasher_ops; + + d->description = NULL; + n = 0; + for (;;) { + n = snprintf(d->description, n ? (n + 1) : 0, "SIMD like acceleration using %u CPUs (using %s x %u) x %u = total x %u", + d->simd_cpus, + d->be_best->info.name, d->be_best->info.simd_degree, + mult_fact, + d->be_simd_degree_mult * d->simd_cpus); + if (d->description) + break; + + d->description = malloc(n + 1); + if (!d->description) + goto err_out; + } + + be->user = d; + + be->info.id = B3BID_CPUSIMD; + be->info.name = "cpusimd"; + be->info.simd_degree = d->simd_cpus * d->be_best->info.simd_degree * mult_fact; + be->info.description = d->description; + + be->hash_many = blake3_hash_many_cpusimd; + be->info.funcs = B3FF_HASH_MANY; + + return 0; + +err_out: + cpusimd_data_free(d); + return -1; +} + +void blake3_backend_cpusimd_cleanup(void) +{ + blake3_backend *be = &blake3_backends[B3BID_CPUSIMD]; + struct cpusimd_data *d; + + d = be->user; + if (!d) + return; + cpusimd_data_free(d); + + memset(be, 0, sizeof(*be)); +} diff --git a/src/blake3/blake3_host_state.c b/src/blake3/blake3_host_state.c new file mode 100644 index 0000000..4699fa3 --- /dev/null +++ b/src/blake3/blake3_host_state.c @@ -0,0 +1,573 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fy-bit64.h" + +#include "blake3.h" +#include "blake3_impl.h" +#include "blake3_internal.h" + +#include "fy-thread.h" + +/* 256K threshold for using alloca */ +#define BLAKE3_ALLOCA_BUFFER_SIZE (256U << 10) +#define BLAKE3_FILE_IO_BUFFER_SIZE BLAKE3_ALLOCA_BUFFER_SIZE +#define BLAKE3_MMAP_MIN_CHUNKSIZE (1U << 20) // minimum chunksize is 1MB +#define BLAKE3_MMAP_MAX_CHUNKSIZE SIZE_MAX + +static int probe_backends(blake3_host_state *hs) +{ + hs->supported_backends = blake3_get_supported_backends(); + hs->detected_backends = blake3_get_detected_backends(); + hs->selectable_backends = hs->supported_backends & hs->detected_backends; + + return 0; +} + +static int select_backends(blake3_host_state *hs) +{ + const blake3_backend *be, *be_portable; + + /* this is the fallback */ + be_portable = blake3_get_backend_by_id(B3BID_PORTABLE); + assert(be_portable); + + /* if there's a backend selected, try to force it */ + if (hs->cfg.backend && hs->cfg.backend[0] && strcmp(hs->cfg.backend, "auto")) { + be = blake3_get_backend_by_name(hs->cfg.backend); + + /* if backend exists and is selectable */ + if (be && (hs->selectable_backends & FY_BIT64(be->info.id))) { + hs->hash_many_be = (be->info.funcs & B3FF_HASH_MANY) ? be : be_portable; + hs->compress_xof_be = (be->info.funcs & B3FF_COMPRESS_XOF) ? be : be_portable; + hs->compress_in_place_be = (be->info.funcs & B3FF_COMPRESS_IN_PLACE) ? be : be_portable; + + /* and select the ops (piggyback on HASH_MANY) */ + hs->hasher_ops = (be->info.funcs & B3FF_HASH_MANY) ? be->hasher_ops : be_portable->hasher_ops; + } + } + + if (!hs->hash_many_be) { + hs->hash_many_be = blake3_backend_select_function(hs->selectable_backends, B3FID_HASH_MANY); + hs->hasher_ops = hs->hash_many_be->hasher_ops; + } + if (!hs->compress_xof_be) + hs->compress_xof_be = blake3_backend_select_function(hs->selectable_backends, B3FID_COMPRESS_XOF); + if (!hs->compress_in_place_be) + hs->compress_in_place_be = blake3_backend_select_function(hs->selectable_backends, B3FID_COMPRESS_IN_PLACE); + + /* select methods */ + hs->hash_many = hs->hash_many_be->hash_many; + hs->compress_xof = hs->compress_xof_be->compress_xof; + hs->compress_in_place = hs->compress_in_place_be->compress_in_place; + + /* select maximum simd degree */ + if (hs->simd_degree < hs->hash_many_be->info.simd_degree) + hs->simd_degree = hs->hash_many_be->info.simd_degree; + if (hs->simd_degree < hs->compress_xof_be->info.simd_degree) + hs->simd_degree = hs->compress_xof_be->info.simd_degree; + if (hs->simd_degree < hs->compress_in_place_be->info.simd_degree) + hs->simd_degree = hs->compress_in_place_be->info.simd_degree; + + return 0; +} + +static void dump_backends(blake3_host_state *hs, uint64_t backends) +{ + const blake3_backend *be; + int i; + + for (i = 0; i < B3BID_COUNT; i++) { + if (!(backends & FY_BIT64(i))) + continue; + be = blake3_get_backend_by_id(i); + assert(be); + + fprintf(stderr, "%sname: %s\n%sdescription: %s\n%ssimd_degree: %u\n%shas_hash_many: %s\n%shas_compress_xof: %s\n%shas_compress_in_place: %s\n", + " -", be->info.name, + " ", be->info.description, + " ", be->info.simd_degree, + " ", (be->info.funcs & B3FF_HASH_MANY) ? "true" : "false", + " ", (be->info.funcs & B3FF_COMPRESS_XOF) ? "true" : "false", + " ", (be->info.funcs & B3FF_COMPRESS_IN_PLACE) ? "true" : "false"); + } +} + +int blake3_host_state_setup(blake3_host_state *hs, const blake3_host_config *cfg) +{ + struct fy_thread_pool_cfg tp_cfg; + long scval; + int rc; + + (void)rc; + + assert(hs); + assert(cfg); + + memset(hs, 0, sizeof(*hs)); + hs->cfg = *cfg; + + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + + hs->num_cpus = (unsigned int)scval; + + rc = probe_backends(hs); + assert(!rc); + + rc = select_backends(hs); + assert(!rc); + + hs->mt_degree = hs->cfg.mt_degree > 0 ? hs->cfg.mt_degree : 64; /* 64K default */ + hs->tp = NULL; + + if (!hs->cfg.no_mthread) { + + if (!hs->cfg.tp) { + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = FYTPCF_STEAL_MODE; + tp_cfg.num_threads = hs->cfg.num_threads ? hs->cfg.num_threads : (hs->num_cpus * 3) / 2; + tp_cfg.userdata = NULL; + hs->tp = fy_thread_pool_create(&tp_cfg); + if (!hs->tp) + goto err_out; + } else + hs->tp = hs->cfg.tp; + } + + hs->file_io_bufsz = hs->cfg.file_io_bufsz ? hs->cfg.file_io_bufsz : BLAKE3_FILE_IO_BUFFER_SIZE; + hs->mmap_min_chunk = hs->cfg.mmap_min_chunk ? hs->cfg.mmap_min_chunk : BLAKE3_MMAP_MIN_CHUNKSIZE; + hs->mmap_max_chunk = hs->cfg.mmap_max_chunk ? hs->cfg.mmap_max_chunk : BLAKE3_MMAP_MAX_CHUNKSIZE; + + if (hs->cfg.debug) { + + fprintf(stderr, "num_cpus: %u\n", hs->num_cpus); + fprintf(stderr, "num_threads: %d\n", hs->tp ? fy_thread_pool_get_num_threads(hs->tp) : 0); + fprintf(stderr, "simd_degree: %u\n", hs->simd_degree); + fprintf(stderr, "mt_degree: %u\n", hs->mt_degree); + fprintf(stderr, "file_io_bufsz: %zu\n", hs->file_io_bufsz); + fprintf(stderr, "mmap_min_chunk: %zu\n", hs->mmap_min_chunk); + fprintf(stderr, "mmap_max_chunk: %zu\n", hs->mmap_max_chunk); + + fprintf(stderr, "supported_backends:\n"); + dump_backends(hs, hs->supported_backends); + fprintf(stderr, "detected_backends:\n"); + dump_backends(hs, hs->detected_backends); + + fprintf(stderr, "selected-backends:\n"); + fprintf(stderr, " hash_many: %s\n", hs->hash_many_be->info.name); + fprintf(stderr, " compress_xof: %s\n", hs->compress_xof_be->info.name); + fprintf(stderr, " compress_in_place: %s\n", hs->compress_in_place_be->info.name); + + } + + return 0; + +err_out: + return -1; +} + +void blake3_host_state_cleanup(blake3_host_state *hs) +{ + assert(hs); + + /* destroy the thread pool if we're the ones created it */ + if (hs->tp && !hs->cfg.tp) + fy_thread_pool_destroy(hs->tp); +} + +blake3_host_state *blake3_host_state_create(const blake3_host_config *cfg) +{ + blake3_host_state *hs; + int rc; + + hs = malloc(sizeof(*hs)); + if (!hs) + return NULL; + rc = blake3_host_state_setup(hs, cfg); + if (rc) + goto err_out; + + return hs; + +err_out: + if (hs) + free(hs); + return NULL; +} + +void blake3_host_state_destroy(blake3_host_state *hs) +{ + if (!hs) + return; + blake3_host_state_cleanup(hs); + free(hs); +} + +blake3_hasher *blake3_hasher_create(blake3_host_state *hs, const uint8_t *key, const void *context, size_t context_len) +{ + blake3_hasher *self; + + self = fy_cacheline_alloc(sizeof(*self)); + if (!self) + return NULL; + + if (!key && !context) + blake3_hasher_init(hs, self); + else if (key) + blake3_hasher_init_keyed(hs, self, key); + else if (context_len == 0) + blake3_hasher_init_derive_key(hs, self, context); + else + blake3_hasher_init_derive_key_raw(hs, self, context, context_len); + return self; +} + +void blake3_hasher_destroy(blake3_hasher *self) +{ + if (!self) + return; + fy_cacheline_free(self); +} + +#if defined(__linux__) + +static int linux_block_dev_is_rotational(dev_t dev) +{ + char *path = NULL; + int rc, fd; + uint8_t c[2]; + int rotational; + + /* by default it's the filesize */ + rotational = -1; /* not found */ + + /* read the rotational attribute of the root */ + rc = asprintf(&path, "/sys/dev/block/%u:%u/queue/rotational", major(dev), 0); + if (rc != -1) { + fd = open(path, O_RDONLY); + if (fd != -1) { + if (read(fd, c, 2) == 2) + rotational = c[0] == '1' && c[1] == '\n'; + close(fd); + } + free(path); + } + + return rotational; +} + +static ssize_t linux_file_cached_size(int fd, void *mem, size_t filesize) +{ + long pagesize = sysconf(_SC_PAGESIZE); + size_t vec_size, resident_size, i; + unsigned char *vec; + int rc; + + (void)rc; + vec_size = (filesize + pagesize - 1) / pagesize; + vec = malloc(vec_size + 1); + if (!vec) + return -1; + + rc = mincore(mem, filesize, vec); + assert(!rc); + + /* this could be parallelized */ + resident_size = 0; + for (i = 0; i < vec_size; i++) { + if (vec[i] & 1) + resident_size += pagesize; + } + + free(vec); + + if (resident_size > filesize) + resident_size = filesize; + + return (ssize_t)resident_size; +} + +static size_t blake3_mmap_file_chunksize(int fd, dev_t dev, void *mem, + size_t filesize, + size_t mmap_min_chunk, size_t mmap_max_chunk) +{ + size_t chunksize; + ssize_t cached_size; + int rotational; + + if (filesize <= mmap_min_chunk) + return filesize; + + /* by default it's the filesize */ + chunksize = filesize; + + /* starting, check if the rotational attribute exists in this dev */ + rotational = linux_block_dev_is_rotational(dev); + if (rotational == -1) { + /* attribute not found? check the non-partition */ + rotational = linux_block_dev_is_rotational(makedev(major(dev), 0)); + } + + if (rotational == 1) { + /* OK, it's rotational, but is it in cache? + * to avoid checking for the cached status of the whole file + * we just probe the MIN_CHUNKSIZE + * We will thrash in the case where the file is only cached + * for the first few bytes, but this is generally unusual. + */ + cached_size = linux_file_cached_size(fd, mem, mmap_min_chunk); + + if (cached_size <= 0) + chunksize = mmap_min_chunk; + } + + if (chunksize > mmap_max_chunk) + chunksize = mmap_max_chunk; + + return chunksize; +} + +#else + +static size_t blake3_mmap_file_chunksize(int fd, dev_t dev, void *mem, + size_t filesize, + size_t mmap_min_chunk, size_t mmap_max_chunk) +{ + if (filesize <= mmap_min_chunk) + return filesize; + + if (filesize > mmap_max_chunk) + return mmap_max_chunk; + + /* for all others, just use mmap at max */ + return filesize; +} + +#endif + +int blake3_hash_file(blake3_hasher *hasher, const char *filename, + uint8_t output[BLAKE3_OUT_LEN]) +{ + blake3_host_state *hs; + FILE *fp = NULL; + void *mem = NULL, *buf = NULL, *p; + int fd = -1, ret = -1; + size_t rdn, filesize, bufsz = 0, max_chunk, left, chunk; + struct stat sb; + dev_t dev = 0; + int rc; + + if (!hasher || !filename || !output) + return -1; + + hs = hasher->hs; + + if (hs->cfg.debug) + fprintf(stderr, "processing file %s\n", filename); + + // reset the hasher (do not initialize again) + blake3_hasher_reset(hasher); + + if (!strcmp(filename, "-")) { + fp = stdin; + } else { + fp = NULL; + + fd = open(filename, O_RDONLY); + if (fd < 0) { + if (hs->cfg.debug) + fprintf(stderr, "unable to open %s - %s\n", + filename, strerror(errno)); + goto err_out; + } + + rc = fstat(fd, &sb); + if (rc < 0) { + if (hs->cfg.debug) + fprintf(stderr, "failed to stat %s - %s\n", + filename, strerror(errno)); + goto err_out; + } + + if (!S_ISREG(sb.st_mode)) { + if (S_ISDIR(sb.st_mode)) + errno = EISDIR; + else + errno = EINVAL; + if (hs->cfg.debug) + fprintf(stderr, "not a regular file %s - %s\n", + filename, strerror(errno)); + goto err_out; + } + + filesize = (size_t)-1; + + /* try to mmap */ + if (sb.st_size > 0 && !hs->cfg.no_mmap) { + filesize = sb.st_size; + mem = mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (mem != MAP_FAILED) { + close(fd); + fd = -1; + dev = sb.st_dev; + } else + mem = NULL; + } + + /* unable to map? fallback to stream mode */ + if (!mem) { + fp = fdopen(fd, "r"); + if (!fp) { + if (hs->cfg.debug) + fprintf(stderr, "unable to fdopen %s - %s\n", + filename, strerror(errno)); + goto err_out; + } + } + } + + /* mmap case, very simple */ + if (mem) { + + /* if the file is small enough don't bother with the rest */ + max_chunk = blake3_mmap_file_chunksize(fd, dev, mem, filesize, + hs->mmap_min_chunk, hs->mmap_max_chunk); + + p = mem; + left = filesize; + while (left > 0) { + chunk = left > max_chunk ? max_chunk : left; + blake3_hasher_update(hasher, p, chunk); + p += chunk; + left -= chunk; + } + } else { + /* slow path using file reads */ + + assert(fp); + + bufsz = hs->file_io_bufsz; + + /* for less than 8MB alloca */ + if (bufsz <= BLAKE3_ALLOCA_BUFFER_SIZE) { + buf = alloca(bufsz); + } else { + buf = malloc(bufsz); + if (!buf) { + if (hs->cfg.debug) + fprintf(stderr, "Unable to allocate buffer of %zu bytes\n", + bufsz); + goto err_out; + } + } + + do { + rdn = fread(buf, 1, bufsz, fp); + if (rdn == 0) + break; + if (rdn > 0) + blake3_hasher_update(hasher, buf, rdn); + } while (rdn >= bufsz); + } + + // Finalize the hash. BLAKE3_OUT_LEN is the default output length, 32 bytes. + blake3_hasher_finalize(hasher, output, BLAKE3_OUT_LEN); + + ret = 0; + +out: + if (mem) + munmap(mem, filesize); + + if (fp && fp != stdin) + fclose(fp); + + if (fd >= 0) + close(fd); + + if (buf && (bufsz > BLAKE3_ALLOCA_BUFFER_SIZE)) + free(buf); + + return ret; + +err_out: + ret = -1; + goto out; +} + +void blake3_hash(struct blake3_hasher *hasher, + const void *mem, size_t size, uint8_t output[BLAKE3_OUT_LEN]) +{ + blake3_hasher_reset(hasher); + blake3_hasher_update(hasher, mem, size); + blake3_hasher_finalize(hasher, output, BLAKE3_OUT_LEN); +} + +const char *blake3_version(void) +{ + return BLAKE3_VERSION_STRING; +} + +void blake3_hasher_init(struct blake3_host_state *hs, struct blake3_hasher *self) +{ + hs->hasher_ops->hasher_init(hs, self); +} + +void blake3_hasher_init_keyed(struct blake3_host_state *hs, struct blake3_hasher *self, const uint8_t key[BLAKE3_KEY_LEN]) +{ + hs->hasher_ops->hasher_init_keyed(hs, self, key); +} + +void blake3_hasher_init_derive_key(struct blake3_host_state *hs, struct blake3_hasher *self, const char *context) +{ + hs->hasher_ops->hasher_init_derive_key(hs, self, context); +} + +void blake3_hasher_init_derive_key_raw(struct blake3_host_state *hs, struct blake3_hasher *self, const void *context, size_t context_len) +{ + hs->hasher_ops->hasher_init_derive_key_raw(hs, self, context, context_len); +} + +void blake3_hasher_update(struct blake3_hasher *self, const void *input, size_t input_len) +{ + blake3_host_state *hs = self->hs; + hs->hasher_ops->hasher_update(self, input, input_len); +} + +void blake3_hasher_finalize(const struct blake3_hasher *self, uint8_t *out, size_t out_len) +{ + blake3_host_state *hs = self->hs; + hs->hasher_ops->hasher_finalize(self, out, out_len); +} + +void blake3_hasher_finalize_seek(const struct blake3_hasher *self, uint64_t seek, uint8_t *out, size_t out_len) +{ + blake3_host_state *hs = self->hs; + hs->hasher_ops->hasher_finalize_seek(self, seek, out, out_len); +} + +void blake3_hasher_reset(struct blake3_hasher *self) +{ + blake3_host_state *hs = self->hs; + hs->hasher_ops->hasher_reset(self); +} diff --git a/src/blake3/blake3_impl.h b/src/blake3/blake3_impl.h new file mode 100644 index 0000000..230dc49 --- /dev/null +++ b/src/blake3/blake3_impl.h @@ -0,0 +1,234 @@ +#ifndef BLAKE3_IMPL_H +#define BLAKE3_IMPL_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#if defined(__linux__) +#include +#endif +#endif + +#if defined(__linux__) +#include +#endif + +#include "blake3.h" + +#include "fy-endian.h" +#include "fy-blob.h" +#include "fy-bit64.h" +#include "fy-align.h" + +#if defined(HAVE_STRING_OP_OVERREAD) && HAVE_STRING_OP_OVERREAD && defined(__GNUC__) && !defined(__clang__) +#define GCC_DISABLE_WSTRING_OP_OVERREAD +#endif + +/* this is the best alignment for blake3 */ +#if defined(__x86_64__) || defined(_M_X64) +#define BLAKE3_ALIGNMENT 64 +#else +#define BLAKE3_ALIGNMENT 32 +#endif + +#define BLAKE3_ALIGN FY_ALIGNED_TO(BLAKE3_ALIGNMENT) + +// This C implementation tries to support recent versions of GCC, Clang, and +// MSVC. +#if defined(_MSC_VER) +#define INLINE static __forceinline +#else +#define INLINE static inline __attribute__((always_inline)) +#endif + +#if defined(__x86_64__) || defined(_M_X64) +#define IS_X86 +#define IS_X86_64 +#endif + +#if defined(__i386__) || defined(_M_IX86) +#define IS_X86 +#define IS_X86_32 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#define IS_AARCH64 +#define IS_ARM +#endif + +#if defined(IS_X86) +#if defined(_MSC_VER) +#include +#endif +#endif + +/* Find index of the highest set bit */ +/* x is assumed to be nonzero. */ +static inline unsigned int highest_one(uint64_t x) +{ +#if defined(__GNUC__) || defined(__clang__) + return 63 ^ (unsigned int)__builtin_clzll(x); +#elif defined(_MSC_VER) && defined(IS_X86_64) + unsigned long index; + _BitScanReverse64(&index, x); + return index; +#elif defined(_MSC_VER) && defined(IS_X86_32) + unsigned long index; + + if(x >> 32) { + _BitScanReverse(&index, (unsigned long)(x >> 32)); + index += 32; + } else + _BitScanReverse(&index, (unsigned long)x); + return index; +#else + unsigned int c = 0; + + if (x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } + if (x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } + if (x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } + if (x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } + if (x & 0x000000000000000cULL) { x >>= 2; c += 2; } + if (x & 0x0000000000000002ULL) { c += 1; } + return c; +#endif +} + +static inline unsigned int lowest_one(uint64_t x) +{ +#if defined(__GNUC__) || defined(__clang__) + return (unsigned int)__builtin_ctzll(x); +#else + unsigned int c = 0; + + if (!(x & 0x00000000ffffffffULL)) { x >>= 32; c += 32; } + if (!(x & 0x000000000000ffffULL)) { x >>= 16; c += 16; } + if (!(x & 0x00000000000000ffULL)) { x >>= 8; c += 8; } + if (!(x & 0x000000000000000fULL)) { x >>= 4; c += 4; } + if (!(x & 0x0000000000000003ULL)) { x >>= 2; c += 2; } + if (!(x & 0x0000000000000001ULL)) { c += 1; } + return c; +#endif +} + +// Count the number of 1 bits. +static inline unsigned int popcnt(uint64_t x) +{ +#if defined(__GNUC__) || defined(__clang__) + return (unsigned int)__builtin_popcountll(x); +#else + unsigned int count = 0; + while (x != 0) { + count++; + x &= x - 1; + } + return count; +#endif +} + +// Largest power of two less than or equal to x. As a special case, returns 1 +// when x is 0. +static inline uint64_t round_down_to_power_of_2(uint64_t x) +{ + return 1ULL << highest_one(x | 1); +} + +static inline uint32_t counter_low(uint64_t counter) +{ + return (uint32_t)counter; +} + +static inline uint32_t counter_high(uint64_t counter) +{ + return (uint32_t)(counter >> 32); +} + +static inline uint32_t load32(const void *src) +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + return *(const uint32_t *)src; +#else + const uint8_t *p = (const uint8_t *)src; + return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | + ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); +#endif +} + +static inline void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], uint32_t key_words[8]) +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + memcpy(key_words, key, BLAKE3_KEY_LEN); +#else + key_words[0] = load32(&key[0 * 4]); + key_words[1] = load32(&key[1 * 4]); + key_words[2] = load32(&key[2 * 4]); + key_words[3] = load32(&key[3 * 4]); + key_words[4] = load32(&key[4 * 4]); + key_words[5] = load32(&key[5 * 4]); + key_words[6] = load32(&key[6 * 4]); + key_words[7] = load32(&key[7 * 4]); +#endif +} + +static inline void store32(void *dst, uint32_t w) +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + *(uint32_t *)dst = w; +#else + uint8_t *p = (uint8_t *)dst; + + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); +#endif +} + +static inline void store_cv_words(uint8_t bytes_out[BLAKE3_OUT_LEN], uint32_t cv_words[BLAKE3_OUT_WORDS]) +{ +#if __BYTE_ORDER == __LITTLE_ENDIAN + memcpy(bytes_out, cv_words, BLAKE3_KEY_LEN); +#else + store32(&bytes_out[0 * 4], cv_words[0]); + store32(&bytes_out[1 * 4], cv_words[1]); + store32(&bytes_out[2 * 4], cv_words[2]); + store32(&bytes_out[3 * 4], cv_words[3]); + store32(&bytes_out[4 * 4], cv_words[4]); + store32(&bytes_out[5 * 4], cv_words[5]); + store32(&bytes_out[6 * 4], cv_words[6]); + store32(&bytes_out[7 * 4], cv_words[7]); +#endif +} + +/* the IV */ +#define B3_IV_0 ((uint32_t)0x6A09E667UL) +#define B3_IV_1 ((uint32_t)0xBB67AE85UL) +#define B3_IV_2 ((uint32_t)0x3C6EF372UL) +#define B3_IV_3 ((uint32_t)0xA54FF53AUL) +#define B3_IV_4 ((uint32_t)0x510E527FUL) +#define B3_IV_5 ((uint32_t)0x9B05688CUL) +#define B3_IV_6 ((uint32_t)0x1F83D9ABUL) +#define B3_IV_7 ((uint32_t)0x5BE0CD19UL) + +/* the message schedule definition */ +#define B3_MSG_SCHEDULE_DEF \ + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, \ + {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, \ + {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, \ + {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, \ + {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, \ + {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, \ + {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13} + +#endif /* BLAKE3_IMPL_H */ diff --git a/src/blake3/blake3_internal.h b/src/blake3/blake3_internal.h new file mode 100644 index 0000000..2bfa42e --- /dev/null +++ b/src/blake3/blake3_internal.h @@ -0,0 +1,158 @@ +#ifndef BLAKE3_INTERNAL_H +#define BLAKE3_INTERNAL_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "blake3.h" + +#include "blake3_impl.h" + +#define BLAKE3_MAX_DEPTH 54 + +// fwd declaration of opaque types +struct blake3_host_state; +struct blake3_hasher; +struct fy_thread_pool; + +// internal flags +enum blake3_flags { + CHUNK_START = 1 << 0, + CHUNK_END = 1 << 1, + PARENT = 1 << 2, + ROOT = 1 << 3, + KEYED_HASH = 1 << 4, + DERIVE_KEY_CONTEXT = 1 << 5, + DERIVE_KEY_MATERIAL = 1 << 6, +}; + +typedef struct blake3_chunk_state { + uint32_t cv[8] BLAKE3_ALIGN; + uint8_t buf[BLAKE3_BLOCK_LEN] BLAKE3_ALIGN; + uint64_t chunk_counter; + uint8_t buf_len; + uint8_t blocks_compressed; + uint8_t flags; +} blake3_chunk_state BLAKE3_ALIGN; + +typedef struct blake3_hasher { + struct blake3_host_state *hs; + uint32_t key[8] BLAKE3_ALIGN; + blake3_chunk_state chunk BLAKE3_ALIGN; + // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example, + // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk + // requires a 4th entry, rather than merging everything down to 1, because we + // don't know whether more input is coming. This is different from how the + // reference implementation does things. + uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN] BLAKE3_ALIGN; + uint8_t cv_stack_len; +} blake3_hasher BLAKE3_ALIGN; + +typedef void (*blake3_hash_many_f)(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out); + +typedef void (*blake3_compress_xof_f)(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags, + uint8_t out[64]); + +typedef void (*blake3_compress_in_place_f)(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +typedef struct blake3_backend { + const blake3_hasher_ops *hasher_ops; + blake3_backend_info info; + blake3_hash_many_f hash_many; + blake3_compress_xof_f compress_xof; + blake3_compress_in_place_f compress_in_place; + void *user; /* per backend data */ +} blake3_backend; + +// the state for the thread when doing compress subtree +typedef struct blake3_compress_subtree_state { + struct blake3_hasher *self; + // inputs + const uint8_t *input; + size_t input_len; + const uint32_t *key; + uint64_t chunk_counter; + uint8_t flags; + // outputs + uint8_t *out; + size_t n; +} blake3_compress_subtree_state; + +typedef struct blake3_hash_many_common_state { + blake3_hash_many_f hash_many; + size_t blocks; + const uint32_t *key; + bool increment_counter; + uint8_t flags, flags_start, flags_end; +} blake3_hash_many_common_state; + +typedef struct blake3_hash_many_state { + const blake3_hash_many_common_state *common; + const uint8_t *const *inputs; + size_t num_inputs; + uint64_t counter; + uint8_t *out; +} blake3_hash_many_state; + +typedef struct blake3_host_state { + blake3_host_config cfg; + unsigned int num_cpus; + uint64_t supported_backends; + uint64_t detected_backends; + uint64_t selectable_backends; + + /* backend for hash_many */ + const blake3_backend *hash_many_be; + blake3_hash_many_f hash_many; + + /* backend for compress_xof */ + const blake3_backend *compress_xof_be; + blake3_compress_xof_f compress_xof; + + /* backend for compress_in_place */ + const blake3_backend *compress_in_place_be; + blake3_compress_in_place_f compress_in_place; + + const blake3_hasher_ops *hasher_ops; + + unsigned int simd_degree; + unsigned int mt_degree; + + unsigned int num_threads; + + struct fy_thread_pool *tp; + + size_t file_io_bufsz; + size_t mmap_min_chunk; + size_t mmap_max_chunk; + +} blake3_host_state; + +int blake3_host_state_setup(blake3_host_state *hs, const blake3_host_config *cfg); +void blake3_host_state_cleanup(blake3_host_state *hs); + +extern blake3_backend blake3_backends[B3BID_COUNT]; + +const blake3_backend *blake3_backend_select_function(uint64_t selectable_backends, blake3_func_id fid); +const blake3_backend *blake3_get_backend_by_id(blake3_backend_id id); +const blake3_backend *blake3_get_backend_by_name(const char *name); +const blake3_backend_info *blake3_get_backend_info(blake3_backend_id id); + +void blake3_cpusimd_setup(unsigned int num_cpus, unsigned int mult_fact); +void blake3_cpusimd_cleanup(void); + +#endif diff --git a/src/blake3/blake3_neon.c b/src/blake3/blake3_neon.c new file mode 100644 index 0000000..e7caba3 --- /dev/null +++ b/src/blake3/blake3_neon.c @@ -0,0 +1,375 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" + +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +#ifdef __ARM_BIG_ENDIAN +#error "This implementation only supports little-endian ARM." +// It might be that all we need for big-endian support here is to get the loads +// and stores right, but step zero would be finding a way to test it in CI. +#endif + +INLINE uint32x4_t loadu_128(const uint8_t src[16]) { + // vld1q_u32 has alignment requirements. Don't use it. + uint32x4_t x; + memcpy(&x, src, 16); + return x; +} + +INLINE void storeu_128(uint32x4_t src, uint8_t dest[16]) { + // vst1q_u32 has alignment requirements. Don't use it. + memcpy(dest, &src, 16); +} + +INLINE uint32x4_t add_128(uint32x4_t a, uint32x4_t b) { + return vaddq_u32(a, b); +} + +INLINE uint32x4_t xor_128(uint32x4_t a, uint32x4_t b) { + return veorq_u32(a, b); +} + +INLINE uint32x4_t set1_128(uint32_t x) { return vld1q_dup_u32(&x); } + +INLINE uint32x4_t set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + uint32_t array[4] = {a, b, c, d}; + return vld1q_u32(array); +} + +INLINE uint32x4_t rot16_128(uint32x4_t x) { + // The straightfoward implementation would be two shifts and an or, but that's + // slower on microarchitectures we've tested. See + // https://github.com/BLAKE3-team/BLAKE3/pull/319. + // return vorrq_u32(vshrq_n_u32(x, 16), vshlq_n_u32(x, 32 - 16)); + return vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x))); +} + +INLINE uint32x4_t rot12_128(uint32x4_t x) { + // See comment in rot16_128. + // return vorrq_u32(vshrq_n_u32(x, 12), vshlq_n_u32(x, 32 - 12)); + return vsriq_n_u32(vshlq_n_u32(x, 32-12), x, 12); +} + +INLINE uint32x4_t rot8_128(uint32x4_t x) { + // See comment in rot16_128. + // return vorrq_u32(vshrq_n_u32(x, 8), vshlq_n_u32(x, 32 - 8)); +#if defined(__clang__) + return vreinterpretq_u32_u8(__builtin_shufflevector(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12)); +#elif __GNUC__ * 10000 + __GNUC_MINOR__ * 100 >=40700 + static const uint8x16_t r8 = {1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12}; + return vreinterpretq_u32_u8(__builtin_shuffle(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), r8)); +#else + return vsriq_n_u32(vshlq_n_u32(x, 32-8), x, 8); +#endif +} + +INLINE uint32x4_t rot7_128(uint32x4_t x) { + // See comment in rot16_128. + // return vorrq_u32(vshrq_n_u32(x, 7), vshlq_n_u32(x, 32 - 7)); + return vsriq_n_u32(vshlq_n_u32(x, 32-7), x, 7); +} + +// TODO: compress_neon + +// TODO: hash2_neon + +/* + * ---------------------------------------------------------------------------- + * hash4_neon + * ---------------------------------------------------------------------------- + */ + +INLINE void round_fn4(uint32x4_t v[16], uint32x4_t m[16], size_t r) { + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[15] = rot16_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot12_128(v[4]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = add_128(v[0], v[4]); + v[1] = add_128(v[1], v[5]); + v[2] = add_128(v[2], v[6]); + v[3] = add_128(v[3], v[7]); + v[12] = xor_128(v[12], v[0]); + v[13] = xor_128(v[13], v[1]); + v[14] = xor_128(v[14], v[2]); + v[15] = xor_128(v[15], v[3]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[15] = rot8_128(v[15]); + v[8] = add_128(v[8], v[12]); + v[9] = add_128(v[9], v[13]); + v[10] = add_128(v[10], v[14]); + v[11] = add_128(v[11], v[15]); + v[4] = xor_128(v[4], v[8]); + v[5] = xor_128(v[5], v[9]); + v[6] = xor_128(v[6], v[10]); + v[7] = xor_128(v[7], v[11]); + v[4] = rot7_128(v[4]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot16_128(v[15]); + v[12] = rot16_128(v[12]); + v[13] = rot16_128(v[13]); + v[14] = rot16_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot12_128(v[5]); + v[6] = rot12_128(v[6]); + v[7] = rot12_128(v[7]); + v[4] = rot12_128(v[4]); + v[0] = add_128(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = add_128(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = add_128(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = add_128(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = add_128(v[0], v[5]); + v[1] = add_128(v[1], v[6]); + v[2] = add_128(v[2], v[7]); + v[3] = add_128(v[3], v[4]); + v[15] = xor_128(v[15], v[0]); + v[12] = xor_128(v[12], v[1]); + v[13] = xor_128(v[13], v[2]); + v[14] = xor_128(v[14], v[3]); + v[15] = rot8_128(v[15]); + v[12] = rot8_128(v[12]); + v[13] = rot8_128(v[13]); + v[14] = rot8_128(v[14]); + v[10] = add_128(v[10], v[15]); + v[11] = add_128(v[11], v[12]); + v[8] = add_128(v[8], v[13]); + v[9] = add_128(v[9], v[14]); + v[5] = xor_128(v[5], v[10]); + v[6] = xor_128(v[6], v[11]); + v[7] = xor_128(v[7], v[8]); + v[4] = xor_128(v[4], v[9]); + v[5] = rot7_128(v[5]); + v[6] = rot7_128(v[6]); + v[7] = rot7_128(v[7]); + v[4] = rot7_128(v[4]); +} + +INLINE void transpose_vecs_128(uint32x4_t vecs[4]) { + // Individually transpose the four 2x2 sub-matrices in each corner. + uint32x4x2_t rows01 = vtrnq_u32(vecs[0], vecs[1]); + uint32x4x2_t rows23 = vtrnq_u32(vecs[2], vecs[3]); + + // Swap the top-right and bottom-left 2x2s (which just got transposed). + vecs[0] = + vcombine_u32(vget_low_u32(rows01.val[0]), vget_low_u32(rows23.val[0])); + vecs[1] = + vcombine_u32(vget_low_u32(rows01.val[1]), vget_low_u32(rows23.val[1])); + vecs[2] = + vcombine_u32(vget_high_u32(rows01.val[0]), vget_high_u32(rows23.val[0])); + vecs[3] = + vcombine_u32(vget_high_u32(rows01.val[1]), vget_high_u32(rows23.val[1])); +} + +INLINE void transpose_msg_vecs4(const uint8_t *const *inputs, + size_t block_offset, uint32x4_t out[16]) { + out[0] = loadu_128(&inputs[0][block_offset + 0 * sizeof(uint32x4_t)]); + out[1] = loadu_128(&inputs[1][block_offset + 0 * sizeof(uint32x4_t)]); + out[2] = loadu_128(&inputs[2][block_offset + 0 * sizeof(uint32x4_t)]); + out[3] = loadu_128(&inputs[3][block_offset + 0 * sizeof(uint32x4_t)]); + out[4] = loadu_128(&inputs[0][block_offset + 1 * sizeof(uint32x4_t)]); + out[5] = loadu_128(&inputs[1][block_offset + 1 * sizeof(uint32x4_t)]); + out[6] = loadu_128(&inputs[2][block_offset + 1 * sizeof(uint32x4_t)]); + out[7] = loadu_128(&inputs[3][block_offset + 1 * sizeof(uint32x4_t)]); + out[8] = loadu_128(&inputs[0][block_offset + 2 * sizeof(uint32x4_t)]); + out[9] = loadu_128(&inputs[1][block_offset + 2 * sizeof(uint32x4_t)]); + out[10] = loadu_128(&inputs[2][block_offset + 2 * sizeof(uint32x4_t)]); + out[11] = loadu_128(&inputs[3][block_offset + 2 * sizeof(uint32x4_t)]); + out[12] = loadu_128(&inputs[0][block_offset + 3 * sizeof(uint32x4_t)]); + out[13] = loadu_128(&inputs[1][block_offset + 3 * sizeof(uint32x4_t)]); + out[14] = loadu_128(&inputs[2][block_offset + 3 * sizeof(uint32x4_t)]); + out[15] = loadu_128(&inputs[3][block_offset + 3 * sizeof(uint32x4_t)]); + transpose_vecs_128(&out[0]); + transpose_vecs_128(&out[4]); + transpose_vecs_128(&out[8]); + transpose_vecs_128(&out[12]); +} + +INLINE void load_counters4(uint64_t counter, bool increment_counter, + uint32x4_t *out_low, uint32x4_t *out_high) { + uint64_t mask = (increment_counter ? ~0 : 0); + *out_low = set4( + counter_low(counter + (mask & 0)), counter_low(counter + (mask & 1)), + counter_low(counter + (mask & 2)), counter_low(counter + (mask & 3))); + *out_high = set4( + counter_high(counter + (mask & 0)), counter_high(counter + (mask & 1)), + counter_high(counter + (mask & 2)), counter_high(counter + (mask & 3))); +} + +void blake3_hash4_neon(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + uint32x4_t h_vecs[8] = { + set1_128(key[0]), set1_128(key[1]), set1_128(key[2]), set1_128(key[3]), + set1_128(key[4]), set1_128(key[5]), set1_128(key[6]), set1_128(key[7]), + }; + uint32x4_t counter_low_vec, counter_high_vec; + load_counters4(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + uint32x4_t block_len_vec = set1_128(BLAKE3_BLOCK_LEN); + uint32x4_t block_flags_vec = set1_128(block_flags); + uint32x4_t msg_vecs[16]; + transpose_msg_vecs4(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + uint32x4_t v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1_128(B3_IV_0), set1_128(B3_IV_1), set1_128(B3_IV_2), set1_128(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn4(v, msg_vecs, 0); + round_fn4(v, msg_vecs, 1); + round_fn4(v, msg_vecs, 2); + round_fn4(v, msg_vecs, 3); + round_fn4(v, msg_vecs, 4); + round_fn4(v, msg_vecs, 5); + round_fn4(v, msg_vecs, 6); + h_vecs[0] = xor_128(v[0], v[8]); + h_vecs[1] = xor_128(v[1], v[9]); + h_vecs[2] = xor_128(v[2], v[10]); + h_vecs[3] = xor_128(v[3], v[11]); + h_vecs[4] = xor_128(v[4], v[12]); + h_vecs[5] = xor_128(v[5], v[13]); + h_vecs[6] = xor_128(v[6], v[14]); + h_vecs[7] = xor_128(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs_128(&h_vecs[0]); + transpose_vecs_128(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu_128(h_vecs[0], &out[0 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[4], &out[1 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[1], &out[2 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[5], &out[3 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[2], &out[4 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[6], &out[5 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[3], &out[6 * sizeof(uint32x4_t)]); + storeu_128(h_vecs[7], &out[7 * sizeof(uint32x4_t)]); +} + +/* + * ---------------------------------------------------------------------------- + * hash_many_neon + * ---------------------------------------------------------------------------- + */ + +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags); + +INLINE void hash_one_neon(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, uint8_t flags_end, + uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + // TODO: Implement compress_neon. However note that according to + // https://github.com/BLAKE2/BLAKE2/commit/7965d3e6e1b4193438b8d3a656787587d2579227, + // compress_neon might not be any faster than compress_portable. + blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= 4) { + blake3_hash4_neon(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += 4; + } + inputs += 4; + num_inputs -= 4; + out = &out[4 * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_neon(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/src/blake3/blake3_portable.c b/src/blake3/blake3_portable.c new file mode 100644 index 0000000..bd07d07 --- /dev/null +++ b/src/blake3/blake3_portable.c @@ -0,0 +1,167 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +INLINE uint32_t rotr32(uint32_t w, uint32_t c) { + return (w >> c) | (w << (32 - c)); +} + +INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, + uint32_t x, uint32_t y) { + state[a] = state[a] + state[b] + x; + state[d] = rotr32(state[d] ^ state[a], 16); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 12); + state[a] = state[a] + state[b] + y; + state[d] = rotr32(state[d] ^ state[a], 8); + state[c] = state[c] + state[d]; + state[b] = rotr32(state[b] ^ state[c], 7); +} + +INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { + // Select the message schedule based on the round. + const uint8_t *schedule = MSG_SCHEDULE[round]; + + // Mix the columns. + g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); + g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); + g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); + g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); + + // Mix the rows. + g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); + g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); + g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); + g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); +} + +INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + uint32_t block_words[16]; + block_words[0] = load32(block + 4 * 0); + block_words[1] = load32(block + 4 * 1); + block_words[2] = load32(block + 4 * 2); + block_words[3] = load32(block + 4 * 3); + block_words[4] = load32(block + 4 * 4); + block_words[5] = load32(block + 4 * 5); + block_words[6] = load32(block + 4 * 6); + block_words[7] = load32(block + 4 * 7); + block_words[8] = load32(block + 4 * 8); + block_words[9] = load32(block + 4 * 9); + block_words[10] = load32(block + 4 * 10); + block_words[11] = load32(block + 4 * 11); + block_words[12] = load32(block + 4 * 12); + block_words[13] = load32(block + 4 * 13); + block_words[14] = load32(block + 4 * 14); + block_words[15] = load32(block + 4 * 15); + + state[0] = cv[0]; + state[1] = cv[1]; + state[2] = cv[2]; + state[3] = cv[3]; + state[4] = cv[4]; + state[5] = cv[5]; + state[6] = cv[6]; + state[7] = cv[7]; + state[8] = B3_IV_0; + state[9] = B3_IV_1; + state[10] = B3_IV_2; + state[11] = B3_IV_3; + state[12] = counter_low(counter); + state[13] = counter_high(counter); + state[14] = (uint32_t)block_len; + state[15] = (uint32_t)flags; + + round_fn(state, &block_words[0], 0); + round_fn(state, &block_words[0], 1); + round_fn(state, &block_words[0], 2); + round_fn(state, &block_words[0], 3); + round_fn(state, &block_words[0], 4); + round_fn(state, &block_words[0], 5); + round_fn(state, &block_words[0], 6); +} + +void blake3_compress_in_place_portable(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + cv[0] = state[0] ^ state[8]; + cv[1] = state[1] ^ state[9]; + cv[2] = state[2] ^ state[10]; + cv[3] = state[3] ^ state[11]; + cv[4] = state[4] ^ state[12]; + cv[5] = state[5] ^ state[13]; + cv[6] = state[6] ^ state[14]; + cv[7] = state[7] ^ state[15]; +} + +void blake3_compress_xof_portable(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + uint32_t state[16]; + compress_pre(state, cv, block, block_len, counter, flags); + + store32(&out[0 * 4], state[0] ^ state[8]); + store32(&out[1 * 4], state[1] ^ state[9]); + store32(&out[2 * 4], state[2] ^ state[10]); + store32(&out[3 * 4], state[3] ^ state[11]); + store32(&out[4 * 4], state[4] ^ state[12]); + store32(&out[5 * 4], state[5] ^ state[13]); + store32(&out[6 * 4], state[6] ^ state[14]); + store32(&out[7 * 4], state[7] ^ state[15]); + store32(&out[8 * 4], state[8] ^ cv[0]); + store32(&out[9 * 4], state[9] ^ cv[1]); + store32(&out[10 * 4], state[10] ^ cv[2]); + store32(&out[11 * 4], state[11] ^ cv[3]); + store32(&out[12 * 4], state[12] ^ cv[4]); + store32(&out[13 * 4], state[13] ^ cv[5]); + store32(&out[14 * 4], state[14] ^ cv[6]); + store32(&out[15 * 4], state[15] ^ cv[7]); +} + +INLINE void hash_one_portable(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + store_cv_words(out, cv); +} + +void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs > 0) { + hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/src/blake3/blake3_sse2.c b/src/blake3/blake3_sse2.c new file mode 100644 index 0000000..f2bb2e3 --- /dev/null +++ b/src/blake3/blake3_sse2.c @@ -0,0 +1,573 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" + +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +#define DEGREE 4 + +#define _mm_shuffle_ps2(a, b, c) \ + (_mm_castps_si128( \ + _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) + +INLINE __m128i loadu(const uint8_t src[16]) { + return _mm_loadu_si128((const __m128i *)src); +} + +INLINE void storeu(__m128i src, uint8_t dest[16]) { + _mm_storeu_si128((__m128i *)dest, src); +} + +INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } + +// Note that clang-format doesn't like the name "xor" for some reason. +INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } + +INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } + +INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); +} + +INLINE __m128i rot16(__m128i x) { + return _mm_shufflehi_epi16(_mm_shufflelo_epi16(x, 0xB1), 0xB1); +} + +INLINE __m128i rot12(__m128i x) { + return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); +} + +INLINE __m128i rot8(__m128i x) { + return xorv(_mm_srli_epi32(x, 8), _mm_slli_epi32(x, 32 - 8)); +} + +INLINE __m128i rot7(__m128i x) { + return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); +} + +INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot16(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot12(*row1); +} + +INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot8(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot7(*row1); +} + +// Note the optimization here of leaving row1 as the unrotated row, rather than +// row0. All the message loads below are adjusted to compensate for this. See +// discussion at https://github.com/sneves/blake2-avx2/pull/4 +INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); +} + +INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); +} + +INLINE __m128i blend_epi16(__m128i a, __m128i b, const int16_t imm8) { + const __m128i bits = _mm_set_epi16(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01); + __m128i mask = _mm_set1_epi16(imm8); + mask = _mm_and_si128(mask, bits); + mask = _mm_cmpeq_epi16(mask, bits); + return _mm_or_si128(_mm_and_si128(mask, b), _mm_andnot_si128(mask, a)); +} + +INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + rows[0] = loadu((uint8_t *)&cv[0]); + rows[1] = loadu((uint8_t *)&cv[4]); + rows[2] = set4(B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3); + rows[3] = set4(counter_low(counter), counter_high(counter), + (uint32_t)block_len, (uint32_t)flags); + + __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); + __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); + __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); + __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); + + __m128i t0, t1, t2, t3, tt; + + // Round 1. The first round permutes the message words from the original + // input order, into the groups that get mixed in parallel. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 + t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 + t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 2. This round and all following rounds apply a fixed permutation + // to the message words from the round before. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 3 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 4 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 5 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 6 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 7 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); +} + +void blake3_compress_in_place_sse2(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); + storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); +} + +void blake3_compress_xof_sse2(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), &out[0]); + storeu(xorv(rows[1], rows[3]), &out[16]); + storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); + storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); +} + +INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[15] = rot16(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot12(v[4]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[15] = rot8(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot7(v[4]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot16(v[15]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[4] = rot12(v[4]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot8(v[15]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + v[4] = rot7(v[4]); +} + +INLINE void transpose_vecs(__m128i vecs[DEGREE]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is + // 22/33. Note that this doesn't split the vector into two lanes, as the + // AVX2 counterparts do. + __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); + __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); + __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); + __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); + + // Interleave 64-bit lanes. + __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); + __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); + __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); + __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); + + vecs[0] = abcd_0; + vecs[1] = abcd_1; + vecs[2] = abcd_2; + vecs[3] = abcd_3; +} + +INLINE void transpose_msg_vecs(const uint8_t *const *inputs, + size_t block_offset, __m128i out[16]) { + out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); + out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); + out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); + out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); + out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); + out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); + out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); + out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); + out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); + out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); + out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); + out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); + out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); + out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); + out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); + out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); + for (size_t i = 0; i < 4; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs(&out[0]); + transpose_vecs(&out[4]); + transpose_vecs(&out[8]); + transpose_vecs(&out[12]); +} + +INLINE void load_counters(uint64_t counter, bool increment_counter, + __m128i *out_lo, __m128i *out_hi) { + const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); + const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); + const __m128i add1 = _mm_and_si128(mask, add0); + __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); + __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), + _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); + __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); + *out_lo = l; + *out_hi = h; +} + +static +void blake3_hash4_sse2(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m128i h_vecs[8] = { + set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), + set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), + }; + __m128i counter_low_vec, counter_high_vec; + load_counters(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); + __m128i block_flags_vec = set1(block_flags); + __m128i msg_vecs[16]; + transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m128i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1(B3_IV_0), set1(B3_IV_1), set1(B3_IV_2), set1(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn(v, msg_vecs, 0); + round_fn(v, msg_vecs, 1); + round_fn(v, msg_vecs, 2); + round_fn(v, msg_vecs, 3); + round_fn(v, msg_vecs, 4); + round_fn(v, msg_vecs, 5); + round_fn(v, msg_vecs, 6); + h_vecs[0] = xorv(v[0], v[8]); + h_vecs[1] = xorv(v[1], v[9]); + h_vecs[2] = xorv(v[2], v[10]); + h_vecs[3] = xorv(v[3], v[11]); + h_vecs[4] = xorv(v[4], v[12]); + h_vecs[5] = xorv(v[5], v[13]); + h_vecs[6] = xorv(v[6], v[14]); + h_vecs[7] = xorv(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs(&h_vecs[0]); + transpose_vecs(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); + storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); + storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); + storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); + storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); + storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); + storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); + storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); +} + +INLINE void hash_one_sse2(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_sse2(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= DEGREE) { + blake3_hash4_sse2(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += DEGREE; + } + inputs += DEGREE; + num_inputs -= DEGREE; + out = &out[DEGREE * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_sse2(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/src/blake3/blake3_sse2_x86-64_unix.S b/src/blake3/blake3_sse2_x86-64_unix.S new file mode 100644 index 0000000..d2d77bc --- /dev/null +++ b/src/blake3/blake3_sse2_x86-64_unix.S @@ -0,0 +1,2291 @@ +#if defined(__ELF__) && defined(__linux__) +.section .note.GNU-stack,"",%progbits +#endif + +#if defined(__ELF__) && defined(__CET__) && defined(__has_include) +#if __has_include() +#include +#endif +#endif + +#if !defined(_CET_ENDBR) +#define _CET_ENDBR +#endif + +.intel_syntax noprefix +.global blake3_hash_many_sse2_asm +.global _blake3_hash_many_sse2_asm +.global blake3_compress_in_place_sse2_asm +.global _blake3_compress_in_place_sse2_asm +.global blake3_compress_xof_sse2_asm +.global _blake3_compress_xof_sse2_asm +#ifdef __APPLE__ +.text +#else +.section .text +#endif + .p2align 6 +_blake3_hash_many_sse2_asm: +blake3_hash_many_sse2_asm: + _CET_ENDBR + push r15 + push r14 + push r13 + push r12 + push rbx + push rbp + mov rbp, rsp + sub rsp, 360 + and rsp, 0xFFFFFFFFFFFFFFC0 + neg r9d + movd xmm0, r9d + pshufd xmm0, xmm0, 0x00 + movdqa xmmword ptr [rsp+0x130], xmm0 + movdqa xmm1, xmm0 + pand xmm1, xmmword ptr [ADD0+rip] + pand xmm0, xmmword ptr [ADD1+rip] + movdqa xmmword ptr [rsp+0x150], xmm0 + movd xmm0, r8d + pshufd xmm0, xmm0, 0x00 + paddd xmm0, xmm1 + movdqa xmmword ptr [rsp+0x110], xmm0 + pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip] + pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip] + pcmpgtd xmm1, xmm0 + shr r8, 32 + movd xmm2, r8d + pshufd xmm2, xmm2, 0x00 + psubd xmm2, xmm1 + movdqa xmmword ptr [rsp+0x120], xmm2 + mov rbx, qword ptr [rbp+0x50] + mov r15, rdx + shl r15, 6 + movzx r13d, byte ptr [rbp+0x38] + movzx r12d, byte ptr [rbp+0x48] + cmp rsi, 4 + jc 3f +2: + movdqu xmm3, xmmword ptr [rcx] + pshufd xmm0, xmm3, 0x00 + pshufd xmm1, xmm3, 0x55 + pshufd xmm2, xmm3, 0xAA + pshufd xmm3, xmm3, 0xFF + movdqu xmm7, xmmword ptr [rcx+0x10] + pshufd xmm4, xmm7, 0x00 + pshufd xmm5, xmm7, 0x55 + pshufd xmm6, xmm7, 0xAA + pshufd xmm7, xmm7, 0xFF + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +9: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movdqu xmm8, xmmword ptr [r8+rdx-0x40] + movdqu xmm9, xmmword ptr [r9+rdx-0x40] + movdqu xmm10, xmmword ptr [r10+rdx-0x40] + movdqu xmm11, xmmword ptr [r11+rdx-0x40] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp], xmm8 + movdqa xmmword ptr [rsp+0x10], xmm9 + movdqa xmmword ptr [rsp+0x20], xmm12 + movdqa xmmword ptr [rsp+0x30], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x30] + movdqu xmm9, xmmword ptr [r9+rdx-0x30] + movdqu xmm10, xmmword ptr [r10+rdx-0x30] + movdqu xmm11, xmmword ptr [r11+rdx-0x30] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0x40], xmm8 + movdqa xmmword ptr [rsp+0x50], xmm9 + movdqa xmmword ptr [rsp+0x60], xmm12 + movdqa xmmword ptr [rsp+0x70], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x20] + movdqu xmm9, xmmword ptr [r9+rdx-0x20] + movdqu xmm10, xmmword ptr [r10+rdx-0x20] + movdqu xmm11, xmmword ptr [r11+rdx-0x20] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0x80], xmm8 + movdqa xmmword ptr [rsp+0x90], xmm9 + movdqa xmmword ptr [rsp+0xA0], xmm12 + movdqa xmmword ptr [rsp+0xB0], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x10] + movdqu xmm9, xmmword ptr [r9+rdx-0x10] + movdqu xmm10, xmmword ptr [r10+rdx-0x10] + movdqu xmm11, xmmword ptr [r11+rdx-0x10] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0xC0], xmm8 + movdqa xmmword ptr [rsp+0xD0], xmm9 + movdqa xmmword ptr [rsp+0xE0], xmm12 + movdqa xmmword ptr [rsp+0xF0], xmm13 + movdqa xmm9, xmmword ptr [BLAKE3_IV_1+rip] + movdqa xmm10, xmmword ptr [BLAKE3_IV_2+rip] + movdqa xmm11, xmmword ptr [BLAKE3_IV_3+rip] + movdqa xmm12, xmmword ptr [rsp+0x110] + movdqa xmm13, xmmword ptr [rsp+0x120] + movdqa xmm14, xmmword ptr [BLAKE3_BLOCK_LEN+rip] + movd xmm15, eax + pshufd xmm15, xmm15, 0x00 + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + paddd xmm0, xmmword ptr [rsp] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x40] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [BLAKE3_IV_0+rip] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x10] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x50] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x80] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0xC0] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x90] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0xD0] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x20] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x70] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x60] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x10] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x90] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xB0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0xE0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x30] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0xD0] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x40] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x20] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x60] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0xB0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x50] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0xF0] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xA0] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0xE0] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x70] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0x30] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x40] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0x50] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x80] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xC0] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0xF0] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xD0] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0xA0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x70] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x20] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x10] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x90] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0x80] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xE0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0xC0] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xD0] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0x20] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x30] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0x60] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xB0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0x10] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xF0] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0x90] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xE0] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x30] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + pshuflw xmm15, xmm15, 0xB1 + pshufhw xmm15, xmm15, 0xB1 + pshuflw xmm12, xmm12, 0xB1 + pshufhw xmm12, xmm12, 0xB1 + pshuflw xmm13, xmm13, 0xB1 + pshufhw xmm13, xmm13, 0xB1 + pshuflw xmm14, xmm14, 0xB1 + pshufhw xmm14, xmm14, 0xB1 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xA0] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x40] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmm15 + psrld xmm15, 8 + pslld xmm8, 24 + pxor xmm15, xmm8 + movdqa xmm8, xmm12 + psrld xmm12, 8 + pslld xmm8, 24 + pxor xmm12, xmm8 + movdqa xmm8, xmm13 + psrld xmm13, 8 + pslld xmm8, 24 + pxor xmm13, xmm8 + movdqa xmm8, xmm14 + psrld xmm14, 8 + pslld xmm8, 24 + pxor xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + pxor xmm0, xmm8 + pxor xmm1, xmm9 + pxor xmm2, xmm10 + pxor xmm3, xmm11 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + pxor xmm4, xmm12 + pxor xmm5, xmm13 + pxor xmm6, xmm14 + pxor xmm7, xmm15 + mov eax, r13d + jne 9b + movdqa xmm9, xmm0 + punpckldq xmm0, xmm1 + punpckhdq xmm9, xmm1 + movdqa xmm11, xmm2 + punpckldq xmm2, xmm3 + punpckhdq xmm11, xmm3 + movdqa xmm1, xmm0 + punpcklqdq xmm0, xmm2 + punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm9 + punpcklqdq xmm9, xmm11 + punpckhqdq xmm3, xmm11 + movdqu xmmword ptr [rbx], xmm0 + movdqu xmmword ptr [rbx+0x20], xmm1 + movdqu xmmword ptr [rbx+0x40], xmm9 + movdqu xmmword ptr [rbx+0x60], xmm3 + movdqa xmm9, xmm4 + punpckldq xmm4, xmm5 + punpckhdq xmm9, xmm5 + movdqa xmm11, xmm6 + punpckldq xmm6, xmm7 + punpckhdq xmm11, xmm7 + movdqa xmm5, xmm4 + punpcklqdq xmm4, xmm6 + punpckhqdq xmm5, xmm6 + movdqa xmm7, xmm9 + punpcklqdq xmm9, xmm11 + punpckhqdq xmm7, xmm11 + movdqu xmmword ptr [rbx+0x10], xmm4 + movdqu xmmword ptr [rbx+0x30], xmm5 + movdqu xmmword ptr [rbx+0x50], xmm9 + movdqu xmmword ptr [rbx+0x70], xmm7 + movdqa xmm1, xmmword ptr [rsp+0x110] + movdqa xmm0, xmm1 + paddd xmm1, xmmword ptr [rsp+0x150] + movdqa xmmword ptr [rsp+0x110], xmm1 + pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip] + pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip] + pcmpgtd xmm0, xmm1 + movdqa xmm1, xmmword ptr [rsp+0x120] + psubd xmm1, xmm0 + movdqa xmmword ptr [rsp+0x120], xmm1 + add rbx, 128 + add rdi, 32 + sub rsi, 4 + cmp rsi, 4 + jnc 2b + test rsi, rsi + jnz 3f +4: + mov rsp, rbp + pop rbp + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + ret +.p2align 5 +3: + test esi, 0x2 + je 3f + movups xmm0, xmmword ptr [rcx] + movups xmm1, xmmword ptr [rcx+0x10] + movaps xmm8, xmm0 + movaps xmm9, xmm1 + movd xmm13, dword ptr [rsp+0x110] + movd xmm14, dword ptr [rsp+0x120] + punpckldq xmm13, xmm14 + movaps xmmword ptr [rsp], xmm13 + movd xmm14, dword ptr [rsp+0x114] + movd xmm13, dword ptr [rsp+0x124] + punpckldq xmm14, xmm13 + movaps xmmword ptr [rsp+0x10], xmm14 + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + movaps xmm10, xmm2 + movups xmm4, xmmword ptr [r8+rdx-0x40] + movups xmm5, xmmword ptr [r8+rdx-0x30] + movaps xmm3, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm3, xmm5, 221 + movaps xmm5, xmm3 + movups xmm6, xmmword ptr [r8+rdx-0x20] + movups xmm7, xmmword ptr [r8+rdx-0x10] + movaps xmm3, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm3, xmm7, 221 + pshufd xmm7, xmm3, 0x93 + movups xmm12, xmmword ptr [r9+rdx-0x40] + movups xmm13, xmmword ptr [r9+rdx-0x30] + movaps xmm11, xmm12 + shufps xmm12, xmm13, 136 + shufps xmm11, xmm13, 221 + movaps xmm13, xmm11 + movups xmm14, xmmword ptr [r9+rdx-0x20] + movups xmm15, xmmword ptr [r9+rdx-0x10] + movaps xmm11, xmm14 + shufps xmm14, xmm15, 136 + pshufd xmm14, xmm14, 0x93 + shufps xmm11, xmm15, 221 + pshufd xmm15, xmm11, 0x93 + shl rax, 0x20 + or rax, 0x40 + movq xmm3, rax + movdqa xmmword ptr [rsp+0x20], xmm3 + movaps xmm3, xmmword ptr [rsp] + movaps xmm11, xmmword ptr [rsp+0x10] + punpcklqdq xmm3, xmmword ptr [rsp+0x20] + punpcklqdq xmm11, xmmword ptr [rsp+0x20] + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm8, xmm12 + movaps xmmword ptr [rsp+0x20], xmm4 + movaps xmmword ptr [rsp+0x30], xmm12 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + pshuflw xmm11, xmm11, 0xB1 + pshufhw xmm11, xmm11, 0xB1 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 20 + psrld xmm4, 12 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 20 + psrld xmm4, 12 + por xmm9, xmm4 + paddd xmm0, xmm5 + paddd xmm8, xmm13 + movaps xmmword ptr [rsp+0x40], xmm5 + movaps xmmword ptr [rsp+0x50], xmm13 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + movdqa xmm13, xmm3 + psrld xmm3, 8 + pslld xmm13, 24 + pxor xmm3, xmm13 + movdqa xmm13, xmm11 + psrld xmm11, 8 + pslld xmm13, 24 + pxor xmm11, xmm13 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 25 + psrld xmm4, 7 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 25 + psrld xmm4, 7 + por xmm9, xmm4 + pshufd xmm0, xmm0, 0x93 + pshufd xmm8, xmm8, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm11, xmm11, 0x4E + pshufd xmm2, xmm2, 0x39 + pshufd xmm10, xmm10, 0x39 + paddd xmm0, xmm6 + paddd xmm8, xmm14 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + pshuflw xmm11, xmm11, 0xB1 + pshufhw xmm11, xmm11, 0xB1 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 20 + psrld xmm4, 12 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 20 + psrld xmm4, 12 + por xmm9, xmm4 + paddd xmm0, xmm7 + paddd xmm8, xmm15 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + movdqa xmm13, xmm3 + psrld xmm3, 8 + pslld xmm13, 24 + pxor xmm3, xmm13 + movdqa xmm13, xmm11 + psrld xmm11, 8 + pslld xmm13, 24 + pxor xmm11, xmm13 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 25 + psrld xmm4, 7 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 25 + psrld xmm4, 7 + por xmm9, xmm4 + pshufd xmm0, xmm0, 0x39 + pshufd xmm8, xmm8, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm11, xmm11, 0x4E + pshufd xmm2, xmm2, 0x93 + pshufd xmm10, xmm10, 0x93 + dec al + je 9f + movdqa xmm12, xmmword ptr [rsp+0x20] + movdqa xmm5, xmmword ptr [rsp+0x40] + pshufd xmm13, xmm12, 0x0F + shufps xmm12, xmm5, 214 + pshufd xmm4, xmm12, 0x39 + movdqa xmm12, xmm6 + shufps xmm12, xmm7, 250 + pand xmm13, xmmword ptr [PBLENDW_0x33_MASK+rip] + pand xmm12, xmmword ptr [PBLENDW_0xCC_MASK+rip] + por xmm13, xmm12 + movdqa xmmword ptr [rsp+0x20], xmm13 + movdqa xmm12, xmm7 + punpcklqdq xmm12, xmm5 + movdqa xmm13, xmm6 + pand xmm12, xmmword ptr [PBLENDW_0x3F_MASK+rip] + pand xmm13, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm12, xmm13 + pshufd xmm12, xmm12, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmmword ptr [rsp+0x40], xmm12 + movdqa xmm5, xmmword ptr [rsp+0x30] + movdqa xmm13, xmmword ptr [rsp+0x50] + pshufd xmm6, xmm5, 0x0F + shufps xmm5, xmm13, 214 + pshufd xmm12, xmm5, 0x39 + movdqa xmm5, xmm14 + shufps xmm5, xmm15, 250 + pand xmm6, xmmword ptr [PBLENDW_0x33_MASK+rip] + pand xmm5, xmmword ptr [PBLENDW_0xCC_MASK+rip] + por xmm6, xmm5 + movdqa xmm5, xmm15 + punpcklqdq xmm5, xmm13 + movdqa xmmword ptr [rsp+0x30], xmm2 + movdqa xmm2, xmm14 + pand xmm5, xmmword ptr [PBLENDW_0x3F_MASK+rip] + pand xmm2, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm5, xmm2 + movdqa xmm2, xmmword ptr [rsp+0x30] + pshufd xmm5, xmm5, 0x78 + punpckhdq xmm13, xmm15 + punpckldq xmm14, xmm13 + pshufd xmm15, xmm14, 0x1E + movdqa xmm13, xmm6 + movdqa xmm14, xmm5 + movdqa xmm5, xmmword ptr [rsp+0x20] + movdqa xmm6, xmmword ptr [rsp+0x40] + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + pxor xmm8, xmm10 + pxor xmm9, xmm11 + mov eax, r13d + cmp rdx, r15 + jne 2b + movups xmmword ptr [rbx], xmm0 + movups xmmword ptr [rbx+0x10], xmm1 + movups xmmword ptr [rbx+0x20], xmm8 + movups xmmword ptr [rbx+0x30], xmm9 + mov eax, dword ptr [rsp+0x130] + neg eax + mov r10d, dword ptr [rsp+0x110+8*rax] + mov r11d, dword ptr [rsp+0x120+8*rax] + mov dword ptr [rsp+0x110], r10d + mov dword ptr [rsp+0x120], r11d + add rdi, 16 + add rbx, 64 + sub rsi, 2 +3: + test esi, 0x1 + je 4b + movups xmm0, xmmword ptr [rcx] + movups xmm1, xmmword ptr [rcx+0x10] + movd xmm13, dword ptr [rsp+0x110] + movd xmm14, dword ptr [rsp+0x120] + punpckldq xmm13, xmm14 + mov r8, qword ptr [rdi] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + shl rax, 32 + or rax, 64 + movq xmm12, rax + movdqa xmm3, xmm13 + punpcklqdq xmm3, xmm12 + movups xmm4, xmmword ptr [r8+rdx-0x40] + movups xmm5, xmmword ptr [r8+rdx-0x30] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [r8+rdx-0x20] + movups xmm7, xmmword ptr [r8+rdx-0x10] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pand xmm9, xmmword ptr [PBLENDW_0x33_MASK+rip] + pand xmm8, xmmword ptr [PBLENDW_0xCC_MASK+rip] + por xmm9, xmm8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm10, xmm6 + pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip] + pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm8, xmm10 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + movups xmmword ptr [rbx], xmm0 + movups xmmword ptr [rbx+0x10], xmm1 + jmp 4b + +.p2align 6 +blake3_compress_in_place_sse2_asm: +_blake3_compress_in_place_sse2_asm: + _CET_ENDBR + movups xmm0, xmmword ptr [rdi] + movups xmm1, xmmword ptr [rdi+0x10] + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + shl r8, 32 + add rdx, r8 + movq xmm3, rcx + movq xmm4, rdx + punpcklqdq xmm3, xmm4 + movups xmm4, xmmword ptr [rsi] + movups xmm5, xmmword ptr [rsi+0x10] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [rsi+0x20] + movups xmm7, xmmword ptr [rsi+0x30] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pand xmm9, xmmword ptr [PBLENDW_0x33_MASK+rip] + pand xmm8, xmmword ptr [PBLENDW_0xCC_MASK+rip] + por xmm9, xmm8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm10, xmm6 + pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip] + pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm8, xmm10 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + movups xmmword ptr [rdi], xmm0 + movups xmmword ptr [rdi+0x10], xmm1 + ret + +.p2align 6 +blake3_compress_xof_sse2_asm: +_blake3_compress_xof_sse2_asm: + _CET_ENDBR + movups xmm0, xmmword ptr [rdi] + movups xmm1, xmmword ptr [rdi+0x10] + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + movzx eax, r8b + movzx edx, dl + shl rax, 32 + add rdx, rax + movq xmm3, rcx + movq xmm4, rdx + punpcklqdq xmm3, xmm4 + movups xmm4, xmmword ptr [rsi] + movups xmm5, xmmword ptr [rsi+0x10] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [rsi+0x20] + movups xmm7, xmmword ptr [rsi+0x30] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshuflw xmm3, xmm3, 0xB1 + pshufhw xmm3, xmm3, 0xB1 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqa xmm14, xmm3 + psrld xmm3, 8 + pslld xmm14, 24 + pxor xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pand xmm9, xmmword ptr [PBLENDW_0x33_MASK+rip] + pand xmm8, xmmword ptr [PBLENDW_0xCC_MASK+rip] + por xmm9, xmm8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm10, xmm6 + pand xmm8, xmmword ptr [PBLENDW_0x3F_MASK+rip] + pand xmm10, xmmword ptr [PBLENDW_0xC0_MASK+rip] + por xmm8, xmm10 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + movdqu xmm4, xmmword ptr [rdi] + movdqu xmm5, xmmword ptr [rdi+0x10] + pxor xmm0, xmm2 + pxor xmm1, xmm3 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + movups xmmword ptr [r9], xmm0 + movups xmmword ptr [r9+0x10], xmm1 + movups xmmword ptr [r9+0x20], xmm2 + movups xmmword ptr [r9+0x30], xmm3 + ret + + +#ifdef __APPLE__ +.static_data +#else +.section .rodata +#endif +.p2align 6 +BLAKE3_IV: + .long 0x6A09E667, 0xBB67AE85 + .long 0x3C6EF372, 0xA54FF53A +ADD0: + .long 0, 1, 2, 3 +ADD1: + .long 4, 4, 4, 4 +BLAKE3_IV_0: + .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667 +BLAKE3_IV_1: + .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85 +BLAKE3_IV_2: + .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372 +BLAKE3_IV_3: + .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A +BLAKE3_BLOCK_LEN: + .long 64, 64, 64, 64 +CMP_MSB_MASK: + .long 0x80000000, 0x80000000, 0x80000000, 0x80000000 +PBLENDW_0x33_MASK: + .long 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000 +PBLENDW_0xCC_MASK: + .long 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF +PBLENDW_0x3F_MASK: + .long 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000 +PBLENDW_0xC0_MASK: + .long 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF diff --git a/src/blake3/blake3_sse41.c b/src/blake3/blake3_sse41.c new file mode 100644 index 0000000..adadd48 --- /dev/null +++ b/src/blake3/blake3_sse41.c @@ -0,0 +1,567 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blake3_impl.h" + +#include + +/* define it here instead of the header */ +static const uint8_t MSG_SCHEDULE[7][16] = { B3_MSG_SCHEDULE_DEF }; + +#define DEGREE 4 + +#define _mm_shuffle_ps2(a, b, c) \ + (_mm_castps_si128( \ + _mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (c)))) + +INLINE __m128i loadu(const uint8_t src[16]) { + return _mm_loadu_si128((const __m128i *)src); +} + +INLINE void storeu(__m128i src, uint8_t dest[16]) { + _mm_storeu_si128((__m128i *)dest, src); +} + +INLINE __m128i addv(__m128i a, __m128i b) { return _mm_add_epi32(a, b); } + +// Note that clang-format doesn't like the name "xor" for some reason. +INLINE __m128i xorv(__m128i a, __m128i b) { return _mm_xor_si128(a, b); } + +INLINE __m128i set1(uint32_t x) { return _mm_set1_epi32((int32_t)x); } + +INLINE __m128i set4(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + return _mm_setr_epi32((int32_t)a, (int32_t)b, (int32_t)c, (int32_t)d); +} + +INLINE __m128i rot16(__m128i x) { + return _mm_shuffle_epi8( + x, _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2)); +} + +INLINE __m128i rot12(__m128i x) { + return xorv(_mm_srli_epi32(x, 12), _mm_slli_epi32(x, 32 - 12)); +} + +INLINE __m128i rot8(__m128i x) { + return _mm_shuffle_epi8( + x, _mm_set_epi8(12, 15, 14, 13, 8, 11, 10, 9, 4, 7, 6, 5, 0, 3, 2, 1)); +} + +INLINE __m128i rot7(__m128i x) { + return xorv(_mm_srli_epi32(x, 7), _mm_slli_epi32(x, 32 - 7)); +} + +INLINE void g1(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot16(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot12(*row1); +} + +INLINE void g2(__m128i *row0, __m128i *row1, __m128i *row2, __m128i *row3, + __m128i m) { + *row0 = addv(addv(*row0, m), *row1); + *row3 = xorv(*row3, *row0); + *row3 = rot8(*row3); + *row2 = addv(*row2, *row3); + *row1 = xorv(*row1, *row2); + *row1 = rot7(*row1); +} + +// Note the optimization here of leaving row1 as the unrotated row, rather than +// row0. All the message loads below are adjusted to compensate for this. See +// discussion at https://github.com/sneves/blake2-avx2/pull/4 +INLINE void diagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(2, 1, 0, 3)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(0, 3, 2, 1)); +} + +INLINE void undiagonalize(__m128i *row0, __m128i *row2, __m128i *row3) { + *row0 = _mm_shuffle_epi32(*row0, _MM_SHUFFLE(0, 3, 2, 1)); + *row3 = _mm_shuffle_epi32(*row3, _MM_SHUFFLE(1, 0, 3, 2)); + *row2 = _mm_shuffle_epi32(*row2, _MM_SHUFFLE(2, 1, 0, 3)); +} + +INLINE void compress_pre(__m128i rows[4], const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, uint8_t flags) { + rows[0] = loadu((uint8_t *)&cv[0]); + rows[1] = loadu((uint8_t *)&cv[4]); + rows[2] = set4(B3_IV_0, B3_IV_1, B3_IV_2, B3_IV_3); + rows[3] = set4(counter_low(counter), counter_high(counter), + (uint32_t)block_len, (uint32_t)flags); + + __m128i m0 = loadu(&block[sizeof(__m128i) * 0]); + __m128i m1 = loadu(&block[sizeof(__m128i) * 1]); + __m128i m2 = loadu(&block[sizeof(__m128i) * 2]); + __m128i m3 = loadu(&block[sizeof(__m128i) * 3]); + + __m128i t0, t1, t2, t3, tt; + + // Round 1. The first round permutes the message words from the original + // input order, into the groups that get mixed in parallel. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(2, 0, 2, 0)); // 6 4 2 0 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 3, 1)); // 7 5 3 1 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(2, 0, 2, 0)); // 14 12 10 8 + t2 = _mm_shuffle_epi32(t2, _MM_SHUFFLE(2, 1, 0, 3)); // 12 10 8 14 + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 1, 3, 1)); // 15 13 11 9 + t3 = _mm_shuffle_epi32(t3, _MM_SHUFFLE(2, 1, 0, 3)); // 13 11 9 15 + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 2. This round and all following rounds apply a fixed permutation + // to the message words from the round before. + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 3 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 4 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 5 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 6 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); + m0 = t0; + m1 = t1; + m2 = t2; + m3 = t3; + + // Round 7 + t0 = _mm_shuffle_ps2(m0, m1, _MM_SHUFFLE(3, 1, 1, 2)); + t0 = _mm_shuffle_epi32(t0, _MM_SHUFFLE(0, 3, 2, 1)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t0); + t1 = _mm_shuffle_ps2(m2, m3, _MM_SHUFFLE(3, 3, 2, 2)); + tt = _mm_shuffle_epi32(m0, _MM_SHUFFLE(0, 0, 3, 3)); + t1 = _mm_blend_epi16(tt, t1, 0xCC); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t1); + diagonalize(&rows[0], &rows[2], &rows[3]); + t2 = _mm_unpacklo_epi64(m3, m1); + tt = _mm_blend_epi16(t2, m2, 0xC0); + t2 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(1, 3, 2, 0)); + g1(&rows[0], &rows[1], &rows[2], &rows[3], t2); + t3 = _mm_unpackhi_epi32(m1, m3); + tt = _mm_unpacklo_epi32(m2, t3); + t3 = _mm_shuffle_epi32(tt, _MM_SHUFFLE(0, 1, 3, 2)); + g2(&rows[0], &rows[1], &rows[2], &rows[3], t3); + undiagonalize(&rows[0], &rows[2], &rows[3]); +} + +void blake3_compress_in_place_sse41(uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), (uint8_t *)&cv[0]); + storeu(xorv(rows[1], rows[3]), (uint8_t *)&cv[4]); +} + +void blake3_compress_xof_sse41(const uint32_t cv[8], + const uint8_t block[BLAKE3_BLOCK_LEN], + uint8_t block_len, uint64_t counter, + uint8_t flags, uint8_t out[64]) { + __m128i rows[4]; + compress_pre(rows, cv, block, block_len, counter, flags); + storeu(xorv(rows[0], rows[2]), &out[0]); + storeu(xorv(rows[1], rows[3]), &out[16]); + storeu(xorv(rows[2], loadu((uint8_t *)&cv[0])), &out[32]); + storeu(xorv(rows[3], loadu((uint8_t *)&cv[4])), &out[48]); +} + +INLINE void round_fn(__m128i v[16], __m128i m[16], size_t r) { + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][0]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][2]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][4]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][6]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[15] = rot16(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot12(v[4]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][1]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][3]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][5]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][7]]); + v[0] = addv(v[0], v[4]); + v[1] = addv(v[1], v[5]); + v[2] = addv(v[2], v[6]); + v[3] = addv(v[3], v[7]); + v[12] = xorv(v[12], v[0]); + v[13] = xorv(v[13], v[1]); + v[14] = xorv(v[14], v[2]); + v[15] = xorv(v[15], v[3]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[15] = rot8(v[15]); + v[8] = addv(v[8], v[12]); + v[9] = addv(v[9], v[13]); + v[10] = addv(v[10], v[14]); + v[11] = addv(v[11], v[15]); + v[4] = xorv(v[4], v[8]); + v[5] = xorv(v[5], v[9]); + v[6] = xorv(v[6], v[10]); + v[7] = xorv(v[7], v[11]); + v[4] = rot7(v[4]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][8]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][10]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][12]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][14]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot16(v[15]); + v[12] = rot16(v[12]); + v[13] = rot16(v[13]); + v[14] = rot16(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot12(v[5]); + v[6] = rot12(v[6]); + v[7] = rot12(v[7]); + v[4] = rot12(v[4]); + v[0] = addv(v[0], m[(size_t)MSG_SCHEDULE[r][9]]); + v[1] = addv(v[1], m[(size_t)MSG_SCHEDULE[r][11]]); + v[2] = addv(v[2], m[(size_t)MSG_SCHEDULE[r][13]]); + v[3] = addv(v[3], m[(size_t)MSG_SCHEDULE[r][15]]); + v[0] = addv(v[0], v[5]); + v[1] = addv(v[1], v[6]); + v[2] = addv(v[2], v[7]); + v[3] = addv(v[3], v[4]); + v[15] = xorv(v[15], v[0]); + v[12] = xorv(v[12], v[1]); + v[13] = xorv(v[13], v[2]); + v[14] = xorv(v[14], v[3]); + v[15] = rot8(v[15]); + v[12] = rot8(v[12]); + v[13] = rot8(v[13]); + v[14] = rot8(v[14]); + v[10] = addv(v[10], v[15]); + v[11] = addv(v[11], v[12]); + v[8] = addv(v[8], v[13]); + v[9] = addv(v[9], v[14]); + v[5] = xorv(v[5], v[10]); + v[6] = xorv(v[6], v[11]); + v[7] = xorv(v[7], v[8]); + v[4] = xorv(v[4], v[9]); + v[5] = rot7(v[5]); + v[6] = rot7(v[6]); + v[7] = rot7(v[7]); + v[4] = rot7(v[4]); +} + +INLINE void transpose_vecs(__m128i vecs[DEGREE]) { + // Interleave 32-bit lanes. The low unpack is lanes 00/11 and the high is + // 22/33. Note that this doesn't split the vector into two lanes, as the + // AVX2 counterparts do. + __m128i ab_01 = _mm_unpacklo_epi32(vecs[0], vecs[1]); + __m128i ab_23 = _mm_unpackhi_epi32(vecs[0], vecs[1]); + __m128i cd_01 = _mm_unpacklo_epi32(vecs[2], vecs[3]); + __m128i cd_23 = _mm_unpackhi_epi32(vecs[2], vecs[3]); + + // Interleave 64-bit lanes. + __m128i abcd_0 = _mm_unpacklo_epi64(ab_01, cd_01); + __m128i abcd_1 = _mm_unpackhi_epi64(ab_01, cd_01); + __m128i abcd_2 = _mm_unpacklo_epi64(ab_23, cd_23); + __m128i abcd_3 = _mm_unpackhi_epi64(ab_23, cd_23); + + vecs[0] = abcd_0; + vecs[1] = abcd_1; + vecs[2] = abcd_2; + vecs[3] = abcd_3; +} + +INLINE void transpose_msg_vecs(const uint8_t *const *inputs, + size_t block_offset, __m128i out[16]) { + out[0] = loadu(&inputs[0][block_offset + 0 * sizeof(__m128i)]); + out[1] = loadu(&inputs[1][block_offset + 0 * sizeof(__m128i)]); + out[2] = loadu(&inputs[2][block_offset + 0 * sizeof(__m128i)]); + out[3] = loadu(&inputs[3][block_offset + 0 * sizeof(__m128i)]); + out[4] = loadu(&inputs[0][block_offset + 1 * sizeof(__m128i)]); + out[5] = loadu(&inputs[1][block_offset + 1 * sizeof(__m128i)]); + out[6] = loadu(&inputs[2][block_offset + 1 * sizeof(__m128i)]); + out[7] = loadu(&inputs[3][block_offset + 1 * sizeof(__m128i)]); + out[8] = loadu(&inputs[0][block_offset + 2 * sizeof(__m128i)]); + out[9] = loadu(&inputs[1][block_offset + 2 * sizeof(__m128i)]); + out[10] = loadu(&inputs[2][block_offset + 2 * sizeof(__m128i)]); + out[11] = loadu(&inputs[3][block_offset + 2 * sizeof(__m128i)]); + out[12] = loadu(&inputs[0][block_offset + 3 * sizeof(__m128i)]); + out[13] = loadu(&inputs[1][block_offset + 3 * sizeof(__m128i)]); + out[14] = loadu(&inputs[2][block_offset + 3 * sizeof(__m128i)]); + out[15] = loadu(&inputs[3][block_offset + 3 * sizeof(__m128i)]); + for (size_t i = 0; i < 4; ++i) { + _mm_prefetch((const void *)&inputs[i][block_offset + 256], _MM_HINT_T0); + } + transpose_vecs(&out[0]); + transpose_vecs(&out[4]); + transpose_vecs(&out[8]); + transpose_vecs(&out[12]); +} + +INLINE void load_counters(uint64_t counter, bool increment_counter, + __m128i *out_lo, __m128i *out_hi) { + const __m128i mask = _mm_set1_epi32(-(int32_t)increment_counter); + const __m128i add0 = _mm_set_epi32(3, 2, 1, 0); + const __m128i add1 = _mm_and_si128(mask, add0); + __m128i l = _mm_add_epi32(_mm_set1_epi32((int32_t)counter), add1); + __m128i carry = _mm_cmpgt_epi32(_mm_xor_si128(add1, _mm_set1_epi32(0x80000000)), + _mm_xor_si128( l, _mm_set1_epi32(0x80000000))); + __m128i h = _mm_sub_epi32(_mm_set1_epi32((int32_t)(counter >> 32)), carry); + *out_lo = l; + *out_hi = h; +} + +static +void blake3_hash4_sse41(const uint8_t *const *inputs, size_t blocks, + const uint32_t key[8], uint64_t counter, + bool increment_counter, uint8_t flags, + uint8_t flags_start, uint8_t flags_end, uint8_t *out) { + __m128i h_vecs[8] = { + set1(key[0]), set1(key[1]), set1(key[2]), set1(key[3]), + set1(key[4]), set1(key[5]), set1(key[6]), set1(key[7]), + }; + __m128i counter_low_vec, counter_high_vec; + load_counters(counter, increment_counter, &counter_low_vec, + &counter_high_vec); + uint8_t block_flags = flags | flags_start; + + for (size_t block = 0; block < blocks; block++) { + if (block + 1 == blocks) { + block_flags |= flags_end; + } + __m128i block_len_vec = set1(BLAKE3_BLOCK_LEN); + __m128i block_flags_vec = set1(block_flags); + __m128i msg_vecs[16]; + transpose_msg_vecs(inputs, block * BLAKE3_BLOCK_LEN, msg_vecs); + + __m128i v[16] = { + h_vecs[0], h_vecs[1], h_vecs[2], h_vecs[3], + h_vecs[4], h_vecs[5], h_vecs[6], h_vecs[7], + set1(B3_IV_0), set1(B3_IV_1), set1(B3_IV_2), set1(B3_IV_3), + counter_low_vec, counter_high_vec, block_len_vec, block_flags_vec, + }; + round_fn(v, msg_vecs, 0); + round_fn(v, msg_vecs, 1); + round_fn(v, msg_vecs, 2); + round_fn(v, msg_vecs, 3); + round_fn(v, msg_vecs, 4); + round_fn(v, msg_vecs, 5); + round_fn(v, msg_vecs, 6); + h_vecs[0] = xorv(v[0], v[8]); + h_vecs[1] = xorv(v[1], v[9]); + h_vecs[2] = xorv(v[2], v[10]); + h_vecs[3] = xorv(v[3], v[11]); + h_vecs[4] = xorv(v[4], v[12]); + h_vecs[5] = xorv(v[5], v[13]); + h_vecs[6] = xorv(v[6], v[14]); + h_vecs[7] = xorv(v[7], v[15]); + + block_flags = flags; + } + + transpose_vecs(&h_vecs[0]); + transpose_vecs(&h_vecs[4]); + // The first four vecs now contain the first half of each output, and the + // second four vecs contain the second half of each output. + storeu(h_vecs[0], &out[0 * sizeof(__m128i)]); + storeu(h_vecs[4], &out[1 * sizeof(__m128i)]); + storeu(h_vecs[1], &out[2 * sizeof(__m128i)]); + storeu(h_vecs[5], &out[3 * sizeof(__m128i)]); + storeu(h_vecs[2], &out[4 * sizeof(__m128i)]); + storeu(h_vecs[6], &out[5 * sizeof(__m128i)]); + storeu(h_vecs[3], &out[6 * sizeof(__m128i)]); + storeu(h_vecs[7], &out[7 * sizeof(__m128i)]); +} + +INLINE void hash_one_sse41(const uint8_t *input, size_t blocks, + const uint32_t key[8], uint64_t counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { + uint32_t cv[8]; + memcpy(cv, key, BLAKE3_KEY_LEN); + uint8_t block_flags = flags | flags_start; + while (blocks > 0) { + if (blocks == 1) { + block_flags |= flags_end; + } + blake3_compress_in_place_sse41(cv, input, BLAKE3_BLOCK_LEN, counter, + block_flags); + input = &input[BLAKE3_BLOCK_LEN]; + blocks -= 1; + block_flags = flags; + } + memcpy(out, cv, BLAKE3_OUT_LEN); +} + +void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, + size_t blocks, const uint32_t key[8], + uint64_t counter, bool increment_counter, + uint8_t flags, uint8_t flags_start, + uint8_t flags_end, uint8_t *out) { + while (num_inputs >= DEGREE) { + blake3_hash4_sse41(inputs, blocks, key, counter, increment_counter, flags, + flags_start, flags_end, out); + if (increment_counter) { + counter += DEGREE; + } + inputs += DEGREE; + num_inputs -= DEGREE; + out = &out[DEGREE * BLAKE3_OUT_LEN]; + } + while (num_inputs > 0) { + hash_one_sse41(inputs[0], blocks, key, counter, flags, flags_start, + flags_end, out); + if (increment_counter) { + counter += 1; + } + inputs += 1; + num_inputs -= 1; + out = &out[BLAKE3_OUT_LEN]; + } +} diff --git a/src/blake3/blake3_sse41_x86-64_unix.S b/src/blake3/blake3_sse41_x86-64_unix.S new file mode 100644 index 0000000..d499a9a --- /dev/null +++ b/src/blake3/blake3_sse41_x86-64_unix.S @@ -0,0 +1,2028 @@ +#if defined(__ELF__) && defined(__linux__) +.section .note.GNU-stack,"",%progbits +#endif + +#if defined(__ELF__) && defined(__CET__) && defined(__has_include) +#if __has_include() +#include +#endif +#endif + +#if !defined(_CET_ENDBR) +#define _CET_ENDBR +#endif + +.intel_syntax noprefix +.global blake3_hash_many_sse41_asm +.global _blake3_hash_many_sse41_asm +.global blake3_compress_in_place_sse41_asm +.global _blake3_compress_in_place_sse41_asm +.global blake3_compress_xof_sse41_asm +.global _blake3_compress_xof_sse41_asm +#ifdef __APPLE__ +.text +#else +.section .text +#endif + .p2align 6 +_blake3_hash_many_sse41_asm: +blake3_hash_many_sse41_asm: + _CET_ENDBR + push r15 + push r14 + push r13 + push r12 + push rbx + push rbp + mov rbp, rsp + sub rsp, 360 + and rsp, 0xFFFFFFFFFFFFFFC0 + neg r9d + movd xmm0, r9d + pshufd xmm0, xmm0, 0x00 + movdqa xmmword ptr [rsp+0x130], xmm0 + movdqa xmm1, xmm0 + pand xmm1, xmmword ptr [ADD0+rip] + pand xmm0, xmmword ptr [ADD1+rip] + movdqa xmmword ptr [rsp+0x150], xmm0 + movd xmm0, r8d + pshufd xmm0, xmm0, 0x00 + paddd xmm0, xmm1 + movdqa xmmword ptr [rsp+0x110], xmm0 + pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip] + pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip] + pcmpgtd xmm1, xmm0 + shr r8, 32 + movd xmm2, r8d + pshufd xmm2, xmm2, 0x00 + psubd xmm2, xmm1 + movdqa xmmword ptr [rsp+0x120], xmm2 + mov rbx, qword ptr [rbp+0x50] + mov r15, rdx + shl r15, 6 + movzx r13d, byte ptr [rbp+0x38] + movzx r12d, byte ptr [rbp+0x48] + cmp rsi, 4 + jc 3f +2: + movdqu xmm3, xmmword ptr [rcx] + pshufd xmm0, xmm3, 0x00 + pshufd xmm1, xmm3, 0x55 + pshufd xmm2, xmm3, 0xAA + pshufd xmm3, xmm3, 0xFF + movdqu xmm7, xmmword ptr [rcx+0x10] + pshufd xmm4, xmm7, 0x00 + pshufd xmm5, xmm7, 0x55 + pshufd xmm6, xmm7, 0xAA + pshufd xmm7, xmm7, 0xFF + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + mov r10, qword ptr [rdi+0x10] + mov r11, qword ptr [rdi+0x18] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +9: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movdqu xmm8, xmmword ptr [r8+rdx-0x40] + movdqu xmm9, xmmword ptr [r9+rdx-0x40] + movdqu xmm10, xmmword ptr [r10+rdx-0x40] + movdqu xmm11, xmmword ptr [r11+rdx-0x40] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp], xmm8 + movdqa xmmword ptr [rsp+0x10], xmm9 + movdqa xmmword ptr [rsp+0x20], xmm12 + movdqa xmmword ptr [rsp+0x30], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x30] + movdqu xmm9, xmmword ptr [r9+rdx-0x30] + movdqu xmm10, xmmword ptr [r10+rdx-0x30] + movdqu xmm11, xmmword ptr [r11+rdx-0x30] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0x40], xmm8 + movdqa xmmword ptr [rsp+0x50], xmm9 + movdqa xmmword ptr [rsp+0x60], xmm12 + movdqa xmmword ptr [rsp+0x70], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x20] + movdqu xmm9, xmmword ptr [r9+rdx-0x20] + movdqu xmm10, xmmword ptr [r10+rdx-0x20] + movdqu xmm11, xmmword ptr [r11+rdx-0x20] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0x80], xmm8 + movdqa xmmword ptr [rsp+0x90], xmm9 + movdqa xmmword ptr [rsp+0xA0], xmm12 + movdqa xmmword ptr [rsp+0xB0], xmm13 + movdqu xmm8, xmmword ptr [r8+rdx-0x10] + movdqu xmm9, xmmword ptr [r9+rdx-0x10] + movdqu xmm10, xmmword ptr [r10+rdx-0x10] + movdqu xmm11, xmmword ptr [r11+rdx-0x10] + movdqa xmm12, xmm8 + punpckldq xmm8, xmm9 + punpckhdq xmm12, xmm9 + movdqa xmm14, xmm10 + punpckldq xmm10, xmm11 + punpckhdq xmm14, xmm11 + movdqa xmm9, xmm8 + punpcklqdq xmm8, xmm10 + punpckhqdq xmm9, xmm10 + movdqa xmm13, xmm12 + punpcklqdq xmm12, xmm14 + punpckhqdq xmm13, xmm14 + movdqa xmmword ptr [rsp+0xC0], xmm8 + movdqa xmmword ptr [rsp+0xD0], xmm9 + movdqa xmmword ptr [rsp+0xE0], xmm12 + movdqa xmmword ptr [rsp+0xF0], xmm13 + movdqa xmm9, xmmword ptr [BLAKE3_IV_1+rip] + movdqa xmm10, xmmword ptr [BLAKE3_IV_2+rip] + movdqa xmm11, xmmword ptr [BLAKE3_IV_3+rip] + movdqa xmm12, xmmword ptr [rsp+0x110] + movdqa xmm13, xmmword ptr [rsp+0x120] + movdqa xmm14, xmmword ptr [BLAKE3_BLOCK_LEN+rip] + movd xmm15, eax + pshufd xmm15, xmm15, 0x00 + prefetcht0 [r8+rdx+0x80] + prefetcht0 [r9+rdx+0x80] + prefetcht0 [r10+rdx+0x80] + prefetcht0 [r11+rdx+0x80] + paddd xmm0, xmmword ptr [rsp] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x40] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [BLAKE3_IV_0+rip] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x10] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x50] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x80] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0xC0] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x90] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0xD0] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x20] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x70] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x60] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x10] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x90] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xB0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0xE0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x30] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0xD0] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x40] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x20] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x60] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0xB0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x50] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0xF0] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xA0] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0xE0] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x70] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0x30] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x40] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0x50] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x80] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xC0] + paddd xmm1, xmmword ptr [rsp+0x90] + paddd xmm2, xmmword ptr [rsp+0xF0] + paddd xmm3, xmmword ptr [rsp+0xE0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xD0] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0xA0] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0x70] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x20] + paddd xmm1, xmmword ptr [rsp+0x30] + paddd xmm2, xmmword ptr [rsp+0x10] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x90] + paddd xmm1, xmmword ptr [rsp+0xB0] + paddd xmm2, xmmword ptr [rsp+0x80] + paddd xmm3, xmmword ptr [rsp+0xF0] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xE0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0xC0] + paddd xmm3, xmmword ptr [rsp+0x10] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xD0] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0x20] + paddd xmm3, xmmword ptr [rsp+0x40] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0x30] + paddd xmm1, xmmword ptr [rsp+0xA0] + paddd xmm2, xmmword ptr [rsp+0x60] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xB0] + paddd xmm1, xmmword ptr [rsp+0x50] + paddd xmm2, xmmword ptr [rsp+0x10] + paddd xmm3, xmmword ptr [rsp+0x80] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xF0] + paddd xmm1, xmmword ptr [rsp] + paddd xmm2, xmmword ptr [rsp+0x90] + paddd xmm3, xmmword ptr [rsp+0x60] + paddd xmm0, xmm4 + paddd xmm1, xmm5 + paddd xmm2, xmm6 + paddd xmm3, xmm7 + pxor xmm12, xmm0 + pxor xmm13, xmm1 + pxor xmm14, xmm2 + pxor xmm15, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + pshufb xmm15, xmm8 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm12 + paddd xmm9, xmm13 + paddd xmm10, xmm14 + paddd xmm11, xmm15 + pxor xmm4, xmm8 + pxor xmm5, xmm9 + pxor xmm6, xmm10 + pxor xmm7, xmm11 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + paddd xmm0, xmmword ptr [rsp+0xE0] + paddd xmm1, xmmword ptr [rsp+0x20] + paddd xmm2, xmmword ptr [rsp+0x30] + paddd xmm3, xmmword ptr [rsp+0x70] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT16+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + movdqa xmmword ptr [rsp+0x100], xmm8 + movdqa xmm8, xmm5 + psrld xmm8, 12 + pslld xmm5, 20 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 12 + pslld xmm6, 20 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 12 + pslld xmm7, 20 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 12 + pslld xmm4, 20 + por xmm4, xmm8 + paddd xmm0, xmmword ptr [rsp+0xA0] + paddd xmm1, xmmword ptr [rsp+0xC0] + paddd xmm2, xmmword ptr [rsp+0x40] + paddd xmm3, xmmword ptr [rsp+0xD0] + paddd xmm0, xmm5 + paddd xmm1, xmm6 + paddd xmm2, xmm7 + paddd xmm3, xmm4 + pxor xmm15, xmm0 + pxor xmm12, xmm1 + pxor xmm13, xmm2 + pxor xmm14, xmm3 + movdqa xmm8, xmmword ptr [ROT8+rip] + pshufb xmm15, xmm8 + pshufb xmm12, xmm8 + pshufb xmm13, xmm8 + pshufb xmm14, xmm8 + paddd xmm10, xmm15 + paddd xmm11, xmm12 + movdqa xmm8, xmmword ptr [rsp+0x100] + paddd xmm8, xmm13 + paddd xmm9, xmm14 + pxor xmm5, xmm10 + pxor xmm6, xmm11 + pxor xmm7, xmm8 + pxor xmm4, xmm9 + pxor xmm0, xmm8 + pxor xmm1, xmm9 + pxor xmm2, xmm10 + pxor xmm3, xmm11 + movdqa xmm8, xmm5 + psrld xmm8, 7 + pslld xmm5, 25 + por xmm5, xmm8 + movdqa xmm8, xmm6 + psrld xmm8, 7 + pslld xmm6, 25 + por xmm6, xmm8 + movdqa xmm8, xmm7 + psrld xmm8, 7 + pslld xmm7, 25 + por xmm7, xmm8 + movdqa xmm8, xmm4 + psrld xmm8, 7 + pslld xmm4, 25 + por xmm4, xmm8 + pxor xmm4, xmm12 + pxor xmm5, xmm13 + pxor xmm6, xmm14 + pxor xmm7, xmm15 + mov eax, r13d + jne 9b + movdqa xmm9, xmm0 + punpckldq xmm0, xmm1 + punpckhdq xmm9, xmm1 + movdqa xmm11, xmm2 + punpckldq xmm2, xmm3 + punpckhdq xmm11, xmm3 + movdqa xmm1, xmm0 + punpcklqdq xmm0, xmm2 + punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm9 + punpcklqdq xmm9, xmm11 + punpckhqdq xmm3, xmm11 + movdqu xmmword ptr [rbx], xmm0 + movdqu xmmword ptr [rbx+0x20], xmm1 + movdqu xmmword ptr [rbx+0x40], xmm9 + movdqu xmmword ptr [rbx+0x60], xmm3 + movdqa xmm9, xmm4 + punpckldq xmm4, xmm5 + punpckhdq xmm9, xmm5 + movdqa xmm11, xmm6 + punpckldq xmm6, xmm7 + punpckhdq xmm11, xmm7 + movdqa xmm5, xmm4 + punpcklqdq xmm4, xmm6 + punpckhqdq xmm5, xmm6 + movdqa xmm7, xmm9 + punpcklqdq xmm9, xmm11 + punpckhqdq xmm7, xmm11 + movdqu xmmword ptr [rbx+0x10], xmm4 + movdqu xmmword ptr [rbx+0x30], xmm5 + movdqu xmmword ptr [rbx+0x50], xmm9 + movdqu xmmword ptr [rbx+0x70], xmm7 + movdqa xmm1, xmmword ptr [rsp+0x110] + movdqa xmm0, xmm1 + paddd xmm1, xmmword ptr [rsp+0x150] + movdqa xmmword ptr [rsp+0x110], xmm1 + pxor xmm0, xmmword ptr [CMP_MSB_MASK+rip] + pxor xmm1, xmmword ptr [CMP_MSB_MASK+rip] + pcmpgtd xmm0, xmm1 + movdqa xmm1, xmmword ptr [rsp+0x120] + psubd xmm1, xmm0 + movdqa xmmword ptr [rsp+0x120], xmm1 + add rbx, 128 + add rdi, 32 + sub rsi, 4 + cmp rsi, 4 + jnc 2b + test rsi, rsi + jnz 3f +4: + mov rsp, rbp + pop rbp + pop rbx + pop r12 + pop r13 + pop r14 + pop r15 + ret +.p2align 5 +3: + test esi, 0x2 + je 3f + movups xmm0, xmmword ptr [rcx] + movups xmm1, xmmword ptr [rcx+0x10] + movaps xmm8, xmm0 + movaps xmm9, xmm1 + movd xmm13, dword ptr [rsp+0x110] + pinsrd xmm13, dword ptr [rsp+0x120], 1 + pinsrd xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + movaps xmmword ptr [rsp], xmm13 + movd xmm14, dword ptr [rsp+0x114] + pinsrd xmm14, dword ptr [rsp+0x124], 1 + pinsrd xmm14, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + movaps xmmword ptr [rsp+0x10], xmm14 + mov r8, qword ptr [rdi] + mov r9, qword ptr [rdi+0x8] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + movaps xmm10, xmm2 + movups xmm4, xmmword ptr [r8+rdx-0x40] + movups xmm5, xmmword ptr [r8+rdx-0x30] + movaps xmm3, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm3, xmm5, 221 + movaps xmm5, xmm3 + movups xmm6, xmmword ptr [r8+rdx-0x20] + movups xmm7, xmmword ptr [r8+rdx-0x10] + movaps xmm3, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm3, xmm7, 221 + pshufd xmm7, xmm3, 0x93 + movups xmm12, xmmword ptr [r9+rdx-0x40] + movups xmm13, xmmword ptr [r9+rdx-0x30] + movaps xmm11, xmm12 + shufps xmm12, xmm13, 136 + shufps xmm11, xmm13, 221 + movaps xmm13, xmm11 + movups xmm14, xmmword ptr [r9+rdx-0x20] + movups xmm15, xmmword ptr [r9+rdx-0x10] + movaps xmm11, xmm14 + shufps xmm14, xmm15, 136 + pshufd xmm14, xmm14, 0x93 + shufps xmm11, xmm15, 221 + pshufd xmm15, xmm11, 0x93 + movaps xmm3, xmmword ptr [rsp] + movaps xmm11, xmmword ptr [rsp+0x10] + pinsrd xmm3, eax, 3 + pinsrd xmm11, eax, 3 + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm8, xmm12 + movaps xmmword ptr [rsp+0x20], xmm4 + movaps xmmword ptr [rsp+0x30], xmm12 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + movaps xmm12, xmmword ptr [ROT16+rip] + pshufb xmm3, xmm12 + pshufb xmm11, xmm12 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 20 + psrld xmm4, 12 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 20 + psrld xmm4, 12 + por xmm9, xmm4 + paddd xmm0, xmm5 + paddd xmm8, xmm13 + movaps xmmword ptr [rsp+0x40], xmm5 + movaps xmmword ptr [rsp+0x50], xmm13 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + movaps xmm13, xmmword ptr [ROT8+rip] + pshufb xmm3, xmm13 + pshufb xmm11, xmm13 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 25 + psrld xmm4, 7 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 25 + psrld xmm4, 7 + por xmm9, xmm4 + pshufd xmm0, xmm0, 0x93 + pshufd xmm8, xmm8, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm11, xmm11, 0x4E + pshufd xmm2, xmm2, 0x39 + pshufd xmm10, xmm10, 0x39 + paddd xmm0, xmm6 + paddd xmm8, xmm14 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + pshufb xmm3, xmm12 + pshufb xmm11, xmm12 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 20 + psrld xmm4, 12 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 20 + psrld xmm4, 12 + por xmm9, xmm4 + paddd xmm0, xmm7 + paddd xmm8, xmm15 + paddd xmm0, xmm1 + paddd xmm8, xmm9 + pxor xmm3, xmm0 + pxor xmm11, xmm8 + pshufb xmm3, xmm13 + pshufb xmm11, xmm13 + paddd xmm2, xmm3 + paddd xmm10, xmm11 + pxor xmm1, xmm2 + pxor xmm9, xmm10 + movdqa xmm4, xmm1 + pslld xmm1, 25 + psrld xmm4, 7 + por xmm1, xmm4 + movdqa xmm4, xmm9 + pslld xmm9, 25 + psrld xmm4, 7 + por xmm9, xmm4 + pshufd xmm0, xmm0, 0x39 + pshufd xmm8, xmm8, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm11, xmm11, 0x4E + pshufd xmm2, xmm2, 0x93 + pshufd xmm10, xmm10, 0x93 + dec al + je 9f + movdqa xmm12, xmmword ptr [rsp+0x20] + movdqa xmm5, xmmword ptr [rsp+0x40] + pshufd xmm13, xmm12, 0x0F + shufps xmm12, xmm5, 214 + pshufd xmm4, xmm12, 0x39 + movdqa xmm12, xmm6 + shufps xmm12, xmm7, 250 + pblendw xmm13, xmm12, 0xCC + movdqa xmm12, xmm7 + punpcklqdq xmm12, xmm5 + pblendw xmm12, xmm6, 0xC0 + pshufd xmm12, xmm12, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmmword ptr [rsp+0x20], xmm13 + movdqa xmmword ptr [rsp+0x40], xmm12 + movdqa xmm5, xmmword ptr [rsp+0x30] + movdqa xmm13, xmmword ptr [rsp+0x50] + pshufd xmm6, xmm5, 0x0F + shufps xmm5, xmm13, 214 + pshufd xmm12, xmm5, 0x39 + movdqa xmm5, xmm14 + shufps xmm5, xmm15, 250 + pblendw xmm6, xmm5, 0xCC + movdqa xmm5, xmm15 + punpcklqdq xmm5, xmm13 + pblendw xmm5, xmm14, 0xC0 + pshufd xmm5, xmm5, 0x78 + punpckhdq xmm13, xmm15 + punpckldq xmm14, xmm13 + pshufd xmm15, xmm14, 0x1E + movdqa xmm13, xmm6 + movdqa xmm14, xmm5 + movdqa xmm5, xmmword ptr [rsp+0x20] + movdqa xmm6, xmmword ptr [rsp+0x40] + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + pxor xmm8, xmm10 + pxor xmm9, xmm11 + mov eax, r13d + cmp rdx, r15 + jne 2b + movups xmmword ptr [rbx], xmm0 + movups xmmword ptr [rbx+0x10], xmm1 + movups xmmword ptr [rbx+0x20], xmm8 + movups xmmword ptr [rbx+0x30], xmm9 + movdqa xmm0, xmmword ptr [rsp+0x130] + movdqa xmm1, xmmword ptr [rsp+0x110] + movdqa xmm2, xmmword ptr [rsp+0x120] + movdqu xmm3, xmmword ptr [rsp+0x118] + movdqu xmm4, xmmword ptr [rsp+0x128] + blendvps xmm1, xmm3, xmm0 + blendvps xmm2, xmm4, xmm0 + movdqa xmmword ptr [rsp+0x110], xmm1 + movdqa xmmword ptr [rsp+0x120], xmm2 + add rdi, 16 + add rbx, 64 + sub rsi, 2 +3: + test esi, 0x1 + je 4b + movups xmm0, xmmword ptr [rcx] + movups xmm1, xmmword ptr [rcx+0x10] + movd xmm13, dword ptr [rsp+0x110] + pinsrd xmm13, dword ptr [rsp+0x120], 1 + pinsrd xmm13, dword ptr [BLAKE3_BLOCK_LEN+rip], 2 + movaps xmm14, xmmword ptr [ROT8+rip] + movaps xmm15, xmmword ptr [ROT16+rip] + mov r8, qword ptr [rdi] + movzx eax, byte ptr [rbp+0x40] + or eax, r13d + xor edx, edx +2: + mov r14d, eax + or eax, r12d + add rdx, 64 + cmp rdx, r15 + cmovne eax, r14d + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + movaps xmm3, xmm13 + pinsrd xmm3, eax, 3 + movups xmm4, xmmword ptr [r8+rdx-0x40] + movups xmm5, xmmword ptr [r8+rdx-0x30] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [r8+rdx-0x20] + movups xmm7, xmmword ptr [r8+rdx-0x10] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pblendw xmm9, xmm8, 0xCC + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + pblendw xmm8, xmm6, 0xC0 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + mov eax, r13d + cmp rdx, r15 + jne 2b + movups xmmword ptr [rbx], xmm0 + movups xmmword ptr [rbx+0x10], xmm1 + jmp 4b + +.p2align 6 +blake3_compress_in_place_sse41_asm: +_blake3_compress_in_place_sse41_asm: + _CET_ENDBR + movups xmm0, xmmword ptr [rdi] + movups xmm1, xmmword ptr [rdi+0x10] + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + shl r8, 32 + add rdx, r8 + movq xmm3, rcx + movq xmm4, rdx + punpcklqdq xmm3, xmm4 + movups xmm4, xmmword ptr [rsi] + movups xmm5, xmmword ptr [rsi+0x10] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [rsi+0x20] + movups xmm7, xmmword ptr [rsi+0x30] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + movaps xmm14, xmmword ptr [ROT8+rip] + movaps xmm15, xmmword ptr [ROT16+rip] + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pblendw xmm9, xmm8, 0xCC + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + pblendw xmm8, xmm6, 0xC0 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + pxor xmm0, xmm2 + pxor xmm1, xmm3 + movups xmmword ptr [rdi], xmm0 + movups xmmword ptr [rdi+0x10], xmm1 + ret + +.p2align 6 +blake3_compress_xof_sse41_asm: +_blake3_compress_xof_sse41_asm: + _CET_ENDBR + movups xmm0, xmmword ptr [rdi] + movups xmm1, xmmword ptr [rdi+0x10] + movaps xmm2, xmmword ptr [BLAKE3_IV+rip] + movzx eax, r8b + movzx edx, dl + shl rax, 32 + add rdx, rax + movq xmm3, rcx + movq xmm4, rdx + punpcklqdq xmm3, xmm4 + movups xmm4, xmmword ptr [rsi] + movups xmm5, xmmword ptr [rsi+0x10] + movaps xmm8, xmm4 + shufps xmm4, xmm5, 136 + shufps xmm8, xmm5, 221 + movaps xmm5, xmm8 + movups xmm6, xmmword ptr [rsi+0x20] + movups xmm7, xmmword ptr [rsi+0x30] + movaps xmm8, xmm6 + shufps xmm6, xmm7, 136 + pshufd xmm6, xmm6, 0x93 + shufps xmm8, xmm7, 221 + pshufd xmm7, xmm8, 0x93 + movaps xmm14, xmmword ptr [ROT8+rip] + movaps xmm15, xmmword ptr [ROT16+rip] + mov al, 7 +9: + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x93 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x39 + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm15 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + pshufb xmm3, xmm14 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + pshufd xmm0, xmm0, 0x39 + pshufd xmm3, xmm3, 0x4E + pshufd xmm2, xmm2, 0x93 + dec al + jz 9f + movdqa xmm8, xmm4 + shufps xmm8, xmm5, 214 + pshufd xmm9, xmm4, 0x0F + pshufd xmm4, xmm8, 0x39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, 250 + pblendw xmm9, xmm8, 0xCC + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + pblendw xmm8, xmm6, 0xC0 + pshufd xmm8, xmm8, 0x78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, 0x1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + jmp 9b +9: + movdqu xmm4, xmmword ptr [rdi] + movdqu xmm5, xmmword ptr [rdi+0x10] + pxor xmm0, xmm2 + pxor xmm1, xmm3 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + movups xmmword ptr [r9], xmm0 + movups xmmword ptr [r9+0x10], xmm1 + movups xmmword ptr [r9+0x20], xmm2 + movups xmmword ptr [r9+0x30], xmm3 + ret + + +#ifdef __APPLE__ +.static_data +#else +.section .rodata +#endif +.p2align 6 +BLAKE3_IV: + .long 0x6A09E667, 0xBB67AE85 + .long 0x3C6EF372, 0xA54FF53A +ROT16: + .byte 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 +ROT8: + .byte 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12 +ADD0: + .long 0, 1, 2, 3 +ADD1: + .long 4, 4, 4, 4 +BLAKE3_IV_0: + .long 0x6A09E667, 0x6A09E667, 0x6A09E667, 0x6A09E667 +BLAKE3_IV_1: + .long 0xBB67AE85, 0xBB67AE85, 0xBB67AE85, 0xBB67AE85 +BLAKE3_IV_2: + .long 0x3C6EF372, 0x3C6EF372, 0x3C6EF372, 0x3C6EF372 +BLAKE3_IV_3: + .long 0xA54FF53A, 0xA54FF53A, 0xA54FF53A, 0xA54FF53A +BLAKE3_BLOCK_LEN: + .long 64, 64, 64, 64 +CMP_MSB_MASK: + .long 0x80000000, 0x80000000, 0x80000000, 0x80000000 diff --git a/src/blake3/fy-blake3.c b/src/blake3/fy-blake3.c new file mode 100644 index 0000000..e14234d --- /dev/null +++ b/src/blake3/fy-blake3.c @@ -0,0 +1,175 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + +#include +#include + +#include "blake3.h" +#include "blake3_impl.h" + +#include + +struct fy_blake3_hasher { + uint8_t output[FY_BLAKE3_OUT_LEN] FY_CACHELINE_ALIGN; + struct fy_blake3_hasher_cfg cfg; + struct blake3_host_state *hs; + struct blake3_hasher *hasher; +}; + +struct fy_blake3_hasher *fy_blake3_hasher_create(const struct fy_blake3_hasher_cfg *cfg) +{ + struct fy_blake3_hasher *fyh = NULL; + blake3_host_config hs_cfg; + + fyh = malloc(sizeof(*fyh)); + if (!fyh) + goto err_out; + + memset(fyh, 0, sizeof(*fyh)); + + /* copy config - if it exists */ + if (cfg) + fyh->cfg = *cfg; + + memset(&hs_cfg, 0, sizeof(hs_cfg)); + hs_cfg.debug = false; + hs_cfg.backend = cfg->backend; + if (!cfg->tp) { + if (cfg->num_threads >= 0) + hs_cfg.num_threads = (unsigned int)cfg->num_threads; + else + hs_cfg.no_mthread = true; + } else + hs_cfg.tp = cfg->tp; + hs_cfg.no_mmap = cfg->no_mmap; + hs_cfg.file_io_bufsz = cfg->file_buffer; + hs_cfg.mmap_min_chunk = cfg->mmap_min_chunk; + hs_cfg.mmap_max_chunk = cfg->mmap_max_chunk; + fyh->hs = blake3_host_state_create(&hs_cfg); + if (!fyh->hs) + goto err_out; + + fyh->hasher = blake3_hasher_create(fyh->hs, fyh->cfg.key, fyh->cfg.context, fyh->cfg.context ? fyh->cfg.context_len : 0); + if (!fyh->hasher) + goto err_out; + + return fyh; + +err_out: + fy_blake3_hasher_destroy(fyh); + return NULL; +} + +void fy_blake3_hasher_destroy(struct fy_blake3_hasher *fyh) +{ + if (!fyh) + return; + if (fyh->hasher) + blake3_hasher_destroy(fyh->hasher); + if (fyh->hs) + blake3_host_state_destroy(fyh->hs); + free(fyh); +} + +void fy_blake3_hasher_update(struct fy_blake3_hasher *fyh, const void *input, size_t input_len) +{ + if (!fyh) + return; + blake3_hasher_update(fyh->hasher, input, input_len); +} + +const uint8_t *fy_blake3_hasher_finalize(struct fy_blake3_hasher *fyh) +{ + if (!fyh) + return NULL; + blake3_hasher_finalize(fyh->hasher, fyh->output, FY_BLAKE3_OUT_LEN); + return fyh->output; +} + +void fy_blake3_hasher_reset(struct fy_blake3_hasher *fyh) +{ + if (!fyh) + return; + blake3_hasher_reset(fyh->hasher); +} + +const uint8_t *fy_blake3_hash_file(struct fy_blake3_hasher *fyh, const char *filename) +{ + int rc; + + if (!fyh || !filename) + return NULL; + + rc = blake3_hash_file(fyh->hasher, filename, fyh->output); + if (rc) + return NULL; + + return fyh->output; +} + +const uint8_t *fy_blake3_hash(struct fy_blake3_hasher *fyh, const void *mem, size_t size) +{ + if (!fyh || !mem) + return NULL; + + blake3_hash(fyh->hasher, mem, size, fyh->output); + + return fyh->output; +} + +const char *fy_blake3_backend_iterate(const char **prevp) +{ + const char *backend_names[64]; /* maximum 64 backends + 1 for NULL */ + const blake3_backend_info *bei; + uint64_t backends, avail; + unsigned int i, count; + + if (!prevp) + return NULL; + + avail = blake3_get_selectable_backends() & + blake3_get_detected_backends(); + + /* first pass, fill the backend name array */ + for (i = 0, backends = avail, count = 0; backends && i < B3BID_COUNT; i++) { + if (!(backends & ((uint64_t)1 << i))) + continue; + bei = blake3_get_backend_info(i); + if (!bei) + continue; + backends &= ~((uint64_t)1 << i); + backend_names[count++] = bei->name; + } + + /* start of the iterator */ + if (!*prevp) { + if (!count) + return NULL; + i = 0; + + } else { + + /* ok, find the current spot */ + for (i = 0; i < count; i++) { + if (!strcmp(*prevp, backend_names[i])) + break; + } + + /* increase for next, last or not found? */ + if (++i >= count) { + *prevp = NULL; + return NULL; + } + } + + return *prevp = backend_names[i]; +} diff --git a/src/internal/fy-allocators.c b/src/internal/fy-allocators.c new file mode 100644 index 0000000..79e83e2 --- /dev/null +++ b/src/internal/fy-allocators.c @@ -0,0 +1,305 @@ +/* + * fy-allocators.c - allocators testing internal utility + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-utils.h" + +#include "fy-allocator.h" +#include "fy-allocator-linear.h" +#include "fy-allocator-malloc.h" +#include "fy-allocator-mremap.h" +#include "fy-allocator-dedup.h" +#include "fy-allocator-auto.h" + +static void dump_allocator_info(struct fy_allocator *a, int tag) +{ + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + struct fy_allocator_arena_info *arena_info; + unsigned int i, j; + + info = fy_allocator_get_info(a, tag); + if (!info) { + fprintf(stderr, "fy_allocator_get_info() failed\n"); + return; + } + + fprintf(stderr, "Allocator %p: free=%zu used=%zu total=%zu\n", a, + info->free, info->used, info->total); + for (i = 0; i < info->num_tag_infos; i++) { + tag_info = &info->tag_infos[i]; + + fprintf(stderr, "\ttag #%d: free=%zu used=%zu total=%zu\n", i, + tag_info->free, tag_info->used, tag_info->total); + for (j = 0; j < tag_info->num_arena_infos; j++) { + arena_info = &tag_info->arena_infos[j]; + + fprintf(stderr, "\t\tarena #%d: free=%zu used=%zu total=%zu data=%p-0x%zx\n", j, + arena_info->free, arena_info->used, arena_info->total, + arena_info->data, arena_info->size); + } + } + + free(info); +} + +static int allocator_test(const char *allocator, const char *parent_allocator, size_t size) +{ + struct fy_linear_allocator_cfg lcfg; + struct fy_dedup_allocator_cfg dcfg; + const void *gcfg = NULL, *pcfg = NULL; + struct fy_allocator *a = NULL, *pa = NULL; + int tag0; + char *names; + bool is_linear, is_dedup; + unsigned int *uintp[16]; + uint8_t *p; + const uint8_t *p1, *p2; + size_t psz = 4096; + unsigned int i; + const void *linear_data; + size_t linear_size; + + names = fy_allocator_get_names(); + assert(names); + fprintf(stderr, "Available allocators: %s\n", names); + free(names); + + is_linear = !strcmp(allocator, "linear"); + is_dedup = !strcmp(allocator, "dedup"); + + if (is_linear) { + memset(&lcfg, 0, sizeof(lcfg)); + lcfg.size = size ? size : 4096; + gcfg = &lcfg; + } else if (is_dedup) { + + fprintf(stderr, "Using parent-allocator: %s\n", parent_allocator); + + if (!strcmp(parent_allocator, "linear")) { + memset(&lcfg, 0, sizeof(lcfg)); + lcfg.size = size ? size : 4096; + + pcfg = &lcfg; + } else + pcfg = NULL; + + pa = fy_allocator_create(parent_allocator, pcfg); + assert(pa); + + memset(&dcfg, 0, sizeof(dcfg)); + dcfg.parent_allocator = pa; + gcfg = &dcfg; + } else + gcfg = NULL; + + fprintf(stderr, "Using allocator: %s\n", allocator); + + a = fy_allocator_create(allocator, gcfg); + assert(a); + + fprintf(stderr, "Allocator created: %p\n", a); + + tag0 = fy_allocator_get_tag(a); + assert(tag0 != FY_ALLOC_TAG_ERROR); + + fprintf(stderr, "tag0 created: %d\n", tag0); + + fprintf(stderr, "Allocating %u integers\n", (unsigned int)ARRAY_SIZE(uintp)); + for (i = 0; i < ARRAY_SIZE(uintp); i++) { + uintp[i] = fy_allocator_alloc(a, tag0, sizeof(unsigned int), _Alignof(unsigned int)); + assert(uintp[i] != NULL); + fprintf(stderr, "\t%u: %p\n", i, uintp[i]); + } + + /* fill in */ + for (i = 0; i < ARRAY_SIZE(uintp); i++) + *uintp[i] = i; + + fprintf(stderr, "Dumping allocator areas before trim\n"); + dump_allocator_info(a, tag0); + + fy_allocator_trim_tag(a, tag0); + + fprintf(stderr, "Dumping allocator areas after trim\n"); + dump_allocator_info(a, tag0); + + fprintf(stderr, "Allocating %zu bytes\n", psz); + p = fy_allocator_alloc(a, tag0, psz, 1); + if (!p) { + fprintf(stderr, "failed to allocate %zu bytes\n", psz); + } else { + for (i = 0; i < psz; i++) + p[i] = i % 251; + } + + /* verify */ + if (p) { + for (i = 0; i < psz; i++) + assert(p[i] == (i % 251)); + } + + for (i = 0; i < ARRAY_SIZE(uintp); i++) + assert(*uintp[i] == i); + + fprintf(stderr, "Dumping allocator areas after alloc\n"); + dump_allocator_info(a, tag0); + + fprintf(stderr, "Storing a copy of p (p1)\n"); + p1 = fy_allocator_store(a, tag0, p, psz, 1); + + fprintf(stderr, "Storing a copy of p (p2)\n"); + p2 = fy_allocator_store(a, tag0, p, psz, 1); + + fprintf(stderr, "Dumping allocator areas after double store p1=%p p2=%p\n", p1, p2); + dump_allocator_info(a, tag0); + + if (p1) { + for (i = 0; i < psz; i++) + assert(p1[i] == (i % 251)); + } + + if (p2) { + for (i = 0; i < psz; i++) + assert(p2[i] == (i % 251)); + } + + fprintf(stderr, "Allocator %p tag %u linear_size %zd\n", + a, tag0, fy_allocator_get_tag_linear_size(a, tag0)); + + linear_size = 0; + linear_data = fy_allocator_get_tag_single_linear(a, tag0, &linear_size); + fprintf(stderr, "Allocator %p tag %u linear_data %p linear_size 0x%zx\n", + a, tag0, linear_data, linear_size); + + fprintf(stderr, "Releasing tag0\n"); + fy_allocator_release_tag(a, tag0); + + fprintf(stderr, "Dumping allocator areas after release\n"); + dump_allocator_info(a, FY_ALLOC_TAG_NONE); + + if (a) + fy_allocator_destroy(a); + + if (pa) + fy_allocator_destroy(pa); + + + return 0; +} + +static struct option lopts[] = { + {"allocator", required_argument, 0, 'a' }, + {"parent", required_argument, 0, 'p' }, + {"size", required_argument, 0, 's' }, + {"help", no_argument, 0, 'h' }, + {0, 0, 0, 0 }, +}; + +static void display_usage(FILE *fp, const char *progname) +{ + char *names; + const char *s; + + names = fy_allocator_get_names(); + assert(names); + + s = strrchr(progname, '/'); + if (s != NULL) + progname = s + 1; + + fprintf(fp, "Usage:\n\t%s [options]\n", progname); + fprintf(fp, "\noptions:\n"); + fprintf(fp, "\t--allocator , -a : Use allocator, one of: %s\n", names); + fprintf(fp, "\t--parent , -p : Use parent allocator, one of: %s\n", names); + fprintf(fp, "\t--size , -s : Size for allocators that require one\n"); + fprintf(fp, "\t--help, -h : Display help message\n"); + fprintf(fp, "\n"); + + free(names); +} + +int main(int argc, char *argv[]) +{ + int opt, lidx, rc; + int exitcode = EXIT_FAILURE; + const char *allocator = "linear"; + const char *parent_allocator = "linear"; + size_t size = 0; + + while ((opt = getopt_long(argc, argv, "a:p:s:h", lopts, &lidx)) != -1) { + switch (opt) { + case 'a': + case 'p': + if (!fy_allocator_is_available(optarg)) { + fprintf(stderr, "Error: illegal allocator name \"%s\"\n", optarg); + goto err_out_usage; + } + if (opt == 'a') + allocator = optarg; + else + parent_allocator = optarg; + break; + case 's': + size = (size_t)atoi(optarg); + break; + case 'h' : + display_usage(stdout, argv[0]); + goto ok_out; + default: + goto err_out_usage; + } + } + + rc = allocator_test(allocator, parent_allocator, size); + if (rc) { + fprintf(stderr, "Error: allocator_test() failed\n"); + goto err_out; + } + +ok_out: + exitcode = EXIT_SUCCESS; + +out: + return exitcode; + +err_out_usage: + display_usage(stderr, argv[0]); +err_out: + exitcode = EXIT_FAILURE; + goto out; +} diff --git a/src/internal/fy-b3sum.c b/src/internal/fy-b3sum.c new file mode 100644 index 0000000..9e81e42 --- /dev/null +++ b/src/internal/fy-b3sum.c @@ -0,0 +1,549 @@ +/* + * fy-b3sum.c - blake3 utility for testing within fyaml + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + +#include +#include +#include +#include + +#include +#include + +#define OPT_NO_MMAP 128 +#define OPT_NO_MTHREAD 129 +#define OPT_NUM_THREADS 130 +#define OPT_BUFFER_SIZE 131 +#define OPT_MT_DEGREE 132 +#define OPT_LIST_BACKENDS 133 +#define OPT_ENABLE_CPUSIMD 134 +#define OPT_CPUSIMD_NUM_CPUS 135 +#define OPT_CPUSIMD_MULT_FACT 136 +#define OPT_DERIVE_KEY 137 +#define OPT_NO_NAMES 138 +#define OPT_RAW 139 +#define OPT_QUIET 140 +#define OPT_KEYED 141 + +static struct option lopts[] = { + {"check", no_argument, 0, 'c' }, + {"derive-key", required_argument, 0, OPT_DERIVE_KEY }, + {"no-names", no_argument, 0, OPT_NO_NAMES }, + {"raw", no_argument, 0, OPT_RAW }, + {"length", required_argument, 0, 'l' }, + {"quiet", no_argument, 0, OPT_QUIET }, + {"keyed", no_argument, 0, OPT_KEYED }, + + {"num-threads", required_argument, 0, OPT_NUM_THREADS }, + {"no-mmap", no_argument, 0, OPT_NO_MMAP }, + {"no-mthread", no_argument, 0, OPT_NO_MTHREAD }, + {"buffer-size", required_argument, 0, OPT_BUFFER_SIZE }, + {"mt-degree", required_argument, 0, OPT_MT_DEGREE }, + {"backend", required_argument, 0, 'b' }, + {"enable-cpusimd", no_argument, 0, OPT_ENABLE_CPUSIMD }, + {"cpusimd-num-cpus", required_argument, 0, OPT_CPUSIMD_NUM_CPUS }, + {"cpusimd-mult-fact", required_argument, 0, OPT_CPUSIMD_MULT_FACT }, + + {"list-backends", no_argument, 0, OPT_LIST_BACKENDS }, + {"help", no_argument, 0, 'h' }, + {"debug", no_argument, 0, 'd' }, + {0, 0, 0, 0 }, +}; + +static void display_usage(FILE *fp, const char *progname) +{ + const char *s; + + s = strrchr(progname, '/'); + if (s != NULL) + progname = s + 1; + + fprintf(fp, "Usage:\n\t%s [options] [args]\n", progname); + fprintf(fp, "\noptions:\n"); + fprintf(fp, "\t--derive-key : Key derivation mode, with the given context string\n"); + fprintf(fp, "\t--no-names : Omit filenames\n"); + fprintf(fp, "\t--raw : Output result in raw bytes (single input allowed)\n"); + fprintf(fp, "\t--length , -l : Output only this amount of bytes per output (max %u)\n", BLAKE3_OUT_LEN); + fprintf(fp, "\t--check, -c : Read files with BLAKE3 checksums and check files\n"); + fprintf(fp, "\t--quiet : Do not print OK for checked files that are correct\n"); + fprintf(fp, "\t--keyed : Keyed mode with secret key read from (32 raw bytes)\n"); + fprintf(fp, "\ntuning options:\n"); + fprintf(fp, "\t--num-threads : Number of threads to use (default: number of CPUs * 3 / 2)\n"); + fprintf(fp, "\t--no-mmap : Disable file mmap\n"); + fprintf(fp, "\t--no-mthread : Disable multithreading\n"); + fprintf(fp, "\t--buffer-size : Buffer size for file I/O\n"); + fprintf(fp, "\t--mt-degree : Set the multi-thread degree (default 128)\n"); + fprintf(fp, "\t--backend , -b : Backend selection\n"); + fprintf(fp, "\t--enable-cpusimd : Enable experimental CPUSIMD support\n"); + fprintf(fp, "\t--cpusimd-num-cpus : Number of CPUs assigned to CPUSIMD\n"); + fprintf(fp, "\t--cpusimd-mult-fact : Multiplication factor for CPUSIMD\n"); + fprintf(fp, "\ninformational options:\n"); + fprintf(fp, "\t--list-backends : List available backends\n"); + fprintf(fp, "\t--debug : Enable debug messages\n"); + fprintf(fp, "\t--help, -h : Display help message\n"); + fprintf(fp, "\n"); + fprintf(fp, "\nargs:\n"); + fprintf(fp, "\t... Files to hash or checkfiles to check.\n\tIf no file given (or file is '-' hash stdin\n"); +} + +static void list_backends(const char *name) +{ + const blake3_backend_info *bei; + uint64_t backends; + bool selected; + unsigned int i; + + backends = blake3_get_selectable_backends() & + blake3_get_detected_backends(); + for (i = 0; backends && i < B3BID_COUNT; i++) { + if (!(backends & ((uint64_t)1 << i))) + continue; + bei = blake3_get_backend_info(i); + if (!bei) + continue; + backends &= ~((uint64_t)1 << i); + + if (!name) + selected = backends == 0; + else + selected = !strcmp(name, bei->name); + + printf("%c %-12s\t%s\n", + selected ? '*' : ' ', bei->name, bei->description); + } +} + +static int do_hash_file(struct blake3_hasher *hasher, const char *filename, bool no_names, bool raw, unsigned int length) +{ + static const char *hexb = "0123456789abcdef"; + uint8_t output[BLAKE3_OUT_LEN]; + size_t filename_sz, line_sz, outsz; + ssize_t wrn; + uint8_t v; + void *outp; + char *line, *s; + unsigned int i; + int rc; + + filename_sz = strlen(filename); + + rc = blake3_hash_file(hasher, filename, output); + if (rc) { + fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno)); + return -1; + } + + if (!raw) { + /* output line (optimized) */ + line_sz = (length * 2); /* the hex output */ + if (!no_names) + line_sz += 2 + filename_sz; /* 2 spaces + filename */ + line_sz++; /* '\n' */ + line = alloca(line_sz + 1); + s = line; + for (i = 0; i < length; i++) { + v = output[i]; + *s++ = hexb[v >> 4]; + *s++ = hexb[v & 15]; + } + if (!no_names) { + *s++ = ' '; + *s++ = ' '; + memcpy(s, filename, filename_sz); + s += filename_sz; + } + *s++ = '\n'; + *s = '\0'; + outp = line; + outsz = (size_t)(s - line); + } else { + outp = output; + outsz = length; + } + + wrn = fwrite(outp, 1, outsz, stdout); + if ((size_t)wrn != outsz) { + fprintf(stderr, "Unable to write to stdout! error: %s\n", strerror(errno)); + return -1; + } + + return 0; +} + +static int do_check_file(struct blake3_hasher *hasher, const char *check_filename, bool quiet) +{ + char *hash, *filename; + FILE *fp = NULL; + char linebuf[8192]; /* maximum size for a line is 8K, should be enough (PATH_MAX is 4K at linux) */ + uint8_t read_hash[BLAKE3_OUT_LEN]; + uint8_t computed_hash[BLAKE3_OUT_LEN]; + uint8_t v; + char *s; + char c; + unsigned int i, j, length; + size_t linesz; + int line, rc, exit_code; + + if (check_filename && strcmp(check_filename, "-")) { + fp = fopen(check_filename, "ra"); + if (!fp) { + fprintf(stderr, "Failed to open check file: \"%s\", error: %s\n", check_filename, strerror(errno)); + goto err_out; + } + } else { + fp = stdin; + } + + /* default error code if all is fine */ + exit_code = 0; + + line = 0; + while (fgets(linebuf, sizeof(linebuf), fp)) { + /* '\0' terminate always */ + linebuf[(sizeof(linebuf)/sizeof(linebuf[0]))-1] = '\0'; + + linesz = strlen(linebuf); + while (linesz > 0 && linebuf[linesz-1] == '\n') + linesz--; + + if (!linesz) { + fprintf(stderr, "Empty line found at file \"%s\" line #%d\n", check_filename, line); + goto err_out; + } + linebuf[linesz] = '\0'; + + length = 0; + s = linebuf; + while (isxdigit((unsigned char)*s)) + s++; + + length = s - linebuf; + + if (length == 0 || length > (BLAKE3_OUT_LEN * 2) || (length % 1) || !isspace((unsigned char)*s)) { + fprintf(stderr, "Bad line found at file \"%s\" line #%d\n", check_filename, line); + fprintf(stderr, "%s\n", linebuf); + goto err_out; + } + + *s++ = '\0'; + + while (isspace((unsigned char)*s)) + s++; + + length >>= 1; /* to bytes */ + hash = linebuf; + filename = s; + + for (i = 0, s = hash; i < length; i++) { + v = 0; + for (j = 0; j < 2; j++) { + v <<= 4; + c = *s++; + if (c >= '0' && c <= '9') + v |= c - '0'; + else if (c >= 'a' && c <= 'f') + v |= c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + v |= c - 'A' + 10; + else + v = 0; + } + read_hash[i] = v; + } + + rc = blake3_hash_file(hasher, filename, computed_hash); + if (rc) { + fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno)); + goto err_out; + } + + /* constant time comparison */ + v = 0; + for (i = 0; i < length; i++) + v |= (read_hash[i] ^ computed_hash[i]); + + if (v) { + printf("%s: FAILED\n", filename); + exit_code = -1; + } else if (!quiet) + printf("%s: OK\n", filename); + } + +out: + if (fp && fp != stdin) + fclose(fp); + + return exit_code; + +err_out: + exit_code = -1; + goto out; +} + +int main(int argc, char *argv[]) +{ + blake3_host_config host_cfg; + struct blake3_host_state *host_state = NULL; + struct blake3_hasher *hasher = NULL; + const char *filename; + int i, opt, lidx, rc, num_inputs, num_ok; + int exitcode = EXIT_FAILURE, opti; + size_t buffer_size = 0; + bool no_names = false, raw = false, quiet = false, keyed = false; + bool no_mmap = false, no_mthread = false, debug = false, enable_cpusimd = false; + bool do_list_backends = false, check = false, derive_key = false; + unsigned int mt_degree = 0, num_threads = 0, cpusimd_num_cpus = 0, cpusimd_mult_fact = 0, length = BLAKE3_OUT_LEN; + const char *backend = NULL, *context = NULL; + uint8_t key[BLAKE3_OUT_LEN]; + ssize_t rdn; + + while ((opt = getopt_long(argc, argv, "cl:b:dh", lopts, &lidx)) != -1) { + switch (opt) { + + case OPT_DERIVE_KEY: + derive_key = true; + context = optarg; + break; + case OPT_NO_NAMES: + no_names = true; + break; + case OPT_RAW: + raw = true; + break; + case 'c': + check = true; + break; + case OPT_QUIET: + quiet = true; + break; + case OPT_KEYED: + keyed = true; + break; + + case 'l': + opti = atoi(optarg);; + if (opti <= 0 || opti > BLAKE3_OUT_LEN) { + fprintf(stderr, "Error: bad length=%d (must be > 0 and <= %u)\n\n", opti, BLAKE3_OUT_LEN); + goto err_out_usage; + } + length = (unsigned int)opti; + break; + + case OPT_NO_MMAP: + no_mmap = true; + break; + case OPT_NO_MTHREAD: + no_mthread = true; + break; + case OPT_NUM_THREADS: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad num_threads=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + num_threads = (unsigned int)opti; + break; + case OPT_BUFFER_SIZE: + opti = atoi(optarg); + if (opti <= 0) { + fprintf(stderr, "Error: bad buffer-size=%d (must be > 0)\n\n", opti); + goto err_out_usage; + } + buffer_size = (unsigned int)opti; + break; + case OPT_MT_DEGREE: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad mt_degree=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + mt_degree = (unsigned int)opti; + break; + + case OPT_LIST_BACKENDS: + do_list_backends = true; + break; + + case OPT_ENABLE_CPUSIMD: + enable_cpusimd = true; + break; + case OPT_CPUSIMD_NUM_CPUS: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad cpusimd_num_cpus=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + cpusimd_num_cpus = (unsigned int)opti; + break; + case OPT_CPUSIMD_MULT_FACT: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad cpusimd_mult_fact=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + cpusimd_mult_fact = (unsigned int)opti; + break; + + case 'b': + backend = optarg; + break; + + case 'd': + debug = true; + break; + case 'h' : + display_usage(stdout, argv[0]); + goto ok_out; + default: + goto err_out_usage; + } + } + + if (enable_cpusimd) { + rc = blake3_backend_cpusimd_setup(cpusimd_num_cpus, cpusimd_mult_fact); + if (rc) { + fprintf(stderr, "Unable to enable CPUSIMD\n"); + goto err_out; + } + } + + if (do_list_backends) { + list_backends(backend); + goto ok_out; + } + + if (quiet && !check) { + fprintf(stderr, "Error: --quiet may only be used together with --check\n\n"); + goto err_out_usage; + } + + if (keyed && derive_key) { + fprintf(stderr, "Error: --keyed and --derive-key may not be used together\n\n"); + goto err_out_usage; + } + + if (check && length != BLAKE3_OUT_LEN) { + fprintf(stderr, "Error: --check and --length may not be used together\n\n"); + goto err_out_usage; + } + + if (keyed) { + rdn = fread(key, 1, BLAKE3_KEY_LEN, stdin); + if (rdn != BLAKE3_KEY_LEN) { + if (rdn >= 0 && rdn < BLAKE3_KEY_LEN) + fprintf(stderr, "Error: could not read secret key from : short key\n\n"); + else + fprintf(stderr, "Error: could not read secret key from : error %s\n\n", strerror(errno)); + goto err_out_usage; + } + rc = fgetc(stdin); + if (rc != EOF) { + fprintf(stderr, "Error: garbage trailing secret key from \n\n"); + goto err_out_usage; + } + } + + memset(&host_cfg, 0, sizeof(host_cfg)); + host_cfg.debug = debug; + host_cfg.no_mthread = no_mthread; + host_cfg.no_mmap = no_mmap; + host_cfg.num_threads = num_threads; + host_cfg.backend = backend; + host_cfg.mt_degree = mt_degree; + host_cfg.file_io_bufsz = buffer_size; + + host_state = blake3_host_state_create(&host_cfg); + if (!host_state) { + fprintf(stderr, "unable to create blake3 host state\n"); + goto err_out; + } + + hasher = blake3_hasher_create(host_state, + keyed ? key : NULL, + derive_key ? context : NULL, 0); + if (!hasher) { + fprintf(stderr, "unable to create blake3 hasher\n"); + goto err_out; + } + + num_inputs = argc - optind; + if (num_inputs <= 0) + num_inputs = 1; /* stdin mode */ + + if (raw && num_inputs > 1) { + fprintf(stderr, "Error: Raw output mode is only supported with a single input\n\n"); + goto err_out_usage; + } + + if (!length) + length = BLAKE3_OUT_LEN; + + /* we will get in the loop even when no arguments (we'll do stdin instead) */ + num_ok = 0; + i = optind; + do { + /* if no arguments, use stdin */ + filename = i < argc ? argv[i] : "-"; + + /* we can't handle '-' in keyed mode */ + if (keyed && !strcmp(filename, "-")) { + fprintf(stderr, "Cannot use in keyed mode\n"); + goto err_out_usage; + } + + if (!check) + rc = do_hash_file(hasher, filename, no_names, raw, length); + else + rc = do_check_file(hasher, filename, quiet); + if (!rc) + num_ok++; + + } while (++i < argc); + + if (num_inputs != num_ok) + goto err_out; + +ok_out: + exitcode = EXIT_SUCCESS; + +out: + if (hasher) + blake3_hasher_destroy(hasher); + + if (host_state) + blake3_host_state_destroy(host_state); + + if (enable_cpusimd) + blake3_backend_cpusimd_cleanup(); + + return exitcode; + +err_out_usage: + display_usage(stderr, argv[0]); +err_out: + exitcode = EXIT_FAILURE; + goto out; +} diff --git a/src/internal/fy-thread.c b/src/internal/fy-thread.c new file mode 100644 index 0000000..ac21563 --- /dev/null +++ b/src/internal/fy-thread.c @@ -0,0 +1,592 @@ +/* + * fy-thread.c - thread testing internal utility + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "fy-atomics.h" +#include "fy-thread.h" + +static void test_worker_thread_fn(void *arg) +{ + FY_ATOMIC(int) *p = arg; + int v, exp_v; + + /* atomically increase the counter */ + v = fy_atomic_load(p); + for (;;) { + exp_v = v; + if (fy_atomic_compare_exchange_strong(p, &exp_v, v + 1)) + return; + v = exp_v; + } +} + +void test_worker_threads(unsigned int num_threads) +{ + struct fy_thread_pool_cfg tp_cfg; + struct fy_thread_pool *tp; + struct fy_thread **threads, *t; + struct fy_thread_work *works; + long scval; + unsigned int i, count, num_cpus; + int rc, test_count; + + (void)rc; + + if (num_threads == 0) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } else + num_cpus = num_threads; + + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = 0; + tp_cfg.num_threads = num_cpus; + tp_cfg.userdata = NULL; + + fprintf(stderr, "calling: fy_thread_pool_create()\n"); + + tp = fy_thread_pool_create(&tp_cfg); + assert(tp); + + count = tp->num_threads; + threads = alloca(count * sizeof(*threads)); + works = alloca(count * sizeof(*works)); + + test_count = 0; + + for (i = 0; i < count; i++) { + fprintf(stderr, "calling: fy_thread_reserve(#%u)\n", i); + threads[i] = fy_thread_reserve(tp); + assert(threads[i]); + t = threads[i]; + assert(t->id == i); + } + + for (i = 0; i < count; i++) { + t = threads[i]; + fprintf(stderr, "calling: fy_thread_submit_work(#%u)\n", i); + works[i].fn = test_worker_thread_fn; + works[i].arg = &test_count; + fy_thread_submit_work(t, &works[i]); + } + + for (i = 0; i < count; i++) { + t = threads[i]; + fprintf(stderr, "calling: fy_thread_wait_work(#%u)\n", i); + fy_thread_wait_work(t); + } + + fprintf(stderr, "%s: test_count=%d\n", __func__, test_count); + if (test_count != (int)num_cpus) { + fprintf(stderr, "error: test_count=%d expected %d\n", test_count, (int)num_cpus); + abort(); + } + + for (i = 0; i < count; i++) { + t = threads[i]; + fprintf(stderr, "calling: fy_thread_pool_unreserve(#%u)\n", i); + fy_thread_unreserve(t); + } + + fprintf(stderr, "calling: fy_thread_pool_destroy()\n"); + fy_thread_pool_destroy(tp); +} + +void test_thread_join(unsigned int num_threads) +{ + struct fy_thread_pool_cfg tp_cfg; + struct fy_thread_pool *tp; + void **args; + long scval; + unsigned int count, num_cpus; + int rc, test_count; + + (void)rc; + + if (num_threads == 0) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } else + num_cpus = num_threads; + + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = 0; + tp_cfg.num_threads = num_cpus; + tp_cfg.userdata = NULL; + + fprintf(stderr, "calling: fy_thread_pool_create()\n"); + tp = fy_thread_pool_create(&tp_cfg); + assert(tp); + + count = tp->num_threads; + args = alloca(count * sizeof(*args)); + + test_count = 0; + + fy_thread_arg_join(tp, test_worker_thread_fn, NULL, &test_count, count); + + fprintf(stderr, "%s: test_count=%d\n", __func__, test_count); + assert(test_count == (int)num_cpus); + + fprintf(stderr, "calling: fy_thread_pool_destroy()\n"); + fy_thread_pool_destroy(tp); +} + +struct thread_latency_state { + struct timespec reserve; + struct timespec reserve_done; + struct timespec submit; + struct timespec execute; + struct timespec wait; + struct timespec wait_done; + struct timespec unreserve; + struct timespec unreserve_done; +}; + +long long delta_ns(struct timespec before, struct timespec after) +{ + if ((before.tv_sec == 0 && before.tv_nsec == 0) || + (after.tv_sec == 0 && after.tv_nsec == 0)) + return -1; + return (long long)((int64_t)(after.tv_sec - before.tv_sec) * (int64_t)1000000000UL + (int64_t)(after.tv_nsec - before.tv_nsec)); +} + +static void test_latency_worker_thread_fn(void *arg) +{ + struct thread_latency_state *s = arg; + + clock_gettime(CLOCK_MONOTONIC, &s->execute); +} + +void test_thread_latency(unsigned int num_threads) +{ + struct fy_thread_pool_cfg tp_cfg; + struct fy_thread_pool *tp; + struct fy_thread **threads, *t; + struct fy_thread_work *works; + long scval; + unsigned int i, count, num_cpus; + int rc; + struct thread_latency_state *states, *s; + + (void)rc; + + if (num_threads == 0) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } else + num_cpus = num_threads; + + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = 0; + tp_cfg.num_threads = num_cpus; + tp_cfg.userdata = NULL; + + tp = fy_thread_pool_create(&tp_cfg); + assert(tp); + + + count = tp->num_threads; + threads = alloca(count * sizeof(*threads)); + works = alloca(count * sizeof(*works)); + states = alloca(count * sizeof(*states)); + + memset(states, 0, count * sizeof(*states)); + + for (i = 0; i < count; i++) { + s = &states[i]; + + clock_gettime(CLOCK_MONOTONIC, &s->reserve); + + threads[i] = fy_thread_reserve(tp); + assert(threads[i]); + t = threads[i]; + assert(t->id == i); + + clock_gettime(CLOCK_MONOTONIC, &s->reserve_done); + + } + + for (i = 0; i < count; i++) { + s = &states[i]; + clock_gettime(CLOCK_MONOTONIC, &s->submit); + + t = threads[i]; + works[i].fn = test_latency_worker_thread_fn; + works[i].arg = s; + fy_thread_submit_work(t, &works[i]); + } + + for (i = 0; i < count; i++) { + s = &states[i]; + clock_gettime(CLOCK_MONOTONIC, &s->wait); + + t = threads[i]; + fy_thread_wait_work(t); + + clock_gettime(CLOCK_MONOTONIC, &s->wait_done); + } + + for (i = 0; i < count; i++) { + + s = &states[i]; + clock_gettime(CLOCK_MONOTONIC, &s->unreserve); + + t = threads[i]; + fy_thread_unreserve(t); + + clock_gettime(CLOCK_MONOTONIC, &s->unreserve_done); + } + + fy_thread_pool_destroy(tp); + + fprintf(stderr, "latency results\n"); + for (i = 0; i < count; i++) { + s = &states[i]; + + fprintf(stderr, "#%2u: reserve:%10lld submit-execute:%10lld execute-waitdone:%10lld wait:%10lld unreserve:%10lld\n", i, + delta_ns(s->reserve, s->reserve_done), + delta_ns(s->submit, s->execute), + delta_ns(s->execute, s->wait_done), + delta_ns(s->wait, s->wait_done), + delta_ns(s->unreserve, s->unreserve_done)); + } +} + +// #define STEAL_LOOP_COUNT 100000000 +#define STEAL_LOOP_COUNT 10000 +static void test_worker_thread_steal_fn(void *arg) +{ + FY_ATOMIC(int) *p = arg; + int v, exp_v; + unsigned int i; + + /* atomically increase the counter STEAL_LOOP_COUNT times */ + for (i = 0; i < STEAL_LOOP_COUNT; i++) { + v = fy_atomic_load(p); + for (;;) { + exp_v = v; + if (fy_atomic_compare_exchange_strong(p, &exp_v, v + 1)) + break; + v = exp_v; + } + } +} + +void test_thread_join_steal(unsigned int num_threads) +{ + struct fy_thread_pool_cfg tp_cfg; + struct fy_thread_pool *tp; + void **args; + long scval; + unsigned int count, num_cpus; + int rc, test_count; + + (void)rc; + + if (num_threads == 0) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } else + num_cpus = num_threads; + + + tp_cfg.flags = FYTPCF_STEAL_MODE; + tp_cfg.num_threads = num_cpus; + tp_cfg.userdata = NULL; + + fprintf(stderr, "calling: fy_thread_pool_create()\n"); + tp = fy_thread_pool_create(&tp_cfg); + assert(tp); + + count = tp->num_threads * 4; + args = alloca(count * sizeof(*args)); + + test_count = 0; + + fy_thread_arg_join(tp, test_worker_thread_steal_fn, NULL, &test_count, count); + + fprintf(stderr, "%s: test_count=%d\n", __func__, test_count); + if (test_count != (int)num_cpus * STEAL_LOOP_COUNT * 4) { + fprintf(stderr, "error: test_count=%d expected %d\n", test_count, (int)num_cpus * STEAL_LOOP_COUNT); + abort(); + } + + fprintf(stderr, "calling: fy_thread_pool_destroy()\n"); + fy_thread_pool_destroy(tp); +} + +struct sum_args { + struct fy_thread_pool *tp; + const uint8_t *values_start; + unsigned int count_start; + const uint8_t *values; + unsigned int count; + uint64_t sum; +}; + +static uint64_t calc_sum(const uint8_t *values, unsigned int count) +{ + unsigned int i; + uint64_t sum = 0; + + for (i = 0; i < count; i++) + sum += values[i]; + + return sum; +} + +static void test_worker_thread_sum_fn(void *arg) +{ + struct sum_args *s = arg; + struct sum_args args[2]; + uint64_t sum; + unsigned int pos; + + pos = s->values - s->values_start; + assert(pos <= s->count_start); + assert(pos + s->count <= s->count_start); + + (void)pos; + + // if (s->count <= (1 << 20) / 8) { + if (s->count <= 4096) { + // fprintf(stderr, "S<%06x-%06x>\n", pos, pos + s->count - 1); + sum = calc_sum(s->values, s->count); + } else { + memset(args, 0, sizeof(args)); + args[0].tp = args[1].tp = s->tp; + args[0].values_start = args[1].values_start = s->values_start; + args[0].count_start = args[1].count_start = s->count_start; + args[0].sum = args[1].sum = 0; + args[0].values = s->values; + args[0].count = s->count / 2; + args[1].values = s->values + args[0].count; + args[1].count = s->count - args[0].count; + + // fprintf(stderr, "M<%06x-%06x,%06x-%06x>\n", + // pos, + // pos + args[0].count - 1, + // pos + args[0].count, + // pos + args[0].count + args[1].count - 1); + + fy_thread_arg_array_join(s->tp, test_worker_thread_sum_fn, NULL, &args, sizeof(args[0]), sizeof(args)/sizeof(args[0])); + sum = args[0].sum + args[1].sum; + } + s->sum = sum; +} + +void test_thread_join_sum(unsigned int num_threads, unsigned int count, bool steal_mode, unsigned int times) +{ + struct fy_thread_pool_cfg tp_cfg; + struct fy_thread_pool *tp; + struct timespec before, after; + unsigned int i, num_cpus; + long scval; + uint8_t *values; + int rc; + uint64_t sum_single, sum_multi; + struct sum_args args[2]; + long long table_multi[times]; + long long ns; + + (void)rc; + + fprintf(stderr, "**********************************************************************\n"); + fprintf(stderr, "%s: steal_mode=%s\n", __func__, steal_mode ? "true" : "false"); + + values = malloc(count * sizeof(*values)); + assert(values); + + clock_gettime(CLOCK_MONOTONIC, &before); + for (i = 0; i < count; i++) + values[i] = (uint8_t)rand(); + + clock_gettime(CLOCK_MONOTONIC, &after); + fprintf(stderr, "%s: seeding done in %lldus\n", __func__, delta_ns(before, after) / 1000); + + clock_gettime(CLOCK_MONOTONIC, &before); + sum_single = calc_sum(values, count); + clock_gettime(CLOCK_MONOTONIC, &after); + ns = delta_ns(before, after); + fprintf(stderr, "%s: calculated sum=%"PRIu64" (single threaded) done in %lldus\n", __func__, sum_single, ns / 1000); + + if (num_threads == 0) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_cpus = (unsigned int)scval; + } else + num_cpus = num_threads; + + memset(&tp_cfg, 0, sizeof(tp_cfg)); + tp_cfg.flags = steal_mode ? FYTPCF_STEAL_MODE : 0; + tp_cfg.num_threads = num_cpus; + tp_cfg.userdata = NULL; + + tp = fy_thread_pool_create(&tp_cfg); + assert(tp); + + fprintf(stderr, "%s: calculating (multi threaded) -", __func__); + for (i = 0; i < times; i++) { + clock_gettime(CLOCK_MONOTONIC, &before); + + memset(args, 0, sizeof(args)); + args[0].tp = args[1].tp = tp; + args[0].values_start = args[1].values_start = values; + args[0].count_start = args[1].count_start = count; + args[0].sum = args[1].sum = 0; + + args[0].values = values; + args[0].count = count / 2; + args[1].values = values + args[0].count; + args[1].count = count - args[0].count; + + // fprintf(stderr, "M<%06x-%06x,%06x-%06x>\n", + // 0, + // args[0].count - 1, + // args[0].count, + // args[0].count + args[1].count - 1); + + fy_thread_arg_array_join(tp, test_worker_thread_sum_fn, NULL, &args, sizeof(args[0]), sizeof(args)/sizeof(args[0])); + + sum_multi = args[0].sum + args[1].sum; + if (sum_multi != sum_single) { + fprintf(stderr, "\nFailed sum_multi %"PRIu64" should be %"PRIu64"\n", sum_multi, sum_single); + abort(); + } + + clock_gettime(CLOCK_MONOTONIC, &after); + ns = delta_ns(before, after); + fprintf(stderr, " %lldus", ns / 1000); + fflush(stderr); + table_multi[i] = ns; + } + ns = 0; + for (i = 0; i < times; i++) + ns += table_multi[i]; + ns /= times; + fprintf(stderr, " : average %lldus\n", ns / 1000); + + fy_thread_pool_destroy(tp); + + free(values); +} + +int thread_test(unsigned int num_threads) +{ +#if 0 + test_worker_threads(num_threads); + test_thread_join(num_threads); + test_thread_latency(num_threads); + test_thread_join_steal(num_threads); +#endif + test_thread_join_sum(num_threads, 1 << 20, false, 10); /* 1M of values */ + test_thread_join_sum(num_threads, 1 << 20, true, 10); /* 1M of values */ + + return 0; +} + +#define OPT_NUM_THREADS 128 + +static struct option lopts[] = { + {"num-threads", required_argument, 0, OPT_NUM_THREADS }, + {"help", no_argument, 0, 'h' }, + {0, 0, 0, 0 }, +}; + +static void display_usage(FILE *fp, const char *progname) +{ + const char *s; + + s = strrchr(progname, '/'); + if (s != NULL) + progname = s + 1; + + fprintf(fp, "Usage:\n\t%s [options]\n", progname); + fprintf(fp, "\noptions:\n"); + fprintf(fp, "\t--num-threads : Number of threads to use (default: number of CPUs * 3 / 2)\n"); + fprintf(fp, "\t--help, -h : Display help message\n"); + fprintf(fp, "\n"); +} + +int main(int argc, char *argv[]) +{ + int opt, lidx, rc; + unsigned int num_threads = 0; + int exitcode = EXIT_FAILURE, opti; + + while ((opt = getopt_long(argc, argv, "h", lopts, &lidx)) != -1) { + switch (opt) { + case OPT_NUM_THREADS: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad num_threads=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + num_threads = (unsigned int)opti; + break; + case 'h' : + display_usage(stdout, argv[0]); + goto ok_out; + default: + goto err_out_usage; + } + } + + rc = thread_test(num_threads); + if (rc) { + fprintf(stderr, "Error: thread_test() failed\n"); + goto err_out; + } + +ok_out: + exitcode = EXIT_SUCCESS; + +out: + return exitcode; + +err_out_usage: + display_usage(stderr, argv[0]); +err_out: + exitcode = EXIT_FAILURE; + goto out; +} diff --git a/src/internal/libfyaml-parser.c b/src/internal/libfyaml-parser.c new file mode 100644 index 0000000..8d0fb8d --- /dev/null +++ b/src/internal/libfyaml-parser.c @@ -0,0 +1,4622 @@ +/* + * libfyaml-parser.c - swiss army knife testing of libfyaml+libyaml + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if defined(HAVE_LIBYAML) && HAVE_LIBYAML +#include +#endif + +#include "fy-parse.h" +#include "fy-walk.h" +#include "fy-blob.h" + +#include "fy-valgrind.h" + +#include "xxhash.h" + +#define QUIET_DEFAULT false +#define INCLUDE_DEFAULT "" +#define MODE_DEFAULT "parse" +#define DEBUG_LEVEL_DEFAULT FYET_WARNING +#define INDENT_DEFAULT 2 +#define WIDTH_DEFAULT 80 +#define RESOLVE_DEFAULT false +#define SORT_DEFAULT false +#define CHUNK_DEFAULT 0 +#define COLOR_DEFAULT "auto" +#define MMAP_DISABLE_DEFAULT false + +#define OPT_DISABLE_MMAP 128 +#define OPT_USE_CALLBACK 129 +#define OPT_DISABLE_ACCEL 130 +#define OPT_DISABLE_BUFFERING 131 +#define OPT_DISABLE_DEPTH_LIMIT 132 +#define OPT_NULL_OUTPUT 133 + +#define OPT_SLOPPY_FLOW_INDENTATION 2007 +#define OPT_YPATH_ALIASES 2008 + +#define OPT_YAML_1_1 4000 +#define OPT_YAML_1_2 4001 +#define OPT_YAML_1_3 4002 + +static struct option lopts[] = { + {"include", required_argument, 0, 'I' }, + {"mode", required_argument, 0, 'm' }, + {"debug-level", required_argument, 0, 'd' }, + {"indent", required_argument, 0, 'i' }, + {"width", required_argument, 0, 'w' }, + {"resolve", no_argument, 0, 'r' }, + {"sort", no_argument, 0, 's' }, + {"chunk", required_argument, 0, 'c' }, + {"color", required_argument, 0, 'C' }, + {"diag", required_argument, 0, 'D' }, + {"module", required_argument, 0, 'M' }, + {"disable-mmap", no_argument, 0, OPT_DISABLE_MMAP }, + {"disable-accel", no_argument, 0, OPT_DISABLE_ACCEL }, + {"disable-buffering", no_argument, 0, OPT_DISABLE_BUFFERING }, + {"disable-depth-limit", no_argument, 0, OPT_DISABLE_DEPTH_LIMIT }, + {"use-callback", no_argument, 0, OPT_USE_CALLBACK }, + {"null-output", no_argument, 0, OPT_NULL_OUTPUT }, + {"walk-path", required_argument, 0, 'W' }, + {"walk-start", required_argument, 0, 'S' }, + {"yaml-1.1", no_argument, 0, OPT_YAML_1_1 }, + {"yaml-1.2", no_argument, 0, OPT_YAML_1_2 }, + {"yaml-1.3", no_argument, 0, OPT_YAML_1_3 }, + {"sloppy-flow-indentation", no_argument, 0, OPT_SLOPPY_FLOW_INDENTATION }, + {"ypath-aliases", no_argument, 0, OPT_YPATH_ALIASES }, + {"quiet", no_argument, 0, 'q' }, + {"help", no_argument, 0, 'h' }, + {0, 0, 0, 0 }, +}; + +#if defined(HAVE_LIBYAML) && HAVE_LIBYAML +#define LIBYAML_MODES "|libyaml-scan|libyaml-parse|libyaml-testsuite|libyaml-dump|libyaml-diff" +#else +#define LIBYAML_MODES "" +#endif + +#define MODES "parse|scan|copy|testsuite|dump|dump2|build|walk|reader|compose|iterate|comment|pathspec|shell-split|parse-timing" LIBYAML_MODES + +static void display_usage(FILE *fp, char *progname) +{ + fprintf(fp, "Usage: %s [options] [files]\n", progname); + fprintf(fp, "\nOptions:\n\n"); + fprintf(fp, "\t--include, -I : Add directory to include path " + " (default path \"%s\")\n", + INCLUDE_DEFAULT); + fprintf(fp, "\t--mode, -m : Set mode [" MODES "]" + " (default mode \"%s\")\n", + MODE_DEFAULT); + fprintf(fp, "\t--debug-level, -d : Set debug level to " + "(default level %d)\n", + DEBUG_LEVEL_DEFAULT); + fprintf(fp, "\t--indent, -i : Set dump indent to " + " (default indent %d)\n", + INDENT_DEFAULT); + fprintf(fp, "\t--width, -w : Set dump width to " + " (default width %d)\n", + WIDTH_DEFAULT); + fprintf(fp, "\t--resolve, -r : Perform anchor and merge key resolution" + " (default %s)\n", + RESOLVE_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--sort, -s : Perform mapping key sort (valid for dump)" + " (default %s)\n", + SORT_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--color, -C : Color output can be one of on, off, auto" + " (default %s)\n", + COLOR_DEFAULT); + fprintf(fp, "\t--chunk, -c : Set buffer chunk to " + " (default is %d - 0 means PAGE_SIZE)\n", + CHUNK_DEFAULT); + fprintf(fp, "\t--diag, -D : Set debug message diagnostic meta" + " (source, position, type, module, all, none)\n"); + fprintf(fp, "\t--module, -M : Set debug message module enable" + " (unknown, atom, scan, parse, doc, build, internal, system, all, none)\n"); + fprintf(fp, "\t--walk-path, -W : Walk path for work mode\n"); + fprintf(fp, "\t--quiet, -q : Quiet operation, do not " + "output messages (default %s)\n", + QUIET_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--help, -h : Display help message\n"); + fprintf(fp, "\ne.g. %s input.yaml\n", progname); + + if (fp == stderr) + exit(EXIT_FAILURE); +} + +void print_escaped(FILE *fp, const char *str, int length) +{ + fprintf(fp, "%s", fy_utf8_format_text_a(str, length, fyue_doublequote)); +} + +static int txt2esc_internal(const char *s, int l, char *out, int *outszp, int delim) +{ + const char *e; + int ll; + char c; + char *o = NULL, *oe = NULL; + + e = s + l; + if (out) { + o = out; + oe = o + *outszp; + } + +#define O_CH(_c) \ + do { \ + ll++; \ + if (o && (oe - o) > 0) \ + *o++ = (_c); \ + } while(0) + + ll = 0; + while (s < e) { + c = *s++; + if (delim > 0 && c == delim) { + O_CH('\\'); + } else if (c == '\0' || strchr("\a\b\t\n\v\f\r\e", c)) { + /* normal 1 -> 2 character escapes */ + O_CH('\\'); + switch (c) { + case '\0': + c = '0'; + break; + case '\a': + c = 'a'; + break; + case '\b': + c = 'b'; + break; + case '\t': + c = 't'; + break; + case '\n': + c = 'n'; + break; + case '\v': + c = 'v'; + break; + case '\f': + c = 'f'; + break; + case '\r': + c = 'r'; + break; + case '\e': + c = 'e'; + break; + } + } else if ((e - s) >= 1 && (uint8_t)c == 0xc2 && + ((uint8_t)s[1] == 0x85 || (uint8_t)s[1] == 0xa0)) { + /* \N & \_ unicode escapes 2 -> 1 */ + O_CH('\\'); + if ((uint8_t)s[1] == 0x85) + c = 'N'; + else + c = '_'; + } else if ((e - s) >= 2 && (uint8_t)c == 0xe2 && (uint8_t)s[1] == 0x80 && + ((uint8_t)s[2] == 0xa8 || (uint8_t)s[2] == 0xa9)) { + /* \L & \P unicode escapes 3 -> 1 */ + O_CH('\\'); + if ((uint8_t)s[2] == 0xa8) + c = 'L'; + else + c = 'P'; + } + O_CH(c); + } + /* terminating \0 */ + O_CH('\0'); + + if (out) + *outszp = oe - o; + + return ll; +} + +static int txt2esc_length(const char *s, int l, int delim) +{ + if (l < 0) + l = strlen(s); + return txt2esc_internal(s, l, NULL, NULL, delim); +} + +static char *txt2esc_format(const char *s, int l, char *buf, int maxsz, int delim) +{ + if (l < 0) + l = strlen(s); + + txt2esc_internal(s, l, buf, &maxsz, delim); + return buf; +} + +#define fy_atom_get_text_a(_atom) \ + ({ \ + struct fy_atom *_a = (_atom); \ + int _len; \ + char *_buf; \ + const char *_txt = ""; \ + \ + if (!_a->direct_output) { \ + _len = fy_atom_format_text_length(_a); \ + if (_len > 0) { \ + _buf = alloca(_len + 1); \ + memset(_buf, 0, _len + 1); \ + fy_atom_format_text(_a, _buf, _len + 1); \ + _buf[_len] = '\0'; \ + _txt = _buf; \ + } \ + } else { \ + _len = fy_atom_size(_a); \ + _buf = alloca(_len + 1); \ + memset(_buf, 0, _len + 1); \ + memcpy(_buf, fy_atom_data(_a), _len); \ + _buf[_len] = '\0'; \ + _txt = _buf; \ + } \ + _txt; \ + }) + +#define txt2esc_a(_s, _l) \ + ({ \ + const char *__s = (const void *)(_s); \ + int __l = (_l); \ + int _ll = txt2esc_length(__s, __l, '\''); \ + txt2esc_format(__s, __l, alloca(_ll + 1), _ll + 1, '\''); \ + }) + +#define fy_atom_get_esc_text_a(_atom) txt2esc_a(fy_atom_get_text_a(_atom), -1) +#define fy_token_get_esc_text_a(_atom) txt2esc_a(fy_token_get_text0(_atom), -1) + +void dump_event(struct fy_parser *fyp, struct fy_event *fye) +{ + char mbuf[40]; + char *mm; + char *anchor = NULL, *tag = NULL, *value = NULL; + + snprintf(mbuf, sizeof(mbuf), " %10s-%-10s ", + "", ""); + mm = mbuf; + mm = " "; + + switch (fye->type) { + case FYET_NONE: + printf("NO\n"); + break; + case FYET_STREAM_START: + printf("%-14s%s|\n", "STREAM_START", mm); + break; + case FYET_STREAM_END: + printf("%-14s%s|\n", "STREAM_END", mm); + break; + case FYET_DOCUMENT_START: + printf("%-14s%s|\n", "DOCUMENT_START", mm); + break; + case FYET_DOCUMENT_END: + printf("%-14s%s|\n", "DOCUMENT_END", mm); + break; + case FYET_ALIAS: + anchor = fy_token_get_esc_text_a(fye->alias.anchor); + printf("%-14s%s| '%s'\n", "ALIAS", mm, anchor); + break; + case FYET_SCALAR: + if (fye->scalar.anchor) + anchor = fy_token_get_esc_text_a(fye->scalar.anchor); + if (fye->scalar.tag) + tag = fy_token_get_esc_text_a(fye->scalar.tag); + if (fye->scalar.value) + value = fy_token_get_esc_text_a(fye->scalar.value); + printf("%-14s%s|%s%s%s%s%s%s '%s'\n", "SCALAR", mm, + anchor ? " anchor='" : "", anchor ? : "", anchor ? "'" : "", + tag ? " tag='" : "", tag ? : "", tag ? "'" : "", + value ? : ""); + break; + case FYET_SEQUENCE_START: + if (fye->sequence_start.anchor) + anchor = fy_token_get_esc_text_a(fye->sequence_start.anchor); + if (fye->sequence_start.tag) + tag = fy_token_get_esc_text_a(fye->sequence_start.tag); + printf("%-14s%s|%s%s%s%s%s%s\n", "SEQUENCE_START", mm, + anchor ? " anchor='" : "", anchor ? : "", anchor ? "'" : "", + tag ? " tag='" : "", tag ? : "", tag ? "'" : ""); + break; + case FYET_SEQUENCE_END: + printf("%-14s%s|\n", "SEQUENCE_END", mm); + break; + case FYET_MAPPING_START: + if (fye->mapping_start.anchor) + anchor = fy_token_get_esc_text_a(fye->mapping_start.anchor); + if (fye->mapping_start.tag) + tag = fy_token_get_esc_text_a(fye->mapping_start.tag); + printf("%-14s%s|%s%s%s%s%s%s\n", "MAPPING_START", mm, + anchor ? " anchor='" : "", anchor ? : "", anchor ? "'" : "", + tag ? " tag='" : "", tag ? : "", tag ? "'" : ""); + break; + case FYET_MAPPING_END: + printf("%-14s%s|\n", "MAPPING_END", mm); + break; + default: + FY_IMPOSSIBLE_ABORT(); + } +} + +int do_parse(struct fy_parser *fyp) +{ + struct fy_eventp *fyep; + + while ((fyep = fy_parse_private(fyp)) != NULL) { + dump_event(fyp, &fyep->e); + fy_parse_eventp_recycle(fyp, fyep); + } + + return fyp->stream_error ? -1 : 0; +} + +void dump_testsuite_event(FILE *fp, struct fy_parser *fyp, struct fy_event *fye) +{ + const char *anchor = NULL, *tag = NULL, *value = NULL; + size_t anchor_len = 0, tag_len = 0, value_len = 0; + enum fy_scalar_style style; + + switch (fye->type) { + case FYET_NONE: + fprintf(fp, "???\n"); + break; + case FYET_STREAM_START: + fprintf(fp, "+STR\n"); + break; + case FYET_STREAM_END: + fprintf(fp, "-STR\n"); + break; + case FYET_DOCUMENT_START: + fprintf(fp, "+DOC%s\n", !fy_document_event_is_implicit(fye) ? " ---" : ""); + break; + case FYET_DOCUMENT_END: + fprintf(fp, "-DOC%s\n", !fy_document_event_is_implicit(fye) ? " ..." : ""); + break; + case FYET_MAPPING_START: + if (fye->mapping_start.anchor) + anchor = fy_token_get_text(fye->mapping_start.anchor, &anchor_len); + if (fye->mapping_start.tag) + tag = fy_token_get_text(fye->mapping_start.tag, &tag_len); + fprintf(fp, "+MAP"); + if (anchor) + fprintf(fp, " &%.*s", (int)anchor_len, anchor); + if (tag) + fprintf(fp, " <%.*s>", (int)tag_len, tag); + fprintf(fp, "\n"); + break; + case FYET_MAPPING_END: + fprintf(fp, "-MAP\n"); + break; + case FYET_SEQUENCE_START: + if (fye->sequence_start.anchor) + anchor = fy_token_get_text(fye->sequence_start.anchor, &anchor_len); + if (fye->sequence_start.tag) + tag = fy_token_get_text(fye->sequence_start.tag, &tag_len); + fprintf(fp, "+SEQ"); + if (anchor) + fprintf(fp, " &%.*s", (int)anchor_len, anchor); + if (tag) + fprintf(fp, " <%.*s>", (int)tag_len, tag); + fprintf(fp, "\n"); + break; + case FYET_SEQUENCE_END: + fprintf(fp, "-SEQ\n"); + break; + case FYET_SCALAR: + if (fye->scalar.anchor) + anchor = fy_token_get_text(fye->scalar.anchor, &anchor_len); + if (fye->scalar.tag) + tag = fy_token_get_text(fye->scalar.tag, &tag_len); + + if (fye->scalar.value) + value = fy_token_get_text(fye->scalar.value, &value_len); + + fprintf(fp, "=VAL"); + if (anchor) + fprintf(fp, " &%.*s", (int)anchor_len, anchor); + if (tag) + fprintf(fp, " <%.*s>", (int)tag_len, tag); + + style = fy_token_scalar_style(fye->scalar.value); + switch (style) { + case FYAS_PLAIN: + fprintf(fp, " :"); + break; + case FYAS_SINGLE_QUOTED: + fprintf(fp, " '"); + break; + case FYAS_DOUBLE_QUOTED: + fprintf(fp, " \""); + break; + case FYAS_LITERAL: + fprintf(fp, " |"); + break; + case FYAS_FOLDED: + fprintf(fp, " >"); + break; + default: + FY_IMPOSSIBLE_ABORT(); + } + print_escaped(fp, value, value_len); + fprintf(fp, "\n"); + break; + case FYET_ALIAS: + anchor = fy_token_get_text(fye->alias.anchor, &anchor_len); + fprintf(fp, "=ALI *%.*s\n", (int)anchor_len, anchor); + break; + default: + FY_IMPOSSIBLE_ABORT(); + } +} + +int do_testsuite(FILE *fp, struct fy_parser *fyp, bool null_output) +{ + struct fy_eventp *fyep; + + while ((fyep = fy_parse_private(fyp)) != NULL) { + if (!null_output) + dump_testsuite_event(fp, fyp, &fyep->e); + fy_parse_eventp_recycle(fyp, fyep); + } + + return fyp->stream_error ? -1 : 0; +} + +static void dump_token(struct fy_token *fyt) +{ + const char *style; + const struct fy_version *vers; + const char *handle, *prefix, *suffix; + const char *typetxt; + + typetxt = fy_token_type_txt[fyt->type]; + assert(typetxt); + + switch (fyt->type) { + case FYTT_VERSION_DIRECTIVE: + vers = fy_version_directive_token_version(fyt); + assert(vers); + printf("%s value=%d.%d\n", typetxt, + vers->major, vers->minor); + break; + case FYTT_TAG_DIRECTIVE: + handle = fy_tag_directive_token_handle0(fyt); + if (!handle) + handle = ""; + prefix = fy_tag_directive_token_prefix0(fyt); + if (!prefix) + prefix = ""; + printf("%s handle='%s' prefix='%s'\n", typetxt, + txt2esc_a(handle, -1), + txt2esc_a(prefix, -1)); + break; + case FYTT_ALIAS: + printf("%s value='%s'\n", typetxt, + fy_atom_get_esc_text_a(&fyt->handle)); + break; + case FYTT_ANCHOR: + printf("%s value='%s'\n", typetxt, + fy_atom_get_esc_text_a(&fyt->handle)); + break; + case FYTT_TAG: + handle = fy_tag_token_handle0(fyt); + if (!handle) + handle = ""; + suffix = fy_tag_token_suffix0(fyt); + if (!suffix) + suffix = ""; + printf("%s handle='%s' suffix='%s'\n", typetxt, + txt2esc_a(handle, -1), + txt2esc_a(suffix, -1)); + break; + case FYTT_SCALAR: + switch (fy_token_scalar_style(fyt)) { + case FYSS_ANY: + style = "ANY"; + break; + case FYSS_PLAIN: + style = "PLAIN"; + break; + case FYSS_SINGLE_QUOTED: + style = "SINGLE_QUOTED"; + break; + case FYSS_DOUBLE_QUOTED: + style = "DOUBLE_QUOTED"; + break; + case FYSS_LITERAL: + style = "LITERAL"; + break; + case FYSS_FOLDED: + style = "FOLDED"; + break; + default: + style = "*illegal*"; + break; + } + printf("%s value='%s' style=%s\n", typetxt, + fy_atom_get_esc_text_a(&fyt->handle), + style); + break; + + case FYTT_INPUT_MARKER: + printf("%s value='%s'\n", typetxt, + fy_atom_get_esc_text_a(&fyt->handle)); + break; + + case FYTT_PE_MAP_KEY: + printf("%s value='%s'\n", typetxt, + fy_atom_get_esc_text_a(&fyt->handle)); + break; + + case FYTT_PE_SEQ_INDEX: + printf("%s value=%d\n", typetxt, + fyt->seq_index.index); + break; + + case FYTT_PE_SEQ_SLICE: + printf("%s value=%d:%d\n", typetxt, + fyt->seq_slice.start_index, fyt->seq_slice.end_index); + break; + + case FYTT_PE_ALIAS: + printf("%s value='%s'\n", "PE-ALIAS", + fy_atom_get_esc_text_a(&fyt->handle)); + break; + + default: + printf("%s\n", typetxt); + break; + } +} + +int do_scan(struct fy_parser *fyp) +{ + struct fy_token *fyt; + + while ((fyt = fy_scan(fyp)) != NULL) { + dump_token(fyt); + fy_token_unref(fyt); + } + + return 0; +} + +int do_copy(struct fy_parser *fyp) +{ + int c, count, line, column; + char buf[5], *s; + const char *str; + + count = 0; + for (;;) { + line = fyp_line(fyp); + column = fyp_column(fyp); + + c = fy_parse_get(fyp); + if (c < 0) { + break; + } + if (c == '\\') { + str = "\\\\"; + } else if (c == '\0') { + str = "\\0"; + } else if (c == '"') { + str = "\\\""; + } else if (c == '\b') { + str = "\\b"; + } else if (c == '\r') { + str = "\\r"; + } else if (c == '\t') { + str = "\\t"; + } else if (c == '\n') { + str = "\\n"; + } else { + s = buf; + if (c < 0x80) + *s++ = c; + else if (c < 0x800) { + *s++ = (c >> 6) | 0xc0; + *s++ = (c & 0x3f) | 0x80; + } else if (c < 0x10000) { + *s++ = (c >> 12) | 0xe0; + *s++ = ((c >> 6) & 0x3f) | 0x80; + *s++ = (c & 0x3f) | 0x80; + } else { + *s++ = (c >> 18) | 0xf0; + *s++ = ((c >> 12) & 0x3f) | 0x80; + *s++ = ((c >> 6) & 0x3f) | 0x80; + *s++ = (c & 0x3f) | 0x80; + } + *s = '\0'; + str = buf; + } + + printf("[%2d,%2d] = \"%s\"\n", line, column, str); + + count++; + } + printf("\ncount=%d\n", count); + + return 0; +} + +int do_dump(struct fy_parser *fyp, int indent, int width, bool resolve, bool sort, bool null_output) +{ + struct fy_document *fyd; + unsigned int flags; + int rc, count; + + flags = 0; + if (sort) + flags |= FYECF_SORT_KEYS; + flags |= FYECF_INDENT(indent) | FYECF_WIDTH(width); + + count = 0; + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + + if (resolve) { + rc = fy_document_resolve(fyd); + if (rc) + return -1; + } + + if (!null_output) + fy_emit_document_to_file(fyd, flags, NULL); + + fy_parse_document_destroy(fyp, fyd); + + count++; + } + + return count > 0 ? 0 : -1; +} + +int do_dump2(struct fy_parser *fyp, int indent, int width, bool resolve, bool sort, bool null_output) +{ + struct fy_document *fyd; + struct fy_document_builder *fydb; + struct fy_document_builder_cfg cfg; + unsigned int flags; + int rc, count; + + flags = 0; + if (sort) + flags |= FYECF_SORT_KEYS; + flags |= FYECF_INDENT(indent) | FYECF_WIDTH(width); + + memset(&cfg, 0, sizeof(cfg)); + cfg.parse_cfg = fyp->cfg; + cfg.diag = fy_diag_ref(fyp->diag); + + fydb = fy_document_builder_create(&cfg); + assert(fydb); + + count = 0; + while ((fyd = fy_document_builder_load_document(fydb, fyp)) != NULL) { + + if (resolve) { + rc = fy_document_resolve(fyd); + if (rc) + goto out; + } + + fy_emit_document_to_file(fyd, flags, NULL); + + fy_parse_document_destroy(fyp, fyd); + + count++; + } + + fy_document_builder_destroy(fydb); + +out: + return count > 0 ? 0 : -1; +} + +struct composer_data { + struct fy_parser *fyp; + struct fy_document *fyd; + bool null_output; + bool document_ready; + bool single_document; +}; + +static enum fy_composer_return +process_event(struct fy_parser *fyp, struct fy_event *fye, struct fy_path *path, void *userdata) +{ + struct composer_data *cd = userdata; + struct fy_document *fyd; + struct fy_path_component *parent, *last; + struct fy_node *fyn, *fyn_parent; + struct fy_node_pair *fynp; + char tbuf[80]; + int rc; + + if (cd->null_output) + return FYCR_OK_CONTINUE; + + fyp_info(fyp, "%s: %c%c%c%c%c %3d - %-32s: %s\n", fy_event_type_txt[fye->type], + fy_path_in_root(path) ? 'R' : '-', + fy_path_in_sequence(path) ? 'S' : '-', + fy_path_in_mapping(path) ? 'M' : '-', + fy_path_in_mapping_key(path) ? 'K' : + fy_path_in_mapping_value(path) ? 'V' : '-', + fy_path_in_collection_root(path) ? '/' : '-', + fy_path_depth(path), + fy_path_get_text_alloca(path), + fy_token_dump_format(fy_event_get_token(fye), tbuf, sizeof(tbuf))); + + switch (fye->type) { + /* nothing to do for those */ + case FYET_NONE: + case FYET_STREAM_START: + case FYET_STREAM_END: + break; + + case FYET_DOCUMENT_START: + if (cd->fyd) { + fy_document_destroy(cd->fyd); + cd->fyd = NULL; + } + cd->document_ready = false; + + cd->fyd = fy_document_create_from_event(fyp, fye); + fyp_error_check(fyp, cd->fyd, err_out, + "fy_document_create_from_event() failed"); + + break; + + case FYET_DOCUMENT_END: + rc = fy_document_update_from_event(cd->fyd, fyp, fye); + fyp_error_check(fyp, !rc, err_out, + "fy_document_update_from_event() failed"); + + cd->document_ready = true; + /* on single document mode we stop here */ + if (cd->single_document) + return FYCR_OK_STOP; + break; + + + case FYET_SCALAR: + case FYET_ALIAS: + case FYET_MAPPING_START: + case FYET_SEQUENCE_START: + + fyd = cd->fyd; + assert(fyd); + + fyn = fy_node_create_from_event(fyd, fyp, fye); + fyp_error_check(fyp, fyn, err_out, + "fy_node_create_from_event() failed"); + + switch (fye->type) { + default: + /* XXX should now happen */ + break; + + case FYET_SCALAR: + case FYET_ALIAS: + last = NULL; + break; + + case FYET_MAPPING_START: + last = fy_path_last_component(path); + assert(last); + + fy_path_component_set_mapping_user_data(last, fyn); + fy_path_component_set_mapping_key_user_data(last, NULL); + + break; + + case FYET_SEQUENCE_START: + + last = fy_path_last_component(path); + assert(last); + + fy_path_component_set_sequence_user_data(last, fyn); + break; + } + + /* parent */ + parent = fy_path_last_not_collection_root_component(path); + + if (fy_path_in_root(path)) { + + rc = fy_document_set_root(cd->fyd, fyn); + fyp_error_check(fyp, !rc, err_out, + "fy_document_set_root() failed"); + + } else if (fy_path_in_sequence(path)) { + + assert(parent); + fyn_parent = fy_path_component_get_sequence_user_data(parent); + assert(fyn_parent); + assert(fy_node_is_sequence(fyn_parent)); + + rc = fy_node_sequence_add_item(fyn_parent, fyn); + fyp_error_check(fyp, !rc, err_out, + "fy_node_sequence_add_item() failed"); + + } else { + /* only thing left */ + assert(fy_path_in_mapping(path)); + + assert(parent); + fyn_parent = fy_path_component_get_mapping_user_data(parent); + assert(fyn_parent); + assert(fy_node_is_mapping(fyn_parent)); + + if (fy_path_in_mapping_key(path)) { + + fynp = fy_node_pair_create_with_key(fyd, fyn_parent, fyn); + fyp_error_check(fyp, fynp, err_out, + "fy_node_pair_create_with_key() failed"); + + fy_path_component_set_mapping_key_user_data(parent, fynp); + + } else { + + assert(fy_path_in_mapping_value(path)); + + fynp = fy_path_component_get_mapping_key_user_data(parent); + assert(fynp); + + rc = fy_node_pair_update_with_value(fynp, fyn); + fyp_error_check(fyp, !rc, err_out, + "fy_node_pair_update_with_value() failed"); + + fy_path_component_set_mapping_key_user_data(parent, NULL); + } + } + + break; + + case FYET_MAPPING_END: + last = fy_path_last_component(path); + assert(last); + + fyn = fy_path_component_get_mapping_user_data(last); + assert(fyn); + assert(fy_node_is_mapping(fyn)); + + rc = fy_node_update_from_event(fyn, fyp, fye); + fyp_error_check(fyp, !rc, err_out, + "fy_node_update_from_event() failed"); + + break; + + case FYET_SEQUENCE_END: + last = fy_path_last_component(path); + assert(last); + + fyn = fy_path_component_get_sequence_user_data(last); + assert(fyn); + assert(fy_node_is_sequence(fyn)); + + rc = fy_node_update_from_event(fyn, fyp, fye); + fyp_error_check(fyp, !rc, err_out, + "fy_node_update_from_event() failed"); + + break; + } + + return FYCR_OK_CONTINUE; + +err_out: + return FYCR_ERROR; +} + +int do_compose(struct fy_parser *fyp, int indent, int width, bool resolve, bool sort, bool null_output) +{ + struct composer_data cd; + int rc; + + memset(&cd, 0, sizeof(cd)); + cd.null_output = null_output; + cd.single_document = true; + + rc = fy_parse_compose(fyp, process_event, &cd); + if (rc == 0 && cd.fyd) + fy_document_default_emit_to_fp(cd.fyd, stdout); + + fy_document_destroy(cd.fyd); + + return 0; +} + +struct fy_node * +fy_node_get_root(struct fy_node *fyn) +{ + if (!fyn) + return NULL; + while (fyn->parent) + fyn = fyn->parent; + return fyn; +} + +bool +fy_node_belongs_to_key(struct fy_document *fyd, struct fy_node *fyn) +{ + return fyd->root != fy_node_get_root(fyn); +} + +int do_iterate(struct fy_parser *fyp) +{ + struct fy_document *fyd; + struct fy_document_iterator *fydi; + int count; + struct fy_node *fyn; + char *path; + size_t len; + const char *text; + char textbuf[16]; + bool belongs_to_key; + + fydi = fy_document_iterator_create(); + assert(fydi); + + count = 0; + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + + fprintf(stderr, "> Start\n"); + fy_document_iterator_node_start(fydi, fy_document_root(fyd)); + + fyn = NULL; + while ((fyn = fy_document_iterator_node_next(fydi)) != NULL) { + + belongs_to_key = fy_node_belongs_to_key(fyd, fyn); + path = fy_node_get_path(fyn); + if (fy_node_is_scalar(fyn)) { + text = fy_node_get_scalar(fyn, &len); + assert(text); + fy_utf8_format_text(text, len, textbuf, sizeof(textbuf), fyue_doublequote); + if (!fy_node_is_alias(fyn)) + fprintf(stderr, "%40s \"%s\"%s\n", path, textbuf, belongs_to_key ? " KEY" : ""); + else + fprintf(stderr, "%40s *%s%s\n", path, textbuf, belongs_to_key ? " KEY" : ""); + } else if (fy_node_is_sequence(fyn)) { + fprintf(stderr, "%40s [%s\n", path, belongs_to_key ? " KEY" : ""); + } else if (fy_node_is_mapping(fyn)) { + fprintf(stderr, "%40s {%s\n", path, belongs_to_key ? " KEY" : ""); + } + + free(path); + } + + fprintf(stderr, "> End\n"); + + fy_parse_document_destroy(fyp, fyd); + + count++; + } + + fy_document_iterator_destroy(fydi); + + return count > 0 ? 0 : -1; +} + +int do_comment(struct fy_parser *fyp) +{ + struct fy_document *fyd; + int count; + struct fy_node *fyn; + struct fy_document_iterator *fydi; + char *path; + struct fy_token *fyt; + struct fy_atom *handle; + enum fy_comment_placement placement; + static const char *placement_txt[] = { "top", "right", "bottom" }; + char buf[1024]; + + fydi = fy_document_iterator_create(); + assert(fydi); + + count = 0; + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + + fy_document_iterator_node_start(fydi, fy_document_root(fyd)); + fyn = NULL; + while ((fyn = fy_document_iterator_node_next(fydi)) != NULL) { + + if (!fy_node_is_scalar(fyn)) + continue; + + fyt = fy_node_get_scalar_token(fyn); + if (!fyt || !fy_token_has_any_comment(fyt)) + continue; + + path = fy_node_get_path(fyn); + + fprintf(stderr, "scalar at %s\n", path); + for (placement = fycp_top; placement < fycp_max; placement++) { + handle = fy_token_comment_handle(fyt, placement, false); + if (!handle || !fy_atom_is_set(handle)) + continue; + + if (!fy_token_get_comment(fyt, buf, sizeof(buf), placement)) + continue; + + fprintf(stderr, "%s: %s\n", placement_txt[placement], buf); + } + + free(path); + } + + fy_parse_document_destroy(fyp, fyd); + + count++; + } + + fy_document_iterator_destroy(fydi); + + return count > 0 ? 0 : -1; +} + +#if defined(HAVE_LIBYAML) && HAVE_LIBYAML + +void dump_libyaml_token(yaml_token_t *token) +{ + const char *style; + + switch (token->type) { + case YAML_NO_TOKEN: + printf("NO\n"); + break; + case YAML_STREAM_START_TOKEN: + printf("STREAM_START\n"); + break; + case YAML_STREAM_END_TOKEN: + printf("STREAM_END\n"); + break; + case YAML_VERSION_DIRECTIVE_TOKEN: + printf("VERSION_DIRECTIVE value=%d.%d\n", + token->data.version_directive.major, + token->data.version_directive.minor); + break; + case YAML_TAG_DIRECTIVE_TOKEN: + printf("TAG_DIRECTIVE handle='%s' prefix='%s'\n", + txt2esc_a(token->data.tag_directive.handle, -1), + txt2esc_a(token->data.tag_directive.prefix, -1)); + break; + case YAML_DOCUMENT_START_TOKEN: + printf("DOCUMENT_START\n"); + break; + case YAML_DOCUMENT_END_TOKEN: + printf("DOCUMENT_END\n"); + break; + case YAML_BLOCK_SEQUENCE_START_TOKEN: + printf("BLOCK_SEQUENCE_START\n"); + break; + case YAML_BLOCK_MAPPING_START_TOKEN: + printf("BLOCK_MAPPING_START\n"); + break; + case YAML_BLOCK_END_TOKEN: + printf("BLOCK_END\n"); + break; + case YAML_FLOW_SEQUENCE_START_TOKEN: + printf("FLOW_SEQUENCE_START\n"); + break; + case YAML_FLOW_SEQUENCE_END_TOKEN: + printf("FLOW_SEQUENCE_END\n"); + break; + case YAML_FLOW_MAPPING_START_TOKEN: + printf("FLOW_MAPPING_START\n"); + break; + case YAML_FLOW_MAPPING_END_TOKEN: + printf("FLOW_MAPPING_END\n"); + break; + case YAML_BLOCK_ENTRY_TOKEN: + printf("BLOCK_ENTRY\n"); + break; + case YAML_FLOW_ENTRY_TOKEN: + printf("FLOW_ENTRY\n"); + break; + case YAML_KEY_TOKEN: + printf("KEY\n"); + break; + case YAML_VALUE_TOKEN: + printf("VALUE\n"); + break; + case YAML_ALIAS_TOKEN: + printf("ALIAS value='%s'\n", + txt2esc_a(token->data.alias.value, -1)); + break; + case YAML_ANCHOR_TOKEN: + printf("ANCHOR value='%s'\n", + txt2esc_a(token->data.anchor.value, -1)); + break; + case YAML_TAG_TOKEN: + printf("TAG handle='%s' suffix='%s'\n", + txt2esc_a(token->data.tag.handle, -1), + txt2esc_a(token->data.tag.suffix, -1)); + break; + case YAML_SCALAR_TOKEN: + switch (token->data.scalar.style) { + case YAML_ANY_SCALAR_STYLE: + style = "ANY"; + break; + case YAML_PLAIN_SCALAR_STYLE: + style = "PLAIN"; + break; + case YAML_SINGLE_QUOTED_SCALAR_STYLE: + style = "SINGLE_QUOTED"; + break; + case YAML_DOUBLE_QUOTED_SCALAR_STYLE: + style = "DOUBLE_QUOTED"; + break; + case YAML_LITERAL_SCALAR_STYLE: + style = "LITERAL"; + break; + case YAML_FOLDED_SCALAR_STYLE: + style = "FOLDED"; + break; + default: + style = "*ERROR*"; + break; + } + printf("SCALAR value='%s' style=%s\n", + txt2esc_a(token->data.scalar.value, token->data.scalar.length), + style); + break; + } +} + +int do_libyaml_scan(yaml_parser_t *parser) +{ + yaml_token_t token; + int done = 0; + + while (!done) + { + if (!yaml_parser_scan(parser, &token)) + return -1; + + dump_libyaml_token(&token); + + done = (token.type == YAML_STREAM_END_TOKEN); + + yaml_token_delete(&token); + } + + return 0; +} + +#define mark_a(_m) \ + ({ \ + yaml_mark_t *__m = (_m); \ + char *_s = alloca(30); \ + snprintf(_s, 30, "%zu/%zu/%zu", __m->index, __m->line, __m->column); \ + _s; \ + }) + +void dump_libyaml_event(yaml_event_t *event) +{ + char mbuf[40]; + char *mm; + char *anchor = NULL, *tag = NULL, *value = NULL; + + snprintf(mbuf, sizeof(mbuf), " %10s-%-10s ", + mark_a(&event->start_mark), + mark_a(&event->end_mark)); + mm = mbuf; + mm = " "; + + switch (event->type) { + case YAML_NO_EVENT: + printf("NO\n"); + break; + case YAML_STREAM_START_EVENT: + printf("%-14s%s|\n", "STREAM_START", mm); + break; + case YAML_STREAM_END_EVENT: + printf("%-14s%s|\n", "STREAM_END", mm); + break; + case YAML_DOCUMENT_START_EVENT: + printf("%-14s%s|\n", "DOCUMENT_START", mm); + break; + case YAML_DOCUMENT_END_EVENT: + printf("%-14s%s|\n", "DOCUMENT_END", mm); + break; + case YAML_ALIAS_EVENT: + anchor = txt2esc_a((char *)event->data.alias.anchor, -1); + printf("%-14s%s| '%s'\n", "ALIAS", mm, anchor); + break; + case YAML_SCALAR_EVENT: + if (event->data.scalar.anchor) + anchor = txt2esc_a((char *)event->data.scalar.anchor, -1); + if (event->data.scalar.tag) + tag = txt2esc_a((char *)event->data.scalar.tag, -1); + value = txt2esc_a((char *)event->data.scalar.value, -1); + printf("%-14s%s|%s%s%s%s%s%s '%s'\n", "SCALAR", mm, + anchor ? " anchor='" : "", anchor ? : "", anchor ? "'" : "", + tag ? " tag='" : "", tag ? : "", tag ? "'" : "", + value); + break; + case YAML_SEQUENCE_START_EVENT: + if (event->data.sequence_start.anchor) + anchor = txt2esc_a((char *)event->data.sequence_start.anchor, -1); + if (event->data.sequence_start.tag) + tag = txt2esc_a((char *)event->data.sequence_start.tag, -1); + printf("%-14s%s|%s%s%s%s%s%s\n", "SEQUENCE_START", mm, + anchor ? " anchor='" : "", anchor ? : "", anchor ? "'" : "", + tag ? " tag='" : "", tag ? : "", tag ? "'" : ""); + break; + case YAML_SEQUENCE_END_EVENT: + printf("%-14s%s|\n", "SEQUENCE_END", mm); + break; + case YAML_MAPPING_START_EVENT: + if (event->data.mapping_start.anchor) + anchor = txt2esc_a((char *)event->data.mapping_start.anchor, -1); + if (event->data.mapping_start.tag) + tag = txt2esc_a((char *)event->data.mapping_start.tag, -1); + printf("%-14s%s|%s%s%s%s%s%s\n", "MAPPING_START", mm, + anchor ? " anchor='" : "", anchor ? : "", anchor ? "'" : "", + tag ? " tag='" : "", tag ? : "", tag ? "'" : ""); + break; + case YAML_MAPPING_END_EVENT: + printf("%-14s%s|\n", "MAPPING_END", mm); + break; + default: + FY_IMPOSSIBLE_ABORT(); + } +} + +int do_libyaml_parse(yaml_parser_t *parser) +{ + yaml_event_t event; + int done = 0; + + while (!done) + { + if (!yaml_parser_parse(parser, &event)) + return -1; + + dump_libyaml_event(&event); + + done = (event.type == YAML_STREAM_END_EVENT); + + yaml_event_delete(&event); + } + + return 0; +} + +void dump_libyaml_testsuite_event(FILE *fp, yaml_event_t *event) +{ + switch (event->type) { + case YAML_NO_EVENT: + fprintf(fp, "???\n"); + break; + case YAML_STREAM_START_EVENT: + fprintf(fp, "+STR\n"); + break; + case YAML_STREAM_END_EVENT: + fprintf(fp, "-STR\n"); + break; + case YAML_DOCUMENT_START_EVENT: + fprintf(fp, "+DOC"); + if (!event->data.document_start.implicit) + fprintf(fp, " ---"); + fprintf(fp, "\n"); + break; + case YAML_DOCUMENT_END_EVENT: + fprintf(fp, "-DOC"); + if (!event->data.document_end.implicit) + fprintf(fp, " ..."); + fprintf(fp, "\n"); + break; + case YAML_MAPPING_START_EVENT: + fprintf(fp, "+MAP"); + if (event->data.mapping_start.anchor) + fprintf(fp, " &%s", event->data.mapping_start.anchor); + if (event->data.mapping_start.tag) + fprintf(fp, " <%s>", event->data.mapping_start.tag); + fprintf(fp, "\n"); + break; + case YAML_MAPPING_END_EVENT: + fprintf(fp, "-MAP\n"); + break; + case YAML_SEQUENCE_START_EVENT: + fprintf(fp, "+SEQ"); + if (event->data.sequence_start.anchor) + fprintf(fp, " &%s", event->data.sequence_start.anchor); + if (event->data.sequence_start.tag) + fprintf(fp, " <%s>", event->data.sequence_start.tag); + fprintf(fp, "\n"); + break; + case YAML_SEQUENCE_END_EVENT: + fprintf(fp, "-SEQ\n"); + break; + case YAML_SCALAR_EVENT: + fprintf(fp, "=VAL"); + if (event->data.scalar.anchor) + fprintf(fp, " &%s", event->data.scalar.anchor); + if (event->data.scalar.tag) + fprintf(fp, " <%s>", event->data.scalar.tag); + switch (event->data.scalar.style) { + case YAML_PLAIN_SCALAR_STYLE: + fprintf(fp, " :"); + break; + case YAML_SINGLE_QUOTED_SCALAR_STYLE: + fprintf(fp, " '"); + break; + case YAML_DOUBLE_QUOTED_SCALAR_STYLE: + fprintf(fp, " \""); + break; + case YAML_LITERAL_SCALAR_STYLE: + fprintf(fp, " |"); + break; + case YAML_FOLDED_SCALAR_STYLE: + fprintf(fp, " >"); + break; + case YAML_ANY_SCALAR_STYLE: + FY_IMPOSSIBLE_ABORT(); + } + print_escaped(fp, (char *)event->data.scalar.value, event->data.scalar.length); + fprintf(fp, "\n"); + break; + case YAML_ALIAS_EVENT: + fprintf(fp, "=ALI *%s\n", event->data.alias.anchor); + break; + default: + FY_IMPOSSIBLE_ABORT(); + } +} + +int do_libyaml_testsuite(FILE *fp, yaml_parser_t *parser, bool null_output) +{ + yaml_event_t event; + int done = 0; + + while (!done) { + + if (!yaml_parser_parse(parser, &event)) + return -1; + + if (!null_output) + dump_libyaml_testsuite_event(fp, &event); + + done = (event.type == YAML_STREAM_END_EVENT); + + yaml_event_delete(&event); + } + + return 0; +} + +int do_libyaml_dump(yaml_parser_t *parser, yaml_emitter_t *emitter, bool null_output) +{ + yaml_document_t document; + int done = 0; + int counter; + + yaml_emitter_set_canonical(emitter, 0); + + counter = 0; + while (!done) + { + if (!yaml_parser_load(parser, &document)) + return -1; + + done = !yaml_document_get_root_node(&document); + + if (!done) { + if (counter > 0) + printf("# document seperator\n"); + + if (!null_output) + yaml_emitter_dump(emitter, &document); + else + yaml_document_delete(&document); + counter++; + + if (!null_output) + yaml_emitter_flush(emitter); + } else + yaml_document_delete(&document); + } + + return 0; +} + +#endif + +struct fy_kv { + struct list_head node; + const char *key; + const char *value; +}; +FY_TYPE_FWD_DECL_LIST(kv); +FY_TYPE_DECL_LIST(kv); + +static int hd_accel_kv_hash(struct fy_accel *xl, const void *key, void *userdata, void *hash) +{ + unsigned int *hashp = hash; + + *hashp = XXH32(key, strlen(key), 2654435761U); + + // printf("%s key=%s hash=%08x\n", __func__, (const char *)key, *hashp); + return 0; +} + +static bool hd_accel_kv_eq(struct fy_accel *xl, const void *hash, const void *key1, const void *key2, void *userdata) +{ + return !strcmp(key1, key2); +} + +struct fy_kv_store { + struct fy_kv_list l; + struct fy_accel xl; + unsigned int count; +}; + +static const struct fy_hash_desc hd_kv_store = { + .size = sizeof(unsigned int), + .max_bucket_grow_limit = 8, + .hash = hd_accel_kv_hash, + .eq = hd_accel_kv_eq, +}; + +int fy_kv_store_setup(struct fy_kv_store *kvs, + unsigned int min_buckets) +{ + int rc; + + if (!kvs) + return -1; + memset(kvs, 0, sizeof(*kvs)); + fy_kv_list_init(&kvs->l); + + rc = fy_accel_setup(&kvs->xl, &hd_kv_store, kvs, min_buckets); + return rc; +} + +void fy_kv_store_cleanup(struct fy_kv_store *kvs) +{ + struct fy_kv *kv; + int rc __FY_DEBUG_UNUSED__; + + if (!kvs) + return; + + while ((kv = fy_kv_list_pop(&kvs->l)) != NULL) { + // printf("%s: removing %s: %s\n", __func__, kv->key, kv->value); + rc = fy_accel_remove(&kvs->xl, kv->key); + assert(!rc); + free(kv); + } + + fy_accel_cleanup(&kvs->xl); +} + +int fy_kv_store_insert(struct fy_kv_store *kvs, const char *key, const char *value) +{ + struct fy_kv *kv; + size_t klen, vlen, size; + char *s; + int rc; + + if (!kvs || !key || !value) + return -1; + + /* no more that UINT_MAX */ + if (kvs->count == UINT_MAX) + return -1; + + klen = strlen(key); + vlen = strlen(value); + size = sizeof(*kv) + klen + 1 + vlen + 1; + + kv = malloc(size); + if (!kv) + return -1; + s = (char *)(kv + 1); + kv->key = s; + memcpy(s, key, klen + 1); + s += klen + 1; + kv->value = s; + memcpy(s, value, vlen + 1); + + rc = fy_accel_insert(&kvs->xl, kv->key, kv); + if (rc) { + free(kv); + return rc; + } + fy_kv_list_add_tail(&kvs->l, kv); + kvs->count++; + + // printf("%s: %s: %s #%d\n", __func__, kv->key, kv->value, kvs->count); + + return 0; +} + +const char *fy_kv_store_lookup(struct fy_kv_store *kvs, const char *key) +{ + const struct fy_kv *kv; + + if (!kvs || !key) + return NULL; + + kv = fy_accel_lookup(&kvs->xl, key); + if (!kv) + return NULL; + + return kv->value; +} + +int fy_kv_store_remove(struct fy_kv_store *kvs, const char *key) +{ + struct fy_kv *kv; + struct fy_accel_entry *xle; + + if (!kvs || !key) + return -1; + + xle = fy_accel_entry_lookup(&kvs->xl, key); + if (!xle) { + printf("%s:%d: %s key=%s\n", __FILE__, __LINE__, __func__, key); + return -1; + } + kv = (void *)xle->value; + fy_accel_entry_remove(&kvs->xl, xle); + + fy_kv_list_del(&kvs->l, kv); + assert(kvs->count > 0); + kvs->count--; + + // printf("%s: %s: %s #%d\n", __func__, kv->key, kv->value, kvs->count); + + free(kv); + + return 0; +} + +const struct fy_kv *fy_kv_store_by_index(struct fy_kv_store *kvs, unsigned int index) +{ + unsigned int i; + struct fy_kv *kv; + + if (!kvs || index >= kvs->count) + return NULL; + + for (i = 0, kv = fy_kv_list_first(&kvs->l); kv && i < index; i++, kv = fy_kv_next(&kvs->l, kv)) + ; + return kv; +} + +const char *fy_kv_store_key_by_index(struct fy_kv_store *kvs, unsigned int index) +{ + const struct fy_kv *kv; + + kv = fy_kv_store_by_index(kvs, index); + return kv ? kv->key : NULL; +} + +static void do_accel_kv(const struct fy_parse_cfg *cfg, int argc, char *argv[]) +{ + struct fy_kv_store kvs; + unsigned int seed, idx; + int i, count; + int rc __FY_DEBUG_UNUSED__; + char keybuf[16], valbuf[16]; + const char *key; + + /* supress warnings about unused functions */ + (void)fy_kv_list_push; + (void)fy_kv_list_push_tail; + (void)fy_kv_list_is_singular; + (void)fy_kv_list_insert_after; + (void)fy_kv_list_insert_before; + (void)fy_kv_list_last; + (void)fy_kv_list_pop_tail; + (void)fy_kv_prev; + (void)fy_kv_lists_splice; + (void)fy_kv_list_splice_after; + (void)fy_kv_list_splice_before; + + seed = 0; /* we don't care much about seed practices right now */ + rc = fy_kv_store_setup(&kvs, 8); + assert(!rc); + + count = 1000; + + printf("creating #%d KVs\n", count); + + for (i = 0; i < count; i++) { + snprintf(keybuf, sizeof(keybuf), "%s-%08x", "key", rand_r(&seed)); + snprintf(valbuf, sizeof(valbuf), "%s-%08x", "val", rand_r(&seed)); + + printf("inserting %s: %s\n", keybuf, valbuf); + + rc = fy_kv_store_insert(&kvs, keybuf, valbuf); + assert(rc == 0); + } + + while (count > 0) { + idx = (unsigned int)rand_r(&seed) % count; + + key = fy_kv_store_key_by_index(&kvs, idx); + assert(key); + + printf("removing #%d - %s\n", idx, key); + + rc = fy_kv_store_remove(&kvs, key); + assert(!rc); + + count--; + } + + printf("\n"); + fy_kv_store_cleanup(&kvs); +} + +int do_accel_test(const struct fy_parse_cfg *cfg, int argc, char *argv[]) +{ + do_accel_kv(cfg, argc, argv); + + return 0; +} + +#if 0 +static void test_diag_output(struct fy_diag *diag, void *user, const char *buf, size_t len) +{ + FILE *fp = user; + static int counter = 0; + + fprintf(fp, "%d: %.*s", ++counter, (int)len, buf); +} +#endif + +int do_build(const struct fy_parse_cfg *cfg, int argc, char *argv[]) +{ +#if 0 + struct fy_parse_cfg cfg_tmp; + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + struct fy_diag_report_ctx drc; + struct fy_token *fyt; + void *iter; + const char *handle, *prefix; + size_t handle_size, prefix_size; + int rc __FY_DEBUG_UNUSED__; + struct fy_atom atom; +#endif + +#if 0 + char *path; + struct fy_node *fynt; + struct fy_document *fydt; + struct fy_node_pair *fynp; + const char *scalar; + size_t len; + int count, i, j; + int rc __FY_DEBUG_UNUSED__; + int ret __FY_DEBUG_UNUSED__; + char tbuf[80]; + struct fy_anchor *fya; + const char *handle, *prefix; + size_t handle_size, prefix_size; + + /****/ + + fydt = fy_document_build_from_string(cfg, "#comment \n[ 42, \n 12 ] # comment\n", FY_NT); + assert(fydt); + + buf = fy_emit_document_to_string(fydt, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fydt); + fydt = NULL; + + /****/ + + fydt = fy_document_build_from_string(cfg, "plain-scalar # comment\n", FY_NT); + assert(fydt); + + scalar = fy_node_get_scalar(fy_document_root(fydt), &len); + printf("root scalar content=\"%.*s\"\n", (int)len, scalar); + + fy_document_destroy(fydt); + fydt = NULL; + + /****/ + + fydt = fy_document_build_from_string(cfg, "[ 10, 11, foo ] # comment", FY_NT); + assert(fydt); + + buf = fy_emit_node_to_string(fy_document_root(fydt), 0); + assert(buf); + printf("resulting node: \"%s\"\n", buf); + free(buf); + + count = fy_node_sequence_item_count(fy_document_root(fydt)); + printf("count=%d\n", count); + assert(count == 3); + + /* try iterator first */ + printf("forward iterator:"); + iter = NULL; + while ((fyn = fy_node_sequence_iterate(fy_document_root(fydt), &iter)) != NULL) { + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf(" \"%s\"", buf); + free(buf); + } + printf("\n"); + + printf("reverse iterator:"); + iter = NULL; + while ((fyn = fy_node_sequence_reverse_iterate(fy_document_root(fydt), &iter)) != NULL) { + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf(" \"%s\"", buf); + free(buf); + } + printf("\n"); + + fy_document_destroy(fydt); + fydt = NULL; + + /*****/ + + fydt = fy_document_build_from_string(cfg, "{ foo: 10, bar : 20, baz: [100, 101], [frob, 1]: boo }", FY_NT); + assert(fydt); + + buf = fy_emit_node_to_string(fy_document_root(fydt), 0); + assert(buf); + printf("resulting node: \"%s\"\n", buf); + free(buf); + + count = fy_node_mapping_item_count(fy_document_root(fydt)); + printf("count=%d\n", count); + assert(count == 4); + + /* try iterator first */ + printf("forward iterator:"); + iter = NULL; + while ((fynp = fy_node_mapping_iterate(fy_document_root(fydt), &iter)) != NULL) { + buf = fy_emit_node_to_string(fynp->key, 0); + assert(buf); + printf(" key=\"%s\"", buf); + free(buf); + buf = fy_emit_node_to_string(fynp->value, 0); + assert(buf); + printf(",value=\"%s\"", buf); + free(buf); + } + printf("\n"); + + printf("reverse iterator:"); + iter = NULL; + while ((fynp = fy_node_mapping_reverse_iterate(fy_document_root(fydt), &iter)) != NULL) { + buf = fy_emit_node_to_string(fynp->key, 0); + assert(buf); + printf(" key=\"%s\"", buf); + free(buf); + buf = fy_emit_node_to_string(fynp->value, 0); + assert(buf); + printf(",value=\"%s\"", buf); + free(buf); + } + printf("\n"); + + printf("index based:"); + for (i = 0; i < count; i++) { + fynp = fy_node_mapping_get_by_index(fy_document_root(fydt), i); + assert(fynp); + buf = fy_emit_node_to_string(fynp->key, 0); + assert(buf); + printf(" key=\"%s\"", buf); + free(buf); + buf = fy_emit_node_to_string(fynp->value, 0); + assert(buf); + printf(",value=\"%s\"", buf); + free(buf); + } + printf("\n"); + + printf("key lookup based:"); + + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fydt), "foo", FY_NT); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s->\"%s\"\n", "foo", buf); + free(buf); + + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fydt), "bar", FY_NT); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s->\"%s\"\n", "bar", buf); + free(buf); + + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fydt), "baz", FY_NT); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s->\"%s\"\n", "baz", buf); + free(buf); + + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fydt), "[ frob, 1 ]", FY_NT); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s->\"%s\"\n", "[ frob, 1 ]", buf); + free(buf); + + printf("\n"); + + fy_document_destroy(fydt); + fydt = NULL; + + /*****/ + + fydt = fy_document_build_from_string(cfg, "{ " + "foo: 10, bar : 20, baz:{ frob: boo }, " + "frooz: [ seq1, { key: value} ], \"zero\\0zero\" : 0, " + "{ key2: value2 }: { key3: value3 } " + "}", FY_NT); + + assert(fydt); + + fyn = fy_node_by_path(fy_document_root(fydt), "/", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "/", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "foo", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "foo", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "bar", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "bar", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "baz", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "baz", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "baz/frob", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "baz/frob", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "frooz", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "frooz", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "/frooz/[0]", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "/frooz/[0]", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "/frooz/[1]", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "/frooz/[1]", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "/frooz/[1]/key", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "/frooz/[1]/key", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "\"foo\"", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "\"foo\"", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "\"zero\\0zero\"", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "zero\\0zero", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "/{ key2: value2 }", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "/{ key2: value2 }", buf); + free(buf); + + fyn = fy_node_by_path(fy_document_root(fydt), "/{ key2: value2 }/key3", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "/{ key2: value2 }/key3", buf); + free(buf); + + printf("\npaths....\n"); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fydt), "/", FY_NT, FYNWF_DONT_FOLLOW)); + printf("%s path is %s\n", "/", path); + if (path) + free(path); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fydt), "/frooz", FY_NT, FYNWF_DONT_FOLLOW)); + printf("%s path is %s\n", "/frooz", path); + if (path) + free(path); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fydt), "/frooz/[0]", FY_NT, FYNWF_DONT_FOLLOW)); + printf("%s path is %s\n", "/frooz/[0]", path); + if (path) + free(path); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fydt), "/{ key2: value2 }/key3", FY_NT, FYNWF_DONT_FOLLOW)); + printf("%s path is %s\n", "/{ key2: value2 }/key3", path); + if (path) + free(path); + + fy_document_destroy(fydt); + fydt = NULL; + + /*****/ + + fyd = fy_document_create(cfg); + assert(fyd); + + fyn = fy_node_build_from_string(fyd, "{ }", FY_NT); + assert(fyn); + + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "/", buf); + free(buf); + + fy_document_set_root(fyd, fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_create(cfg); + assert(fyd); + + fyn = fy_node_create_scalar(fyd, "foo", 3); + assert(fyn); + fy_document_set_root(fyd, fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_create(cfg); + assert(fyd); + + fyn = fy_node_create_scalar(fyd, "foo\nfoo", 7); + assert(fyn); + fy_document_set_root(fyd, fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_create(cfg); + assert(fyd); + + fyn = fy_node_create_sequence(fyd); + assert(fyn); + fy_document_set_root(fyd, fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_create(cfg); + assert(fyd); + + fyn = fy_node_create_mapping(fyd); + assert(fyn); + fy_document_set_root(fyd, fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_create(cfg); + assert(fyd); + + fyn = fy_node_create_sequence(fyd); + assert(fyn); + + fy_node_sequence_append(fyn, fy_node_create_scalar(fyd, "foo", FY_NT)); + fy_node_sequence_append(fyn, fy_node_create_scalar(fyd, "bar", FY_NT)); + fy_node_sequence_append(fyn, fy_node_build_from_string(fyd, "{ baz: frooz }", FY_NT)); + + fy_document_set_root(fyd, fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_create(cfg); + assert(fyd); + + fyn = fy_node_create_mapping(fyd); + assert(fyn); + + rc = fy_node_mapping_append(fyn, NULL, fy_node_build_from_string(fyd, "[ 0, 1 ]", FY_NT)); + assert(!rc); + + fynt = fy_node_build_from_string(fyd, "foo", FY_NT); + assert(fynt); + rc = fy_node_mapping_append(fyn, NULL, fynt); + assert(rc); + fy_node_free(fynt); + + rc = fy_node_mapping_append(fyn, fy_node_build_from_string(fyd, "bar", FY_NT), NULL); + assert(!rc); + + fy_document_set_root(fyd, fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_create(cfg); + assert(fyd); + + fy_document_set_root(fyd, fy_node_build_from_string(fyd, "scalar", FY_NT)); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_create(cfg); + assert(fyd); + + fyn = fy_node_create_sequence(fyd); + assert(fyn); + + fy_node_sequence_append(fyn, fy_node_build_from_string(fyd, "foo", FY_NT)); + fy_node_sequence_append(fyn, fy_node_build_from_string(fyd, "bar", FY_NT)); + + fy_document_set_root(fyd, fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_build_from_string(cfg, "[ one, two, four ]", FY_NT); + + fy_node_sequence_append(fy_document_root(fyd), fy_node_build_from_string(fyd, "five", FY_NT)); + fy_node_sequence_prepend(fy_document_root(fyd), fy_node_build_from_string(fyd, "zero", FY_NT)); + + fy_node_sequence_insert_after(fy_document_root(fyd), + fy_node_by_path(fy_document_root(fyd), "/[2]", FY_NT, FYNWF_DONT_FOLLOW), + fy_node_build_from_string(fyd, "three", FY_NT)); + + fy_node_sequence_insert_before(fy_document_root(fyd), + fy_node_by_path(fy_document_root(fyd), "/[3]", FY_NT, FYNWF_DONT_FOLLOW), + fy_node_build_from_string(fyd, "two-and-a-half", FY_NT)); + + fyn = fy_node_sequence_remove(fy_document_root(fyd), + fy_node_by_path(fy_document_root(fyd), "/[3]", FY_NT, FYNWF_DONT_FOLLOW)); + + fy_node_free(fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_build_from_string(cfg, "{ one: 1, two: 2, four: 4 }", FY_NT); + + fy_node_mapping_append(fy_document_root(fyd), + fy_node_build_from_string(fyd, "three", FY_NT), + fy_node_build_from_string(fyd, "3", FY_NT)); + + fy_node_mapping_prepend(fy_document_root(fyd), + fy_node_build_from_string(fyd, "zero", FY_NT), + fy_node_build_from_string(fyd, "0", FY_NT)); + + fy_node_mapping_append(fy_document_root(fyd), + fy_node_build_from_string(fyd, "two-and-a-half", FY_NT), + fy_node_build_from_string(fyd, "2.5", FY_NT)); + + fyn = fy_node_mapping_remove_by_key(fy_document_root(fyd), + fy_node_build_from_string(fyd, "two-and-a-half", FY_NT)); + assert(fyn != NULL); + + fy_node_free(fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_build_from_string(cfg, "{ aaa: 1, zzz: 2, bbb: 4 }", FY_NT); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_build_from_string(cfg, "\naaa: 1\nzzz: 2\nbbb:\n ccc: foo\n", FY_NT); + + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_build_from_string(cfg, "{ aaa: 1, zzz: 2, bbb: 4 }", FY_NT); + + buf = fy_emit_document_to_string(fyd, FYECF_MODE_BLOCK); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + /* {? {z: bar} : map-value1, ? {a: whee} : map-value2, ? [a, b, c] : seq-value, ? [z] : {frooz: ting}, aaa: 1, bbb: 4, zzz: 2} */ + fyd = fy_document_build_from_string(cfg, "{ a: 5, { z: bar }: 1, z: 7, " + "[ a, b, c] : 3, { a: whee } : 2 , b: 6, [ z ]: 4 }", FY_NT); + + buf = fy_emit_document_to_string(fyd, FYECF_SORT_KEYS); + assert(buf); + printf("resulting document (sorted1):\n"); + fputs(buf, stdout); + free(buf); + + ret = fy_node_sort(fy_document_root(fyd), NULL, NULL); + assert(!ret); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document (sorted2):\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + +#if 0 + /******/ + + cfg_tmp = *cfg; + cfg_tmp.flags |= FYPCF_COLLECT_DIAG; + + /* this is an error */ + fyd = fy_document_build_from_string(&cfg_tmp, "{ a: 5 ] }"); + assert(fyd); + assert(!fyd->root); + + fprintf(stderr, "error log:\n%s", fy_document_get_log(fyd, NULL)); + + fy_document_destroy(fyd); +#endif + + /******/ + + fyd = fy_document_buildf(cfg, "{ %s: %d, zzz: 2, bbb: 4 }", "aaa", 1); + assert(fyd); + + buf = fy_emit_document_to_string(fyd, FYECF_MODE_BLOCK); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_buildf(cfg, "{ %s: %d, zzz: \"this is\n\ntext\", bbb: 4 }", "aaa", 1); + assert(fyd); + + i = j = -1; + count = fy_document_scanf(fyd, "aaa %d bbb %d zzz %79[^\xc0]s", &i, &j, tbuf); + printf("count=%d, aaa=%d bbb=%d zzz=\"%s\"\n", count, i, j, tbuf); + + fy_document_destroy(fyd); + + /******/ + + fyd = fy_document_create(cfg); + assert(fyd); + + rc = fy_document_tag_directive_add(fyd, "!foo!", "tag:bar.com,2019:"); + assert(!rc); + + rc = fy_document_tag_directive_add(fyd, "!e!", "tag%21"); + assert(!rc); + + fyn = fy_node_build_from_string(fyd, "{ foo: bar }", FY_NT); + assert(fyn); + + fy_document_set_root(fyd, fyn); + + /* rc = fy_node_set_tag(fyn, "!!", -1); */ + /* rc = fy_node_set_tag(fyn, "!!int", -1); */ + /* rc = fy_node_set_tag(fyn, "!", -1); */ + rc = fy_node_set_tag(fyn, "!foo!baz", FY_NT); + /* rc = fy_node_set_tag(fyn, "!e!tag%21", -1); */ + /* rc = fy_node_set_tag(fyn, "!e!tag:12:", -1); */ + assert(!rc); + + // fy_document_tag_directive_remove(fyd, "!foo!"); + + rc = fy_node_set_anchor(fy_node_by_path(fyn, "/foo", FY_NT, FYNWF_DONT_FOLLOW), "anchor", FY_NT); + assert(!rc); + + rc = fy_node_mapping_append(fyn, + fy_node_build_from_string(fyd, "!foo!whoa baz", FY_NT), + fy_node_build_from_string(fyd, "frooz", FY_NT)); + assert(!rc); + + rc = fy_node_mapping_append(fyn, + fy_node_build_from_string(fyd, "test", FY_NT), + fy_node_build_from_string(fyd, "*anchor", FY_NT)); + assert(!rc); + + rc = fy_node_mapping_append(fyn, + fy_node_build_from_string(fyd, "test-2", FY_NT), + fy_node_create_alias(fyd, "anchor", FY_NT)); + assert(!rc); + + if (cfg->flags & FYPCF_RESOLVE_DOCUMENT) { + rc = fy_document_resolve(fyd); + assert(!rc); + } + + fprintf(stderr, "tag directives of document\n"); + /* dump directives */ + iter = NULL; + while ((fyt = fy_document_tag_directive_iterate(fyd, &iter)) != NULL) { + handle = fy_tag_directive_token_handle(fyt, &handle_size); + assert(handle); + + prefix = fy_tag_directive_token_prefix(fyt, &prefix_size); + assert(prefix); + + fprintf(stderr, "tag-directive \"%.*s\" \"%.*s\"\n", + (int)handle_size, handle, + (int)prefix_size, prefix); + } + + fprintf(stderr, "anchors of document\n"); + iter = NULL; + while ((fya = fy_document_anchor_iterate(fyd, &iter)) != NULL) { + + path = fy_node_get_path(fy_anchor_node(fya)); + assert(path); + + anchor = fy_anchor_get_text(fya, &anchor_size); + assert(anchor); + + fprintf(stderr, "&%.*s %s\n", (int)anchor_size, anchor, path); + + free(path); + } + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + + /*****/ + + printf("\nJSON pointer tests\n"); + + fyd = fy_document_build_from_string(cfg, + "{\n" + " \"foo\": [\"bar\", \"baz\"],\n" + " \"\": 0,\n" + " \"a/b\": 1,\n" + " \"c%d\": 2,\n" + " \"e^f\": 3,\n" + " \"g|h\": 4,\n" + " \"i\\\\j\": 5,\n" + " \"k\\\"l\": 6,\n" + " \" \": 7,\n" + " \"m~n\": 8\n" + "}" , FY_NT); + + assert(fyd); + + fyn = fy_node_by_path(fy_document_root(fyd), "/", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "/", buf); + free(buf); + + { + const char *json_paths[] = { + "", + "/foo", + "/foo/0", + "/", + "/a~1b", + "/c%d", + "/e^f", + "/g|h", + "/i\\j", + "/k\"l", + "/ ", + "/m~0n" + }; + unsigned int ii; + + for (ii = 0; ii < sizeof(json_paths)/sizeof(json_paths[0]); ii++) { + fyn = fy_node_by_path(fy_document_root(fyd), json_paths[ii], FY_NT, FYNWF_PTR_JSON); + if (!fyn) { + printf("Unable to lookup JSON path: '%s'\n", json_paths[ii]); + } else { + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("JSON path: '%s' is '%s'\n", json_paths[ii], buf); + free(buf); + } + printf("\n"); + } + } + + fy_document_destroy(fyd); + fyd = NULL; + + /*****/ + + printf("\nRelative JSON pointer tests\n"); + + fyd = fy_document_build_from_string(cfg, + "{\n" + " \"foo\": [\"bar\", \"baz\"],\n" + " \"highly\": {\n" + " \"nested\": {\n" + " \"objects\": true\n" + " }\n" + " }\n" + "}\n", FY_NT); + assert(fyd); + + fyn = fy_node_by_path(fy_document_root(fyd), "/", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("%s is \"%s\"\n", "/", buf); + free(buf); + + { + const char *reljson_paths[] = { + "0", + "1/0", + "2/highly/nested/objects", + }; + unsigned int ii; + + for (ii = 0; ii < sizeof(reljson_paths)/sizeof(reljson_paths[0]); ii++) { + fyn = fy_node_by_path( + fy_node_by_path(fy_document_root(fyd), "/foo/1", FY_NT, FYNWF_DONT_FOLLOW), /* "baz" */ + reljson_paths[ii], FY_NT, FYNWF_PTR_RELJSON); + if (!fyn) { + printf("Unable to lookup relative JSON path: '%s'\n", reljson_paths[ii]); + } else { + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("Relative JSON path: '%s' is '%s'\n", reljson_paths[ii], buf); + free(buf); + } + printf("\n"); + } + } + + { + const char *reljson_paths[] = { + "0/objects", + "1/nested/objects", + "2/foo/0", + }; + unsigned int ii; + + for (ii = 0; ii < sizeof(reljson_paths)/sizeof(reljson_paths[0]); ii++) { + fyn = fy_node_by_path( + fy_node_by_path(fy_document_root(fyd), "/highly/nested", FY_NT, FYNWF_DONT_FOLLOW), /* "baz" */ + reljson_paths[ii], FY_NT, FYNWF_PTR_RELJSON); + if (!fyn) { + printf("Unable to lookup relative JSON path: '%s'\n", reljson_paths[ii]); + } else { + buf = fy_emit_node_to_string(fyn, 0); + assert(buf); + printf("Relative JSON path: '%s' is '%s'\n", reljson_paths[ii], buf); + free(buf); + } + printf("\n"); + } + } + + fy_document_destroy(fyd); + fyd = NULL; + + /*****/ + + fyd = fy_document_create(NULL); + assert(fyd); + + fyn = fy_node_create_sequence(fyd); + assert(fyn); + + fy_document_set_root(fyd, fyn); + fyn = NULL; + + fyn = fy_node_build_from_string(fyd, "%TAG !e! tag:example.com,2000:app/\n---\n- foo\n- !e!foo bar\n", FY_NT); + assert(fyn); + + rc = fy_node_sequence_append(fy_document_root(fyd), fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + printf("tag directives:\n"); + iter = NULL; + while ((fyt = fy_document_tag_directive_iterate(fyd, &iter)) != NULL) { + handle = fy_tag_directive_token_handle(fyt, &handle_size); + prefix = fy_tag_directive_token_prefix(fyt, &prefix_size); + printf("> handle='%.*s' prefix='%.*s'\n", + (int)handle_size, handle, + (int)prefix_size, prefix); + } + + /* try to build another, but with a different !e! prefix, it must fail */ + fyn = fy_node_build_from_string(fyd, "%TAG !e! tag:example.com,2019:app/\n---\n- foo\n- !e!foo bar\n", FY_NT); + assert(!fyn); + + rc = fy_document_tag_directive_add(fyd, "!f!", "tag:example.com,2019:f/"); + assert(!rc); + + printf("new tag directives:\n"); + iter = NULL; + while ((fyt = fy_document_tag_directive_iterate(fyd, &iter)) != NULL) { + handle = fy_tag_directive_token_handle(fyt, &handle_size); + prefix = fy_tag_directive_token_prefix(fyt, &prefix_size); + printf("> handle='%.*s' prefix='%.*s'\n", + (int)handle_size, handle, + (int)prefix_size, prefix); + } + + fyn = fy_node_build_from_string(fyd, "!f!whiz frooz\n", FY_NT); + assert(fyn); + + rc = fy_node_sequence_append(fy_document_root(fyd), fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("resulting document:\n"); + fputs(buf, stdout); + free(buf); + + fy_document_destroy(fyd); + fyd = NULL; + + /***************/ + + fyp_debug(NULL, FYEM_INTERNAL, "(debug) test"); + fyp_info(NULL, "(info) test"); + fyp_notice(NULL, "(notice) test"); + fyp_warning(NULL, "(warning) test"); + fyp_error(NULL, "(error) test"); + + /****/ + + fyd = fy_document_build_from_string(cfg, + "{\n" + " { foo: bar }: baz,\n" + " frooz: whee,\n" + " houston: [ we, have, a, problem ]\n" + "}", FY_NT); + assert(fyd); + + printf("***************************\n"); + printf("fy_node_is_synthetic(\"/\") = %s\n", + fy_node_is_synthetic(fy_document_root(fyd)) ? "true" : "false"); + printf("***************************\n"); + + fyn = fy_node_by_path(fy_document_root(fyd), "/{ foo: bar }", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + printf("fy_node_is_synthetic(\"/{ foo: bar }\") = %s\n", + fy_node_is_synthetic(fyn) ? "true" : "false"); + + buf = fy_emit_node_to_string(fyn, FYECF_MODE_FLOW_ONELINE | FYECF_WIDTH_INF); + assert(buf); + printf("/{ foo: bar }: %s\n", buf); + free(buf); + + memset(&drc, 0, sizeof(drc)); + drc.type = FYET_NOTICE; + drc.module = FYEM_DOC; + drc.fyt = fy_token_ref(fyn->scalar); + fy_document_diag_report(fyd, &drc, "Test %d", 12); + + fyn = fy_node_by_path(fy_document_root(fyd), "/houston", FY_NT, FYNWF_DONT_FOLLOW); + assert(fyn); + printf("fy_node_is_synthetic(/houston) = %s\n", + fy_node_is_synthetic(fyn) ? "true" : "false"); + + buf = fy_emit_node_to_string(fyn, FYECF_MODE_FLOW_ONELINE | FYECF_WIDTH_INF); + assert(buf); + printf("/houston: %s\n", buf); + free(buf); + + memset(&drc, 0, sizeof(drc)); + drc.type = FYET_NOTICE; + drc.module = FYEM_DOC; + drc.fyt = fy_token_ref(fyn->sequence_start); + fy_document_diag_report(fyd, &drc, "Test %d", 13); + + fyt = fy_node_token(fyn); + assert(fyt); + + memset(&drc, 0, sizeof(drc)); + drc.type = FYET_NOTICE; + drc.module = FYEM_DOC; + drc.fyt = fyt; + fy_document_diag_report(fyd, &drc, "Test %d", 14); + + printf("***************************\n"); + printf("fy_node_is_synthetic(\"/\") = %s\n", + fy_node_is_synthetic(fy_document_root(fyd)) ? "true" : "false"); + printf("***************************\n"); + + fy_node_sequence_append(fy_node_by_path(fy_document_root(fyd), "/houston", FY_NT, FYNWF_DONT_FOLLOW), + fy_node_create_scalar(fyd, "synthesonic", FY_NT)); + + printf("***************************\n"); + printf("fy_node_is_synthetic(\"/\") = %s\n", + fy_node_is_synthetic(fy_document_root(fyd)) ? "true" : "false"); + printf("***************************\n"); + + fyt = fy_node_token(fy_document_root(fyd)); + assert(fyt); + + memset(&drc, 0, sizeof(drc)); + drc.type = FYET_NOTICE; + drc.module = FYEM_DOC; + drc.fyt = fyt; + fy_document_diag_report(fyd, &drc, "Test %d", 16); + + fy_node_mapping_append(fy_document_root(fyd), + fy_node_create_scalar(fyd, "key", FY_NT), + fy_node_create_scalar(fyd, "value", FY_NT)); + + printf("***************************\n"); + printf("fy_node_is_synthetic(\"/\") = %s\n", + fy_node_is_synthetic(fy_document_root(fyd)) ? "true" : "false"); + printf("***************************\n"); + + fy_node_sequence_append(fy_node_by_path(fy_document_root(fyd), "/houston", FY_NT, FYNWF_DONT_FOLLOW), + fy_node_create_scalar(fyd, "item", FY_NT)); + + fyt = fy_node_token(fy_node_by_path(fy_document_root(fyd), "/houston", FY_NT, FYNWF_DONT_FOLLOW)); + assert(fyt); + + memset(&drc, 0, sizeof(drc)); + drc.type = FYET_NOTICE; + drc.module = FYEM_DOC; + drc.fyt = fyt; + fy_document_diag_report(fyd, &drc, "Test %d", 17); + + fy_node_report(fy_node_by_path(fy_document_root(fyd), "/houston/0", FY_NT, FYNWF_DONT_FOLLOW), + FYET_WARNING, "/houston/0 checking report"); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + printf("/:\n"); + fputs(buf, stdout); + free(buf); + + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE | FYECF_WIDTH_INF); + assert(buf); + printf("%s\n", buf); + free(buf); + + fy_diag_printf(fyd->diag, "Outputting on document diag\n"); + + fy_debug(fyd->diag, "Debug level\n"); + fy_info(fyd->diag, "Info level\n"); + fy_notice(fyd->diag, "Notice level\n"); + fy_warning(fyd->diag, "Warning level\n"); + fy_error(fyd->diag, "Error level\n"); + + fy_document_destroy(fyd); + fyd = NULL; + + { + struct fy_diag *diag; + struct fy_diag_cfg dcfg; + struct fy_parse_cfg pcfg; + FILE *fp; + char *mbuf = NULL; + size_t msize; + + fp = open_memstream(&mbuf, &msize); + assert(fp); + + fy_diag_cfg_default(&dcfg); + dcfg.fp = fp; + dcfg.colorize = isatty(fileno(stderr)) == 1; + + diag = fy_diag_create(&dcfg); + assert(diag); + + fy_error(diag, "Writting in the diagnostic\n"); + + memset(&pcfg, 0, sizeof(pcfg)); + pcfg.flags = FYPCF_DEFAULT_DOC; + pcfg.diag = diag; + fyd = fy_document_build_from_string(&pcfg, "{ foo: \"\\xeh\", foo: baz }", FY_NT); + /* the document must not be created (duplicate key) */ + assert(!fyd); + + fy_diag_destroy(diag); + + fclose(fp); + + assert(mbuf); + + printf("checking diagnostic\n"); + fwrite(mbuf, msize, 1, stdout); + + free(mbuf); + } + + { + struct fy_diag *diag; + struct fy_diag_cfg dcfg; + struct fy_parse_cfg pcfg; + FILE *fp; + char *mbuf = NULL; + size_t msize; + + fp = open_memstream(&mbuf, &msize); + assert(fp); + + fy_diag_cfg_default(&dcfg); + dcfg.fp = NULL; + dcfg.colorize = isatty(fileno(stderr)) == 1; + dcfg.output_fn = test_diag_output; + dcfg.user = stderr; + + diag = fy_diag_create(&dcfg); + assert(diag); + + fy_error(diag, "Writting in the diagnostic\n"); + + memset(&pcfg, 0, sizeof(pcfg)); + pcfg.flags = FYPCF_DEFAULT_DOC; + pcfg.diag = diag; + fyd = fy_document_build_from_string(&pcfg, "{ foo: \"\\xeh\", foo: baz }", FY_NT); + /* the document must not be created (duplicate key) */ + assert(!fyd); + + fy_diag_destroy(diag); + + fclose(fp); + + assert(mbuf); + + printf("checking diagnostic\n"); + fwrite(mbuf, msize, 1, stdout); + + free(mbuf); + } +#endif + /*****/ + +#if 0 + { +//#define MANUAL_SCALAR_STR "val\"quote'\0null\0&the\nrest " +//#define MANUAL_SCALAR_STR "0\n1" +//#define MANUAL_SCALAR_STR "\\\"\0\a\b\t\v\f\r\e\xc2\x85\xc2\xa0\xe2\x80\xa8\xe2\x80\xa9" +//#define MANUAL_SCALAR_STR "\\\"\0\a\b\t\v\f\r\e\n" +//#define MANUAL_SCALAR_STR "\xc2\x85" +//#define MANUAL_SCALAR_STR "\xc2\xa0" +#define MANUAL_SCALAR_STR "\xff\xff\xff\xff" +//#define MANUAL_SCALAR_STR "foo\xf9\xff\xffzbar\xff\xffwz" + + const char *what = MANUAL_SCALAR_STR; + size_t what_sz = sizeof(MANUAL_SCALAR_STR) - 1; + struct fy_document *fyd; + struct fy_node *fyn; + char *buf, *buf2; + + fyd = fy_document_create(cfg); + assert(fyd); + + fyn = fy_node_create_scalar(fyd, what, what_sz); + assert(fyn); + + fy_document_set_root(fyd, fyn); + + buf = fy_emit_document_to_string(fyd, 0); + assert(buf); + fputs(buf, stdout); + fy_document_destroy(fyd); + + fyd = fy_document_build_from_string(cfg, buf, FY_NT); + assert(fyd); + + buf2 = fy_emit_document_to_string(fyd, 0); + assert(buf2); + fputs(buf2, stdout); + fy_document_destroy(fyd); + + free(buf); + free(buf2); + } +#endif + +#if 0 + do_accel_test(cfg, argc, argv); +#endif + + struct fy_emitter_cfg ecfg; + struct fy_emitter* emit; + + memset(&ecfg, 0, sizeof(ecfg)); + // ecfg.flags = FYECF_MODE_BLOCK; + ecfg.flags = FYECF_MODE_MANUAL; + + emit = fy_emitter_create(&ecfg); + + // key: + // - a: 1 + + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_START)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_START, false, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, + fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "key", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "a", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "1", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_END, true, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_END)); + + fy_emitter_destroy(emit); + + emit = fy_emitter_create(&ecfg); + + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_START)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_START, false, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_FLOW, NULL, NULL)); + fy_emit_event(emit, + fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "key", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_START, FYNS_FLOW, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_FLOW, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "a", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "1", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_END, true, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_END)); + + fy_emitter_destroy(emit); + + emit = fy_emitter_create(&ecfg); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_START)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_START, false, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, + fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "key", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "a", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "1", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_END, true, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_END)); + fy_emitter_destroy(emit); + + return 0; +} + +struct pathspec_arg { + const char *arg; + size_t argsz; + struct fy_document *fyd; + bool is_numeric; + int number; + void *user; +}; + +struct pathspec { + const char *func; + size_t funcsz; + unsigned int argc; + struct pathspec_arg arg[10]; + void *user; +}; + +struct path { + bool absolute; /* starts at root */ + bool trailing_slash; + unsigned int count; + unsigned int alloc; + struct pathspec *ps; + struct pathspec ps_local[10]; +}; + +int setup_path(struct path *path) +{ + if (!path) + return -1; + + memset(path, 0, sizeof(*path)); + path->count = 0; + path->alloc = sizeof(path->ps_local)/sizeof(path->ps_local[0]); + path->absolute = false; + path->trailing_slash = false; + path->ps = path->ps_local; + + return 0; +} + +void cleanup_path(struct path *path) +{ + struct pathspec *ps; + unsigned int i; + + if (!path) + return; + + while (path->count > 0) { + ps = &path->ps[--path->count]; + for (i = 0; i < ps->argc; i++) { + if (ps->arg[i].fyd) + fy_document_destroy(ps->arg[i].fyd); + } + } + if (path->ps != path->ps_local) + free(path->ps); + path->ps = path->ps_local; + path->count = 0; + path->alloc = sizeof(path->ps_local)/sizeof(path->ps_local[0]); + path->absolute = false; + path->trailing_slash = false; + path->ps = path->ps_local; +} + +size_t parse_pathspec(const char *str, size_t len, struct pathspec *ps) +{ + const char *s, *e, *ss, *ee; + unsigned int maxargs; + bool is_func; + size_t adv; + struct fy_document *fyd; + struct pathspec_arg *psa; + + s = str; + e = s + len; + + ps->func = NULL; + ps->funcsz = 0; + ps->argc = 0; + + maxargs = sizeof(ps->arg)/sizeof(ps->arg[0]); + + if (s >= e) + return 0; + + is_func = false; + + /* either . .. or .func( */ + if (*s == '.') { + /* . */ + if (s + 1 >= e || s[1] == '/') { + ps->func = s; + ps->funcsz = 1; + s++; + goto out; + } + /* .. */ + if (s[1] == '.') { + if (s + 2 >= e || s[2] == '/') { + ps->func = s; + ps->funcsz = 2; + s += 2; + goto out; + } + + /* error; stop */ + goto err_out; + } + /* skip over . */ + ss = ++s; + while (s < e && *s != '(') + s++; + + /* error; no ( found */ + if (s >= e) + goto err_out; + + ps->func = ss; + ps->funcsz = (size_t)(s - ss); + + is_func = true; + } + + if (is_func && *s == '(') + s++; + + for (;;) { + ss = s; + + fyd = NULL; + if (fy_is_path_flow_key_start(*s)) { + fyd = fy_flow_document_build_from_string(NULL, s, e - s, &adv); + if (!fyd) + goto err_out; + + s = ss + adv; + } else { + if (!is_func) { + while (s < e && *s != '/') + s++; + } else { + while (s < e && *s != ',' && *s != ')') + s++; + } + } + if (ps->argc >= maxargs) + goto err_out; + + psa = &ps->arg[ps->argc++]; + + psa->arg = ss; + psa->argsz = s - ss; + psa->fyd = fyd; + + ee = s; + + /* check for numeric */ + if (ss < ee && (fy_is_num(*ss) || *ss == '-')) { + bool is_neg; + int idx, len, digit; + + if (*ss == '-') { + ss++; + is_neg = true; + } else + is_neg = false; + + idx = 0; + len = 0; + while (ss < ee && fy_is_num(*ss)) { + digit = *ss - '0'; + /* no 0 prefixed numbers allowed */ + if (len > 0 && idx == 0) + goto not_numeric; + + /* number overflow */ + if (idx * 10 < idx) + goto not_numeric; + + idx = idx * 10; + idx += digit; + len++; + ss++; + } + + if (is_neg) + idx = -idx; + psa->is_numeric = true; + psa->number = idx; + + } else { +not_numeric: + psa->is_numeric = false; + psa->number = 0; + } + + if (!is_func || s >= e || *s == ')') + break; + + if (s < e && *s == ',') + s++; + } + + if (is_func && s < e && *s == ')') + s++; +out: + return s - str; + +err_out: + return (size_t)-1; +} + +int parse_path(const char *str, size_t len, struct path *path) +{ + const char *s, *e; + unsigned int i; + struct pathspec *ps, *psnew; + size_t adv; + + path->count = 0; + path->absolute = false; + path->trailing_slash = false; + + if (len == (size_t)-1) + len = strlen(str); + + s = str; + e = s + len; + + if (s >= e) + return -1; + + /* starts at root */ + if (*s == '/') { + s++; + path->absolute = true; + } + + /* check for trailing slash and remove it */ + if (e[-1] == '/') { + e--; + path->trailing_slash = true; + }; + + while (s < e) { + if (path->count >= path->alloc) { + psnew = realloc(path->ps == path->ps_local ? NULL : path->ps, + path->alloc * 2 * sizeof(*psnew)); + if (!psnew) + goto err_out; + if (path->ps == path->ps_local) + memcpy(psnew, path->ps, path->count * sizeof(*psnew)); + path->ps = psnew; + path->alloc *= 2; + } + ps = &path->ps[path->count++]; + adv = parse_pathspec(s, (size_t)(e - s), ps); + if (adv == (size_t)-1) + goto err_out; + + if (adv == 0) + break; + + s += adv; + if (s < e && *s == '/') + s++; + } + + /* error */ + if (s < e) + goto err_out; + + return 0; + +err_out: + while (path->count > 0) { + ps = &path->ps[--path->count]; + for (i = 0; i < ps->argc; i++) { + if (ps->arg[i].fyd) + fy_document_destroy(ps->arg[i].fyd); + } + } + return -1; +} + +int do_pathspec(int argc, char *argv[]) +{ + int i; + const char *s, *e; + size_t adv; + struct pathspec ps; + + assert(argc > 0); + + s = argv[0]; + e = s + strlen(s); + + if (s >= e) + return -1; + + /* starts at root */ + if (*s == '/') { + fprintf(stderr, "starts with /\n"); + s++; + } + + while (s < e) { + fprintf(stderr, "parsing: %.*s\n", (int)(e - s), s); + adv = parse_pathspec(s, (size_t)(e - s), &ps); + if (adv == 0) { + fprintf(stderr, "parse_pathspec() returns 0\n"); + break; + } + fprintf(stderr, "full-ps: %.*s\n", (int)adv, s); + fprintf(stderr, "func: %.*s\n", (int)ps.funcsz, ps.func); + for (i = 0; i < (int)ps.argc; i++) { + fprintf(stderr, "arg[%d]: %.*s\n", i, (int)ps.arg[i].argsz, ps.arg[i].arg); + if (ps.arg[i].fyd) + fy_document_destroy(ps.arg[i].fyd); + } + fprintf(stderr, "\n"); + + s += adv; + + if (s < e && *s == '/') + s++; + } + + return 0; +} + +struct fy_node * +node_find(struct fy_document *fyd, struct fy_node *fyn_start, struct path *path) +{ + struct fy_node *fyn; + struct fy_anchor *fya; + struct pathspec *ps; + struct pathspec_arg *psa; + unsigned int i; + + if (!fyd || !fyn_start || !path) + return NULL; + + if (path->absolute) + fyn_start = fyd->root; + + fyn = fyn_start; + for (i = 0; fyn && i < path->count; i++) { + + ps = &path->ps[i]; + if (ps->funcsz > 0) { + if (ps->funcsz == 1 && !memcmp(ps->func, ".", 1)) { + /* current; nop */ + } else if (ps->funcsz == 2 && !memcmp(ps->func, "..", 2)) { + /* parent */ + fyn = fy_node_get_document_parent(fyn); + } else if (ps->funcsz == 3 && !memcmp(ps->func, "key", 3)) { + if (ps->argc != 1) { + fprintf(stderr, "illegal number of arguments at key\n"); + return NULL; + } + + if (!fy_node_is_mapping(fyn)) { + fprintf(stderr, "key function only works on mappings\n"); + return NULL; + } + + psa = &ps->arg[0]; + + if (psa->fyd) { + fyn = fy_node_mapping_lookup_key_by_key(fyn, fy_document_root(psa->fyd)); + if (!fyn) { + fprintf(stderr, "failed to find complex key\n"); + return NULL; + } + } else { + fyn = fy_node_mapping_lookup_key_by_string(fyn, psa->arg, psa->argsz); + if (!fyn) { + fprintf(stderr, "failed to find simple key\n"); + return NULL; + } + } + + } else { + fprintf(stderr, "unkown function %.*s\n", (int)ps->funcsz, ps->func); + return NULL; + } + } else { + if (ps->argc != 1) { + fprintf(stderr, "illegal number of arguments at key\n"); + return NULL; + } + psa = &ps->arg[0]; + + /* check for alias */ + if (psa->arg[0] == '*') { + /* alias must be the first component and not absolute */ + if (path->absolute) { + fprintf(stderr, "bad alias when absolute\n"); + return NULL; + } + if (i > 0) { + fprintf(stderr, "bad alias not at start\n"); + return NULL; + } + fya = fy_document_lookup_anchor(fyd, &psa->arg[1], psa->argsz-1); + if (!fya) { + fprintf(stderr, "bad alias unable to find anchor\n"); + return NULL; + } + /* continue */ + fyn = fya->fyn; + continue; + } + + switch (fyn->type) { + case FYNT_SCALAR: + if (!fy_node_is_alias(fyn)) { + fprintf(stderr, "at scalar; this is the end\n"); + return NULL; + } + fya = fy_document_lookup_anchor_by_token(fyd, fyn->scalar); + if (!fya) { + fprintf(stderr, "unable to lookup alias\n"); + return NULL; + } + fyn = fya->fyn; + break; + + case FYNT_SEQUENCE: + if (!psa->is_numeric) { + fprintf(stderr, "sequence requires numeric argument\n"); + return NULL; + } + + fyn = fy_node_sequence_get_by_index(fyn, psa->number); + if (!fyn) { + fprintf(stderr, "failed to find sequence idx\n"); + return NULL; + } + + break; + + case FYNT_MAPPING: + if (psa->fyd) { + fyn = fy_node_mapping_lookup_value_by_key(fyn, fy_document_root(psa->fyd)); + if (!fyn) { + fprintf(stderr, "failed to find complex key\n"); + return NULL; + } + } else { + fyn = fy_node_mapping_lookup_by_string(fyn, psa->arg, psa->argsz); + if (!fyn) { + fprintf(stderr, "failed to find simple key\n"); + return NULL; + } + } + break; + } + } + + if (fy_node_is_alias(fyn)) { + struct fy_node *referred[FYPCF_GUARANTEED_MINIMUM_DEPTH_LIMIT]; + unsigned int derefs, k; + + for (derefs = 0; derefs < FYPCF_GUARANTEED_MINIMUM_DEPTH_LIMIT; derefs++) { + if (!fy_node_is_alias(fyn)) + break; + fya = fy_document_lookup_anchor_by_token(fyd, fyn->scalar); + if (!fya) { + fprintf(stderr, "unable to deref alias\n"); + return NULL; + } + for (k = 0; k < derefs; k++) { + if (fya->fyn == referred[k]) { + fprintf(stderr, "alias loop detected\n"); + return NULL; + } + } + fyn = fya->fyn; + referred[derefs] = fyn; + } + } + } + + return fyn; +} + +struct fy_node * +node_find_exec(struct fy_document *fyd, struct fy_node *fyn_start, const char *path) +{ + struct fy_input *fyi; + struct fy_path_parser fypp_local, *fypp = &fypp_local; + struct fy_path_parse_cfg pcfg_local, *pcfg = &pcfg_local; + struct fy_path_exec *fypx = NULL; + struct fy_path_exec_cfg xcfg_local, *xcfg = &xcfg_local; + struct fy_emitter fye_local, *fye = &fye_local; + struct fy_emitter_cfg ecfg_local, *ecfg = &ecfg_local; + struct fy_path_expr *expr; + struct fy_document *fyd_pe; + struct fy_node *fyn; + void *iterp; + int rc; + + fyi = fy_input_from_data(path, strlen(path), NULL, false); + assert(fyi); + + memset(pcfg, 0, sizeof(*pcfg)); + pcfg->diag = fyd->diag; + fy_path_parser_setup(fypp, pcfg); + + memset(xcfg, 0, sizeof(*xcfg)); + xcfg->diag = fyd->diag; + fypx = fy_path_exec_create(xcfg); + assert(fypx); + + rc = fy_path_parser_open(fypp, fyi, NULL); + assert(!rc); + + fyn = NULL; + expr = fy_path_parse_expression(fypp); + if (!expr) { + fprintf(stderr, "Failed to parse expression \"%s\"\n", path); + goto do_close; + } + fprintf(stderr, "OK; parsed expression \"%s\"\n", path); + + fy_path_expr_dump(expr, fyd->diag, FYET_WARNING, 0, "expression dump"); + + fyd_pe = fy_path_expr_to_document(expr); + if (!fyd_pe) { + fprintf(stderr, "Failed to create YAML path expression tree for \"%s\"\n", path); + goto do_free; + } + + memset(ecfg, 0, sizeof(*ecfg)); + ecfg->diag = fyd->diag; + fy_emit_setup(fye, ecfg); + + fy_emit_document(fye, fyd_pe); + + fy_emit_cleanup(fye); + + fy_document_destroy(fyd_pe); + + rc = fy_path_exec_execute(fypx, expr, fyn_start); + if (rc) { + fprintf(stderr, "Failed to execute expression \"%s\"\n", path); + goto do_free; + } + + iterp = NULL; + fyn = fy_path_exec_results_iterate(fypx, &iterp); + +do_free: + fy_path_expr_free(expr); + +do_close: + fy_path_parser_close(fypp); + + fy_path_exec_unref(fypx); + + fy_path_parser_cleanup(fypp); + fy_input_unref(fyi); + + return fyn; +} + +int do_bypath(struct fy_parser *fyp, const char *pathstr, const char *start) +{ + struct path path; + struct pathspec *ps; + struct pathspec_arg *arg; + struct fy_document *fyd; + struct fy_node *fyn; + int count; + unsigned int i, j; + int rc __FY_DEBUG_UNUSED__; + + setup_path(&path); + + rc = parse_path(pathstr, (size_t)-1, &path); + assert(!rc); + + fprintf(stderr, "%s: pathstr=%s absolute=%s trailing_slash=%s\n", __func__, + pathstr, + path.absolute ? "true" : "false", + path.trailing_slash ? "true" : "false"); + for (i = 0; i < path.count; i++) { + ps = &path.ps[i]; + fprintf(stderr, " func=%.*s argc=%u", (int)ps->funcsz, ps->func, ps->argc); + for (j = 0; j < ps->argc; j++) { + arg = &ps->arg[j]; + fprintf(stderr, " %.*s", (int)arg->argsz, arg->arg); + } + fprintf(stderr, "\n"); + } + + count = 0; + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + + fyn = node_find_exec(fyd, fy_document_root(fyd), pathstr); + if (!fyn) { + fprintf(stderr, "exec: did not find node for %s\n", pathstr); + } else { + fprintf(stderr, "exec: path %s - return %s\n", pathstr, fy_node_get_path_alloca(fyn)); + } + + fyn = node_find(fyd, fy_document_root(fyd), &path); + if (!fyn) { + fprintf(stderr, "norm: did not find node for %s\n", pathstr); + } else { + fprintf(stderr, "norm: path %s - return %s\n", pathstr, fy_node_get_path_alloca(fyn)); + } + + fy_parse_document_destroy(fyp, fyd); + + count++; + } + + cleanup_path(&path); + + return count > 0 ? 0 : -1; +} + +struct test_parser { + struct fy_reader reader; + struct fy_diag *diag; + struct fy_input *fyi; +}; + +struct fy_diag *test_parser_reader_get_diag(struct fy_reader *fyr) +{ + struct test_parser *parser = container_of(fyr, struct test_parser, reader); + return parser->diag; +} + +static const struct fy_reader_ops test_parser_reader_ops = { + .get_diag = test_parser_reader_get_diag, + .file_open = NULL, +}; + +int do_reader(struct fy_parser *fyp, int indent, int width, bool resolve, bool sort) +{ + const char *data = "this is a test-testing: more data"; + struct test_parser parser; + struct fy_diag_cfg dcfg; + struct fy_diag *diag; + struct fy_reader *fyr; + struct fy_input *fyi; + struct fy_document *fyd; + char ubuf[5]; + int c; + int r __FY_DEBUG_UNUSED__; + + fy_diag_cfg_default(&dcfg); + diag = fy_diag_create(&dcfg); + assert(diag); + + fyi = fy_input_from_data(data, FY_NT, NULL, false); + assert(fyi); + + memset(&parser, 0, sizeof(parser)); + fyr = &parser.reader; + parser.diag = diag; + + fy_reader_setup(fyr, &test_parser_reader_ops); + fyr_notice(fyr, "Reader initialized\n"); + + r = fy_reader_input_open(fyr, fyi, NULL); + assert(!r); + fyr_notice(fyr, "Reader input opened\n"); + + while ((c = fy_reader_peek(fyr)) >= 0 && c != '{' && c != '[' && c != '"' && c != '\'' && c != '-') { + fy_reader_advance(fyr, c); + fy_utf8_put_unchecked(ubuf, c); + fyr_notice(fyr, "%.*s %d\n", (int)fy_utf8_width(c), ubuf, c); + } + + if (c > 0) { + fy_parser_set_reader(fyp, fyr); + fy_parser_set_flow_only_mode(fyp, true); + + fyd = fy_parse_load_document(fyp); + if (fyd) { + fyr_notice(fyr, "parsed a yaml document\n"); + } + + (void)fy_emit_document_to_file(fyd, 0, NULL); + + fy_document_destroy(fyd); + + /* remaining */ + while ((c = fy_reader_peek(fyr)) >= 0) { + fy_reader_advance(fyr, c); + fy_utf8_put_unchecked(ubuf, c); + fyr_notice(fyr, "%.*s %d\n", (int)fy_utf8_width(c), ubuf, c); + } + } + + fy_reader_input_done(fyr); + fyr_notice(fyr, "Reader input done\n"); + + fy_input_close(fyi); + + fy_reader_cleanup(fyr); + + fy_input_unref(fyi); + fy_diag_destroy(diag); + + return 0; +} + +int do_walk(struct fy_parser *fyp, const char *walkpath, const char *walkstart, int indent, int width, bool resolve, bool sort) +{ + struct fy_path_parse_cfg pcfg; + struct fy_path_parser fypp_data, *fypp = &fypp_data; + struct fy_path_expr *expr; + struct fy_walk_result_list results; + struct fy_walk_result *result; + struct fy_input *fyi; + struct fy_document *fyd, *fyd2; + struct fy_node *fyn, *fyn2; + struct fy_walk_result *fwr; + struct fy_path_exec *fypx = NULL; + struct fy_path_exec_cfg xcfg_local, *xcfg = &xcfg_local; + char *path; + unsigned int flags; + int rc; + + flags = 0; + if (sort) + flags |= FYECF_SORT_KEYS; + flags |= FYECF_INDENT(indent) | FYECF_WIDTH(width); + + fy_notice(fyp->diag, "setting up path parser for \"%s\"\n", walkpath); + + memset(&pcfg, 0, sizeof(pcfg)); + pcfg.diag = fyp->diag; + fy_path_parser_setup(fypp, &pcfg); + + fyi = fy_input_from_data(walkpath, FY_NT, NULL, false); + assert(fyi); + + rc = fy_path_parser_open(fypp, fyi, NULL); + assert(!rc); + + fy_notice(fyp->diag, "path parser input set for \"%s\"\n", walkpath); + + /* while ((fyt = fy_path_scan(fypp)) != NULL) { + dump_token(fyt); + fy_token_unref(fyt); + } */ + expr = fy_path_parse_expression(fypp); + if (!expr) { + fy_error(fyp->diag, "failed to parse expression\n"); + } else + fy_path_expr_dump(expr, fyp->diag, FYET_NOTICE, 0, "fypp root "); + + memset(xcfg, 0, sizeof(*xcfg)); + xcfg->diag = fyp->diag; + fypx = fy_path_exec_create(xcfg); + assert(fypx); + + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + + if (resolve) { + rc = fy_document_resolve(fyd); + if (rc) + return -1; + } + + fyn = fy_node_by_path(fy_document_root(fyd), walkstart, FY_NT, FYNWF_DONT_FOLLOW); + if (!fyn) { + printf("could not find walkstart node %s\n", walkstart); + continue; + } + + fy_emit_document_to_file(fyd, flags, NULL); + + path = fy_node_get_path(fyn); + assert(path); + printf("# walking starting at %s\n", path); + free(path); + + fy_walk_result_list_init(&results); + fwr = fy_walk_result_alloc_rl(NULL); + assert(fwr); + fwr->type = fwrt_node_ref; + fwr->fyn = fyn; + result = fy_path_expr_execute(fypx, 0, expr, fwr, fpet_none); + + printf("\n"); + if (!result) { + printf("# no results\n"); + goto next; + } + + if (result->type == fwrt_node_ref) { + printf("# single reference result\n"); + + path = fy_node_get_path(result->fyn); + assert(path); + + printf("# %s\n", path); + free(path); + + fyd2 = fy_document_create(&fyp->cfg); + assert(fyd2); + + fyn2 = fy_node_copy(fyd2, result->fyn); + assert(fyn2); + + fy_document_set_root(fyd2, fyn2); + + fy_emit_document_to_file(fyd2, flags, NULL); + + fy_document_destroy(fyd2); + + goto next; + } + + printf("# multiple results\n"); + while ((fwr = fy_walk_result_list_pop(&result->refs)) != NULL) { + + if (fwr->type != fwrt_node_ref) { + fy_walk_result_free_rl(NULL, fwr); + continue; + } + + path = fy_node_get_path(fwr->fyn); + assert(path); + + printf("# %s\n", path); + free(path); + + fyd2 = fy_document_create(&fyp->cfg); + assert(fyd2); + + fyn2 = fy_node_copy(fyd2, fwr->fyn); + assert(fyn2); + + fy_document_set_root(fyd2, fyn2); + + printf("---\n"); + fy_emit_document_to_file(fyd2, flags, NULL); + + fy_document_destroy(fyd2); + + fy_walk_result_free_rl(NULL, fwr); + } +next: + fy_walk_result_free_rl(NULL, result); + + fy_parse_document_destroy(fyp, fyd); + } + + fy_path_exec_unref(fypx); + + fy_path_expr_free(expr); + + fy_path_parser_close(fypp); + + fy_input_unref(fyi); + fy_path_parser_cleanup(fypp); + + return 0; +} + +int do_crash(const struct fy_parse_cfg *cfg, int argc, char *argv[]) +{ + struct fy_document *fyd = NULL; + struct fy_node *fyn = NULL; + // illegal> + char key[12] = {0x26, 0x2b, 0x74, 0x68, 0x65, 0x62, 0x65, 0x86, 0x6e, 0x67, 0x77, 0x00}; + int rc = -1; + + fyd = fy_document_build_from_string(cfg, "base: &base\n name: this-is-a-name\n", FY_NT); + if (!fyd) { + fprintf(stderr, "failed to build document"); + goto failed; + } + + fyn = fy_node_buildf(fyd, "abc"); + if (!fyn) { + fprintf(stderr, "failed to build a node"); + goto failed; + } + + rc = fy_document_insert_at(fyd, key, FY_NT, fyn); + fyn = NULL; + if (rc) { + fprintf(stderr, "failed to insert document\n"); + goto failed; + } + rc = fy_emit_document_to_fp(fyd, FYECF_DEFAULT | FYECF_SORT_KEYS, stdout); + if (rc) { + fprintf(stderr, "failed to emit document to stdout\n"); + goto failed; + } + + rc = 0; +failed: + fy_node_free(fyn); + fy_document_destroy(fyd); + return rc; +} + +int do_bad_utf8(const struct fy_parse_cfg *cfg, int argc, char *argv[]) +{ + // char key[12] = {0x26, 0x2b, 0x74, 0x68, 0x65, 0x62, 0x65, 0x86, 0x6e, 0x67, 0x77, 0x00}; + // char key[11] = {0x26, 0x2b, 0x74, 0x68, 0x65, 0x62, 0x65, 0x6e, 0x67, 0x77, 0x00}; + // char key[] = { + // 0x22, 0xCE, 0xA4, 0xCE, 0xB9, 0xCE, 0xBC, 0xCE, + // 0xAE, 0x20, 0xCE, 0xB5, 0xCE, 0xBB, 0xCE, 0xBB, + // 0xCE, 0xB7, 0xCE, 0xBD, 0xCE, 0xB9, 0xCE, 0xBA, + // 0xCE, 0xAE, 0x22, 0x0A, 0x00 + //}; + char key[] = { + 0x67, 0xe7, 0x67, 0x54, 0x67, 0x67, 0x67, 0x67, 0xe8, + 0x67, 0x4e, 0x64, 0x6a, 0x67, 0x67, 0xaa, 0x6b, 0x73, 0x00 + }; + int *fwd; + int *bwd; + const char *s; + const char *e; + int len, i, c, w, pos; + + len = strlen(key); + + fwd = alloca(sizeof(*fwd) * len); + bwd = alloca(sizeof(*bwd) * len); + + memset(fwd, 0, sizeof(*fwd) * len); + memset(bwd, 0, sizeof(*bwd) * len); + + s = key; + e = s + strlen(key); + + printf("forward utf8 check\n"); + pos = 0; + while (s < e) { + c = fy_utf8_get(s, e - s, &w); + if (c < 0) { + switch (c) { + case FYUG_EOF: + printf("EOF before end at pos %d\n", pos); + break; + case FYUG_INV: + printf("INV before end at pos %d\n", pos); + break; + case FYUG_PARTIAL: + printf("PARTIAL before end at pos %d\n", pos); + break; + default: + printf("UKNNOWN %d before end at pos %d\n", c, pos); + break; + } + break; + } + fwd[pos] = c; + s += w; + pos++; + } + printf("forward utf8 check complete (end pos %d)\n", pos); + + for (i = 0; i < pos; i++) + printf("0x%02x%s", fwd[i], i < (pos - 1) ? " " : "\n"); + + printf("backward utf8 check\n"); + pos = 0; + s = key; + while (s < e) { + c = fy_utf8_get_right(s, e - s, &w); + if (c < 0) { + switch (c) { + case FYUG_EOF: + printf("EOF before end at pos %d\n", pos); + break; + case FYUG_INV: + printf("INV before end at pos %d\n", pos); + break; + case FYUG_PARTIAL: + printf("PARTIAL before end at pos %d\n", pos); + break; + default: + printf("UKNNOWN %d before end at pos %d\n", c, pos); + break; + } + break; + } + bwd[pos] = c; + e -= w; + pos++; + } + printf("backward utf8 check complete (end pos %d)\n", pos); + + for (i = pos - 1; i >= 0; i--) + printf("0x%02x%s", bwd[i], i > 0 ? " " : "\n"); + + return 0; +} + +int do_shell_split(int in_argc, char *in_argv[]) +{ + char buf[256], line[256 + 1]; + char *s, *e; + const char * const *argv; + int i, argc; + void *mem; + + printf("shell split; Ctrl-D to exit\n"); + buf[sizeof(buf)-1] = '\0'; + while (fgets(buf, sizeof(buf) - 1, stdin)) { + buf[sizeof(buf)-1] = '\0'; + strcpy(line, buf); + s = line; + e = s + strlen(line); + while (e > s && e[-1] == '\n') + *--e = '\0'; + + printf("input: '%s'\n", line); + + mem = fy_utf8_split_posix(line, &argc, &argv); + if (!mem) { + fprintf(stderr, "Bad input '%s'\n", line); + } else { + for (i = 0; i < argc; i++) { + fprintf(stderr, "%d: %s\n", i, argv[i]); + } + assert(argv[argc] == NULL); + free(mem); + } + } + return 0; +} + +int do_parse_timing(int argc, char *argv[], bool disable_mmap) +{ + void *blob; + size_t blob_size; + int i, c, w; + struct timespec before, after; + int64_t ns; + const uint8_t *s, *e, *ss; + size_t count; + struct fy_utf8_result res; + +#undef BEFORE +#define BEFORE() \ + do { \ + clock_gettime(CLOCK_MONOTONIC, &before); \ + } while(0) + +#undef AFTER +#define AFTER() \ + ({ \ + clock_gettime(CLOCK_MONOTONIC, &after); \ + (int64_t)(after.tv_sec - before.tv_sec) * (int64_t)1000000000UL + (int64_t)(after.tv_nsec - before.tv_nsec); \ + }) + + for (i = optind; i < argc; i++) { + + printf("file=%s\n", argv[i]); + + BEFORE(); + blob = fy_blob_read(argv[i], &blob_size); + assert(blob); + ns = AFTER(); + + printf("read %zu bytes in %"PRId64"ns\n", blob_size, ns); + + BEFORE(); + s = blob; + e = s + blob_size; + count = 0; + while (s < e) { + if (*s++ == 'e') + count++; + } + ns = AFTER(); + printf("%zu 'e' chars method 1 in %"PRId64"ns\n", count, ns); + + BEFORE(); + s = blob; + e = s + blob_size; + count = 0; + while ((ss = memchr(s, 'e', e - s)) != NULL) { + count++; + s = ss + 1; + } + ns = AFTER(); + printf("%zu 'e' chars method 2 in %"PRId64"ns\n", count, ns); + + BEFORE(); + s = blob; + e = s + blob_size; + count = 0; + while ((c = fy_utf8_get(s, e - s, &w)) >= 0) { + if (c == 'e') + count++; + s += w; + } + ns = AFTER(); + printf("%zu 'e' utf8 characters using method 1 in %"PRId64"ns\n", count, ns); + + BEFORE(); + s = blob; + e = s + blob_size; + count = 0; + while ((c = fy_utf8_get_s(s, e, &w)) >= 0) { + if (c == 'e') + count++; + s += w; + } + ns = AFTER(); + printf("%zu 'e' utf8 characters using method 2 in %"PRId64"ns\n", count, ns); + + BEFORE(); + s = blob; + e = s + blob_size - FY_UTF8_MAX_WIDTH; + count = 0; + while (s < e && (c = fy_utf8_get_s_nocheck(s, &w)) >= 0) { + if (c == 'e') + count++; + s += w; + } + while ((c = fy_utf8_get_s(s, e, &w)) >= 0) { + if (c == 'e') + count++; + s += w; + } + ns = AFTER(); + printf("%zu 'e' utf8 characters using method 3 in %"PRId64"ns\n", count, ns); + + BEFORE(); + s = blob; + e = s + blob_size; + count = 0; + while ((res = fy_utf8_get_s_res(s, e)).c >= 0) { + if (res.c == 'e') + count++; + s += res.w; + } + ns = AFTER(); + printf("%zu 'e' utf8 characters using method 4 in %"PRId64"ns\n", count, ns); + + free(blob); + + } + + return 0; +} + +int apply_flags_option(const char *arg, unsigned int *flagsp, + int (*modify_flags)(const char *what, unsigned int *flagsp)) +{ + const char *s, *e, *sn; + char *targ; + int len, ret; + + if (!arg || !flagsp || !modify_flags) + return -1; + + s = arg; + e = arg + strlen(s); + while (s < e) { + sn = strchr(s, ','); + if (!sn) + sn = e; + + len = sn - s; + targ = alloca(len + 1); + memcpy(targ, s, len); + targ[len] = '\0'; + + ret = modify_flags(targ, flagsp); + if (ret) + return ret; + + s = sn < e ? (sn + 1) : sn; + } + + return 0; +} + +static ssize_t callback_stdin_input(void *user, void *buf, size_t count) +{ + return fread(buf, 1, count, stdin); +} + +int main(int argc, char *argv[]) +{ + struct fy_parser ctx, *fyp = &ctx; + struct fy_parse_cfg cfg = { + .search_path = INCLUDE_DEFAULT, + .flags = (QUIET_DEFAULT ? FYPCF_QUIET : 0), + }; + enum fy_error_type error_level = FYET_MAX; + int color_diag = -1; + struct fy_input_cfg *fyic, *fyic_array = NULL; + int i, j, icount, rc, exitcode = EXIT_FAILURE, opt, lidx; + char *tmp, *s; + const char *mode = MODE_DEFAULT; + int indent = INDENT_DEFAULT; + int width = WIDTH_DEFAULT; + bool resolve = RESOLVE_DEFAULT; + bool sort = SORT_DEFAULT; + size_t chunk = CHUNK_DEFAULT; + const char *color = COLOR_DEFAULT; + const char *walkpath = "/"; + const char *walkstart = "/"; + bool use_callback = false; + bool null_output = false; + + fy_valgrind_check(&argc, &argv); + + while ((opt = getopt_long(argc, argv, "I:m:i:w:d:rsc:C:D:M:W:S:qh", lopts, &lidx)) != -1) { + switch (opt) { + case 'I': + tmp = alloca(strlen(cfg.search_path) + 1 + strlen(optarg) + 1); + s = tmp; + strcpy(s, cfg.search_path); + if (cfg.search_path && cfg.search_path[0]) { + s += strlen(cfg.search_path); + *s++ = ':'; + } + strcpy(s, optarg); + s += strlen(optarg); + *s = '\0'; + cfg.search_path = tmp; + break; + case 'm': + mode = optarg; + break; + case 'i': + indent = atoi(optarg); + break; + case 'w': + width = atoi(optarg); + break; + case 'd': + error_level = fy_string_to_error_type(optarg); + if (error_level == FYET_MAX) { + fprintf(stderr, "bad diag option %s\n", optarg); + display_usage(stderr, argv[0]); + } + break; + case 'r': + resolve = true; + cfg.flags |= FYPCF_RESOLVE_DOCUMENT; + break; + case 's': + sort = true; + break; + case 'c': + chunk = atoi(optarg); + break; + case 'C': + color = optarg; + if (!strcmp(color, "auto")) + color_diag = -1; + else if (!strcmp(color, "yes") || !strcmp(color, "1") || !strcmp(color, "on")) + color_diag = 1; + else if (!strcmp(color, "no") || !strcmp(color, "0") || !strcmp(color, "off")) + color_diag = 0; + else { + fprintf(stderr, "bad color option %s\n", optarg); + display_usage(stderr, argv[0]); + } + break; + case 'D': + /* XXX TODO if I'm ever bothered */ + break; + case 'M': + /* XXX TODO if I'm ever bothered */ + break; + case 'W': + walkpath = optarg; + break; + case 'S': + walkstart = optarg; + break; + case OPT_DISABLE_MMAP: + cfg.flags |= FYPCF_DISABLE_MMAP_OPT; + break; + case OPT_DISABLE_ACCEL: + cfg.flags |= FYPCF_DISABLE_ACCELERATORS; + break; + case OPT_DISABLE_BUFFERING: + cfg.flags |= FYPCF_DISABLE_BUFFERING; + break; + case OPT_DISABLE_DEPTH_LIMIT: + cfg.flags |= FYPCF_DISABLE_DEPTH_LIMIT; + break; + case OPT_USE_CALLBACK: + use_callback = true; + break; + case OPT_NULL_OUTPUT: + null_output = true; + break; + case OPT_YAML_1_1: + cfg.flags &= ~(FYPCF_DEFAULT_VERSION_MASK << FYPCF_DEFAULT_VERSION_SHIFT); + cfg.flags |= FYPCF_DEFAULT_VERSION_1_1; + break; + case OPT_YAML_1_2: + cfg.flags &= ~(FYPCF_DEFAULT_VERSION_MASK << FYPCF_DEFAULT_VERSION_SHIFT); + cfg.flags |= FYPCF_DEFAULT_VERSION_1_2; + break; + case OPT_YAML_1_3: + cfg.flags &= ~(FYPCF_DEFAULT_VERSION_MASK << FYPCF_DEFAULT_VERSION_SHIFT); + cfg.flags |= FYPCF_DEFAULT_VERSION_1_3; + break; + case OPT_SLOPPY_FLOW_INDENTATION: + cfg.flags |= FYPCF_SLOPPY_FLOW_INDENTATION; + break; + case OPT_YPATH_ALIASES: + cfg.flags |= FYPCF_YPATH_ALIASES; + break; + case 'q': + cfg.flags |= FYPCF_QUIET; + break; + case 'h' : + default: + if (opt != 'h') + fprintf(stderr, "Unknown option\n"); + display_usage(opt == 'h' ? stdout : stderr, argv[0]); + return EXIT_SUCCESS; + } + } + + /* check mode */ + if (strcmp(mode, "parse") && + strcmp(mode, "scan") && + strcmp(mode, "copy") && + strcmp(mode, "testsuite") && + strcmp(mode, "dump") && + strcmp(mode, "dump2") && + strcmp(mode, "build") && + strcmp(mode, "walk") && + strcmp(mode, "reader") && + strcmp(mode, "compose") && + strcmp(mode, "iterate") && + strcmp(mode, "comment") && + strcmp(mode, "pathspec") && + strcmp(mode, "bypath") && + strcmp(mode, "crash") && + strcmp(mode, "badutf8") && + strcmp(mode, "shell-split") && + strcmp(mode, "parse-timing") +#if defined(HAVE_LIBYAML) && HAVE_LIBYAML + && strcmp(mode, "libyaml-scan") + && strcmp(mode, "libyaml-parse") + && strcmp(mode, "libyaml-testsuite") + && strcmp(mode, "libyaml-dump") + && strcmp(mode, "libyaml-diff") +#endif + + ) { + fprintf(stderr, "Unknown mode %s\n", mode); + display_usage(opt == 'h' ? stdout : stderr, argv[0]); + } + + /* libyaml options are first */ +#if defined(HAVE_LIBYAML) && HAVE_LIBYAML + if (!strcmp(mode, "libyaml-scan") || + !strcmp(mode, "libyaml-parse") || + !strcmp(mode, "libyaml-testsuite") || + !strcmp(mode, "libyaml-dump")) { + FILE *fp; + yaml_parser_t parser; + yaml_emitter_t emitter; + + if (optind >= argc) { + fprintf(stderr, "Missing file argument\n"); + goto cleanup; + } + + fp = fopen(argv[optind], "rb"); + if (!fp) { + fprintf(stderr, "Failed to open file %s\n", argv[optind]); + goto cleanup; + } + + rc = yaml_parser_initialize(&parser); + assert(rc); + rc = yaml_emitter_initialize(&emitter); + assert(rc); + + yaml_parser_set_input_file(&parser, fp); + yaml_emitter_set_output_file(&emitter, stdout); + + if (!strcmp(mode, "libyaml-scan")) { + rc = do_libyaml_scan(&parser); + if (rc < 0) { + fprintf(stderr, "do_libyaml_scan() error %d\n", rc); + fprintf(stderr, " problem='%s' context='%s'\n", parser.problem, parser.context); + } + } else if (!strcmp(mode, "libyaml-parse")) { + rc = do_libyaml_parse(&parser); + if (rc < 0) { + fprintf(stderr, "do_libyaml_parse() error %d\n", rc); + fprintf(stderr, " problem='%s' context='%s'\n", parser.problem, parser.context); + } + } else if (!strcmp(mode, "libyaml-testsuite")) { + rc = do_libyaml_testsuite(stdout, &parser, null_output); + if (rc < 0) { + fprintf(stderr, "do_libyaml_testsuite() error %d\n", rc); + fprintf(stderr, " problem='%s' context='%s'\n", parser.problem, parser.context); + } + } else if (!strcmp(mode, "libyaml-dump")) { + rc = do_libyaml_dump(&parser, &emitter, null_output); + if (rc < 0) { + fprintf(stderr, "do_libyaml_dump() error %d\n", rc); + if (parser.problem) + fprintf(stderr, " problem='%s' context='%s'\n", parser.problem, parser.context); + } + } else + rc = -1; + + yaml_parser_delete(&parser); + yaml_emitter_delete(&emitter); + + fclose(fp); + + return !rc ? EXIT_SUCCESS : EXIT_FAILURE; + } +#endif + +#if defined(HAVE_LIBYAML) && HAVE_LIBYAML + /* set yaml 1.1 mode with sloppy indentation to match libyaml */ + if (!strcmp(mode, "libyaml-diff")) { + cfg.flags &= ~(FYPCF_DEFAULT_VERSION_MASK << FYPCF_DEFAULT_VERSION_SHIFT); + cfg.flags |= FYPCF_DEFAULT_VERSION_1_1; + cfg.flags |= FYPCF_SLOPPY_FLOW_INDENTATION; + } +#endif + + if (!strcmp(mode, "build")) { + rc = do_build(&cfg, argc - optind, argv + optind); + return !rc ? EXIT_SUCCESS : EXIT_FAILURE; + } + + if (!strcmp(mode, "crash")) { + rc = do_crash(&cfg, argc - optind, argv + optind); + return !rc ? EXIT_SUCCESS : EXIT_FAILURE; + } + if (!strcmp(mode, "badutf8")) { + rc = do_bad_utf8(&cfg, argc - optind, argv + optind); + return !rc ? EXIT_SUCCESS : EXIT_FAILURE; + } + + if (!strcmp(mode, "pathspec")) { + rc = do_pathspec(argc - optind, argv + optind); + return !rc ? EXIT_SUCCESS : EXIT_FAILURE; + } + + /* turn on comment parsing for comment mode */ + if (!strcmp(mode, "comment")) + cfg.flags |= FYPCF_PARSE_COMMENTS; + + rc = fy_parse_setup(fyp, &cfg); + if (rc) { + fprintf(stderr, "fy_parse_setup() failed\n"); + goto cleanup; + } + + if (error_level != FYET_MAX) + fy_diag_set_level(fy_parser_get_diag(fyp), error_level); + if (color_diag != -1) + fy_diag_set_colorize(fy_parser_get_diag(fyp), !!color_diag); + + icount = argc - optind; + if (!icount) + icount++; + + fyic_array = alloca(sizeof(*fyic_array) * icount); + memset(fyic_array, 0, sizeof(*fyic_array) * icount); + + j = 0; + for (i = optind; i < argc; i++) { + fyic = &fyic_array[i - optind]; + if (!strcmp(argv[i], "-")) { + if (!use_callback) { + fyic->type = fyit_stream; + fyic->stream.name = "stdin"; + fyic->stream.fp = stdin; + fyic->chunk = chunk; + } else { + fyic->type = fyit_callback; + fyic->userdata = stdin; + fyic->callback.input = callback_stdin_input; + } + } else { + fyic->type = fyit_file; + fyic->file.filename = argv[i]; + } + + rc = fy_parse_input_append(fyp, fyic); + if (rc) { + fprintf(stderr, "fy_input_append() failed\n"); + goto cleanup; + } + j++; + } + + if (!j) { + fyic = &fyic_array[0]; + + if (!use_callback) { + fyic->type = fyit_stream; + fyic->stream.name = "stdin"; + fyic->stream.fp = stdin; + fyic->chunk = chunk; + } else { + fyic->type = fyit_callback; + fyic->userdata = stdin; + fyic->callback.input = callback_stdin_input; + } + + rc = fy_parse_input_append(fyp, fyic); + if (rc) { + fprintf(stderr, "fy_input_append() failed\n"); + goto cleanup; + } + } + + if (!strcmp(mode, "parse")) { + rc = do_parse(fyp); + if (rc < 0) { + /* fprintf(stderr, "do_parse() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "scan")) { + rc = do_scan(fyp); + if (rc < 0) { + fprintf(stderr, "do_scan() error %d\n", rc); + goto cleanup; + } + } else if (!strcmp(mode, "copy")) { + rc = do_copy(fyp); + if (rc < 0) { + fprintf(stderr, "do_copy() error %d\n", rc); + goto cleanup; + } + } else if (!strcmp(mode, "testsuite")) { + rc = do_testsuite(stdout, fyp, null_output); + if (rc < 0) { + /* fprintf(stderr, "do_testsuite() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "dump")) { + rc = do_dump(fyp, indent, width, resolve, sort, null_output); + if (rc < 0) { + /* fprintf(stderr, "do_dump() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "dump2")) { + rc = do_dump2(fyp, indent, width, resolve, sort, null_output); + if (rc < 0) { + /* fprintf(stderr, "do_dump() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "walk")) { + rc = do_walk(fyp, walkpath, walkstart, indent, width, resolve, sort); + if (rc < 0) { + /* fprintf(stderr, "do_walk() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "reader")) { + rc = do_reader(fyp, indent, width, resolve, sort); + if (rc < 0) { + /* fprintf(stderr, "do_reader() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "compose")) { + rc = do_compose(fyp, indent, width, resolve, sort, null_output); + if (rc < 0) { + /* fprintf(stderr, "do_compose() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "iterate")) { + rc = do_iterate(fyp); + if (rc < 0) { + /* fprintf(stderr, "do_iterate() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "comment")) { + rc = do_comment(fyp); + if (rc < 0) { + /* fprintf(stderr, "do_comment() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "bypath")) { + rc = do_bypath(fyp, walkpath, walkstart); + if (rc < 0) { + /* fprintf(stderr, "do_bypath() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "shell-split")) { + rc = do_shell_split(argc, argv); + if (rc < 0) { + /* fprintf(stderr, "do_shell_split() error %d\n", rc); */ + goto cleanup; + } + } else if (!strcmp(mode, "parse-timing")) { + rc = do_parse_timing(argc, argv, !!(cfg.flags & FYPCF_DISABLE_MMAP_OPT)); + if (rc < 0) { + /* fprintf(stderr, "do_parse_timing() error %d\n", rc); */ + goto cleanup; + } + } +#if defined(HAVE_LIBYAML) && HAVE_LIBYAML + if (!strcmp(mode, "libyaml-diff")) { + FILE *fp; + // FILE *t1fp, *t2fp; + + yaml_parser_t parser; + + if (optind >= argc) { + fprintf(stderr, "Missing file argument\n"); + goto cleanup; + } + + fp = fopen(argv[optind], "rb"); + if (!fp) { + fprintf(stderr, "Failed to open file %s\n", argv[optind]); + goto cleanup; + } + + rc = yaml_parser_initialize(&parser); + assert(rc); + + yaml_parser_set_input_file(&parser, fp); + + fprintf(stdout, "LIBYAML:\n"); + rc = do_libyaml_testsuite(stdout, &parser, false); + if (rc < 0) { + fprintf(stderr, "do_libyaml_testsuite() failed\n"); + goto cleanup; + } + + fprintf(stdout, "LIBFYAML:\n"); + rc = do_testsuite(stdout, fyp, false); + if (rc < 0) { + fprintf(stderr, "do_libyaml_testsuite() failed\n"); + goto cleanup; + } + + yaml_parser_delete(&parser); + + fclose(fp); + + if (rc < 0) + goto cleanup; + } +#endif + + exitcode = EXIT_SUCCESS; + +cleanup: + fy_parse_cleanup(&ctx); + + return exitcode; +} diff --git a/src/lib/fy-accel.c b/src/lib/fy-accel.c new file mode 100644 index 0000000..87a9b23 --- /dev/null +++ b/src/lib/fy-accel.c @@ -0,0 +1,419 @@ +/* + * fy-accel.c - YAML accelerated access methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include + +#include "fy-parse.h" +#include "fy-doc.h" + +#include "fy-accel.h" + +#include "xxhash.h" + +/* powers of two and the closest primes before + * + * pow2: 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 + * prime: 1 2 3 7 13 31 61 127 251 509 1021 2039 4093 8191 16381 32749 65521 + * + * pow2: 131072 262144 524288 + * prime: 130657 262051 524201 + */ + +/* 64K bucket should be enough for everybody */ +static const uint32_t prime_lt_pow2[] = { + 1, 2, 3, 7, 13, 31, 61, 127, 251, 509, 1021, + 2039, 4093, 8191, 16381, 32749, 65521, + 130657, 262051, 524201 +}; + +static inline unsigned int +fy_accel_hash_to_pos(struct fy_accel *xl, const void *hash, unsigned int nbuckets) +{ + uint64_t pos; + + switch (xl->hd->size) { + case 1: + pos = *(const uint8_t *)hash; + break; + case 2: + assert(!((uintptr_t)hash & 1)); + pos = *(const uint16_t *)hash; + break; + case 4: + assert(!((uintptr_t)hash & 3)); + pos = *(const uint32_t *)hash; + break; + case 8: + assert(!((uintptr_t)hash & 7)); + pos = *(const uint64_t *)hash; + break; + default: + /* sigh, what ever */ + pos = XXH32(hash, xl->hd->size, 0); + break; + } + + return (unsigned int)(pos % nbuckets); +} + +static inline bool +fy_accel_hash_eq(struct fy_accel *xl, const void *hash1, const void *hash2) +{ + switch (xl->hd->size) { + case 1: + return *(const uint8_t *)hash1 == *(const uint8_t *)hash2; + case 2: + assert(!((uintptr_t)hash1 & 1)); + assert(!((uintptr_t)hash2 & 1)); + return *(const uint16_t *)hash1 == *(const uint16_t *)hash2; + case 4: + assert(!((uintptr_t)hash1 & 3)); + assert(!((uintptr_t)hash2 & 3)); + return *(const uint32_t *)hash1 == *(const uint32_t *)hash2; + case 8: + assert(!((uintptr_t)hash1 & 7)); + assert(!((uintptr_t)hash2 & 7)); + return *(const uint64_t *)hash1 == *(const uint64_t *)hash2; + default: + break; + } + + return !memcmp(hash1, hash2, xl->hd->size); +} + +int fy_accel_resize(struct fy_accel *xl, unsigned int min_buckets) +{ + unsigned int next_pow2, exp, i, nbuckets, pos; + struct fy_accel_entry_list *xlel; + struct fy_accel_entry *xle; + struct fy_accel_entry_list *buckets_new; + + /* get the next power of two larger or equal */ + next_pow2 = 1; + exp = 0; + while (next_pow2 < min_buckets && + exp < sizeof(prime_lt_pow2)/sizeof(prime_lt_pow2[0])) { + next_pow2 <<= 1; + exp++; + } + + nbuckets = prime_lt_pow2[exp]; + if (nbuckets == xl->nbuckets) + return 0; + + buckets_new = malloc(sizeof(*buckets_new) * nbuckets); + if (!buckets_new) + return -1; + + for (i = 0, xlel = buckets_new; i < nbuckets; i++, xlel++) + fy_accel_entry_list_init(xlel); + + if (xl->buckets) { + for (i = 0, xlel = xl->buckets; i < xl->nbuckets; i++, xlel++) { + while ((xle = fy_accel_entry_list_pop(xlel)) != NULL) { + pos = fy_accel_hash_to_pos(xl, xle->hash, nbuckets); + fy_accel_entry_list_add_tail(&buckets_new[pos], xle); + } + } + free(xl->buckets); + } + xl->buckets = buckets_new; + xl->nbuckets = nbuckets; + xl->next_exp2 = exp; + + return 0; +} + +int fy_accel_grow(struct fy_accel *xl) +{ + if (!xl) + return -1; + + /* should not grow indefinetely */ + if (xl->next_exp2 >= sizeof(prime_lt_pow2)/sizeof(prime_lt_pow2[0])) + return -1; + + return fy_accel_resize(xl, prime_lt_pow2[xl->next_exp2 + 1]); +} + +int fy_accel_shrink(struct fy_accel *xl) +{ + if (!xl) + return -1; + + /* should not shrink indefinetely */ + if (xl->next_exp2 <= 0) + return -1; + + return fy_accel_resize(xl, prime_lt_pow2[xl->next_exp2 - 1]); +} + +int +fy_accel_setup(struct fy_accel *xl, + const struct fy_hash_desc *hd, + void *userdata, + unsigned int min_buckets) +{ + if (!xl || !hd || !hd->size || !hd->hash) + return -1; + + memset(xl, 0, sizeof(*xl)); + xl->hd = hd; + xl->userdata = userdata; + xl->count = 0; + + return fy_accel_resize(xl, min_buckets); +} + +void fy_accel_cleanup(struct fy_accel *xl) +{ + unsigned int i; + struct fy_accel_entry_list *xlel; + struct fy_accel_entry *xle; + + if (!xl) + return; + + for (i = 0, xlel = xl->buckets; i < xl->nbuckets; i++, xlel++) { + while ((xle = fy_accel_entry_list_pop(xlel)) != NULL) { + free(xle); + assert(xl->count > 0); + xl->count--; + } + } + + free(xl->buckets); +} + +struct fy_accel_entry * +fy_accel_entry_insert(struct fy_accel *xl, const void *key, const void *value) +{ + struct fy_accel_entry *xle, *xlet; + struct fy_accel_entry_list *xlel; + unsigned int pos, bucket_size; + int rc; + + if (!xl) + return NULL; + + xle = malloc(sizeof(*xle) + xl->hd->size); + if (!xle) + goto err_out; + + rc = xl->hd->hash(xl, key, xl->userdata, xle->hash); + if (rc) + goto err_out; + xle->key = key; + xle->value = value; + + pos = fy_accel_hash_to_pos(xl, xle->hash, xl->nbuckets); + xlel = &xl->buckets[pos]; + + fy_accel_entry_list_add_tail(xlel, xle); + + assert(xl->count < UINT_MAX); + xl->count++; + + /* if we don't auto-resize, return */ + if (xl->hd->max_bucket_grow_limit) { + bucket_size = 0; + for (xlet = fy_accel_entry_list_first(xlel); xlet; xlet = fy_accel_entry_next(xlel, xlet)) { + bucket_size++; + if (bucket_size >= xl->hd->max_bucket_grow_limit) + break; + } + + /* we don't really care whether the grow up succeeds or not */ + if (bucket_size >= xl->hd->max_bucket_grow_limit) + (void)fy_accel_grow(xl); + } + + return xle; +err_out: + if (xle) + free(xle); + return NULL; +} + +struct fy_accel_entry * +fy_accel_entry_lookup(struct fy_accel *xl, const void *key) +{ + struct fy_accel_entry_iter xli; + struct fy_accel_entry *xle; + + xle = fy_accel_entry_iter_start(&xli, xl, key); + if (xle) + fy_accel_entry_iter_finish(&xli); + + return xle; +} + +struct fy_accel_entry * +fy_accel_entry_lookup_key_value(struct fy_accel *xl, const void *key, const void *value) +{ + struct fy_accel_entry_iter xli; + struct fy_accel_entry *xle, *xle_start; + + xle_start = fy_accel_entry_iter_start(&xli, xl, key); + if (!xle_start) + return NULL; + + for (xle = xle_start; xle; xle = fy_accel_entry_iter_next(&xli)) { + + if (xle->value == value) + break; + } + fy_accel_entry_iter_finish(&xli); + + return xle; +} + +void +fy_accel_entry_remove(struct fy_accel *xl, struct fy_accel_entry *xle) +{ + unsigned int pos; + + if (!xl || !xle) + return; + + pos = fy_accel_hash_to_pos(xl, xle->hash, xl->nbuckets); + + fy_accel_entry_list_del(&xl->buckets[pos], xle); + + assert(xl->count > 0); + xl->count--; + + free(xle); +} + +int +fy_accel_insert(struct fy_accel *xl, const void *key, const void *value) +{ + struct fy_accel_entry *xle; + + xle = fy_accel_entry_lookup(xl, key); + if (xle) + return -1; /* exists */ + + xle = fy_accel_entry_insert(xl, key, value); + if (!xle) + return -1; /* failure to insert */ + + return 0; +} + +const void * +fy_accel_lookup(struct fy_accel *xl, const void *key) +{ + struct fy_accel_entry *xle; + + xle = fy_accel_entry_lookup(xl, key); + return xle ? xle->value : NULL; +} + +int +fy_accel_remove(struct fy_accel *xl, const void *data) +{ + struct fy_accel_entry *xle; + + xle = fy_accel_entry_lookup(xl, data); + if (!xle) + return -1; + + fy_accel_entry_remove(xl, xle); + + return 0; +} + +struct fy_accel_entry * +fy_accel_entry_iter_next_internal(struct fy_accel_entry_iter *xli) +{ + struct fy_accel *xl; + struct fy_accel_entry *xle; + struct fy_accel_entry_list *xlel; + const void *key; + void *hash; + + if (!xli) + return NULL; + + xl = xli->xl; + hash = xli->hash; + xlel = xli->xlel; + if (!xl || !hash || !xlel) + return NULL; + key = xli->key; + + xle = !xli->xle ? fy_accel_entry_list_first(xlel) : + fy_accel_entry_next(xlel, xli->xle); + for (; xle; xle = fy_accel_entry_next(xlel, xle)) { + if (fy_accel_hash_eq(xl, hash, xle->hash) && + xl->hd->eq(xl, hash, xle->key, key, xl->userdata)) + break; + } + return xli->xle = xle; +} + +struct fy_accel_entry * +fy_accel_entry_iter_start(struct fy_accel_entry_iter *xli, struct fy_accel *xl, const void *key) +{ + unsigned int pos; + int rc; + + if (!xli || !xl) + return NULL; + xli->xl = xl; + xli->key = key; + if (xl->hd->size <= sizeof(xli->hash_inline)) + xli->hash = xli->hash_inline; + else + xli->hash = malloc(xl->hd->size); + xli->xlel = NULL; + + if (!xli->hash) + goto err_out; + + rc = xl->hd->hash(xl, key, xl->userdata, xli->hash); + if (rc) + goto err_out; + + pos = fy_accel_hash_to_pos(xl, xli->hash, xl->nbuckets); + xli->xlel = &xl->buckets[pos]; + + xli->xle = NULL; + + return fy_accel_entry_iter_next_internal(xli); + +err_out: + fy_accel_entry_iter_finish(xli); + return NULL; +} + +void fy_accel_entry_iter_finish(struct fy_accel_entry_iter *xli) +{ + if (!xli) + return; + + if (xli->hash && xli->hash != xli->hash_inline) + free(xli->hash); +} + +struct fy_accel_entry * +fy_accel_entry_iter_next(struct fy_accel_entry_iter *xli) +{ + if (!xli || !xli->xle) + return NULL; + + return fy_accel_entry_iter_next_internal(xli); +} diff --git a/src/lib/fy-accel.h b/src/lib/fy-accel.h new file mode 100644 index 0000000..1531e8c --- /dev/null +++ b/src/lib/fy-accel.h @@ -0,0 +1,92 @@ +/* + * fy-accel.h - YAML accelerated access methods + * + * Copyright (c) 2020 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ACCEL_H +#define FY_ACCEL_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include + +#include "fy-list.h" +#include "fy-typelist.h" + +struct fy_accel_entry { + struct list_head node; + const void *key; + const void *value; + uint8_t hash[0]; +}; +FY_TYPE_FWD_DECL_LIST(accel_entry); +FY_TYPE_DECL_LIST(accel_entry); + +struct fy_accel; + +struct fy_hash_desc { + unsigned int size; + unsigned int max_bucket_grow_limit; + bool unique; + int (*hash)(struct fy_accel *xl, const void *key, void *userdata, void *hash); + bool (*eq)(struct fy_accel *xl, const void *hash, const void *key1, const void *key2, void *userdata); +}; + +struct fy_accel { + const struct fy_hash_desc *hd; + void *userdata; + unsigned int count; + unsigned int nbuckets; + unsigned int next_exp2; + struct fy_accel_entry_list *buckets; +}; + +int +fy_accel_setup(struct fy_accel *xl, + const struct fy_hash_desc *hd, + void *userdata, + unsigned int min_buckets); + +void fy_accel_cleanup(struct fy_accel *xl); + +int fy_accel_resize(struct fy_accel *xl, unsigned int min_buckets); +int fy_accel_grow(struct fy_accel *xl); +int fy_accel_shrink(struct fy_accel *xl); + +int fy_accel_insert(struct fy_accel *xl, const void *key, const void *value); +const void *fy_accel_lookup(struct fy_accel *xl, const void *key); +int fy_accel_remove(struct fy_accel *xl, const void *key); + +struct fy_accel_entry_iter { + struct fy_accel *xl; + const void *key; + void *hash; + struct fy_accel_entry_list *xlel; + struct fy_accel_entry *xle; + uint64_t hash_inline[4]; /* to avoid allocation */ +}; + +struct fy_accel_entry * +fy_accel_entry_insert(struct fy_accel *xl, const void *key, const void *value); + +struct fy_accel_entry * +fy_accel_entry_lookup(struct fy_accel *xl, const void *key); +struct fy_accel_entry * +fy_accel_entry_lookup_key_value(struct fy_accel *xl, const void *key, const void *value); + +void fy_accel_entry_remove(struct fy_accel *xl, struct fy_accel_entry *xle); + +struct fy_accel_entry * +fy_accel_entry_iter_start(struct fy_accel_entry_iter *xli, + struct fy_accel *xl, const void *key); +void fy_accel_entry_iter_finish(struct fy_accel_entry_iter *xli); +struct fy_accel_entry * +fy_accel_entry_iter_next(struct fy_accel_entry_iter *xli); + +#endif diff --git a/src/lib/fy-atom.c b/src/lib/fy-atom.c new file mode 100644 index 0000000..4d554fe --- /dev/null +++ b/src/lib/fy-atom.c @@ -0,0 +1,1897 @@ +/* + * fy-atom.c - YAML atom methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" +#include "fy-doc.h" +#include "fy-input.h" + +#include "fy-atom.h" + +const char *fy_atom_data(const struct fy_atom *atom) +{ + if (!atom) + return NULL; + + return fy_input_start(atom->fyi) + atom->start_mark.input_pos; +} + + +struct fy_atom *fy_reader_fill_atom(struct fy_reader *fyr, int advance, struct fy_atom *handle) +{ + /* start mark */ + fy_reader_fill_atom_start(fyr, handle); + + /* advance the given number of characters */ + if (advance > 0) + fy_reader_advance_by(fyr, advance); + + fy_reader_fill_atom_end(fyr, handle); + + return handle; +} + +int fy_reader_advance_mark(struct fy_reader *fyr, int advance, struct fy_mark *m) +{ + int i, c, tabsize; + bool is_line_break; + + tabsize = fy_reader_tabsize(fyr); + i = 0; + while (advance-- > 0) { + c = fy_reader_peek_at(fyr, i++); + if (c == -1) + return -1; + m->input_pos += fy_utf8_width(c); + + /* first check for CR/LF */ + if (c == '\r' && fy_reader_peek_at(fyr, i) == '\n') { + m->input_pos++; + i++; + is_line_break = true; + } else if (fy_reader_is_lb(fyr, c)) + is_line_break = true; + else + is_line_break = false; + + if (is_line_break) { + m->column = 0; + m->line++; + } else if (tabsize > 0 && fy_is_tab(c)) + m->column += (tabsize - (fy_reader_column(fyr) % tabsize)); + else + m->column++; + } + + return 0; +} + +struct fy_atom *fy_reader_fill_atom_mark(struct fy_reader *fyr, + const struct fy_mark *start_mark, + const struct fy_mark *end_mark, + struct fy_atom *handle) +{ + if (!fyr || !start_mark || !end_mark || !handle) + return NULL; + + memset(handle, 0, sizeof(*handle)); + + handle->start_mark = *start_mark; + handle->end_mark = *end_mark; + handle->fyi = fy_reader_current_input(fyr); + handle->fyi_generation = fy_reader_current_input_generation(fyr); + + /* default is plain, modify at return */ + handle->style = FYAS_PLAIN; + handle->chomp = FYAC_CLIP; + /* by default we don't do storage hints, it's the job of the caller */ + handle->storage_hint = 0; + handle->storage_hint_valid = false; + + return handle; +} + +struct fy_atom *fy_reader_fill_atom_at(struct fy_reader *fyr, int advance, int count, struct fy_atom *handle) +{ + struct fy_mark start_mark, end_mark; + int rc; + + if (!fyr || !handle || !fyr->current_input) + return NULL; + + /* start mark */ + fy_reader_get_mark(fyr, &start_mark); + rc = fy_reader_advance_mark(fyr, advance, &start_mark); + (void)rc; + /* ignore the return, if the advance failed, it's the end of input */ + + /* end mark */ + end_mark = start_mark; + rc = fy_reader_advance_mark(fyr, count, &end_mark); + (void)rc; + /* ignore the return, if the advance failed, it's the end of input */ + + return fy_reader_fill_atom_mark(fyr, &start_mark, &end_mark, handle); +} + +static inline void +fy_atom_iter_chunk_reset(struct fy_atom_iter *iter) +{ + iter->top = 0; + iter->read = 0; +} + +static int +fy_atom_iter_grow_chunk(struct fy_atom_iter *iter) +{ + struct fy_atom_iter_chunk *chunks, *c; + size_t asz; + const char *old_s, *old_e, *ss; + unsigned int i; + size_t offset; + + old_s = (const char *)iter->chunks; + old_e = (const char *)(iter->chunks + iter->alloc); + + asz = sizeof(*chunks) * iter->alloc * 2; + chunks = realloc(iter->chunks == iter->startup_chunks ? NULL : iter->chunks, asz); + if (!chunks) /* out of memory */ + return -1; + if (iter->chunks == iter->startup_chunks) + memcpy(chunks, iter->startup_chunks, sizeof(iter->startup_chunks)); + + /* for chunks that point to the inplace buffer, reassign pointers */ + for (ss = old_s, c = chunks, i = 0; i < iter->top; ss += sizeof(*c), c++, i++) { + if (c->ic.str < old_s || c->ic.str >= old_e || + c->ic.len > sizeof(c->inplace_buf)) + continue; + + /* get offset */ + offset = (size_t)(c->ic.str - ss); + + /* verify that it points to the inplace_buf area */ + assert(offset >= offsetof(struct fy_atom_iter_chunk, inplace_buf)); + offset -= offsetof(struct fy_atom_iter_chunk, inplace_buf); + + c->ic.str = c->inplace_buf + offset; + } + + iter->alloc *= 2; + iter->chunks = chunks; + + return 0; +} + +static int +_fy_atom_iter_add_chunk(struct fy_atom_iter *iter, const char *str, size_t len) +{ + struct fy_atom_iter_chunk *c; + int ret; + + if (!len) + return 0; + + /* grow iter chunks? */ + if (iter->top >= iter->alloc) { + ret = fy_atom_iter_grow_chunk(iter); + if (ret) + return ret; + } + assert(iter->top < iter->alloc); + c = &iter->chunks[iter->top++]; + + c->ic.str = str; + c->ic.len = len; + + return 0; +} + +static int +_fy_atom_iter_add_chunk_copy(struct fy_atom_iter *iter, const char *str, size_t len) +{ + struct fy_atom_iter_chunk *c; + int ret; + + if (!len) + return 0; + + assert(len <= sizeof(c->inplace_buf)); + + if (iter->top >= iter->alloc) { + ret = fy_atom_iter_grow_chunk(iter); + if (ret) + return ret; + } + assert(iter->top < iter->alloc); + c = &iter->chunks[iter->top++]; + + memcpy(c->inplace_buf, str, len); + + c->ic.str = c->inplace_buf; + c->ic.len = len; + + return 0; +} + +/* keep it around without a warning even though it's unused */ +static int +_fy_atom_iter_add_utf8(struct fy_atom_iter *iter, int c) + __attribute__((__unused__)); + +static int +_fy_atom_iter_add_utf8(struct fy_atom_iter *iter, int c) +{ + char buf[FY_UTF8_FORMAT_BUFMIN]; + char *e; + + /* only fails if invalid utf8 */ + e = fy_utf8_put(buf, sizeof(buf), c); + if (!e) + return -1; + + return _fy_atom_iter_add_chunk_copy(iter, buf, e - buf); +} + +/* optimized linebreaks */ +static int +_fy_atom_iter_add_lb(struct fy_atom_iter *iter, int c) +{ + switch (c) { + /* those are generic linebreaks */ + case '\r': + case '\n': + case 0x85: + return _fy_atom_iter_add_chunk(iter, "\n", 1); + /* these are specific linebreaks */ + case 0x2028: + return _fy_atom_iter_add_chunk(iter, "\xe2\x80\xa8", 3); + case 0x2029: + return _fy_atom_iter_add_chunk(iter, "\xe2\x80\xa9", 3); + } + /* not a linebreak */ + return -1; +} + +// #define DEBUG_CHUNK + +#ifndef DEBUG_CHUNK +#define fy_atom_iter_add_chunk _fy_atom_iter_add_chunk +#define fy_atom_iter_add_chunk_copy _fy_atom_iter_add_chunk_copy +#define fy_atom_iter_add_utf8 _fy_atom_iter_add_utf8 +#define fy_atom_iter_add_lb _fy_atom_iter_add_lb +#else +#define fy_atom_iter_add_chunk(_iter, _str, _len) \ + ({ \ + const char *__str = (_str); \ + size_t __len2 = (_len); \ + char *__out = NULL; \ + int __ret = 0; \ + \ + if (__len2 > 0) { \ + __out = fy_utf8_format_text_alloc(__str, __len2, fyue_doublequote); \ + assert(__out); \ + fprintf(stderr, "%s:%d chunk #%zu \"%s\"\n", __func__, __LINE__, __len2, __out); \ + __ret = _fy_atom_iter_add_chunk((_iter), __str, __len2); \ + free(__out); \ + } \ + __ret; \ + }) + +#define fy_atom_iter_add_chunk_copy(_iter, _str, _len) \ + ({ \ + const char *__str = (_str); \ + size_t __len2 = (_len); \ + char *__out = NULL; \ + int __ret = 0; \ + \ + if (__len2 > 0) { \ + __out = fy_utf8_format_text_alloc(__str, __len2, fyue_doublequote); \ + assert(__out); \ + fprintf(stderr, "%s:%d chunk-copy #%zu \"%s\"\n", __func__, __LINE__, __len2, __out); \ + /* fprintf(stderr, "%s:%d chunk-copy \"%.*s\"\n", __func__, __LINE__, (int)__len, __str); */ \ + __ret = _fy_atom_iter_add_chunk_copy((_iter), __str, __len2); \ + free(__out); \ + } \ + __ret; \ + }) +#define fy_atom_iter_add_utf8(_iter, _c) \ + ({ \ + int __c = (_c); \ + fprintf(stderr, "%s:%d utf8 %d\n", __func__, __LINE__, __c); \ + _fy_atom_iter_add_utf8((_iter), (_c)); \ + }) +#define fy_atom_iter_add_lb(_iter, _c) \ + ({ \ + int __c = (_c); \ + fprintf(stderr, "%s:%d lb 0x%02x\n", __func__, __LINE__, __c); \ + _fy_atom_iter_add_lb((_iter), (_c)); \ + }) +#endif + +static void +fy_atom_iter_line_analyze(struct fy_atom_iter *iter, struct fy_atom_iter_line_info *li, + const char *line_start, size_t len) +{ + const struct fy_atom *atom = iter->atom; + const char *s, *e, *ss; + int col, c, w, ts, cws, advws; + bool last_was_ws, is_block; + int lastc; + + s = line_start; + e = line_start + len; + + is_block = atom->style == FYAS_LITERAL || atom->style == FYAS_FOLDED; + + /* short circuit non multiline, non ws atoms */ + if ((atom->direct_output && !atom->has_lb && !atom->has_ws) || + atom->style == FYAS_DOUBLE_QUOTED_MANUAL) { + li->start = s; + li->end = e; + li->nws_start = s; + li->nws_end = e; + li->chomp_start = s; + li->final = true; + li->empty = atom->empty; + li->trailing_breaks = 0; + li->trailing_breaks_ws = false; + li->start_ws = 0; + li->end_ws = 0; + li->indented = false; + li->lb_end = is_block ? atom->ends_with_lb : false; + li->final = true; + li->actual_lb = -1; + li->s_tb = li->e_tb = NULL; + li->ends_with_backslash = false; + return; + } + + li->start = s; + li->end = NULL; + li->nws_start = NULL; + li->nws_end = NULL; + li->chomp_start = NULL; + li->empty = true; + li->trailing_breaks = 0; + li->trailing_breaks_ws = false; + li->first = false; + li->start_ws = (size_t)-1; + li->end_ws = (size_t)-1; + li->indented = false; + li->lb_end = false; + li->final = false; + li->actual_lb = -1; + li->ends_with_backslash = false; + + last_was_ws = false; + + ts = atom->tabsize ? : 8; /* pick it up from the atom (if there is) */ + + /* consecutive whitespace */ + cws = 0; + + lastc = -1; + li->s_tb = s; + for (col = 0, ss = s; (c = fy_utf8_get(ss, (e - ss), &w)) >= 0; ss += w) { + + lastc = c; + + /* mark start of chomp */ + if (is_block && !li->chomp_start && (unsigned int)col >= iter->chomp) { + li->chomp_start = ss; + + /* if the character at the chomp point is whitespace + * then we're indented + */ + li->indented = fy_is_ws(c); + +#if defined(DEBUG_CHUNK) + fprintf(stderr, "%s:%d chomp_start=%d\n", __FILE__, __LINE__, (int)(li->chomp_start - li->start)); +#endif + + } + + if (fy_is_lb_m(c, atom->lb_mode)) { + +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d lb=0x%x\n", __FILE__, __LINE__, c); +#endif + + col = 0; + if (!li->end) { + + li->end = ss; + li->end_ws = cws; + li->lb_end = true; + li->actual_lb = c; + +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d set actual_lb=0x%x\n", __FILE__, __LINE__, li->actual_lb); +#endif + + cws = 0; + } + + /* no chomp point hit, use whatever we have here */ + if (is_block && !li->chomp_start) { + li->chomp_start = ss; +#if defined(DEBUG_CHUNK) + fprintf(stderr, "%s:%d chomp_start=%d\n", __FILE__, __LINE__, (int)(li->chomp_start - li->start)); +#endif + + } + + if (!last_was_ws) { + cws = 0; + li->nws_end = ss; + last_was_ws = true; + } + + } else if (fy_is_space(c)) { + + col++; + cws++; + + if (!last_was_ws) { + li->nws_end = ss; + last_was_ws = true; + } + + } else if (fy_is_tab(c)) { + + bool can_be_nws_end; + + advws = ts - (col % ts); + col += advws; + +#if defined(DEBUG_CHUNK) + fprintf(stderr, "%s:%d tab col=%d chomp=%d\n", __FILE__, __LINE__, col, (int)(li->chomp_start - li->start)); +#endif + + if (fy_atom_style_is_block(atom->style) && col >= (int)(li->chomp_start - li->start)) { +#if defined(DEBUG_CHUNK) + fprintf(stderr, "%s:%d tab col=%d chomp=%d\n", __FILE__, __LINE__, col, (int)(li->chomp_start - li->start)); +#endif + goto do_nws; + } + + cws += advws; + + can_be_nws_end = true; + if (atom->style == FYAS_DOUBLE_QUOTED && ss > li->start && ss[-1] == '\\') { + can_be_nws_end = false; +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d backslashed tab\n", __FILE__, __LINE__); +#endif + } + + if (can_be_nws_end && !last_was_ws) { + li->nws_end = ss; + last_was_ws = true; + } + + } else { + + col++; +do_nws: + /* mark start of non whitespace */ + if (!li->nws_start) + li->nws_start = ss; + + if (li->empty) + li->empty = false; + + if (li->start_ws == (size_t)-1) + li->start_ws = cws; + + last_was_ws = false; + cws = 0; + } + + /* if we got both break */ + if (li->end && iter->chomp >= 0) + break; + } + li->e_tb = ss; + + li->final = c < 0; + + if (li->final && atom->ends_with_eof) { + + /* mark start of chomp */ + if (is_block && !li->chomp_start && (unsigned int)col >= iter->chomp) { + li->chomp_start = ss; + + /* if the character at the chomp point is whitespace + * then we're indented + */ + li->indented = fy_is_ws(lastc); +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d is_block && !li->chomp_start && (unsigned int)col >= iter->chomp\n", __FILE__, __LINE__); +#endif +#if defined(DEBUG_CHUNK) + fprintf(stderr, "%s:%d chomp_start=%d\n", __FILE__, __LINE__, (int)(li->chomp_start - li->start)); +#endif + + } + + if (!li->end) { + li->end = ss; + li->end_ws = cws; + li->lb_end = true; + +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d li->final && atom->ends_with_eof && !li->end\n", __FILE__, __LINE__); +#endif + cws = 0; + } + + /* no chomp point hit, use whatever we have here */ + if (is_block && !li->chomp_start) { + li->chomp_start = ss; +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d li->final && atom->ends_with_eof && !li->chomp_start\n", __FILE__, __LINE__); +#endif +#if defined(DEBUG_CHUNK) + fprintf(stderr, "%s:%d chomp_start=%d\n", __FILE__, __LINE__, (int)(li->chomp_start - li->start)); +#endif + } + + if (!last_was_ws) { + cws = 0; + li->nws_end = ss; + last_was_ws = true; +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d li->final && atom->ends_with_eof && !last_was_ws\n", __FILE__, __LINE__); +#endif + } + } + + if (!last_was_ws) + li->nws_end = ss; + + if (!li->nws_start) + li->nws_start = ss; + + if (!li->nws_end) + li->nws_end = ss; + + /* if we haven't hit the chomp, point use whatever we're now */ + if (is_block && !li->chomp_start) { + li->chomp_start = ss; +#if defined(DEBUG_CHUNK) + fprintf(stderr, "%s:%d chomp_start=%d\n", __FILE__, __LINE__, (int)(li->chomp_start - li->start)); +#endif + } + + if (li->start_ws == (size_t)-1) + li->start_ws = 0; + + if (li->end_ws == (size_t)-1) + li->end_ws = 0; + + /* mark next line to the end if no linebreak found */ + if (!li->end) { + li->end = iter->e; + li->last = true; + li->end_ws = cws; + li->lb_end = false; + goto out; + } + + /* find out if any trailing breaks exist afterwards */ + for (; (c = fy_utf8_get(ss, (e - ss), &w)) >= 0 && (fy_is_ws(c) || fy_is_lb_m(c, atom->lb_mode)); ss += w) { + + if (!li->trailing_breaks_ws && is_block && (unsigned int)col > iter->chomp) + li->trailing_breaks_ws = true; + + if (fy_is_lb_m(c, atom->lb_mode)) { + li->trailing_breaks++; + col = 0; + } else { + /* indented whitespace counts as break */ + if (fy_is_tab(c)) + col += (ts - (col % ts)); + else + col++; + } + + } + + /* and mark as last if only whitespace and breaks after this point */ + li->last = ss >= e; + +out: + assert(li->start); + assert(li->end); + assert(li->nws_start); + assert(li->nws_end); + assert(!is_block || li->chomp_start); + + li->ends_with_backslash = atom->style == FYAS_DOUBLE_QUOTED && + !li->empty && + (li->nws_end > li->nws_start && li->nws_end[-1] == '\\') && + ((li->nws_end - li->nws_start) <= 1 || li->nws_end[-2] != '\\'); +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d ends_with_backslash=%s\n", __FILE__, __LINE__, li->ends_with_backslash ? "true" : "false"); +#endif +} + +void fy_atom_iter_start(const struct fy_atom *atom, struct fy_atom_iter *iter) +{ + struct fy_atom_iter_line_info *li; + size_t len; + + if (!atom || !iter) + return; + + memset(iter, 0, sizeof(*iter)); + + iter->atom = atom; + iter->s = fy_atom_data(atom); + len = fy_atom_size(atom); + iter->e = iter->s + len; + + iter->chomp = atom->increment; + + /* default tab size is 8 */ + iter->tabsize = atom->tabsize ? : 8; + + memset(iter->li, 0, sizeof(iter->li)); + li = &iter->li[1]; + fy_atom_iter_line_analyze(iter, li, iter->s, len); + li->first = true; + + /* if there's single quote at the start of a line ending the atom */ + iter->dangling_end_quote = atom->end_mark.column == 0; + iter->single_line = atom->start_mark.line == atom->end_mark.line; + iter->empty = atom->empty; + iter->last_ends_with_backslash = li->ends_with_backslash; +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d single_line=%s empty=%s last_ends_with_backslash=%s\n", __FILE__, __LINE__, + iter->single_line ? "true" : "false", + iter->empty ? "true" : "false", + iter->last_ends_with_backslash ? "true" : "false"); +#endif + + /* current is 0, next is 1 */ + iter->current = 0; + + iter->alloc = sizeof(iter->startup_chunks)/sizeof(iter->startup_chunks[0]); + iter->top = 0; + iter->read = 0; + iter->chunks = iter->startup_chunks; + + iter->done = false; + + iter->unget_c = -1; +} + +void fy_atom_iter_finish(struct fy_atom_iter *iter) +{ + if (iter->chunks && iter->chunks != iter->startup_chunks) + free(iter->chunks); + + iter->chunks = NULL; +} + +static const struct fy_atom_iter_line_info * +fy_atom_iter_line(struct fy_atom_iter *iter) +{ + const struct fy_atom *atom = iter->atom; + struct fy_atom_iter_line_info *li, *nli; + const char *ss; + + /* return while there's a next line */ + if (!iter) + return NULL; + + /* make next line the current one */ + iter->current = !iter->current; + + li = &iter->li[iter->current]; + + /* if we're out, we're out */ + if (li->start >= iter->e) + return NULL; + + /* scan next line (special handling for '\r\n') */ + ss = li->end; + if (ss < iter->e) { + if (*ss == '\r' && (ss + 1) < iter->e && ss[1] == '\n') + ss += 2; + else + ss += fy_utf8_width_by_first_octet((uint8_t)*ss); + } + + /* get current and next line */ + fy_atom_iter_line_analyze(iter, &iter->li[!iter->current], ss, iter->e - ss); + + /* if no more, mark the next as NULL */ + nli = &iter->li[!iter->current]; + if (nli->start >= iter->e) + nli = NULL; + + /* for quoted, output the white space start */ + if (atom->style == FYAS_SINGLE_QUOTED || atom->style == FYAS_DOUBLE_QUOTED) { + li->s = li->first ? li->start : li->nws_start; + li->e = li->last ? li->end : li->nws_end; + + /* just empty */ + if (li->empty && li->first && li->last && !iter->single_line) + li->s = li->e; + + } else if (atom->style == FYAS_LITERAL || atom->style == FYAS_FOLDED) { + li->s = li->chomp_start; + li->e = li->end; + if (li->empty && li->first && li->last) + li->s = li->e; + } else { + li->s = li->nws_start; + li->e = li->nws_end; + } + + /* bah, I hate this, */ + if (li->s > li->e) + li->s = li->e; + + assert(li->s <= li->e); + + /* we never fold LS or PS linebreaks (on yaml 1.1) */ + li->need_nl = fy_is_lb_LS_PS(li->actual_lb) && fy_is_lb_m(li->actual_lb, iter->atom->lb_mode) && !li->ends_with_backslash; + li->need_sep = false; + +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d need_nl=%s\n", __FILE__, __LINE__, li->need_nl ? "true" : "false"); +#endif + if (li->need_nl) + return li; + + switch (atom->style) { + case FYAS_PLAIN: + case FYAS_URI: + li->need_nl = !li->last && li->empty; + li->need_sep = !li->need_nl && nli && !nli->empty; + break; + + case FYAS_DOUBLE_QUOTED_MANUAL: + li->need_nl = false; + li->need_sep = false; + break; + + case FYAS_COMMENT: + li->need_nl = !li->final; + li->need_sep = false; + break; + + case FYAS_DOUBLE_QUOTED: +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d ends_with_backslash=%s\n", __FILE__, __LINE__, li->ends_with_backslash ? "true" : "false"); +#endif + if (li->ends_with_backslash) { + + li->need_nl = false; + li->need_sep = false; + break; + } + + /* fall-through */ + case FYAS_SINGLE_QUOTED: + li->need_nl = (!li->last && !li->first && li->empty) || + (nli && iter->empty && !li->first); + + if (li->need_nl) + break; + + li->need_sep = (nli && !nli->empty) || + (!nli && li->last && iter->dangling_end_quote) || + (nli && nli->final && nli->empty); + + break; + case FYAS_LITERAL: + li->need_nl = true; + break; + case FYAS_FOLDED: + li->need_nl = !li->last && (li->empty || li->indented || li->trailing_breaks_ws || (nli && nli->indented)); + if (li->need_nl) + break; + li->need_sep = nli && !nli->indented && !nli->empty; + break; + default: + break; + } + + return li; +} + +static int +fy_atom_iter_format(struct fy_atom_iter *iter) +{ + const struct fy_atom *atom = iter->atom; + const struct fy_atom_iter_line_info *li; + const char *s, *e, *t; + int value, code_length, rlen, ret; + uint8_t code[4], *tt; + int j, pending_nl; + int *pending_lb = NULL, *pending_lb_new = NULL; + int pending_lb_size = 0; + enum fy_utf8_escape esc_mode; + size_t i; + + /* done? */ + li = fy_atom_iter_line(iter); + if (!li) { + iter->done = true; + return 0; + } + if (iter->done) + return 0; + + s = li->s; + e = li->e; + + switch (atom->style) { + + case FYAS_LITERAL: + case FYAS_PLAIN: + case FYAS_FOLDED: + case FYAS_COMMENT: + if (s < e) { + ret = fy_atom_iter_add_chunk(iter, s, e - s); + if (ret) + goto out; + } + + break; + + case FYAS_SINGLE_QUOTED: + if (li->last) + e = li->nws_end; + while (s < e) { + /* find next single quote */ + t = memchr(s, '\'', e - s); + rlen = (t ? t : e) - s; + + ret = fy_atom_iter_add_chunk(iter, s, rlen); + if (ret) + goto out; + + /* end of string */ + if (!t) + break; + + s = t; + /* next character single quote too */ + if ((e - s) >= 2 && s[1] == '\'') + fy_atom_iter_add_chunk(iter, s, 1); + + /* skip over this single quote char */ + s++; + } + break; + + case FYAS_DOUBLE_QUOTED: + if (li->last) + e = li->nws_end; + + esc_mode = atom->json_mode ? fyue_doublequote_json : + atom->lb_mode == fylb_cr_nl ? fyue_doublequote : fyue_doublequote_yaml_1_1; + + while (s < e) { + /* find next escape */ + t = memchr(s, '\\', e - s); + /* copy up to there (or end) */ + rlen = (t ? t : e) - s; + ret = fy_atom_iter_add_chunk(iter, s, rlen); + if (ret) + goto out; + + if (!t || (e - t) < 2) + break; + + ret = fy_utf8_parse_escape(&t, e - t, esc_mode); + if (ret < 0) + goto out; + s = t; + value = ret; + + tt = fy_utf8_put(code, sizeof(code), value); + if (!tt) { + ret = -1; + goto out; + } + + ret = fy_atom_iter_add_chunk_copy(iter, (const char *)code, tt - code); + if (ret) + goto out; + } + break; + + case FYAS_URI: + while (s < e) { + /* find next escape */ + t = memchr(s, '%', e - s); + rlen = (t ? t : e) - s; + ret = fy_atom_iter_add_chunk(iter, s, rlen); + if (ret) + goto out; + + /* end of string */ + if (!t) + break; + s = t; + + code_length = sizeof(code); + t = fy_uri_esc(s, e - s, code, &code_length); + if (!t) { + ret = -1; + goto out; + } + + /* output escaped utf8 */ + ret = fy_atom_iter_add_chunk_copy(iter, (const char *)code, code_length); + if (ret) + goto out; + s = t; + } + break; + + case FYAS_DOUBLE_QUOTED_MANUAL: + /* manual scalar just goes out */ + ret = fy_atom_iter_add_chunk(iter, s, e - s); + if (ret) + goto out; + s = e; + break; + + default: + ret = -1; + goto out; + } + + if (li->last) { + + if (fy_atom_style_is_block(atom->style)) { + + switch (atom->chomp) { + case FYAC_STRIP: + case FYAC_CLIP: + + pending_lb_size = 16; + pending_lb = alloca(sizeof(*pending_lb) * pending_lb_size); + + pending_nl = 0; + if (!li->empty) { + pending_lb[0] = li->actual_lb > 0 ? li->actual_lb : '\n'; + pending_nl = 1; + } + while ((li = fy_atom_iter_line(iter)) != NULL) { + if (!iter->empty && li->chomp_start < li->end) { + for (j = 0; j < pending_nl; j++) { + ret = fy_atom_iter_add_lb(iter, pending_lb[j]); + if (ret) + goto out; + } + pending_nl = 0; + + ret = fy_atom_iter_add_chunk(iter, li->chomp_start, li->end - li->chomp_start); + if (ret) + goto out; + } + if (li->lb_end && !iter->empty) { + if (pending_nl >= pending_lb_size) { + pending_lb_new = alloca(sizeof(*pending_lb) * pending_lb_size * 2); + memcpy(pending_lb_new, pending_lb, sizeof(*pending_lb) * pending_lb_size); + pending_lb_size *= 2; + pending_lb = pending_lb_new; + } + pending_lb[pending_nl] = li->actual_lb > 0 ? li->actual_lb : '\n'; + pending_nl++; + } + } + + if (atom->chomp == FYAC_CLIP && (pending_nl || atom->ends_with_eof)) { + ret = fy_atom_iter_add_lb(iter, pending_lb[0]); + if (ret) + goto out; + } + break; + case FYAC_KEEP: + if (li->lb_end || atom->ends_with_eof) { + ret = fy_atom_iter_add_lb(iter, li->actual_lb > 0 ? li->actual_lb : '\n'); + if (ret) + goto out; + } + + /* nothing more if it's an EOF */ + if (atom->ends_with_eof && atom->empty) + break; + + while ((li = fy_atom_iter_line(iter)) != NULL) { + if (!iter->empty && li->chomp_start < li->end) { + ret = fy_atom_iter_add_chunk(iter, li->chomp_start, li->end - li->chomp_start); + if (ret) + goto out; + } + if (li->lb_end) { + ret = fy_atom_iter_add_lb(iter, li->actual_lb > 0 ? li->actual_lb : '\n'); + if (ret) + goto out; + } + } + break; + } + + iter->done = true; + + } else { + +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d trailing_breaks=%zu end_ws=%zu empty=%s\n", __func__, __LINE__, + li->trailing_breaks, li->end_ws, + li->empty ? "true" : "false"); +#endif + + if (li->trailing_breaks == 0 && li->end_ws > 0) { + /* end of quote in a non-blank line having white space */ + ret = fy_atom_iter_add_chunk(iter, li->nws_end, (size_t)(li->e - li->nws_end)); + if (ret) + goto out; + } else if (li->trailing_breaks == 1) { + + if (atom->style == FYAS_DOUBLE_QUOTED) { +#ifdef DEBUG_CHUNK + fprintf(stderr, "%s:%d %s <>%d single trailing break %c\n", __FILE__, __LINE__, __func__, + (int)(li->s_tb - li->s), li->s_tb[-1]); +#endif + } + + if (!li->ends_with_backslash) { + ret = fy_atom_iter_add_chunk(iter, " ", 1); + if (ret) + goto out; + } + } else if (li->trailing_breaks > 1) { + for (i = 0; i < li->trailing_breaks - 1; i++) { + ret = fy_atom_iter_add_lb(iter, '\n'); + if (ret) + goto out; + } + } + + iter->done = true; + } + + } else { + if (li->need_sep) { + ret = fy_atom_iter_add_chunk(iter, " ", 1); + if (ret) + goto out; + } + + if (li->need_nl) { + ret = fy_atom_iter_add_lb(iter, li->actual_lb > 0 ? li->actual_lb : '\n'); + if (ret) + goto out; + } + } + + /* got more */ + ret = 1; + +out: + return ret; +} + +const struct fy_iter_chunk * +fy_atom_iter_peek_chunk(struct fy_atom_iter *iter) +{ + if (iter->read >= iter->top) + return NULL; + + return &iter->chunks[iter->read].ic; +} + +void fy_atom_iter_advance(struct fy_atom_iter *iter, size_t len) +{ + struct fy_atom_iter_chunk *ac; + size_t rlen; + + /* while more and not out */ + while (len > 0 && iter->read < iter->top) { + + ac = iter->chunks + iter->read; + + /* get next run length */ + rlen = len > ac->ic.len ? ac->ic.len : len; + + /* remove from chunk */ + ac->ic.str += rlen; + ac->ic.len -= rlen; + + /* advance if out of data */ + if (ac->ic.len == 0) + iter->read++; + + /* remove run from length */ + len -= rlen; + } + + /* reset when everything is gone */ + if (iter->read >= iter->top) + fy_atom_iter_chunk_reset(iter); +} + +const struct fy_iter_chunk * +fy_atom_iter_chunk_next(struct fy_atom_iter *iter, const struct fy_iter_chunk *curr, int *errp) +{ + const struct fy_iter_chunk *ic; + int ret; + + ic = fy_atom_iter_peek_chunk(iter); + if (curr && curr == ic) + fy_atom_iter_advance(iter, ic->len); + + /* need to pull in data? */ + ic = fy_atom_iter_peek_chunk(iter); + if (!curr || !ic) { + fy_atom_iter_chunk_reset(iter); + + do { + ret = fy_atom_iter_format(iter); + + /* either end or error, means we don't have data */ + if (ret <= 0) { + if (errp) + *errp = ret < 0 ? -1 : 0; + return NULL; + } + + } while (!fy_atom_iter_peek_chunk(iter)); + } + ic = fy_atom_iter_peek_chunk(iter); + if (errp) + *errp = 0; + return ic; +} + +int fy_atom_format_text_length(struct fy_atom *atom) +{ + struct fy_atom_iter iter; + const struct fy_iter_chunk *ic; + size_t len; + int ret; + + if (!atom) + return -1; + + if (atom->storage_hint_valid) + return atom->storage_hint; + + len = 0; + fy_atom_iter_start(atom, &iter); + ic = NULL; + while ((ic = fy_atom_iter_chunk_next(&iter, ic, &ret)) != NULL) + len += ic->len; + fy_atom_iter_finish(&iter); + + /* something funky going on here */ + if ((int)len < 0) + return -1; + + if (ret != 0) + return ret; + + atom->storage_hint = (size_t)len; + atom->storage_hint_valid = true; + return (int)len; +} + +const char *fy_atom_format_text(struct fy_atom *atom, char *buf, size_t maxsz) +{ + struct fy_atom_iter iter; + const struct fy_iter_chunk *ic; + char *s, *e; + int ret; + + if (!atom || !buf) + return NULL; + + s = buf; + e = s + maxsz; + fy_atom_iter_start(atom, &iter); + ic = NULL; + while ((ic = fy_atom_iter_chunk_next(&iter, ic, &ret)) != NULL) { + /* must fit */ + if ((size_t)(e - s) < ic->len) + return NULL; + memcpy(s, ic->str, ic->len); + s += ic->len; + } + fy_atom_iter_finish(&iter); + + if (ret != 0 || s >= e) + return NULL; + *s = '\0'; + + return buf; +} + +int fy_atom_format_utf8_length(struct fy_atom *atom) +{ + struct fy_atom_iter iter; + const struct fy_iter_chunk *ic; + const char *s, *e; + size_t len; + int ret, rem, run, w; + + if (!atom) + return -1; + + len = 0; + rem = 0; + fy_atom_iter_start(atom, &iter); + ic = NULL; + while ((ic = fy_atom_iter_chunk_next(&iter, ic, &ret)) != NULL) { + s = ic->str; + e = s + ic->len; + + /* add the remainder */ + run = (e - s) > rem ? rem : (e - s); + s += run; + + /* count utf8 characters */ + while (s < e) { + w = fy_utf8_width_by_first_octet(*(uint8_t *)s); + + /* how many bytes of this run */ + run = (e - s) > w ? w : (e - s); + /* the remainder of this run */ + rem = w - run; + /* one more character */ + len++; + /* and advance */ + s += run; + } + } + fy_atom_iter_finish(&iter); + + /* something funky going on here */ + if ((int)len < 0) + return -1; + + if (ret != 0) + return ret; + + return (int)len; +} + + +struct fy_atom_iter * +fy_atom_iter_create(const struct fy_atom *atom) +{ + struct fy_atom_iter *iter; + + iter = malloc(sizeof(*iter)); + if (!iter) + return NULL; + if (atom) + fy_atom_iter_start(atom, iter); + else + memset(iter, 0, sizeof(*iter)); + return iter; +} + +void fy_atom_iter_destroy(struct fy_atom_iter *iter) +{ + if (!iter) + return; + + fy_atom_iter_finish(iter); + free(iter); +} + +ssize_t fy_atom_iter_read(struct fy_atom_iter *iter, void *buf, size_t count) +{ + ssize_t nread; + size_t nrun; + const struct fy_iter_chunk *ic; + int ret; + + if (!iter || !buf) + return -1; + + ret = 0; + nread = 0; + while (count > 0) { + ic = fy_atom_iter_peek_chunk(iter); + if (ic) { + nrun = count > ic->len ? ic->len : count; + memcpy(buf, ic->str, nrun); + nread += nrun; + count -= nrun; + fy_atom_iter_advance(iter, nrun); + continue; + } + + fy_atom_iter_chunk_reset(iter); + do { + ret = fy_atom_iter_format(iter); + + /* either end or error, means we don't have data */ + if (ret <= 0) + return ret == 0 ? nread : -1; + + } while (!fy_atom_iter_peek_chunk(iter)); + } + + return nread; +} + +int fy_atom_iter_getc(struct fy_atom_iter *iter) +{ + uint8_t buf; + ssize_t nread; + int c; + + if (!iter) + return -1; + + /* first try the pushed ungetc */ + if (iter->unget_c >= 0) { + c = iter->unget_c; + /* unmatched getc/ungetc */ + if (fy_utf8_width(c) != 1) + return -1; + iter->unget_c = -1; + return c; + } + + /* read first octet */ + nread = fy_atom_iter_read(iter, &buf, 1); + if (nread != 1) + return -1; + + return (int)buf & 0xff; +} + +int fy_atom_iter_ungetc(struct fy_atom_iter *iter, int c) +{ + if (!iter || c >= 0x80) + return -1; + + if (iter->unget_c >= 0) + return -1; + if (c < 0) { + iter->unget_c = -1; + return 0; + } + iter->unget_c = c; + return c; +} + +int fy_atom_iter_peekc(struct fy_atom_iter *iter) +{ + int c; + + c = fy_atom_iter_getc(iter); + if (c == -1) + return -1; + + return fy_atom_iter_ungetc(iter, c); +} + +int fy_atom_iter_utf8_get(struct fy_atom_iter *iter) +{ + uint8_t buf[4]; /* maximum utf8 is 4 octets */ + ssize_t nread; + int c, w; + + if (!iter) + return -1; + + /* first try the pushed ungetc */ + if (iter->unget_c >= 0) { + c = iter->unget_c; + iter->unget_c = -1; + return c; + } + + /* read first octet */ + nread = fy_atom_iter_read(iter, &buf[0], 1); + if (nread != 1) + return -1; + + /* get width from it (0 means illegal) */ + w = fy_utf8_width_by_first_octet(buf[0]); + if (!w) + return -1; + + /* read the rest octets (if possible) */ + if (w > 1) { + nread = fy_atom_iter_read(iter, buf + 1, w - 1); + if (nread != (w - 1)) + return -1; + } + + /* and return the decoded utf8 character */ + return fy_utf8_get(buf, w, &w); +} + +int fy_atom_iter_utf8_quoted_get(struct fy_atom_iter *iter, size_t *lenp, uint8_t *buf) +{ + ssize_t nread; + int c, w, ww; + + if (!iter || !lenp || !buf || *lenp < 4) + return -1; + + /* first try the pushed ungetc */ + if (iter->unget_c >= 0) { + c = iter->unget_c; + iter->unget_c = -1; + *lenp = 0; + return c; + } + + /* read first octet */ + nread = fy_atom_iter_read(iter, &buf[0], 1); + if (nread != 1) + return -1; + + /* get width from it (0 means illegal) - return it and mark it */ + w = fy_utf8_width_by_first_octet(buf[0]); + if (!w) { + *lenp = 1; + return 0; + } + + /* read the rest octets (if possible) */ + if (w > 1) { + nread = fy_atom_iter_read(iter, buf + 1, w - 1); + if (nread != (w - 1)) { + if (nread != -1 && nread < (w - 1)) + *lenp = nread; + return 0; + } + } + + /* and return the decoded utf8 character */ + c = fy_utf8_get(buf, w, &ww); + if (c >= 0) { + *lenp = 0; + return c; + } + *lenp = w; + return 0; +} + +int fy_atom_iter_utf8_unget(struct fy_atom_iter *iter, int c) +{ + if (!iter) + return -1; + + if (iter->unget_c >= 0) + return -1; + if (c < 0) { + iter->unget_c = -1; + return 0; + } + if (!fy_utf8_is_valid(c)) + return -1; + iter->unget_c = c; + return c; +} + +int fy_atom_iter_utf8_peek(struct fy_atom_iter *iter) +{ + int c; + + c = fy_atom_iter_utf8_get(iter); + if (c == -1) + return -1; + + return fy_atom_iter_utf8_unget(iter, c); +} + +int fy_atom_memcmp(struct fy_atom *atom, const void *ptr, size_t len) +{ + const char *dstr, *str; + size_t dlen, tlen; + struct fy_atom_iter iter; + int c, ct, ret; + + /* empty? just fine */ + if (!atom && !ptr && !len) + return 0; + + /* empty atom but not ptr */ + if (!atom && (ptr || len)) + return -1; + + /* non empty atom and empty ptr */ + if (atom && (!ptr || !len)) + return 1; + + /* direct output, nice */ + if (atom->direct_output) { + dlen = fy_atom_size(atom); + dstr = fy_atom_data(atom); + tlen = dlen > len ? len : dlen; + ret = memcmp(dstr, ptr, tlen); + if (ret) + return ret; + + return dlen == len ? 0 : len > dlen ? -1 : 1; + } + + str = ptr; + ct = -1; + fy_atom_iter_start(atom, &iter); + while ((c = fy_atom_iter_getc(&iter)) >= 0 && len) { + ct = *str & 0xff; + if (ct != c) + break; + str++; + len--; + } + fy_atom_iter_finish(&iter); + + /* out of data on both */ + if (c == -1 && !len) + return 0; + + return ct > c ? -1 : 1; +} + +int fy_atom_strcmp(struct fy_atom *atom, const char *str) +{ + size_t len; + + len = str ? strlen(str) : 0; + + return fy_atom_memcmp(atom, str, len); +} + +bool fy_atom_is_number(struct fy_atom *atom) +{ + struct fy_atom_iter iter; + int c, len, dec, fract, enot; + bool first_zero; + + /* empty? just fine */ + if (!atom || atom->size0) + return false; + + len = 0; + + fy_atom_iter_start(atom, &iter); + + /* skip minus sign if it's there */ + c = fy_atom_iter_utf8_peek(&iter); + if (c == '-') { + (void)fy_atom_iter_utf8_get(&iter); + len++; + } + + /* skip digits */ + first_zero = false; + dec = 0; + while ((c = fy_atom_iter_utf8_peek(&iter)) >= 0 && fy_is_digit(c)) { + if (dec == 0 && c == '0') + first_zero = true; + else if (dec == 1 && first_zero) + goto err_out; /* 0[0-9] is bad */ + (void)fy_atom_iter_utf8_get(&iter); + dec++; + len++; + } + + /* no digits is bad */ + if (!dec) + goto err_out; + + fract = 0; + /* dot? */ + c = fy_atom_iter_utf8_peek(&iter); + if (c == '.') { + + (void)fy_atom_iter_utf8_get(&iter); + len++; + /* skip decimal part */ + while ((c = fy_atom_iter_utf8_peek(&iter)) >= 0 && fy_is_digit(c)) { + (void)fy_atom_iter_utf8_get(&iter); + len++; + fract++; + } + + /* . without fractional */ + if (!fract) + goto err_out; + } + + enot = 0; + /* scientific notation */ + c = fy_atom_iter_utf8_peek(&iter); + if (c == 'e' || c == 'E') { + (void)fy_atom_iter_utf8_get(&iter); + len++; + + /* skip sign if it's there */ + c = fy_atom_iter_utf8_peek(&iter); + if (c == '+' || c == '-') { + (void)fy_atom_iter_utf8_get(&iter); + len++; + } + + /* skip exponent part */ + while ((c = fy_atom_iter_utf8_peek(&iter)) >= 0 && fy_is_digit(c)) { + (void)fy_atom_iter_utf8_get(&iter); + len++; + enot++; + } + + if (!enot) + goto err_out; + } + + c = fy_atom_iter_utf8_peek(&iter); + + fy_atom_iter_finish(&iter); + + /* everything must be consumed (and something must) */ + return c < 0 && len > 0; + +err_out: + fy_atom_iter_finish(&iter); + + return false; +} + +int fy_atom_cmp(struct fy_atom *atom1, struct fy_atom *atom2) +{ + struct fy_atom_iter iter1, iter2; + const char *d1, *d2; + size_t l1, l2, l; + int c1, c2, ret; + + /* handles NULL case too */ + if (atom1 == atom2) + return true; + + /* either null, can't do */ + if (!atom1 || !atom2) + return false; + + /* direct output? */ + if (atom1->direct_output) { + d1 = fy_atom_data(atom1); + l1 = fy_atom_size(atom1); + } else { + d1 = NULL; + l1 = 0; + } + if (atom2->direct_output) { + d2 = fy_atom_data(atom2); + l2 = fy_atom_size(atom2); + } else { + d2 = NULL; + l2 = 0; + } + + /* we have both atoms with direct output */ + if (d1 && d2) { + l = l1 > l2 ? l2 : l1; + ret = memcmp(d1, d2, l); + if (ret) + return ret; + return l1 == l2 ? 0 : l2 > l1 ? -1 : 1; + } + + /* only atom2 is direct */ + if (d2) + return fy_atom_memcmp(atom1, d2, l2); + + /* only atom1 is direct, (note reversing sign) */ + if (d1) + return -fy_atom_memcmp(atom2, d1, l1); + + /* neither is direct, do it with iterators */ + fy_atom_iter_start(atom1, &iter1); + fy_atom_iter_start(atom2, &iter2); + do { + c1 = fy_atom_iter_getc(&iter1); + c2 = fy_atom_iter_getc(&iter2); + } while (c1 == c2 && c1 >= 0 && c2 >= 0); + fy_atom_iter_finish(&iter2); + fy_atom_iter_finish(&iter1); + + if (c1 == -1 && c2 == -1) + return 0; + + return c2 > c1 ? -1 : 1; +} + +const struct fy_raw_line * +fy_atom_raw_line_iter_next(struct fy_atom_raw_line_iter *iter) +{ + struct fy_raw_line *l; + int c, w, col, col8, count; + unsigned int ts; + const char *s; + + if (!iter || !iter->rs || iter->rs > iter->ae) + return NULL; + + l = &iter->line; + + ts = iter->atom->tabsize; + + /* track back to the start of the line */ + s = iter->rs; + + /* we allow a single zero size iteration */ + if (l->lineno > 0 && iter->rs >= iter->ae) + return NULL; + + while (s > iter->is) { + c = fy_utf8_get_right(iter->is, (size_t)(s - iter->is), &w); + if (c <= 0 || fy_is_lb_m(c, iter->atom->lb_mode)) + break; + s -= w; + } + + l->line_start = s; + col = col8 = 0; + count = 0; + c = -1; + w = 0; + + /* track until the start of the content */ + while (s < iter->as) { + c = fy_utf8_get(s, (size_t)(iter->ae - s), &w); + /* we should never hit that */ + if (c <= 0) + return NULL; + if (fy_is_tab(c)) { + col8 += (8 - (col8 % 8)); + if (ts) + col += (ts - (col % ts)); + else + col++; + } else if (!fy_is_lb_m(c, iter->atom->lb_mode)) { + col++; + col8++; + } else + return NULL; + count++; + + s += w; + } + /* mark start of content */ + l->content_start = s; + l->content_start_col = col; + l->content_start_col8 = col8; + l->content_start_count = count; + + /* track until the end of the content (or lb) */ + while (s < iter->ae) { + c = fy_utf8_get(s, (size_t)(iter->ae - s), &w); + /* we should never hit that */ + if (c <= 0) + return NULL; + if (fy_is_tab(c)) { + col8 += (8 - (col8 % 8)); + if (ts) + col += (ts - (col % ts)); + else + col++; + } else if (!fy_is_lb_m(c, iter->atom->lb_mode)) { + col++; + col8++; + } else + break; + count++; + + s += w; + } + + l->content_len = (size_t)(s - l->content_start); + l->content_count = count - l->content_start_count; + l->content_end_col = col; + l->content_end_col8 = col8; + + /* if the stop was due to end of the atom */ + if (s >= iter->ae) { + while (s < iter->ie) { + c = fy_utf8_get(s, (size_t)(iter->ie - s), &w); + /* just end of input */ + if (c <= 0) + break; + + if (fy_is_tab(c)) { + col8 += (8 - (col8 % 8)); + if (ts) + col += (ts - (col % ts)); + else + col++; + } else if (!fy_is_lb_m(c, iter->atom->lb_mode)) { + col++; + col8++; + } else + break; + count++; + + s += w; + } + } + + l->line_len = (size_t)(s - l->line_start); + l->line_count = count; + + if (fy_is_lb_m(c, iter->atom->lb_mode)) { + s += w; + /* special case for MSDOS */ + if (c == '\r' && (s < iter->ie && s[1] == '\n')) + s++; + /* len_lb includes the lb */ + l->line_len_lb = (size_t)(s - l->line_start); + } else + l->line_len_lb = l->line_len; + + /* start at line #1 */ + l->lineno++; + + iter->rs = s; + + return l; +} + +void fy_atom_raw_line_iter_start(const struct fy_atom *atom, + struct fy_atom_raw_line_iter *iter) +{ + struct fy_input *fyi; + + if (!atom || !iter) + return; + + memset(iter, 0, sizeof(*iter)); + + fyi = atom->fyi; + if (!fyi) + return; + + iter->atom = atom; + + iter->as = fy_atom_data(atom); + iter->ae = iter->as + fy_atom_size(atom); + + iter->is = fy_input_start(fyi); + iter->ie = iter->is + fy_input_size(fyi); + + iter->rs = iter->as; +} + +void fy_atom_raw_line_iter_finish(struct fy_atom_raw_line_iter *iter) +{ + /* nothing */ +} diff --git a/src/lib/fy-atom.h b/src/lib/fy-atom.h new file mode 100644 index 0000000..3f32275 --- /dev/null +++ b/src/lib/fy-atom.h @@ -0,0 +1,300 @@ +/* + * fy-atom.h - internal YAML atom methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_ATOM_H +#define FY_ATOM_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include + +#include "fy-list.h" +#include "fy-ctype.h" + +struct fy_reader; +struct fy_input; +struct fy_node; + +enum fy_atom_style { + /* YAML atoms */ + FYAS_PLAIN, + FYAS_SINGLE_QUOTED, + FYAS_DOUBLE_QUOTED, + FYAS_LITERAL, + FYAS_FOLDED, + FYAS_URI, /* special style for URIs */ + FYAS_DOUBLE_QUOTED_MANUAL, + FYAS_COMMENT /* (possibly multi line) comment */ +}; + +static inline bool fy_atom_style_is_quoted(enum fy_atom_style style) +{ + return style == FYAS_SINGLE_QUOTED || style == FYAS_DOUBLE_QUOTED; +} + +static inline bool fy_atom_style_is_block(enum fy_atom_style style) +{ + return style == FYAS_LITERAL || style == FYAS_FOLDED; +} + +enum fy_atom_chomp { + FYAC_STRIP, + FYAC_CLIP, + FYAC_KEEP, +}; + +struct fy_atom { + struct fy_mark start_mark; + struct fy_mark end_mark; + size_t storage_hint; /* guaranteed to fit in this amount of bytes */ + struct fy_input *fyi; /* input on which atom is on */ + uint64_t fyi_generation; /* to detect reallocs */ + unsigned int increment; + union { + uint64_t tozero; /* fast way to zero everything here */ + struct { + /* save a little bit of space with bitfields */ + unsigned int style : 8; /* enum fy_atom_style, note that it's a big perf win for bytes */ + unsigned int chomp : 8; /* enum fy_atom_chomp */ + unsigned int tabsize : 8; + unsigned int lb_mode : 1; /* enum fy_lb_mode */ + unsigned int fws_mode : 1; /* enum fy_flow_ws_mode */ + unsigned int directive0_mode : 1; + bool direct_output : 1; /* can directly output */ + bool storage_hint_valid : 1; + bool empty : 1; /* atom contains whitespace and linebreaks only if length > 0 */ + bool has_lb : 1; /* atom contains at least one linebreak */ + bool has_ws : 1; /* atom contains at least one whitespace */ + bool starts_with_ws : 1; /* atom starts with whitespace */ + bool starts_with_lb : 1; /* atom starts with linebreak */ + bool ends_with_ws : 1; /* atom ends with whitespace */ + bool ends_with_lb : 1; /* atom ends with linebreak */ + bool trailing_lb : 1; /* atom ends with trailing linebreaks > 1 */ + bool size0 : 1; /* atom contains absolutely nothing */ + bool valid_anchor : 1; /* atom is a valid anchor */ + bool json_mode : 1; /* atom was read in json mode */ + bool ends_with_eof : 1; /* atom ends at EOF of input */ + bool is_merge_key: 1; /* atom is just << */ + bool simple_key_allowed : 1; /* atom allows a simple key */ + bool high_ascii : 1; /* atom has utf code point >= 0x80 (only for plains) */ + }; + }; +}; + +static inline bool fy_atom_is_set(const struct fy_atom *atom) +{ + return atom && atom->fyi; +} + +static inline void fy_atom_reset(struct fy_atom *atom) +{ + if (atom) + atom->fyi = NULL; +} + +static inline bool fy_atom_json_mode(struct fy_atom *handle) +{ + if (!handle) + return false; + + return handle->json_mode; +} + +static inline enum fy_lb_mode fy_atom_lb_mode(struct fy_atom *handle) +{ + if (!handle) + return fylb_cr_nl; + + return handle->lb_mode; +} + +static inline bool fy_atom_is_merge_key(struct fy_atom *handle) +{ + return handle && handle->is_merge_key; +} + +static inline enum fy_flow_ws_mode fy_atom_flow_ws_mode(struct fy_atom *handle) +{ + if (!handle) + return fyfws_space_tab; + + return handle->fws_mode; +} + +/* all atoms are scalars so... */ +static inline bool fy_atom_is_lb(struct fy_atom *handle, int c) +{ + return fy_is_generic_lb_m(c, fy_atom_lb_mode(handle)); +} + +static inline bool fy_atom_is_flow_ws(struct fy_atom *handle, int c) +{ + return fy_is_flow_ws_m(c, fy_atom_flow_ws_mode(handle)); +} + +int fy_atom_format_text_length(struct fy_atom *atom); +const char *fy_atom_format_text(struct fy_atom *atom, char *buf, size_t maxsz); + +int fy_atom_format_utf8_length(struct fy_atom *atom); + +static inline void +fy_atom_reset_storage_hints(struct fy_atom *handle) +{ + handle->storage_hint = 0; + handle->storage_hint_valid = false; +} + +struct fy_atom *fy_reader_fill_atom(struct fy_reader *fyr, int advance, struct fy_atom *handle); +struct fy_atom *fy_reader_fill_atom_mark(struct fy_reader *fyr, const struct fy_mark *start_mark, + const struct fy_mark *end_mark, struct fy_atom *handle); +struct fy_atom *fy_reader_fill_atom_at(struct fy_reader *fyr, int advance, int count, struct fy_atom *handle); + +#define fy_reader_fill_atom_a(_fyr, _advance) fy_reader_fill_atom((_fyr), (_advance), alloca(sizeof(struct fy_atom))) + +struct fy_atom *fy_fill_node_atom(struct fy_node *fyn, struct fy_atom *handle); + +#define fy_fill_node_atom_a(_fyn) fy_fill_node_atom((_fyn), alloca(sizeof(struct fy_atom))) + +struct fy_atom_iter_line_info { + const char *start; + const char *end; + const char *nws_start; + const char *nws_end; + const char *chomp_start; + bool empty : 1; + bool trailing_breaks_ws : 1; + bool first : 1; /* first */ + bool last : 1; /* last (only ws/lb afterwards */ + bool final : 1; /* the final iterator */ + bool indented : 1; + bool lb_end : 1; + bool need_nl : 1; + bool need_sep : 1; + bool ends_with_backslash : 1; /* last ended in \\ */ + size_t trailing_ws; + size_t trailing_breaks; + size_t start_ws, end_ws; + const char *s; + const char *e; + int actual_lb; /* the line break */ + const char *s_tb; /* start of trailing breaks run */ + const char *e_tb; /* end of trailing breaks run */ +}; + +struct fy_atom_iter_chunk { + struct fy_iter_chunk ic; + /* note that it is guaranteed for copied chunks to be + * less or equal to 10 characters (the maximum digitbuf + * for double quoted escapes */ + char inplace_buf[10]; /* small copies in place */ +}; + +#define NR_STARTUP_CHUNKS 8 +#define SZ_STARTUP_COPY_BUFFER 32 + +struct fy_atom_iter { + const struct fy_atom *atom; + const char *s, *e; + unsigned int chomp; + int tabsize; + bool single_line : 1; + bool dangling_end_quote : 1; + bool last_ends_with_backslash : 1; + bool empty : 1; + bool current : 1; + bool done : 1; /* last iteration (for block styles) */ + struct fy_atom_iter_line_info li[2]; + unsigned int alloc; + unsigned int top; + unsigned int read; + struct fy_atom_iter_chunk *chunks; + struct fy_atom_iter_chunk startup_chunks[NR_STARTUP_CHUNKS]; + int unget_c; +}; + +void fy_atom_iter_start(const struct fy_atom *atom, struct fy_atom_iter *iter); +void fy_atom_iter_finish(struct fy_atom_iter *iter); +const struct fy_iter_chunk *fy_atom_iter_peek_chunk(struct fy_atom_iter *iter); +const struct fy_iter_chunk *fy_atom_iter_chunk_next(struct fy_atom_iter *iter, const struct fy_iter_chunk *curr, int *errp); +void fy_atom_iter_advance(struct fy_atom_iter *iter, size_t len); + +struct fy_atom_iter *fy_atom_iter_create(const struct fy_atom *atom); +void fy_atom_iter_destroy(struct fy_atom_iter *iter); +ssize_t fy_atom_iter_read(struct fy_atom_iter *iter, void *buf, size_t count); +int fy_atom_iter_getc(struct fy_atom_iter *iter); +int fy_atom_iter_ungetc(struct fy_atom_iter *iter, int c); +int fy_atom_iter_peekc(struct fy_atom_iter *iter); +int fy_atom_iter_utf8_get(struct fy_atom_iter *iter); +int fy_atom_iter_utf8_quoted_get(struct fy_atom_iter *iter, size_t *lenp, uint8_t *buf); +int fy_atom_iter_utf8_unget(struct fy_atom_iter *iter, int c); +int fy_atom_iter_utf8_peek(struct fy_atom_iter *iter); + +int fy_atom_memcmp(struct fy_atom *atom, const void *ptr, size_t len); +int fy_atom_strcmp(struct fy_atom *atom, const char *str); +bool fy_atom_is_number(struct fy_atom *atom); +int fy_atom_cmp(struct fy_atom *atom1, struct fy_atom *atom2); + +const char *fy_atom_data(const struct fy_atom *atom); + +static inline size_t fy_atom_size(const struct fy_atom *atom) +{ + if (!atom) + return 0; + + return atom->end_mark.input_pos - atom->start_mark.input_pos; +} + +static inline bool fy_plain_atom_streq(const struct fy_atom *atom, const char *str) +{ + size_t size = strlen(str); + + if (!atom || !str || atom->style != FYAS_PLAIN || fy_atom_size(atom) != size) + return false; + + return !memcmp(str, fy_atom_data(atom), size); +} + +struct fy_raw_line { + int lineno; + const char *line_start; + size_t line_len; + size_t line_len_lb; + size_t line_count; + const char *content_start; + size_t content_len; + size_t content_start_count; + size_t content_count; + int content_start_col; + int content_start_col8; /* this is the tab 8 */ + int content_end_col; + int content_end_col8; +}; + +struct fy_atom_raw_line_iter { + const struct fy_atom *atom; + const char *is, *ie; /* input start, end */ + const char *as, *ae; /* atom start, end */ + const char *rs; + struct fy_raw_line line; +}; + +void fy_atom_raw_line_iter_start(const struct fy_atom *atom, + struct fy_atom_raw_line_iter *iter); +void fy_atom_raw_line_iter_finish(struct fy_atom_raw_line_iter *iter); + +const struct fy_raw_line * +fy_atom_raw_line_iter_next(struct fy_atom_raw_line_iter *iter); + +#endif diff --git a/src/lib/fy-composer-diag.c b/src/lib/fy-composer-diag.c new file mode 100644 index 0000000..bc75b24 --- /dev/null +++ b/src/lib/fy-composer-diag.c @@ -0,0 +1,91 @@ +/* + * fy-composer-diag.c - composer diagnostics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-diag.h" +#include "fy-composer.h" + +int fy_composer_vdiag(struct fy_composer *fyc, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap) +{ + struct fy_diag_ctx fydc; + int rc; + + if (!fyc || !fmt || !fyc->cfg.diag) + return -1; + + /* perform the enable tests early to avoid the overhead */ + if (((flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT) < fyc->cfg.diag->cfg.level) + return 0; + + /* fill in fy_diag_ctx */ + memset(&fydc, 0, sizeof(fydc)); + + fydc.level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; + fydc.module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; + fydc.source_file = file; + fydc.source_line = line; + fydc.source_func = func; + fydc.line = -1; + fydc.column = -1; + + rc = fy_vdiag(fyc->cfg.diag, &fydc, fmt, ap); + + return rc; +} + +int fy_composer_diag(struct fy_composer *fyc, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_composer_vdiag(fyc, flags, file, line, func, fmt, ap); + va_end(ap); + + return rc; +} + +void fy_composer_diag_vreport(struct fy_composer *fyc, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap) +{ + if (!fyc || !fyc->cfg.diag || !fydrc || !fmt) + return; + + fy_diag_vreport(fyc->cfg.diag, fydrc, fmt, ap); +} + +void fy_composer_diag_report(struct fy_composer *fyc, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_composer_diag_vreport(fyc, fydrc, fmt, ap); + va_end(ap); +} diff --git a/src/lib/fy-composer.c b/src/lib/fy-composer.c new file mode 100644 index 0000000..a4c7ea4 --- /dev/null +++ b/src/lib/fy-composer.c @@ -0,0 +1,445 @@ +/* + * fy-composer.c - Composer support + * + * Copyright (c) 2021 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" +#include "fy-doc.h" + +#include "fy-utils.h" + +struct fy_composer * +fy_composer_create(struct fy_composer_cfg *cfg) +{ + struct fy_composer *fyc; + struct fy_path *fypp; + + /* verify configuration and mandatory ops */ + if (!cfg || !cfg->ops || + !cfg->ops->process_event) + return NULL; + + fyc = malloc(sizeof(*fyc)); + if (!fyc) + return NULL; + memset(fyc, 0, sizeof(*fyc)); + fyc->cfg = *cfg; + + fy_path_list_init(&fyc->paths); + fypp = fy_path_create(); + if (!fypp) + goto err_no_path; + fy_path_list_add_tail(&fyc->paths, fypp); + + return fyc; + +err_no_path: + free(fyc); + return NULL; +} + +void fy_composer_destroy(struct fy_composer *fyc) +{ + struct fy_path *fypp; + + if (!fyc) + return; + + fy_diag_unref(fyc->cfg.diag); + while ((fypp = fy_path_list_pop(&fyc->paths)) != NULL) + fy_path_destroy(fypp); + free(fyc); +} + +static enum fy_composer_return +fy_composer_process_event_private(struct fy_composer *fyc, struct fy_event *fye, struct fy_path *fypp) +{ + const struct fy_composer_ops *ops; + struct fy_eventp *fyep; + struct fy_path_component *fypc, *fypc_last; + struct fy_path *fyppt; + struct fy_document *fyd; + bool is_collection, is_map, is_start, is_end; + int rc = 0; + enum fy_composer_return ret; + bool stop_req = false; + + assert(fyc); + assert(fye); + assert(fypp); + + fyep = container_of(fye, struct fy_eventp, e); + + ops = fyc->cfg.ops; + assert(ops); + + rc = 0; + + switch (fye->type) { + case FYET_MAPPING_START: + is_collection = true; + is_start = true; + is_end = false; + is_map = true; + break; + + case FYET_MAPPING_END: + is_collection = true; + is_start = false; + is_end = true; + is_map = true; + break; + + case FYET_SEQUENCE_START: + is_collection = true; + is_start = true; + is_end = false; + is_map = false; + break; + + case FYET_SEQUENCE_END: + is_collection = true; + is_start = false; + is_end = true; + is_map = false; + break; + + case FYET_SCALAR: + is_collection = false; + is_start = true; + is_end = true; + is_map = false; + break; + + case FYET_ALIAS: + is_collection = false; + is_start = true; + is_end = true; + is_map = false; + break; + + case FYET_STREAM_START: + case FYET_STREAM_END: + case FYET_DOCUMENT_START: + case FYET_DOCUMENT_END: + return ops->process_event(fyc, fypp, fye); + + default: + return FYCR_OK_CONTINUE; + } + + fypc_last = fy_path_component_list_tail(&fypp->components); + + if (fy_path_component_is_mapping(fypc_last) && fypc_last->map.accumulating_complex_key) { + + /* get the next one */ + fyppt = fy_path_next(&fyc->paths, fypp); + assert(fyppt); + assert(fyppt != fypp); + assert(fyppt->parent == fypp); + + /* and pass along */ + ret = fy_composer_process_event_private(fyc, fye, fyppt); + if (!fy_composer_return_is_ok(ret)) { + /* XXX TODO handle skip */ + return ret; + } + if (!stop_req) + stop_req = ret == FYCR_OK_STOP; + + rc = fy_document_builder_process_event(fypp->fydb, fyep ? &fyep->e : NULL); + if (rc == 0) + return FYCR_OK_CONTINUE; + fyc_error_check(fyc, rc > 0, err_out, + "fy_document_builder_process_event() failed\n"); + + /* get the document */ + fyd = fy_document_builder_take_document(fypp->fydb); + fyc_error_check(fyc, fyd, err_out, + "fy_document_builder_take_document() failed\n"); + + fy_document_builder_destroy(fypp->fydb); + fypp->fydb = NULL; + + fypc_last->map.is_complex_key = true; + fypc_last->map.accumulating_complex_key = false; + fypc_last->map.complex_key = fyd; + fypc_last->map.has_key = true; + fypc_last->map.await_key = false; + fypc_last->map.complex_key_complete = true; + fypc_last->map.root = false; + + fyppt = fy_path_list_pop_tail(&fyc->paths); + assert(fyppt); + + fy_path_destroy(fyppt); + + fyc_error_check(fyc, rc >= 0, err_out, + "fy_path_component_build_text() failed\n"); + + return !stop_req ? FYCR_OK_CONTINUE : FYCR_OK_STOP; + } + + /* start of something on a mapping */ + if (is_start && fy_path_component_is_mapping(fypc_last) && fypc_last->map.await_key && is_collection) { + + /* the configuration must support a document builder for complex keys */ + FYC_TOKEN_ERROR_CHECK(fyc, fy_event_get_token(fye), FYEM_DOC, + ops->create_document_builder, err_out, + "composer configuration does not support complex keys"); + + /* call out for creating the document builder */ + fypp->fydb = ops->create_document_builder(fyc); + fyc_error_check(fyc, fypp->fydb, err_out, + "ops->create_document_builder() failed\n"); + + /* and pass the current event; must return 0 since we know it's a collection start */ + rc = fy_document_builder_process_event(fypp->fydb, fyep ? &fyep->e : NULL); + fyc_error_check(fyc, !rc, err_out, + "fy_document_builder_process_event() failed\n"); + + fypc_last->map.is_complex_key = true; + fypc_last->map.accumulating_complex_key = true; + fypc_last->map.complex_key = NULL; + fypc_last->map.complex_key_complete = false; + + /* create new path */ + fyppt = fy_path_create(); + fyc_error_check(fyc, fyppt, err_out, + "fy_path_create() failed\n"); + + /* append it to the end */ + fyppt->parent = fypp; + fy_path_list_add_tail(&fyc->paths, fyppt); + + /* and pass along */ + ret = fy_composer_process_event_private(fyc, fye, fyppt); + if (!fy_composer_return_is_ok(ret)) { + /* XXX TODO handle skip */ + return ret; + } + if (!stop_req) + stop_req = ret == FYCR_OK_STOP; + + return !stop_req ? FYCR_OK_CONTINUE : FYCR_OK_STOP; + } + + if (is_start && fy_path_component_is_sequence(fypc_last)) { /* start in a sequence */ + + if (fypc_last->seq.idx < 0) + fypc_last->seq.idx = 0; + else + fypc_last->seq.idx++; + } + + if (is_collection && is_start) { + + /* collection start */ + if (is_map) { + fypc = fy_path_component_create_mapping(fypp); + fyc_error_check(fyc, fypc, err_out, + "fy_path_component_create_mapping() failed\n"); + } else { + fypc = fy_path_component_create_sequence(fypp); + fyc_error_check(fyc, fypc, err_out, + "fy_path_component_create_sequence() failed\n"); + } + + /* append to the tail */ + fy_path_component_list_add_tail(&fypp->components, fypc); + + } else if (is_collection && is_end) { + + /* collection end */ + assert(fypc_last); + fy_path_component_clear_state(fypc_last); + + } else if (!is_collection && fy_path_component_is_mapping(fypc_last) && fypc_last->map.await_key) { + + fypc_last->map.is_complex_key = false; + fypc_last->map.scalar.tag = fy_token_ref(fy_event_get_tag_token(fye)); + fypc_last->map.scalar.key = fy_token_ref(fy_event_get_token(fye)); + fypc_last->map.has_key = true; + fypc_last->map.root = false; + + } + + /* process the event */ + ret = ops->process_event(fyc, fypp, fye); + if (!fy_composer_return_is_ok(ret)) { + /* XXX TODO handle skip */ + return ret; + } + if (!stop_req) + stop_req = ret == FYCR_OK_STOP; + + if (is_collection && is_end) { + /* for the end of a collection, pop the last component */ + fypc = fy_path_component_list_pop_tail(&fypp->components); + assert(fypc); + + assert(fypc == fypc_last); + + fy_path_component_recycle(fypp, fypc); + + /* and get the new last */ + fypc_last = fy_path_component_list_tail(&fypp->components); + } + + /* at the end of something */ + if (is_end && fy_path_component_is_mapping(fypc_last)) { + if (!fypc_last->map.await_key) { + fy_path_component_clear_state(fypc_last); + fypc_last->map.await_key = true; + } else + fypc_last->map.await_key = false; + } + + return !stop_req ? FYCR_OK_CONTINUE : FYCR_OK_STOP; + +err_out: + return FYCR_ERROR; +} + +void +fy_composer_halt_one(struct fy_composer *fyc, struct fy_path *fypp, enum fy_composer_return rc) +{ + const struct fy_composer_ops *ops; + struct fy_eventp ev_none; + struct fy_path_component *fypc; + + /* for normal stop, we don't teardown */ + if (rc == FYCR_OK_STOP) + return; + + ops = fyc->cfg.ops; + assert(ops); + + memset(&ev_none, 0, sizeof(ev_none)); + ev_none.e.type = FYET_NONE; + + /* pump FYET_NONEs to clean the stack */ + + while ((fypc = fy_path_component_list_tail(&fypp->components)) != NULL) { + + ops->process_event(fyc, fypp, &ev_none.e); + + fy_path_component_list_del(&fypp->components, fypc); + fy_path_component_free(fypc); + } + + /* and the final one to clean the root */ + ops->process_event(fyc, fypp, &ev_none.e); +} + +void fy_composer_halt(struct fy_composer *fyc, enum fy_composer_return rc) +{ + struct fy_path *fypp; + + /* for normal stop, we don't teardown */ + if (rc == FYCR_OK_STOP) + return; + + for (fypp = fy_path_list_head(&fyc->paths); fypp; fypp = fy_path_next(&fyc->paths, fypp)) + fy_composer_halt_one(fyc, fypp, rc); +} + +enum fy_composer_return +fy_composer_process_event(struct fy_composer *fyc, struct fy_event *fye) +{ + struct fy_path *fypp; + int rc; + + if (!fyc || !fye) + return FYCR_ERROR; + + /* start at the head */ + fypp = fy_path_list_head(&fyc->paths); + + /* no top? something's very out of order */ + if (!fypp) + return FYCR_ERROR; + + rc = fy_composer_process_event_private(fyc, fye, fypp); + + if (rc == FYCR_ERROR || rc == FYCR_OK_STOP) + fy_composer_halt(fyc, rc); + + return rc; +} + +struct fy_composer_cfg *fy_composer_get_cfg(struct fy_composer *fyc) +{ + if (!fyc) + return NULL; + return &fyc->cfg; +} + +void *fy_composer_get_cfg_userdata(struct fy_composer *fyc) +{ + if (!fyc) + return NULL; + return fyc->cfg.userdata; +} + +struct fy_diag *fy_composer_get_diag(struct fy_composer *fyc) +{ + if (!fyc) + return NULL; + return fyc->cfg.diag; +} + +struct fy_path *fy_composer_get_root_path(struct fy_composer *fyc) +{ + if (!fyc) + return NULL; + return fy_path_list_head(&fyc->paths); +} + +struct fy_path *fy_composer_get_next_path(struct fy_composer *fyc, struct fy_path *fypp) +{ + if (!fyc || !fypp) + return NULL; + return fy_path_next(&fyc->paths, fypp); +} + +struct fy_path *fy_composer_get_path(struct fy_composer *fyc) +{ + struct fy_path *fypp, *fyppt; + struct fy_path_component *fypc_last; + + if (!fyc) + return NULL; + + fypp = fy_path_list_head(&fyc->paths); + if (!fypp) + return NULL; + + /* traverse until we find the last non complex key acculating */ + for (;;) { + fypc_last = fy_path_component_list_tail(&fypp->components); + if (!fy_path_component_is_mapping(fypc_last) || !fypc_last->map.accumulating_complex_key) + break; + fyppt = fy_path_next(&fyc->paths, fypp); + if (!fyppt) + break; + fypp = fyppt; + } + + return fypp; +} diff --git a/src/lib/fy-composer.h b/src/lib/fy-composer.h new file mode 100644 index 0000000..2c1634b --- /dev/null +++ b/src/lib/fy-composer.h @@ -0,0 +1,137 @@ +/* + * fy-composer.h - YAML composer + * + * Copyright (c) 2021 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_COMPOSER_H +#define FY_COMPOSER_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include + +#include "fy-list.h" +#include "fy-typelist.h" + +#include "fy-emit-accum.h" +#include "fy-path.h" + +#include "fy-diag.h" + +struct fy_composer; +struct fy_token; +struct fy_diag; +struct fy_event; +struct fy_eventp; +struct fy_document_builder; + +struct fy_composer { + struct fy_composer_cfg cfg; + struct fy_path_list paths; +}; + +void fy_composer_halt_one(struct fy_composer *fyc, struct fy_path *fypp, enum fy_composer_return rc); +void fy_composer_halt(struct fy_composer *fyc, enum fy_composer_return rc); + +/* diagnostics */ + +static inline bool +fyc_debug_log_level_is_enabled(struct fy_composer *fyc, enum fy_error_module module) +{ + return fyc && fy_diag_log_level_is_enabled(fyc->cfg.diag, FYET_DEBUG, module); +} + +int fy_composer_vdiag(struct fy_composer *fyc, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap); + +int fy_composer_diag(struct fy_composer *fyc, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) + __attribute__((format(printf, 6, 7))); + +void fy_composer_diag_vreport(struct fy_composer *fyc, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap); +void fy_composer_diag_report(struct fy_composer *fyc, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +#ifdef FY_DEVMODE + +#define fyc_debug(_fyc, _module, _fmt, ...) \ + do { \ + struct fy_composer *__fyc = (_fyc); \ + enum fy_error_module __module = (_module); \ + \ + if (fyc_debug_log_level_is_enabled(__fyc, __module)) \ + fy_composer_diag(__fyc, FYET_DEBUG | FYDF_MODULE(_module), \ + __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__); \ + } while(0) +#else + +#define fyc_debug(_fyc, _module, _fmt, ...) \ + do { } while(0) + +#endif + +#define fyc_info(_fyc, _fmt, ...) \ + fy_composer_diag((_fyc), FYET_INFO, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyc_notice(_fyc, _fmt, ...) \ + fy_composer_diag((_fyc), FYET_NOTICE, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyc_warning(_fyc, _fmt, ...) \ + fy_composer_diag((_fyc), FYET_WARNING, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyc_error(_fyc, _fmt, ...) \ + fy_composer_diag((_fyc), FYET_ERROR, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) + +#define fyc_error_check(_fyc, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + fyc_error((_fyc), _fmt, ## __VA_ARGS__); \ + goto _label ; \ + } \ + } while(0) + +#define _FYC_TOKEN_DIAG(_fyc, _fyt, _type, _module, _fmt, ...) \ + do { \ + struct fy_diag_report_ctx _drc; \ + memset(&_drc, 0, sizeof(_drc)); \ + _drc.type = (_type); \ + _drc.module = (_module); \ + _drc.fyt = (_fyt); \ + fy_composer_diag_report((_fyc), &_drc, (_fmt) , ## __VA_ARGS__); \ + } while(0) + +#define FYC_TOKEN_DIAG(_fyc, _fyt, _type, _module, _fmt, ...) \ + _FYC_TOKEN_DIAG(_fyc, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYC_NODE_DIAG(_fyc, _fyn, _type, _module, _fmt, ...) \ + _FYC_TOKEN_DIAG(_fyc, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYC_TOKEN_ERROR(_fyc, _fyt, _module, _fmt, ...) \ + FYC_TOKEN_DIAG(_fyc, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYC_TOKEN_ERROR_CHECK(_fyc, _fyt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYC_TOKEN_ERROR(_fyc, _fyt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYC_TOKEN_WARNING(_fyc, _fyt, _module, _fmt, ...) \ + FYC_TOKEN_DIAG(_fyc, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#endif diff --git a/src/lib/fy-diag.c b/src/lib/fy-diag.c new file mode 100644 index 0000000..58a6dad --- /dev/null +++ b/src/lib/fy-diag.c @@ -0,0 +1,1173 @@ +/* + * fy-diag.c - diagnostics + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-diag.h" + +#include "fy-parse.h" +#include "fy-input.h" + + +static const char *error_type_txt[] = { + [FYET_DEBUG] = "debug", + [FYET_INFO] = "info", + [FYET_NOTICE] = "notice", + [FYET_WARNING] = "warning", + [FYET_ERROR] = "error", +}; + +const char *fy_error_type_to_string(enum fy_error_type type) +{ + + if ((unsigned int)type >= FYET_MAX) + return ""; + return error_type_txt[type]; +} + +enum fy_error_type fy_string_to_error_type(const char *str) +{ + unsigned int i; + int level; + + if (!str) + return FYET_MAX; + + if (isdigit((unsigned char)*str)) { + level = atoi(str); + if (level >= 0 && level < FYET_MAX) + return (enum fy_error_type)level; + } + + for (i = 0; i < FYET_MAX; i++) { + if (!strcmp(str, error_type_txt[i])) + return (enum fy_error_type)i; + } + + return FYET_MAX; +} + +static const char *error_module_txt[] = { + [FYEM_UNKNOWN] = "unknown", + [FYEM_ATOM] = "atom", + [FYEM_SCAN] = "scan", + [FYEM_PARSE] = "parse", + [FYEM_DOC] = "doc", + [FYEM_BUILD] = "build", + [FYEM_INTERNAL] = "internal", + [FYEM_SYSTEM] = "system", + [FYEM_EMIT] = "emit", + [FYEM_TYPESET] = "typeset", + [FYEM_DECODE] = "decode", + [FYEM_ENCODE] = "encode", +}; + +const char *fy_error_module_to_string(enum fy_error_module module) +{ + + if ((unsigned int)module >= FYEM_MAX) + return ""; + return error_module_txt[module]; +} + +enum fy_error_module fy_string_to_error_module(const char *str) +{ + unsigned int i; + + if (!str) + return FYEM_MAX; + + for (i = 0; i < FYEM_MAX; i++) { + if (!strcmp(str, error_module_txt[i])) + return (enum fy_error_module)i; + } + + return FYEM_MAX; +} + +static const char *fy_error_level_str(enum fy_error_type level) +{ + static const char *txt[] = { + [FYET_DEBUG] = "DBG", + [FYET_INFO] = "INF", + [FYET_NOTICE] = "NOT", + [FYET_WARNING] = "WRN", + [FYET_ERROR] = "ERR", + }; + + if ((unsigned int)level >= FYET_MAX) + return "*unknown*"; + return txt[level]; +} + +static const char *fy_error_module_str(enum fy_error_module module) +{ + static const char *txt[] = { + [FYEM_UNKNOWN] = "UNKWN", + [FYEM_ATOM] = "ATOM ", + [FYEM_SCAN] = "SCAN ", + [FYEM_PARSE] = "PARSE", + [FYEM_DOC] = "DOC ", + [FYEM_BUILD] = "BUILD", + [FYEM_INTERNAL] = "INTRL", + [FYEM_SYSTEM] = "SYSTM", + [FYEM_EMIT] = "EMIT ", + [FYEM_TYPESET] = "TYPES", + [FYEM_DECODE] = "DEC ", + [FYEM_ENCODE] = "ENC ", + }; + + if ((unsigned int)module >= FYEM_MAX) + return "*unknown*"; + return txt[module]; +} + +/* really concervative options */ +static const struct fy_diag_term_info default_diag_term_info_template = { + .rows = 25, + .columns = 80 +}; + +static const struct fy_diag_cfg default_diag_cfg_template = { + .fp = NULL, /* must be overriden */ + .level = FYET_INFO, + .module_mask = (1U << FYEM_MAX) - 1, /* all modules */ + .show_source = false, + .show_position = false, + .show_type = true, + .show_module = false, + .colorize = false, /* can be overriden */ + .source_width = 50, + .position_width = 10, + .type_width = 5, + .module_width = 6, +}; + +void fy_diag_cfg_default(struct fy_diag_cfg *cfg) +{ + if (!cfg) + return; + + *cfg = default_diag_cfg_template; + cfg->fp = stderr; + cfg->colorize = isatty(fileno(stderr)) == 1; +} + +void fy_diag_cfg_from_parser_flags(struct fy_diag_cfg *cfg, enum fy_parse_cfg_flags pflags) +{ + /* nothing */ +} + +static bool fy_diag_isatty(struct fy_diag *diag) +{ + return diag && diag->cfg.fp && isatty(fileno(diag->cfg.fp)); +} + +static void fy_diag_update_term_info(struct fy_diag *diag) +{ + int fd, rows, columns, ret; + + /* start by setting things to the default */ + diag->term_info = default_diag_term_info_template; + + fd = diag->cfg.fp && isatty(fileno(diag->cfg.fp)) ? + fileno(diag->cfg.fp) : -1; + + if (fd == -1) + goto out; + + rows = columns = 0; + ret = fy_term_query_size(fd, &rows, &columns); + if (ret != 0) + goto out; + + if (rows > 0 && columns > 0) { + diag->term_info.rows = rows; + diag->term_info.columns = columns; + } +out: + diag->terminal_probed = true; +} + +void fy_diag_errorp_free(struct fy_diag_errorp *errp) +{ + if (errp->space) + free(errp->space); + fy_token_unref(errp->e.fyt); + free(errp); +} + +struct fy_diag *fy_diag_create(const struct fy_diag_cfg *cfg) +{ + struct fy_diag *diag; + + diag = malloc(sizeof(*diag)); + if (!diag) + return NULL; + memset(diag, 0, sizeof(*diag)); + + if (!cfg) + fy_diag_cfg_default(&diag->cfg); + else + diag->cfg = *cfg; + diag->on_error = false; + diag->refs = 1; + + diag->terminal_probed = false; + if (!fy_diag_isatty(diag)) + fy_diag_update_term_info(diag); + + fy_diag_errorp_list_init(&diag->errors); + + return diag; +} + +void fy_diag_destroy(struct fy_diag *diag) +{ + struct fy_diag_errorp *errp; + + if (!diag) + return; + + diag->destroyed = true; + + /* free everything */ + while ((errp = fy_diag_errorp_list_pop(&diag->errors)) != NULL) + fy_diag_errorp_free(errp); + + return fy_diag_unref(diag); +} + +bool fy_diag_got_error(struct fy_diag *diag) +{ + return diag && diag->on_error; +} + +void fy_diag_set_error(struct fy_diag *diag, bool on_error) +{ + if (diag) + diag->on_error = on_error; +} + +void fy_diag_reset_error(struct fy_diag *diag) +{ + struct fy_diag_errorp *errp; + + if (!diag) + return; + + diag->on_error = false; + + while ((errp = fy_diag_errorp_list_pop(&diag->errors)) != NULL) + fy_diag_errorp_free(errp); +} + +void fy_diag_set_collect_errors(struct fy_diag *diag, bool collect_errors) +{ + struct fy_diag_errorp *errp; + + if (!diag || diag->destroyed) + return; + + diag->collect_errors = collect_errors; + + /* clear collected errors on disable */ + if (!diag->collect_errors) { + while ((errp = fy_diag_errorp_list_pop(&diag->errors)) != NULL) + fy_diag_errorp_free(errp); + } +} + +struct fy_diag_error *fy_diag_errors_iterate(struct fy_diag *diag, void **prevp) +{ + struct fy_diag_errorp *errp; + + if (!diag || !prevp) + return NULL; + + if (!*prevp) + errp = fy_diag_errorp_list_head(&diag->errors); + else { + errp = *prevp; + errp = fy_diag_errorp_next(&diag->errors, errp); + } + + if (!errp) + return NULL; + *prevp = errp; + return &errp->e; +} + +void fy_diag_free(struct fy_diag *diag) +{ + if (!diag) + return; + free(diag); +} + +const struct fy_diag_cfg *fy_diag_get_cfg(struct fy_diag *diag) +{ + if (!diag) + return NULL; + return &diag->cfg; +} + +void fy_diag_set_cfg(struct fy_diag *diag, const struct fy_diag_cfg *cfg) +{ + if (!diag) + return; + + if (!cfg) + fy_diag_cfg_default(&diag->cfg); + else + diag->cfg = *cfg; + + fy_diag_update_term_info(diag); +} + +void fy_diag_set_level(struct fy_diag *diag, enum fy_error_type level) +{ + if (!diag || (unsigned int)level >= FYET_MAX) + return; + diag->cfg.level = level; +} + +void fy_diag_set_colorize(struct fy_diag *diag, bool colorize) +{ + if (!diag) + return; + diag->cfg.colorize = colorize; +} + +struct fy_diag *fy_diag_ref(struct fy_diag *diag) +{ + if (!diag) + return NULL; + + assert(diag->refs + 1 > 0); + diag->refs++; + + return diag; +} + +void fy_diag_unref(struct fy_diag *diag) +{ + if (!diag) + return; + + assert(diag->refs > 0); + + if (diag->refs == 1) + fy_diag_free(diag); + else + diag->refs--; +} + +ssize_t fy_diag_write(struct fy_diag *diag, const void *buf, size_t count) +{ + size_t ret; + + if (!diag || !buf) + return -1; + + /* no more output */ + if (diag->destroyed) + return 0; + + ret = 0; + if (diag->cfg.fp) { + ret = fwrite(buf, 1, count, diag->cfg.fp); + } else if (diag->cfg.output_fn) { + diag->cfg.output_fn(diag, diag->cfg.user, buf, count); + ret = count; + } + + return ret == count ? (ssize_t)count : -1; +} + +int fy_diag_vprintf(struct fy_diag *diag, const char *fmt, va_list ap) +{ + char *buf; + int rc; + + if (!diag || !fmt) + return -1; + + /* no more output */ + if (diag->destroyed) + return 0; + + if (diag->cfg.fp) + return vfprintf(diag->cfg.fp, fmt, ap); + + if (diag->cfg.output_fn) { + rc = vasprintf(&buf, fmt, ap); + if (rc < 0) + return rc; + diag->cfg.output_fn(diag, diag->cfg.user, buf, (size_t)rc); + free(buf); + return rc; + } + + return -1; +} + +int fy_diag_printf(struct fy_diag *diag, const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_diag_vprintf(diag, fmt, ap); + va_end(ap); + + return rc; +} + +int fy_vdiag(struct fy_diag *diag, const struct fy_diag_ctx *fydc, + const char *fmt, va_list ap) +{ + char *msg = NULL; + char *source = NULL, *position = NULL, *typestr = NULL, *modulestr = NULL; + const char *file_stripped = NULL; + const char *color_start = NULL, *color_end = NULL; + enum fy_error_type level; + int rc; + + if (!diag || !fydc || !fmt) + return -1; + + level = fydc->level; + + /* turn errors into debugs while not reset */ + if (level >= FYET_ERROR && diag->on_error) + level = FYET_DEBUG; + + if (!fy_diag_log_level_is_enabled(diag, level, fydc->module)) { + rc = 0; + goto out; + } + + msg = fy_vsprintfa(fmt, ap); + fy_strip_trailing_nl(msg); + + /* source part */ + if (diag->cfg.show_source) { + if (fydc->source_file) { + file_stripped = strrchr(fydc->source_file, '/'); + if (!file_stripped) + file_stripped = fydc->source_file; + else + file_stripped++; + } else + file_stripped = ""; + source = fy_sprintfa("%s:%d @%s()%s", + file_stripped, fydc->source_line, fydc->source_func, " "); + } + + /* position part */ + if (diag->cfg.show_position && fydc->line >= 0 && fydc->column >= 0) + position = fy_sprintfa("<%3d:%2d>%s", fydc->line, fydc->column, ": "); + + /* type part */ + if (diag->cfg.show_type) + typestr = fy_sprintfa("[%s]%s", fy_error_level_str(level), ": "); + + /* module part */ + if (diag->cfg.show_module) + modulestr = fy_sprintfa("<%s>%s", fy_error_module_str(fydc->module), ": "); + + if (diag->cfg.colorize) { + switch (level) { + case FYET_DEBUG: + color_start = "\x1b[37m"; /* normal white */ + break; + case FYET_INFO: + color_start = "\x1b[37;1m"; /* bright white */ + break; + case FYET_NOTICE: + color_start = "\x1b[34;1m"; /* bright blue */ + break; + case FYET_WARNING: + color_start = "\x1b[33;1m"; /* bright yellow */ + break; + case FYET_ERROR: + color_start = "\x1b[31;1m"; /* bright red */ + break; + default: /* handles FYET_MAX */ + break; + } + if (color_start) + color_end = "\x1b[0m"; + } + + rc = fy_diag_printf(diag, "%s" "%*s" "%*s" "%*s" "%*s" "%s" "%s\n", + color_start ? : "", + source ? diag->cfg.source_width : 0, source ? : "", + position ? diag->cfg.position_width : 0, position ? : "", + typestr ? diag->cfg.type_width : 0, typestr ? : "", + modulestr ? diag->cfg.module_width : 0, modulestr ? : "", + msg, + color_end ? : ""); + + if (rc > 0) + rc++; + +out: + /* if it's the first error we're generating set the + * on_error flag until the top caller clears it + */ + if (!diag->on_error && fydc->level >= FYET_ERROR) + diag->on_error = true; + + return rc; +} + +int fy_diagf(struct fy_diag *diag, const struct fy_diag_ctx *fydc, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_vdiag(diag, fydc, fmt, ap); + va_end(ap); + + return rc; +} + +static void fy_diag_get_error_colors(struct fy_diag *diag, enum fy_error_type type, + const char **start, const char **end, const char **white) +{ + if (!diag->cfg.colorize) { + *start = *end = *white = ""; + return; + } + + switch (type) { + case FYET_DEBUG: + *start = "\x1b[37m"; /* normal white */ + break; + case FYET_INFO: + *start = "\x1b[37;1m"; /* bright white */ + break; + case FYET_NOTICE: + *start = "\x1b[34;1m"; /* bright blue */ + break; + case FYET_WARNING: + *start = "\x1b[33;1m"; /* bright yellow */ + break; + case FYET_ERROR: + *start = "\x1b[31;1m"; /* bright red */ + break; + default: + *start = "\x1b[0m"; /* catch-all reset */ + break; + } + *end = "\x1b[0m"; + *white = "\x1b[37;1m"; +} + +void fy_diag_error_atom_display(struct fy_diag *diag, enum fy_error_type type, struct fy_atom *atom) +{ + const struct fy_raw_line *l, *ln; + struct fy_raw_line l_tmp; + struct fy_atom_raw_line_iter iter; + int content_start_col, content_end_col, content_width; + int pass, cols, min_col, max_col, max_line_count, max_line_col8, max_width; + int start_col, end_col; + const char *color_start, *color_end, *white; + bool first_line, last_line; + const char *display; + int display_len, line_shift; + char qc, first_mark; + char *rowbuf = NULL, *rbs = NULL, *rbe = NULL; + const char *s, *e; + int col8, c, w; + int tab8_len, tilde_start, tilde_width, tilde_width_m1; + size_t rowbufsz; + + (void)end_col; + + if (!diag || !atom) + return; + + fy_diag_get_error_colors(diag, type, &color_start, &color_end, &white); + + /* pacify compilers that think these are uninitialized */ + cols = 0; + rowbufsz = 0; + + /* two passes, first one collects extents */ + + start_col = -1; + end_col = -1; + min_col = -1; + max_col = -1; + max_line_count = -1; + max_line_col8 = -1; + line_shift = -1; + for (pass = 0; pass < 2; pass++) { + + /* on the start of the second pass */ + if (pass > 0) { + + cols = 0; + + /* if it's probed, use what's there */ + if (diag->terminal_probed && diag->term_info.columns > 0) + cols = diag->term_info.columns; + + /* heuristic, avoid probing terminal size if maximum column is less than 80 + * columns. This is faster and avoid problems with terminals... + */ + if (!cols && max_line_col8 < 80) + cols = 80; + + /* no choice but to probe */ + if (!cols) { + /* only need the terminal width when outputting an error */ + if (!diag->terminal_probed && fy_diag_isatty(diag)) + fy_diag_update_term_info(diag); + + cols = diag->term_info.columns; + } + + /* worse case utf8 + 2 color sequences + zero terminated */ + rowbufsz = cols * 4 + 2 * 16 + 1; + rowbuf = alloca(rowbufsz); + rbe = rowbuf + rowbufsz; + + /* if the maximum column number is less than the terminal + * width everything fits, and we're fine */ + if (max_line_col8 < cols) { + line_shift = 0; + } else { + max_width = max_col - min_col; + + /* try to center */ + line_shift = min_col + (max_width - cols) / 2; + + /* the start of the content must always be included */ + if (start_col < line_shift) + line_shift = start_col; + } + } + + fy_atom_raw_line_iter_start(atom, &iter); + l = fy_atom_raw_line_iter_next(&iter); + for (; l != NULL; l = ln) { + + /* save it */ + l_tmp = *l; + l = &l_tmp; + + /* get the next too */ + ln = fy_atom_raw_line_iter_next(&iter); + + first_line = l->lineno <= 1; + last_line = ln == NULL; + + content_start_col = l->content_start_col8; + content_end_col = l->content_end_col8; + + /* adjust for single and double quoted to include the quote marks (usually works) */ + if (fy_atom_style_is_quoted(atom->style)) { + qc = atom->style == FYAS_SINGLE_QUOTED ? '\'' : '"'; + if (first_line && l->content_start > l->line_start && + l->content_start[-1] == qc) + content_start_col--; + if (last_line && (l->content_start + l->content_len) < (l->line_start + l->line_len) && + l->content_start[l->content_len] == qc) + content_end_col++; + } + + content_width = content_end_col - content_start_col; + + if (pass == 0) { + + if (min_col < 0 || content_start_col < min_col) + min_col = content_start_col; + if (max_col < 0 || content_end_col > max_col) + max_col = content_end_col; + if (max_line_count < 0 || (int)l->line_count > max_line_count) + max_line_count = (int)l->line_count; + if (first_line) + start_col = content_start_col; + if (last_line) + end_col = content_end_col; + + /* optimize by using the content end as a starting point */ + s = l->content_start + l->content_len; + e = l->line_start + l->line_count; + + /* guard against something stupid */ + if (s > e) + s = e; + + col8 = l->content_end_col8; + while ((c = fy_utf8_get(s, (e - s), &w)) >= 0) { + s += w; + if (fy_is_tab(c)) + col8 += 8 - (col8 % 8); + else + col8++; + } + /* update the max column number of the lines */ + if (max_line_col8 < 0 || col8 > max_line_col8) + max_line_col8 = col8; + + continue; + } + + /* output pass */ + + /* the defaults if everything fits */ + first_mark = first_line ? '^' : '~'; + + tab8_len = 0; + + /* find the starting point */ + s = l->line_start; + e = s + l->line_len; + col8 = 0; + while (col8 < line_shift && (c = fy_utf8_get(s, (e - s), &w)) >= 0) { + if (fy_is_tab(c)) + col8 += 8 - (col8 % 8); + else + col8++; + s += w; + } + if (col8 > line_shift) + tab8_len = col8 - line_shift; /* the remaining of the tab */ + else + tab8_len = 0; + + /* start filling the row buffer */ + assert(rowbuf); + rbs = rowbuf; + rbe = rowbuf + rowbufsz; + + /* remaining tabs */ + while (tab8_len > 0) { + *rbs++ = ' '; + tab8_len--; + } + + /* go forward until end of line or cols */ + while (col8 < (line_shift + cols) && (c = fy_utf8_get(s, (e - s), &w)) >= 0 && rbs < rbe) { + if (fy_is_tab(c)) { + s++; + tab8_len = 8 - (col8 % 8); + col8 += tab8_len; + while (tab8_len > 0 && rbs < rbe) { + *rbs++ = ' '; + tab8_len--; + } + } else { + while (w > 0 && rbs < rbe) { + *rbs++ = *s++; + w--; + } + col8++; + } + } + display = rowbuf; + display_len = rbs - rowbuf; + + tilde_start = content_start_col - line_shift; + tilde_width = content_width; + if (tilde_start + tilde_width > cols) + tilde_width = cols - tilde_start; + if ((size_t)tilde_width >= rowbufsz) + tilde_width = rowbufsz - 1; /* guard */ + tilde_width_m1 = tilde_width > 0 ? (tilde_width - 1) : 0; + + /* output the line */ + fy_diag_write(diag, display, display_len); + + /* set the tildes */ + assert((int)rowbufsz > tilde_width_m1 + 1); + memset(rowbuf, '~', tilde_width_m1); + rowbuf[tilde_width_m1] = '\0'; + + fy_diag_printf(diag, "\n%*s%s%c%.*s%s\n", + tilde_start, "", + color_start, first_mark, tilde_width_m1, rowbuf, color_end); + + } + fy_atom_raw_line_iter_finish(&iter); + } +} + +void fy_diag_error_token_display(struct fy_diag *diag, enum fy_error_type type, struct fy_token *fyt) +{ + if (!diag || !fyt) + return; + + fy_diag_error_atom_display(diag, type, fy_token_atom(fyt)); +} + +void fy_diag_vreport(struct fy_diag *diag, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap) +{ + const char *name, *color_start = NULL, *color_end = NULL, *white = NULL; + char *msg_str = NULL, *name_str = NULL; + const struct fy_mark *start_mark; + int line, column; + struct fy_diag_errorp *errp; + struct fy_diag_error *err; + size_t spacesz, msgsz, filesz; + char *s; + + if (!diag || !fydrc || !fmt || !fydrc->fyt) + return; + + start_mark = fy_token_start_mark(fydrc->fyt); + + if (fydrc->has_override) { + name = fydrc->override_file; + line = fydrc->override_line; + column = fydrc->override_column; + } else { + name = fy_input_get_filename(fy_token_get_input(fydrc->fyt)); + line = start_mark->line + 1; + column = start_mark->column + 1; + } + + /* it will strip trailing newlines */ + msg_str = fy_vsprintfa(fmt, ap); + fy_strip_trailing_nl(msg_str); + + /* get the colors */ + fy_diag_get_error_colors(diag, fydrc->type, &color_start, &color_end, &white); + + if (name || (line > 0 && column > 0)) + name_str = (line > 0 && column > 0) ? + fy_sprintfa("%s%s:%d:%d: ", white, name, line, column) : + fy_sprintfa("%s%s: ", white, name); + + if (!diag->collect_errors) { + fy_diag_printf(diag, "%s" "%s%s: %s" "%s\n", + name_str ? : "", + color_start, fy_error_type_to_string(fydrc->type), color_end, + msg_str); + + fy_diag_error_token_display(diag, fydrc->type, fydrc->fyt); + + fy_token_unref(fydrc->fyt); + + } else if ((errp = malloc(sizeof(*errp))) != NULL) { + + msgsz = strlen(msg_str) + 1; + filesz = strlen(name) + 1; + spacesz = msgsz + filesz; + + errp->space = malloc(spacesz); + if (!errp->space) { + free(errp); + goto out; + } + s = errp->space; + + err = &errp->e; + memset(err, 0, sizeof(*err)); + err->type = fydrc->type; + err->module = fydrc->module; + err->fyt = fydrc->fyt; + err->msg = s; + memcpy(s, msg_str, msgsz); + s += msgsz; + err->file = s; + memcpy(s, name, filesz); + s += filesz; + err->line = line; + err->column = column; + + fy_diag_errorp_list_add_tail(&diag->errors, errp); + } +out: + if (!diag->on_error && fydrc->type == FYET_ERROR) + diag->on_error = true; +} + +void fy_diag_report(struct fy_diag *diag, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_diag_vreport(diag, fydrc, fmt, ap); + va_end(ap); +} + +void fy_diag_token_vreport(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *fmt, va_list ap) +{ + struct fy_diag_report_ctx drc; + bool save_on_error; + + if (!fyt || !diag) + return; + + save_on_error = diag->on_error; + diag->on_error = false; + + memset(&drc, 0, sizeof(drc)); + drc.type = type; + drc.module = FYEM_UNKNOWN; + drc.fyt = fyt; + fy_diag_vreport(diag, &drc, fmt, ap); + + diag->on_error = save_on_error; +} + +void fy_diag_token_report(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_diag_token_vreport(diag, fyt, type, fmt, ap); + va_end(ap); +} + +void fy_diag_token_override_vreport(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *file, + int line, int column, + const char *fmt, va_list ap) +{ + struct fy_diag_report_ctx drc; + bool save_on_error; + + if (!fyt || !diag) + return; + + save_on_error = diag->on_error; + diag->on_error = false; + + memset(&drc, 0, sizeof(drc)); + drc.type = type; + drc.module = FYEM_UNKNOWN; + drc.fyt = fyt; + drc.has_override = true; + drc.override_file = file; + drc.override_line = line; + drc.override_column = column; + fy_diag_vreport(diag, &drc, fmt, ap); + + diag->on_error = save_on_error; +} + +void fy_diag_token_override_report(struct fy_diag *diag, struct fy_token *fyt, + enum fy_error_type type, const char *file, + int line, int column, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_diag_token_override_vreport(diag, fyt, type, file, line, column, fmt, ap); + va_end(ap); +} + +void fy_diag_node_vreport(struct fy_diag *diag, struct fy_node *fyn, + enum fy_error_type type, const char *fmt, va_list ap) +{ + struct fy_diag_report_ctx drc; + bool save_on_error; + + if (!fyn || !diag) + return; + + save_on_error = diag->on_error; + diag->on_error = false; + + memset(&drc, 0, sizeof(drc)); + drc.type = type; + drc.module = FYEM_UNKNOWN; + drc.fyt = fy_node_token(fyn); + fy_diag_vreport(diag, &drc, fmt, ap); + + diag->on_error = save_on_error; +} + +void fy_diag_node_report(struct fy_diag *diag, struct fy_node *fyn, + enum fy_error_type type, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_diag_node_vreport(diag, fyn, type, fmt, ap); + va_end(ap); +} + +void fy_diag_node_override_vreport(struct fy_diag *diag, struct fy_node *fyn, + enum fy_error_type type, const char *file, + int line, int column, + const char *fmt, va_list ap) +{ + struct fy_diag_report_ctx drc; + bool save_on_error; + + if (!fyn || !diag) + return; + + save_on_error = diag->on_error; + diag->on_error = false; + + memset(&drc, 0, sizeof(drc)); + drc.type = type; + drc.module = FYEM_UNKNOWN; + drc.fyt = fy_node_token(fyn); + drc.has_override = true; + drc.override_file = file; + drc.override_line = line; + drc.override_column = column; + fy_diag_vreport(diag, &drc, fmt, ap); + + diag->on_error = save_on_error; +} + +void fy_diag_node_override_report(struct fy_diag *diag, struct fy_node *fyn, + enum fy_error_type type, const char *file, + int line, int column, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_diag_node_override_vreport(diag, fyn, type, file, line, column, fmt, ap); + va_end(ap); +} + +void fy_node_vreport(struct fy_node *fyn, enum fy_error_type type, + const char *fmt, va_list ap) +{ + if (!fyn || !fyn->fyd) + return; + + fy_diag_node_vreport(fyn->fyd->diag, fyn, type, fmt, ap); +} + +void fy_node_report(struct fy_node *fyn, enum fy_error_type type, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_node_vreport(fyn, type, fmt, ap); + va_end(ap); +} + +void fy_node_override_vreport(struct fy_node *fyn, enum fy_error_type type, + const char *file, int line, int column, + const char *fmt, va_list ap) +{ + if (!fyn || !fyn->fyd) + return; + + fy_diag_node_override_vreport(fyn->fyd->diag, fyn, type, + file, line, column, fmt, ap); +} + +void fy_node_override_report(struct fy_node *fyn, enum fy_error_type type, + const char *file, int line, int column, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_node_override_vreport(fyn, type, file, line, column, fmt, ap); + va_end(ap); +} + +void fy_diag_event_vreport(struct fy_diag *diag, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, va_list ap) +{ + struct fy_diag_report_ctx drc; + struct fy_token *fyt; + bool save_on_error; + + if (!fye || !diag) + return; + + save_on_error = diag->on_error; + diag->on_error = false; + + memset(&drc, 0, sizeof(drc)); + drc.type = type; + drc.module = FYEM_UNKNOWN; + drc.fyt = NULL; + switch (fyep) { + case FYEP_VALUE: + fyt = fy_event_get_token(fye); + break; + case FYEP_TAG: + fyt = fy_event_get_tag_token(fye); + break; + case FYEP_ANCHOR: + fyt = fy_event_get_anchor_token(fye); + break; + default: + fyt = NULL; + break; + } + drc.fyt = fy_token_ref(fyt); + fy_diag_vreport(diag, &drc, fmt, ap); + + diag->on_error = save_on_error; +} + +void fy_diag_event_report(struct fy_diag *diag, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_diag_event_vreport(diag, fye, fyep, type, fmt, ap); + va_end(ap); +} + +void fy_event_vreport(struct fy_parser *fyp, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, va_list ap) +{ + if (fye) + fy_diag_event_vreport(fyp->diag, fye, fyep, type, fmt, ap); + else + fy_parser_vlog(fyp, type, fmt, ap); +} + +void +fy_event_report(struct fy_parser *fyp, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_event_vreport(fyp, fye, fyep, type, fmt, ap); + va_end(ap); +} diff --git a/src/lib/fy-diag.h b/src/lib/fy-diag.h new file mode 100644 index 0000000..a37d356 --- /dev/null +++ b/src/lib/fy-diag.h @@ -0,0 +1,133 @@ +/* + * fy-diag.h - diagnostics + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_DIAG_H +#define FY_DIAG_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include + +#include "fy-utils.h" + +#include "fy-list.h" +#include "fy-token.h" + +/* error flags (above 0x100 is library specific) */ +#define FYEF_SOURCE 0x0001 +#define FYEF_POSITION 0x0002 +#define FYEF_TYPE 0x0004 +#define FYEF_USERSTART 0x0100 + +#define FYDF_LEVEL_SHIFT 0 +#define FYDF_LEVEL_MASK (0x0f << FYDF_LEVEL_SHIFT) +#define FYDF_LEVEL(x) (((unsigned int)(x) << FYDF_LEVEL_SHIFT) & FYDF_LEVEL_MASK) +#define FYDF_DEBUG FYDF_LEVEL(FYET_DEBUG) +#define FYDF_INFO FYDF_LEVEL(FYET_INFO) +#define FYDF_NOTICE FYDF_LEVEL(FYET_NOTICE) +#define FYDF_WARNING FYDF_LEVEL(FYET_WARNING) +#define FYDF_ERROR FYDF_LEVEL(FYET_ERROR) + +#define FYDF_MODULE_SHIFT 4 +#define FYDF_MODULE_MASK (0x0f << FYDF_MODULE_SHIFT) +#define FYDF_MODULE(x) (((unsigned int)(x) << FYDF_MODULE_SHIFT) & FYDF_MODULE_MASK) +#define FYDF_ATOM FYDF_MODULE(FYEM_ATOM) +#define FYDF_SCANNER FYDF_MODULE(FYEM_SCANNER) +#define FYDF_PARSER FYDF_MODULE(FYEM_PARSER) +#define FYDF_TREE FYDF_MODULE(FYEM_TREE) +#define FYDF_BUILDER FYDF_MODULE(FYEM_BUILDER) +#define FYDF_INTERNAL FYDF_MODULE(FYEM_INTERNAL) +#define FYDF_SYSTEM FYDF_MODULE(FYEM_SYSTEM) +#define FYDF_MODULE_USER_MASK 7 +#define FYDF_MODULE_USER(x) FYDF_MODULE(8 + ((x) & FYDF_MODULE_USER_MASK)) + +struct fy_diag_term_info { + int rows; + int columns; +}; + +struct fy_diag_report_ctx { + enum fy_error_type type; + enum fy_error_module module; + struct fy_token *fyt; + bool has_override; + const char *override_file; + int override_line; + int override_column; +}; + +FY_TYPE_FWD_DECL_LIST(diag_errorp); +struct fy_diag_errorp { + struct list_head node; + char *space; + struct fy_diag_error e; +}; +FY_TYPE_DECL_LIST(diag_errorp); + +struct fy_diag { + struct fy_diag_cfg cfg; + int refs; + bool on_error : 1; + bool destroyed : 1; + bool collect_errors : 1; + bool terminal_probed : 1; + struct fy_diag_term_info term_info; + struct fy_diag_errorp_list errors; +}; + +void fy_diag_free(struct fy_diag *diag); + +void fy_diag_vreport(struct fy_diag *diag, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap); +void fy_diag_report(struct fy_diag *diag, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +static inline bool +fy_diag_log_level_is_enabled(struct fy_diag *diag, enum fy_error_type level, enum fy_error_module module) +{ + if (!diag) + return false; + + /* check level */ + if ((unsigned int)level < FYET_MAX) { + + /* turn errors into debugs while not reset */ + if (level >= FYET_ERROR && diag->on_error) + level = FYET_DEBUG; + + if (level < diag->cfg.level) + return false; + } + + /* check module enable mask */ + if ((unsigned int)module < FYEM_MAX) { + if (!(diag->cfg.module_mask & FY_BIT(module))) + return false; + } + + /* ok, clear to generate */ + return true; +} + +void fy_diag_error_atom_display(struct fy_diag *diag, enum fy_error_type type, + struct fy_atom *atom); +void fy_diag_error_token_display(struct fy_diag *diag, enum fy_error_type type, + struct fy_token *fyt); + +void fy_diag_cfg_from_parser_flags(struct fy_diag_cfg *cfg, enum fy_parse_cfg_flags pflags); + +#endif diff --git a/src/lib/fy-doc-diag.c b/src/lib/fy-doc-diag.c new file mode 100644 index 0000000..04247e3 --- /dev/null +++ b/src/lib/fy-doc-diag.c @@ -0,0 +1,91 @@ +/* + * fy-doc-diag.c - document diagnostics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-diag.h" +#include "fy-doc.h" + +int fy_document_vdiag(struct fy_document *fyd, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap) +{ + struct fy_diag_ctx fydc; + int rc; + + if (!fyd || !fmt || !fyd->diag) + return -1; + + /* perform the enable tests early to avoid the overhead */ + if (((flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT) < fyd->diag->cfg.level) + return 0; + + /* fill in fy_diag_ctx */ + memset(&fydc, 0, sizeof(fydc)); + + fydc.level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; + fydc.module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; + fydc.source_file = file; + fydc.source_line = line; + fydc.source_func = func; + fydc.line = -1; + fydc.column = -1; + + rc = fy_vdiag(fyd->diag, &fydc, fmt, ap); + + return rc; +} + +int fy_document_diag(struct fy_document *fyd, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_document_vdiag(fyd, flags, file, line, func, fmt, ap); + va_end(ap); + + return rc; +} + +void fy_document_diag_vreport(struct fy_document *fyd, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap) +{ + if (!fyd || !fyd->diag || !fydrc || !fmt) + return; + + fy_diag_vreport(fyd->diag, fydrc, fmt, ap); +} + +void fy_document_diag_report(struct fy_document *fyd, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_document_diag_vreport(fyd, fydrc, fmt, ap); + va_end(ap); +} diff --git a/src/lib/fy-doc.c b/src/lib/fy-doc.c new file mode 100644 index 0000000..c3b10ff --- /dev/null +++ b/src/lib/fy-doc.c @@ -0,0 +1,7649 @@ +/* + * fy-doc.c - YAML document methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" +#include "fy-doc.h" + +#include "fy-utils.h" + +#include "xxhash.h" + +static const struct fy_hash_desc hd_anchor; +static const struct fy_hash_desc hd_nanchor; +static const struct fy_hash_desc hd_mapping; + +int fy_node_hash_uint(struct fy_node *fyn, unsigned int *hashp); + +static struct fy_node * +fy_node_by_path_internal(struct fy_node *fyn, + const char *path, size_t pathlen, + enum fy_node_walk_flags flags); + +#define FY_NODE_PATH_WALK_DEPTH_DEFAULT 16 + +static inline unsigned int +fy_node_walk_max_depth_from_flags(enum fy_node_walk_flags flags) +{ + unsigned int max_depth; + + max_depth = ((unsigned int)flags >> FYNWF_MAXDEPTH_SHIFT) & FYNWF_MAXDEPTH_MASK; + if (max_depth == 0) + max_depth = FY_NODE_PATH_WALK_DEPTH_DEFAULT; + + return max_depth; +} + +static inline unsigned int +fy_node_walk_marker_from_flags(enum fy_node_walk_flags flags) +{ + return ((unsigned int)flags >> FYNWF_MARKER_SHIFT) & FYNWF_MARKER_MASK; +} + +/* internal simple key to optimize string lookups */ +static inline bool is_simple_key(const char *str, size_t len) +{ + const char *s, *e; + char c; + + if (!str) + return false; + + if (len == (size_t)-1) + len = strlen(str); + + for (s = str, e = s + len; s < e; s++) { + + c = *s; + + /* note no isalpha() it's locale specific */ + if (!((c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || + (c == '_'))) + break; + } + + return s == e; +} + +static void fy_resolve_parent_node(struct fy_document *fyd, struct fy_node *fyn, struct fy_node *fyn_parent); + +void fy_anchor_destroy(struct fy_anchor *fya) +{ + if (!fya) + return; + fy_token_unref(fya->anchor); + free(fya); +} + +struct fy_anchor *fy_anchor_create(struct fy_document *fyd, + struct fy_node *fyn, struct fy_token *anchor) +{ + struct fy_anchor *fya = NULL; + + fya = malloc(sizeof(*fya)); + if (!fya) + return NULL; + + fya->fyn = fyn; + fya->anchor = anchor; + fya->multiple = false; + + return fya; +} + +struct fy_anchor *fy_document_anchor_iterate(struct fy_document *fyd, void **prevp) +{ + struct fy_anchor_list *fyal; + + if (!fyd || !prevp) + return NULL; + + fyal = &fyd->anchors; + + return *prevp = *prevp ? fy_anchor_next(fyal, *prevp) : fy_anchor_list_head(fyal); +} + +#define FYDSAF_COPY FY_BIT(0) +#define FYDSAF_MALLOCED FY_BIT(1) + +static int fy_document_set_anchor_internal(struct fy_document *fyd, struct fy_node *fyn, const char *text, size_t len, + unsigned int flags) +{ + const bool copy = !!(flags & FYDSAF_COPY); + const bool malloced = !!(flags & FYDSAF_MALLOCED); + struct fy_anchor *fya = NULL, *fyam = NULL; + struct fy_input *fyi = NULL; + struct fy_token *fyt = NULL; + struct fy_accel_entry *xle; + struct fy_atom handle; + char *data_copy = NULL; + const char *origtext; + size_t origlen; + int rc; + + if (!fyd || !fyn || fyn->fyd != fyd) + return -1; + + if (text && len == (size_t)-1) + len = strlen(text); + + fya = fy_document_lookup_anchor_by_node(fyd, fyn); + + if (!text) { + /* no anchor, and trying to delete? OK */ + if (fya) + return 0; + /* remove the anchor */ + fy_anchor_list_del(&fyd->anchors, fya); + + if (fy_document_is_accelerated(fyd)) { + xle = fy_accel_entry_lookup_key_value(fyd->axl, fya->anchor, fya); + fy_accel_entry_remove(fyd->axl, xle); + + xle = fy_accel_entry_lookup_key_value(fyd->naxl, fya->fyn, fya); + fy_accel_entry_remove(fyd->naxl, xle); + } + + fy_anchor_destroy(fya); + return 0; + } + + /* trying to add duplicate anchor */ + if (fya) { + origtext = fy_token_get_text(fya->anchor, &origlen); + fyd_error_check(fyd, origtext, err_out, + "fy_token_get_text() failed"); + + FYD_NODE_ERROR(fyd, fyn, FYEM_DOC, + "cannot set anchor %.*s (anchor %.*s already exists)", + (int)len, text, (int)origlen, origtext); + if (malloced && text) + free((void *)text); + fya = NULL; + goto err_out; + } + + if (copy) { + data_copy = malloc(len); + fyd_error_check(fyd, data_copy, err_out, + "malloc() failed"); + memcpy(data_copy, text, len); + } else if (malloced) + data_copy = (char *)text; + else + data_copy = NULL; + + if (data_copy) + fyi = fy_input_from_malloc_data(data_copy, len, &handle, true); + else + fyi = fy_input_from_data(text, len, &handle, true); + fyd_error_check(fyd, fyi, err_out, + "fy_input_from_data() failed"); + data_copy = NULL; + + /* it must not be something funky */ + if (!handle.valid_anchor) + goto err_out; + + fyt = fy_token_create(FYTT_ANCHOR, &handle); + if (!fyt) + goto err_out; + + fya = fy_anchor_create(fyd, fyn, fyt); + if (!fya) + goto err_out; + + fy_anchor_list_add(&fyd->anchors, fya); + if (fy_document_is_accelerated(fyd)) { + xle = fy_accel_entry_lookup(fyd->axl, fya->anchor); + if (xle) { + fyam = (void *)xle->value; + /* multiple */ + if (!fyam->multiple) + fyam->multiple = true; + fya->multiple = true; + + fyd_notice(fyd, "register anchor %.*s is multiple", (int)len, text); + } + + xle = fy_accel_entry_insert(fyd->axl, fya->anchor, fya); + fyd_error_check(fyd, xle, err_out, + "fy_accel_entry_insert() fyd->axl failed"); + } + + if (fy_document_is_accelerated(fyd)) { + rc = fy_accel_insert(fyd->naxl, fyn, fya); + fyd_error_check(fyd, !rc, err_out_rc, + "fy_accel_insert() fyd->naxl failed"); + } + + /* take away the input reference */ + fy_input_unref(fyi); + + return 0; +err_out: + rc = -1; +err_out_rc: + if (data_copy) + free(data_copy); + fy_anchor_destroy(fya); + fy_token_unref(fyt); + fy_input_unref(fyi); + fyd->diag->on_error = false; + return rc; +} + +int fy_document_set_anchor(struct fy_document *fyd, struct fy_node *fyn, const char *text, size_t len) +{ + return fy_document_set_anchor_internal(fyd, fyn, text, len, 0); +} + +int fy_node_set_anchor(struct fy_node *fyn, const char *text, size_t len) +{ + if (!fyn) + return -1; + return fy_document_set_anchor_internal(fyn->fyd, fyn, text, len, 0); +} + +int fy_node_set_anchor_copy(struct fy_node *fyn, const char *text, size_t len) +{ + if (!fyn) + return -1; + return fy_document_set_anchor_internal(fyn->fyd, fyn, text, len, FYDSAF_COPY); +} + +int fy_node_set_vanchorf(struct fy_node *fyn, const char *fmt, va_list ap) +{ + if (!fyn || !fmt) + return -1; + + return fy_document_set_anchor_internal(fyn->fyd, fyn, fy_vsprintfa(fmt, ap), FY_NT, FYDSAF_COPY); +} + +int fy_node_set_anchorf(struct fy_node *fyn, const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = fy_node_set_vanchorf(fyn, fmt, ap); + va_end(ap); + + return ret; +} + +int fy_node_remove_anchor(struct fy_node *fyn) +{ + return fy_node_set_anchor(fyn, NULL, 0); +} + +struct fy_anchor *fy_node_get_anchor(struct fy_node *fyn) +{ + if (!fyn) + return NULL; + + return fy_document_lookup_anchor_by_node(fyn->fyd, fyn); +} + +struct fy_anchor *fy_node_get_nearest_anchor(struct fy_node *fyn) +{ + struct fy_anchor *fya; + struct fy_node *fynt; + + while ((fya = fy_node_get_anchor(fyn)) == NULL && (fynt = fy_node_get_parent(fyn))) + fyn = fynt; + + return fya; +} + +struct fy_node *fy_node_get_nearest_child_of(struct fy_node *fyn_base, + struct fy_node *fyn) +{ + struct fy_node *fynp; + + if (!fyn) + return NULL; + if (!fyn_base) + fyn_base = fy_document_root(fy_node_document(fyn)); + if (!fyn_base) + return NULL; + + /* move up until we hit a node that's a child of fyn_base */ + fynp = fyn; + while (fyn && (fynp = fy_node_get_parent(fyn)) != NULL && fyn_base != fynp) + fyn = fynp; + + return fyn; +} + +void fy_parse_document_destroy(struct fy_parser *fyp, struct fy_document *fyd) +{ + struct fy_node *fyn; + + if (!fyd) + return; + + fy_document_cleanup_path_expr_data(fyd); + + fyn = fyd->root; + fyd->root = NULL; + fy_node_detach_and_free(fyn); + + fy_document_purge_anchors(fyd); + + fy_document_state_unref(fyd->fyds); + + fy_diag_unref(fyd->diag); + + free(fyd); +} + +struct fy_document *fy_parse_document_create(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_document *fyd = NULL; + struct fy_document_state *fyds; + struct fy_event *fye = NULL; + int rc; + + if (!fyp || !fyep) + return NULL; + + fye = &fyep->e; + + FYP_TOKEN_ERROR_CHECK(fyp, fy_event_get_token(fye), FYEM_DOC, + fye->type == FYET_DOCUMENT_START, err_out, + "invalid start of event stream"); + + fyd = malloc(sizeof(*fyd)); + fyp_error_check(fyp, fyd, err_out, + "malloc() failed"); + + memset(fyd, 0, sizeof(*fyd)); + + fyd->diag = fy_diag_ref(fyp->diag); + fyd->parse_cfg = fyp->cfg; + + fy_anchor_list_init(&fyd->anchors); + if (fy_document_can_be_accelerated(fyd)) { + fyd->axl = malloc(sizeof(*fyd->axl)); + fyp_error_check(fyp, fyd->axl, err_out, + "malloc() failed"); + + /* start with a very small bucket list */ + rc = fy_accel_setup(fyd->axl, &hd_anchor, fyd, 8); + fyp_error_check(fyp, !rc, err_out, + "fy_accel_setup() failed"); + + fyd->naxl = malloc(sizeof(*fyd->naxl)); + fyp_error_check(fyp, fyd->axl, err_out, + "malloc() failed"); + + /* start with a very small bucket list */ + rc = fy_accel_setup(fyd->naxl, &hd_nanchor, fyd, 8); + fyp_error_check(fyp, !rc, err_out, + "fy_accel_setup() failed"); + } + + fyd->root = NULL; + + fyds = fye->document_start.document_state; + fye->document_start.document_state = NULL; + + /* and we're done with this event */ + fy_parse_eventp_recycle(fyp, fyep); + + /* drop the old reference */ + fy_document_state_unref(fyd->fyds); + + /* note that we keep the reference */ + fyd->fyds = fyds; + + fy_document_list_init(&fyd->children); + + return fyd; + +err_out: + fy_parse_document_destroy(fyp, fyd); + fy_parse_eventp_recycle(fyp, fyep); + fyd->diag->on_error = false; + return NULL; +} + +const struct fy_parse_cfg *fy_document_get_cfg(struct fy_document *fyd) +{ + if (!fyd) + return NULL; + return &fyd->parse_cfg; +} + +struct fy_diag *fy_document_get_diag(struct fy_document *fyd) +{ + if (!fyd || !fyd->diag) + return NULL; + return fy_diag_ref(fyd->diag); +} + +int fy_document_set_diag(struct fy_document *fyd, struct fy_diag *diag) +{ + struct fy_diag_cfg dcfg; + + if (!fyd) + return -1; + + /* default? */ + if (!diag) { + fy_diag_cfg_default(&dcfg); + diag = fy_diag_create(&dcfg); + if (!diag) + return -1; + } + + fy_diag_unref(fyd->diag); + fyd->diag = fy_diag_ref(diag); + + return 0; +} + +struct fy_document *fy_node_document(struct fy_node *fyn) +{ + return fyn ? fyn->fyd : NULL; +} + +static inline struct fy_anchor * +fy_document_accel_lookup_anchor_by_token(struct fy_document *fyd, struct fy_token *fyt) +{ + assert(fyd); + assert(fyd->axl); + return (void *)fy_accel_lookup(fyd->axl, fyt); +} + +static inline struct fy_anchor * +fy_document_accel_lookup_anchor_by_node(struct fy_document *fyd, struct fy_node *fyn) +{ + assert(fyd); + assert(fyd->naxl); + return (void *)fy_accel_lookup(fyd->naxl, fyn); +} + +static inline struct fy_node_pair * +fy_node_accel_lookup_by_node(struct fy_node *fyn, struct fy_node *fyn_key) +{ + assert(fyn); + assert(fyn->xl); + + return (void *)fy_accel_lookup(fyn->xl, (const void *)fyn_key); +} + +struct fy_anchor * +fy_document_lookup_anchor(struct fy_document *fyd, const char *anchor, size_t len) +{ + struct fy_anchor *fya; + struct fy_anchor_list *fyal; + struct fy_input *fyi; + struct fy_atom handle; + struct fy_token *fyt; + const char *text; + size_t text_len; + + if (!fyd || !anchor) + return NULL; + + if (len == (size_t)-1) + len = strlen(anchor); + + if (fy_document_is_accelerated(fyd)) { + fyi = fy_input_from_data(anchor, len, &handle, true); + if (!fyi) + return NULL; + + fyt = fy_token_create(FYTT_ANCHOR, &handle); + + if (!fyt) { + fy_input_unref(fyi); + return NULL; + } + + fya = fy_document_accel_lookup_anchor_by_token(fyd, fyt); + + fy_input_unref(fyi); + fy_token_unref(fyt); + + if (!fya) + return NULL; + + /* single anchor? return it */ + if (!fya->multiple) + return fya; + + /* multiple anchors, fall-through */ + } + + /* note that we're performing the lookup in reverse creation order + * so that we pick the most recent + */ + fyal = &fyd->anchors; + for (fya = fy_anchor_list_tail(fyal); fya; fya = fy_anchor_prev(fyal, fya)) { + text = fy_anchor_get_text(fya, &text_len); + if (!text) + return NULL; + + if (len == text_len && !memcmp(anchor, text, len)) + return fya; + } + + return NULL; +} + +struct fy_anchor * +fy_document_lookup_anchor_by_token(struct fy_document *fyd, + struct fy_token *anchor) +{ + struct fy_anchor *fya, *fya_found, *fya_found2; + struct fy_anchor_list *fyal; + const char *anchor_text, *text; + size_t anchor_len, text_len; + int count; + + if (!fyd || !anchor) + return NULL; + + /* first try direct match (it's faster and the common case) */ + if (fy_document_is_accelerated(fyd)) { + fya = fy_document_accel_lookup_anchor_by_token(fyd, anchor); + if (!fya) + return NULL; + + /* single anchor? return it */ + if (!fya->multiple) + return fya; + + /* multiple anchors, fall-through */ + } + + anchor_text = fy_token_get_text(anchor, &anchor_len); + if (!anchor_text) + return NULL; + + fyal = &fyd->anchors; + + /* first pass, try with a single match */ + count = 0; + fya_found = NULL; + for (fya = fy_anchor_list_head(fyal); fya; fya = fy_anchor_next(fyal, fya)) { + text = fy_anchor_get_text(fya, &text_len); + if (!text) + return NULL; + + if (anchor_len == text_len && !memcmp(anchor_text, text, anchor_len)) { + count++; + fya_found = fya; + } + } + + /* not found */ + if (!count) + return NULL; + + /* single one? fine */ + if (count == 1) + return fya_found; + + /* multiple ones, must pick the one that's the last one before + * the requesting token */ + /* fyd_notice(fyd, "multiple anchors for %.*s", (int)anchor_len, anchor_text); */ + + /* only try the ones on the same input + * we don't try to cover the case where the label is referenced + * by other constructed documents + */ + fya_found2 = NULL; + for (fya = fy_anchor_list_head(fyal); fya; fya = fy_anchor_next(fyal, fya)) { + + /* only on the same input */ + if (fy_token_get_input(fya->anchor) != fy_token_get_input(anchor)) + continue; + + text = fy_anchor_get_text(fya, &text_len); + if (!text) + return NULL; + + if (anchor_len == text_len && !memcmp(anchor_text, text, anchor_len) && + fy_token_start_pos(fya->anchor) < fy_token_start_pos(anchor)) { + fya_found2 = fya; + } + } + + /* just return the one find earlier */ + if (!fya_found2) + return fya_found; + + /* return the one that was the latest */ + return fya_found2; +} + +struct fy_anchor *fy_document_lookup_anchor_by_node(struct fy_document *fyd, struct fy_node *fyn) +{ + struct fy_anchor *fya; + struct fy_anchor_list *fyal; + + if (!fyd || !fyn) + return NULL; + + if (fy_document_is_accelerated(fyd)) { + fya = fy_document_accel_lookup_anchor_by_node(fyd, fyn); + } else { + fyal = &fyd->anchors; + for (fya = fy_anchor_list_head(fyal); fya; fya = fy_anchor_next(fyal, fya)) { + if (fya->fyn == fyn) + break; + } + } + + return fya; +} + +const char *fy_anchor_get_text(struct fy_anchor *fya, size_t *lenp) +{ + if (!fya || !lenp) + return NULL; + return fy_token_get_text(fya->anchor, lenp); +} + +struct fy_node *fy_anchor_node(struct fy_anchor *fya) +{ + if (!fya) + return NULL; + return fya->fyn; +} + +int fy_node_pair_free(struct fy_node_pair *fynp) +{ + int rc, rc_ret = 0; + + if (!fynp) + return 0; + + rc = fy_node_free(fynp->key); + if (rc) + rc_ret = -1; + rc = fy_node_free(fynp->value); + if (rc) + rc_ret = -1; + + free(fynp); + + return rc_ret; +} + +void fy_node_pair_detach_and_free(struct fy_node_pair *fynp) +{ + if (!fynp) + return; + + fy_node_detach_and_free(fynp->key); + fy_node_detach_and_free(fynp->value); + free(fynp); +} + +struct fy_node_pair *fy_node_pair_alloc(struct fy_document *fyd) +{ + struct fy_node_pair *fynp = NULL; + + fynp = malloc(sizeof(*fynp)); + if (!fynp) + return NULL; + + fynp->key = NULL; + fynp->value = NULL; + fynp->fyd = fyd; + fynp->parent = NULL; + return fynp; +} + +int fy_node_free(struct fy_node *fyn) +{ + struct fy_document *fyd; + struct fy_node *fyni; + struct fy_node_pair *fynp; + struct fy_anchor *fya, *fyan; + struct fy_accel_entry_iter xli; + struct fy_accel_entry *xle, *xlen; + + if (!fyn) + return 0; + + /* a document must exist */ + fyd = fyn->fyd; + if (!fyd) + return -1; + + if (fyn->attached) + return -1; + + if (fy_document_is_accelerated(fyd)) { + for (xle = fy_accel_entry_iter_start(&xli, fyd->naxl, fyn); + xle; xle = xlen) { + xlen = fy_accel_entry_iter_next(&xli); + + fya = (void *)xle->value; + + fy_anchor_list_del(&fyd->anchors, fya); + + xle = fy_accel_entry_lookup_key_value(fyd->axl, fya->anchor, fya); + fy_accel_entry_remove(fyd->axl, xle); + + xle = fy_accel_entry_lookup_key_value(fyd->naxl, fya->fyn, fya); + fy_accel_entry_remove(fyd->naxl, xle); + + fy_anchor_destroy(fya); + } + fy_accel_entry_iter_finish(&xli); + } else { + /* remove anchors that are located on this node */ + for (fya = fy_anchor_list_head(&fyd->anchors); fya; fya = fyan) { + fyan = fy_anchor_next(&fyd->anchors, fya); + if (fya->fyn == fyn) { + fy_anchor_list_del(&fyd->anchors, fya); + fy_anchor_destroy(fya); + } + } + } + + /* clear the meta data of this node */ + fy_node_clear_meta(fyn); + + fy_token_unref(fyn->tag); + fyn->tag = NULL; + switch (fyn->type) { + case FYNT_SCALAR: + fy_token_unref(fyn->scalar); + fyn->scalar = NULL; + break; + case FYNT_SEQUENCE: + while ((fyni = fy_node_list_pop(&fyn->sequence)) != NULL) + fy_node_detach_and_free(fyni); + fy_token_unref(fyn->sequence_start); + fy_token_unref(fyn->sequence_end); + fyn->sequence_start = NULL; + fyn->sequence_end = NULL; + break; + case FYNT_MAPPING: + while ((fynp = fy_node_pair_list_pop(&fyn->mapping)) != NULL) { + if (fyn->xl) + fy_accel_remove(fyn->xl, fynp->key); + fy_node_pair_detach_and_free(fynp); + } + fy_token_unref(fyn->mapping_start); + fy_token_unref(fyn->mapping_end); + fyn->mapping_start = NULL; + fyn->mapping_end = NULL; + break; + } + + if (fyn->xl) { + fy_accel_cleanup(fyn->xl); + free(fyn->xl); + } + + fy_node_cleanup_path_expr_data(fyn); + + free(fyn); + + return 0; +} + +void fy_node_detach_and_free(struct fy_node *fyn) +{ + int rc __FY_DEBUG_UNUSED__; + + if (!fyn || !fyn->fyd) + return; + + fyn->attached = false; + + /* it must always succeed */ + rc = fy_node_free(fyn); + assert(!rc); +} + +struct fy_node *fy_node_alloc(struct fy_document *fyd, enum fy_node_type type) +{ + struct fy_node *fyn = NULL; + int rc; + + fyn = malloc(sizeof(*fyn)); + if (!fyn) + return NULL; + + memset(fyn, 0, sizeof(*fyn)); + + fyn->style = FYNS_ANY; + fyn->fyd = fyd; + fyn->type = type; + + switch (fyn->type) { + case FYNT_SCALAR: + break; + + case FYNT_SEQUENCE: + fy_node_list_init(&fyn->sequence); + break; + case FYNT_MAPPING: + fy_node_pair_list_init(&fyn->mapping); + + if (fy_document_is_accelerated(fyd)) { + fyn->xl = malloc(sizeof(*fyn->xl)); + fyd_error_check(fyd, fyn->xl, err_out, + "malloc() failed"); + + /* start with a very small bucket list */ + rc = fy_accel_setup(fyn->xl, &hd_mapping, fyd, 8); + fyd_error_check(fyd, !rc, err_out, + "fy_accel_setup() failed"); + } + break; + } + return fyn; + +err_out: + if (fyn) { + if (fyn->xl) { + fy_accel_cleanup(fyn->xl); + free(fyn->xl); + } + free(fyn); + } + return NULL; +} + +struct fy_token *fy_node_non_synthesized_token(struct fy_node *fyn) +{ + struct fy_token *fyt_start = NULL, *fyt_end = NULL; + struct fy_token *fyt; + struct fy_input *fyi; + struct fy_atom handle; + unsigned int aflags; + const char *s, *e; + size_t size; + + if (!fyn) + return NULL; + + fyi = fy_node_get_input(fyn); + if (!fyi) + return NULL; + + switch (fyn->type) { + case FYNT_SCALAR: + return fy_token_ref(fyn->scalar); + + case FYNT_SEQUENCE: + fyt_start = fyn->sequence_start; + fyt_end = fyn->sequence_end; + break; + + case FYNT_MAPPING: + fyt_start = fyn->mapping_start; + fyt_end = fyn->mapping_end; + break; + + } + + if (!fyt_start || !fyt_end) + return NULL; + + s = fy_input_start(fyi) + fyt_start->handle.start_mark.input_pos; + e = fy_input_start(fyi) + fyt_end->handle.end_mark.input_pos; + size = (size_t)(e - s); + + if (size > 0) + aflags = fy_analyze_scalar_content(s, size, + fy_token_atom_json_mode(fyt_start), + fy_token_atom_lb_mode(fyt_start), + fy_token_atom_flow_ws_mode(fyt_start)); + else + aflags = FYACF_EMPTY | FYACF_FLOW_PLAIN | FYACF_BLOCK_PLAIN; + + memset(&handle, 0, sizeof(handle)); + handle.start_mark = fyt_start->handle.start_mark; + handle.end_mark = fyt_end->handle.end_mark; + + /* if it's plain, all is good */ + if (aflags & FYACF_FLOW_PLAIN) { + handle.storage_hint = size; /* maximum */ + handle.storage_hint_valid = false; + handle.direct_output = !!(aflags & FYACF_JSON_ESCAPE); /* direct only when no json escape */ + handle.style = FYAS_PLAIN; + } else { + handle.storage_hint = 0; /* just calculate */ + handle.storage_hint_valid = false; + handle.direct_output = false; + handle.style = FYAS_DOUBLE_QUOTED_MANUAL; + } + handle.empty = !!(aflags & FYACF_EMPTY); + handle.has_lb = !!(aflags & FYACF_LB); + handle.has_ws = !!(aflags & FYACF_WS); + handle.starts_with_ws = !!(aflags & FYACF_STARTS_WITH_WS); + handle.starts_with_lb = !!(aflags & FYACF_STARTS_WITH_LB); + handle.ends_with_ws = !!(aflags & FYACF_ENDS_WITH_WS); + handle.ends_with_lb = !!(aflags & FYACF_ENDS_WITH_LB); + handle.trailing_lb = !!(aflags & FYACF_TRAILING_LB); + handle.size0 = !!(aflags & FYACF_SIZE0); + handle.valid_anchor = !!(aflags & FYACF_VALID_ANCHOR); + handle.json_mode = false; /* always false */ + handle.lb_mode = fylb_cr_nl; /* always \r\n */ + handle.fws_mode = fyfws_space_tab; /* always space + tab */ + handle.directive0_mode = false; + + handle.chomp = FYAC_STRIP; + handle.increment = 0; + handle.fyi = fyi; + handle.tabsize = 0; + + fyt = fy_token_create(FYTT_INPUT_MARKER, &handle); + if (!fyt) + return NULL; + + return fyt; +} + +struct fy_token *fy_node_token(struct fy_node *fyn) +{ + struct fy_atom atom; + struct fy_input *fyi = NULL; + struct fy_token *fyt = NULL; + char *buf = NULL; + + if (!fyn) + return NULL; + + /* if it's non synthetic we can use the node extends */ + if (!fy_node_is_synthetic(fyn)) + return fy_node_non_synthesized_token(fyn); + + /* emit to a string and create the token there */ + buf = fy_emit_node_to_string(fyn, FYECF_MODE_FLOW_ONELINE | FYECF_WIDTH_INF); + if (!buf) + goto err_out; + + fyi = fy_input_from_malloc_data(buf, FY_NT, &atom, true); + if (!fyi) + goto err_out; + + fyt = fy_token_create(FYTT_INPUT_MARKER, &atom); + if (!fyt) + goto err_out; + + /* take away the input reference */ + fy_input_unref(fyi); + + return fyt; + +err_out: + fy_input_unref(fyi); + if (buf) + free(buf); + return NULL; +} + +bool fy_node_uses_single_input_only(struct fy_node *fyn, struct fy_input *fyi) +{ + struct fy_node *fyni; + struct fy_node_pair *fynp; + + if (!fyn || !fyi) + return false; + + switch (fyn->type) { + case FYNT_SCALAR: + return fy_token_get_input(fyn->scalar) == fyi; + + case FYNT_SEQUENCE: + if (fy_token_get_input(fyn->sequence_start) != fyi) + return false; + + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) { + + if (!fy_node_uses_single_input_only(fyni, fyi)) + return false; + } + + if (fy_token_get_input(fyn->sequence_end) != fyi) + return false; + break; + + case FYNT_MAPPING: + if (fy_token_get_input(fyn->mapping_start) != fyi) + return false; + + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; + fynp = fy_node_pair_next(&fyn->mapping, fynp)) { + + if (fynp->key && !fy_node_uses_single_input_only(fynp->key, fyi)) + return false; + + if (fynp->value && !fy_node_uses_single_input_only(fynp->value, fyi)) + return false; + } + + if (fy_token_get_input(fyn->mapping_end) != fyi) + return false; + + break; + } + + return true; +} + +struct fy_input *fy_node_get_first_input(struct fy_node *fyn) +{ + if (!fyn) + return NULL; + + switch (fyn->type) { + case FYNT_SCALAR: + return fy_token_get_input(fyn->scalar); + + case FYNT_SEQUENCE: + return fy_token_get_input(fyn->sequence_start); + + case FYNT_MAPPING: + return fy_token_get_input(fyn->mapping_start); + } + + /* should never happen */ + return NULL; +} + +/* a node is synthetic if any of it's tokens reside in + * different inputs, or any sequence/mapping has been + * created via the manual sequence/mapping creation methods + */ +bool fy_node_is_synthetic(struct fy_node *fyn) +{ + return fyn && fyn->synthetic; +} + +/* map this node and all of it's parents synthetic */ +void fy_node_mark_synthetic(struct fy_node *fyn) +{ + if (!fyn) + return; + fyn->synthetic = true; + while ((fyn = fy_node_get_document_parent(fyn)) != NULL) + fyn->synthetic = true; +} + +struct fy_input *fy_node_get_input(struct fy_node *fyn) +{ + struct fy_input *fyi = NULL; + + fyi = fy_node_get_first_input(fyn); + if (!fyi) + return NULL; + + return fy_node_uses_single_input_only(fyn, fyi) ? fyi : NULL; +} + +int fy_document_register_anchor(struct fy_document *fyd, + struct fy_node *fyn, struct fy_token *anchor) +{ + struct fy_anchor *fya, *fyam; + struct fy_accel_entry *xle; + const char *text; + size_t text_len; + int rc; + + fya = fy_anchor_create(fyd, fyn, anchor); + fyd_error_check(fyd, fya, err_out, + "fy_anchor_create() failed"); + + fy_anchor_list_add_tail(&fyd->anchors, fya); + if (fy_document_is_accelerated(fyd)) { + xle = fy_accel_entry_lookup(fyd->axl, fya->anchor); + if (xle) { + fyam = (void *)xle->value; + /* multiple */ + if (!fyam->multiple) + fyam->multiple = true; + fya->multiple = true; + + text = fy_anchor_get_text(fya, &text_len); + fyd_notice(fyd, "register anchor %.*s is multiple", (int)text_len, text); + } + + xle = fy_accel_entry_insert(fyd->axl, fya->anchor, fya); + fyd_error_check(fyd, xle, err_out, + "fy_accel_entry_insert() fyd->axl failed"); + } + + if (fy_document_is_accelerated(fyd)) { + rc = fy_accel_insert(fyd->naxl, fyn, fya); + fyd_error_check(fyd, !rc, err_out_rc, + "fy_accel_insert() fyd->naxl failed"); + } + + return 0; + +err_out: + rc = -1; +err_out_rc: + fyd->diag->on_error = false; + return rc; +} + +struct fy_node_cmp_arg { + fy_node_scalar_compare_fn cmp_fn; + void *arg; +}; + +static int fy_node_scalar_cmp_default(struct fy_node *fyn_a, + struct fy_node *fyn_b, + void *arg); + +static int fy_node_mapping_sort_cmp_default(const struct fy_node_pair *fynp_a, + const struct fy_node_pair *fynp_b, + void *arg); + +bool fy_node_compare_user(struct fy_node *fyn1, struct fy_node *fyn2, + fy_node_mapping_sort_fn sort_fn, void *sort_fn_arg, + fy_node_scalar_compare_fn cmp_fn, void *cmp_fn_arg) +{ + struct fy_node *fyni1, *fyni2; + struct fy_node_pair *fynp1, *fynp2; + bool ret, null1, null2; + struct fy_node_pair **fynpp1, **fynpp2; + int i, count1, count2; + bool alias1, alias2; + struct fy_node_cmp_arg def_arg; + + if (!cmp_fn) { + cmp_fn = fy_node_scalar_cmp_default; + cmp_fn_arg = NULL; + } + if (!sort_fn) { + sort_fn = fy_node_mapping_sort_cmp_default; + def_arg.cmp_fn = cmp_fn; + def_arg.arg = cmp_fn_arg; + sort_fn_arg = &def_arg; + } else { + def_arg.cmp_fn = NULL; + def_arg.arg = NULL; + } + /* equal pointers? */ + if (fyn1 == fyn2) + return true; + + null1 = !fyn1 || (fyn1->type == FYNT_SCALAR && fy_token_get_text_length(fyn1->scalar) == 0); + null2 = !fyn2 || (fyn2->type == FYNT_SCALAR && fy_token_get_text_length(fyn2->scalar) == 0); + + /* both null */ + if (null1 && null2) + return true; + + /* either is NULL, no match */ + if (null1 || null2) + return false; + + /* types must match */ + if (fyn1->type != fyn2->type) + return false; + + ret = true; + + switch (fyn1->type) { + case FYNT_SEQUENCE: + fyni1 = fy_node_list_head(&fyn1->sequence); + fyni2 = fy_node_list_head(&fyn2->sequence); + while (fyni1 && fyni2) { + + ret = fy_node_compare_user(fyni1, fyni2, sort_fn, sort_fn_arg, cmp_fn, cmp_fn_arg); + if (!ret) + break; + + fyni1 = fy_node_next(&fyn1->sequence, fyni1); + fyni2 = fy_node_next(&fyn2->sequence, fyni2); + } + if (ret && fyni1 != fyni2 && (!fyni1 || !fyni2)) + ret = false; + + break; + + case FYNT_MAPPING: + count1 = fy_node_mapping_item_count(fyn1); + count2 = fy_node_mapping_item_count(fyn2); + + /* mapping counts must match */ + if (count1 != count2) { + ret = false; + break; + } + + fynpp1 = alloca(sizeof(*fynpp1) * (count1 + 1)); + fy_node_mapping_fill_array(fyn1, fynpp1, count1); + fy_node_mapping_perform_sort(fyn1, sort_fn, sort_fn_arg, fynpp1, count1); + + fynpp2 = alloca(sizeof(*fynpp2) * (count2 + 1)); + fy_node_mapping_fill_array(fyn2, fynpp2, count2); + fy_node_mapping_perform_sort(fyn2, sort_fn, sort_fn_arg, fynpp2, count2); + + for (i = 0; i < count1; i++) { + fynp1 = fynpp1[i]; + fynp2 = fynpp2[i]; + + ret = fy_node_compare_user(fynp1->key, fynp2->key, sort_fn, sort_fn_arg, cmp_fn, cmp_fn_arg); + if (!ret) + break; + + ret = fy_node_compare_user(fynp1->value, fynp2->value, sort_fn, sort_fn_arg, cmp_fn, cmp_fn_arg); + if (!ret) + break; + } + if (i >= count1) + ret = true; + + break; + + case FYNT_SCALAR: + alias1 = fy_node_is_alias(fyn1); + alias2 = fy_node_is_alias(fyn2); + + /* either both must be aliases or both not */ + if (alias1 != alias2) + return false; + + ret = !cmp_fn(fyn1, fyn2, cmp_fn_arg); + break; + } + + return ret; +} + +bool fy_node_compare(struct fy_node *fyn1, struct fy_node *fyn2) +{ + return fy_node_compare_user(fyn1, fyn2, NULL, NULL, NULL, NULL); +} + +bool fy_node_compare_string(struct fy_node *fyn, const char *str, size_t len) +{ + struct fy_document *fyd = NULL; + bool ret; + + fyd = fy_document_build_from_string(NULL, str, len); + if (!fyd) + return false; + + ret = fy_node_compare(fyn, fy_document_root(fyd)); + + fy_document_destroy(fyd); + + return ret; +} + +bool fy_node_compare_token(struct fy_node *fyn, struct fy_token *fyt) +{ + /* check if there's NULL */ + if (!fyn || !fyt) + return false; + + /* only valid for scalars */ + if (!fy_node_is_scalar(fyn) || fyt->type != FYTT_SCALAR) + return false; + + return fy_token_cmp(fyn->scalar, fyt) == 0; +} + +bool fy_node_compare_text(struct fy_node *fyn, const char *text, size_t len) +{ + const char *textn; + size_t lenn; + + if (!fyn || !text) + return false; + + textn = fy_node_get_scalar(fyn, &lenn); + if (!textn) + return false; + + if (len == FY_NT) + len = strlen(text); + + if (len != lenn) + return false; + + return memcmp(text, textn, len) == 0; +} + +struct fy_node_pair *fy_node_mapping_lookup_pair(struct fy_node *fyn, struct fy_node *fyn_key) +{ + struct fy_node_pair *fynpi, *fynp; + + /* sanity check */ + if (!fy_node_is_mapping(fyn)) + return NULL; + + fynp = NULL; + + + if (fyn->xl) { + fynp = fy_node_accel_lookup_by_node(fyn, fyn_key); + } else { + for (fynpi = fy_node_pair_list_head(&fyn->mapping); fynpi; + fynpi = fy_node_pair_next(&fyn->mapping, fynpi)) { + if (fy_node_compare(fynpi->key, fyn_key)) { + fynp = fynpi; + break; + } + } + } + + return fynp; +} + +int fy_node_mapping_get_pair_index(struct fy_node *fyn, const struct fy_node_pair *fynp) +{ + struct fy_node_pair *fynpi; + int i; + + if (!fy_node_is_mapping(fyn)) + return -1; + + for (i = 0, fynpi = fy_node_pair_list_head(&fyn->mapping); fynpi; + fynpi = fy_node_pair_next(&fyn->mapping, fynpi), i++) { + + if (fynpi == fynp) + return i; + } + + return -1; +} + +bool fy_node_mapping_key_is_duplicate(struct fy_node *fyn, struct fy_node *fyn_key) +{ + return fy_node_mapping_lookup_pair(fyn, fyn_key) != NULL; +} + +static int +fy_parse_document_load_node(struct fy_parser *fyp, struct fy_document *fyd, + struct fy_eventp *fyep, struct fy_node **fynp, + int *depthp); + +int fy_parse_document_load_alias(struct fy_parser *fyp, struct fy_document *fyd, struct fy_eventp *fyep, struct fy_node **fynp) +{ + *fynp = NULL; + + fyp_doc_debug(fyp, "in %s", __func__); + + /* TODO verify aliases etc */ + fy_parse_eventp_recycle(fyp, fyep); + return 0; +} + +static int +fy_parse_document_load_scalar(struct fy_parser *fyp, struct fy_document *fyd, + struct fy_eventp *fyep, struct fy_node **fynp, + int *depthp) +{ + struct fy_node *fyn = NULL; + struct fy_event *fye; + int rc; + + if (!fyd) + return -1; + + fyp_error_check(fyp, fyep || !fyp->stream_error, err_out, + "no event to process"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 0, FYEM_DOC, + fyep, err_out, + "premature end of event stream"); + + fyp_doc_debug(fyp, "in %s [%s]", __func__, fy_event_type_txt[fyep->e.type]); + + *fynp = NULL; + + fye = &fyep->e; + + /* we don't free nodes that often, so no need for recycling */ + fyn = fy_node_alloc(fyd, FYNT_SCALAR); + fyp_error_check(fyp, fyn, err_out, + "fy_node_alloc() failed"); + + if (fye->type == FYET_SCALAR) { + + /* move the tags and value to the node */ + if (fye->scalar.value) + fyn->style = fy_node_style_from_scalar_style(fye->scalar.value->scalar.style); + else + fyn->style = FYNS_PLAIN; + fyn->tag = fye->scalar.tag; + fye->scalar.tag = NULL; + + fyn->scalar = fye->scalar.value; + fye->scalar.value = NULL; + + if (fye->scalar.anchor) { + rc = fy_document_register_anchor(fyd, fyn, fye->scalar.anchor); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_document_register_anchor() failed"); + fye->scalar.anchor = NULL; + } + + } else if (fye->type == FYET_ALIAS) { + fyn->style = FYNS_ALIAS; + fyn->scalar = fye->alias.anchor; + fye->alias.anchor = NULL; + } else + FY_IMPOSSIBLE_ABORT(); + + *fynp = fyn; + fyn = NULL; + + /* everything OK */ + fy_parse_eventp_recycle(fyp, fyep); + return 0; + +err_out: + rc = -1; +err_out_rc: + fy_parse_eventp_recycle(fyp, fyep); + fyd->diag->on_error = false; + return rc; +} + +static int +fy_parse_document_load_sequence(struct fy_parser *fyp, struct fy_document *fyd, + struct fy_eventp *fyep, struct fy_node **fynp, + int *depthp) +{ + struct fy_node *fyn = NULL, *fyn_item = NULL; + struct fy_event *fye = NULL; + struct fy_token *fyt_ss = NULL; + int rc; + + fyp_error_check(fyp, fyep || !fyp->stream_error, err_out, + "no event to process"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 0, FYEM_DOC, + fyep, err_out, + "premature end of event stream"); + + fyp_doc_debug(fyp, "in %s [%s]", __func__, fy_event_type_txt[fyep->e.type]); + + *fynp = NULL; + + fye = &fyep->e; + + fyt_ss = fye->sequence_start.sequence_start; + + /* we don't free nodes that often, so no need for recycling */ + fyn = fy_node_alloc(fyd, FYNT_SEQUENCE); + fyp_error_check(fyp, fyn, err_out, + "fy_node_alloc() failed"); + + fyn->style = fyt_ss && fyt_ss->type == FYTT_FLOW_SEQUENCE_START ? FYNS_FLOW : FYNS_BLOCK; + + fyn->tag = fye->sequence_start.tag; + fye->sequence_start.tag = NULL; + + if (fye->sequence_start.anchor) { + rc = fy_document_register_anchor(fyd, fyn, fye->sequence_start.anchor); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_document_register_anchor() failed"); + fye->sequence_start.anchor = NULL; + } + + if (fye->sequence_start.sequence_start) { + fyn->sequence_start = fye->sequence_start.sequence_start; + fye->sequence_start.sequence_start = NULL; + } else + fyn->sequence_start = NULL; + + assert(fyn->sequence_start); + + /* done with this */ + fy_parse_eventp_recycle(fyp, fyep); + fyep = NULL; + + while ((fyep = fy_parse_private(fyp)) != NULL) { + fye = &fyep->e; + if (fye->type == FYET_SEQUENCE_END) + break; + + rc = fy_parse_document_load_node(fyp, fyd, fyep, &fyn_item, depthp); + fyep = NULL; + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_document_load_node() failed"); + + fy_node_list_add_tail(&fyn->sequence, fyn_item); + fyn_item->attached = true; + fyn_item = NULL; + } + + if (!fyep) + goto err_out; + + if (fye->sequence_end.sequence_end) { + fyn->sequence_end = fye->sequence_end.sequence_end; + fye->sequence_end.sequence_end = NULL; + } else + fyn->sequence_end = NULL; + + assert(fyn->sequence_end); + + *fynp = fyn; + fyn = NULL; + + fy_parse_eventp_recycle(fyp, fyep); + return 0; + + /* fallthrough */ +err_out: + rc = -1; +err_out_rc: + fy_parse_eventp_recycle(fyp, fyep); + fy_node_detach_and_free(fyn_item); + fy_node_detach_and_free(fyn); + return rc; +} + +static int +fy_parse_document_load_mapping(struct fy_parser *fyp, struct fy_document *fyd, + struct fy_eventp *fyep, struct fy_node **fynp, + int *depthp) +{ + struct fy_node *fyn = NULL, *fyn_key = NULL, *fyn_value = NULL; + struct fy_node_pair *fynp_item = NULL; + struct fy_event *fye = NULL; + struct fy_token *fyt_ms = NULL; + bool duplicate; + int rc; + + fyp_error_check(fyp, fyep || !fyp->stream_error, err_out, + "no event to process"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 0, FYEM_DOC, + fyep, err_out, + "premature end of event stream"); + + fyp_doc_debug(fyp, "in %s [%s]", __func__, fy_event_type_txt[fyep->e.type]); + + *fynp = NULL; + + fye = &fyep->e; + + fyt_ms = fye->mapping_start.mapping_start; + + /* we don't free nodes that often, so no need for recycling */ + fyn = fy_node_alloc(fyd, FYNT_MAPPING); + fyp_error_check(fyp, fyn, err_out, + "fy_node_alloc() failed"); + + fyn->style = fyt_ms && fyt_ms->type == FYTT_FLOW_MAPPING_START ? FYNS_FLOW : FYNS_BLOCK; + + fyn->tag = fye->mapping_start.tag; + fye->mapping_start.tag = NULL; + + if (fye->mapping_start.anchor) { + rc = fy_document_register_anchor(fyd, fyn, fye->mapping_start.anchor); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_document_register_anchor() failed"); + fye->mapping_start.anchor = NULL; + } + + if (fye->mapping_start.mapping_start) { + fyn->mapping_start = fye->mapping_start.mapping_start; + fye->mapping_start.mapping_start = NULL; + } + + assert(fyn->mapping_start); + + /* done with this */ + fy_parse_eventp_recycle(fyp, fyep); + fyep = NULL; + + while ((fyep = fy_parse_private(fyp)) != NULL) { + fye = &fyep->e; + if (fye->type == FYET_MAPPING_END) + break; + + fynp_item = fy_node_pair_alloc(fyd); + fyp_error_check(fyp, fynp_item, err_out, + "fy_node_pair_alloc() failed"); + + fyn_key = NULL; + fyn_value = NULL; + + rc = fy_parse_document_load_node(fyp, fyd, + fyep, &fyn_key, depthp); + fyep = NULL; + + assert(fyn_key); + + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_document_load_node() failed"); + + /* if we don't allow duplicate keys */ + if (!(fyd->parse_cfg.flags & FYPCF_ALLOW_DUPLICATE_KEYS)) { + + /* make sure we don't add an already existing key */ + duplicate = fy_node_mapping_key_is_duplicate(fyn, fyn_key); + + FYP_NODE_ERROR_CHECK(fyp, fyn_key, FYEM_DOC, + !duplicate, err_out, + "duplicate key"); + } + + fyep = fy_parse_private(fyp); + + fyp_error_check(fyp, fyep || !fyp->stream_error, err_out, + "fy_parse_private() failed"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 0, FYEM_DOC, + fyep, err_out, + "missing mapping value"); + + fye = &fyep->e; + + rc = fy_parse_document_load_node(fyp, fyd, + fyep, &fyn_value, depthp); + fyep = NULL; + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_document_load_node() failed"); + + assert(fyn_value); + + fynp_item->key = fyn_key; + fynp_item->value = fyn_value; + fyn_key = NULL; + fyn_value = NULL; + if (fyn->xl) { + rc = fy_accel_insert(fyn->xl, fynp_item->key, fynp_item); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_accel_insert() failed"); + } + + fy_node_pair_list_add_tail(&fyn->mapping, fynp_item); + if (fynp_item->key) + fynp_item->key->attached = true; + if (fynp_item->value) + fynp_item->value->attached = true; + fynp_item = NULL; + } + + if (!fyep) + goto err_out; + + if (fye->mapping_end.mapping_end) { + fyn->mapping_end = fye->mapping_end.mapping_end; + fye->mapping_end.mapping_end = NULL; + } + + assert(fyn->mapping_end); + + *fynp = fyn; + fyn = NULL; + + fy_parse_eventp_recycle(fyp, fyep); + + return 0; + +err_out: + rc = -1; +err_out_rc: + fy_parse_eventp_recycle(fyp, fyep); + fy_node_pair_free(fynp_item); + fy_node_detach_and_free(fyn_key); + fy_node_detach_and_free(fyn_value); + fy_node_detach_and_free(fyn); + return rc; +} + +static int +fy_parse_document_load_node(struct fy_parser *fyp, struct fy_document *fyd, + struct fy_eventp *fyep, struct fy_node **fynp, + int *depthp) +{ + struct fy_event *fye; + enum fy_event_type type; + int ret; + + *fynp = NULL; + + fyp_error_check(fyp, fyep || !fyp->stream_error, err_out, + "no event to process"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 0, FYEM_DOC, + fyep, err_out, + "premature end of event stream"); + + fyp_doc_debug(fyp, "in %s [%s]", __func__, fy_event_type_txt[fyep->e.type]); + + fye = &fyep->e; + + type = fye->type; + + FYP_TOKEN_ERROR_CHECK(fyp, fy_event_get_token(fye), FYEM_DOC, + type == FYET_ALIAS || type == FYET_SCALAR || + type == FYET_SEQUENCE_START || type == FYET_MAPPING_START, err_out, + "bad event"); + + (*depthp)++; + + FYP_TOKEN_ERROR_CHECK(fyp, fy_event_get_token(fye), FYEM_DOC, + ((fyp->cfg.flags & FYPCF_DISABLE_DEPTH_LIMIT) || + *depthp <= fy_depth_limit()), err_out, + "depth limit exceeded"); + + switch (type) { + + case FYET_ALIAS: + case FYET_SCALAR: + ret = fy_parse_document_load_scalar(fyp, fyd, + fyep, fynp, depthp); + break; + + case FYET_SEQUENCE_START: + ret = fy_parse_document_load_sequence(fyp, fyd, + fyep, fynp, depthp); + break; + + case FYET_MAPPING_START: + ret = fy_parse_document_load_mapping(fyp, fyd, + fyep, fynp, depthp); + break; + + default: + ret = 0; + break; + } + + --(*depthp); + return ret; + +err_out: + fy_parse_eventp_recycle(fyp, fyep); + return -1; +} + +int fy_parse_document_load_end(struct fy_parser *fyp, struct fy_document *fyd, struct fy_eventp *fyep) +{ + struct fy_event *fye; + int rc; + + fyp_error_check(fyp, fyep || !fyp->stream_error, err_out, + "no event to process"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 0, FYEM_DOC, + fyep, err_out, + "premature end of event stream"); + + fyp_doc_debug(fyp, "in %s [%s]", __func__, fy_event_type_txt[fyep->e.type]); + + fye = &fyep->e; + + FYP_TOKEN_ERROR_CHECK(fyp, fy_event_get_token(fye), FYEM_DOC, + fye->type == FYET_DOCUMENT_END, err_out, + "bad event"); + + /* recycle the document end event */ + fy_parse_eventp_recycle(fyp, fyep); + + return 0; +err_out: + rc = -1; + fy_parse_eventp_recycle(fyp, fyep); + return rc; +} + +struct fy_document *fy_parse_load_document_recursive(struct fy_parser *fyp) +{ + struct fy_document *fyd = NULL; + struct fy_eventp *fyep = NULL; + struct fy_event *fye = NULL; + int rc, depth; + bool was_stream_start; + +again: + was_stream_start = false; + do { + /* get next event */ + fyep = fy_parse_private(fyp); + + /* no more */ + if (!fyep) + return NULL; + + was_stream_start = fyep->e.type == FYET_STREAM_START; + + if (was_stream_start) { + fy_parse_eventp_recycle(fyp, fyep); + fyep = NULL; + } + + } while (was_stream_start); + + fye = &fyep->e; + + /* STREAM_END */ + if (fye->type == FYET_STREAM_END) { + fy_parse_eventp_recycle(fyp, fyep); + + /* final STREAM_END? */ + if (fyp->state == FYPS_END) + return NULL; + + /* multi-stream */ + goto again; + } + + FYP_TOKEN_ERROR_CHECK(fyp, fy_event_get_token(fye), FYEM_DOC, + fye->type == FYET_DOCUMENT_START, err_out, + "bad event"); + + fyd = fy_parse_document_create(fyp, fyep); + fyep = NULL; + + fyp_error_check(fyp, fyd, err_out, + "fy_parse_document_create() failed"); + + fyp_doc_debug(fyp, "calling load_node() for root"); + depth = 0; + rc = fy_parse_document_load_node(fyp, fyd, fy_parse_private(fyp), + &fyd->root, &depth); + fyp_error_check(fyp, !rc, err_out, + "fy_parse_document_load_node() failed"); + + rc = fy_parse_document_load_end(fyp, fyd, fy_parse_private(fyp)); + fyp_error_check(fyp, !rc, err_out, + "fy_parse_document_load_node() failed"); + + /* always resolve parents */ + fy_resolve_parent_node(fyd, fyd->root, NULL); + + if (fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT) { + rc = fy_document_resolve(fyd); + fyp_error_check(fyp, !rc, err_out, + "fy_document_resolve() failed"); + } + + return fyd; + +err_out: + fy_parse_eventp_recycle(fyp, fyep); + fy_parse_document_destroy(fyp, fyd); + return NULL; +} + +struct fy_document *fy_parse_load_document_with_builder(struct fy_parser *fyp) +{ + struct fy_document_builder_cfg cfg; + struct fy_document *fyd; + int rc; + + if (!fyp) + return NULL; + + if (!fyp->fydb) { + memset(&cfg, 0, sizeof(cfg)); + cfg.parse_cfg = fyp->cfg; + cfg.userdata = fyp; + cfg.diag = fy_diag_ref(fyp->diag); + + fyp->fydb = fy_document_builder_create(&cfg); + if (!fyp->fydb) + return NULL; + } + + fyd = fy_document_builder_load_document(fyp->fydb, fyp); + if (!fyd) + return NULL; + + if (fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT) { + rc = fy_document_resolve(fyd); + if (rc) { + fy_document_destroy(fyd); + fyp->stream_error = true; + return NULL; + } + } + + return fyd; +} + +struct fy_document *fy_parse_load_document(struct fy_parser *fyp) +{ + if (!fyp) + return NULL; + + return !(fyp->cfg.flags & FYPCF_PREFER_RECURSIVE) ? + fy_parse_load_document_with_builder(fyp) : + fy_parse_load_document_recursive(fyp); +} + +struct fy_node *fy_node_copy_internal(struct fy_document *fyd, struct fy_node *fyn_from, + struct fy_node *fyn_parent) +{ + struct fy_document *fyd_from; + struct fy_node *fyn, *fyni, *fynit; + struct fy_node_pair *fynp, *fynpt; + struct fy_anchor *fya, *fya_from; + const char *anchor; + size_t anchor_len; + int rc; + + if (!fyd || !fyn_from || !fyn_from->fyd) + return NULL; + + fyd_from = fyn_from->fyd; + + fyn = fy_node_alloc(fyd, fyn_from->type); + fyd_error_check(fyd, fyn, err_out, + "fy_node_alloc() failed"); + + fyn->tag = fy_token_ref(fyn_from->tag); + fyn->style = fyn_from->style; + fyn->parent = fyn_parent; + + switch (fyn->type) { + case FYNT_SCALAR: + fyn->scalar = fy_token_ref(fyn_from->scalar); + break; + + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn_from->sequence); fyni; + fyni = fy_node_next(&fyn_from->sequence, fyni)) { + + fynit = fy_node_copy_internal(fyd, fyni, fyn); + fyd_error_check(fyd, fynit, err_out, + "fy_node_copy_internal() failed"); + + fy_node_list_add_tail(&fyn->sequence, fynit); + fynit->attached = true; + } + + break; + case FYNT_MAPPING: + for (fynp = fy_node_pair_list_head(&fyn_from->mapping); fynp; + fynp = fy_node_pair_next(&fyn_from->mapping, fynp)) { + + fynpt = fy_node_pair_alloc(fyd); + fyd_error_check(fyd, fynpt, err_out, + "fy_node_pair_alloc() failed"); + + fynpt->key = fy_node_copy_internal(fyd, fynp->key, fyn); + fynpt->value = fy_node_copy_internal(fyd, fynp->value, fyn); + fynp->parent = fyn; + + fy_node_pair_list_add_tail(&fyn->mapping, fynpt); + if (fyn->xl) { + rc = fy_accel_insert(fyn->xl, fynpt->key, fynpt); + fyd_error_check(fyd, !rc, err_out, + "fy_accel_insert() failed"); + } + if (fynpt->key) { + fynpt->key->attached = true; + fynpt->key->key_root = true; + } + if (fynpt->value) + fynpt->value->attached = true; + } + break; + } + + /* drop an anchor to the copy */ + for (fya_from = fy_anchor_list_head(&fyd_from->anchors); fya_from; + fya_from = fy_anchor_next(&fyd_from->anchors, fya_from)) { + if (fyn_from == fya_from->fyn) + break; + } + + /* source node has an anchor */ + if (fya_from) { + fya = fy_document_lookup_anchor_by_token(fyd, fya_from->anchor); + if (!fya) { + fyd_doc_debug(fyd, "new anchor"); + /* update the new anchor position */ + rc = fy_document_register_anchor(fyd, fyn, fya_from->anchor); + fyd_error_check(fyd, !rc, err_out, + "fy_document_register_anchor() failed"); + + fy_token_ref(fya_from->anchor); + } else { + anchor = fy_anchor_get_text(fya, &anchor_len); + fyd_error_check(fyd, anchor, err_out, + "fy_anchor_get_text() failed"); + fyd_doc_debug(fyd, "not overwritting anchor %.*s", (int)anchor_len, anchor); + } + } + + return fyn; + +err_out: + return NULL; +} + +struct fy_node *fy_node_copy(struct fy_document *fyd, struct fy_node *fyn_from) +{ + struct fy_node *fyn; + + if (!fyd) + return NULL; + + fyn = fy_node_copy_internal(fyd, fyn_from, NULL); + if (!fyn) { + fyd->diag->on_error = false; + return NULL; + } + + return fyn; +} + +struct fy_document *fy_document_clone(struct fy_document *fydsrc) +{ + struct fy_document *fyd = NULL; + + if (!fydsrc) + return NULL; + + fyd = fy_document_create(&fydsrc->parse_cfg); + if (!fyd) + return NULL; + + /* drop the default document state */ + fy_document_state_unref(fyd->fyds); + /* and use the source document state (and ref it) */ + fyd->fyds = fy_document_state_ref(fydsrc->fyds); + assert(fyd->fyds); + + if (fydsrc->root) { + fyd->root = fy_node_copy(fyd, fydsrc->root); + if (!fyd->root) + goto err_out; + } + + return fyd; +err_out: + fy_document_destroy(fyd); + return NULL; +} + +int fy_node_copy_to_scalar(struct fy_document *fyd, struct fy_node *fyn_to, struct fy_node *fyn_from) +{ + struct fy_node *fyn, *fyni; + struct fy_node_pair *fynp; + + fyn = fy_node_copy(fyd, fyn_from); + if (!fyn) + return -1; + + /* the node is guaranteed to be a scalar */ + fy_token_unref(fyn_to->tag); + fyn_to->tag = NULL; + fy_token_unref(fyn_to->scalar); + fyn_to->scalar = NULL; + + fyn_to->type = fyn->type; + fyn_to->tag = fy_token_ref(fyn->tag); + fyn_to->style = fyn->style; + + switch (fyn->type) { + case FYNT_SCALAR: + fyn_to->scalar = fyn->scalar; + fyn->scalar = NULL; + break; + case FYNT_SEQUENCE: + fy_node_list_init(&fyn_to->sequence); + while ((fyni = fy_node_list_pop(&fyn->sequence)) != NULL) + fy_node_list_add_tail(&fyn_to->sequence, fyni); + break; + case FYNT_MAPPING: + fy_node_pair_list_init(&fyn_to->mapping); + while ((fynp = fy_node_pair_list_pop(&fyn->mapping)) != NULL) { + if (fyn->xl) + fy_accel_remove(fyn->xl, fynp->key); + fy_node_pair_list_add_tail(&fyn_to->mapping, fynp); + if (fyn_to->xl) + fy_accel_insert(fyn_to->xl, fynp->key, fynp); + } + break; + } + + /* and free */ + fy_node_free(fyn); + + return 0; +} + +static int fy_document_node_update_tags(struct fy_document *fyd, struct fy_node *fyn) +{ + struct fy_node *fyni; + struct fy_node_pair *fynp, *fynpi; + struct fy_token *fyt_td; + const char *handle; + size_t handle_size; + int rc; + + if (!fyd || !fyn) + return 0; + + /* replace tag reference with the one that the document contains */ + if (fyn->tag) { + fyd_error_check(fyd, fyn->tag->type == FYTT_TAG, err_out, + "bad node tag"); + + handle = fy_tag_directive_token_handle(fyn->tag->tag.fyt_td, &handle_size); + fyd_error_check(fyd, handle, err_out, + "bad tag directive token"); + + fyt_td = fy_document_state_lookup_tag_directive(fyd->fyds, handle, handle_size); + fyd_error_check(fyd, fyt_td, err_out, + "Missing tag directive with handle=%.*s", (int)handle_size, handle); + + /* need to replace this */ + if (fyt_td != fyn->tag->tag.fyt_td) { + fy_token_unref(fyn->tag->tag.fyt_td); + fyn->tag->tag.fyt_td = fy_token_ref(fyt_td); + + } + } + + + switch (fyn->type) { + case FYNT_SCALAR: + break; + + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) { + + rc = fy_document_node_update_tags(fyd, fyni); + if (rc) + goto err_out_rc; + } + break; + + case FYNT_MAPPING: + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; fynp = fynpi) { + + fynpi = fy_node_pair_next(&fyn->mapping, fynp); + + /* the parent of the key is always NULL */ + rc = fy_document_node_update_tags(fyd, fynp->key); + if (rc) + goto err_out_rc; + + rc = fy_document_node_update_tags(fyd, fynp->value); + if (rc) + goto err_out_rc; + } + break; + } + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_node_insert(struct fy_node *fyn_to, struct fy_node *fyn_from) +{ + struct fy_document *fyd; + struct fy_node *fyn_parent, *fyn_cpy, *fyni, *fyn_prev; + struct fy_node_pair *fynp, *fynpi, *fynpj; + int rc; + + if (!fyn_to || !fyn_to->fyd) + return -1; + + fyd = fyn_to->fyd; + assert(fyd); + + fyn_parent = fy_node_get_document_parent(fyn_to); + fynp = NULL; + if (fyn_parent) { + fyd_error_check(fyd, fyn_parent->type != FYNT_SCALAR, err_out, + "Illegal scalar parent node type"); + + if (fyn_parent->type == FYNT_MAPPING) { + /* find mapping pair that contains the `to` node */ + for (fynp = fy_node_pair_list_head(&fyn_parent->mapping); fynp; + fynp = fy_node_pair_next(&fyn_parent->mapping, fynp)) { + if (fynp->value == fyn_to) + break; + } + } + } + + /* verify no funkiness on root */ + assert(fyn_parent || fyn_to == fyd->root); + + /* deleting target */ + if (!fyn_from) { + fyn_to->parent = NULL; + + if (!fyn_parent) { + fyd_doc_debug(fyd, "Deleting root node"); + fy_node_detach_and_free(fyn_to); + fyd->root = NULL; + } else if (fyn_parent->type == FYNT_SEQUENCE) { + fyd_doc_debug(fyd, "Deleting sequence node"); + fy_node_list_del(&fyn_parent->sequence, fyn_to); + fy_node_detach_and_free(fyn_to); + } else { + fyd_doc_debug(fyd, "Deleting mapping node"); + /* should never happen, it's checked right above, but play safe */ + assert(fyn_parent->type == FYNT_MAPPING); + + fyd_error_check(fyd, fynp, err_out, + "Illegal mapping node found"); + + fy_node_pair_list_del(&fyn_parent->mapping, fynp); + if (fyn_parent->xl) + fy_accel_remove(fyn_parent->xl, fynp->key); + /* this will also delete fyn_to */ + fy_node_pair_detach_and_free(fynp); + } + return 0; + } + + /* + * from: scalar + * + * to: another-scalar -> scalar + * to: { key: value } -> scalar + * to: [ seq0, seq1 ] -> scalar + * + * from: [ seq2 ] + * to: scalar -> [ seq2 ] + * to: { key: value } -> [ seq2 ] + * to: [ seq0, seq1 ] -> [ seq0, seq1, sec2 ] + * + * from: { another-key: another-value } + * to: scalar -> { another-key: another-value } + * to: { key: value } -> { key: value, another-key: another-value } + * to: [ seq0, seq1 ] -> { another-key: another-value } + * + * from: { key: another-value } + * to: scalar -> { key: another-value } + * to: { key: value } -> { key: another-value } + * to: [ seq0, seq1 ] -> { key: another-value } + * + */ + + /* if types of `from` and `to` differ (or it's a scalar), it's a replace */ + if (fyn_from->type != fyn_to->type || fyn_from->type == FYNT_SCALAR) { + + fyn_cpy = fy_node_copy(fyd, fyn_from); + fyd_error_check(fyd, fyn_cpy, err_out, + "fy_node_copy() failed"); + + if (!fyn_parent) { + fyd_doc_debug(fyd, "Replacing root node"); + fy_node_detach_and_free(fyd->root); + fyd->root = fyn_cpy; + } else if (fyn_parent->type == FYNT_SEQUENCE) { + fyd_doc_debug(fyd, "Replacing sequence node"); + + /* get previous */ + fyn_prev = fy_node_prev(&fyn_parent->sequence, fyn_to); + + /* delete */ + fy_node_list_del(&fyn_parent->sequence, fyn_to); + fy_node_detach_and_free(fyn_to); + + /* if there's no previous insert to head */ + if (!fyn_prev) + fy_node_list_add(&fyn_parent->sequence, fyn_cpy); + else + fy_node_list_insert_after(&fyn_parent->sequence, fyn_prev, fyn_cpy); + } else { + fyd_doc_debug(fyd, "Replacing mapping node value"); + /* should never happen, it's checked right above, but play safe */ + assert(fyn_parent->type == FYNT_MAPPING); + fyd_error_check(fyd, fynp, err_out, + "Illegal mapping node found"); + + fy_node_detach_and_free(fynp->value); + fynp->value = fyn_cpy; + } + + return 0; + } + + /* types match, if it's a sequence append */ + if (fyn_to->type == FYNT_SEQUENCE) { + + fyd_doc_debug(fyd, "Appending to sequence node"); + + for (fyni = fy_node_list_head(&fyn_from->sequence); fyni; + fyni = fy_node_next(&fyn_from->sequence, fyni)) { + + fyn_cpy = fy_node_copy(fyd, fyni); + fyd_error_check(fyd, fyn_cpy, err_out, + "fy_node_copy() failed"); + + fy_node_list_add_tail(&fyn_to->sequence, fyn_cpy); + fyn_cpy->attached = true; + } + } else { + /* only mapping is possible here */ + + /* iterate over all the keys in the `from` */ + for (fynpi = fy_node_pair_list_head(&fyn_from->mapping); fynpi; + fynpi = fy_node_pair_next(&fyn_from->mapping, fynpi)) { + + if (fyn_to->xl) { + fynpj = fy_node_accel_lookup_by_node(fyn_to, fynpi->key); + } else { + /* find whether the key already exists */ + for (fynpj = fy_node_pair_list_head(&fyn_to->mapping); fynpj; + fynpj = fy_node_pair_next(&fyn_to->mapping, fynpj)) { + + if (fy_node_compare(fynpi->key, fynpj->key)) + break; + } + } + + if (!fynpj) { + fyd_doc_debug(fyd, "Appending to mapping node"); + + /* not found? append it */ + fynpj = fy_node_pair_alloc(fyd); + fyd_error_check(fyd, fynpj, err_out, + "fy_node_pair_alloc() failed"); + + fynpj->key = fy_node_copy(fyd, fynpi->key); + fyd_error_check(fyd, !fynpi->key || fynpj->key, err_out, + "fy_node_copy() failed"); + fynpj->value = fy_node_copy(fyd, fynpi->value); + fyd_error_check(fyd, !fynpi->value || fynpj->value, err_out, + "fy_node_copy() failed"); + + fy_node_pair_list_add_tail(&fyn_to->mapping, fynpj); + if (fyn_to->xl) + fy_accel_insert(fyn_to->xl, fynpj->key, fynpj); + + if (fynpj->key) + fynpj->key->attached = true; + if (fynpj->value) + fynpj->value->attached = true; + + } else { + fyd_doc_debug(fyd, "Updating mapping node value (deep merge)"); + + rc = fy_node_insert(fynpj->value, fynpi->value); + fyd_error_check(fyd, !rc, err_out_rc, + "fy_node_insert() failed"); + } + } + } + + /* adjust parents */ + switch (fyn_to->type) { + case FYNT_SCALAR: + break; + + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn_to->sequence); fyni; + fyni = fy_node_next(&fyn_to->sequence, fyni)) { + + fyni->parent = fyn_to; + } + break; + + case FYNT_MAPPING: + for (fynp = fy_node_pair_list_head(&fyn_to->mapping); fynp; fynp = fynpi) { + + fynpi = fy_node_pair_next(&fyn_to->mapping, fynp); + + if (fynp->key) { + fynp->key->parent = fyn_to; + fynp->key->key_root = true; + } + if (fynp->value) + fynp->value->parent = fyn_to; + fynp->parent = fyn_to; + } + break; + } + + /* if the documents differ, merge their states */ + if (fyn_to->fyd != fyn_from->fyd) { + rc = fy_document_state_merge(fyn_to->fyd->fyds, fyn_from->fyd->fyds); + fyd_error_check(fyd, !rc, err_out_rc, + "fy_document_state_merge() failed"); + + rc = fy_document_node_update_tags(fyd, fy_document_root(fyd)); + fyd_error_check(fyd, !rc, err_out_rc, + "fy_document_node_update_tags() failed"); + } + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_node_delete(struct fy_node *fyn) +{ + return fy_node_insert(fyn, NULL); +} + +int fy_document_insert_at(struct fy_document *fyd, + const char *path, size_t pathlen, + struct fy_node *fyn) +{ + int rc; + struct fy_node *fyn2; + + fyn2 = fy_node_by_path(fy_document_root(fyd), path, pathlen, FYNWF_DONT_FOLLOW); + rc = fy_node_insert(fyn2, fyn); + + fy_node_free(fyn); + + return rc; +} + +struct fy_token *fy_document_tag_directive_iterate(struct fy_document *fyd, void **prevp) +{ + struct fy_token_list *fytl; + + if (!fyd || !fyd->fyds || !prevp) + return NULL; + + fytl = &fyd->fyds->fyt_td; + + return *prevp = *prevp ? fy_token_next(fytl, *prevp) : fy_token_list_head(fytl); +} + +struct fy_token *fy_document_tag_directive_lookup(struct fy_document *fyd, const char *handle) +{ + struct fy_token *fyt; + void *iter; + const char *h; + size_t h_size, len; + + if (!fyd || !handle) + return NULL; + len = strlen(handle); + + iter = NULL; + while ((fyt = fy_document_tag_directive_iterate(fyd, &iter)) != NULL) { + h = fy_tag_directive_token_handle(fyt, &h_size); + if (!h) + continue; + if (h_size == len && !memcmp(h, handle, len)) + return fyt; + } + return NULL; +} + +int fy_document_tag_directive_add(struct fy_document *fyd, const char *handle, const char *prefix) +{ + struct fy_token *fyt; + + if (!fyd || !fyd->fyds || !handle || !prefix) + return -1; + + /* it must not exist */ + fyt = fy_document_tag_directive_lookup(fyd, handle); + if (fyt) + return -1; + + return fy_document_state_append_tag(fyd->fyds, handle, prefix, false); +} + +int fy_document_tag_directive_remove(struct fy_document *fyd, const char *handle) +{ + struct fy_token *fyt; + + if (!fyd || !fyd->fyds || !handle) + return -1; + + /* it must not exist */ + fyt = fy_document_tag_directive_lookup(fyd, handle); + if (!fyt || fyt->refs != 1) + return -1; + + fy_token_list_del(&fyd->fyds->fyt_td, fyt); + fy_token_unref(fyt); + + return 0; +} + +static int fy_resolve_alias(struct fy_document *fyd, struct fy_node *fyn) +{ + struct fy_node *fyn_copy = NULL; + int rc; + + fyn_copy = fy_node_resolve_alias(fyn); + FYD_NODE_ERROR_CHECK(fyd, fyn, FYEM_DOC, + fyn_copy, err_out, + "invalid alias"); + + rc = fy_node_copy_to_scalar(fyd, fyn, fyn_copy); + fyd_error_check(fyd, !rc, err_out, + "fy_node_copy_to_scalar() failed"); + + return 0; + +err_out: + fyd->diag->on_error = false; + return -1; +} + +static struct fy_node * +fy_node_follow_alias(struct fy_node *fyn, enum fy_node_walk_flags flags) +{ + enum fy_node_walk_flags ptr_flags; + struct fy_anchor *fya; + const char *anchor_text, *s, *e, *p, *path; + size_t anchor_len, path_len; + struct fy_node *fyn_path_root; + unsigned int marker; + + if (!fyn || !fy_node_is_alias(fyn)) + return NULL; + + ptr_flags = flags & FYNWF_PTR(FYNWF_PTR_MASK); + if (ptr_flags == FYNWF_PTR_YPATH) + return fy_node_alias_resolve_by_ypath(fyn); + + /* try regular label target */ + fya = fy_document_lookup_anchor_by_token(fyn->fyd, fyn->scalar); + if (fya) + return fya->fyn; + + anchor_text = fy_token_get_text(fyn->scalar, &anchor_len); + if (!anchor_text) + return NULL; + + s = anchor_text; + e = s + anchor_len; + + fyn_path_root = NULL; + + if (ptr_flags == FYNWF_PTR_YAML && (p = memchr(s, '/', e - s)) != NULL) { + /* fyd_notice(fyn->fyd, "%s: alias contains a path component %.*s", + __func__, (int)(e - p - 1), p + 1); */ + + if (p > s) { + + fya = fy_document_lookup_anchor(fyn->fyd, s, p - s); + if (!fya) { + /* fyd_notice(fyn->fyd, "%s: unable to resolve alias %.*s @%s", + __func__, (int)(p - s), s, fy_node_get_path(fya->fyn)); */ + return NULL; + } + + /* fyd_notice(fyn->fyd, "%s: alias base %.*s @%s", + __func__, (int)(p - s), s, fy_node_get_path(fya->fyn)); */ + path = ++p; + path_len = e - p; + + fyn_path_root = fya->fyn; + + } else { + /* fyd_notice(fyn->fyd, "%s: absolute %.*s @%s", + __func__, (int)(p - s), s, fy_node_get_path(fya->fyn)); */ + path = s; + path_len = e - s; + + fyn_path_root = fyn->fyd->root; + } + } + + if (!fyn_path_root) + return NULL; + + marker = fy_node_walk_marker_from_flags(flags); + if (marker >= FYNWF_MAX_USER_MARKER) + return NULL; + + /* use the next marker */ + flags &= ~FYNWF_MARKER(FYNWF_MARKER_MASK); + flags |= FYNWF_MARKER(marker + 1); + + return fy_node_by_path_internal(fyn_path_root, path, path_len, flags); +} + +static bool fy_node_pair_is_merge_key(struct fy_node_pair *fynp) +{ + struct fy_node *fyn = fynp->key; + + return fyn && fyn->type == FYNT_SCALAR && fyn->style == FYNS_PLAIN && + fy_plain_atom_streq(fy_token_atom(fyn->scalar), "<<"); +} + +static struct fy_node *fy_alias_get_merge_mapping(struct fy_document *fyd, struct fy_node *fyn) +{ + struct fy_anchor *fya; + + /* must be an alias */ + if (!fy_node_is_alias(fyn)) + return NULL; + + /* anchor must exist */ + fya = fy_document_lookup_anchor_by_token(fyd, fyn->scalar); + if (!fya) + return NULL; + + /* and it must be a mapping */ + if (fya->fyn->type != FYNT_MAPPING) + return NULL; + + return fya->fyn; +} + +static bool fy_node_pair_is_valid_merge_key(struct fy_document *fyd, struct fy_node_pair *fynp) +{ + struct fy_node *fyn, *fyni, *fynm; + + fyn = fynp->value; + + /* value must exist */ + if (!fyn) + return false; + + /* scalar alias */ + fynm = fy_alias_get_merge_mapping(fyd, fyn); + if (fynm) + return true; + + /* it must be a sequence then */ + if (fyn->type != FYNT_SEQUENCE) + return false; + + /* the sequence must only contain valid aliases for mapping */ + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) { + + /* sequence of aliases only! */ + fynm = fy_alias_get_merge_mapping(fyd, fyni); + if (!fynm) + return false; + + } + + return true; +} + +static int fy_resolve_merge_key_populate(struct fy_document *fyd, struct fy_node *fyn, + struct fy_node_pair *fynp, struct fy_node *fynm) +{ + struct fy_node_pair *fynpi, *fynpn; + + if (!fyd) + return -1; + + fyd_error_check(fyd, + fyn && fynp && fynm && fyn->type == FYNT_MAPPING && fynm->type == FYNT_MAPPING, + err_out, "bad inputs to %s", __func__); + + for (fynpi = fy_node_pair_list_head(&fynm->mapping); fynpi; + fynpi = fy_node_pair_next(&fynm->mapping, fynpi)) { + + /* if we don't allow duplicate keys */ + if (!(fyd->parse_cfg.flags & FYPCF_ALLOW_DUPLICATE_KEYS)) { + + /* make sure we don't override an already existing key */ + if (fy_node_mapping_key_is_duplicate(fyn, fynpi->key)) + continue; + } + + fynpn = fy_node_pair_alloc(fyd); + fyd_error_check(fyd, fynpn, err_out, + "fy_node_pair_alloc() failed"); + + fynpn->key = fy_node_copy(fyd, fynpi->key); + fynpn->value = fy_node_copy(fyd, fynpi->value); + + fy_node_pair_list_insert_after(&fyn->mapping, fynp, fynpn); + if (fyn->xl) + fy_accel_insert(fyn->xl, fynpn->key, fynpn); + } + + return 0; + +err_out: + return -1; +} + +static int fy_resolve_merge_key(struct fy_document *fyd, struct fy_node *fyn, struct fy_node_pair *fynp) +{ + struct fy_node *fynv, *fyni, *fynm; + int rc; + + /* it must be a valid merge key value */ + FYD_NODE_ERROR_CHECK(fyd, fynp->value, FYEM_DOC, + fy_node_pair_is_valid_merge_key(fyd, fynp), err_out, + "invalid merge key value"); + + fynv = fynp->value; + fynm = fy_alias_get_merge_mapping(fyd, fynv); + if (fynm) { + rc = fy_resolve_merge_key_populate(fyd, fyn, fynp, fynm); + fyd_error_check(fyd, !rc, err_out_rc, + "fy_resolve_merge_key_populate() failed"); + + return 0; + } + + /* it must be a sequence then */ + fyd_error_check(fyd, fynv->type == FYNT_SEQUENCE, err_out, + "invalid node type to use for merge key"); + + /* the sequence must only contain valid aliases for mapping */ + for (fyni = fy_node_list_head(&fynv->sequence); fyni; + fyni = fy_node_next(&fynv->sequence, fyni)) { + + fynm = fy_alias_get_merge_mapping(fyd, fyni); + fyd_error_check(fyd, fynm, err_out, + "invalid merge key sequence item (not an alias)"); + + rc = fy_resolve_merge_key_populate(fyd, fyn, fynp, fynm); + fyd_error_check(fyd, !rc, err_out_rc, + "fy_resolve_merge_key_populate() failed"); + } + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +/* the anchors are scalars that have the FYNS_ALIAS style */ +static int fy_resolve_anchor_node(struct fy_document *fyd, struct fy_node *fyn) +{ + struct fy_node *fyni; + struct fy_node_pair *fynp, *fynpi, *fynpit; + int rc, ret_rc = 0; + struct fy_token *fyt; + + if (!fyn) + return 0; + + if (fy_node_is_alias(fyn)) + return fy_resolve_alias(fyd, fyn); + + if (fyn->type == FYNT_SEQUENCE) { + + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) { + + rc = fy_resolve_anchor_node(fyd, fyni); + if (rc && !ret_rc) + ret_rc = rc; + } + + } else if (fyn->type == FYNT_MAPPING) { + + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; fynp = fynpi) { + + fynpi = fy_node_pair_next(&fyn->mapping, fynp); + + if (fy_node_pair_is_merge_key(fynp)) { + rc = fy_resolve_merge_key(fyd, fyn, fynp); + if (rc && !ret_rc) + ret_rc = rc; + + /* remove this node pair */ + if (!rc) { + fy_node_pair_list_del(&fyn->mapping, fynp); + if (fyn->xl) + fy_accel_remove(fyn->xl, fynp->key); + fy_node_pair_detach_and_free(fynp); + } + + } else { + + rc = fy_resolve_anchor_node(fyd, fynp->key); + + if (!rc) { + + /* check whether the keys are duplicate */ + for (fynpit = fy_node_pair_list_head(&fyn->mapping); fynpit; + fynpit = fy_node_pair_next(&fyn->mapping, fynpit)) { + + /* skip this node pair */ + if (fynpit == fynp) + continue; + + if (!fy_node_compare(fynpit->key, fynp->key)) + continue; + + /* whoops, duplicate key after resolution */ + fyt = NULL; + switch (fyn->type) { + case FYNT_SCALAR: + fyt = fyn->scalar; + break; + case FYNT_SEQUENCE: + fyt = fyn->sequence_start; + break; + case FYNT_MAPPING: + fyt = fyn->mapping_start; + break; + } + + FYD_TOKEN_ERROR_CHECK(fyd, fyt, FYEM_DOC, + false, err_out, + "duplicate key after resolving"); + + } + + } + + if (rc && !ret_rc) + ret_rc = rc; + + rc = fy_resolve_anchor_node(fyd, fynp->value); + if (rc && !ret_rc) + ret_rc = rc; + + } + } + } + + return ret_rc; + +err_out: + return -1; +} + +static void fy_resolve_parent_node(struct fy_document *fyd, struct fy_node *fyn, struct fy_node *fyn_parent) +{ + struct fy_node *fyni; + struct fy_node_pair *fynp, *fynpi; + + if (!fyn) + return; + + fyn->parent = fyn_parent; + + switch (fyn->type) { + case FYNT_SCALAR: + break; + + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) { + + fy_resolve_parent_node(fyd, fyni, fyn); + } + break; + + case FYNT_MAPPING: + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; fynp = fynpi) { + + fynpi = fy_node_pair_next(&fyn->mapping, fynp); + + fy_resolve_parent_node(fyd, fynp->key, fyn); + fy_resolve_parent_node(fyd, fynp->value, fyn); + fynp->parent = fyn; + } + break; + } +} + +void fy_document_purge_anchors(struct fy_document *fyd) +{ + struct fy_anchor *fya; + struct fy_anchor *fyan; + struct fy_accel_entry *xle; + + /* remove all anchors */ + for (fya = fy_anchor_list_head(&fyd->anchors); fya; fya = fyan) { + fyan = fy_anchor_next(&fyd->anchors, fya); + fy_anchor_list_del(&fyd->anchors, fya); + + if (fy_document_is_accelerated(fyd)) { + xle = fy_accel_entry_lookup_key_value(fyd->axl, fya->anchor, fya); + fy_accel_entry_remove(fyd->axl, xle); + + xle = fy_accel_entry_lookup_key_value(fyd->naxl, fya->fyn, fya); + fy_accel_entry_remove(fyd->naxl, xle); + } + + fy_anchor_destroy(fya); + } + + if (fy_document_is_accelerated(fyd)) { + fy_accel_cleanup(fyd->axl); + free(fyd->axl); + fyd->axl = NULL; + + fy_accel_cleanup(fyd->naxl); + free(fyd->naxl); + fyd->naxl = NULL; + } +} + +typedef void (*fy_node_applyf)(struct fy_node *fyn, void *user); + +void fy_node_apply(struct fy_node *fyn, fy_node_applyf func, void *user) +{ + struct fy_node *fyni; + struct fy_node_pair *fynp; + + if (!fyn || !func) + return; + + (*func)(fyn, user); + + switch (fyn->type) { + case FYNT_SCALAR: + break; + + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) + fy_node_apply(fyni, func, user); + break; + + case FYNT_MAPPING: + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; + fynp = fy_node_pair_next(&fyn->mapping, fynp)) { + + fy_node_apply(fynp->key, func, user); + fy_node_apply(fynp->value, func, user); + } + break; + } +} + +static void clear_system_marks(struct fy_node *fyn, void *user) +{ + fyn->marks &= ~FYNWF_SYSTEM_MARKS; +} + +/* clear all the system markers */ +void fy_node_clear_system_marks(struct fy_node *fyn) +{ + fy_node_apply(fyn, clear_system_marks, NULL); +} + +static void count_aliases(struct fy_node *fyn, void *user) +{ + int *countp = user; + + if (fyn->style == FYNS_ALIAS) + (*countp)++; +} + +int fy_node_count_aliases(struct fy_node *fyn) +{ + int count; + + count = 0; + + fy_node_apply(fyn, count_aliases, &count); + + return count; +} + +int fy_document_resolve(struct fy_document *fyd) +{ + int rc, num_aliases, num_aliases_prev; + bool ret; + + if (!fyd) + return 0; + + num_aliases_prev = INT_MAX; + do { + fy_node_clear_system_marks(fyd->root); + + /* for resolution to work, no reference loops should exist */ + ret = fy_check_ref_loop(fyd, fyd->root, + FYNWF_MAXDEPTH_DEFAULT | FYNWF_FOLLOW, NULL); + + fy_node_clear_system_marks(fyd->root); + + if (ret) + goto err_out; + + /* now resolve any anchor nodes */ + rc = fy_resolve_anchor_node(fyd, fyd->root); + if (rc) + goto err_out_rc; + + /* redo parent resolution */ + fy_resolve_parent_node(fyd, fyd->root, NULL); + + /* count the remaining aliases */ + num_aliases = fy_node_count_aliases(fyd->root); + if (num_aliases == num_aliases_prev) + goto err_out; + + } while (num_aliases > 0); + + /* remove all anchors after resolution */ + fy_document_purge_anchors(fyd); + + return 0; + +err_out: + rc = -1; +err_out_rc: + fyd->diag->on_error = false; + return rc; +} + +void fy_document_free_nodes(struct fy_document *fyd) +{ + struct fy_document *fyd_child; + + for (fyd_child = fy_document_list_first(&fyd->children); fyd_child; + fyd_child = fy_document_next(&fyd->children, fyd_child)) + fy_document_free_nodes(fyd_child); + + fy_node_detach_and_free(fyd->root); + fyd->root = NULL; +} + +void fy_document_destroy(struct fy_document *fyd) +{ + struct fy_document *fyd_child; + + /* both the document and the parser object must exist */ + if (!fyd) + return; + + /* we have to free the nodes first */ + fy_document_free_nodes(fyd); + + /* recursively delete children */ + while ((fyd_child = fy_document_list_pop(&fyd->children)) != NULL) { + fyd_child->parent = NULL; + fy_document_destroy(fyd_child); + } + + fy_parse_document_destroy(NULL, fyd); +} + +int fy_document_set_parent(struct fy_document *fyd, struct fy_document *fyd_child) +{ + + if (!fyd || !fyd_child || fyd_child->parent) + return -1; + + fyd_child->parent = fyd; + fy_document_list_add_tail(&fyd->children, fyd_child); + + return 0; +} + +static const struct fy_parse_cfg doc_parse_default_cfg = { + .flags = FYPCF_DEFAULT_DOC, +}; + +struct fy_document *fy_document_create(const struct fy_parse_cfg *cfg) +{ + struct fy_document *fyd = NULL; + struct fy_diag *diag; + int rc; + + if (!cfg) + cfg = &doc_parse_default_cfg; + + fyd = malloc(sizeof(*fyd)); + if (!fyd) + goto err_out; + + memset(fyd, 0, sizeof(*fyd)); + fyd->parse_cfg = *cfg; + + diag = cfg->diag; + if (!diag) { + diag = fy_diag_create(NULL); + if (!diag) + goto err_out; + } else + fy_diag_ref(diag); + + fyd->diag = diag; + + fy_anchor_list_init(&fyd->anchors); + if (fy_document_is_accelerated(fyd)) { + fyd->axl = malloc(sizeof(*fyd->axl)); + fyd_error_check(fyd, fyd->axl, err_out, + "malloc() failed"); + + /* start with a very small bucket list */ + rc = fy_accel_setup(fyd->axl, &hd_anchor, fyd, 8); + fyd_error_check(fyd, !rc, err_out, + "fy_accel_setup() failed"); + + fyd->naxl = malloc(sizeof(*fyd->naxl)); + fyd_error_check(fyd, fyd->axl, err_out, + "malloc() failed"); + + /* start with a very small bucket list */ + rc = fy_accel_setup(fyd->naxl, &hd_nanchor, fyd, 8); + fyd_error_check(fyd, !rc, err_out, + "fy_accel_setup() failed"); + } + fyd->root = NULL; + + /* we don't do document create version setting, + * perhaps we should in the future + */ + fyd->fyds = fy_document_state_default(NULL, NULL); + fyd_error_check(fyd, fyd->fyds, err_out, + "fy_document_state_default() failed"); + + /* turn on JSON mode if it's forced */ + fyd->fyds->json_mode = (cfg->flags & + (FYPCF_JSON_MASK << FYPCF_JSON_SHIFT)) == FYPCF_JSON_FORCE; + + fy_document_list_init(&fyd->children); + + return fyd; + +err_out: + fy_parse_document_destroy(NULL, fyd); + return NULL; +} + +struct fy_document_build_string_ctx { + const char *str; + size_t len; +}; + +static int parser_setup_from_string(struct fy_parser *fyp, void *user) +{ + struct fy_document_build_string_ctx *ctx = user; + + return fy_parser_set_string(fyp, ctx->str, ctx->len); +} + +struct fy_document_build_malloc_string_ctx { + char *str; + size_t len; +}; + +static int parser_setup_from_malloc_string(struct fy_parser *fyp, void *user) +{ + struct fy_document_build_malloc_string_ctx *ctx = user; + + return fy_parser_set_malloc_string(fyp, ctx->str, ctx->len); +} + +struct fy_document_build_file_ctx { + const char *file; +}; + +static int parser_setup_from_file(struct fy_parser *fyp, void *user) +{ + struct fy_document_build_file_ctx *ctx = user; + + return fy_parser_set_input_file(fyp, ctx->file); +} + +struct fy_document_build_fp_ctx { + const char *name; + FILE *fp; +}; + +static int parser_setup_from_fp(struct fy_parser *fyp, void *user) +{ + struct fy_document_build_fp_ctx *ctx = user; + + return fy_parser_set_input_fp(fyp, ctx->name, ctx->fp); +} + +struct fy_document_vbuildf_ctx { + const char *fmt; + va_list ap; +}; + +static int parser_setup_from_fmt_ap(struct fy_parser *fyp, void *user) +{ + struct fy_document_vbuildf_ctx *vctx = user; + va_list ap, ap_orig; + int size, sizew; + char *buf; + + /* first try without allocating */ + va_copy(ap_orig, vctx->ap); + size = vsnprintf(NULL, 0, vctx->fmt, ap_orig); + va_end(ap_orig); + + fyp_error_check(fyp, size >= 0, err_out, + "vsnprintf() failed"); + + buf = malloc(size + 1); + fyp_error_check(fyp, buf, err_out, + "malloc() failed"); + + va_copy(ap, vctx->ap); + sizew = vsnprintf(buf, size + 1, vctx->fmt, ap); + fyp_error_check(fyp, sizew == size, err_out, + "vsnprintf() failed"); + va_end(ap); + + buf[size] = '\0'; + + return fy_parser_set_malloc_string(fyp, buf, size); + +err_out: + return -1; +} + +static struct fy_document *fy_document_build_internal(const struct fy_parse_cfg *cfg, + int (*parser_setup)(struct fy_parser *fyp, void *user), + void *user) +{ + struct fy_parser fyp_data, *fyp = &fyp_data; + struct fy_document *fyd = NULL; + struct fy_eventp *fyep; + bool got_stream_end; + int rc; + + if (!parser_setup) + return NULL; + + if (!cfg) + cfg = &doc_parse_default_cfg; + + rc = fy_parse_setup(fyp, cfg); + if (rc) + return NULL; + + rc = (*parser_setup)(fyp, user); + fyp_error_check(fyp, !rc, err_out, + "parser_setup() failed"); + + fyd = fy_parse_load_document(fyp); + + /* we're going to handle stream errors from now */ + if (!fyd) + fyp->stream_error = false; + + /* if we collect diagnostics, we can continue */ + fyp_error_check(fyp, fyd || (fyp->cfg.flags & FYPCF_COLLECT_DIAG), err_out, + "fy_parse_load_document() failed"); + + /* no document, but we're collecting diagnostics */ + if (!fyd) { + + fyp_error(fyp, "fy_parse_load_document() failed"); + + fyp->stream_error = false; + fyd = fy_parse_document_create(fyp, NULL); + fyp_error_check(fyp, fyd, err_out, + "fy_parse_document_create() failed"); + fyd->parse_error = true; + + /* XXX */ + goto out; + } + + got_stream_end = false; + while (!got_stream_end && (fyep = fy_parse_private(fyp)) != NULL) { + if (fyep->e.type == FYET_STREAM_END) + got_stream_end = true; + fy_parse_eventp_recycle(fyp, fyep); + } + + if (got_stream_end) { + fyep = fy_parse_private(fyp); + fyp_error_check(fyp, !fyep, err_out, + "more events after stream end"); + fy_parse_eventp_recycle(fyp, fyep); + } + +out: + fy_parse_cleanup(fyp); + return fyd; + +err_out: + fy_document_destroy(fyd); + fy_parse_cleanup(fyp); + return NULL; +} + +struct fy_document *fy_document_build_from_string(const struct fy_parse_cfg *cfg, + const char *str, size_t len) +{ + struct fy_document_build_string_ctx ctx = { + .str = str, + .len = len, + }; + + return fy_document_build_internal(cfg, parser_setup_from_string, &ctx); +} + +struct fy_document *fy_document_build_from_malloc_string(const struct fy_parse_cfg *cfg, + char *str, size_t len) +{ + struct fy_document_build_malloc_string_ctx ctx = { + .str = str, + .len = len, + }; + + return fy_document_build_internal(cfg, parser_setup_from_malloc_string, &ctx); +} + +struct fy_document *fy_document_build_from_file(const struct fy_parse_cfg *cfg, + const char *file) +{ + struct fy_document_build_file_ctx ctx = { + .file = file, + }; + + return fy_document_build_internal(cfg, parser_setup_from_file, &ctx); +} + +struct fy_document *fy_document_build_from_fp(const struct fy_parse_cfg *cfg, + FILE *fp) +{ + struct fy_document_build_fp_ctx ctx = { + .name = NULL, + .fp = fp, + }; + + return fy_document_build_internal(cfg, parser_setup_from_fp, &ctx); +} + +enum fy_node_type fy_node_get_type(struct fy_node *fyn) +{ + /* a NULL is a plain scalar node */ + return fyn ? fyn->type : FYNT_SCALAR; +} + +enum fy_node_style fy_node_get_style(struct fy_node *fyn) +{ + /* a NULL is a plain scalar node */ + return fyn ? fyn->style : FYNS_PLAIN; +} + +struct fy_token *fy_node_get_start_token(struct fy_node *fyn) +{ + if (!fyn) + return NULL; + + switch (fyn->type) { + case FYNT_MAPPING: + return fyn->mapping_start; + case FYNT_SEQUENCE: + return fyn->sequence_start; + case FYNT_SCALAR: + return fyn->scalar; + default: + /* should not happen, but play it safe */ + break; + } + return NULL; +} + +struct fy_token *fy_node_get_end_token(struct fy_node *fyn) +{ + if (!fyn) + return NULL; + + switch (fyn->type) { + case FYNT_MAPPING: + return fyn->mapping_end; + case FYNT_SEQUENCE: + return fyn->sequence_end; + case FYNT_SCALAR: + return fyn->scalar; + default: + /* should not happen, but play it safe */ + break; + } + return NULL; +} + +bool fy_node_is_null(struct fy_node *fyn) +{ + if (!fyn) + return true; + + if (fyn->type != FYNT_SCALAR) + return false; + + return fyn->scalar == NULL || fyn->scalar->scalar.is_null; +} + +bool fy_node_is_attached(struct fy_node *fyn) +{ + return fyn ? fyn->attached : false; +} + +struct fy_node *fy_node_get_parent(struct fy_node *fyn) +{ + return fyn && !fyn->key_root ? fyn->parent : NULL; +} + +struct fy_node *fy_node_get_document_parent(struct fy_node *fyn) +{ + return fyn ? fyn->parent : NULL; +} + +struct fy_token *fy_node_get_tag_token(struct fy_node *fyn) +{ + return fyn ? fyn->tag : NULL; +} + +struct fy_token *fy_node_get_scalar_token(struct fy_node *fyn) +{ + return fyn && fyn->type == FYNT_SCALAR ? fyn->scalar : NULL; +} + +struct fy_node *fy_node_pair_key(struct fy_node_pair *fynp) +{ + return fynp ? fynp->key : NULL; +} + +struct fy_node *fy_node_pair_value(struct fy_node_pair *fynp) +{ + return fynp ? fynp->value : NULL; +} + +int fy_node_pair_set_key(struct fy_node_pair *fynp, struct fy_node *fyn) +{ + struct fy_node *fyn_map; + struct fy_node_pair *fynpi; + + if (!fynp) + return -1; + + /* the node must not be attached */ + if (fyn && fyn->attached) + return -1; + + /* sanity check and duplication check */ + fyn_map = fynp->parent; + if (fyn_map) { + + /* (in)sanity check */ + if (!fy_node_is_mapping(fyn_map)) + return -1; + + if (fyn_map->xl) { + /* either we're already on the parent list (and it's OK) */ + /* or we're not and we have a duplicate key */ + fynpi = fy_node_accel_lookup_by_node(fyn_map, fyn); + if (fynpi && fynpi != fynp) + return -1; + /* remove that key */ + fy_accel_remove(fyn_map->xl, fynp->key); + } else { + /* check whether the key is a duplicate + * skipping ourselves since our key gets replaced + */ + for (fynpi = fy_node_pair_list_head(&fyn_map->mapping); fynpi; + fynpi = fy_node_pair_next(&fyn_map->mapping, fynpi)) { + + if (fynpi != fynp && fy_node_compare(fynpi->key, fyn)) + return -1; + } + } + + fy_node_mark_synthetic(fyn_map); + } + + fy_node_detach_and_free(fynp->key); + fynp->key = fyn; + + if (fyn_map && fyn_map->xl) + fy_accel_insert(fyn_map->xl, fynp->key, fynp); + + fyn->attached = true; + + return 0; +} + +int fy_node_pair_set_value(struct fy_node_pair *fynp, struct fy_node *fyn) +{ + if (!fynp) + return -1; + /* the node must not be attached */ + if (fyn && fyn->attached) + return -1; + fy_node_detach_and_free(fynp->value); + fynp->value = fyn; + fyn->attached = true; + + if (fynp->parent) + fy_node_mark_synthetic(fynp->parent); + + return 0; +} + +struct fy_node *fy_document_root(struct fy_document *fyd) +{ + if (!fyd) + return NULL; + + return fyd->root; +} + +const char *fy_node_get_tag(struct fy_node *fyn, size_t *lenp) +{ + size_t tmplen; + + if (!lenp) + lenp = &tmplen; + + if (!fyn || !fyn->tag) { + *lenp = 0; + return NULL; + } + + return fy_token_get_text(fyn->tag, lenp); +} + +const char *fy_node_get_tag0(struct fy_node *fyn) +{ + if (!fyn || !fyn->tag) + return NULL; + + return fy_token_get_text0(fyn->tag); +} + +size_t fy_node_get_tag_length(struct fy_node *fyn) +{ + + if (!fyn || !fyn->tag) + return 0; + + return fy_token_get_text_length(fyn->tag); +} + +const char *fy_node_get_scalar(struct fy_node *fyn, size_t *lenp) +{ + size_t tmplen; + + if (!lenp) + lenp = &tmplen; + + if (!fyn || fyn->type != FYNT_SCALAR) { + *lenp = 0; + return NULL; + } + + return fy_token_get_text(fyn->scalar, lenp); +} + +const char *fy_node_get_scalar0(struct fy_node *fyn) +{ + if (!fyn || fyn->type != FYNT_SCALAR) + return NULL; + + return fy_token_get_text0(fyn->scalar); +} + +size_t fy_node_get_scalar_length(struct fy_node *fyn) +{ + + if (!fyn || fyn->type != FYNT_SCALAR) + return 0; + + return fy_token_get_text_length(fyn->scalar); +} + +size_t fy_node_get_scalar_utf8_length(struct fy_node *fyn) +{ + + if (!fyn || fyn->type != FYNT_SCALAR) + return 0; + + return fy_token_format_utf8_length(fyn->scalar); +} + +struct fy_node *fy_node_sequence_iterate(struct fy_node *fyn, void **prevp) +{ + if (!fyn || fyn->type != FYNT_SEQUENCE || !prevp) + return NULL; + + return *prevp = *prevp ? fy_node_next(&fyn->sequence, *prevp) : fy_node_list_head(&fyn->sequence); +} + +struct fy_node *fy_node_sequence_reverse_iterate(struct fy_node *fyn, void **prevp) +{ + if (!fyn || fyn->type != FYNT_SEQUENCE || !prevp) + return NULL; + + return *prevp = *prevp ? fy_node_prev(&fyn->sequence, *prevp) : fy_node_list_tail(&fyn->sequence); +} + +int fy_node_sequence_item_count(struct fy_node *fyn) +{ + struct fy_node *fyni; + int count; + + if (!fyn || fyn->type != FYNT_SEQUENCE) + return -1; + + count = 0; + for (fyni = fy_node_list_head(&fyn->sequence); fyni; fyni = fy_node_next(&fyn->sequence, fyni)) + count++; + return count; +} + +bool fy_node_sequence_is_empty(struct fy_node *fyn) +{ + return !fyn || fyn->type != FYNT_SEQUENCE || fy_node_list_empty(&fyn->sequence); +} + +struct fy_node *fy_node_sequence_get_by_index(struct fy_node *fyn, int index) +{ + struct fy_node *fyni; + void *iterp = NULL; + + if (!fyn || fyn->type != FYNT_SEQUENCE) + return NULL; + + if (index >= 0) { + do { + fyni = fy_node_sequence_iterate(fyn, &iterp); + } while (fyni && --index >= 0); + } else { + do { + fyni = fy_node_sequence_reverse_iterate(fyn, &iterp); + } while (fyni && ++index < 0); + } + + return fyni; +} + +struct fy_node_pair *fy_node_mapping_iterate(struct fy_node *fyn, void **prevp) +{ + if (!fyn || fyn->type != FYNT_MAPPING || !prevp) + return NULL; + + return *prevp = *prevp ? fy_node_pair_next(&fyn->mapping, *prevp) : fy_node_pair_list_head(&fyn->mapping); +} + +struct fy_node_pair *fy_node_mapping_reverse_iterate(struct fy_node *fyn, void **prevp) +{ + if (!fyn || fyn->type != FYNT_MAPPING || !prevp) + return NULL; + + return *prevp = *prevp ? fy_node_pair_prev(&fyn->mapping, *prevp) : fy_node_pair_list_tail(&fyn->mapping); +} + +struct fy_node *fy_node_collection_iterate(struct fy_node *fyn, void **prevp) +{ + struct fy_node_pair *fynp; + + if (!fyn || !prevp) + return NULL; + + switch (fyn->type) { + case FYNT_SEQUENCE: + return fy_node_sequence_iterate(fyn, prevp); + + case FYNT_MAPPING: + fynp = fy_node_mapping_iterate(fyn, prevp); + if (!fynp) + return NULL; + return fynp->value; + + case FYNT_SCALAR: + fyn = !*prevp ? fyn : NULL; + *prevp = fyn; + return fyn; + } + + return NULL; +} + + +int fy_node_mapping_item_count(struct fy_node *fyn) +{ + struct fy_node_pair *fynpi; + int count; + + if (!fyn || fyn->type != FYNT_MAPPING) + return -1; + + count = 0; + for (fynpi = fy_node_pair_list_head(&fyn->mapping); fynpi; fynpi = fy_node_pair_next(&fyn->mapping, fynpi)) + count++; + return count; +} + +bool fy_node_mapping_is_empty(struct fy_node *fyn) +{ + return !fyn || fyn->type != FYNT_MAPPING || fy_node_pair_list_empty(&fyn->mapping); +} + +struct fy_node_pair *fy_node_mapping_get_by_index(struct fy_node *fyn, int index) +{ + struct fy_node_pair *fynpi; + void *iterp = NULL; + + if (!fyn || fyn->type != FYNT_MAPPING) + return NULL; + + if (index >= 0) { + do { + fynpi = fy_node_mapping_iterate(fyn, &iterp); + } while (fynpi && --index >= 0); + } else { + do { + fynpi = fy_node_mapping_reverse_iterate(fyn, &iterp); + } while (fynpi && ++index < 0); + } + + return fynpi; +} + +struct fy_node_pair * +fy_node_mapping_lookup_pair_by_simple_key(struct fy_node *fyn, + const char *key, size_t len) +{ + struct fy_node_pair *fynpi; + struct fy_node *fyn_scalar; + + if (!fyn || fyn->type != FYNT_MAPPING || !key) + return NULL; + + if (len == (size_t)-1) + len = strlen(key); + + if (fyn->xl) { + fyn_scalar = fy_node_create_scalar(fyn->fyd, key, len); + if (!fyn_scalar) + return NULL; + + fynpi = fy_node_accel_lookup_by_node(fyn, fyn_scalar); + + fy_node_free(fyn_scalar); + + if (fynpi) + return fynpi; + } else { + for (fynpi = fy_node_pair_list_head(&fyn->mapping); fynpi; + fynpi = fy_node_pair_next(&fyn->mapping, fynpi)) { + + if (!fy_node_is_scalar(fynpi->key) || fy_node_is_alias(fynpi->key)) + continue; + + if (!fynpi->key && len == 0) + return fynpi; + + if (fynpi->key && !fy_token_memcmp(fynpi->key->scalar, key, len)) + return fynpi; + } + } + + return NULL; +} + +struct fy_node * +fy_node_mapping_lookup_value_by_simple_key(struct fy_node *fyn, + const char *key, size_t len) +{ + struct fy_node_pair *fynp; + + fynp = fy_node_mapping_lookup_pair_by_simple_key(fyn, key, len); + return fynp ? fy_node_pair_value(fynp) : NULL; +} + +struct fy_node_pair * +fy_node_mapping_lookup_pair_by_null_key(struct fy_node *fyn) +{ + struct fy_node_pair *fynpi; + + if (!fyn || fyn->type != FYNT_MAPPING) + return NULL; + + /* no acceleration for NULLs */ + for (fynpi = fy_node_pair_list_head(&fyn->mapping); fynpi; + fynpi = fy_node_pair_next(&fyn->mapping, fynpi)) { + + if (fy_node_is_null(fynpi->key)) + return fynpi; + } + + return NULL; +} + +struct fy_node * +fy_node_mapping_lookup_value_by_null_key(struct fy_node *fyn) +{ + struct fy_node_pair *fynp; + + fynp = fy_node_mapping_lookup_pair_by_null_key(fyn); + return fynp ? fy_node_pair_value(fynp) : NULL; +} + +const char * +fy_node_mapping_lookup_scalar_by_simple_key(struct fy_node *fyn, size_t *lenp, + const char *key, size_t keylen) +{ + struct fy_node *fyn_value; + + fyn_value = fy_node_mapping_lookup_value_by_simple_key(fyn, key, keylen); + if (!fyn_value || !fy_node_is_scalar(fyn_value)) + return NULL; + return fy_node_get_scalar(fyn_value, lenp); +} + +const char * +fy_node_mapping_lookup_scalar0_by_simple_key(struct fy_node *fyn, + const char *key, size_t keylen) +{ + struct fy_node *fyn_value; + + fyn_value = fy_node_mapping_lookup_value_by_simple_key(fyn, key, keylen); + if (!fyn_value || !fy_node_is_scalar(fyn_value)) + return NULL; + return fy_node_get_scalar0(fyn_value); +} + +struct fy_node *fy_node_mapping_lookup_value_by_key(struct fy_node *fyn, struct fy_node *fyn_key) +{ + struct fy_node_pair *fynp; + + fynp = fy_node_mapping_lookup_pair(fyn, fyn_key); + return fynp ? fynp->value : NULL; +} + +struct fy_node *fy_node_mapping_lookup_key_by_key(struct fy_node *fyn, struct fy_node *fyn_key) +{ + struct fy_node_pair *fynp; + + fynp = fy_node_mapping_lookup_pair(fyn, fyn_key); + return fynp ? fynp->key : NULL; +} + +struct fy_node_pair * +fy_node_mapping_lookup_pair_by_string(struct fy_node *fyn, const char *key, size_t len) +{ + struct fy_document *fyd; + struct fy_node_pair *fynp; + + /* try quick and dirty simple scan */ + if (is_simple_key(key, len)) + return fy_node_mapping_lookup_pair_by_simple_key(fyn, key, len); + + fyd = fy_document_build_from_string(NULL, key, len); + if (!fyd) + return NULL; + + fynp = fy_node_mapping_lookup_pair(fyn, fy_document_root(fyd)); + + fy_document_destroy(fyd); + + return fynp; +} + +struct fy_node * +fy_node_mapping_lookup_by_string(struct fy_node *fyn, + const char *key, size_t len) +{ + struct fy_node_pair *fynp; + + fynp = fy_node_mapping_lookup_pair_by_string(fyn, key, len); + return fynp ? fynp->value : NULL; +} + +struct fy_node * +fy_node_mapping_lookup_value_by_string(struct fy_node *fyn, + const char *key, size_t len) +{ + return fy_node_mapping_lookup_by_string(fyn, key, len); +} + +struct fy_node * +fy_node_mapping_lookup_key_by_string(struct fy_node *fyn, + const char *key, size_t len) +{ + struct fy_node_pair *fynp; + + fynp = fy_node_mapping_lookup_pair_by_string(fyn, key, len); + return fynp ? fynp->key : NULL; +} + +bool fy_node_is_empty(struct fy_node *fyn) +{ + struct fy_node *fyni; + struct fy_node_pair *fynp; + struct fy_atom *atom; + + /* skip if no value node or token */ + if (!fyn) + return true; + + switch (fyn->type) { + case FYNT_SCALAR: + atom = fy_token_atom(fyn->scalar); + if (atom && !atom->size0 && !atom->empty) + return false; + break; + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) { + if (!fy_node_is_empty(fyni)) + return false; + } + break; + case FYNT_MAPPING: + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; + fynp = fy_node_pair_next(&fyn->mapping, fynp)) { + if (!fy_node_is_empty(fynp->value)) + return false; + } + break; + } + return true; +} + +#define fy_node_walk_ctx_create_a(_max_depth, _mark) \ + ({ \ + unsigned int __max_depth = (_max_depth); \ + struct fy_node_walk_ctx *_ctx; \ + \ + _ctx = alloca(sizeof(*_ctx) + sizeof(struct fy_node *) * __max_depth); \ + _ctx->max_depth = _max_depth; \ + _ctx->next_slot = 0; \ + _ctx->mark = (_mark); \ + _ctx; \ + }) + +static inline void fy_node_walk_mark_start(struct fy_node_walk_ctx *ctx) +{ + ctx->next_slot = 0; +} + +static inline void fy_node_walk_mark_end(struct fy_node_walk_ctx *ctx) +{ + struct fy_node *fyn; + + while (ctx->next_slot > 0) { + fyn = ctx->marked[--ctx->next_slot]; + fyn->marks &= ~ctx->mark; + } +} + +/* fyn is guaranteed to be non NULL and an alias */ +static inline bool fy_node_walk_mark(struct fy_node_walk_ctx *ctx, struct fy_node *fyn) +{ + struct fy_document *fyd = fyn->fyd; + struct fy_token *fyt = NULL; + + switch (fyn->type) { + case FYNT_SCALAR: + fyt = fyn->scalar; + break; + case FYNT_SEQUENCE: + fyt = fyn->sequence_start; + break; + case FYNT_MAPPING: + fyt = fyn->mapping_start; + break; + } + + /* depth error */ + FYD_TOKEN_ERROR_CHECK(fyd, fyt, FYEM_DOC, + ctx->next_slot < ctx->max_depth, err_out, + "max recursion depth exceeded (%u)", ctx->max_depth); + + /* mark found, loop */ + FYD_TOKEN_ERROR_CHECK(fyd, fyt, FYEM_DOC, + !(fyn->marks & ctx->mark), err_out, + "cyclic reference detected"); + + fyn->marks |= ctx->mark; + ctx->marked[ctx->next_slot++] = fyn; + + return true; + +err_out: + return false; +} + +static struct fy_node * +fy_node_follow_aliases(struct fy_node *fyn, enum fy_node_walk_flags flags, bool single) +{ + enum fy_node_walk_flags ptr_flags; + struct fy_ptr_node_list nl; + struct fy_ptr_node *fypn; + + if (!fyn || !fy_node_is_alias(fyn) || !(flags & FYNWF_FOLLOW)) + return fyn; + + ptr_flags = flags & FYNWF_PTR(FYNWF_PTR_MASK); + if (ptr_flags != FYNWF_PTR_YAML && ptr_flags != FYNWF_PTR_YPATH) + return fyn; + + fy_ptr_node_list_init(&nl); + + while (fyn && fy_node_is_alias(fyn)) { + + // fprintf(stderr, "%s: %s\n", __func__, fy_node_get_path_alloca(fyn)); + + /* check for loops */ + if (fy_ptr_node_list_contains(&nl, fyn)) { + fyn = NULL; + break; + } + + /* out of memory? */ + fypn = fy_ptr_node_create(fyn); + if (!fypn) { + fyn = NULL; + break; + } + fy_ptr_node_list_add_tail(&nl, fypn); + + fyn = fy_node_follow_alias(fyn, flags); + + if (single) + break; + } + + /* release */ + while ((fypn = fy_ptr_node_list_pop(&nl)) != NULL) + fy_ptr_node_destroy(fypn); + + return fyn; +} + +struct fy_node *fy_node_resolve_alias(struct fy_node *fyn) +{ + enum fy_node_walk_flags flags; + + if (!fyn) + return NULL; + + flags = FYNWF_FOLLOW | FYNWF_MAXDEPTH_DEFAULT | FYNWF_MARKER_DEFAULT; + if (fyn->fyd->parse_cfg.flags & FYPCF_YPATH_ALIASES) + flags |= FYNWF_PTR_YPATH; + else + flags |= FYNWF_PTR_YAML; + return fy_node_follow_aliases(fyn, flags, false); +} + +struct fy_node *fy_node_dereference(struct fy_node *fyn) +{ + enum fy_node_walk_flags flags; + + if (!fyn || !fy_node_is_alias(fyn)) + return NULL; + + flags = FYNWF_FOLLOW | FYNWF_MAXDEPTH_DEFAULT | FYNWF_MARKER_DEFAULT; + if (fyn->fyd->parse_cfg.flags & FYPCF_YPATH_ALIASES) + flags |= FYNWF_PTR_YPATH; + else + flags |= FYNWF_PTR_YAML; + return fy_node_follow_aliases(fyn, flags, true); +} + +static struct fy_node * +fy_node_by_path_internal(struct fy_node *fyn, + const char *path, size_t pathlen, + enum fy_node_walk_flags flags) +{ + enum fy_node_walk_flags ptr_flags; + struct fy_node *fynt, *fyni; + const char *s, *e, *ss, *ee; + char *end_idx, *json_key, *t, *p, *uri_path; + char c; + int idx, rlen; + size_t len, json_key_len, uri_path_len; + bool has_json_key_esc; + uint8_t code[4]; + int code_length; + bool trailing_slash; + + if (!fyn || !path) + return NULL; + + ptr_flags = flags & FYNWF_PTR(FYNWF_PTR_MASK); + if (ptr_flags == FYNWF_PTR_YPATH) + return fy_node_by_ypath(fyn, path, pathlen); + + s = path; + if (pathlen == (size_t)-1) + pathlen = strlen(path); + e = s + pathlen; + + /* a trailing slash works just like unix and symbolic links + * if it does not exist no symbolic link lookups are performed + * at the end of the operation. + * if it exists they are followed upon resolution + */ + trailing_slash = pathlen > 0 && path[pathlen - 1] == '/'; + + /* and continue on path lookup with the rest */ + + /* skip all prefixed / */ + switch (ptr_flags) { + default: + case FYNWF_PTR_YAML: + while (s < e && *s == '/') + s++; + /* for a last component / always match this one */ + if (s >= e) + goto out; + break; + + case FYNWF_PTR_JSON: + /* "" -> everything here */ + if (s == e) + return fyn; + /* it must have a separator here */ + if (*s != '/') + return NULL; + s++; + break; + + case FYNWF_PTR_RELJSON: + break; + } + + /* fyd_notice(fyn->fyd, "%s:%d following alias @%s \"%.*s\"", + __func__, __LINE__, fy_node_get_path(fyn), (int)(e - s), s); */ + fyn = fy_node_follow_aliases(fyn, flags, true); + + /* scalar can be only last element in the path (it has no key) */ + if (fy_node_is_scalar(fyn)) { + if (*s) + fyn = NULL; /* not end of the path - fail */ + goto out; + } + + /* for a sequence the only allowed key is [n] where n is the index to follow */ + if (fy_node_is_sequence(fyn)) { + + c = -1; + switch (ptr_flags) { + default: + case FYNWF_PTR_YAML: + while (s < e && isspace((unsigned char)*s)) + s++; + + c = *s; + if (c == '[') + s++; + else if (!isdigit(c) && c != '-') + return NULL; + + idx = (int)strtol(s, &end_idx, 10); + + /* no digits found at all */ + if (idx == 0 && end_idx == s) + return NULL; + + s = end_idx; + + while (s < e && isspace((unsigned char)*s)) + s++; + + if (c == '[' && *s++ != ']') + return NULL; + + while (s < e && isspace((unsigned char)*s)) + s++; + + break; + + case FYNWF_PTR_JSON: + case FYNWF_PTR_RELJSON: + + /* special array end - always fails */ + if (*s == '-') + return NULL; + + idx = (int)strtol(s, &end_idx, 10); + + /* no digits found at all */ + if (idx == 0 && end_idx == s) + return NULL; + + /* no negatives */ + if (idx < 0) + return NULL; + + s = end_idx; + + if (s < e && *s == '/') + s++; + + break; + } + + len = e - s; + + fyn = fy_node_sequence_get_by_index(fyn, idx); + if (trailing_slash) + fyn = fy_node_follow_aliases(fyn, flags, false); + fyn = fy_node_by_path_internal(fyn, s, len, flags); + goto out; + } + + /* be a little bit paranoid */ + assert(fy_node_is_mapping(fyn)); + + path = s; + pathlen = (size_t)(e - s); + + switch (ptr_flags) { + default: + case FYNWF_PTR_YAML: + + /* scan ahead for the end of the path component + * note that we don't do UTF8 here, because all the + * escapes are regular ascii characters, i.e. + * '/', '*', '&', '.', '{', '}', '[', ']' and '\\' + */ + + while (s < e) { + c = *s; + /* end of path component? */ + if (c == '/') + break; + s++; + + if (c == '\\') { + /* it must be a valid escape */ + if (s >= e || !strchr("/*&.{}[]\\", *s)) + return NULL; + s++; + } else if (c == '"') { + while (s < e && *s != '"') { + c = *s++; + if (c == '\\' && (s < e && *s == '"')) + s++; + } + /* not a normal double quote end */ + if (s >= e || *s != '"') + return NULL; + s++; + } else if (c == '\'') { + while (s < e && *s != '\'') { + c = *s++; + if (c == '\'' && (s < e && *s == '\'')) + s++; + } + /* not a normal single quote end */ + if (s >= e || *s != '\'') + return NULL; + s++; + } + } + len = s - path; + + fynt = fyn; + fyn = fy_node_mapping_lookup_by_string(fyn, path, len); + + /* failed! last ditch attempt, is there a merge key? */ + if (!fyn && fynt && (flags & FYNWF_FOLLOW) && ptr_flags == FYNWF_PTR_YAML) { + fyn = fy_node_mapping_lookup_by_string(fynt, "<<", 2); + if (!fyn) + goto out; + + if (fy_node_is_alias(fyn)) { + + /* single alias '<<: *foo' */ + fyn = fy_node_mapping_lookup_by_string( + fy_node_follow_aliases(fyn, flags, false), path, len); + + } else if (fy_node_is_sequence(fyn)) { + + /* multi aliases '<<: [ *foo, *bar ]' */ + fynt = fyn; + for (fyni = fy_node_list_head(&fynt->sequence); fyni; + fyni = fy_node_next(&fynt->sequence, fyni)) { + if (!fy_node_is_alias(fyni)) + continue; + fyn = fy_node_mapping_lookup_by_string( + fy_node_follow_aliases(fyni, flags, false), + path, len); + if (fyn) + break; + } + } else + fyn = NULL; + } + break; + + case FYNWF_PTR_JSON: + case FYNWF_PTR_RELJSON: + + has_json_key_esc = false; + while (s < e) { + c = *s; + /* end of path component? */ + if (c == '/') + break; + s++; + if (c == '~') + has_json_key_esc = true; + } + len = s - path; + + if (has_json_key_esc) { + /* note that the escapes reduce the length, so allocating the + * same size is guaranteed safe */ + json_key = alloca(len + 1); + + ss = path; + ee = s; + t = json_key; + while (ss < ee) { + if (*ss != '~') { + *t++ = *ss++; + continue; + } + /* unterminated ~ escape, or neither ~0, ~1 */ + if (ss + 1 >= ee || (ss[1] < '0' && ss[1] > '1')) + return NULL; + *t++ = ss[1] == '0' ? '~' : '/'; + ss += 2; + } + json_key_len = t - json_key; + + path = json_key; + len = json_key_len; + } + + /* URI encoded escaped */ + if ((flags & FYNWF_URI_ENCODED) && memchr(path, '%', len)) { + /* escapes shrink, so safe to allocate as much */ + uri_path = alloca(len + 1); + + ss = path; + ee = path + len; + t = uri_path; + while (ss < ee) { + /* copy run until '%' or end */ + p = memchr(ss, '%', ee - ss); + rlen = (p ? p : ee) - ss; + memcpy(t, ss, rlen); + ss += rlen; + t += rlen; + + /* if end, break */ + if (!p) + break; + + /* collect a utf8 character sequence */ + code_length = sizeof(code); + ss = fy_uri_esc(ss, ee - ss, code, &code_length); + if (!ss) { + /* bad % escape sequence */ + return NULL; + } + memcpy(t, code, code_length); + t += code_length; + } + uri_path_len = t - uri_path; + + path = uri_path; + len = uri_path_len; + } + + fynt = fyn; + fyn = fy_node_mapping_lookup_value_by_simple_key(fyn, path, len); + break; + } + + len = e - s; + + if (len > 0 && trailing_slash) { + /* fyd_notice(fyn->fyd, "%s:%d following alias @%s \"%.*s\"", + __func__, __LINE__, fy_node_get_path(fyn), (int)(e - s), s); */ + fyn = fy_node_follow_aliases(fyn, flags, true); + } + + fyn = fy_node_by_path_internal(fyn, s, len, flags); + +out: + len = e - s; + + if (len > 0 && trailing_slash) { + /* fyd_notice(fyn->fyd, "%s:%d following alias @%s \"%.*s\"", + __func__, __LINE__, fy_node_get_path(fyn), (int)(e - s), s); */ + fyn = fy_node_follow_aliases(fyn, flags, true); + } + + return fyn; +} + +struct fy_node *fy_node_by_path(struct fy_node *fyn, + const char *path, size_t len, + enum fy_node_walk_flags flags) +{ + struct fy_document *fyd; + struct fy_anchor *fya; + const char *s, *e, *t, *anchor; + size_t alen; + char c; + int idx, w; + char *end_idx; + + if (!fyn || !path) + return NULL; + + if (len == (size_t)-1) + len = strlen(path); + + /* verify that the path string is well formed UTF8 */ + s = path; + e = s + len; + + fyd = fyn->fyd; + while (s < e) { + c = fy_utf8_get(s, e - s, &w); + if (c < 0) { + fyd_error(fyd, "fy_node_by_path() malformed path string\n"); + return NULL; + } + s += w; + } + + /* rewind */ + s = path; + + /* if it's a YPATH, just punt to that method */ + if ((flags & FYNWF_PTR(FYNWF_PTR_MASK)) == FYNWF_PTR_YPATH) + return fy_node_by_ypath(fyn, path, len); + + /* fyd_notice(fyn->fyd, "%s: %.*s", __func__, (int)(len), s); */ + + /* first path component may be an alias */ + if ((flags & FYNWF_FOLLOW) && fyn && path) { + while (s < e && isspace((unsigned char)*s)) + s++; + + if (s >= e || *s != '*') + goto regular_path_lookup; + + s++; + + c = -1; + for (t = s; t < e; t++) { + c = *t; + /* it ends on anything non alias */ + if (c == '[' || c == ']' || + c == '{' || c == '}' || + c == ',' || c == ' ' || c == '\t' || + c == '/') + break; + } + + /* bad alias form for path */ + if (c == '[' || c == ']' || c == '{' || c == '}' || c == ',') + return NULL; + + anchor = s; + alen = t - s; + + if (alen) { + /* we must be terminated by '/' or space followed by '/' */ + /* strip until spaces and '/' end */ + while (t < e && (*t == ' ' || *t == '\t')) + t++; + + while (t < e && *t == '/') + t++; + + /* update path */ + path = t; + len = e - t; + + /* fyd_notice(fyn->fyd, "%s: looking up anchor=%.*s", __func__, (int)(alen), anchor); */ + + /* lookup anchor */ + fya = fy_document_lookup_anchor(fyn->fyd, anchor, alen); + if (!fya) { + /* fyd_notice(fyn->fyd, "%s: failed to lookup anchor=%.*s", __func__, (int)(alen), anchor); */ + return NULL; + } + + /* fyd_notice(fyn->fyd, "%s: found anchor=%.*s at %s", + __func__, (int)(alen), anchor, fy_node_get_path(fya->fyn)); */ + + /* nothing more? we're done */ + if (*path == '\0') + return fya->fyn; + + /* anchor found... all good */ + + fyn = fya->fyn; + } else { + /* no anchor it must be of the form *\/ */ + + path = s; + len = e - s; + } + + /* fyd_notice(fyn->fyd, "%s: continuing looking for %.*s", + __func__, (int)(len), path); */ + + } + +regular_path_lookup: + + /* if it's a relative json pointer... */ + if ((flags & FYNWF_PTR(FYNWF_PTR_MASK)) == FYNWF_PTR_RELJSON) { + + /* it must at least be one digit */ + if (len == 0) + return NULL; + + idx = (int)strtol(path, &end_idx, 10); + + /* at least one digit must exist */ + if (idx == 0 && path == end_idx) + return NULL; + + e = path + len; + len = e - end_idx; + path = end_idx; + + /* we don't do the trailing # here */ + if (len == 1 && *path == '#') + return NULL; + + while (idx-- > 0) + fyn = fy_node_get_parent(fyn); + + /* convert to regular json pointer from now on */ + flags &= ~FYNWF_PTR(FYNWF_PTR_MASK); + flags |= FYNWF_PTR_JSON; + } + + return fy_node_by_path_internal(fyn, path, len, flags); +} + +static char * +fy_node_get_reference_internal(struct fy_node *fyn_base, struct fy_node *fyn, bool near) +{ + struct fy_anchor *fya; + const char *path; + char *path2, *path3; + const char *text; + size_t len; + + if (!fyn) + return NULL; + + path2 = NULL; + + /* if the node has an anchor use it (ie return *foo) */ + if (!fyn_base && (fya = fy_node_get_anchor(fyn)) != NULL) { + text = fy_anchor_get_text(fya, &len); + if (!text) + return NULL; + path2 = alloca(1 + len + 1); + path2[0] = '*'; + memcpy(path2 + 1, text, len); + path2[len + 1] = '\0'; + + } else { + + fya = fyn_base ? fy_node_get_anchor(fyn_base) : NULL; + if (!fya && near) + fya = fy_node_get_nearest_anchor(fyn); + if (!fya) { + /* no anchor, direct reference (ie return *\/foo\/bar */ + path = fy_node_get_path_alloca(fyn); + if (!*path) + return NULL; + path2 = alloca(1 + strlen(path) + 1); + path2[0] = '*'; + strcpy(path2 + 1, path); + } else { + text = fy_anchor_get_text(fya, &len); + if (!text) + return NULL; + if (fy_anchor_node(fya) != fyn) { + path = fy_node_get_path_relative_to_alloca(fy_anchor_node(fya), fyn); + if (*path) { + /* we have a relative path */ + path2 = alloca(1 + len + 1 + strlen(path) + 1); + path2[0] = '*'; + memcpy(path2 + 1, text, len); + path2[len + 1] = '/'; + memcpy(1 + path2 + len + 1, path, strlen(path) + 1); + } else { + /* absolute path */ + path = fy_node_get_path_alloca(fyn); + if (!*path) + return NULL; + path2 = alloca(1 + strlen(path) + 1); + path2[0] = '*'; + strcpy(path2 + 1, path); + } + } else { + path2 = alloca(1 + len + 1); + path2[0] = '*'; + memcpy(path2 + 1, text, len); + path2[len + 1] = '\0'; + } + } + } + + if (!path2) + return NULL; + + path3 = strdup(path2); + if (!path3) + return NULL; + + return path3; +} + +char *fy_node_get_reference(struct fy_node *fyn) +{ + return fy_node_get_reference_internal(NULL, fyn, false); +} + +struct fy_node *fy_node_create_reference(struct fy_node *fyn) +{ + struct fy_node *fyn_ref; + char *path, *alias; + + path = fy_node_get_reference(fyn); + if (!path) + return NULL; + + alias = path; + if (*alias == '*') + alias++; + + fyn_ref = fy_node_create_alias_copy(fy_node_document(fyn), alias, FY_NT); + + free(path); + + return fyn_ref; +} + +char *fy_node_get_relative_reference(struct fy_node *fyn_base, struct fy_node *fyn) +{ + return fy_node_get_reference_internal(fyn_base, fyn, false); +} + +struct fy_node *fy_node_create_relative_reference(struct fy_node *fyn_base, struct fy_node *fyn) +{ + struct fy_node *fyn_ref; + char *path, *alias; + + path = fy_node_get_relative_reference(fyn_base, fyn); + if (!path) + return NULL; + + alias = path; + if (*alias == '*') + alias++; + + fyn_ref = fy_node_create_alias_copy(fy_node_document(fyn), alias, FY_NT); + + free(path); + + return fyn_ref; +} + +bool fy_check_ref_loop(struct fy_document *fyd, struct fy_node *fyn, + enum fy_node_walk_flags flags, + struct fy_node_walk_ctx *ctx) +{ + struct fy_node *fyni; + struct fy_node_pair *fynp, *fynpi; + struct fy_node_walk_ctx *ctxn; + bool ret; + + if (!fyn) + return false; + + /* visited? no need to check */ + if (fyn->marks & FY_BIT(FYNWF_VISIT_MARKER)) + return false; + + /* marked node, it's a loop */ + if (ctx && !fy_node_walk_mark(ctx, fyn)) + return true; + + ret = false; + + switch (fyn->type) { + case FYNT_SCALAR: + + /* if it's not an alias, we're done */ + if (!fy_node_is_alias(fyn)) + break; + + ctxn = ctx; + if (!ctxn) + ctxn = fy_node_walk_ctx_create_a( + fy_node_walk_max_depth_from_flags(flags), FYNWF_REF_MARKER); + + + if (!ctx) { + fy_node_walk_mark_start(ctxn); + + /* mark this node */ + fy_node_walk_mark(ctxn, fyn); + } + + fyni = fy_node_follow_alias(fyn, flags); + + ret = fy_check_ref_loop(fyd, fyni, flags, ctxn); + + if (!ctx) + fy_node_walk_mark_end(ctxn); + + if (ret) + break; + + break; + + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) { + + ret = fy_check_ref_loop(fyd, fyni, flags, ctx); + if (ret) + break; + } + break; + + case FYNT_MAPPING: + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; fynp = fynpi) { + + fynpi = fy_node_pair_next(&fyn->mapping, fynp); + + ret = fy_check_ref_loop(fyd, fynp->key, flags, ctx); + if (ret) + break; + + ret = fy_check_ref_loop(fyd, fynp->value, flags, ctx); + if (ret) + break; + } + break; + } + + /* mark as visited */ + fyn->marks |= FY_BIT(FYNWF_VISIT_MARKER); + + return ret; +} + +char *fy_node_get_parent_address(struct fy_node *fyn) +{ + struct fy_node *parent, *fyni; + struct fy_node_pair *fynp; + struct fy_node *fyna; + char *path = NULL; + const char *str; + size_t len; + int idx; + bool is_key_root; + int ret; + const char *fmt; + char *new_path, *old_path; + + if (!fyn) + return NULL; + + parent = fy_node_get_document_parent(fyn); + if (!parent) + return NULL; + + if (fy_node_is_sequence(parent)) { + + /* for a sequence, find the index */ + idx = 0; + for (fyni = fy_node_list_head(&parent->sequence); fyni; + fyni = fy_node_next(&parent->sequence, fyni)) { + if (fyni == fyn) + break; + idx++; + } + + if (!fyni) + return NULL; + + ret = asprintf(&path, "%d", idx); + if (ret == -1) + return NULL; + } + + if (fy_node_is_mapping(parent)) { + + is_key_root = fyn->key_root; + + idx = 0; + fyna = NULL; + for (fynp = fy_node_pair_list_head(&parent->mapping); fynp; + fynp = fy_node_pair_next(&parent->mapping, fynp)) { + + if ((!is_key_root && fynp->value == fyn) || (is_key_root && fynp->key == fyn)) + break; + idx++; + } + + if (!fynp) + return NULL; + + fyna = fynp->key; + if (!fyna) + return NULL; + + /* if key is a plain scalar try to not use a complex style (even for quoted) */ + if (fyna && fy_node_is_scalar(fyna) && !fy_node_is_alias(fyna) && + (str = fy_token_get_scalar_path_key(fyna->scalar, &len)) != NULL) { + + fmt = !is_key_root ? "%.*s" : ".key(%.*s)"; + ret = asprintf(&path, fmt, (int)len, str); + if (ret == -1) + return NULL; + + } else { + + /* something complex, emit it */ + path = fy_emit_node_to_string(fyna, + FYECF_MODE_FLOW_ONELINE | FYECF_WIDTH_INF | + FYECF_STRIP_LABELS | FYECF_STRIP_TAGS | + FYECF_NO_ENDING_NEWLINE); + if (!path) + return NULL; + + if (is_key_root) { + old_path = path; + ret = asprintf(&new_path, ".key(%s)", path); + if (ret == -1) { + free(path); + return NULL; + } + free(old_path); + path = new_path; + } + } + } + + return path; +} + +char *fy_node_get_path(struct fy_node *fyn) +{ + struct path_track { + struct path_track *prev; + char *path; + }; + struct path_track *track, *newtrack; + char *path, *s, *path_mem; + size_t len; + struct fy_node *parent; + + if (!fyn) + return NULL; + + /* easy on the root */ + parent = fy_node_get_document_parent(fyn); + if (!parent) { + path_mem = strdup("/"); + return path_mem; + } + + track = NULL; + len = 0; + while ((path = fy_node_get_parent_address(fyn))) { + newtrack = alloca(sizeof(*newtrack)); + newtrack->prev = track; + newtrack->path = path; + + track = newtrack; + + len += strlen(path) + 1; + + fyn = fy_node_get_document_parent(fyn); + } + len += 2; + + path_mem = malloc(len); + + s = path_mem; + + while (track) { + len = strlen(track->path); + if (s) { + *s++ = '/'; + memcpy(s, track->path, len); + s += len; + } + free(track->path); + track = track->prev; + } + + if (s) + *s = '\0'; + + return path_mem; +} + +char *fy_node_get_path_relative_to(struct fy_node *fyn_parent, struct fy_node *fyn) +{ + char *path, *ppath, *path2, *path_ret; + size_t pathlen, ppathlen; + struct fy_node *ni, *nj; + + if (!fyn) + return NULL; + + /* must be on the same document */ + if (fyn_parent && (fyn_parent->fyd != fyn->fyd)) + return NULL; + + if (!fyn_parent) + fyn_parent = fyn->fyd->root; + + /* verify that it's a parent */ + ni = fyn; + while ((nj = fy_node_get_parent(ni)) != NULL && nj != fyn_parent) + ni = nj; + + /* not a parent, illegal */ + if (!nj) + return NULL; + + /* here we go... */ + path = ""; + pathlen = 0; + + ni = fyn; + while ((nj = fy_node_get_parent(ni)) != NULL) { + ppath = fy_node_get_parent_address(ni); + if (!ppath) + return NULL; + + ppathlen = strlen(ppath); + + if (pathlen > 0) { + path2 = alloca(pathlen + 1 + ppathlen + 1); + memcpy(path2, ppath, ppathlen); + path2[ppathlen] = '/'; + memcpy(path2 + ppathlen + 1, path, pathlen); + path2[ppathlen + 1 + pathlen] = '\0'; + } else { + path2 = alloca(ppathlen + 1); + memcpy(path2, ppath, ppathlen); + path2[ppathlen] = '\0'; + } + + path = path2; + pathlen = strlen(path); + + free(ppath); + ni = nj; + + if (ni == fyn_parent) + break; + } + + path_ret = strdup(path); + return path_ret; +} + +char *fy_node_get_short_path(struct fy_node *fyn) +{ + struct fy_node *fyn_anchor; + struct fy_anchor *fya; + const char *text; + size_t len; + char *str, *path; + + if (!fyn) + return NULL; + + /* get the nearest anchor traversing upwards */ + fya = fy_node_get_nearest_anchor(fyn); + if (!fya) + return fy_node_get_path(fyn); + + fyn_anchor = fy_anchor_node(fya); + + text = fy_anchor_get_text(fya, &len); + if (!text) + return NULL; + + if (fyn_anchor == fyn) + str = fy_sprintfa("*%.*s", (int)len, text); + else + str = fy_sprintfa("*%.*s/%s", (int)len, text, + fy_node_get_path_relative_to_alloca(fyn_anchor, fyn)); + fy_strip_trailing_nl(str); + + path = strdup(str); + return path; +} + +static struct fy_node * +fy_document_load_node(struct fy_document *fyd, struct fy_parser *fyp, + struct fy_document_state **fydsp) +{ + struct fy_eventp *fyep = NULL; + struct fy_event *fye = NULL; + struct fy_node *fyn = NULL; + int rc, depth; + bool was_stream_start; + + if (!fyd || !fyp) + return NULL; + + /* only single documents */ + fy_parser_set_next_single_document(fyp); + fy_parser_set_default_document_state(fyp, fyd->fyds); + +again: + was_stream_start = false; + do { + /* get next event */ + fyep = fy_parse_private(fyp); + + /* no more */ + if (!fyep) + return NULL; + + was_stream_start = fyep->e.type == FYET_STREAM_START; + + if (was_stream_start) { + fy_parse_eventp_recycle(fyp, fyep); + fyep = NULL; + } + + } while (was_stream_start); + + fye = &fyep->e; + + /* STREAM_END */ + if (fye->type == FYET_STREAM_END) { + fy_parse_eventp_recycle(fyp, fyep); + + /* final STREAM_END? */ + if (fyp->state == FYPS_END) + return NULL; + + /* multi-stream */ + goto again; + } + + FYD_TOKEN_ERROR_CHECK(fyd, fy_event_get_token(fye), FYEM_DOC, + fye->type == FYET_DOCUMENT_START, err_out, + "bad event"); + + fy_parse_eventp_recycle(fyp, fyep); + fyep = NULL; + fye = NULL; + + fyd_doc_debug(fyd, "calling load_node() for root"); + depth = 0; + rc = fy_parse_document_load_node(fyp, fyd, fy_parse_private(fyp), &fyn, &depth); + fyd_error_check(fyd, !rc, err_out, + "fy_parse_document_load_node() failed"); + + rc = fy_parse_document_load_end(fyp, fyd, fy_parse_private(fyp)); + fyd_error_check(fyd, !rc, err_out, + "fy_parse_document_load_node() failed"); + + /* always resolve parents */ + fy_resolve_parent_node(fyd, fyn, NULL); + + if (fydsp) + *fydsp = fy_document_state_ref(fyp->current_document_state); + + return fyn; + +err_out: + fy_parse_eventp_recycle(fyp, fyep); + fyd->diag->on_error = false; + return NULL; +} + +static struct fy_node * +fy_node_build_internal(struct fy_document *fyd, + int (*parser_setup)(struct fy_parser *fyp, void *user), + void *user) +{ + struct fy_document_state *fyds = NULL; + struct fy_node *fyn = NULL; + struct fy_parser fyp_data, *fyp = &fyp_data; + struct fy_parse_cfg cfg; + struct fy_eventp *fyep; + int rc; + bool got_stream_end; + + if (!fyd || !parser_setup) + return NULL; + + cfg = fyd->parse_cfg; + cfg.diag = fyd->diag; + rc = fy_parse_setup(fyp, &cfg); + if (rc) { + fyd->diag->on_error = false; + return NULL; + } + + rc = (*parser_setup)(fyp, user); + fyd_error_check(fyd, !rc, err_out, + "parser_setup() failed"); + + fyn = fy_document_load_node(fyd, fyp, &fyds); + fyd_error_check(fyd, fyn, err_out, + "fy_document_load_node() failed"); + + got_stream_end = false; + while (!got_stream_end && (fyep = fy_parse_private(fyp)) != NULL) { + if (fyep->e.type == FYET_STREAM_END) + got_stream_end = true; + fy_parse_eventp_recycle(fyp, fyep); + } + + if (got_stream_end) { + fyep = fy_parse_private(fyp); + + FYD_TOKEN_ERROR_CHECK(fyd, fy_event_get_token(&fyep->e), FYEM_DOC, + !fyep, err_out, + "trailing events after the last"); + + fy_parse_eventp_recycle(fyp, fyep); + } + + rc = fy_document_state_merge(fyd->fyds, fyds); + fyd_error_check(fyd, !rc, err_out, + "fy_document_state_merge() failed"); + + fy_document_state_unref(fyds); + + fy_parse_cleanup(fyp); + + return fyn; + +err_out: + fy_node_detach_and_free(fyn); + fy_document_state_unref(fyds); + fy_parse_cleanup(fyp); + fyd->diag->on_error = false; + return NULL; +} + +struct fy_node *fy_node_build_from_string(struct fy_document *fyd, const char *str, size_t len) +{ + struct fy_document_build_string_ctx ctx = { + .str = str, + .len = len, + }; + + return fy_node_build_internal(fyd, parser_setup_from_string, &ctx); +} + +struct fy_node *fy_node_build_from_malloc_string(struct fy_document *fyd, char *str, size_t len) +{ + struct fy_document_build_malloc_string_ctx ctx = { + .str = str, + .len = len, + }; + + return fy_node_build_internal(fyd, parser_setup_from_malloc_string, &ctx); +} + +struct fy_node *fy_node_build_from_file(struct fy_document *fyd, const char *file) +{ + struct fy_document_build_file_ctx ctx = { + .file = file, + }; + + return fy_node_build_internal(fyd, parser_setup_from_file, &ctx); +} + +struct fy_node *fy_node_build_from_fp(struct fy_document *fyd, FILE *fp) +{ + struct fy_document_build_fp_ctx ctx = { + .name = NULL, + .fp = fp, + }; + + return fy_node_build_internal(fyd, parser_setup_from_fp, &ctx); +} + +int fy_document_set_root(struct fy_document *fyd, struct fy_node *fyn) +{ + if (!fyd) + return -1; + + if (fyn && fyn->attached) + return -1; + + fy_node_detach_and_free(fyd->root); + fyd->root = NULL; + + fyn->parent = NULL; + fyd->root = fyn; + + if (fyn) + fyn->attached = true; + + return 0; +} + +#define FYNCSIF_ALIAS FY_BIT(0) +#define FYNCSIF_SIMPLE FY_BIT(1) +#define FYNCSIF_COPY FY_BIT(2) +#define FYNCSIF_MALLOCED FY_BIT(3) + +static struct fy_node * +fy_node_create_scalar_internal(struct fy_document *fyd, const char *data, size_t size, + unsigned int flags) +{ + const bool alias = !!(flags & FYNCSIF_ALIAS); + const bool simple = !!(flags & FYNCSIF_SIMPLE); + const bool copy = !!(flags & FYNCSIF_COPY); + const bool malloced = !!(flags & FYNCSIF_MALLOCED); + struct fy_node *fyn = NULL; + struct fy_input *fyi; + struct fy_atom handle; + enum fy_scalar_style style; + char *data_copy = NULL; + + if (!fyd) + return NULL; + + if (data && size == (size_t)-1) + size = strlen(data); + + fyn = fy_node_alloc(fyd, FYNT_SCALAR); + fyd_error_check(fyd, fyn, err_out, + "fy_node_alloc() failed"); + + if (copy) { + data_copy = malloc(size); + fyd_error_check(fyd, data_copy, err_out, + "malloc() failed"); + memcpy(data_copy, data, size); + fyi = fy_input_from_malloc_data(data_copy, size, &handle, simple); + } else if (malloced) + fyi = fy_input_from_malloc_data((void *)data, size, &handle, simple); + else + fyi = fy_input_from_data(data, size, &handle, simple); + fyd_error_check(fyd, fyi, err_out, + "fy_input_from_data() failed"); + data_copy = NULL; + + if (!alias) { + style = handle.style == FYAS_PLAIN ? FYSS_PLAIN : FYSS_DOUBLE_QUOTED; + fyn->scalar = fy_token_create(FYTT_SCALAR, &handle, style); + } else + fyn->scalar = fy_token_create(FYTT_ALIAS, &handle, NULL); + + fyd_error_check(fyd, fyn->scalar, err_out, + "fy_token_create() failed"); + + fyn->style = !alias ? (style == FYSS_PLAIN ? FYNS_PLAIN : FYNS_DOUBLE_QUOTED) : FYNS_ALIAS; + + /* take away the input reference */ + fy_input_unref(fyi); + + return fyn; + +err_out: + if (data_copy) + free(data_copy); + fy_node_detach_and_free(fyn); + fyd->diag->on_error = false; + return NULL; +} + +struct fy_node *fy_node_create_scalar(struct fy_document *fyd, const char *data, size_t size) +{ + return fy_node_create_scalar_internal(fyd, data, size, 0); +} + +struct fy_node *fy_node_create_alias(struct fy_document *fyd, const char *data, size_t size) +{ + return fy_node_create_scalar_internal(fyd, data, size, FYNCSIF_ALIAS); +} + +struct fy_node *fy_node_create_scalar_copy(struct fy_document *fyd, const char *data, size_t size) +{ + return fy_node_create_scalar_internal(fyd, data, size, FYNCSIF_COPY); +} + +struct fy_node *fy_node_create_alias_copy(struct fy_document *fyd, const char *data, size_t size) +{ + return fy_node_create_scalar_internal(fyd, data, size, FYNCSIF_ALIAS | FYNCSIF_COPY); +} + +struct fy_node *fy_node_create_vscalarf(struct fy_document *fyd, const char *fmt, va_list ap) +{ + if (!fyd || !fmt) + return NULL; + + return fy_node_create_scalar_internal(fyd, fy_vsprintfa(fmt, ap), FY_NT, FYNCSIF_COPY); +} + +struct fy_node *fy_node_create_scalarf(struct fy_document *fyd, const char *fmt, ...) +{ + va_list ap; + struct fy_node *fyn; + + va_start(ap, fmt); + fyn = fy_node_create_vscalarf(fyd, fmt, ap); + va_end(ap); + + return fyn; +} + +int fy_node_set_tag(struct fy_node *fyn, const char *data, size_t len) +{ + struct fy_document *fyd; + struct fy_tag_scan_info info; + int handle_length, uri_length, prefix_length; + const char *handle_start; + int rc; + struct fy_atom handle; + struct fy_input *fyi = NULL; + struct fy_token *fyt = NULL, *fyt_td = NULL; + + if (!fyn || !data || !len || !fyn->fyd) + return -1; + + fyd = fyn->fyd; + + if (len == (size_t)-1) + len = strlen(data); + + memset(&info, 0, sizeof(info)); + + rc = fy_tag_scan(data, len, &info); + if (rc) + goto err_out; + + handle_length = info.handle_length; + uri_length = info.uri_length; + prefix_length = info.prefix_length; + + handle_start = data + prefix_length; + + fyt_td = fy_document_state_lookup_tag_directive(fyd->fyds, + handle_start, handle_length); + if (!fyt_td) + goto err_out; + + fyi = fy_input_from_data(data, len, &handle, true); + if (!fyi) + goto err_out; + + handle.style = FYAS_URI; + handle.direct_output = false; + handle.storage_hint = 0; + handle.storage_hint_valid = false; + + fyt = fy_token_create(FYTT_TAG, &handle, prefix_length, + handle_length, uri_length, fyt_td); + if (!fyt) + goto err_out; + + fy_token_unref(fyn->tag); + fyn->tag = fyt; + + /* take away the input reference */ + fy_input_unref(fyi); + + return 0; +err_out: + fyd->diag->on_error = false; + return -1; +} + +int fy_node_remove_tag(struct fy_node *fyn) +{ + if (!fyn || !fyn->tag) + return -1; + + fy_token_unref(fyn->tag); + fyn->tag = NULL; + + return 0; +} + +struct fy_node *fy_node_create_sequence(struct fy_document *fyd) +{ + struct fy_node *fyn; + + fyn = fy_node_alloc(fyd, FYNT_SEQUENCE); + if (!fyn) + return NULL; + + return fyn; +} + +struct fy_node *fy_node_create_mapping(struct fy_document *fyd) +{ + struct fy_node *fyn; + + fyn = fy_node_alloc(fyd, FYNT_MAPPING); + if (!fyn) + return NULL; + + return fyn; +} + +static int fy_node_sequence_insert_prepare(struct fy_node *fyn_seq, struct fy_node *fyn) +{ + struct fy_document *fyd; + + if (!fyn_seq || !fyn || fyn_seq->type != FYNT_SEQUENCE) + return -1; + + /* can't insert a node that's attached already */ + if (fyn->attached) + return -1; + + /* a document must be associated with the sequence */ + fyd = fyn_seq->fyd; + if (!fyd) + return -1; + + /* the documents of the nodes must match */ + if (fyn->fyd != fyd) + return -1; + + fyn->parent = fyn_seq; + + return 0; +} + +int fy_node_sequence_append(struct fy_node *fyn_seq, struct fy_node *fyn) +{ + int ret; + + ret = fy_node_sequence_insert_prepare(fyn_seq, fyn); + if (ret) + return ret; + + fy_node_mark_synthetic(fyn_seq); + fy_node_list_add_tail(&fyn_seq->sequence, fyn); + fyn->attached = true; + return 0; +} + +int fy_node_sequence_prepend(struct fy_node *fyn_seq, struct fy_node *fyn) +{ + int ret; + + ret = fy_node_sequence_insert_prepare(fyn_seq, fyn); + if (ret) + return ret; + + fy_node_mark_synthetic(fyn_seq); + fy_node_list_add(&fyn_seq->sequence, fyn); + fyn->attached = true; + return 0; +} + +static bool fy_node_sequence_contains_node(struct fy_node *fyn_seq, struct fy_node *fyn) +{ + struct fy_node *fyni; + + if (!fyn_seq || !fyn || fyn_seq->type != FYNT_SEQUENCE) + return false; + + for (fyni = fy_node_list_head(&fyn_seq->sequence); fyni; fyni = fy_node_next(&fyn_seq->sequence, fyni)) + if (fyni == fyn) + return true; + + return false; +} + +int fy_node_sequence_insert_before(struct fy_node *fyn_seq, + struct fy_node *fyn_mark, struct fy_node *fyn) +{ + int ret; + + if (!fy_node_sequence_contains_node(fyn_seq, fyn_mark)) + return -1; + + ret = fy_node_sequence_insert_prepare(fyn_seq, fyn); + if (ret) + return ret; + + fy_node_mark_synthetic(fyn_seq); + fy_node_list_insert_before(&fyn_seq->sequence, fyn_mark, fyn); + fyn->attached = true; + + return 0; +} + +int fy_node_sequence_insert_after(struct fy_node *fyn_seq, + struct fy_node *fyn_mark, struct fy_node *fyn) +{ + int ret; + + if (!fy_node_sequence_contains_node(fyn_seq, fyn_mark)) + return -1; + + ret = fy_node_sequence_insert_prepare(fyn_seq, fyn); + if (ret) + return ret; + + fy_node_mark_synthetic(fyn_seq); + fy_node_list_insert_after(&fyn_seq->sequence, fyn_mark, fyn); + fyn->attached = true; + + return 0; +} + +struct fy_node *fy_node_sequence_remove(struct fy_node *fyn_seq, struct fy_node *fyn) +{ + if (!fy_node_sequence_contains_node(fyn_seq, fyn)) + return NULL; + + fy_node_list_del(&fyn_seq->sequence, fyn); + fyn->parent = NULL; + fyn->attached = false; + + fy_node_mark_synthetic(fyn_seq); + + return fyn; +} + +static struct fy_node_pair * +fy_node_mapping_pair_insert_prepare(struct fy_node *fyn_map, + struct fy_node *fyn_key, struct fy_node *fyn_value) +{ + struct fy_document *fyd; + struct fy_node_pair *fynp; + + if (!fyn_map || fyn_map->type != FYNT_MAPPING) + return NULL; + + /* a document must be associated with the mapping */ + fyd = fyn_map->fyd; + if (!fyd) + return NULL; + + /* if not NULL, the documents of the nodes must match */ + if ((fyn_key && fyn_key->fyd != fyd) || + (fyn_value && fyn_value->fyd != fyd)) + return NULL; + + /* if not NULL neither the key nor the value must be attached */ + if ((fyn_key && fyn_key->attached) || + (fyn_value && fyn_value->attached)) + return NULL; + + /* if we don't allow duplicate keys */ + if (!(fyd->parse_cfg.flags & FYPCF_ALLOW_DUPLICATE_KEYS)) { + + if (fy_node_mapping_key_is_duplicate(fyn_map, fyn_key)) + return NULL; + } + + fynp = fy_node_pair_alloc(fyd); + if (!fynp) + return NULL; + + if (fyn_key) { + fyn_key->parent = fyn_map; + fyn_key->key_root = true; + } + if (fyn_value) + fyn_value->parent = fyn_map; + + fynp->key = fyn_key; + fynp->value = fyn_value; + fynp->parent = fyn_map; + + return fynp; +} + +int fy_node_mapping_append(struct fy_node *fyn_map, + struct fy_node *fyn_key, struct fy_node *fyn_value) +{ + struct fy_node_pair *fynp; + + fynp = fy_node_mapping_pair_insert_prepare(fyn_map, fyn_key, fyn_value); + if (!fynp) + return -1; + + fy_node_pair_list_add_tail(&fyn_map->mapping, fynp); + if (fyn_map->xl) + fy_accel_insert(fyn_map->xl , fyn_key, fynp); + + if (fyn_key) + fyn_key->attached = true; + if (fyn_value) + fyn_value->attached = true; + + fy_node_mark_synthetic(fyn_map); + + return 0; +} + +int fy_node_mapping_prepend(struct fy_node *fyn_map, + struct fy_node *fyn_key, struct fy_node *fyn_value) +{ + struct fy_node_pair *fynp; + + fynp = fy_node_mapping_pair_insert_prepare(fyn_map, fyn_key, fyn_value); + if (!fynp) + return -1; + + if (fyn_key) + fyn_key->attached = true; + if (fyn_value) + fyn_value->attached = true; + fy_node_pair_list_add(&fyn_map->mapping, fynp); + if (fyn_map->xl) + fy_accel_insert(fyn_map->xl, fyn_key, fynp); + + fy_node_mark_synthetic(fyn_map); + + return 0; +} + +bool fy_node_mapping_contains_pair(struct fy_node *fyn_map, struct fy_node_pair *fynp) +{ + struct fy_node_pair *fynpi; + + if (!fyn_map || !fynp || fyn_map->type != FYNT_MAPPING) + return false; + + if (fyn_map->xl) { + fynpi = fy_node_accel_lookup_by_node(fyn_map, fynp->key); + if (fynpi == fynp) + return true; + } else { + for (fynpi = fy_node_pair_list_head(&fyn_map->mapping); fynpi; fynpi = fy_node_pair_next(&fyn_map->mapping, fynpi)) + if (fynpi == fynp) + return true; + } + + return false; +} + +int fy_node_mapping_remove(struct fy_node *fyn_map, struct fy_node_pair *fynp) +{ + if (!fy_node_mapping_contains_pair(fyn_map, fynp)) + return -1; + + fy_node_pair_list_del(&fyn_map->mapping, fynp); + if (fyn_map->xl) + fy_accel_remove(fyn_map->xl, fynp->key); + + if (fynp->key) { + fynp->key->parent = NULL; + fynp->key->attached = false; + } + + if (fynp->value) { + fynp->value->parent = NULL; + fynp->value->attached = false; + } + + fynp->parent = NULL; + + return 0; +} + +/* returns value */ +struct fy_node *fy_node_mapping_remove_by_key(struct fy_node *fyn_map, struct fy_node *fyn_key) +{ + struct fy_node_pair *fynp; + struct fy_node *fyn_value; + + fynp = fy_node_mapping_lookup_pair(fyn_map, fyn_key); + if (!fynp) + return NULL; + + fyn_value = fynp->value; + if (fyn_value) { + fyn_value->parent = NULL; + fyn_value->attached = false; + } + + /* do not free the key if it's the same pointer */ + if (fyn_key != fynp->key) + fy_node_detach_and_free(fyn_key); + fynp->value = NULL; + + fy_node_pair_list_del(&fyn_map->mapping, fynp); + if (fyn_map->xl) + fy_accel_remove(fyn_map->xl, fynp->key); + + fy_node_pair_detach_and_free(fynp); + + fy_node_mark_synthetic(fyn_map); + + return fyn_value; +} + +void *fy_node_mapping_sort_ctx_arg(struct fy_node_mapping_sort_ctx *ctx) +{ + return ctx->arg; +} + +static int fy_node_mapping_sort_cmp( +#ifdef __APPLE__ +void *arg, const void *a, const void *b +#else +const void *a, const void *b, void *arg +#endif +) +{ + struct fy_node_mapping_sort_ctx *ctx = arg; + struct fy_node_pair * const *fynppa = a, * const *fynppb = b; + + assert(fynppa >= ctx->fynpp && fynppa < ctx->fynpp + ctx->count); + assert(fynppb >= ctx->fynpp && fynppb < ctx->fynpp + ctx->count); + + return ctx->key_cmp(*fynppa, *fynppb, ctx->arg); +} + +/* not! thread safe! */ +#if !defined(HAVE_QSORT_R) || !HAVE_QSORT_R || defined(__EMSCRIPTEN__) +static struct fy_node_mapping_sort_ctx *fy_node_mapping_sort_ctx_no_qsort_r; + +static int fy_node_mapping_sort_cmp_no_qsort_r(const void *a, const void *b) +{ +#ifdef __APPLE__ + return fy_node_mapping_sort_cmp( + fy_node_mapping_sort_ctx_no_qsort_r, + a, b); +#else + return fy_node_mapping_sort_cmp( a, b, + fy_node_mapping_sort_ctx_no_qsort_r); +#endif +} + +#endif + +static int fy_node_scalar_cmp_default(struct fy_node *fyn_a, + struct fy_node *fyn_b, + void *arg) +{ + /* handles NULL cases */ + if (fyn_a == fyn_b) + return 0; + if (!fyn_a) + return 1; + if (!fyn_b) + return -1; + return fy_token_cmp(fyn_a->scalar, fyn_b->scalar); +} + +/* the default sort method */ +static int fy_node_mapping_sort_cmp_default(const struct fy_node_pair *fynp_a, + const struct fy_node_pair *fynp_b, + void *arg) +{ + int idx_a, idx_b; + bool alias_a, alias_b, scalar_a, scalar_b; + struct fy_node_cmp_arg *cmp_arg; + fy_node_scalar_compare_fn cmp_fn; + void *cmp_fn_arg; + + cmp_arg = arg; + cmp_fn = cmp_arg ? cmp_arg->cmp_fn : fy_node_scalar_cmp_default; + cmp_fn_arg = cmp_arg ? cmp_arg->arg : NULL; + + /* order is: maps first, followed by sequences, and last scalars sorted */ + scalar_a = !fynp_a->key || fy_node_is_scalar(fynp_a->key); + scalar_b = !fynp_b->key || fy_node_is_scalar(fynp_b->key); + + /* scalar? perform comparison */ + if (scalar_a && scalar_b) { + + /* if both are aliases, sort skipping the '*' */ + alias_a = fy_node_is_alias(fynp_a->key); + alias_b = fy_node_is_alias(fynp_b->key); + + /* aliases win */ + if (alias_a && !alias_b) + return -1; + + if (!alias_a && alias_b) + return 1; + + return cmp_fn(fynp_a->key, fynp_b->key, cmp_fn_arg); + } + + /* b is scalar, a is not */ + if (!scalar_a && scalar_b) + return -1; + + /* a is scalar, b is not */ + if (scalar_a && !scalar_b) + return 1; + + /* different types, mappings win */ + if (fynp_a->key->type != fynp_b->key->type) + return fynp_a->key->type == FYNT_MAPPING ? -1 : 1; + + /* ok, need to compare indices now */ + idx_a = fy_node_mapping_get_pair_index(fynp_a->parent, fynp_a); + idx_b = fy_node_mapping_get_pair_index(fynp_b->parent, fynp_b); + + return idx_a > idx_b ? 1 : (idx_a < idx_b ? -1 : 0); +} + +void fy_node_mapping_fill_array(struct fy_node *fyn_map, + struct fy_node_pair **fynpp, int count) +{ + struct fy_node_pair *fynpi; + int i; + + for (i = 0, fynpi = fy_node_pair_list_head(&fyn_map->mapping); i < count && fynpi; + fynpi = fy_node_pair_next(&fyn_map->mapping, fynpi), i++) + fynpp[i] = fynpi; + + /* if there's enough space, put down a NULL at the end */ + if (i < count) + fynpp[i++] = NULL; + assert(i == count); + +} + +void fy_node_mapping_perform_sort(struct fy_node *fyn_map, + fy_node_mapping_sort_fn key_cmp, void *arg, + struct fy_node_pair **fynpp, int count) +{ + struct fy_node_mapping_sort_ctx ctx; + struct fy_node_cmp_arg def_arg; + + if (!key_cmp) { + def_arg.cmp_fn = fy_node_scalar_cmp_default; + def_arg.arg = arg; + } else { + def_arg.cmp_fn = NULL; + def_arg.arg = NULL; + } + ctx.key_cmp = key_cmp ? : fy_node_mapping_sort_cmp_default; + ctx.arg = key_cmp ? arg : &def_arg; + ctx.fynpp = fynpp; + ctx.count = count; +#if defined(HAVE_QSORT_R) && HAVE_QSORT_R && !defined(__EMSCRIPTEN__) +#ifdef __APPLE__ + qsort_r(fynpp, count, sizeof(*fynpp), &ctx, fy_node_mapping_sort_cmp); +#else + qsort_r(fynpp, count, sizeof(*fynpp), fy_node_mapping_sort_cmp, &ctx); +#endif +#else + /* caution, not thread safe */ + fy_node_mapping_sort_ctx_no_qsort_r = &ctx; + qsort(fynpp, count, sizeof(*fynpp), fy_node_mapping_sort_cmp_no_qsort_r); + fy_node_mapping_sort_ctx_no_qsort_r = NULL; +#endif +} + +struct fy_node_pair **fy_node_mapping_sort_array(struct fy_node *fyn_map, + fy_node_mapping_sort_fn key_cmp, void *arg, int *countp) +{ + int count; + struct fy_node_pair **fynpp; + + count = fy_node_mapping_item_count(fyn_map); + if (count < 0) + return NULL; + + fynpp = malloc((count + 1) * sizeof(*fynpp)); + if (!fynpp) + return NULL; + + memset(fynpp, 0, (count + 1) * sizeof(*fynpp)); + + fy_node_mapping_fill_array(fyn_map, fynpp, count); + fy_node_mapping_perform_sort(fyn_map, key_cmp, arg, fynpp, count); + + if (countp) + *countp = count; + + return fynpp; +} + +void fy_node_mapping_release_array(struct fy_node *fyn_map, struct fy_node_pair **fynpp) +{ + if (!fyn_map || !fynpp) + return; + + free(fynpp); +} + +int fy_node_mapping_sort(struct fy_node *fyn_map, + fy_node_mapping_sort_fn key_cmp, + void *arg) +{ + int count, i; + struct fy_node_pair **fynpp, *fynpi; + + fynpp = fy_node_mapping_sort_array(fyn_map, key_cmp, arg, &count); + if (!fynpp) + return -1; + + fy_node_pair_list_init(&fyn_map->mapping); + for (i = 0; i < count; i++) { + fynpi = fynpp[i]; + fy_node_pair_list_add_tail(&fyn_map->mapping, fynpi); + } + + fy_node_mapping_release_array(fyn_map, fynpp); + + return 0; +} + +int fy_node_sort(struct fy_node *fyn, fy_node_mapping_sort_fn key_cmp, void *arg) +{ + struct fy_node *fyni; + struct fy_node_pair *fynp, *fynpi; + int ret; + + if (!fyn) + return 0; + + switch (fyn->type) { + case FYNT_SCALAR: + break; + + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) { + + fy_node_sort(fyni, key_cmp, arg); + } + break; + + case FYNT_MAPPING: + ret = fy_node_mapping_sort(fyn, key_cmp, arg); + if (ret) + return ret; + + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; fynp = fynpi) { + + fynpi = fy_node_pair_next(&fyn->mapping, fynp); + + /* the parent of the key is always NULL */ + ret = fy_node_sort(fynp->key, key_cmp, arg); + if (ret) + return ret; + + ret = fy_node_sort(fynp->value, key_cmp, arg); + if (ret) + return ret; + + fynp->parent = fyn; + } + break; + } + + return 0; +} + +struct fy_node *fy_node_vbuildf(struct fy_document *fyd, const char *fmt, va_list ap) +{ + struct fy_document_vbuildf_ctx vctx; + struct fy_node *fyn; + + vctx.fmt = fmt; + va_copy(vctx.ap, ap); + fyn = fy_node_build_internal(fyd, parser_setup_from_fmt_ap, &vctx); + va_end(ap); + + return fyn; +} + +struct fy_node *fy_node_buildf(struct fy_document *fyd, const char *fmt, ...) +{ + struct fy_node *fyn; + va_list ap; + + va_start(ap, fmt); + fyn = fy_node_vbuildf(fyd, fmt, ap); + va_end(ap); + + return fyn; +} + +struct fy_document *fy_document_vbuildf(const struct fy_parse_cfg *cfg, const char *fmt, va_list ap) +{ + struct fy_document *fyd; + struct fy_document_vbuildf_ctx vctx; + + vctx.fmt = fmt; + va_copy(vctx.ap, ap); + fyd = fy_document_build_internal(cfg, parser_setup_from_fmt_ap, &vctx); + va_end(ap); + + return fyd; +} + +struct fy_document *fy_document_buildf(const struct fy_parse_cfg *cfg, const char *fmt, ...) +{ + struct fy_document *fyd; + va_list ap; + + va_start(ap, fmt); + fyd = fy_document_vbuildf(cfg, fmt, ap); + va_end(ap); + + return fyd; +} + +struct flow_reader_container { + struct fy_reader reader; + const struct fy_parse_cfg *cfg; +}; + +static struct fy_diag *flow_reader_get_diag(struct fy_reader *fyr) +{ + struct flow_reader_container *frc = container_of(fyr, struct flow_reader_container, reader); + return frc->cfg ? frc->cfg->diag : NULL; +} + +static const struct fy_reader_ops reader_ops = { + .get_diag = flow_reader_get_diag, +}; + +struct fy_document * +fy_flow_document_build_from_string(const struct fy_parse_cfg *cfg, + const char *str, size_t len, size_t *consumed) +{ + struct flow_reader_container frc; + struct fy_reader *fyr = NULL; + struct fy_parser fyp_data, *fyp = &fyp_data; + struct fy_parse_cfg cfg_data; + struct fy_input *fyi; + struct fy_document *fyd; + struct fy_mark mark; + int rc; + + if (!str) + return NULL; + + if (consumed) + *consumed = 0; + + if (!cfg) { + memset(&cfg_data, 0, sizeof(cfg_data)); + cfg_data.flags = FYPCF_DEFAULT_PARSE; + cfg = &cfg_data; + } + + memset(&frc, 0, sizeof(frc)); + fyr = &frc.reader; + frc.cfg = cfg; + + fy_reader_setup(fyr, &reader_ops); + + rc = fy_parse_setup(fyp, cfg); + if (rc) + goto err_no_parse; + + fyi = fy_input_from_data(str, len, NULL, false); + if (!fyi) + goto err_no_input; + + rc = fy_reader_input_open(fyr, fyi, NULL); + if (rc) + goto err_no_input_open; + + fy_parser_set_reader(fyp, fyr); + fy_parser_set_flow_only_mode(fyp, true); + + fyd = fy_parse_load_document(fyp); + + fy_parse_cleanup(fyp); + + if (fyd && consumed) { + fy_reader_get_mark(fyr, &mark); + *consumed = mark.input_pos; + } + + fy_reader_cleanup(fyr); + fy_input_unref(fyi); + + return fyd; + +err_no_input_open: + fy_input_unref(fyi); +err_no_input: + fy_parse_cleanup(fyp); +err_no_parse: + fy_reader_cleanup(fyr); + return NULL; +} + +struct fy_document * +fy_block_document_build_from_string(const struct fy_parse_cfg *cfg, + const char *str, size_t len, size_t *consumed) +{ + struct flow_reader_container frc; + struct fy_reader *fyr = NULL; + struct fy_parser fyp_data, *fyp = &fyp_data; + struct fy_parse_cfg cfg_data; + struct fy_input *fyi; + struct fy_document *fyd; + struct fy_mark mark; + int rc; + + if (!str) + return NULL; + + if (consumed) + *consumed = 0; + + if (!cfg) { + memset(&cfg_data, 0, sizeof(cfg_data)); + cfg_data.flags = FYPCF_DEFAULT_PARSE; + cfg = &cfg_data; + } + + memset(&frc, 0, sizeof(frc)); + fyr = &frc.reader; + frc.cfg = cfg; + + fy_reader_setup(fyr, &reader_ops); + + rc = fy_parse_setup(fyp, cfg); + if (rc) + goto err_no_parse; + + fyi = fy_input_from_data(str, len, NULL, false); + if (!fyi) + goto err_no_input; + + rc = fy_reader_input_open(fyr, fyi, NULL); + if (rc) + goto err_no_input_open; + + fy_parser_set_reader(fyp, fyr); + fy_parser_set_block_only_mode(fyp, true); + + fyd = fy_parse_load_document(fyp); + + fy_parse_cleanup(fyp); + + if (fyd && consumed) { + fy_reader_get_mark(fyr, &mark); + *consumed = mark.input_pos; + } + + fy_reader_cleanup(fyr); + fy_input_unref(fyi); + + return fyd; + +err_no_input_open: + fy_input_unref(fyi); +err_no_input: + fy_parse_cleanup(fyp); +err_no_parse: + fy_reader_cleanup(fyr); + return NULL; +} + +int fy_node_vscanf(struct fy_node *fyn, const char *fmt, va_list ap) +{ + size_t len; + char *fmt_cpy, *s, *e, *t, *te, *key, *fmtspec; + const char *value; + char *value0; + size_t value_len, value0_len; + int count, ret; + struct fy_node *fynv; + va_list apt; + + if (!fyn || !fmt) + goto err_out; + + len = strlen(fmt); + fmt_cpy = alloca(len + 1); + memcpy(fmt_cpy, fmt, len + 1); + s = fmt_cpy; + e = s + len; + + /* the format is of the form 'access key' %fmt[...] */ + /* so we search for a (non escaped '%) */ + value0 = NULL; + value0_len = 0; + count = 0; + while (s < e) { + /* a '%' format must exist */ + t = strchr(s, '%'); + if (!t) + goto err_out; + + /* skip escaped % */ + if (t + 1 < e && t[1] == '%') { + s = t + 2; + continue; + } + + /* trim spaces from key */ + while (isspace((unsigned char)*s)) + s++; + te = t; + while (te > s && isspace((unsigned char)te[-1])) + *--te = '\0'; + + key = s; + + /* we have to scan until the next space that's not in char set */ + fmtspec = t; + while (t < e) { + if (isspace((unsigned char)*t)) + break; + /* character set (may include space) */ + if (*t == '[') { + t++; + /* skip caret */ + if (t < e && *t == '^') + t++; + /* if first character in the set is ']' accept it */ + if (t < e && *t == ']') + t++; + /* now skip until end of character set */ + while (t < e && *t != ']') + t++; + continue; + } + t++; + } + if (t < e) + *t++ = '\0'; + + /* find by (relative) path */ + fynv = fy_node_by_path(fyn, key, t - s, FYNWF_DONT_FOLLOW); + if (!fynv || fynv->type != FYNT_SCALAR) + break; + + /* there must be a text */ + value = fy_token_get_text(fynv->scalar, &value_len); + if (!value) + break; + + /* allocate buffer it's smaller than the one we have already */ + if (!value0 || value0_len < value_len) { + value0 = alloca(value_len + 1); + value0_len = value_len; + } + + memcpy(value0, value, value_len); + value0[value_len] = '\0'; + + va_copy(apt, ap); + /* scanf, all arguments are pointers */ + (void)va_arg(ap, void *); /* advance argument pointer */ + + /* pass it to the system's scanf method */ + ret = vsscanf(value0, fmtspec, apt); + + /* since it's a single specifier, it must be one on success */ + if (ret != 1) + break; + + s = t; + count++; + } + + return count; + +err_out: + errno = -EINVAL; + return -1; +} + +int fy_node_scanf(struct fy_node *fyn, const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = fy_node_vscanf(fyn, fmt, ap); + va_end(ap); + + return ret; +} + +int fy_document_vscanf(struct fy_document *fyd, const char *fmt, va_list ap) +{ + return fy_node_vscanf(fyd->root, fmt, ap); +} + +int fy_document_scanf(struct fy_document *fyd, const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = fy_document_vscanf(fyd, fmt, ap); + va_end(ap); + + return ret; +} + +bool fy_document_has_directives(const struct fy_document *fyd) +{ + struct fy_document_state *fyds; + + if (!fyd) + return false; + + fyds = fyd->fyds; + if (!fyds) + return false; + + return fyds->fyt_vd || !fy_token_list_empty(&fyds->fyt_td); +} + +bool fy_document_has_explicit_document_start(const struct fy_document *fyd) +{ + return fyd ? !fyd->fyds->start_implicit : false; +} + +bool fy_document_has_explicit_document_end(const struct fy_document *fyd) +{ + return fyd ? !fyd->fyds->end_implicit : false; +} + +void *fy_node_get_meta(struct fy_node *fyn) +{ + return fyn && fyn->has_meta ? fyn->meta : NULL; +} + +int fy_node_set_meta(struct fy_node *fyn, void *meta) +{ + struct fy_document *fyd; + + if (!fyn || !fyn->fyd) + return -1; + + fyd = fyn->fyd; + if (fyn->has_meta && fyd->meta_clear_fn) + fyd->meta_clear_fn(fyn, fyn->meta, fyd->meta_user); + fyn->meta = meta; + fyn->has_meta = true; + + return 0; +} + +void fy_node_clear_meta(struct fy_node *fyn) +{ + struct fy_document *fyd; + + if (!fyn || !fyn->has_meta || !fyn->fyd) + return; + + fyd = fyn->fyd; + if (fyd->meta_clear_fn) + fyd->meta_clear_fn(fyn, fyn->meta, fyd->meta_user); + fyn->meta = NULL; + fyn->has_meta = false; +} + +static void fy_node_clear_meta_internal(struct fy_node *fyn) +{ + struct fy_node *fyni; + struct fy_node_pair *fynp, *fynpi; + + if (!fyn) + return; + + switch (fyn->type) { + case FYNT_SCALAR: + break; + + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) { + + fy_node_clear_meta_internal(fyni); + } + break; + + case FYNT_MAPPING: + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; fynp = fynpi) { + + fynpi = fy_node_pair_next(&fyn->mapping, fynp); + + fy_node_clear_meta_internal(fynp->key); + fy_node_clear_meta_internal(fynp->value); + } + break; + } + + fy_node_clear_meta(fyn); +} + +int fy_document_register_meta(struct fy_document *fyd, + fy_node_meta_clear_fn clear_fn, + void *user) +{ + if (!fyd || !clear_fn || fyd->meta_clear_fn) + return -1; + + fyd->meta_clear_fn = clear_fn; + fyd->meta_user = user; + + return 0; +} + +void fy_document_unregister_meta(struct fy_document *fyd) +{ + if (!fyd) + return; + + fy_node_clear_meta_internal(fy_document_root(fyd)); + + fyd->meta_clear_fn = NULL; + fyd->meta_user = NULL; +} + +bool fy_node_set_marker(struct fy_node *fyn, unsigned int marker) +{ + unsigned int prev_marks; + + if (!fyn || marker > FYNWF_MAX_USER_MARKER) + return false; + prev_marks = fyn->marks; + fyn->marks |= FY_BIT(marker); + return !!(prev_marks & FY_BIT(marker)); +} + +bool fy_node_clear_marker(struct fy_node *fyn, unsigned int marker) +{ + unsigned int prev_marks; + + if (!fyn || marker > FYNWF_MAX_USER_MARKER) + return false; + prev_marks = fyn->marks; + fyn->marks &= ~FY_BIT(marker); + return !!(prev_marks & FY_BIT(marker)); +} + +bool fy_node_is_marker_set(struct fy_node *fyn, unsigned int marker) +{ + if (!fyn || marker > FYNWF_MAX_USER_MARKER) + return false; + return !!(fyn->marks & FY_BIT(marker)); +} + +FILE *fy_document_get_error_fp(struct fy_document *fyd) +{ + /* just this for now */ + return stderr; +} + +enum fy_parse_cfg_flags fy_document_get_cfg_flags(const struct fy_document *fyd) +{ + if (!fyd) + return fy_parser_get_cfg_flags(NULL); + + return fyd->parse_cfg.flags; +} + +bool fy_document_can_be_accelerated(struct fy_document *fyd) +{ + if (!fyd) + return false; + + return !(fyd->parse_cfg.flags & FYPCF_DISABLE_ACCELERATORS); +} + +bool fy_document_is_accelerated(struct fy_document *fyd) +{ + if (!fyd) + return false; + + return fyd->axl && fyd->naxl; +} + +static int hd_anchor_hash(struct fy_accel *xl, const void *key, void *userdata, void *hash) +{ + struct fy_token *fyt = (void *)key; + unsigned int *hashp = hash; + const char *text; + size_t len; + + text = fy_token_get_text(fyt, &len); + if (!text) + return -1; + + *hashp = XXH32(text, len, 2654435761U); + return 0; +} + +static bool hd_anchor_eq(struct fy_accel *xl, const void *hash, const void *key1, const void *key2, void *userdata) +{ + struct fy_token *fyt1 = (void *)key1, *fyt2 = (void *)key2; + const char *text1, *text2; + size_t len1, len2; + + text1 = fy_token_get_text(fyt1, &len1); + if (!text1) + return false; + text2 = fy_token_get_text(fyt2, &len2); + if (!text2) + return false; + + return len1 == len2 && !memcmp(text1, text2, len1); +} + +static const struct fy_hash_desc hd_anchor = { + .size = sizeof(unsigned int), + .max_bucket_grow_limit = 6, /* TODO allow tuning */ + .hash = hd_anchor_hash, + .eq = hd_anchor_eq, +}; + +static int hd_nanchor_hash(struct fy_accel *xl, const void *key, void *userdata, void *hash) +{ + struct fy_node *fyn = (void *)key; + unsigned int *hashp = hash; + uintptr_t ptr = (uintptr_t)fyn; + + *hashp = XXH32(&ptr, sizeof(ptr), 2654435761U); + + return 0; +} + +static bool hd_nanchor_eq(struct fy_accel *xl, const void *hash, const void *key1, const void *key2, void *userdata) +{ + struct fy_node *fyn1 = (void *)key1, *fyn2 = (void *)key2; + + return fyn1 == fyn2; +} + +static const struct fy_hash_desc hd_nanchor = { + .size = sizeof(unsigned int), + .max_bucket_grow_limit = 6, /* TODO allow tuning */ + .hash = hd_nanchor_hash, + .eq = hd_nanchor_eq, +}; + + +static int hd_mapping_hash(struct fy_accel *xl, const void *key, void *userdata, void *hash) +{ + return fy_node_hash_uint((struct fy_node *)key, hash); +} + +static bool hd_mapping_eq(struct fy_accel *xl, const void *hash, const void *key1, const void *key2, void *userdata) +{ + return fy_node_compare((struct fy_node *)key1, (struct fy_node *)key2); +} + +static const struct fy_hash_desc hd_mapping = { + .size = sizeof(unsigned int), + .max_bucket_grow_limit = 6, /* TODO allow tuning */ + .hash = hd_mapping_hash, + .eq = hd_mapping_eq, +}; + +typedef void (*fy_hash_update_fn)(void *state, const void *ptr, size_t size); + +static int +fy_node_hash_internal(struct fy_node *fyn, fy_hash_update_fn update_fn, void *state) +{ + struct fy_node *fyni; + struct fy_node_pair *fynp; + struct fy_node_pair **fynpp; + struct fy_token_iter iter; + int i, count, rc; + const struct fy_iter_chunk *ic; + + if (!fyn) { + /* NULL */ + update_fn(state, "s", 1); /* as zero length scalar */ + return 0; + } + + switch (fyn->type) { + case FYNT_SEQUENCE: + /* SEQUENCE */ + update_fn(state, "S", 1); + + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) { + + rc = fy_node_hash_internal(fyni, update_fn, state); + if (rc) + return rc; + } + + break; + + case FYNT_MAPPING: + count = fy_node_mapping_item_count(fyn); + + fynpp = alloca(sizeof(*fynpp) * (count + 1)); + + fy_node_mapping_fill_array(fyn, fynpp, count); + fy_node_mapping_perform_sort(fyn, NULL, NULL, fynpp, count); + + /* MAPPING */ + update_fn(state, "M", 1); + + for (i = 0; i < count; i++) { + fynp = fynpp[i]; + + /* MAPPING KEY */ + update_fn(state, "K", 1); + rc = fy_node_hash_internal(fynp->key, update_fn, state); + if (rc) + return rc; + + /* MAPPING VALUE */ + update_fn(state, "V", 1); + rc = fy_node_hash_internal(fynp->value, update_fn, state); + if (rc) + return rc; + } + + break; + + case FYNT_SCALAR: + update_fn(state, !fy_node_is_alias(fyn) ? "s" : "A", 1); + + fy_token_iter_start(fyn->scalar, &iter); + ic = NULL; + while ((ic = fy_token_iter_chunk_next(&iter, ic, &rc)) != NULL) + update_fn(state, ic->str, ic->len); + fy_token_iter_finish(&iter); + + break; + } + + return 0; +} + +static void update_xx32(void *state, const void *ptr, size_t size) +{ + XXH32_update(state, ptr, size); +} + +int fy_node_hash_uint(struct fy_node *fyn, unsigned int *hashp) +{ + XXH32_state_t state; + int rc; + + XXH32_reset(&state, 2654435761U); + + rc = fy_node_hash_internal(fyn, update_xx32, &state); + if (rc) + return rc; + + *hashp = XXH32_digest(&state); + return 0; +} + +struct fy_document_state *fy_document_get_document_state(struct fy_document *fyd) +{ + return fyd ? fyd->fyds : NULL; +} + +int fy_document_set_document_state(struct fy_document *fyd, struct fy_document_state *fyds) +{ + /* document must exist and not have any contents */ + if (!fyd || fyd->root) + return -1; + + if (!fyds) + fyds = fy_document_state_default(NULL, NULL); + else + fyds = fy_document_state_ref(fyds); + + if (!fyds) + return -1; + + /* drop the previous document state */ + fy_document_state_unref(fyd->fyds); + /* and use the new document state from now on */ + fyd->fyds = fyds; + + return 0; +} + +struct fy_ptr_node *fy_ptr_node_create(struct fy_node *fyn) +{ + struct fy_ptr_node *fypn; + + if (!fyn) + return NULL; + + fypn = malloc(sizeof(*fypn)); + if (!fypn) + return NULL; + memset(&fypn->node, 0, sizeof(fypn->node)); + fypn->fyn = fyn; + return fypn; +} + +void fy_ptr_node_destroy(struct fy_ptr_node *fypn) +{ + free(fypn); +} + +void fy_ptr_node_list_free_all(struct fy_ptr_node_list *fypnl) +{ + struct fy_ptr_node *fypn; + + while ((fypn = fy_ptr_node_list_pop(fypnl)) != NULL) + fy_ptr_node_destroy(fypn); +} + +bool fy_ptr_node_list_contains(struct fy_ptr_node_list *fypnl, struct fy_node *fyn) +{ + struct fy_ptr_node *fypn; + + if (!fypnl || !fyn) + return false; + for (fypn = fy_ptr_node_list_head(fypnl); fypn; fypn = fy_ptr_node_next(fypnl, fypn)) { + if (fypn->fyn == fyn) + return true; + } + return false; +} + +struct fy_document * +fy_document_create_from_event(struct fy_parser *fyp, struct fy_event *fye) +{ + struct fy_document *fyd; + int rc; + + if (!fyp || !fye || fye->type != FYET_DOCUMENT_START) + return NULL; + + /* TODO update document end */ + fyd = fy_document_create(&fyp->cfg); + fyp_error_check(fyp, fyd, err_out, + "fy_document_create() failed"); + + rc = fy_document_set_document_state(fyd, fye->document_start.document_state); + fyp_error_check(fyp, !rc, err_out, + "fy_document_set_document_state() failed"); + + return fyd; + +err_out: + fy_document_destroy(fyd); + return NULL; +} + +int +fy_document_update_from_event(struct fy_document *fyd, struct fy_parser *fyp, struct fy_event *fye) +{ + if (!fyd || !fyp || !fye || fye->type != FYET_DOCUMENT_END) + return -1; + + /* nothing besides checks */ + return 0; +} + +struct fy_node * +fy_node_create_from_event(struct fy_document *fyd, struct fy_parser *fyp, struct fy_event *fye) +{ + struct fy_node *fyn = NULL; + struct fy_token *value = NULL, *anchor = NULL; + int rc; + + if (!fyd || !fye) + return NULL; + + switch (fye->type) { + default: + break; + + case FYET_SCALAR: + fyn = fy_node_alloc(fyd, FYNT_SCALAR); + fyp_error_check(fyp, fyn, err_out, + "fy_node_alloc() scalar failed"); + + value = fye->scalar.value; + + if (value) /* NULL scalar */ + fyn->style = fy_node_style_from_scalar_style(value->scalar.style); + else + fyn->style = FYNS_PLAIN; + + /* NULLs are OK */ + fyn->tag = fy_token_ref(fye->scalar.tag); + fyn->scalar = fy_token_ref(value); + anchor = fye->scalar.anchor; + break; + + case FYET_ALIAS: + fyn = fy_node_alloc(fyd, FYNT_SCALAR); + fyp_error_check(fyp, fyn, err_out, + "fy_node_alloc() alias failed"); + + value = fye->alias.anchor; + fyn->style = FYNS_ALIAS; + fyn->scalar = fy_token_ref(value); + anchor = NULL; + break; + + case FYET_MAPPING_START: + fyn = fy_node_create_mapping(fyd); + fyp_error_check(fyp, fyn, err_out, + "fy_node_create_mapping() failed"); + + value = fye->mapping_start.mapping_start; + fyn->style = value->type == FYTT_FLOW_MAPPING_START ? FYNS_FLOW : FYNS_BLOCK; + + fyn->tag = fy_token_ref(fye->mapping_start.tag); + fyn->mapping_start = fy_token_ref(value); + fyn->mapping_end = NULL; + anchor = fye->mapping_start.anchor; + break; + + case FYET_SEQUENCE_START: + fyn = fy_node_create_sequence(fyd); + fyp_error_check(fyp, fyn, err_out, + "fy_node_create_sequence() failed"); + + value = fye->sequence_start.sequence_start; + + fyn->style = value->type == FYTT_FLOW_SEQUENCE_START ? FYNS_FLOW : FYNS_BLOCK; + + fyn->tag = fy_token_ref(fye->sequence_start.tag); + fyn->sequence_start = fy_token_ref(value); + fyn->sequence_end = NULL; + anchor = fye->sequence_start.anchor; + + break; + + } + + if (fyn && anchor) { + rc = fy_document_register_anchor(fyd, fyn, fy_token_ref(anchor)); + fyp_error_check(fyp, !rc, err_out, + "fy_document_register_anchor() failed"); + } + + return fyn; + +err_out: + /* NULL OK */ + fy_node_free(fyn); + return NULL; +} + +int +fy_node_update_from_event(struct fy_node *fyn, struct fy_parser *fyp, struct fy_event *fye) +{ + if (!fyn || !fyp || !fye) + return -1; + + switch (fye->type) { + + case FYET_MAPPING_END: + if (!fy_node_is_mapping(fyn)) + return -1; + fy_token_unref(fyn->mapping_end); + fyn->mapping_end = fy_token_ref(fye->mapping_end.mapping_end); + + break; + + case FYET_SEQUENCE_END: + if (!fy_node_is_sequence(fyn)) + return -1; + fy_token_unref(fyn->sequence_end); + fyn->sequence_end = fy_token_ref(fye->sequence_end.sequence_end); + + break; + + default: + return -1; + } + + return 0; +} + +struct fy_node_pair * +fy_node_pair_create_with_key(struct fy_document *fyd, struct fy_node *fyn_parent, struct fy_node *fyn) +{ + struct fy_node_pair *fynp; + bool is_duplicate; + + if (!fyd || !fyn_parent || !fy_node_is_mapping(fyn_parent)) + return NULL; + + /* if we don't allow duplicate keys */ + if (!(fyd->parse_cfg.flags & FYPCF_ALLOW_DUPLICATE_KEYS)) { + + /* make sure we don't add an already existing key */ + is_duplicate = fy_node_mapping_key_is_duplicate(fyn_parent, fyn); + if (is_duplicate) { + FYD_NODE_ERROR(fyd, fyn, FYEM_DOC, + "duplicate mapping key"); + return NULL; + } + } + + fynp = fy_node_pair_alloc(fyd); + fyd_error_check(fyd, fynp, err_out, + "fy_node_pair_alloc() failed"); + + fynp->parent = fyn_parent; + + fynp->key = fyn; + if (fynp->key) + fynp->key->attached = true; + + return fynp; + +err_out: + fy_node_pair_free(fynp); + return NULL; + +} + +int +fy_node_pair_update_with_value(struct fy_node_pair *fynp, struct fy_node *fyn) +{ + struct fy_node *fyn_parent; + int rc; + + /* node pair must exist and value must be NULL */ + if (!fynp || fynp->value || !fynp->parent || !fy_node_is_mapping(fynp->parent) || !fyn->fyd) + return -1; + + fynp->value = fyn; + if (fynp->value) + fynp->value->attached = true; + + fyn_parent = fynp->parent; + + fy_node_pair_list_add_tail(&fyn_parent->mapping, fynp); + if (fyn_parent->xl) { + rc = fy_accel_insert(fyn_parent->xl, fynp->key, fynp); + fyd_error_check(fyn->fyd, !rc, err_out, + "fy_accel_insert() failed"); + } + + return 0; + +err_out: + fy_node_pair_list_del(&fyn_parent->mapping, fynp); + if (fyn) + fyn->attached = false; + fynp->value = NULL; + return -1; +} + +int +fy_node_sequence_add_item(struct fy_node *fyn_parent, struct fy_node *fyn) +{ + /* node pair must exist and value must be NULL */ + if (!fyn_parent || !fyn || !fy_node_is_sequence(fyn_parent) || !fyn->fyd) + return -1; + + fyn->parent = fyn_parent; + fy_node_list_add_tail(&fyn_parent->sequence, fyn); + fyn->attached = true; + return 0; +} + +int fy_document_iterator_setup(struct fy_document_iterator *fydi, const struct fy_document_iterator_cfg *cfg) +{ + memset(fydi, 0, sizeof(*fydi)); + + if (cfg) + fydi->cfg = *cfg; + + fydi->state = FYDIS_WAITING_STREAM_START; + fydi->fyd = NULL; + fydi->iterate_root = NULL; + + /* suppress recycling if we must */ + fydi->suppress_recycling_force = getenv("FY_VALGRIND") && !getenv("FY_VALGRIND_RECYCLING"); + fydi->suppress_recycling = fydi->suppress_recycling_force; + + fy_eventp_list_init(&fydi->recycled_eventp); + fy_token_list_init(&fydi->recycled_token); + + if (!fydi->suppress_recycling) { + fydi->recycled_eventp_list = &fydi->recycled_eventp; + fydi->recycled_token_list = &fydi->recycled_token; + } else { + fydi->recycled_eventp_list = NULL; + fydi->recycled_token_list = NULL; + } + + /* start with the stack pointing to the in place data */ + fydi->stack_top = (unsigned int)-1; + fydi->stack_alloc = sizeof(fydi->in_place) / sizeof(fydi->in_place[0]); + fydi->stack = fydi->in_place; + + /* set generator state accordingly */ + fydi->generator_state = 0; + + /* without configuration, is the default */ + if (!cfg) + return 0; + + /* set the generator flags accordingly */ + switch (fydi->cfg.flags & FYDICF_WANT_MASK) { + case FYDICF_WANT_STREAM_DOCUMENT_BODY_EVENTS: + fydi->generator_state |= FYDIGF_WANTS_STREAM | FYDIGF_WANTS_DOC | FYDIGF_ENDS_AFTER_DOC; + break; + case FYDICF_WANT_DOCUMENT_BODY_EVENTS: + fydi->generator_state |= FYDIGF_WANTS_DOC | FYDIGF_ENDS_AFTER_DOC; + fydi->state = FYDIS_WAITING_DOCUMENT_START; + break; + case FYDICF_WANT_BODY_EVENTS: + fydi->generator_state |= FYDIGF_ENDS_AFTER_BODY; + fydi->state = FYDIS_WAITING_BODY_START_OR_DOCUMENT_END; + default: + break; + } + + return 0; +} + +void fy_document_iterator_cleanup(struct fy_document_iterator *fydi) +{ + struct fy_token *fyt; + struct fy_eventp *fyep; + + /* free the stack if it's not the inplace one */ + if (fydi->stack != fydi->in_place) + free(fydi->stack); + fydi->stack_top = (unsigned int)-1; + fydi->stack_alloc = sizeof(fydi->in_place) / sizeof(fydi->in_place[0]); + fydi->stack = fydi->in_place; + + while ((fyt = fy_token_list_pop(&fydi->recycled_token)) != NULL) + fy_token_free(fyt); + + while ((fyep = fy_eventp_list_pop(&fydi->recycled_eventp)) != NULL) + fy_eventp_free(fyep); + + fydi->state = FYDIS_WAITING_STREAM_START; + fydi->fyd = NULL; + fydi->iterate_root = NULL; +} + +struct fy_document_iterator * +fy_document_iterator_create_cfg(const struct fy_document_iterator_cfg *cfg) +{ + struct fy_document_iterator *fydi = NULL; + int rc; + + fydi = malloc(sizeof(*fydi)); + if (!fydi) + goto err_out; + + rc = fy_document_iterator_setup(fydi, cfg); + if (rc) + goto err_out; + + return fydi; + +err_out: + fy_document_iterator_destroy(fydi); + return NULL; +} + +struct fy_document_iterator *fy_document_iterator_create(void) +{ + return fy_document_iterator_create_cfg(NULL); +} + +struct fy_document_iterator * +fy_document_iterator_create_on_document(struct fy_document *fyd) +{ + struct fy_document_iterator_cfg cfg; + + if (!fyd) + return NULL; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYDICF_WANT_STREAM_DOCUMENT_BODY_EVENTS; + cfg.fyd = fyd; + cfg.iterate_root = fyd->root; + + return fy_document_iterator_create_cfg(&cfg); +} + +struct fy_document_iterator * +fy_document_iterator_create_on_node(struct fy_node *fyn) +{ + struct fy_document_iterator_cfg cfg; + + if (!fyn) + return NULL; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYDICF_WANT_STREAM_DOCUMENT_BODY_EVENTS; + cfg.fyd = fy_node_document(fyn); + cfg.iterate_root = fyn; + + return fy_document_iterator_create_cfg(&cfg); +} + +void fy_document_iterator_destroy(struct fy_document_iterator *fydi) +{ + if (!fydi) + return; + fy_document_iterator_cleanup(fydi); + free(fydi); +} + +static struct fy_event * +fydi_event_create(struct fy_document_iterator *fydi, struct fy_node *fyn, bool start) +{ + struct fy_eventp *fyep; + struct fy_event *fye; + struct fy_anchor *fya; + struct fy_token *anchor = NULL; + + fyep = fy_document_iterator_eventp_alloc(fydi); + if (!fyep) { + fydi->state = FYDIS_ERROR; + return NULL; + } + fye = &fyep->e; + + if (start) { + fya = fy_node_get_anchor(fyn); + anchor = fya ? fya->anchor : NULL; + } + + switch (fyn->type) { + + case FYNT_SCALAR: + if (fyn->style != FYNS_ALIAS) { + fye->type = FYET_SCALAR; + fye->scalar.anchor = fy_token_ref(anchor); + fye->scalar.tag = fy_token_ref(fyn->tag); + fye->scalar.value = fy_token_ref(fyn->scalar); + } else { + fye->type = FYET_ALIAS; + fye->alias.anchor = fy_token_ref(fyn->scalar); + } + break; + + case FYNT_SEQUENCE: + if (start) { + fye->type = FYET_SEQUENCE_START; + fye->sequence_start.anchor = fy_token_ref(anchor); + fye->sequence_start.tag = fy_token_ref(fyn->tag); + fye->sequence_start.sequence_start = fy_token_ref(fyn->sequence_start); + } else { + fye->type = FYET_SEQUENCE_END; + fye->sequence_end.sequence_end = fy_token_ref(fyn->sequence_end); + } + break; + + case FYNT_MAPPING: + if (start) { + fye->type = FYET_MAPPING_START; + fye->mapping_start.anchor = fy_token_ref(anchor); + fye->mapping_start.tag = fy_token_ref(fyn->tag); + fye->mapping_start.mapping_start = fy_token_ref(fyn->mapping_start); + } else { + fye->type = FYET_MAPPING_END; + fye->mapping_end.mapping_end = fy_token_ref(fyn->mapping_end); + } + break; + } + + return fye; +} + +struct fy_event * +fy_document_iterator_stream_start(struct fy_document_iterator *fydi) +{ + struct fy_event *fye; + + if (!fydi || fydi->state == FYDIS_ERROR) + return NULL; + + /* both none and stream start are the same for this */ + if (fydi->state != FYDIS_WAITING_STREAM_START && + fydi->state != FYDIS_WAITING_STREAM_END_OR_DOCUMENT_START) + goto err_out; + + fye = fy_document_iterator_event_create(fydi, FYET_STREAM_START); + if (!fye) + goto err_out; + + fydi->state = FYDIS_WAITING_DOCUMENT_START; + return fye; + +err_out: + fydi->state = FYDIS_ERROR; + return NULL; +} + +struct fy_event * +fy_document_iterator_stream_end(struct fy_document_iterator *fydi) +{ + struct fy_event *fye; + + if (!fydi || fydi->state == FYDIS_ERROR) + return NULL; + + if (fydi->state != FYDIS_WAITING_STREAM_END_OR_DOCUMENT_START && + fydi->state != FYDIS_WAITING_DOCUMENT_START) + goto err_out; + + fye = fy_document_iterator_event_create(fydi, FYET_STREAM_END); + if (!fye) + goto err_out; + + fydi->state = FYDIS_WAITING_STREAM_START; + return fye; + +err_out: + fydi->state = FYDIS_ERROR; + return NULL; +} + +struct fy_event * +fy_document_iterator_document_start_internal(struct fy_document_iterator *fydi, struct fy_document *fyd, + struct fy_node *iterate_root) +{ + struct fy_event *fye = NULL; + struct fy_eventp *fyep; + + if (!fydi || fydi->state == FYDIS_ERROR) + return NULL; + + if (!fyd) + goto err_out; + + /* we can transition to document start only from document start or stream end */ + if (fydi->state != FYDIS_WAITING_DOCUMENT_START && + fydi->state != FYDIS_WAITING_STREAM_END_OR_DOCUMENT_START) + goto err_out; + + fyep = fy_document_iterator_eventp_alloc(fydi); + if (!fyep) + goto err_out; + fye = &fyep->e; + + fydi->fyd = fyd; + + /* the iteration root is the document root if not given */ + fydi->iterate_root = iterate_root ? iterate_root : fyd->root; + + /* suppress recycling if we must */ + fydi->suppress_recycling = (fyd->parse_cfg.flags & FYPCF_DISABLE_RECYCLING) || + fydi->suppress_recycling_force; + + if (!fydi->suppress_recycling) { + fydi->recycled_eventp_list = &fydi->recycled_eventp; + fydi->recycled_token_list = &fydi->recycled_token; + } else { + fydi->recycled_eventp_list = NULL; + fydi->recycled_token_list = NULL; + } + + fye->type = FYET_DOCUMENT_START; + fye->document_start.document_start = NULL; + fye->document_start.document_state = fy_document_state_ref(fyd->fyds); + fye->document_start.implicit = fyd->fyds->start_implicit; + + /* and go into body */ + fydi->state = FYDIS_WAITING_BODY_START_OR_DOCUMENT_END; + + return fye; + +err_out: + fy_document_iterator_event_free(fydi, fye); + fydi->state = FYDIS_ERROR; + return NULL; +} + +struct fy_event * +fy_document_iterator_document_start(struct fy_document_iterator *fydi, struct fy_document *fyd) +{ + return fy_document_iterator_document_start_internal(fydi, fyd, NULL); +} + +struct fy_event * +fy_document_iterator_document_end(struct fy_document_iterator *fydi) +{ + struct fy_event *fye; + + if (!fydi || fydi->state == FYDIS_ERROR) + return NULL; + + if (!fydi->fyd || !fydi->fyd->fyds || + fydi->state != FYDIS_WAITING_DOCUMENT_END) + goto err_out; + + fye = fy_document_iterator_event_create(fydi, FYET_DOCUMENT_END, (int)fydi->fyd->fyds->end_implicit); + if (!fye) + goto err_out; + + fydi->fyd = NULL; + fydi->iterate_root = NULL; + + fydi->state = FYDIS_WAITING_STREAM_END_OR_DOCUMENT_START; + return fye; + +err_out: + fydi->state = FYDIS_ERROR; + return NULL; +} + +static bool +fy_document_iterator_ensure_space(struct fy_document_iterator *fydi, unsigned int space) +{ + struct fy_document_iterator_body_state *new_stack; + size_t new_size, copy_size; + unsigned int new_stack_alloc; + + /* empty stack should always have enough space */ + if (fydi->stack_top == (unsigned int)-1) { + assert(fydi->stack_alloc >= space); + return true; + } + + if (fydi->stack_top + space < fydi->stack_alloc) + return true; + + /* make sure we have enough space */ + new_stack_alloc = fydi->stack_alloc * 2; + while (fydi->stack_top + space >= new_stack_alloc) + new_stack_alloc *= 2; + + new_size = new_stack_alloc * sizeof(*new_stack); + + if (fydi->stack == fydi->in_place) { + new_stack = malloc(new_size); + if (!new_stack) + return false; + copy_size = (fydi->stack_top + 1) * sizeof(*new_stack); + memcpy(new_stack, fydi->stack, copy_size); + } else { + new_stack = realloc(fydi->stack, new_size); + if (!new_stack) + return false; + } + fydi->stack = new_stack; + fydi->stack_alloc = new_stack_alloc; + return true; +} + +static bool +fydi_push_collection(struct fy_document_iterator *fydi, struct fy_node *fyn) +{ + struct fy_document_iterator_body_state *s; + + /* make sure there's enough space */ + if (!fy_document_iterator_ensure_space(fydi, 1)) + return false; + + /* get the next */ + fydi->stack_top++; + s = &fydi->stack[fydi->stack_top]; + s->fyn = fyn; + + switch (fyn->type) { + case FYNT_SEQUENCE: + s->fyni = fy_node_list_head(&fyn->sequence); + break; + + case FYNT_MAPPING: + s->fynp = fy_node_pair_list_head(&fyn->mapping); + s->processed_key = false; + break; + + default: + FY_IMPOSSIBLE_ABORT(); + } + + return true; +} + +static inline void +fydi_pop_collection(struct fy_document_iterator *fydi) +{ + assert(fydi->stack_top != (unsigned int)-1); + fydi->stack_top--; +} + +static inline struct fy_document_iterator_body_state * +fydi_last_collection(struct fy_document_iterator *fydi) +{ + if (fydi->stack_top == (unsigned int)-1) + return NULL; + return &fydi->stack[fydi->stack_top]; +} + +bool +fy_document_iterator_body_next_internal(struct fy_document_iterator *fydi, + struct fy_document_iterator_body_result *res) +{ + struct fy_document_iterator_body_state *s; + struct fy_node *fyn, *fyn_col; + bool end; + + if (!fydi || !res || fydi->state == FYDIS_ERROR) + return false; + + if (fydi->state != FYDIS_WAITING_BODY_START_OR_DOCUMENT_END && + fydi->state != FYDIS_BODY) + goto err_out; + + end = false; + s = fydi_last_collection(fydi); + if (!s) { + + fyn = fydi->iterate_root; + /* empty root, or last */ + if (!fyn || fydi->state == FYDIS_BODY) { + fydi->state = FYDIS_WAITING_DOCUMENT_END; + return false; + } + + /* ok, in body proper */ + fydi->state = FYDIS_BODY; + + } else { + + fyn_col = s->fyn; + assert(fyn_col); + + fyn = NULL; + if (fyn_col->type == FYNT_SEQUENCE) { + fyn = s->fyni; + if (fyn) + s->fyni = fy_node_next(&fyn_col->sequence, s->fyni); + } else { + assert(fyn_col->type == FYNT_MAPPING); + if (s->fynp) { + if (!s->processed_key) { + fyn = s->fynp->key; + s->processed_key = true; + } else { + fyn = s->fynp->value; + s->processed_key = false; + + /* next in mapping after value */ + s->fynp = fy_node_pair_next(&fyn_col->mapping, s->fynp); + } + } + } + + /* if no next node in the collection, it's the end of the collection */ + if (!fyn) { + fyn = fyn_col; + end = true; + } + } + + assert(fyn); + + /* only for collections */ + if (fyn->type != FYNT_SCALAR) { + if (!end) { + /* push the new sequence */ + if (!fydi_push_collection(fydi, fyn)) + goto err_out; + } else + fydi_pop_collection(fydi); + } + + res->fyn = fyn; + res->end = end; + return true; + +err_out: + fydi->state = FYDIS_ERROR; + return false; +} + +struct fy_event *fy_document_iterator_body_next(struct fy_document_iterator *fydi) +{ + struct fy_document_iterator_body_result res; + + if (!fydi) + return NULL; + + if (!fy_document_iterator_body_next_internal(fydi, &res)) + return NULL; + + return fydi_event_create(fydi, res.fyn, !res.end); +} + +void +fy_document_iterator_node_start(struct fy_document_iterator *fydi, struct fy_node *fyn) +{ + /* do nothing on error */ + if (!fydi || fydi->state == FYDIS_ERROR) + return; + + /* and go into body */ + fydi->state = FYDIS_WAITING_BODY_START_OR_DOCUMENT_END; + fydi->iterate_root = fyn; + fydi->fyd = NULL; +} + +struct fy_node *fy_document_iterator_node_next(struct fy_document_iterator *fydi) +{ + struct fy_document_iterator_body_result res; + + if (!fydi) + return NULL; + + /* do not return ending nodes, are not interested in them */ + do { + if (!fy_document_iterator_body_next_internal(fydi, &res)) + return NULL; + + } while (res.end); + + return res.fyn; +} + +bool fy_document_iterator_get_error(struct fy_document_iterator *fydi) +{ + if (!fydi) + return true; + + if (fydi->state != FYDIS_ERROR) + return false; + + fy_document_iterator_cleanup(fydi); + + return true; +} + +struct fy_event * +fy_document_iterator_generate_next(struct fy_document_iterator *fydi) +{ + struct fy_event *fye = NULL; + + if (!fydi || fydi->state == FYDIS_ERROR) + return NULL; + + if (fydi->generator_state & FYDIGF_GENERATED_NULL) + return NULL; + + /* wants stream events and not generated yet */ + if ((fydi->generator_state & (FYDIGF_WANTS_STREAM | FYDIGF_GENERATED_SS)) == FYDIGF_WANTS_STREAM) { + fye = fy_document_iterator_stream_start(fydi); + if (!fye) + return NULL; + fydi->generator_state |= FYDIGF_GENERATED_SS; + return fye; + } + + /* wants document events and not generated yet */ + if ((fydi->generator_state & (FYDIGF_WANTS_DOC | FYDIGF_GENERATED_DS)) == FYDIGF_WANTS_DOC) { + if (!fydi->cfg.fyd) { + fydi->state = FYDIS_ERROR; + return NULL; + } + fye = fy_document_iterator_document_start_internal(fydi, fydi->cfg.fyd, fydi->cfg.iterate_root); + if (!fye) + return NULL; + fydi->generator_state |= FYDIGF_GENERATED_DS; + return fye; + } + + /* generate body events... */ + if (!(fydi->generator_state & FYDIGF_GENERATED_BODY)) { + if (!fydi->iterate_root) + fydi->iterate_root = fydi->cfg.iterate_root ? fydi->cfg.iterate_root : fy_document_root(fydi->fyd); + fye = fy_document_iterator_body_next(fydi); + if (fye) + return fye; + + fydi->generator_state |= FYDIGF_GENERATED_BODY; + } + + /* wants document events and not generated yet */ + if ((fydi->generator_state & (FYDIGF_WANTS_DOC | FYDIGF_GENERATED_DE)) == FYDIGF_WANTS_DOC) { + fye = fy_document_iterator_document_end(fydi); + if (!fye) + return NULL; + fydi->generator_state |= FYDIGF_GENERATED_DE; + return fye; + } + + /* wants stream events and not generated yet */ + if ((fydi->generator_state & (FYDIGF_WANTS_STREAM | FYDIGF_GENERATED_SE)) == FYDIGF_WANTS_STREAM) { + fye = fy_document_iterator_stream_end(fydi); + if (!fye) + return NULL; + fydi->generator_state |= FYDIGF_GENERATED_SE; + return fye; + } + + fydi->generator_state |= FYDIGF_GENERATED_NULL; + return NULL; +} diff --git a/src/lib/fy-doc.h b/src/lib/fy-doc.h new file mode 100644 index 0000000..c9c46c9 --- /dev/null +++ b/src/lib/fy-doc.h @@ -0,0 +1,391 @@ +/* + * fy-doc.h - YAML document internal header file + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_DOC_H +#define FY_DOC_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include + +#include "fy-ctype.h" +#include "fy-utf8.h" +#include "fy-list.h" +#include "fy-typelist.h" +#include "fy-types.h" +#include "fy-diag.h" +#include "fy-dump.h" +#include "fy-docstate.h" +#include "fy-accel.h" +#include "fy-walk.h" +#include "fy-path.h" + +struct fy_eventp; + +/* TODO vary according to platfom */ +static inline int fy_depth_limit(void) +{ + return FYPCF_GUARANTEED_MINIMUM_DEPTH_LIMIT; +} + +FY_TYPE_FWD_DECL_LIST(document); + +struct fy_node; + +struct fy_node_pair { + struct list_head node; + struct fy_node *key; + struct fy_node *value; + struct fy_document *fyd; + struct fy_node *parent; +}; +FY_TYPE_FWD_DECL_LIST(node_pair); +FY_TYPE_DECL_LIST(node_pair); + +FY_TYPE_FWD_DECL_LIST(node); +struct fy_node { + struct list_head node; + struct fy_token *tag; + enum fy_node_style style; + struct fy_node *parent; + struct fy_document *fyd; + unsigned int marks; + unsigned int type : 2; /* enum fy_node_type: 2 bits are enough for 3 types */ + bool has_meta : 1; + bool attached : 1; /* when it's attached somewhere */ + bool synthetic : 1; /* node has been modified programmaticaly */ + bool key_root : 1; /* node is the root of key fy_node_get_parent() will return NULL */ + void *meta; + struct fy_accel *xl; /* mapping access accelerator */ + struct fy_path_expr_node_data *pxnd; + union { + struct fy_token *scalar; + struct fy_node_list sequence; + struct fy_node_pair_list mapping; + }; + union { + struct fy_token *sequence_start; + struct fy_token *mapping_start; + }; + union { + struct fy_token *sequence_end; + struct fy_token *mapping_end; + }; +}; +FY_TYPE_DECL_LIST(node); + +struct fy_node *fy_node_alloc(struct fy_document *fyd, enum fy_node_type type); +struct fy_node_pair *fy_node_pair_alloc(struct fy_document *fyd); +int fy_node_pair_free(struct fy_node_pair *fynp); + +void fy_node_detach_and_free(struct fy_node *fyn); +void fy_node_pair_detach_and_free(struct fy_node_pair *fynp); + +struct fy_anchor { + struct list_head node; + struct fy_node *fyn; + struct fy_token *anchor; + bool multiple : 1; +}; +FY_TYPE_FWD_DECL_LIST(anchor); +FY_TYPE_DECL_LIST(anchor); + +struct fy_document { + struct list_head node; + struct fy_anchor_list anchors; + struct fy_accel *axl; /* name -> anchor access accelerator */ + struct fy_accel *naxl; /* node -> anchor access accelerator */ + struct fy_document_state *fyds; + struct fy_diag *diag; + struct fy_parse_cfg parse_cfg; + struct fy_node *root; + bool parse_error : 1; + + struct fy_document *parent; + struct fy_document_list children; + + fy_node_meta_clear_fn meta_clear_fn; + void *meta_user; + + struct fy_path_expr_document_data *pxdd; +}; +/* only the list declaration/methods */ +FY_TYPE_DECL_LIST(document); + +struct fy_document *fy_parse_document_create(struct fy_parser *fyp, struct fy_eventp *fyep); +void fy_document_purge_anchors(struct fy_document *fyd); + +struct fy_node_mapping_sort_ctx { + fy_node_mapping_sort_fn key_cmp; + void *arg; + struct fy_node_pair **fynpp; + int count; +}; + +void fy_node_mapping_perform_sort(struct fy_node *fyn_map, + fy_node_mapping_sort_fn key_cmp, void *arg, + struct fy_node_pair **fynpp, int count); + +void fy_node_mapping_fill_array(struct fy_node *fyn_map, + struct fy_node_pair **fynpp, int count); + +struct fy_node_pair **fy_node_mapping_sort_array(struct fy_node *fyn_map, + fy_node_mapping_sort_fn key_cmp, + void *arg, int *countp); + +void fy_node_mapping_release_array(struct fy_node *fyn_map, struct fy_node_pair **fynpp); + +struct fy_node_walk_ctx { + unsigned int max_depth; + unsigned int next_slot; + unsigned int mark; + struct fy_node *marked[0]; +}; + +bool fy_node_is_empty(struct fy_node *fyn); + +bool fy_check_ref_loop(struct fy_document *fyd, struct fy_node *fyn, + enum fy_node_walk_flags flags, + struct fy_node_walk_ctx *ctx); + +#define FYNWF_VISIT_MARKER (FYNWF_MAX_USER_MARKER + 1) +#define FYNWF_REF_MARKER (FYNWF_MAX_USER_MARKER + 2) +#define FYNWF_INSET_MARKER (FYNWF_MAX_USER_MARKER + 3) + +#define FYNWF_SYSTEM_MARKS (FY_BIT(FYNWF_VISIT_MARKER) | \ + FY_BIT(FYNWF_REF_MARKER) | \ + FY_BIT(FYNWF_INSET_MARKER) ) + +void fy_node_clear_system_marks(struct fy_node *fyn); + +bool fy_node_uses_single_input_only(struct fy_node *fyn, struct fy_input *fyi); +struct fy_input *fy_node_get_first_input(struct fy_node *fyn); +bool fy_node_is_synthetic(struct fy_node *fyn); +void fy_node_mark_synthetic(struct fy_node *fyn); +struct fy_input *fy_node_get_input(struct fy_node *fyn); +int fy_document_register_anchor(struct fy_document *fyd, + struct fy_node *fyn, struct fy_token *anchor); +bool fy_node_mapping_key_is_duplicate(struct fy_node *fyn, struct fy_node *fyn_key); + +struct fy_token *fy_node_non_synthesized_token(struct fy_node *fyn); +struct fy_token *fy_node_token(struct fy_node *fyn); + +FILE *fy_document_get_error_fp(struct fy_document *fyd); +enum fy_parse_cfg_flags fy_document_get_cfg_flags(const struct fy_document *fyd); +bool fy_document_is_accelerated(struct fy_document *fyd); +bool fy_document_can_be_accelerated(struct fy_document *fyd); + +/* TODO move to main include */ +struct fy_node *fy_node_collection_iterate(struct fy_node *fyn, void **prevp); + +/* indirect node */ +FY_TYPE_FWD_DECL_LIST(ptr_node); +struct fy_ptr_node { + struct list_head node; + struct fy_node *fyn; +}; +FY_TYPE_DECL_LIST(ptr_node); + +struct fy_ptr_node *fy_ptr_node_create(struct fy_node *fyn); +void fy_ptr_node_destroy(struct fy_ptr_node *fypn); +void fy_ptr_node_list_free_all(struct fy_ptr_node_list *fypnl); +bool fy_ptr_node_list_contains(struct fy_ptr_node_list *fypnl, struct fy_node *fyn); +int fy_node_linearize_recursive(struct fy_ptr_node_list *fypnl, struct fy_node *fyn); +int fy_node_linearize(struct fy_ptr_node_list *fypnl, struct fy_node *fyn); +void fy_node_iterator_check(struct fy_node *fyn); + + +enum fy_document_iterator_state { + FYDIS_WAITING_STREAM_START, + FYDIS_WAITING_DOCUMENT_START, + FYDIS_WAITING_BODY_START_OR_DOCUMENT_END, + FYDIS_BODY, + FYDIS_WAITING_DOCUMENT_END, + FYDIS_WAITING_STREAM_END_OR_DOCUMENT_START, + FYDIS_ENDED, + FYDIS_ERROR, +}; + +struct fy_document_iterator_body_state { + struct fy_node *fyn; /* the collection node */ + bool processed_key : 1; /* for mapping only */ + union { + struct fy_node *fyni; /* for sequence */ + struct fy_node_pair *fynp; /* for mapping */ + }; +}; + +struct fy_document_iterator { + struct fy_document_iterator_cfg cfg; + enum fy_document_iterator_state state; + struct fy_document *fyd; + struct fy_node *iterate_root; + bool suppress_recycling_force : 1; + bool suppress_recycling : 1; + + struct fy_eventp_list recycled_eventp; + struct fy_token_list recycled_token; + + struct fy_eventp_list *recycled_eventp_list; /* NULL when suppressing */ + struct fy_token_list *recycled_token_list; /* NULL when suppressing */ + + unsigned int stack_top; + unsigned int stack_alloc; + struct fy_document_iterator_body_state *stack; + struct fy_document_iterator_body_state in_place[FYPCF_GUARANTEED_MINIMUM_DEPTH_LIMIT]; + +#define FYDIGF_GENERATED_SS FY_BIT(0) +#define FYDIGF_GENERATED_DS FY_BIT(1) +#define FYDIGF_GENERATED_DE FY_BIT(2) +#define FYDIGF_GENERATED_SE FY_BIT(3) +#define FYDIGF_GENERATED_BODY FY_BIT(4) +#define FYDIGF_GENERATED_NULL FY_BIT(5) +#define FYDIGF_ENDS_AFTER_BODY FY_BIT(6) +#define FYDIGF_ENDS_AFTER_DOC FY_BIT(7) +#define FYDIGF_WANTS_STREAM FY_BIT(8) +#define FYDIGF_WANTS_DOC FY_BIT(9) + unsigned int generator_state; +}; + +int fy_document_iterator_setup(struct fy_document_iterator *fydi, const struct fy_document_iterator_cfg *cfg); +void fy_document_iterator_cleanup(struct fy_document_iterator *fydi); +struct fy_document_iterator *fy_document_iterator_create(void); +void fy_document_iterator_destroy(struct fy_document_iterator *fydi); + +struct fy_event * +fy_document_iterator_document_start_internal(struct fy_document_iterator *fydi, struct fy_document *fyd, + struct fy_node *iterate_root); + +struct fy_document_iterator_body_result { + struct fy_node *fyn; + bool end; +}; + +bool +fy_document_iterator_body_next_internal(struct fy_document_iterator *fydi, + struct fy_document_iterator_body_result *res); + +/* diagnostics */ + +static inline bool +fyd_debug_log_level_is_enabled(struct fy_document *fyd, enum fy_error_module module) +{ + return fyd && fy_diag_log_level_is_enabled(fyd->diag, FYET_DEBUG, module); +} + +int fy_document_vdiag(struct fy_document *fyd, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap); + +int fy_document_diag(struct fy_document *fyd, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) + __attribute__((format(printf, 6, 7))); + +void fy_document_diag_vreport(struct fy_document *fyd, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap); +void fy_document_diag_report(struct fy_document *fyd, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +#ifdef FY_DEVMODE + +#define fyd_debug(_fyd, _module, _fmt, ...) \ + do { \ + struct fy_document *__fyd = (_fyd); \ + enum fy_error_module __module = (_module); \ + \ + if (fyd_debug_log_level_is_enabled(__fyd, __module)) \ + fy_document_diag(__fyd, FYET_DEBUG | FYDF_MODULE(_module), \ + __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__); \ + } while(0) +#else + +#define fyd_debug(_fyd, _module, _fmt, ...) \ + do { } while(0) + +#endif + +#define fyd_info(_fyd, _fmt, ...) \ + fy_document_diag((_fyd), FYET_INFO, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyd_notice(_fyd, _fmt, ...) \ + fy_document_diag((_fyd), FYET_NOTICE, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyd_warning(_fyd, _fmt, ...) \ + fy_document_diag((_fyd), FYET_WARNING, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyd_error(_fyd, _fmt, ...) \ + fy_document_diag((_fyd), FYET_ERROR, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) + +#define fyd_doc_debug(_fyd, _fmt, ...) \ + fyd_debug((_fyd), FYEM_DOC, (_fmt) , ## __VA_ARGS__) + +#define fyd_error_check(_fyd, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + fyd_error((_fyd), _fmt, ## __VA_ARGS__); \ + goto _label ; \ + } \ + } while(0) + +#define _FYD_TOKEN_DIAG(_fyd, _fyt, _type, _module, _fmt, ...) \ + do { \ + struct fy_diag_report_ctx _drc; \ + memset(&_drc, 0, sizeof(_drc)); \ + _drc.type = (_type); \ + _drc.module = (_module); \ + _drc.fyt = (_fyt); \ + fy_document_diag_report((_fyd), &_drc, (_fmt) , ## __VA_ARGS__); \ + } while(0) + +#define FYD_TOKEN_DIAG(_fyd, _fyt, _type, _module, _fmt, ...) \ + _FYD_TOKEN_DIAG(_fyd, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYD_NODE_DIAG(_fyd, _fyn, _type, _module, _fmt, ...) \ + _FYD_TOKEN_DIAG(_fyd, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYD_TOKEN_ERROR(_fyd, _fyt, _module, _fmt, ...) \ + FYD_TOKEN_DIAG(_fyd, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYD_NODE_ERROR(_fyd, _fyn, _module, _fmt, ...) \ + FYD_NODE_DIAG(_fyd, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYD_TOKEN_ERROR_CHECK(_fyd, _fyt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYD_TOKEN_ERROR(_fyd, _fyt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYD_NODE_ERROR_CHECK(_fyd, _fyn, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYD_NODE_ERROR(_fyd, _fyn, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYD_TOKEN_WARNING(_fyd, _fyt, _module, _fmt, ...) \ + FYD_TOKEN_DIAG(_fyd, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYD_NODE_WARNING(_fyd, _fyn, _type, _module, _fmt, ...) \ + FYD_NODE_DIAG(_fyd, _fyn, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#endif diff --git a/src/lib/fy-docbuilder-diag.c b/src/lib/fy-docbuilder-diag.c new file mode 100644 index 0000000..841469f --- /dev/null +++ b/src/lib/fy-docbuilder-diag.c @@ -0,0 +1,92 @@ +/* + * fy-docbuilder-diag.c - document builder diagnostics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-diag.h" +#include "fy-docbuilder.h" + +int fy_document_builder_vdiag(struct fy_document_builder *fydb, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap) +{ + struct fy_diag_ctx fydc; + int rc; + + if (!fydb || !fmt || !fydb->cfg.diag) + return -1; + + /* perform the enable tests early to avoid the overhead */ + if (((flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT) < fydb->cfg.diag->cfg.level) + return 0; + + /* fill in fy_diag_ctx */ + memset(&fydc, 0, sizeof(fydc)); + + fydc.level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; + fydc.module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; + fydc.source_file = file; + fydc.source_line = line; + fydc.source_func = func; + fydc.line = -1; + fydc.column = -1; + + rc = fy_vdiag(fydb->cfg.diag, &fydc, fmt, ap); + + return rc; +} + +int fy_document_builder_diag(struct fy_document_builder *fydb, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_document_builder_vdiag(fydb, flags, file, line, func, fmt, ap); + va_end(ap); + + return rc; +} + +void fy_document_builder_diag_vreport(struct fy_document_builder *fydb, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap) +{ + if (!fydb || !fydb->cfg.diag || !fydrc || !fmt) + return; + + fy_diag_vreport(fydb->cfg.diag, fydrc, fmt, ap); +} + +void fy_document_builder_diag_report(struct fy_document_builder *fydb, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_document_builder_diag_vreport(fydb, fydrc, fmt, ap); + va_end(ap); +} + diff --git a/src/lib/fy-docbuilder.c b/src/lib/fy-docbuilder.c new file mode 100644 index 0000000..f4a8d09 --- /dev/null +++ b/src/lib/fy-docbuilder.c @@ -0,0 +1,601 @@ +/* + * fy-docbuilder.c - YAML document builder methods + * + * Copyright (c) 2022 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-utils.h" + +#include "fy-docbuilder.h" + +#include "fy-parse.h" +#include "fy-doc.h" + +const char *fy_document_builder_state_txt[] = { + [FYDBS_NODE] = "node", + [FYDBS_MAP_KEY] = "map-key", + [FYDBS_MAP_VAL] = "map-val", + [FYDBS_SEQ] = "seq", +}; + +void +fy_document_builder_reset(struct fy_document_builder *fydb) +{ + struct fy_document_builder_ctx *c; + unsigned int i; + + if (!fydb) + return; + + for (i = 0, c = fydb->stack; i < fydb->next; i++, c++) { + fy_node_free(c->fyn); + c->fyn = NULL; + fy_node_pair_free(c->fynp); + c->fynp = NULL; + } + fydb->next = 0; + + if (fydb->fyd) { + fy_document_destroy(fydb->fyd); + fydb->fyd = NULL; + } + fydb->in_stream = false; + fydb->doc_done = false; +} + +static const struct fy_document_builder_cfg docbuilder_default_cfg = { + .parse_cfg = { + .flags = FYPCF_DEFAULT_DOC, + } +}; + +struct fy_document_builder * +fy_document_builder_create(const struct fy_document_builder_cfg *cfg) +{ + struct fy_document_builder *fydb = NULL; + + if (!cfg) + cfg = &docbuilder_default_cfg; + + fydb = malloc(sizeof(*fydb)); + if (!fydb) + goto err_out; + + memset(fydb, 0, sizeof(*fydb)); + fydb->cfg = *cfg; + fydb->next = 0; + fydb->in_stream = false; + fydb->doc_done = false; + fydb->alloc = fy_depth_limit(); /* always start with this */ + fydb->max_depth = (cfg->parse_cfg.flags & FYPCF_DISABLE_DEPTH_LIMIT) ? 0 : fy_depth_limit(); + + fydb->stack = malloc(fydb->alloc * sizeof(*fydb->stack)); + if (!fydb->stack) + goto err_out; + + return fydb; + +err_out: + if (fydb) { + if (fydb->stack) + free(fydb->stack); + free(fydb); + } + + return NULL; +} + +struct fy_document_builder * +fy_document_builder_create_on_parser(struct fy_parser *fyp) +{ + struct fy_document_builder *fydb = NULL; + struct fy_document_state *fyds; + struct fy_document_builder_cfg cfg; + int rc; + + if (!fyp) + return NULL; + + memset(&cfg, 0, sizeof(cfg)); + cfg.parse_cfg = fyp->cfg; + cfg.diag = fy_diag_ref(fyp->diag); + + fydb = fy_document_builder_create(&cfg); + if (!fydb) { + fy_diag_unref(cfg.diag); + return NULL; + } + + fyds = fy_parser_get_document_state(fyp); + if (fyds) { + rc = fy_document_builder_set_in_document(fydb, fyds, true); + if (rc) { + fy_document_builder_destroy(fydb); + return NULL; + } + } + + return fydb; +} + +void +fy_document_builder_destroy(struct fy_document_builder *fydb) +{ + if (!fydb) + return; + + fy_document_builder_reset(fydb); + + fy_diag_unref(fydb->cfg.diag); + if (fydb->stack) + free(fydb->stack); + free(fydb); +} + +struct fy_document * +fy_document_builder_get_document(struct fy_document_builder *fydb) +{ + return fydb ? fydb->fyd : NULL; +} + +bool +fy_document_builder_is_in_stream(struct fy_document_builder *fydb) +{ + return fydb && fydb->in_stream; +} + +bool +fy_document_builder_is_in_document(struct fy_document_builder *fydb) +{ + return fydb && fydb->fyd != NULL && !fydb->doc_done; +} + +bool +fy_document_builder_is_document_complete(struct fy_document_builder *fydb) +{ + return fydb && fydb->fyd != NULL && fydb->doc_done; +} + +struct fy_document * +fy_document_builder_take_document(struct fy_document_builder *fydb) +{ + struct fy_document *fyd; + + if (!fy_document_builder_is_document_complete(fydb)) + return NULL; + fyd = fydb->fyd; + fydb->fyd = NULL; + fydb->doc_done = false; + return fyd; +} + +struct fy_document * +fy_document_builder_peek_document(struct fy_document_builder *fydb) +{ + struct fy_document *fyd; + struct fy_document_builder_ctx *c; + + if (!fydb) + return NULL; + + /* just peek; may be incomplete */ + fyd = fydb->fyd; + + assert(fydb->next > 0); + c = &fydb->stack[0]; + + /* wire the root */ + if (!fyd->root) + fyd->root = c->fyn; + + return fyd; +} + + +void +fy_document_builder_set_in_stream(struct fy_document_builder *fydb) +{ + if (!fydb) + return; + /* reset */ + fy_document_builder_reset(fydb); + + fydb->in_stream = true; +} + +int +fy_document_builder_set_in_document(struct fy_document_builder *fydb, struct fy_document_state *fyds, bool single) +{ + struct fy_document_builder_ctx *c; + int rc; + + if (!fydb) + return -1; + + /* reset */ + fy_document_builder_reset(fydb); + + fydb->in_stream = true; + + fydb->fyd = fy_document_create(&fydb->cfg.parse_cfg); + if (!fydb->fyd) + return -1; + + if (fyds) { + rc = fy_document_set_document_state(fydb->fyd, fyds); + if (rc) + return rc; + } + + fydb->doc_done = false; + fydb->single_mode = single; + + /* be paranoid */ + assert(fydb->next < fydb->alloc); + + c = &fydb->stack[++fydb->next - 1]; + memset(c, 0, sizeof(*c)); + c->s = FYDBS_NODE; + + return 0; +} + +int +fy_document_builder_process_event(struct fy_document_builder *fydb, struct fy_event *fye) +{ + enum fy_event_type etype; + struct fy_document *fyd; + struct fy_document_builder_ctx *c, *cp; + struct fy_node *fyn, *fyn_parent; + struct fy_node_pair *fynp; + struct fy_document_builder_ctx *newc; + struct fy_token *fyt; + int rc; + + etype = fye ? fye->type : FYET_NONE; + fyt = fye ? fy_event_get_token(fye) : NULL; + + /* not in document */ + if (!fydb->next) { + switch (etype) { + case FYET_STREAM_START: + FYDB_TOKEN_ERROR_CHECK(fydb, fyt, FYEM_DOC, + !fydb->in_stream, err_out, + "STREAM_START while in stream error"); + fydb->in_stream = true; + break; + + case FYET_STREAM_END: + FYDB_TOKEN_ERROR_CHECK(fydb, fyt, FYEM_DOC, + fydb->in_stream, err_out, + "STREAM_END while not in stream error"); + fydb->in_stream = false; + return 1; + + case FYET_DOCUMENT_START: + FYDB_TOKEN_ERROR_CHECK(fydb, fyt, FYEM_DOC, + fydb->in_stream, err_out, + "DOCUMENT_START while not in stream error"); + + /* no-one cares, destroy the document */ + if (!fydb->fyd) + fy_document_destroy(fydb->fyd); + + fydb->fyd = fy_document_create(&fydb->cfg.parse_cfg); + fydb_error_check(fydb, fydb->fyd, err_out, + "fy_document_create() failed"); + + rc = fy_document_set_document_state(fydb->fyd, fye->document_start.document_state); + fydb_error_check(fydb, !rc, err_out, + "fy_document_set_document_state() failed"); + + fydb->doc_done = false; + goto push; + + case FYET_DOCUMENT_END: + FYDB_TOKEN_ERROR_CHECK(fydb, fyt, FYEM_DOC, + fydb->in_stream, err_out, + "DOCUMENT_END while not in stream error"); + + FYDB_TOKEN_ERROR_CHECK(fydb, fyt, FYEM_DOC, + fydb->fyd, err_out, + "DOCUMENT_END without a document"); + + fydb->doc_done = true; + break; + + default: + /* unexpected event */ + FYDB_TOKEN_ERROR(fydb, fyt, FYEM_DOC, + "Unexpected event %s in non-build mode\n", + fy_event_type_txt[etype]); + goto err_out; + } + + return 0; + } + + fyd = fydb->fyd; + + c = &fydb->stack[fydb->next - 1]; + fyn = NULL; + + /* verify that we have a document */ + assert(fydb->fyd); + /* the top state must always be NODE for processing the event */ + assert(c->s == FYDBS_NODE); + + switch (etype) { + case FYET_SCALAR: + case FYET_ALIAS: + fyn = fy_node_alloc(fyd, FYNT_SCALAR); + fydb_error_check(fydb, fyn, err_out, + "fy_node_alloc() SCALAR failed"); + + if (etype == FYET_SCALAR) { + if (fye->scalar.value) + fyn->style = fy_node_style_from_scalar_style(fye->scalar.value->scalar.style); + else + fyn->style = FYNS_PLAIN; + fyn->tag = fy_token_ref(fye->scalar.tag); + if (fye->scalar.anchor) { + rc = fy_document_register_anchor(fyd, fyn, fy_token_ref(fye->scalar.anchor)); + fydb_error_check(fydb, !rc, err_out, + "fy_document_register_anchor() failed"); + } + fyn->scalar = fy_token_ref(fye->scalar.value); + } else { + fyn->style = FYNS_ALIAS; + fyn->scalar = fy_token_ref(fye->alias.anchor); + } + goto complete; + + case FYET_MAPPING_START: + c->s = FYDBS_MAP_KEY; + + fyn = fy_node_alloc(fyd, FYNT_MAPPING); + fydb_error_check(fydb, fyn, err_out, + "fy_node_alloc() MAPPING failed"); + + c->fyn = fyn; + if (fye->mapping_start.mapping_start) + fyn->style = fye->mapping_start.mapping_start->type == FYTT_FLOW_MAPPING_START ? FYNS_FLOW : FYNS_BLOCK; + else + fyn->style = FYNS_ANY; + fyn->tag = fy_token_ref(fye->mapping_start.tag); + if (fye->mapping_start.anchor) { + rc = fy_document_register_anchor(fyd, fyn, fy_token_ref(fye->mapping_start.anchor)); + fydb_error_check(fydb, !rc, err_out, + "fy_document_register_anchor() failed"); + } + fyn->mapping_start = fy_token_ref(fye->mapping_start.mapping_start); + break; + + case FYET_MAPPING_END: + + FYDB_TOKEN_ERROR_CHECK(fydb, fyt, FYEM_DOC, fydb->next > 1, err_out, + "Unexpected MAPPING_END (unexpected end of mapping)"); + + cp = &fydb->stack[fydb->next - 2]; + + FYDB_TOKEN_ERROR_CHECK(fydb, fyt, FYEM_DOC, cp->s == FYDBS_MAP_KEY, err_out, + "Unexpected MAPPING_END (not in mapping)"); + + fyn = cp->fyn; + fyn->mapping_end = fy_token_ref(fye->mapping_end.mapping_end); + fydb->next--; + goto complete; + + case FYET_SEQUENCE_START: + c->s = FYDBS_SEQ; + fyn = fy_node_alloc(fyd, FYNT_SEQUENCE); + fydb_error_check(fydb, fyn, err_out, + "fy_node_alloc() SEQUENCE failed"); + + c->fyn = fyn; + if (fye->sequence_start.sequence_start) + fyn->style = fye->sequence_start.sequence_start->type == FYTT_FLOW_SEQUENCE_START ? FYNS_FLOW : FYNS_BLOCK; + else + fyn->style = FYNS_ANY; + fyn->tag = fy_token_ref(fye->sequence_start.tag); + if (fye->sequence_start.anchor) { + rc = fy_document_register_anchor(fyd, fyn, fy_token_ref(fye->sequence_start.anchor)); + fydb_error_check(fydb, !rc, err_out, + "fy_document_register_anchor() failed"); + } + fyn->sequence_start = fy_token_ref(fye->sequence_start.sequence_start); + break; + + case FYET_SEQUENCE_END: + + FYDB_TOKEN_ERROR_CHECK(fydb, fyt, FYEM_DOC, fydb->next > 1, err_out, + "Unexpected SEQUENCE_END (unexpected end of sequence)"); + + cp = &fydb->stack[fydb->next - 2]; + + FYDB_TOKEN_ERROR_CHECK(fydb, fyt, FYEM_DOC, cp->s == FYDBS_SEQ, err_out, + "Unexpected MAPPING_SEQUENCE (not in sequence)"); + + fyn = cp->fyn; + fyn->sequence_end = fy_token_ref(fye->sequence_end.sequence_end); + fydb->next--; + goto complete; + + default: + /* unexpected event */ + FYDB_TOKEN_ERROR(fydb, fyt, FYEM_DOC, + "Unexpected event %s in build mode\n", + fy_event_type_txt[etype]); + goto err_out; + } + +push: + FYDB_TOKEN_ERROR_CHECK(fydb, fyt, FYEM_DOC, + !fydb->max_depth || fydb->next < fydb->max_depth, err_out, + "Max depth (%d) exceeded\n", fydb->next); + + /* grow the stack? */ + if (fydb->next >= fydb->alloc) { + newc = realloc(fydb->stack, fydb->alloc * 2 * sizeof(*fydb->stack)); + fydb_error_check(fydb, newc, err_out, + "Unable to grow the context stack"); + fydb->alloc *= 2; + fydb->stack = newc; + } + assert(fydb->next < fydb->alloc); + + c = &fydb->stack[++fydb->next - 1]; + memset(c, 0, sizeof(*c)); + c->s = FYDBS_NODE; + return 0; + +err_out: + return -1; + +complete: + assert(fydb->next > 0); + c = &fydb->stack[fydb->next - 1]; + c->fyn = fyn; + assert(fydb->next > 0); + fydb->next--; + + /* root */ + if (fydb->next == 0) { + fyd->root = fyn; + /* if we're in single mode, don't wait for doc end */ + if (fydb->single_mode) + fydb->doc_done = true; + return 1; + } + + c = &fydb->stack[fydb->next - 1]; + + fyn_parent = c->fyn; + + switch (c->s) { + + case FYDBS_MAP_KEY: + fynp = fy_node_pair_alloc(fyd); + assert(fynp); + fynp->key = fyn; + c->fynp = fynp; + + /* if we don't allow duplicate keys */ + if (!(fyd->parse_cfg.flags & FYPCF_ALLOW_DUPLICATE_KEYS)) { + + /* make sure we don't add an already existing key */ + if (fy_node_mapping_key_is_duplicate(fyn_parent, fyn)) { + FYDB_NODE_ERROR(fydb, fyn, FYEM_DOC, "duplicate key"); + goto err_out; + } + } + + c->s = FYDBS_MAP_VAL; + goto push; + + case FYDBS_MAP_VAL: + fynp = c->fynp; + assert(fynp); + fynp->value = fyn; + + /* set the parent of the node pair and value */ + fynp->parent = fyn_parent; + if (fynp->key) { + fynp->key->parent = fyn_parent; + fynp->key->key_root = true; + } + if (fynp->value) + fynp->value->parent = fyn_parent; + + fy_node_pair_list_add_tail(&c->fyn->mapping, fynp); + if (fyn->xl) { + rc = fy_accel_insert(fyn->xl, fynp->key, fynp); + assert(!rc); + } + if (fynp->key) + fynp->key->attached = true; + if (fynp->value) + fynp->value->attached = true; + + c->fynp = NULL; + c->s = FYDBS_MAP_KEY; + goto push; + + case FYDBS_SEQ: + /* append sequence */ + fyn->parent = fyn_parent; + fy_node_list_add_tail(&c->fyn->sequence, fyn); + fyn->attached = true; + goto push; + + case FYDBS_NODE: + /* complete is a scalar */ + fyn->parent = fyn_parent; + return 0; + } + + return 0; +} + +struct fy_document * +fy_document_builder_load_document(struct fy_document_builder *fydb, + struct fy_parser *fyp) +{ + struct fy_eventp *fyep = NULL; + int rc; + + if (fyp->state == FYPS_END) + return NULL; + + while (!fy_document_builder_is_document_complete(fydb) && + (fyep = fy_parse_private(fyp)) != NULL) { + rc = fy_document_builder_process_event(fydb, &fyep->e); + fy_parse_eventp_recycle(fyp, fyep); + if (rc < 0) { + fyp->stream_error = true; + return NULL; + } + } + + /* get ownership of the document */ + return fy_document_builder_take_document(fydb); +} + +struct fy_document * +fy_document_builder_event_document(struct fy_document_builder *fydb, struct fy_eventp_list *evpl) +{ + struct fy_eventp *fyep = NULL; + int rc; + + if (!fydb || !evpl) + return NULL; + + for (fyep = fy_eventp_list_head(evpl); fyep; fyep = fy_eventp_next(evpl, fyep)) { + + if (fy_document_builder_is_document_complete(fydb)) + break; + + rc = fy_document_builder_process_event(fydb, &fyep->e); + if (rc < 0) + return NULL; + + } + + /* get ownership of the document */ + return fy_document_builder_take_document(fydb); +} diff --git a/src/lib/fy-docbuilder.h b/src/lib/fy-docbuilder.h new file mode 100644 index 0000000..5c8780d --- /dev/null +++ b/src/lib/fy-docbuilder.h @@ -0,0 +1,161 @@ +/* + * fy-docbuilder.h - YAML document builder internal header file + * + * Copyright (c) 2022 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_DOCBUILDER_H +#define FY_DOCBUILDER_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include + +#include "fy-doc.h" + +#include "fy-diag.h" + +enum fy_document_builder_state { + FYDBS_NODE, + FYDBS_MAP_KEY, + FYDBS_MAP_VAL, + FYDBS_SEQ, +}; + +struct fy_document_builder_ctx { + enum fy_document_builder_state s; + struct fy_node *fyn; + struct fy_node_pair *fynp; /* for mapping */ +}; + +struct fy_document_builder { + struct fy_document_builder_cfg cfg; + struct fy_document *fyd; + bool single_mode; + bool in_stream; + bool doc_done; + unsigned int next; + unsigned int alloc; + unsigned int max_depth; + struct fy_document_builder_ctx *stack; +}; + +/* internal only */ +struct fy_document * +fy_document_builder_event_document(struct fy_document_builder *fydb, struct fy_eventp_list *evpl); + +/* diagnostics */ + +static inline bool +fydb_debug_log_level_is_enabled(struct fy_document_builder *fydb, enum fy_error_module module) +{ + return fydb && fy_diag_log_level_is_enabled(fydb->cfg.diag, FYET_DEBUG, module); +} + +int fy_document_builder_vdiag(struct fy_document_builder *fydb, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap); + +int fy_document_builder_diag(struct fy_document_builder *fydb, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) + __attribute__((format(printf, 6, 7))); + +void fy_document_builder_diag_vreport(struct fy_document_builder *fydb, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap); +void fy_document_builder_diag_report(struct fy_document_builder *fydb, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +#ifdef FY_DEVMODE + +#define fydb_debug(_fydb, _module, _fmt, ...) \ + do { \ + struct fy_document_builder *__fydb = (_fydb); \ + enum fy_error_module __module = (_module); \ + \ + if (fydb_debug_log_level_is_enabled(__fydb, __module)) \ + fy_document_builder_diag(__fydb, FYET_DEBUG | FYDF_MODULE(_module), \ + __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__); \ + } while(0) +#else + +#define fydb_debug(_fydb, _module, _fmt, ...) \ + do { } while(0) + +#endif + +#define fydb_info(_fydb, _fmt, ...) \ + fy_document_builder_diag((_fydb), FYET_INFO, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fydb_notice(_fydb, _fmt, ...) \ + fy_document_builder_diag((_fydb), FYET_NOTICE, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fydb_warning(_fydb, _fmt, ...) \ + fy_document_builder_diag((_fydb), FYET_WARNING, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fydb_error(_fydb, _fmt, ...) \ + fy_document_builder_diag((_fydb), FYET_ERROR, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) + +#define fydb_error_check(_fydb, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + fydb_error((_fydb), _fmt, ## __VA_ARGS__); \ + goto _label ; \ + } \ + } while(0) + +#define _FYDB_TOKEN_DIAG(_fydb, _fyt, _type, _module, _fmt, ...) \ + do { \ + struct fy_diag_report_ctx _drc; \ + memset(&_drc, 0, sizeof(_drc)); \ + _drc.type = (_type); \ + _drc.module = (_module); \ + _drc.fyt = (_fyt); \ + fy_document_builder_diag_report((_fydb), &_drc, (_fmt) , ## __VA_ARGS__); \ + } while(0) + +#define FYDB_TOKEN_DIAG(_fydb, _fyt, _type, _module, _fmt, ...) \ + _FYDB_TOKEN_DIAG(_fydb, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYDB_NODE_DIAG(_fydb, _fyn, _type, _module, _fmt, ...) \ + _FYDB_TOKEN_DIAG(_fydb, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYDB_TOKEN_ERROR(_fydb, _fyt, _module, _fmt, ...) \ + FYDB_TOKEN_DIAG(_fydb, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYDB_NODE_ERROR(_fydb, _fyn, _module, _fmt, ...) \ + FYDB_NODE_DIAG(_fydb, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYDB_TOKEN_ERROR_CHECK(_fydb, _fyt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYDB_TOKEN_ERROR(_fydb, _fyt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYDB_NODE_ERROR_CHECK(_fydb, _fyn, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYDB_NODE_ERROR(_fydb, _fyn, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYDB_TOKEN_WARNING(_fydb, _fyt, _module, _fmt, ...) \ + FYDB_TOKEN_DIAG(_fydb, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#endif diff --git a/src/lib/fy-docstate.c b/src/lib/fy-docstate.c new file mode 100644 index 0000000..6c530bb --- /dev/null +++ b/src/lib/fy-docstate.c @@ -0,0 +1,447 @@ +/* + * fy-docstate.c - YAML document state methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" +#include "fy-doc.h" + +#include "fy-docstate.h" + +struct fy_document_state *fy_document_state_alloc(void) +{ + struct fy_document_state *fyds; + + fyds = malloc(sizeof(*fyds)); + if (!fyds) + return NULL; + memset(fyds, 0, sizeof(*fyds)); + + fyds->fyt_vd = NULL; + fy_token_list_init(&fyds->fyt_td); + + fyds->refs = 1; + + return fyds; +} + +void fy_document_state_free(struct fy_document_state *fyds) +{ + if (!fyds) + return; + + assert(fyds->refs == 1); + + fy_token_unref(fyds->fyt_vd); + fy_token_list_unref_all(&fyds->fyt_td); + + free(fyds); +} + +struct fy_document_state *fy_document_state_ref(struct fy_document_state *fyds) +{ + if (!fyds) + return NULL; + + assert(fyds->refs + 1 > 0); + + fyds->refs++; + + return fyds; +} + +void fy_document_state_unref(struct fy_document_state *fyds) +{ + if (!fyds) + return; + + assert(fyds->refs > 0); + + if (fyds->refs == 1) + fy_document_state_free(fyds); + else + fyds->refs--; +} + +int fy_document_state_append_tag(struct fy_document_state *fyds, + const char *handle, const char *prefix, + bool is_default) +{ + struct fy_token *fyt = NULL; + struct fy_input *fyi = NULL; + char *data; + size_t size, handle_size, prefix_size; + struct fy_atom atom; + + size = strlen(handle) + 1 + strlen(prefix); + data = malloc(size + 1); + if (!data) + goto err_out; + + snprintf(data, size + 1, "%s %s", handle, prefix); + + fyi = fy_input_from_malloc_data(data, size, &atom, true); + if (!fyi) + goto err_out; + data = NULL; /* ownership now at input */ + + handle_size = strlen(handle); + prefix_size = strlen(prefix); + + fyt = fy_token_create(FYTT_TAG_DIRECTIVE, &atom, + handle_size, prefix_size, + is_default); + if (!fyt) + goto err_out; + + fy_token_list_add_tail(&fyds->fyt_td, fyt); + + if (!fy_tag_is_default_internal(handle, handle_size, prefix, prefix_size)) + fyds->tags_explicit = true; + + /* take away the input reference */ + fy_input_unref(fyi); + + return 0; + +err_out: + fy_token_unref(fyt); + fy_input_unref(fyi); + if (data) + free(data); + return -1; +} + +struct fy_document_state *fy_document_state_default( + const struct fy_version *default_version, + const struct fy_tag * const *default_tags) +{ + struct fy_document_state *fyds = NULL; + const struct fy_tag *fytag; + int i, rc; + + if (!default_version) + default_version = &fy_default_version; + + if (!default_tags) + default_tags = fy_default_tags; + + fyds = fy_document_state_alloc(); + if (!fyds) + goto err_out; + + fyds->version = *default_version; + + fyds->version_explicit = false; + fyds->tags_explicit = false; + fyds->start_implicit = true; + fyds->end_implicit = true; + fyds->json_mode = false; + + memset(&fyds->start_mark, 0, sizeof(fyds->start_mark)); + memset(&fyds->end_mark, 0, sizeof(fyds->end_mark)); + + fyds->fyt_vd = NULL; + fy_token_list_init(&fyds->fyt_td); + + for (i = 0; (fytag = default_tags[i]) != NULL; i++) { + + rc = fy_document_state_append_tag(fyds, fytag->handle, fytag->prefix, true); + if (rc) + goto err_out; + } + + return fyds; +err_out: + fy_document_state_unref(fyds); + return NULL; +} + +struct fy_document_state *fy_document_state_copy(struct fy_document_state *fyds) +{ + struct fy_document_state *fyds_new = NULL; + struct fy_token *fyt_td, *fyt; + + fyds_new = fy_document_state_alloc(); + if (!fyds_new) + goto err_out; + + fyds_new->version = fyds->version; + fyds_new->version_explicit = fyds->version_explicit; + fyds_new->tags_explicit = fyds->tags_explicit; + fyds_new->start_implicit = fyds->start_implicit; + fyds_new->end_implicit = fyds->end_implicit; + fyds_new->json_mode = fyds->json_mode; + + fyds_new->start_mark = fyds->start_mark; + fyds_new->end_mark = fyds->end_mark; + + if (fyds->fyt_vd) { + fyt = fy_token_alloc(); + if (!fyt) + goto err_out; + + fyt->type = FYTT_VERSION_DIRECTIVE; + fyt->handle = fyds->fyt_vd->handle; + fyt->version_directive.vers = fyds->fyt_vd->version_directive.vers; + + /* take reference */ + fy_input_ref(fyt->handle.fyi); + + fyds_new->fyt_vd = fyt; + } + + for (fyt = fy_token_list_first(&fyds->fyt_td); fyt; fyt = fy_token_next(&fyds->fyt_td, fyt)) { + fyt_td = fy_token_alloc(); + if (!fyt_td) + goto err_out; + + fyt_td->type = FYTT_TAG_DIRECTIVE; + fyt_td->tag_directive.tag_length = fyt->tag_directive.tag_length; + fyt_td->tag_directive.uri_length = fyt->tag_directive.uri_length; + fyt_td->tag_directive.is_default = fyt->tag_directive.is_default; + fyt_td->handle = fyt->handle; + fyt_td->tag_directive.prefix0 = NULL; + fyt_td->tag_directive.handle0 = NULL; + + /* take reference */ + fy_input_ref(fyt_td->handle.fyi); + + /* append to the new document state */ + fy_token_list_add_tail(&fyds_new->fyt_td, fyt_td); + } + + return fyds_new; + +err_out: + fy_document_state_unref(fyds_new); + return NULL; +} + +struct fy_token *fy_document_state_lookup_tag_directive(struct fy_document_state *fyds, + const char *handle, size_t handle_size) +{ + const char *td_handle; + size_t td_handle_size; + struct fy_token *fyt; + + if (!fyds) + return NULL; + + for (fyt = fy_token_list_first(&fyds->fyt_td); fyt; fyt = fy_token_next(&fyds->fyt_td, fyt)) { + + td_handle = fy_tag_directive_token_handle(fyt, &td_handle_size); + assert(td_handle); + + if (handle_size == td_handle_size && !memcmp(handle, td_handle, handle_size)) + return fyt; + + } + + return NULL; +} + +int fy_document_state_merge(struct fy_document_state *fyds, + struct fy_document_state *fydsc) +{ + const char *td_prefix, *tdc_handle, *tdc_prefix; + size_t td_prefix_size, tdc_handle_size, tdc_prefix_size; + struct fy_token *fyt, *fytc_td, *fyt_td; + + if (!fyds || !fydsc) + return -1; + + /* check if there's a duplicate handle (which differs */ + for (fytc_td = fy_token_list_first(&fydsc->fyt_td); fytc_td; + fytc_td = fy_token_next(&fydsc->fyt_td, fytc_td)) { + + tdc_handle = fy_tag_directive_token_handle(fytc_td, &tdc_handle_size); + if (!tdc_handle) + goto err_out; + tdc_prefix = fy_tag_directive_token_prefix(fytc_td, &tdc_prefix_size); + if (!tdc_prefix) + goto err_out; + + fyt_td = fy_document_state_lookup_tag_directive(fyds, tdc_handle, tdc_handle_size); + if (fyt_td) { + /* exists, must check whether the prefixes match */ + td_prefix = fy_tag_directive_token_prefix(fyt_td, &td_prefix_size); + assert(td_prefix); + + /* match? do nothing */ + if (tdc_prefix_size == td_prefix_size && + !memcmp(tdc_prefix, td_prefix, td_prefix_size)) + continue; + + if (!fy_token_tag_directive_is_overridable(fyt_td)) + goto err_out; + + /* override tag directive */ + fy_token_list_del(&fyds->fyt_td, fyt_td); + fy_token_unref(fyt_td); + } + + fyt = fy_token_create(FYTT_TAG_DIRECTIVE, + &fytc_td->handle, + fytc_td->tag_directive.tag_length, + fytc_td->tag_directive.uri_length, + fytc_td->tag_directive.is_default); + if (!fyt) + goto err_out; + + fy_token_list_add_tail(&fyds->fyt_td, fyt); + } + + /* merge other document state */ + fyds->version_explicit |= fydsc->version_explicit; + fyds->tags_explicit |= fydsc->tags_explicit; + /* NOTE: json mode is not carried over */ + + if (fyds->version.major < fydsc->version.major || + (fyds->version.major == fydsc->version.major && + fyds->version.minor < fydsc->version.minor)) + fyds->version = fydsc->version; + + return 0; + +err_out: + return -1; +} + +const struct fy_version * +fy_document_state_version(struct fy_document_state *fyds) +{ + /* return the default if not set */ + return fyds ? &fyds->version : &fy_default_version; +} + +const struct fy_mark *fy_document_state_start_mark(struct fy_document_state *fyds) +{ + return fyds ? &fyds->start_mark : NULL; +} + +const struct fy_mark *fy_document_state_end_mark(struct fy_document_state *fyds) +{ + return fyds ? &fyds->end_mark : NULL; +} + +bool fy_document_state_version_explicit(struct fy_document_state *fyds) +{ + return fyds ? fyds->version_explicit : false; +} + +bool fy_document_state_tags_explicit(struct fy_document_state *fyds) +{ + return fyds ? fyds->tags_explicit : false; +} + +bool fy_document_state_start_implicit(struct fy_document_state *fyds) +{ + return fyds ? fyds->start_implicit : true; +} + +bool fy_document_state_end_implicit(struct fy_document_state *fyds) +{ + return fyds ? fyds->end_implicit : true; +} + +bool fy_document_state_json_mode(struct fy_document_state *fyds) +{ + return fyds ? fyds->json_mode : true; +} + +const struct fy_tag * +fy_document_state_tag_directive_iterate(struct fy_document_state *fyds, void **iterp) +{ + struct fy_token *fyt; + const struct fy_tag *tag; + + if (!fyds || !iterp) + return NULL; + + fyt = *iterp; + fyt = !fyt ? fy_token_list_head(&fyds->fyt_td) : fy_token_next(&fyds->fyt_td, fyt); + if (!fyt) + return NULL; + + /* sanity check */ + assert(fyt->type == FYTT_TAG_DIRECTIVE); + + /* always refresh, should be relatively infrequent */ + fyt->tag_directive.tag.handle = fy_tag_directive_token_handle0(fyt); + fyt->tag_directive.tag.prefix = fy_tag_directive_token_prefix0(fyt); + + tag = &fyt->tag_directive.tag; + + *iterp = fyt; + + return tag; +} + +struct fy_tag ** +fy_document_state_tag_directives(struct fy_document_state *fyds) +{ + struct fy_tag *tags, **tagsp; + const struct fy_tag *fytag; + size_t size, len; + void *iter; + char *s; + int i; + + if (!fyds) + return NULL; + + /* first find the extents of the area we're going to need */ + i = 0; + size = 0; + iter = NULL; + while ((fytag = fy_document_state_tag_directive_iterate(fyds, &iter)) != NULL) { + size += strlen(fytag->handle) + 1 + strlen(fytag->prefix) + 1; + i++; + } + size += sizeof(struct fy_tag) * i; + size += sizeof(struct fy_tag *) * (i + 1); + + tagsp = malloc(size); + if (!tagsp) + return NULL; + memset(tagsp, 0, size); + tags = (void *)(tagsp + i + 1); + s = (void *)(tags + i); + + i = 0; + iter = NULL; + while ((fytag = fy_document_state_tag_directive_iterate(fyds, &iter)) != NULL) { + tags[i].handle = s; + len = strlen(fytag->handle); + memcpy(s, fytag->handle, len + 1); + s += len + 1; + + tags[i].prefix = s; + len = strlen(fytag->prefix); + memcpy(s, fytag->prefix, len + 1); + s += len + 1; + + tagsp[i] = &tags[i]; + + i++; + } + tagsp[i] = NULL; /* NULL terminated */ + + return tagsp; +} diff --git a/src/lib/fy-docstate.h b/src/lib/fy-docstate.h new file mode 100644 index 0000000..eb8df53 --- /dev/null +++ b/src/lib/fy-docstate.h @@ -0,0 +1,64 @@ +/* + * fy-docstate.h - YAML document state header. + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_DOCSTATE_H +#define FY_DOCSTATE_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include + +#include "fy-ctype.h" +#include "fy-list.h" +#include "fy-typelist.h" +#include "fy-token.h" + +struct fy_document; + +struct fy_document_state { + int refs; + struct fy_version version; + bool version_explicit : 1; + bool tags_explicit : 1; + bool start_implicit : 1; + bool end_implicit : 1; + bool started_explicit : 1; + bool json_mode : 1; + struct fy_mark start_mark; + struct fy_mark end_mark; + struct fy_token *fyt_vd; /* version directive */ + struct fy_token_list fyt_td; /* tag directives */ +}; + +struct fy_document_state *fy_document_state_alloc(void); +void fy_document_state_free(struct fy_document_state *fyds); +struct fy_document_state *fy_document_state_ref(struct fy_document_state *fyds); +void fy_document_state_unref(struct fy_document_state *fyds); + +int fy_document_state_append_tag(struct fy_document_state *fyds, + const char *handle, const char *prefix, + bool is_default); + +struct fy_document_state *fy_document_state_default( + const struct fy_version *default_version, + const struct fy_tag * const *default_tags); + +struct fy_document_state *fy_document_state_copy(struct fy_document_state *fyds); +int fy_document_state_merge(struct fy_document_state *fyds, + struct fy_document_state *fydsc); + +struct fy_token *fy_document_state_lookup_tag_directive(struct fy_document_state *fyds, + const char *handle, size_t handle_size); + +#endif diff --git a/src/lib/fy-dump.c b/src/lib/fy-dump.c new file mode 100644 index 0000000..17a7543 --- /dev/null +++ b/src/lib/fy-dump.c @@ -0,0 +1,314 @@ +/* + * fy-dump.c - various debugging methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" +#include "fy-ctype.h" +#include "fy-utf8.h" + +const char *fy_token_type_txt[FYTT_COUNT] = { + [FYTT_NONE] = "", + [FYTT_STREAM_START] = "STRM+", + [FYTT_STREAM_END] = "STRM-", + [FYTT_VERSION_DIRECTIVE] = "VRSD", + [FYTT_TAG_DIRECTIVE] = "TAGD", + [FYTT_DOCUMENT_START] = "DOC+", + [FYTT_DOCUMENT_END] = "DOC-", + [FYTT_BLOCK_SEQUENCE_START] = "BSEQ+", + [FYTT_BLOCK_MAPPING_START] = "BMAP+", + [FYTT_BLOCK_END] = "BEND", + [FYTT_FLOW_SEQUENCE_START] = "FSEQ+", + [FYTT_FLOW_SEQUENCE_END] = "FSEQ-", + [FYTT_FLOW_MAPPING_START] = "FMAP+", + [FYTT_FLOW_MAPPING_END] = "FMAP-", + [FYTT_BLOCK_ENTRY] = "BENTR", + [FYTT_FLOW_ENTRY] = "FENTR", + [FYTT_KEY] = "KEY", + [FYTT_SCALAR] = "SCLR", + [FYTT_VALUE] = "VAL", + [FYTT_ALIAS] = "ALIAS", + [FYTT_ANCHOR] = "ANCHR", + [FYTT_TAG] = "TAG", + [FYTT_INPUT_MARKER] = "INPUT_MARKER", + + [FYTT_PE_SLASH] = "PE_SLASH", + [FYTT_PE_ROOT] = "PE_ROOT", + [FYTT_PE_THIS] = "PE_THIS", + [FYTT_PE_PARENT] = "PE_PARENT", + [FYTT_PE_MAP_KEY] = "PE_MAP_KEY", + [FYTT_PE_SEQ_INDEX] = "PE_SEQ_INDEX", + [FYTT_PE_SEQ_SLICE] = "PE_SEQ_SLICE", + [FYTT_PE_SCALAR_FILTER] = "PE_SCALAR_FILTER", + [FYTT_PE_COLLECTION_FILTER] = "PE_COLLECTION_FILTER", + [FYTT_PE_SEQ_FILTER] = "PE_SEQ_FILTER", + [FYTT_PE_MAP_FILTER] = "PE_MAP_FILTER", + [FYTT_PE_UNIQUE_FILTER] = "PE_UNIQUE_FILTER", + [FYTT_PE_EVERY_CHILD] = "PE_EVERY_CHILD", + [FYTT_PE_EVERY_CHILD_R] = "PE_EVERY_CHILD_R", + [FYTT_PE_ALIAS] = "PE_ALIAS", + [FYTT_PE_SIBLING] = "PE_SIBLING", + [FYTT_PE_COMMA] = "PE_COMMA", + [FYTT_PE_BARBAR] = "PE_BARBAR", + [FYTT_PE_AMPAMP] = "PE_AMPAMP", + [FYTT_PE_LPAREN] = "PE_LPAREN", + [FYTT_PE_RPAREN] = "PE_RPAREN", + + [FYTT_PE_EQEQ] = "PE_EQEQ", + [FYTT_PE_NOTEQ] = "PE_NOTEQ", + [FYTT_PE_LT] = "PE_LT", + [FYTT_PE_GT] = "PE_GT", + [FYTT_PE_LTE] = "PE_LTE", + [FYTT_PE_GTE] = "PE_GTE", + + [FYTT_SE_PLUS] = "SE_PLUS", + [FYTT_SE_MINUS] = "SE_MINUS", + [FYTT_SE_MULT] = "SE_MULT", + [FYTT_SE_DIV] = "SE_DIV", + + [FYTT_PE_METHOD] = "PE_METHOD", + [FYTT_SE_METHOD] = "SE_METHOD", + + [FYTT_PE_BANG] = "PE_BANG", + [FYTT_PE_AT] = "PE_AT", +}; + +char *fy_token_dump_format(struct fy_token *fyt, char *buf, size_t bufsz) +{ + const char *typetxt, *text; + size_t size; + enum fy_token_type type; + const char *pfx, *sfx; + + if (fyt && (unsigned int)fyt->type < sizeof(fy_token_type_txt)/ + sizeof(fy_token_type_txt[0])) { + typetxt = fy_token_type_txt[fyt->type]; + type = fyt->type; + } else { + typetxt = ""; + type = FYTT_NONE; + } + + size = 0; + switch (type) { + case FYTT_SCALAR: + case FYTT_ALIAS: + case FYTT_ANCHOR: + text = fy_token_get_text(fyt, &size); + break; + default: + text = NULL; + break; + } + + if (!text) { + snprintf(buf, bufsz, "%s", typetxt); + return buf; + } + + pfx = typetxt; + sfx = ""; + switch (type) { + case FYTT_SCALAR: + + pfx = "\""; + + /* not too large */ + if (size > 20) + size = 20; + text = fy_utf8_format_text_a(text, size, fyue_doublequote); + size = strlen(text); + if (size > 10) { + sfx = "...\""; + size = 7; + } else { + sfx = "\""; + } + break; + case FYTT_ALIAS: + case FYTT_ANCHOR: + sfx = type == FYTT_ALIAS ? "*" : "&"; + if (size > 10) { + sfx = "..."; + size = 7; + } else + sfx = ""; + break; + + default: + break; + } + + snprintf(buf, bufsz, "%s%.*s%s", pfx, (int)size, text, sfx); + + return buf; +} + +char *fy_token_list_dump_format(struct fy_token_list *fytl, + struct fy_token *fyt_highlight, char *buf, size_t bufsz) +{ + char *s, *e; + struct fy_token *fyt; + + s = buf; + e = buf + bufsz - 1; + for (fyt = fy_token_list_first(fytl); fyt; fyt = fy_token_next(fytl, fyt)) { + + if (s >= (e - 1)) + break; + + s += snprintf(s, e - s, "%s%s", + fyt != fy_token_list_first(fytl) ? "," : "", + fyt_highlight == fyt ? "*" : ""); + + fy_token_dump_format(fyt, s, e - s); + + s += strlen(s); + } + *s = '\0'; + + return buf; +} + +char *fy_simple_key_dump_format(struct fy_parser *fyp, struct fy_simple_key *fysk, char *buf, size_t bufsz) +{ + char tbuf[80]; + + if (!fysk) { + if (bufsz > 0) + *buf = '\0'; + return buf; + } + + fy_token_dump_format(fysk->token, tbuf, sizeof(tbuf)); + + snprintf(buf, bufsz, "%s/%c%c/%d/<%d-%d,%d-%d>", tbuf, + fysk->required ? 'R' : '-', + fysk->implicit_complex ? 'C' : '-', + fysk->flow_level, + fysk->mark.line, fysk->mark.column, + fysk->end_mark.line, fysk->end_mark.column); + return buf; +} + +char *fy_simple_key_list_dump_format(struct fy_parser *fyp, struct fy_simple_key_list *fyskl, + struct fy_simple_key *fysk_highlight, char *buf, size_t bufsz) +{ + char *s, *e; + struct fy_simple_key *fysk; + + s = buf; + e = buf + bufsz - 1; + for (fysk = fy_simple_key_list_first(fyskl); fysk; fysk = fy_simple_key_next(fyskl, fysk)) { + + if (s >= (e - 1)) + break; + + s += snprintf(s, e - s, "%s%s", + fysk != fy_simple_key_list_first(fyskl) ? "," : "", + fysk_highlight == fysk ? "*" : ""); + + fy_simple_key_dump_format(fyp, fysk, s, e - s); + + s += strlen(s); + } + *s = '\0'; + + return buf; +} + +#ifdef FY_DEVMODE + +void fyp_debug_dump_token_list(struct fy_parser *fyp, struct fy_token_list *fytl, + struct fy_token *fyt_highlight, const char *banner) +{ + char buf[4096]; + + if (!fyp || !fyp->diag || FYET_DEBUG < fyp->diag->cfg.level) + return; + + fyp_scan_debug(fyp, "%s%s\n", banner, + fy_token_list_dump_format(fytl, fyt_highlight, buf, sizeof(buf))); +} + +void fyp_debug_dump_token(struct fy_parser *fyp, struct fy_token *fyt, const char *banner) +{ + char buf[80]; + + if (!fyp || !fyp->diag || FYET_DEBUG < fyp->diag->cfg.level) + return; + + fyp_scan_debug(fyp, "%s%s\n", banner, + fy_token_dump_format(fyt, buf, sizeof(buf))); +} + +void fyp_debug_dump_simple_key_list(struct fy_parser *fyp, struct fy_simple_key_list *fyskl, + struct fy_simple_key *fysk_highlight, const char *banner) +{ + char buf[4096]; + + if (!fyp || !fyp->diag || FYET_DEBUG < fyp->diag->cfg.level) + return; + + fyp_scan_debug(fyp, "%s%s\n", banner, + fy_simple_key_list_dump_format(fyp, fyskl, fysk_highlight, buf, sizeof(buf))); +} + +void fyp_debug_dump_simple_key(struct fy_parser *fyp, struct fy_simple_key *fysk, const char *banner) +{ + char buf[80]; + + if (!fyp || !fyp->diag || FYET_DEBUG < fyp->diag->cfg.level) + return; + + fyp_scan_debug(fyp, "%s%s\n", banner, + fy_simple_key_dump_format(fyp, fysk, buf, sizeof(buf))); +} + +void fyp_debug_dump_input(struct fy_parser *fyp, const struct fy_input_cfg *fyic, + const char *banner) +{ + switch (fyic->type) { + case fyit_file: + fyp_scan_debug(fyp, "%s: filename=\"%s\"\n", banner, + fyic->file.filename); + break; + case fyit_stream: + fyp_scan_debug(fyp, "%s: stream=\"%s\" fileno=%d\n", banner, + fyic->stream.name, fileno(fyic->stream.fp)); + break; + case fyit_memory: + fyp_scan_debug(fyp, "%s: start=%p size=%zu\n", banner, + fyic->memory.data, fyic->memory.size); + break; + case fyit_alloc: + fyp_scan_debug(fyp, "%s: start=%p size=%zu\n", banner, + fyic->alloc.data, fyic->alloc.size); + break; + default: + break; + } +} + +#endif diff --git a/src/lib/fy-dump.h b/src/lib/fy-dump.h new file mode 100644 index 0000000..13b1c9a --- /dev/null +++ b/src/lib/fy-dump.h @@ -0,0 +1,92 @@ +/* + * fy-dump.h - dumps for various internal structures + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_DUMP_H +#define FY_DUMP_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include + +#include "fy-list.h" +#include "fy-diag.h" + +struct fy_parser; +struct fy_token; +struct fy_token_list; +struct fy_simple_key; +struct fy_simple_key_list; +struct fy_input_cfg; + +extern const char *fy_token_type_txt[]; + +char *fy_token_dump_format(struct fy_token *fyt, char *buf, size_t bufsz); +char *fy_token_list_dump_format(struct fy_token_list *fytl, + struct fy_token *fyt_highlight, char *buf, size_t bufsz); + +char *fy_simple_key_dump_format(struct fy_parser *fyp, struct fy_simple_key *fysk, char *buf, size_t bufsz); +char *fy_simple_key_list_dump_format(struct fy_parser *fyp, struct fy_simple_key_list *fyskl, + struct fy_simple_key *fysk_highlight, char *buf, size_t bufsz); + +#ifdef FY_DEVMODE + +void fyp_debug_dump_token_list(struct fy_parser *fyp, struct fy_token_list *fytl, + struct fy_token *fyt_highlight, const char *banner); +void fyp_debug_dump_token(struct fy_parser *fyp, struct fy_token *fyt, const char *banner); + +void fyp_debug_dump_simple_key_list(struct fy_parser *fyp, struct fy_simple_key_list *fyskl, + struct fy_simple_key *fysk_highlight, const char *banner); +void fyp_debug_dump_simple_key(struct fy_parser *fyp, struct fy_simple_key *fysk, const char *banner); + +void fyp_debug_dump_input(struct fy_parser *fyp, const struct fy_input_cfg *fyic, + const char *banner); + +#else + +static inline void +fyp_debug_dump_token_list(struct fy_parser *fyp, struct fy_token_list *fytl, + struct fy_token *fyt_highlight, const char *banner) +{ + /* nothing */ +} + +static inline void +fyp_debug_dump_token(struct fy_parser *fyp, struct fy_token *fyt, const char *banner) +{ + /* nothing */ +} + +static inline void +fyp_debug_dump_simple_key_list(struct fy_parser *fyp, struct fy_simple_key_list *fyskl, + struct fy_simple_key *fysk_highlight, const char *banner) +{ + /* nothing */ +} + +static inline void +fyp_debug_dump_simple_key(struct fy_parser *fyp, struct fy_simple_key *fysk, const char *banner) +{ + /* nothing */ +} + +static inline void +fy_debug_dump_input(struct fy_parser *fyp, const struct fy_input_cfg *fyic, + const char *banner) +{ + /* nothing */ +} + +#endif + +#endif diff --git a/src/lib/fy-emit-accum.h b/src/lib/fy-emit-accum.h new file mode 100644 index 0000000..fd9bb67 --- /dev/null +++ b/src/lib/fy-emit-accum.h @@ -0,0 +1,383 @@ +/* + * fy-emit-accum.h - internal YAML emitter accumulator header + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_EMIT_ACCUM_H +#define FY_EMIT_ACCUM_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#include + +#include "fy-utf8.h" +#include "fy-event.h" + +struct fy_emit_accum { + char *accum; + size_t alloc; + size_t next; + char *inplace; + size_t inplacesz; + int col, row; + int ts; + enum fy_lb_mode lb_mode; +}; + +static inline void +fy_emit_accum_init(struct fy_emit_accum *ea, + void *inplace, size_t inplacesz, + int ts, enum fy_lb_mode lb_mode) +{ + memset(ea, 0, sizeof(*ea)); + + ea->inplace = inplace; + ea->inplacesz = inplacesz; + ea->accum = ea->inplace; + ea->alloc = ea->inplacesz; + ea->ts = ts ? ts : 8; + ea->lb_mode = lb_mode; +} + +static inline void +fy_emit_accum_reset(struct fy_emit_accum *ea) +{ + ea->next = 0; + ea->col = 0; + ea->row = 0; +} + +static inline void +fy_emit_accum_cleanup(struct fy_emit_accum *ea) +{ + if (ea->accum && ea->accum != ea->inplace) + free(ea->accum); + ea->accum = ea->inplace; + ea->alloc = ea->inplacesz; + fy_emit_accum_reset(ea); +} + +static inline void +fy_emit_accum_start(struct fy_emit_accum *ea, int col, enum fy_lb_mode lb_mode) +{ + fy_emit_accum_reset(ea); + ea->col = col; + ea->lb_mode = lb_mode; +} + +static inline void +fy_emit_accum_finish(struct fy_emit_accum *ea) +{ + fy_emit_accum_reset(ea); +} + +static inline int +fy_emit_accum_grow(struct fy_emit_accum *ea, size_t need) +{ + size_t atleast, asz; + char *new_accum; + + atleast = ea->alloc + need; + asz = ea->alloc; + /* minimum buffer is 32 */ + if (asz < 32) + asz = 32; + do { + asz *= 2; + } while (asz < atleast); + assert(asz > ea->inplacesz); + new_accum = realloc(ea->accum == ea->inplace ? NULL : ea->accum, asz); + if (!new_accum) /* out of memory */ + return -1; + if (ea->accum && ea->accum == ea->inplace) + memcpy(new_accum, ea->accum, ea->next); + ea->alloc = asz; + ea->accum = new_accum; + + return 0; +} + +static inline int +fy_emit_accum_utf8_put_raw(struct fy_emit_accum *ea, int c) +{ + size_t w, avail; + int ret; + + /* grow if needed */ + w = fy_utf8_width(c); + if (w > (avail = (ea->alloc - ea->next))) { + ret = fy_emit_accum_grow(ea, w - avail); + if (ret != 0) + return ret; + } + (void)fy_utf8_put_unchecked(ea->accum + ea->next, c); + ea->next += w; + + return 0; +} + +static inline int +fy_emit_accum_put_raw(struct fy_emit_accum *ea, int c) +{ + int ret; + + /* only lower ascii please */ + if (c >= 0x80) + return -1; + + /* grow if needed */ + if (ea->next >= ea->alloc) { + ret = fy_emit_accum_grow(ea, 1); + if (ret != 0) + return ret; + } + *(ea->accum + ea->next) = (char)c; + ea->next++; + + return 0; +} + +static inline int +fy_emit_accum_utf8_put(struct fy_emit_accum *ea, int c) +{ + int ret; + + if (!fy_utf8_is_valid(c)) + return -1; + + if (fy_is_lb_m(c, ea->lb_mode)) { + ret = fy_emit_accum_put_raw(ea, '\n'); + if (ret) + return ret; + ea->col = 0; + ea->row++; + } else if (fy_is_tab(c)) { + ret = fy_emit_accum_put_raw(ea, '\t'); + if (ret) + return ret; + ea->col += (ea->ts - (ea->col % ea->ts)); + } else { + if (c < 0x80) { + ret = fy_emit_accum_put_raw(ea, c); + if (ret) + return ret; + } else { + ret = fy_emit_accum_utf8_put_raw(ea, c); + } + ea->col++; + } + + return 0; +} + +static inline int +fy_emit_accum_utf8_write_raw(struct fy_emit_accum *ea, const void *data, size_t len) +{ + size_t avail; + int ret; + + /* grow if needed */ + if (len > (avail = (ea->alloc - ea->next))) { + ret = fy_emit_accum_grow(ea, len - avail); + if (ret != 0) + return ret; + } + memcpy(ea->accum + ea->next, data, len); + ea->next += len; + + return 0; +} + +static inline int +fy_emit_accum_utf8_write(struct fy_emit_accum *ea, const void *data, size_t len) +{ + const char *s, *e; + int c, w, ret; + + for (s = data, e = s + len; (c = fy_utf8_get(s, (e - s), &w)) >= 0; s += w) { + ret = fy_emit_accum_utf8_put(ea, c); + if (ret) + break; + } + return c == FYUG_EOF ? 0 : -1; +} + +static inline int +fy_emit_accum_utf8_printf_raw(struct fy_emit_accum *ea, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); + +static inline int +fy_emit_accum_utf8_printf_raw(struct fy_emit_accum *ea, const char *fmt, ...) +{ + va_list ap; + size_t avail, len; + int ret; + + /* get the size of the string */ + va_start(ap, fmt); + len = vsnprintf(NULL, 0, fmt, ap); + va_end(ap); + + /* grow if needed */ + if ((len + 1) > (avail = (ea->alloc - ea->next))) { + ret = fy_emit_accum_grow(ea, (len + 1) - avail); + if (ret != 0) + return ret; + } + + va_start(ap, fmt); + (void)vsnprintf(ea->accum + ea->next, len + 1, fmt, ap); + va_end(ap); + + ea->next += len; + + return 0; +} + +static inline const char * +fy_emit_accum_get(struct fy_emit_accum *ea, size_t *lenp) +{ + *lenp = ea->next; + if (!ea->next) { + return ""; + } + return ea->accum; +} + +static inline int +fy_emit_accum_make_0_terminated(struct fy_emit_accum *ea) +{ + int ret; + + /* the empty case is special cased */ + if (!ea->next) + return 0; + + /* grow if needed for the '\0' */ + if (ea->next >= ea->alloc) { + ret = fy_emit_accum_grow(ea, 1); + if (ret != 0) + return ret; + } + assert(ea->next < ea->alloc); + *(ea->accum + ea->next) = '\0'; + return 0; +} + +static inline const char * +fy_emit_accum_get0(struct fy_emit_accum *ea) +{ + int ret; + + ret = fy_emit_accum_make_0_terminated(ea); + if (ret) + return NULL; + return ea->accum; +} + +static inline char * +fy_emit_accum_steal(struct fy_emit_accum *ea, size_t *lenp) +{ + int ret; + char *buf; + + /* empty, return a malloc'ed buffer to "" */ + if (!ea->next) { + buf = strdup(""); + if (!buf) { + *lenp = 0; + return NULL; + } + *lenp = ea->next; + } else if (ea->inplace && ea->accum == ea->inplace) { + buf = malloc(ea->next + 1); + if (!buf) { + *lenp = 0; + return NULL; + } + memcpy(buf, ea->accum, ea->next); + buf[ea->next] = '\0'; + *lenp = ea->next; + } else { + ret = fy_emit_accum_make_0_terminated(ea); + if (ret) { + *lenp = 0; + return NULL; + } + assert(ea->accum && ea->accum != ea->inplace); + buf = ea->accum; + *lenp = ea->next; + /* reset to inplace */ + ea->accum = ea->inplace; + ea->alloc = ea->inplacesz; + } + + fy_emit_accum_cleanup(ea); + return buf; +} + +static inline char * +fy_emit_accum_steal0(struct fy_emit_accum *ea) +{ + size_t len; + + return fy_emit_accum_steal(ea, &len); +} + +static inline bool +fy_emit_accum_empty(struct fy_emit_accum *ea) +{ + return ea->next == 0; +} + +static inline int +fy_emit_accum_size(struct fy_emit_accum *ea) +{ + return ea->next; +} + +static inline int +fy_emit_accum_column(struct fy_emit_accum *ea) +{ + return ea->col; +} + +static inline int +fy_emit_accum_row(struct fy_emit_accum *ea) +{ + return ea->row; +} + +struct fy_emit_accum_state { + int col; + int row; + size_t next; +}; + +static inline void +fy_emit_accum_get_state(struct fy_emit_accum *ea, struct fy_emit_accum_state *s) +{ + s->col = ea->col; + s->row = ea->row; + s->next = ea->next; +} + +static inline void +fy_emit_accum_rewind_state(struct fy_emit_accum *ea, const struct fy_emit_accum_state *s) +{ + /* we can only go back */ + assert(s->next <= ea->next); + ea->col = s->col; + ea->row = s->row; + ea->next = s->next; +} + +#endif diff --git a/src/lib/fy-emit.c b/src/lib/fy-emit.c new file mode 100644 index 0000000..cf1ef91 --- /dev/null +++ b/src/lib/fy-emit.c @@ -0,0 +1,3983 @@ +/* + * fy-emit.c - Internal YAML emitter methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" +#include "fy-emit.h" + +static void fy_emitter_fill_default_colors(struct fy_emitter *fye); +static FILE *fy_emitter_get_output_fp(struct fy_emitter *fye); +static int fy_emitter_get_output_fd(struct fy_emitter *fye); +static int fy_emitter_null_output(struct fy_emitter *fye, enum fy_emitter_write_type type, const char *str, int len, void *userdata); + +void fy_emit_save_ctx_cleanup(struct fy_emitter *emit, struct fy_emit_save_ctx *sc); + +static inline struct fy_token_list *token_recycle_list(struct fy_emitter *emit, struct fy_parser *fyp) +{ + if (fyp && fyp->recycled_token_list) + return fyp->recycled_token_list; + if (emit && emit->recycled_token_list) + return emit->recycled_token_list; + return NULL; +} + +static inline void fy_emit_token_unref(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_token *fyt) +{ + fy_token_unref_rl(token_recycle_list(emit, fyp), fyt); +} + +/* fwd decl */ +void fy_emit_write(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int len); +void fy_emit_printf(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +static inline bool fy_emit_is_json_mode(const struct fy_emitter *emit) +{ + enum fy_emitter_cfg_flags flags; + + if (emit->force_json) + return true; + + flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + return flags == FYECF_MODE_JSON || flags == FYECF_MODE_JSON_TP || + flags == FYECF_MODE_JSON_ONELINE || flags == FYECF_MODE_JSON_COMPACT; +} + +static inline bool fy_emit_is_flow_mode(const struct fy_emitter *emit) +{ + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + + return flags == FYECF_MODE_FLOW || flags == FYECF_MODE_FLOW_ONELINE || + flags == FYECF_MODE_FLOW_COMPACT || fy_emit_is_json_mode(emit); +} + +static inline bool fy_emit_is_block_mode(const struct fy_emitter *emit) +{ + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + + return flags == FYECF_MODE_BLOCK || flags == FYECF_MODE_DEJSON || flags == FYECF_MODE_PRETTY; +} + +static inline bool fy_emit_is_oneline(const struct fy_emitter *emit) +{ + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + + return flags == FYECF_MODE_FLOW_ONELINE || flags == FYECF_MODE_JSON_ONELINE; +} + +static inline bool fy_emit_is_compact(const struct fy_emitter *emit) +{ + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + + return flags == FYECF_MODE_FLOW_COMPACT || flags == FYECF_MODE_JSON_COMPACT; +} + +static inline bool fy_emit_is_oneline_or_compact(const struct fy_emitter *emit) +{ + return fy_emit_is_oneline(emit) || fy_emit_is_compact(emit); +} + +static inline bool fy_emit_is_dejson_mode(const struct fy_emitter *emit) +{ + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + + return flags == FYECF_MODE_DEJSON; +} + +static inline bool fy_emit_is_pretty_mode(const struct fy_emitter *emit) +{ + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + + return flags == FYECF_MODE_PRETTY; +} + +static inline bool fy_emit_is_manual(const struct fy_emitter *emit) +{ + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + + return flags == FYECF_MODE_MANUAL; +} + +static inline int fy_emit_indent(struct fy_emitter *emit) +{ + int indent; + + indent = (emit->xcfg.cfg.flags & FYECF_INDENT(FYECF_INDENT_MASK)) >> FYECF_INDENT_SHIFT; + return indent ? indent : 2; +} + +static inline int fy_emit_width(struct fy_emitter *emit) +{ + int width; + + if (fy_emit_is_oneline(emit)) + return INT_MAX; + + width = (emit->xcfg.cfg.flags & FYECF_WIDTH(FYECF_WIDTH_MASK)) >> FYECF_WIDTH_SHIFT; + if (width == 0) + return 80; + if (width == FYECF_WIDTH_MASK) + return INT_MAX; + return width; +} + +static inline bool fy_emit_output_comments(struct fy_emitter *emit) +{ + return !!(emit->xcfg.cfg.flags & FYECF_OUTPUT_COMMENTS); +} + +static int fy_emit_node_check_json(struct fy_emitter *emit, struct fy_node *fyn) +{ + struct fy_document *fyd; + struct fy_node *fyni; + struct fy_node_pair *fynp, *fynpi; + int ret; + + if (!fyn) + return 0; + + fyd = fyn->fyd; + + switch (fyn->type) { + case FYNT_SCALAR: + FYD_TOKEN_ERROR_CHECK(fyd, fyn->scalar, FYEM_INTERNAL, + !fy_node_is_alias(fyn), err_out, + "aliases not allowed in JSON emit mode"); + break; + + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn->sequence); fyni; + fyni = fy_node_next(&fyn->sequence, fyni)) { + ret = fy_emit_node_check_json(emit, fyni); + if (ret) + return ret; + } + break; + + case FYNT_MAPPING: + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; fynp = fynpi) { + + fynpi = fy_node_pair_next(&fyn->mapping, fynp); + + ret = fy_emit_node_check_json(emit, fynp->key); + if (ret) + return ret; + ret = fy_emit_node_check_json(emit, fynp->value); + if (ret) + return ret; + } + break; + } + return 0; +err_out: + return -1; +} + +static int fy_emit_node_check(struct fy_emitter *emit, struct fy_node *fyn) +{ + int ret; + + if (!fyn) + return 0; + + if (fy_emit_is_json_mode(emit) && !emit->source_json) { + ret = fy_emit_node_check_json(emit, fyn); + if (ret) + return ret; + } + + return 0; +} + +void fy_emit_node_internal(struct fy_emitter *emit, struct fy_node *fyn, int flags, int indent, bool is_key); +void fy_emit_scalar(struct fy_emitter *emit, struct fy_node *fyn, int flags, int indent, bool is_key); +void fy_emit_sequence(struct fy_emitter *emit, struct fy_node *fyn, int flags, int indent); +void fy_emit_mapping(struct fy_emitter *emit, struct fy_node *fyn, int flags, int indent); + +void fy_emit_write(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int len); +void fy_emit_puts(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str); +void fy_emit_putc(struct fy_emitter *emit, enum fy_emitter_write_type type, int c); + +/* simple write, just ascii, advance column */ +void fy_emit_write_simple(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int len) +{ + int outlen; + + if (!len) + return; + + outlen = emit->xcfg.cfg.output(emit, type, str, len, emit->xcfg.cfg.userdata); + if (outlen != len) + emit->output_error = true; + + emit->column += len; +} + +/* simple puts as above */ +void fy_emit_puts_simple(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str) +{ + fy_emit_write_simple(emit, type, str, strlen(str)); +} + +void fy_emit_putc_simple(struct fy_emitter *emit, enum fy_emitter_write_type type, int c) +{ + char cc = (char)c; + + fy_emit_write_simple(emit, type, &cc, 1); + if (cc == '\n') { + emit->column = 0; + emit->line++; + } +} + +void fy_emit_write(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int len) +{ + int c, w; + const char *m, *e; + int outlen; + + if (!len) + return; + + outlen = emit->xcfg.cfg.output(emit, type, str, len, emit->xcfg.cfg.userdata); + if (outlen != len) + emit->output_error = true; + + e = str + len; + while ((c = fy_utf8_get(str, (e - str), &w)) >= 0) { + + /* special handling for MSDOS */ + if (c == '\r' && (e - str) > 1 && str[1] == '\n') { + str += 2; + emit->column = 0; + emit->line++; + continue; + } + + /* regular line break */ + if (fy_is_lb_r_n(c)) { + emit->column = 0; + emit->line++; + str += w; + continue; + } + + /* completely ignore ANSI color escape sequences */ + if (c == '\x1b' && (e - str) > 2 && str[1] == '[' && + (m = memchr(str, 'm', e - str)) != NULL) { + str = m + 1; + continue; + } + + emit->column++; + str += w; + } +} + +void fy_emit_puts(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str) +{ + fy_emit_write(emit, type, str, strlen(str)); +} + +void fy_emit_putc(struct fy_emitter *emit, enum fy_emitter_write_type type, int c) +{ + char buf[FY_UTF8_FORMAT_BUFMIN]; + + fy_utf8_format(c, buf, fyue_none); + fy_emit_puts(emit, type, buf); +} + +void fy_emit_vprintf(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *fmt, va_list ap) +{ + char *str; + int size; + va_list ap2; + + va_copy(ap2, ap); + + size = vsnprintf(NULL, 0, fmt, ap); + if (size < 0) + return; + + str = alloca(size + 1); + size = vsnprintf(str, size + 1, fmt, ap2); + if (size < 0) + return; + + fy_emit_write(emit, type, str, size); +} + +void fy_emit_printf(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_emit_vprintf(emit, type, fmt, ap); + va_end(ap); +} + +void fy_emit_write_ws(struct fy_emitter *emit) +{ + fy_emit_putc_simple(emit, fyewt_whitespace, ' '); + emit->flags |= FYEF_WHITESPACE; +} + +void fy_emit_write_indent(struct fy_emitter *emit, int indent) +{ + int len; + char *ws; + + indent = indent > 0 ? indent : 0; + + if (!fy_emit_indentation(emit) || emit->column > indent || + (emit->column == indent && !fy_emit_whitespace(emit))) + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + + if (emit->column < indent) { + len = indent - emit->column; + ws = alloca(len + 1); + memset(ws, ' ', len); + ws[len] = '\0'; + fy_emit_write_simple(emit, fyewt_indent, ws, len); + } + + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; +} + +enum document_indicator { + di_question_mark, + di_colon, + di_dash, + di_left_bracket, + di_right_bracket, + di_left_brace, + di_right_brace, + di_comma, + di_bar, + di_greater, + di_single_quote_start, + di_single_quote_end, + di_double_quote_start, + di_double_quote_end, + di_ambersand, + di_star, +}; + +void fy_emit_write_indicator(struct fy_emitter *emit, + enum document_indicator indicator, + int flags, int indent, + enum fy_emitter_write_type wtype) +{ + /* extended indicators mode? */ + if (wtype == fyewt_indicator && + (emit->xcfg.xflags & FYEXCF_EXTENDED_INDICATORS)) { + wtype = FYEWT_EXTENDED_INDICATORS_FIRST + (unsigned int)indicator; + assert(wtype >= FYEWT_EXTENDED_INDICATORS_FIRST && + wtype <= FYEWT_EXTENDED_INDICATORS_LAST); + } + + switch (indicator) { + + case di_question_mark: + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + fy_emit_putc_simple(emit, wtype, '?'); + emit->flags &= ~(FYEF_WHITESPACE | FYEF_OPEN_ENDED); + break; + + case di_colon: + fy_emit_putc_simple(emit, wtype, ':'); + emit->flags &= ~(FYEF_WHITESPACE | FYEF_OPEN_ENDED); + break; + + case di_dash: + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + fy_emit_putc_simple(emit, wtype, '-'); + emit->flags &= ~(FYEF_WHITESPACE | FYEF_OPEN_ENDED); + break; + + case di_left_bracket: + case di_left_brace: + emit->flow_level++; + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + fy_emit_putc_simple(emit, wtype, indicator == di_left_bracket ? '[' : '{'); + emit->flags |= FYEF_WHITESPACE; + emit->flags &= ~(FYEF_INDENTATION | FYEF_OPEN_ENDED); + break; + + case di_right_bracket: + case di_right_brace: + emit->flow_level--; + fy_emit_putc_simple(emit, wtype, indicator == di_right_bracket ? ']' : '}'); + emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION | FYEF_OPEN_ENDED); + break; + + case di_comma: + fy_emit_putc_simple(emit, wtype, ','); + emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION | FYEF_OPEN_ENDED); + break; + + case di_bar: + case di_greater: + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + fy_emit_putc_simple(emit, wtype, indicator == di_bar ? '|' : '>'); + emit->flags &= ~(FYEF_INDENTATION | FYEF_WHITESPACE | FYEF_OPEN_ENDED); + break; + + case di_single_quote_start: + case di_double_quote_start: + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + fy_emit_putc_simple(emit, wtype, indicator == di_single_quote_start ? '\'' : '"'); + emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION | FYEF_OPEN_ENDED); + break; + + case di_single_quote_end: + case di_double_quote_end: + fy_emit_putc_simple(emit, wtype, indicator == di_single_quote_end ? '\'' : '"'); + emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION | FYEF_OPEN_ENDED); + break; + + case di_ambersand: + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + fy_emit_putc_simple(emit, wtype, '&'); + emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION); + break; + + case di_star: + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + fy_emit_putc_simple(emit, wtype, '*'); + emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION); + break; + } +} + +int fy_emit_increase_indent(struct fy_emitter *emit, int flags, int indent) +{ + if (indent < 0) + return (flags & DDNF_FLOW) ? fy_emit_indent(emit) : 0; + + if (!(flags & DDNF_INDENTLESS)) + return indent + fy_emit_indent(emit); + + return indent; +} + +void fy_emit_write_comment(struct fy_emitter *emit, int flags, int indent, const char *str, size_t len, struct fy_atom *handle) +{ + const char *s, *e, *sr; + int c, w; + bool breaks; + + if (!str || !len) + return; + + if (len == (size_t)-1) + len = strlen(str); + + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + indent = emit->column; + + s = str; + e = str + len; + + sr = s; /* start of normal output run */ + breaks = false; + while (s < e && (c = fy_utf8_get(s, e - s, &w)) > 0) { + + if (fy_is_lb_m(c, fy_atom_lb_mode(handle))) { + + /* output run */ + fy_emit_write(emit, fyewt_comment, sr, s - sr); + sr = s + w; + fy_emit_write_indent(emit, indent); + emit->flags |= FYEF_INDENTATION; + breaks = true; + } else { + + if (breaks) { + fy_emit_write(emit, fyewt_comment, sr, s - sr); + sr = s; + fy_emit_write_indent(emit, indent); + } + emit->flags &= ~FYEF_INDENTATION; + breaks = false; + } + + s += w; + } + + /* dump what's remaining */ + fy_emit_write(emit, fyewt_comment, sr, s - sr); + + emit->flags |= (FYEF_WHITESPACE | FYEF_INDENTATION); +} + +struct fy_atom *fy_emit_token_comment_handle(struct fy_emitter *emit, struct fy_token *fyt, enum fy_comment_placement placement) +{ + struct fy_atom *handle; + + handle = fy_token_comment_handle(fyt, placement, false); + return handle && fy_atom_is_set(handle) ? handle : NULL; +} + +void fy_emit_document_start_indicator(struct fy_emitter *emit) +{ + /* do not emit twice */ + if (emit->flags & FYEF_HAD_DOCUMENT_START) + return; + + /* do not try to emit if it's json mode */ + if (fy_emit_is_json_mode(emit)) + goto no_doc_emit; + + /* output linebreak anyway */ + if (emit->column) + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + + /* ok, emit document start indicator */ + fy_emit_puts_simple(emit, fyewt_document_indicator, "---"); + emit->flags &= ~FYEF_WHITESPACE; + emit->flags |= FYEF_HAD_DOCUMENT_START; + return; + +no_doc_emit: + emit->flags &= ~FYEF_HAD_DOCUMENT_START; +} + +struct fy_token *fy_node_value_token(struct fy_node *fyn) +{ + struct fy_token *fyt; + + if (!fyn) + return NULL; + + switch (fyn->type) { + case FYNT_SCALAR: + fyt = fyn->scalar; + break; + case FYNT_SEQUENCE: + fyt = fyn->sequence_start; + break; + case FYNT_MAPPING: + fyt = fyn->mapping_start; + break; + default: + fyt = NULL; + break; + } + + return fyt; +} + +bool fy_emit_token_has_comment(struct fy_emitter *emit, struct fy_token *fyt, enum fy_comment_placement placement) +{ + return fy_emit_token_comment_handle(emit, fyt, placement) ? true : false; +} + +bool fy_emit_node_has_comment(struct fy_emitter *emit, struct fy_node *fyn, enum fy_comment_placement placement) +{ + return fy_emit_token_has_comment(emit, fy_node_value_token(fyn), placement); +} + +void fy_emit_token_comment(struct fy_emitter *emit, struct fy_token *fyt, int flags, int indent, + enum fy_comment_placement placement) +{ + struct fy_atom *handle; + char *text; + const char *t; + int len; + + handle = fy_emit_token_comment_handle(emit, fyt, placement); + if (!handle) + return; + + len = fy_atom_format_text_length(handle); + if (len < 0) + return; + + text = alloca(len + 1); + + if (placement == fycp_top || placement == fycp_bottom) { + fy_emit_write_indent(emit, indent); + emit->flags |= FYEF_WHITESPACE; + } + + t = fy_atom_format_text(handle, text, len + 1); + + fy_emit_write_comment(emit, flags, indent, t, len, handle); + + emit->flags &= ~FYEF_INDENTATION; + + if (placement == fycp_top || placement == fycp_bottom) { + fy_emit_write_indent(emit, indent); + emit->flags |= FYEF_WHITESPACE; + } +} + +void fy_emit_node_comment(struct fy_emitter *emit, struct fy_node *fyn, int flags, int indent, + enum fy_comment_placement placement) +{ + struct fy_token *fyt; + + if (!fy_emit_output_comments(emit) || (unsigned int)placement >= fycp_max) + return; + + fyt = fy_node_value_token(fyn); + if (!fyt) + return; + + fy_emit_token_comment(emit, fyt, flags, indent, placement); +} + +void fy_emit_common_node_preamble(struct fy_emitter *emit, + struct fy_token *fyt_value, + struct fy_token *fyt_anchor, + struct fy_token *fyt_tag, + int flags, int indent) +{ + const char *anchor = NULL; + const char *tag = NULL; + const char *td_prefix __FY_DEBUG_UNUSED__; + const char *td_handle; + size_t td_prefix_size, td_handle_size; + size_t tag_len = 0, anchor_len = 0; + bool json_mode = false; + struct fy_emit_save_ctx *sc = &emit->s_sc; + + json_mode = fy_emit_is_json_mode(emit); + + if (!json_mode) { + + if (fy_emit_token_has_comment(emit, fyt_value, fycp_top)) + fy_emit_token_comment(emit, fyt_value, sc->flags, sc->indent, fycp_top); + + if (!(emit->xcfg.cfg.flags & FYECF_STRIP_LABELS)) { + if (fyt_anchor) + anchor = fy_token_get_text(fyt_anchor, &anchor_len); + } + + if (!(emit->xcfg.cfg.flags & FYECF_STRIP_TAGS)) { + if (fyt_tag) + tag = fy_token_get_text(fyt_tag, &tag_len); + } + + if (anchor) { + fy_emit_write_indicator(emit, di_ambersand, flags, indent, fyewt_anchor); + fy_emit_write(emit, fyewt_anchor, anchor, anchor_len); + } + + if (tag) { + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + + td_handle = fy_tag_token_get_directive_handle(fyt_tag, &td_handle_size); + assert(td_handle); + td_prefix = fy_tag_token_get_directive_prefix(fyt_tag, &td_prefix_size); + assert(td_prefix); + + if (!td_handle_size) + fy_emit_printf(emit, fyewt_tag, "!<%.*s>", (int)tag_len, tag); + else + fy_emit_printf(emit, fyewt_tag, "%.*s%.*s", + (int)td_handle_size, td_handle, + (int)(tag_len - td_prefix_size), tag + td_prefix_size); + + emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION); + } + } + + /* content for root always starts on a new line */ + if ((flags & DDNF_ROOT) && emit->column != 0 && + !(emit->flags & FYEF_HAD_DOCUMENT_START)) { + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + } +} + +void fy_emit_node_internal(struct fy_emitter *emit, struct fy_node *fyn, int flags, int indent, bool is_key) +{ + enum fy_node_type type; + struct fy_anchor *fya; + struct fy_token *fyt_anchor = NULL; + struct fy_token *fyt_value = NULL; + + if (!(emit->xcfg.cfg.flags & FYECF_STRIP_LABELS)) { + fya = fy_document_lookup_anchor_by_node(emit->fyd, fyn); + if (fya) + fyt_anchor = fya->anchor; + } + + type = fyn ? fyn->type : FYNT_SCALAR; + + if (type == FYNT_SCALAR) + fyt_value = fyn ? fyn->scalar : NULL; + else if (type == FYNT_SEQUENCE) + fyt_value = fyn->sequence_start; + else if (type == FYNT_MAPPING) + fyt_value = fyn->mapping_start; + + fy_emit_common_node_preamble(emit, fyt_value, fyt_anchor, fyn->tag, flags, indent); + + if (type != FYNT_SCALAR && (flags & DDNF_ROOT) && emit->column != 0) { + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + } + + switch (type) { + case FYNT_SCALAR: + /* if we're pretty and root at column 0 (meaning it's a single scalar document) output --- */ + if ((flags & DDNF_ROOT) && fy_emit_is_pretty_mode(emit) && !emit->column && + !fy_emit_is_flow_mode(emit) && !(flags & DDNF_FLOW)) + fy_emit_document_start_indicator(emit); + fy_emit_scalar(emit, fyn, flags, indent, is_key); + break; + case FYNT_SEQUENCE: + FYD_TOKEN_ERROR_CHECK(fyn->fyd, fyn->sequence_start, FYEM_INTERNAL, + !is_key || !fy_emit_is_json_mode(emit), err_out, + "JSON does not allow sequences as keys"); + fy_emit_sequence(emit, fyn, flags, indent); + break; + case FYNT_MAPPING: + FYD_TOKEN_ERROR_CHECK(fyn->fyd, fyn->mapping_start, FYEM_INTERNAL, + !is_key || !fy_emit_is_json_mode(emit), err_out, + "JSON does not allow mappings as keys"); + fy_emit_mapping(emit, fyn, flags, indent); + break; + } +err_out: + /* nothing */ + return; +} + +void fy_emit_token_write_plain(struct fy_emitter *emit, struct fy_token *fyt, int flags, int indent) +{ + bool allow_breaks, should_indent, spaces, breaks; + int c; + enum fy_emitter_write_type wtype; + const char *str = NULL; + size_t len = 0; + struct fy_atom *atom; + struct fy_atom_iter iter; + + /* null and not json mode */ + if (!fyt && !fy_emit_is_json_mode(emit)) + goto out; + + wtype = (flags & DDNF_SIMPLE_SCALAR_KEY) ? fyewt_plain_scalar_key : fyewt_plain_scalar; + + atom = fy_token_atom(fyt); + + /* null and json mode */ + if (fy_emit_is_json_mode(emit) && fyt->scalar.is_null) { + fy_emit_puts_simple(emit, wtype, "null"); + goto out; + } + + allow_breaks = !(flags & DDNF_SIMPLE) && !fy_emit_is_json_mode(emit) && !fy_emit_is_oneline(emit); + + /* very very simple case first (90% of cases) */ + str = fy_token_get_direct_simple_output(fyt, &len); + if (str && (!allow_breaks || (fy_emit_accum_column(&emit->ea) + (int)len <= fy_emit_width(emit)))) { + fy_emit_write_simple(emit, wtype, str, len); + goto out; + } + + /* simple case first (90% of cases) */ + str = fy_token_get_direct_output(fyt, &len); + if (str && fy_token_atom_style(fyt) == FYAS_PLAIN) { + fy_emit_write(emit, wtype, str, len); + goto out; + } + + if (!atom) + goto out; + + spaces = false; + breaks = false; + + fy_atom_iter_start(atom, &iter); + fy_emit_accum_start(&emit->ea, emit->column, fy_token_atom_lb_mode(fyt)); + while ((c = fy_atom_iter_utf8_get(&iter)) > 0) { + + if (fy_is_ws(c)) { + + should_indent = allow_breaks && !spaces && + fy_emit_accum_column(&emit->ea) > fy_emit_width(emit); + + if (should_indent && !fy_is_ws(fy_atom_iter_utf8_peek(&iter))) { + fy_emit_output_accum(emit, wtype, &emit->ea); + emit->flags &= ~FYEF_INDENTATION; + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + } else + fy_emit_accum_utf8_put(&emit->ea, c); + + spaces = true; + + } else if (fy_is_lb_m(c, fy_token_atom_lb_mode(fyt))) { + + /* blergh */ + if (!allow_breaks) + break; + + /* output run */ + if (!breaks) { + fy_emit_output_accum(emit, wtype, &emit->ea); + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + } + + emit->flags &= ~FYEF_INDENTATION; + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + + breaks = true; + + } else { + + if (breaks) { + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + } + + fy_emit_accum_utf8_put(&emit->ea, c); + + emit->flags &= ~FYEF_INDENTATION; + + spaces = false; + breaks = false; + } + } + fy_emit_output_accum(emit, wtype, &emit->ea); + fy_emit_accum_finish(&emit->ea); + fy_atom_iter_finish(&iter); + +out: + emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION); +} + +void fy_emit_token_write_alias(struct fy_emitter *emit, struct fy_token *fyt, int flags, int indent) +{ + const char *str = NULL; + size_t len = 0; + struct fy_atom_iter iter; + int c; + + if (!fyt) + return; + + fy_emit_write_indicator(emit, di_star, flags, indent, fyewt_alias); + + /* try direct output first (99% of cases) */ + str = fy_token_get_direct_output(fyt, &len); + if (str) { + fy_emit_write(emit, fyewt_alias, str, len); + return; + } + + /* corner case, use iterator */ + fy_atom_iter_start(fy_token_atom(fyt), &iter); + fy_emit_accum_start(&emit->ea, emit->column, fy_token_atom_lb_mode(fyt)); + while ((c = fy_atom_iter_utf8_get(&iter)) > 0) + fy_emit_accum_utf8_put(&emit->ea, c); + fy_emit_output_accum(emit, fyewt_alias, &emit->ea); + fy_emit_accum_finish(&emit->ea); + fy_atom_iter_finish(&iter); +} + +void fy_emit_token_write_quoted(struct fy_emitter *emit, struct fy_token *fyt, int flags, int indent, char qc) +{ + const struct fy_token_analysis *ta; + bool allow_breaks, spaces, breaks; + int c, i, w, digit; + enum fy_emitter_write_type wtype; + const char *str = NULL; + size_t len = 0; + bool should_indent, done_esc; + struct fy_atom *atom; + struct fy_atom_iter iter; + enum fy_atom_style target_style; + uint32_t hi_surrogate, lo_surrogate; + uint8_t non_utf8[4]; + size_t non_utf8_len, k; + int emit_width; + + wtype = qc == '\'' ? + ((flags & DDNF_SIMPLE_SCALAR_KEY) ? + fyewt_single_quoted_scalar_key : fyewt_single_quoted_scalar) : + ((flags & DDNF_SIMPLE_SCALAR_KEY) ? + fyewt_double_quoted_scalar_key : fyewt_double_quoted_scalar); + + fy_emit_write_indicator(emit, + qc == '\'' ? di_single_quote_start : di_double_quote_start, + flags, indent, wtype); + + /* XXX check whether this is ever the case */ + if (!fyt) + goto out; + + /* note that if the original target style and the target differ + * we can note use direct output + */ + target_style = qc == '"' ? FYAS_DOUBLE_QUOTED : FYAS_SINGLE_QUOTED; + + allow_breaks = !(flags & DDNF_SIMPLE) && !fy_emit_is_json_mode(emit) && !fy_emit_is_oneline(emit); + emit_width = fy_emit_width(emit); + + ta = fy_token_text_analyze(fyt); + + /* simple case of direct output (large amount of cases) */ + str = fy_token_get_direct_output(fyt, &len); + if (str && fy_token_atom_style(fyt) == target_style && + (ta->flags & FYTTAF_DIRECT_OUTPUT) && emit->column + ta->maxcol < emit_width) { + fy_emit_write(emit, wtype, str, len); + goto out; + } + + /* no atom? i.e. empty */ + atom = fy_token_atom(fyt); + if (!atom) + goto out; + + spaces = false; + breaks = false; + + fy_atom_iter_start(atom, &iter); + fy_emit_accum_start(&emit->ea, emit->column, fy_token_atom_lb_mode(fyt)); + for (;;) { + non_utf8_len = sizeof(non_utf8); + c = fy_atom_iter_utf8_quoted_get(&iter, &non_utf8_len, non_utf8); + if (c < 0) + break; + + if (c == 0 && non_utf8_len > 0) { + for (k = 0; k < non_utf8_len; k++) { + c = (int)non_utf8[k] & 0xff; + fy_emit_accum_utf8_put(&emit->ea, '\\'); + fy_emit_accum_utf8_put(&emit->ea, 'x'); + digit = ((unsigned int)c >> 4) & 15; + fy_emit_accum_utf8_put(&emit->ea, + digit <= 9 ? ('0' + digit) : ('A' + digit - 10)); + digit = (unsigned int)c & 15; + fy_emit_accum_utf8_put(&emit->ea, + digit <= 9 ? ('0' + digit) : ('A' + digit - 10)); + } + continue; + } + + if (fy_is_space(c) || (qc == '\'' && fy_is_ws(c))) { + should_indent = allow_breaks && !spaces && + fy_emit_accum_column(&emit->ea) >= emit_width; + + if (should_indent && + ((qc == '\'' && fy_is_ws(fy_atom_iter_utf8_peek(&iter))) || + qc == '"')) { + fy_emit_output_accum(emit, wtype, &emit->ea); + + if (qc == '"' && fy_is_ws(fy_atom_iter_utf8_peek(&iter))) + fy_emit_putc_simple(emit, wtype, '\\'); + + emit->flags &= ~FYEF_INDENTATION; + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + } else + fy_emit_accum_utf8_put(&emit->ea, c); + + spaces = true; + breaks = false; + + } else if (qc == '\'' && fy_is_lb_m(c, fy_token_atom_lb_mode(fyt))) { + + /* blergh */ + if (!allow_breaks) + break; + + /* output run */ + if (!breaks) { + fy_emit_output_accum(emit, wtype, &emit->ea); + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + } + + emit->flags &= ~FYEF_INDENTATION; + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + + breaks = true; + } else { + /* output run */ + if (breaks) { + fy_emit_output_accum(emit, wtype, &emit->ea); + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + + } else if (qc == '"' && allow_breaks && fy_emit_accum_column(&emit->ea) >= emit_width) { + + fy_emit_output_accum(emit, wtype, &emit->ea); + + fy_emit_putc_simple(emit, wtype, '\\'); + + emit->flags &= ~FYEF_INDENTATION; + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + } + + /* escape */ + if (qc == '\'' && c == '\'') { + fy_emit_accum_utf8_put(&emit->ea, '\''); + fy_emit_accum_utf8_put(&emit->ea, '\''); + } else if (qc == '"' && + ((!fy_is_printq(c) || c == '"' || c == '\\') || + (fy_emit_is_json_mode(emit) && !fy_is_json_unescaped(c)))) { + + fy_emit_accum_utf8_put(&emit->ea, '\\'); + + /* common YAML and JSON escapes */ + done_esc = false; + switch (c) { + case '\b': + fy_emit_accum_utf8_put(&emit->ea, 'b'); + done_esc = true; + break; + case '\f': + fy_emit_accum_utf8_put(&emit->ea, 'f'); + done_esc = true; + break; + case '\n': + fy_emit_accum_utf8_put(&emit->ea, 'n'); + done_esc = true; + break; + case '\r': + fy_emit_accum_utf8_put(&emit->ea, 'r'); + done_esc = true; + break; + case '\t': + fy_emit_accum_utf8_put(&emit->ea, 't'); + done_esc = true; + break; + case '"': + fy_emit_accum_utf8_put(&emit->ea, '"'); + done_esc = true; + break; + case '\\': + fy_emit_accum_utf8_put(&emit->ea, '\\'); + done_esc = true; + break; + } + + if (done_esc) + goto done; + + if (!fy_emit_is_json_mode(emit)) { + switch (c) { + case '\0': + fy_emit_accum_utf8_put(&emit->ea, '0'); + break; + case '\a': + fy_emit_accum_utf8_put(&emit->ea, 'a'); + break; + case '\v': + fy_emit_accum_utf8_put(&emit->ea, 'v'); + break; + case '\e': + fy_emit_accum_utf8_put(&emit->ea, 'e'); + break; + case 0x85: + fy_emit_accum_utf8_put(&emit->ea, 'N'); + break; + case 0xa0: + fy_emit_accum_utf8_put(&emit->ea, '_'); + break; + case 0x2028: + fy_emit_accum_utf8_put(&emit->ea, 'L'); + break; + case 0x2029: + fy_emit_accum_utf8_put(&emit->ea, 'P'); + break; + default: + if ((unsigned int)c <= 0xff) { + fy_emit_accum_utf8_put(&emit->ea, 'x'); + w = 2; + } else if ((unsigned int)c <= 0xffff) { + fy_emit_accum_utf8_put(&emit->ea, 'u'); + w = 4; + } else if ((unsigned int)c <= 0xffffffff) { + fy_emit_accum_utf8_put(&emit->ea, 'U'); + w = 8; + } + + for (i = w - 1; i >= 0; i--) { + digit = ((unsigned int)c >> (i * 4)) & 15; + fy_emit_accum_utf8_put(&emit->ea, + digit <= 9 ? ('0' + digit) : ('A' + digit - 10)); + } + break; + } + + } else { + /* JSON escapes all others in \uXXXX and \uXXXX\uXXXX */ + w = 4; + if ((unsigned int)c <= 0xffff) { + fy_emit_accum_utf8_put(&emit->ea, 'u'); + for (i = w - 1; i >= 0; i--) { + digit = ((unsigned int)c >> (i * 4)) & 15; + fy_emit_accum_utf8_put(&emit->ea, + digit <= 9 ? ('0' + digit) : ('A' + digit - 10)); + } + } else { + hi_surrogate = 0xd800 | ((((c >> 16) & 0x1f) - 1) << 6) | ((c >> 10) & 0x3f); + lo_surrogate = 0xdc00 | (c & 0x3ff); + + fy_emit_accum_utf8_put(&emit->ea, 'u'); + for (i = w - 1; i >= 0; i--) { + digit = ((unsigned int)hi_surrogate >> (i * 4)) & 15; + fy_emit_accum_utf8_put(&emit->ea, + digit <= 9 ? ('0' + digit) : ('A' + digit - 10)); + } + + fy_emit_accum_utf8_put(&emit->ea, '\\'); + fy_emit_accum_utf8_put(&emit->ea, 'u'); + for (i = w - 1; i >= 0; i--) { + digit = ((unsigned int)lo_surrogate >> (i * 4)) & 15; + fy_emit_accum_utf8_put(&emit->ea, + digit <= 9 ? ('0' + digit) : ('A' + digit - 10)); + } + } + } + } else + fy_emit_accum_utf8_put(&emit->ea, c); +done: + emit->flags &= ~FYEF_INDENTATION; + spaces = false; + breaks = false; + } + } + fy_emit_output_accum(emit, wtype, &emit->ea); + fy_emit_accum_finish(&emit->ea); + fy_atom_iter_finish(&iter); + +out: + fy_emit_write_indicator(emit, + qc == '\'' ? di_single_quote_end : di_double_quote_end, + flags, indent, wtype); +} + +bool fy_emit_token_write_block_hints(struct fy_emitter *emit, struct fy_token *fyt, int flags, int indent, char *chompp) +{ + char chomp = '\0'; + bool explicit_chomp = false; + struct fy_atom *atom; + + atom = fy_token_atom(fyt); + if (!atom) { + emit->flags &= ~FYEF_OPEN_ENDED; + chomp = '-'; + goto out; + } + + if (atom->starts_with_ws || atom->starts_with_lb) { + fy_emit_putc_simple(emit, fyewt_indicator, '0' + fy_emit_indent(emit)); + explicit_chomp = true; + } + + if (!atom->ends_with_lb) { + emit->flags &= ~FYEF_OPEN_ENDED; + chomp = '-'; + goto out; + } + + if (atom->trailing_lb) { + emit->flags |= FYEF_OPEN_ENDED; + chomp = '+'; + goto out; + } + emit->flags &= ~FYEF_OPEN_ENDED; + +out: + if (chomp) + fy_emit_putc_simple(emit, fyewt_indicator, chomp); + *chompp = chomp; + return explicit_chomp; +} + +void fy_emit_token_write_literal(struct fy_emitter *emit, struct fy_token *fyt, int flags, int indent) +{ + bool breaks; + int c; + char chomp; + struct fy_atom *atom; + struct fy_atom_iter iter; + + fy_emit_write_indicator(emit, di_bar, flags, indent, fyewt_indicator); + + fy_emit_token_write_block_hints(emit, fyt, flags, indent, &chomp); + if (flags & DDNF_ROOT) + indent += fy_emit_indent(emit); + + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + + atom = fy_token_atom(fyt); + if (!atom) + goto out; + + breaks = true; + + fy_atom_iter_start(atom, &iter); + fy_emit_accum_start(&emit->ea, emit->column, fy_token_atom_lb_mode(fyt)); + while ((c = fy_atom_iter_utf8_get(&iter)) > 0) { + + if (breaks) { + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + breaks = false; + } + + if (fy_is_lb_m(c, fy_token_atom_lb_mode(fyt))) { + fy_emit_output_accum(emit, fyewt_literal_scalar, &emit->ea); + emit->flags &= ~FYEF_INDENTATION; + breaks = true; + } else + fy_emit_accum_utf8_put(&emit->ea, c); + } + fy_emit_output_accum(emit, fyewt_literal_scalar, &emit->ea); + fy_emit_accum_finish(&emit->ea); + fy_atom_iter_finish(&iter); + +out: + emit->flags &= ~FYEF_INDENTATION; +} + +void fy_emit_token_write_folded(struct fy_emitter *emit, struct fy_token *fyt, int flags, int indent) +{ + bool leading_spaces, breaks; + int c, nrbreaks, nrbreakslim; + char chomp; + struct fy_atom *atom; + struct fy_atom_iter iter; + + fy_emit_write_indicator(emit, di_greater, flags, indent, fyewt_indicator); + + fy_emit_token_write_block_hints(emit, fyt, flags, indent, &chomp); + if (flags & DDNF_ROOT) + indent += fy_emit_indent(emit); + + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + + atom = fy_token_atom(fyt); + if (!atom) + return; + + breaks = true; + leading_spaces = true; + + fy_atom_iter_start(atom, &iter); + fy_emit_accum_start(&emit->ea, emit->column, fy_token_atom_lb_mode(fyt)); + while ((c = fy_atom_iter_utf8_get(&iter)) > 0) { + + if (fy_is_lb_m(c, fy_token_atom_lb_mode(fyt))) { + + /* output run */ + if (!fy_emit_accum_empty(&emit->ea)) { + fy_emit_output_accum(emit, fyewt_literal_scalar, &emit->ea); + /* do not output a newline (indent) if at the end or + * this is a leading spaces line */ + if (!fy_is_z(fy_atom_iter_utf8_peek(&iter)) && !leading_spaces) { + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + } + } + + /* count the number of consecutive breaks */ + nrbreaks = 1; + while (fy_is_lb_m((c = fy_atom_iter_utf8_peek(&iter)), fy_token_atom_lb_mode(fyt))) { + nrbreaks++; + (void)fy_atom_iter_utf8_get(&iter); + } + + /* NOTE: Because the number of indents is tricky + * if it's a non blank, non end, it's the number of breaks + * if it's a blank, it's the number of breaks minus 1 + * if it's the end, it's the number of breaks minus 2 + */ + nrbreakslim = fy_is_z(c) ? 2 : fy_is_blank(c) ? 1 : 0; + while (nrbreaks-- > nrbreakslim) { + emit->flags &= ~FYEF_INDENTATION; + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + } + + breaks = true; + + } else { + + /* if we had a break, output an indent */ + if (breaks) { + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + + /* if this line starts with whitespace we need to know */ + leading_spaces = fy_is_ws(c); + } + + if (!breaks && fy_is_space(c) && + !fy_is_space(fy_atom_iter_utf8_peek(&iter)) && + fy_emit_accum_column(&emit->ea) > fy_emit_width(emit)) { + fy_emit_output_accum(emit, fyewt_folded_scalar, &emit->ea); + emit->flags &= ~FYEF_INDENTATION; + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + } else + fy_emit_accum_utf8_put(&emit->ea, c); + + breaks = false; + } + } + fy_emit_output_accum(emit, fyewt_folded_scalar, &emit->ea); + fy_emit_accum_finish(&emit->ea); + fy_atom_iter_finish(&iter); +} + +static enum fy_node_style +fy_emit_token_scalar_style(struct fy_emitter *emit, struct fy_token *fyt, + int flags, int indent, enum fy_node_style style, + struct fy_token *fyt_tag) +{ + const struct fy_token_analysis *ta = NULL; + bool json, flow, is_null_scalar, is_json_plain; + struct fy_atom *atom; + const char *tag; + size_t tag_len; + + atom = fy_token_atom(fyt); + + flow = fy_emit_is_flow_mode(emit) || (flags & DDNF_FLOW); + + /* check if style is allowed (i.e. no block styles in flow context) */ + if (flow && (style == FYNS_LITERAL || style == FYNS_FOLDED)) + style = FYNS_ANY; + + json = fy_emit_is_json_mode(emit); + + is_null_scalar = !atom || fyt->scalar.is_null; + + /* is this a plain json atom? */ + is_json_plain = (json || emit->source_json || fy_emit_is_dejson_mode(emit)) && + (is_null_scalar || + !fy_atom_strcmp(atom, "false") || + !fy_atom_strcmp(atom, "true") || + !fy_atom_strcmp(atom, "null") || + fy_atom_is_number(atom)); + + if (json) { + /* literal in JSON mode is output as quoted */ + if (style == FYNS_LITERAL || style == FYNS_FOLDED) + return FYNS_DOUBLE_QUOTED; + + if (is_json_plain) { + tag = fy_token_get_text(fyt_tag, &tag_len); + + /* XXX hardcoded string tag resultion */ + if (tag && tag_len && + ((tag_len == 1 && *tag == '!') || + (tag_len == 21 && !memcmp(tag, "tag:yaml.org,2002:str", 21)))) + return FYNS_DOUBLE_QUOTED; + } + + /* JSON NULL, but with plain style */ + if ((style == FYNS_PLAIN || style == FYNS_ANY) && + (is_null_scalar || (is_json_plain && !atom->size0))) + return FYNS_PLAIN; + + return FYNS_DOUBLE_QUOTED; + } + + ta = fy_token_text_analyze(fyt); + + if (flow && (style == FYNS_ANY || style == FYNS_LITERAL || style == FYNS_FOLDED)) { + + /* if there's a linebreak, or ends in a colon, use double quoted style */ + if (ta->flags & (FYTTAF_HAS_ANY_LB | FYTTAF_ENDS_WITH_COLON)) { + style = FYNS_DOUBLE_QUOTED; + goto out; + } + + /* anything not empty is double quoted here */ + style = !(ta->flags & FYTTAF_EMPTY) ? FYNS_PLAIN : FYNS_DOUBLE_QUOTED; + } + + /* try to pretify */ + if (!flow && fy_emit_is_pretty_mode(emit) && + (style == FYNS_ANY || style == FYNS_DOUBLE_QUOTED || style == FYNS_SINGLE_QUOTED)) { + + /* any original style can be a plain, but contains linebreaks, do a literal */ + if ((ta->flags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) == (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) { + style = FYNS_LITERAL; + goto out; + } + + /* any style, can be just a plain, just make it so */ + if ((ta->flags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) == FYTTAF_CAN_BE_PLAIN) { + style = FYNS_PLAIN; + goto out; + } + } + + if (!flow && emit->source_json && fy_emit_is_dejson_mode(emit)) { + if (is_json_plain || (ta->flags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) == FYTTAF_CAN_BE_PLAIN) { + style = FYNS_PLAIN; + goto out; + } + } + +out: + if (style == FYNS_ANY) + style = (ta->flags & FYTTAF_CAN_BE_PLAIN) ? FYNS_PLAIN : FYNS_DOUBLE_QUOTED; + + if (style == FYNS_PLAIN) { + /* plains in flow mode not being able to be plains + * - plain in block mode that can't be plain in flow mode + * - special handling for plains on start of line + */ + if ((flow && !(ta->flags & FYTTAF_CAN_BE_PLAIN_FLOW) && !(ta->flags & FYTTAF_CAN_BE_SIMPLE_KEY) && !is_null_scalar) || + ((ta->flags & FYTTAF_QUOTE_AT_0) && indent == 0)) + style = FYNS_DOUBLE_QUOTED; + + /* make it double quoted, but only if not already over the width (and contains a space/lb) */ + if (style == FYNS_PLAIN && (ta->flags & (FYTTAF_HAS_LB | FYTTAF_HAS_WS)) && + emit->column < fy_emit_width(emit) && (emit->column + ta->maxspan) > fy_emit_width(emit)) + style = FYNS_DOUBLE_QUOTED; + } + + return style; +} + +void fy_emit_token_scalar(struct fy_emitter *emit, struct fy_token *fyt, int flags, int indent, + enum fy_node_style style, struct fy_token *fyt_tag) +{ + assert(style != FYNS_FLOW && style != FYNS_BLOCK); + + indent = fy_emit_increase_indent(emit, flags, indent); + + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + + style = fy_emit_token_scalar_style(emit, fyt, flags, indent, style, fyt_tag); + + switch (style) { + case FYNS_ALIAS: + fy_emit_token_write_alias(emit, fyt, flags, indent); + break; + case FYNS_PLAIN: + fy_emit_token_write_plain(emit, fyt, flags, indent); + break; + case FYNS_DOUBLE_QUOTED: + fy_emit_token_write_quoted(emit, fyt, flags, indent, '"'); + break; + case FYNS_SINGLE_QUOTED: + fy_emit_token_write_quoted(emit, fyt, flags, indent, '\''); + break; + case FYNS_LITERAL: + fy_emit_token_write_literal(emit, fyt, flags, indent); + break; + case FYNS_FOLDED: + fy_emit_token_write_folded(emit, fyt, flags, indent); + break; + default: + break; + } +} + +void fy_emit_scalar(struct fy_emitter *emit, struct fy_node *fyn, int flags, int indent, bool is_key) +{ + enum fy_node_style style; + + /* default style */ + style = fyn ? fyn->style : FYNS_ANY; + + /* all JSON keys are double quoted */ + if (fy_emit_is_json_mode(emit) && is_key) + style = FYNS_DOUBLE_QUOTED; + + fy_emit_token_scalar(emit, + fyn ? fyn->scalar : NULL, + flags, indent, + style, fyn->tag); +} + +static void fy_emit_sequence_prolog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) +{ + bool json = fy_emit_is_json_mode(emit); + bool was_flow = sc->flow; + + sc->old_indent = sc->indent; + if (!json) { + if (fy_emit_is_manual(emit)) + sc->flow = (sc->xstyle == FYNS_BLOCK && was_flow) || sc->xstyle == FYNS_FLOW; + else if (fy_emit_is_block_mode(emit)) + sc->flow = sc->empty; + else + sc->flow = fy_emit_is_flow_mode(emit) || emit->flow_level || sc->flow_token || sc->empty; + } else + sc->flow = true; + + if (sc->flow) { + if (!emit->flow_level) { + sc->indent = fy_emit_increase_indent(emit, sc->flags, sc->indent); + sc->old_indent = sc->indent; + } + + sc->flags = (sc->flags | DDNF_FLOW) | (sc->flags & ~DDNF_INDENTLESS); + fy_emit_write_indicator(emit, di_left_bracket, sc->flags, sc->indent, fyewt_indicator); + + /* we need an indent afterward if not compact */ + if (!fy_emit_is_oneline_or_compact(emit)) + sc->flags |= DDNF_HANGING_INDENT; + } else { + sc->flags = (sc->flags & ~DDNF_FLOW); + } + + if (!fy_emit_is_oneline_or_compact(emit)) { + if (was_flow || (sc->flags & (DDNF_ROOT | DDNF_SEQ))) + sc->indent = fy_emit_increase_indent(emit, sc->flags, sc->indent); + } + + sc->flags &= ~DDNF_ROOT; +} + +static void fy_emit_sequence_epilog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) +{ + if (sc->flow && (sc->flags & DDNF_HANGING_INDENT) && !fy_emit_is_oneline_or_compact(emit) && !sc->empty) + fy_emit_write_indent(emit, sc->old_indent); + + if (sc->flow || fy_emit_is_json_mode(emit)) { + fy_emit_write_indicator(emit, di_right_bracket, sc->flags, sc->old_indent, fyewt_indicator); + } +} + +static void fy_emit_sequence_item_prolog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc, + struct fy_token *fyt_value) +{ + int tmp_indent; + + sc->flags |= DDNF_SEQ; + + if (fy_emit_token_has_comment(emit, fyt_value, fycp_top) || + !fy_emit_is_oneline_or_compact(emit) || + ((fy_emit_is_compact(emit) || sc->flow) && emit->column >= fy_emit_width(emit))) + fy_emit_write_indent(emit, sc->indent); + + if (!sc->flow && !fy_emit_is_json_mode(emit)) + fy_emit_write_indicator(emit, di_dash, sc->flags, sc->indent, fyewt_indicator); + + tmp_indent = sc->indent; + if (fy_emit_token_has_comment(emit, fyt_value, fycp_top)) { + if (!sc->flow && !fy_emit_is_json_mode(emit)) + tmp_indent = fy_emit_increase_indent(emit, sc->flags, sc->indent); + fy_emit_token_comment(emit, fyt_value, sc->flags, tmp_indent, fycp_top); + } +} + +static void fy_emit_sequence_item_epilog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc, + bool last, struct fy_token *fyt_value) +{ + bool needs_hanging_indent; + + if ((sc->flow || fy_emit_is_json_mode(emit)) && !last) + fy_emit_write_indicator(emit, di_comma, sc->flags, sc->indent, fyewt_indicator); + + fy_emit_token_comment(emit, fyt_value, sc->flags, sc->indent, fycp_right); + + needs_hanging_indent = sc->flow && !fy_emit_is_oneline_or_compact(emit) && !sc->empty; + + if (last && needs_hanging_indent && (sc->flags & DDNF_HANGING_INDENT)) + fy_emit_write_indent(emit, sc->old_indent); + else if (needs_hanging_indent) + sc->flags |= DDNF_HANGING_INDENT; + + sc->flags &= ~DDNF_SEQ; +} + +void fy_emit_sequence(struct fy_emitter *emit, struct fy_node *fyn, int flags, int indent) +{ + struct fy_node *fyni, *fynin; + struct fy_token *fyt_value; + bool last; + struct fy_emit_save_ctx sct, *sc = &sct; + + memset(sc, 0, sizeof(*sc)); + + sc->flags = flags; + sc->indent = indent; + sc->empty = fy_node_list_empty(&fyn->sequence); + sc->flow_token = fyn->style == FYNS_FLOW; + sc->flow = !!(flags & DDNF_FLOW); + sc->xstyle = fyn->style; + sc->old_indent = sc->indent; + + fy_emit_sequence_prolog(emit, sc); + + for (fyni = fy_node_list_head(&fyn->sequence); fyni; fyni = fynin) { + + fynin = fy_node_next(&fyn->sequence, fyni); + last = !fynin; + fyt_value = fy_node_value_token(fyni); + + fy_emit_sequence_item_prolog(emit, sc, fyt_value); + fy_emit_node_internal(emit, fyni, (sc->flags & ~DDNF_ROOT), sc->indent, false); + fy_emit_sequence_item_epilog(emit, sc, last, fyt_value); + } + + fy_emit_sequence_epilog(emit, sc); +} + +static void fy_emit_mapping_prolog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) +{ + bool json = fy_emit_is_json_mode(emit); + bool was_flow = sc->flow; + + sc->old_indent = sc->indent; + if (!json) { + if (fy_emit_is_manual(emit)) + sc->flow = (sc->xstyle == FYNS_BLOCK && was_flow) || sc->xstyle == FYNS_FLOW; + else if (fy_emit_is_block_mode(emit)) + sc->flow = sc->empty; + else + sc->flow = fy_emit_is_flow_mode(emit) || emit->flow_level || sc->flow_token || sc->empty; + } else + sc->flow = true; + + if (sc->flow) { + if (!emit->flow_level) { + sc->indent = fy_emit_increase_indent(emit, sc->flags, sc->indent); + sc->old_indent = sc->indent; + } + + sc->flags = (sc->flags | DDNF_FLOW) | (sc->flags & ~DDNF_INDENTLESS); + fy_emit_write_indicator(emit, di_left_brace, sc->flags, sc->indent, fyewt_indicator); + + /* we need an indent afterward if not compact */ + if (!fy_emit_is_oneline_or_compact(emit)) + sc->flags |= DDNF_HANGING_INDENT; + } else { + sc->flags &= ~(DDNF_FLOW | DDNF_INDENTLESS); + } + + if (!fy_emit_is_oneline_or_compact(emit) && !sc->empty) + sc->indent = fy_emit_increase_indent(emit, sc->flags, sc->indent); + + sc->flags &= ~DDNF_ROOT; +} + +static void fy_emit_mapping_epilog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) +{ + if (sc->flow || fy_emit_is_json_mode(emit)) { + if ((sc->flags & DDNF_HANGING_INDENT) && !fy_emit_is_oneline_or_compact(emit) && !sc->empty) + fy_emit_write_indent(emit, sc->old_indent); + fy_emit_write_indicator(emit, di_right_brace, sc->flags, sc->old_indent, fyewt_indicator); + } +} + +static void fy_emit_mapping_key_prolog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc, + struct fy_token *fyt_key, bool simple_key) +{ + const struct fy_token_analysis *ta; + bool do_indent, key_over; + + sc->flags = DDNF_MAP | (sc->flags & DDNF_FLOW); + + if (!fy_emit_is_oneline_or_compact(emit) || + ((fy_emit_is_compact(emit) || sc->flow) && emit->column >= fy_emit_width(emit))) + fy_emit_write_indent(emit, sc->indent); + + if (simple_key) { + sc->flags |= DDNF_SIMPLE; + if (fyt_key && fyt_key->type == FYTT_SCALAR) + sc->flags |= DDNF_SIMPLE_SCALAR_KEY; + } else { + /* do not emit the ? in flow modes at all */ + if (fy_emit_is_flow_mode(emit)) + sc->flags |= DDNF_SIMPLE; + } + + do_indent = false; + if (!fy_emit_is_oneline_or_compact(emit)) { + if (fyt_key && fyt_key->type != FYTT_SCALAR) + /* always indent on non scalar keys */ + key_over = true; + else { + /* for scalar keys, check if key + 2 + quotes go over */ + ta = fy_token_text_analyze(fyt_key); + key_over = (emit->column + ta->maxspan + 2 + (!simple_key ? 2 : 0)) >= fy_emit_width(emit); + } + + do_indent = key_over; + if (!do_indent && !fy_emit_is_compact(emit)) + do_indent = !sc->flow || (sc->flags & DDNF_HANGING_INDENT); + } else + do_indent = false; + + if (do_indent) + fy_emit_write_indent(emit, sc->indent); + + /* complex? */ + if (!(sc->flags & DDNF_SIMPLE)) + fy_emit_write_indicator(emit, di_question_mark, sc->flags, sc->indent, fyewt_indicator); +} + +static void fy_emit_mapping_key_epilog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc, + struct fy_token *fyt_key) +{ + int tmp_indent; + + /* if the key is an alias, always output an extra whitespace */ + if (fyt_key && fyt_key->type == FYTT_ALIAS) + fy_emit_write_ws(emit); + + sc->flags &= ~DDNF_MAP; + + if (!(sc->flags & DDNF_SIMPLE)) { + if (!emit->flow_level && !fy_emit_is_oneline_or_compact(emit)) + fy_emit_write_indent(emit, sc->indent); + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + } + fy_emit_write_indicator(emit, di_colon, sc->flags, sc->indent, fyewt_indicator); + + tmp_indent = sc->indent; + if (fy_emit_token_has_comment(emit, fyt_key, fycp_right)) { + + if (!sc->flow && !fy_emit_is_json_mode(emit)) + tmp_indent = fy_emit_increase_indent(emit, sc->flags, sc->indent); + + fy_emit_token_comment(emit, fyt_key, sc->flags, tmp_indent, fycp_right); + fy_emit_write_indent(emit, tmp_indent); + } + + sc->flags = DDNF_MAP | (sc->flags & DDNF_FLOW); +} + +static void fy_emit_mapping_value_prolog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc, + struct fy_token *fyt_value) +{ + const struct fy_token_analysis *ta; + bool value_over; + + /* don't do anything for those cases */ + if (!sc->flow || fy_emit_is_oneline_or_compact(emit) || !fyt_value || fyt_value->type != FYTT_SCALAR) + return; + + ta = fy_token_text_analyze(fyt_value); + value_over = (emit->column + ta->maxspan) >= fy_emit_width(emit); + + if (value_over) { + sc->indent = fy_emit_increase_indent(emit, sc->flags, sc->indent); + fy_emit_write_indent(emit, sc->indent); + } +} + +static void fy_emit_mapping_value_epilog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc, + bool last, struct fy_token *fyt_value) +{ + bool needs_hanging_indent; + + if ((sc->flow || fy_emit_is_json_mode(emit)) && !last) + fy_emit_write_indicator(emit, di_comma, sc->flags, sc->indent, fyewt_indicator); + + fy_emit_token_comment(emit, fyt_value, sc->flags, sc->indent, fycp_right); + + needs_hanging_indent = sc->flow && !fy_emit_is_oneline_or_compact(emit) && !sc->empty; + + if (last && needs_hanging_indent && (sc->flags & DDNF_HANGING_INDENT)) + fy_emit_write_indent(emit, sc->old_indent); + else if (needs_hanging_indent) + sc->flags |= DDNF_HANGING_INDENT; + + sc->flags &= ~DDNF_MAP; +} + +void fy_emit_mapping(struct fy_emitter *emit, struct fy_node *fyn, int flags, int indent) +{ + struct fy_node_pair *fynp, *fynpn, **fynpp = NULL; + struct fy_token *fyt_key, *fyt_value; + bool last, simple_key, used_malloc = false; + const struct fy_token_analysis *tak = NULL, *tav; + int i, count; + struct fy_emit_save_ctx sct, *sc = &sct; + + memset(sc, 0, sizeof(*sc)); + + sc->flags = flags; + sc->indent = indent; + sc->empty = fy_node_pair_list_empty(&fyn->mapping); + sc->flow_token = fyn->style == FYNS_FLOW; + sc->flow = !!(flags & DDNF_FLOW); + sc->xstyle = fyn->style; + sc->old_indent = sc->indent; + + fy_emit_mapping_prolog(emit, sc); + + if (!(emit->xcfg.cfg.flags & (FYECF_SORT_KEYS | FYECF_STRIP_EMPTY_KV))) { + fynp = fy_node_pair_list_head(&fyn->mapping); + fynpp = NULL; + } else { + count = fy_node_mapping_item_count(fyn); + + /* heuristic, avoid allocation for small maps */ + if (count > 64) { + fynpp = malloc((count + 1) * sizeof(*fynpp)); + fyd_error_check(fyn->fyd, fynpp, err_out, + "malloc() failed"); + used_malloc = true; + } else + fynpp = alloca((count + 1) * sizeof(*fynpp)); + + /* fill (removing empty KVs) */ + i = 0; + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; + fynp = fy_node_pair_next(&fyn->mapping, fynp)) { + + /* strip key/value pair from the output if it's empty */ + if ((emit->xcfg.cfg.flags & FYECF_STRIP_EMPTY_KV) && fy_node_is_empty(fynp->value)) + continue; + + fynpp[i++] = fynp; + } + count = i; + fynpp[count] = NULL; + + /* sort the keys */ + if (emit->xcfg.cfg.flags & FYECF_SORT_KEYS) + fy_node_mapping_perform_sort(fyn, NULL, NULL, fynpp, count); + + i = 0; + fynp = fynpp[i]; + } + + for (; fynp; fynp = fynpn) { + + if (!fynpp) + fynpn = fy_node_pair_next(&fyn->mapping, fynp); + else + fynpn = fynpp[++i]; + + last = !fynpn; + fyt_key = fy_node_value_token(fynp->key); + fyt_value = fy_node_value_token(fynp->value); + + FYD_NODE_ERROR_CHECK(fynp->fyd, fynp->key, FYEM_INTERNAL, + !fy_emit_is_json_mode(emit) || + (fynp->key && fynp->key->type == FYNT_SCALAR), + err_out, "Non scalar keys are not allowed in JSON emit mode"); + + simple_key = false; + if (fynp->key) { + switch (fynp->key->type) { + case FYNT_SCALAR: + tak = fy_token_text_analyze(fynp->key->scalar); + simple_key = fy_emit_is_json_mode(emit) || + !!(tak->flags & FYTTAF_CAN_BE_SIMPLE_KEY); + break; + case FYNT_SEQUENCE: + simple_key = fy_node_list_empty(&fynp->key->sequence); + break; + case FYNT_MAPPING: + simple_key = fy_node_pair_list_empty(&fynp->key->mapping); + break; + } + } + + /* special handling for compact mode simple key + scalar value*/ + if (fy_emit_is_compact(emit) && simple_key && + fynp->value && fynp->value->type == FYNT_SCALAR && tak) { + tav = fy_token_text_analyze(fynp->value->scalar); + + /* for plain key + value if we go over the width */ + if ((tav->flags & FYTTAF_CAN_BE_PLAIN) && + (emit->column + tak->maxspan + 2 + tav->maxspan + 1) >= fy_emit_width(emit)) + fy_emit_write_indent(emit, sc->indent); + } + + fy_emit_mapping_key_prolog(emit, sc, fyt_key, simple_key); + if (fynp->key) + fy_emit_node_internal(emit, fynp->key, (sc->flags & ~DDNF_ROOT), sc->indent, true); + fy_emit_mapping_key_epilog(emit, sc, fyt_key); + + fy_emit_mapping_value_prolog(emit, sc, fyt_value); + if (fynp->value) + fy_emit_node_internal(emit, fynp->value, (sc->flags & ~DDNF_ROOT), sc->indent, false); + fy_emit_mapping_value_epilog(emit, sc, last, fyt_value); + } + + if (fynpp && used_malloc) + free(fynpp); + + fy_emit_mapping_epilog(emit, sc); + +err_out: + return; +} + +int fy_emit_common_document_start(struct fy_emitter *emit, + struct fy_document_state *fyds, + bool root_tag_or_anchor) +{ + struct fy_token *fyt_chk; + const char *td_handle, *td_prefix; + size_t td_handle_size, td_prefix_size; + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags; + enum fy_emitter_cfg_flags vd_flags = flags & FYECF_VERSION_DIR(FYECF_VERSION_DIR_MASK); + enum fy_emitter_cfg_flags td_flags = flags & FYECF_TAG_DIR(FYECF_TAG_DIR_MASK); + enum fy_emitter_cfg_flags dsm_flags = flags & FYECF_DOC_START_MARK(FYECF_DOC_START_MARK_MASK); + bool vd, td, dsm; + bool had_non_default_tag = false; + bool strip_doc = false; + + if (!emit || !fyds || emit->fyds) + return -1; + + strip_doc = !!(emit->xcfg.cfg.flags & FYECF_STRIP_DOC); + + emit->fyds = fy_document_state_ref(fyds); + + vd = ((vd_flags == FYECF_VERSION_DIR_AUTO && fyds->version_explicit) || + vd_flags == FYECF_VERSION_DIR_ON) && + !strip_doc; + td = ((td_flags == FYECF_TAG_DIR_AUTO && fyds->tags_explicit) || + td_flags == FYECF_TAG_DIR_ON) && + !strip_doc; + + /* if either a version or directive tags exist, and no previous + * explicit document end existed, output one now + */ + if (!fy_emit_is_json_mode(emit) && (vd || td) && !(emit->flags & FYEF_HAD_DOCUMENT_END)) { + if (emit->column) + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + if (!strip_doc) { + fy_emit_puts_simple(emit, fyewt_document_indicator, "..."); + emit->flags &= ~FYEF_WHITESPACE; + emit->flags |= FYEF_HAD_DOCUMENT_END; + } + } + + if (!fy_emit_is_json_mode(emit) && vd) { + if (emit->column) + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + fy_emit_printf(emit, fyewt_version_directive, "%%YAML %d.%d", + fyds->version.major, fyds->version.minor); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + } + + if (!fy_emit_is_json_mode(emit) && td) { + + for (fyt_chk = fy_token_list_first(&fyds->fyt_td); fyt_chk; fyt_chk = fy_token_next(&fyds->fyt_td, fyt_chk)) { + + td_handle = fy_tag_directive_token_handle(fyt_chk, &td_handle_size); + td_prefix = fy_tag_directive_token_prefix(fyt_chk, &td_prefix_size); + assert(td_handle && td_prefix); + + if (fy_tag_is_default_internal(td_handle, td_handle_size, td_prefix, td_prefix_size)) + continue; + + had_non_default_tag = true; + + if (emit->column) + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + fy_emit_printf(emit, fyewt_tag_directive, "%%TAG %.*s %.*s", + (int)td_handle_size, td_handle, + (int)td_prefix_size, td_prefix); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + } + } + + /* always output document start indicator: + * - was explicit + * - document has tags + * - document has an explicit version + * - root exists & has a tag or an anchor + */ + dsm = (dsm_flags == FYECF_DOC_START_MARK_AUTO && + (!fyds->start_implicit || + (fyds->tags_explicit && had_non_default_tag) || + fyds->version_explicit )) || + dsm_flags == FYECF_DOC_START_MARK_ON; + + /* if there was previous output without document end */ + if (!dsm && (emit->flags & FYEF_HAD_DOCUMENT_OUTPUT) && + !(emit->flags & FYEF_HAD_DOCUMENT_END)) + dsm = true; + + /* there was *any* document end output (and no document end) */ + if (!dsm && (emit->flags & FYEF_HAD_DOCUMENT_END_OUTPUT) && + !(emit->flags & FYEF_HAD_DOCUMENT_END)) + dsm = true; + + /* output document start indicator if we should */ + if (dsm && !fy_emit_is_json_mode(emit)) + fy_emit_document_start_indicator(emit); + + /* clear that in any case */ + emit->flags &= ~FYEF_HAD_DOCUMENT_END; + + return 0; +} + +int fy_emit_document_start(struct fy_emitter *emit, struct fy_document *fyd, + struct fy_node *fyn_root) +{ + struct fy_node *root; + bool root_tag_or_anchor; + int ret; + + if (!emit || !fyd || !fyd->fyds) + return -1; + + root = fyn_root ? : fy_document_root(fyd); + + root_tag_or_anchor = root && (root->tag || fy_document_lookup_anchor_by_node(fyd, root)); + + ret = fy_emit_common_document_start(emit, fyd->fyds, root_tag_or_anchor); + if (ret) + return ret; + + emit->fyd = fyd; + + return 0; +} + +int fy_emit_common_document_end(struct fy_emitter *emit, bool override_state, bool implicit_override) +{ + const struct fy_document_state *fyds; + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags; + enum fy_emitter_cfg_flags dem_flags = flags & FYECF_DOC_END_MARK(FYECF_DOC_END_MARK_MASK); + bool implicit, dem; + + if (!emit || !emit->fyds) + return -1; + + fyds = emit->fyds; + + implicit = fyds->end_implicit; + if (fyds->started_explicit) + implicit = false; + else if (override_state) + implicit = implicit_override; + + dem = ((dem_flags == FYECF_DOC_END_MARK_AUTO && !implicit) || + dem_flags == FYECF_DOC_END_MARK_ON) && + !(emit->xcfg.cfg.flags & FYECF_STRIP_DOC); + + if (!(emit->xcfg.cfg.flags & FYECF_NO_ENDING_NEWLINE)) { + if (emit->column != 0) { + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + } + + if (!fy_emit_is_json_mode(emit) && dem) { + fy_emit_puts_simple(emit, fyewt_document_indicator, "..."); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + emit->flags |= FYEF_HAD_DOCUMENT_END; + } else + emit->flags &= ~FYEF_HAD_DOCUMENT_END; + } else { + if (!fy_emit_is_json_mode(emit) && dem) { + if (emit->column != 0) { + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + } + fy_emit_puts_simple(emit, fyewt_document_indicator, "..."); + emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION); + emit->flags |= FYEF_HAD_DOCUMENT_END; + } else + emit->flags &= ~FYEF_HAD_DOCUMENT_END; + } + + /* mark that we did output a document earlier */ + emit->flags |= FYEF_HAD_DOCUMENT_OUTPUT; + + /* stop our association with the document */ + fy_document_state_unref(emit->fyds); + emit->fyds = NULL; + + return 0; +} + +int fy_emit_document_end(struct fy_emitter *emit) +{ + int ret; + + ret = fy_emit_common_document_end(emit, false, false); + if (ret) + return ret; + + emit->fyd = NULL; + return 0; +} + +int fy_emit_common_explicit_document_end(struct fy_emitter *emit) +{ + if (!emit) + return -1; + + if (emit->column != 0) { + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + } + + if (!fy_emit_is_json_mode(emit)) { + fy_emit_puts_simple(emit, fyewt_document_indicator, "..."); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + emit->flags |= FYEF_HAD_DOCUMENT_END; + } else + emit->flags &= ~FYEF_HAD_DOCUMENT_END; + + /* mark that we did output a document earlier */ + emit->flags |= FYEF_HAD_DOCUMENT_OUTPUT; + + /* stop our association with the document */ + fy_document_state_unref(emit->fyds); + emit->fyds = NULL; + + return 0; +} + +int fy_emit_explicit_document_end(struct fy_emitter *emit) +{ + int ret; + + ret = fy_emit_common_explicit_document_end(emit); + if (ret) + return ret; + + emit->fyd = NULL; + return 0; +} + +void fy_emit_reset(struct fy_emitter *emit, bool reset_events) +{ + struct fy_eventp *fyep; + + emit->line = 0; + emit->column = 0; + emit->flow_level = 0; + emit->output_error = 0; + /* clear/set flags for a fresh start */ + emit->flags &= ~(FYEF_OPEN_ENDED | FYEF_HAD_DOCUMENT_START | FYEF_HAD_DOCUMENT_OUTPUT); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + + emit->state = FYES_NONE; + + /* reset the accumulator */ + fy_emit_accum_reset(&emit->ea); + + /* streaming mode indent */ + emit->s_indent = -1; + /* streaming mode flags */ + emit->s_flags = DDNF_ROOT; + + fy_emit_save_ctx_cleanup(emit, &emit->s_sc); + while (emit->sc_stack_top > 0) + fy_emit_save_ctx_cleanup(emit, &emit->sc_stack[--emit->sc_stack_top]); + + emit->state_stack_top = 0; + + /* and release any queued events */ + if (reset_events) { + while ((fyep = fy_eventp_list_pop(&emit->queued_events)) != NULL) + fy_eventp_release(fyep); + } +} + +int fy_emit_setup(struct fy_emitter *emit, const struct fy_emitter_cfg *cfg) +{ + struct fy_diag *diag; + + if (!cfg) + return -1; + + memset(emit, 0, sizeof(*emit)); + emit->output_fd = -1; + + /* is it extended config or not? */ + if (!(cfg->flags & FYECF_EXTENDED_CFG)) + emit->xcfg.cfg = *cfg; + else + emit->xcfg = *container_of(cfg, struct fy_emitter_xcfg, cfg); + + fy_emitter_fill_default_colors(emit); + emit->output_fp = fy_emitter_get_output_fp(emit); + if (!emit->output_fp) + emit->output_fd = fy_emitter_get_output_fd(emit); + + if (!emit->xcfg.cfg.output) { + if (!emit->output_fp && emit->output_fd < 0) { + emit->xcfg.cfg.output = fy_emitter_null_output; + } else { + emit->xcfg.cfg.output = fy_emitter_default_output; + /* for the default, turn on extended indicators */ + emit->xcfg.xflags |= FYEXCF_EXTENDED_INDICATORS; + + switch (emit->xcfg.xflags & (FYEXCF_COLOR_MASK << FYEXCF_COLOR_SHIFT)) { + case FYEXCF_COLOR_AUTO: + emit->output_colorize = emit->output_fp ? + isatty(fileno(emit->output_fp)) : + isatty(emit->output_fd); + break; + case FYEXCF_COLOR_FORCE: + emit->output_colorize = true; + break; + default: + emit->output_colorize = false; + break; + } + } + } + + diag = cfg->diag; + + if (!diag) { + diag = fy_diag_create(NULL); + if (!diag) + return -1; + } else + fy_diag_ref(diag); + + emit->diag = diag; + + fy_emit_accum_init(&emit->ea, emit->ea_inplace_buf, sizeof(emit->ea_inplace_buf), 0, fylb_cr_nl); + fy_eventp_list_init(&emit->queued_events); + + emit->state_stack = emit->state_stack_inplace; + emit->state_stack_alloc = sizeof(emit->state_stack_inplace)/sizeof(emit->state_stack_inplace[0]); + + emit->sc_stack = emit->sc_stack_inplace; + emit->sc_stack_alloc = sizeof(emit->sc_stack_inplace)/sizeof(emit->sc_stack_inplace[0]); + + fy_eventp_list_init(&emit->recycled_eventp); + fy_token_list_init(&emit->recycled_token); + + /* suppress recycling if we must */ + emit->suppress_recycling_force = getenv("FY_VALGRIND") && !getenv("FY_VALGRIND_RECYCLING"); + emit->suppress_recycling = emit->suppress_recycling_force; + + if (!emit->suppress_recycling) { + emit->recycled_eventp_list = &emit->recycled_eventp; + emit->recycled_token_list = &emit->recycled_token; + } else { + emit->recycled_eventp_list = NULL; + emit->recycled_token_list = NULL; + } + + fy_emit_reset(emit, false); + + return 0; +} + +void fy_emit_save_ctx_cleanup(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) +{ + sc = &emit->s_sc; + fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_key); + sc->fyt_last_key = NULL; + fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_value); + sc->fyt_last_value = NULL; + memset(sc, 0, sizeof(*sc)); +} + +void fy_emit_cleanup(struct fy_emitter *emit) +{ + struct fy_eventp *fyep; + struct fy_token *fyt; + + /* call the finalizer if it exists */ + if (emit->finalizer) + emit->finalizer(emit); + + fy_emit_save_ctx_cleanup(emit, &emit->s_sc); + while (emit->sc_stack_top > 0) + fy_emit_save_ctx_cleanup(emit, &emit->sc_stack[--emit->sc_stack_top]); + + while ((fyt = fy_token_list_pop(&emit->recycled_token)) != NULL) + fy_token_free(fyt); + + while ((fyep = fy_eventp_list_pop(&emit->recycled_eventp)) != NULL) + fy_eventp_free(fyep); + + fy_document_state_unref(emit->fyds); + emit->fyds = NULL; + + fy_emit_accum_cleanup(&emit->ea); + + while ((fyep = fy_eventp_list_pop(&emit->queued_events)) != NULL) + fy_eventp_release(fyep); + + if (emit->state_stack && emit->state_stack != emit->state_stack_inplace) + free(emit->state_stack); + + if (emit->sc_stack && emit->sc_stack != emit->sc_stack_inplace) + free(emit->sc_stack); + + fy_diag_unref(emit->diag); + + if (emit->owned_output_fp) + fclose(emit->owned_output_fp); +} + +int fy_emit_node_no_check(struct fy_emitter *emit, struct fy_node *fyn) +{ + if (fyn) + fy_emit_node_internal(emit, fyn, DDNF_ROOT, -1, false); + return 0; +} + +int fy_emit_node(struct fy_emitter *emit, struct fy_node *fyn) +{ + int ret; + + ret = fy_emit_node_check(emit, fyn); + if (ret) + return ret; + + return fy_emit_node_no_check(emit, fyn); +} + +int fy_emit_root_node_no_check(struct fy_emitter *emit, struct fy_node *fyn) +{ + if (!emit || !fyn) + return -1; + + /* top comment first */ + fy_emit_node_comment(emit, fyn, DDNF_ROOT, -1, fycp_top); + + fy_emit_node_internal(emit, fyn, DDNF_ROOT, -1, false); + + /* right comment next */ + fy_emit_node_comment(emit, fyn, DDNF_ROOT, -1, fycp_right); + + /* bottom comment last */ + fy_emit_node_comment(emit, fyn, DDNF_ROOT, -1, fycp_bottom); + + return 0; +} + +int fy_emit_root_node(struct fy_emitter *emit, struct fy_node *fyn) +{ + int ret; + + if (!emit || !fyn) + return -1; + + ret = fy_emit_node_check(emit, fyn); + if (ret) + return ret; + + return fy_emit_root_node_no_check(emit, fyn); +} + +void fy_emit_prepare_document_state(struct fy_emitter *emit, struct fy_document_state *fyds) +{ + if (!emit || !fyds) + return; + + /* if the original document was JSON and the mode is ORIGINAL turn on JSON mode */ + emit->source_json = fyds && fyds->json_mode; + emit->force_json = (emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK)) == FYECF_MODE_ORIGINAL && + emit->source_json; +} + +int fy_emit_document_no_check(struct fy_emitter *emit, struct fy_document *fyd) +{ + int rc; + + rc = fy_emit_document_start(emit, fyd, NULL); + if (rc) + return rc; + + rc = fy_emit_root_node_no_check(emit, fyd->root); + if (rc) + return rc; + + rc = fy_emit_document_end(emit); + + return rc; +} + +int fy_emit_document(struct fy_emitter *emit, struct fy_document *fyd) +{ + int ret; + + if (!emit) + return -1; + + if (fyd) { + fy_emit_prepare_document_state(emit, fyd->fyds); + + if (fyd->root) { + ret = fy_emit_node_check(emit, fyd->root); + if (ret) + return ret; + } + } + + return fy_emit_document_no_check(emit, fyd); +} + +struct fy_emitter *fy_emitter_create(const struct fy_emitter_cfg *cfg) +{ + struct fy_emitter *emit; + int rc; + + if (!cfg) + return NULL; + + emit = malloc(sizeof(*emit)); + if (!emit) + return NULL; + + rc = fy_emit_setup(emit, cfg); + if (rc) { + free(emit); + return NULL; + } + + return emit; +} + +void fy_emitter_destroy(struct fy_emitter *emit) +{ + if (!emit) + return; + + fy_emit_cleanup(emit); + + free(emit); +} + +const struct fy_emitter_cfg *fy_emitter_get_cfg(struct fy_emitter *emit) +{ + if (!emit) + return NULL; + + return &emit->xcfg.cfg; +} + +struct fy_diag *fy_emitter_get_diag(struct fy_emitter *emit) +{ + if (!emit || !emit->diag) + return NULL; + return fy_diag_ref(emit->diag); +} + +int fy_emitter_set_diag(struct fy_emitter *emit, struct fy_diag *diag) +{ + struct fy_diag_cfg dcfg; + + if (!emit) + return -1; + + /* default? */ + if (!diag) { + fy_diag_cfg_default(&dcfg); + diag = fy_diag_create(&dcfg); + if (!diag) + return -1; + } + + fy_diag_unref(emit->diag); + emit->diag = fy_diag_ref(diag); + + return 0; +} + +void fy_emitter_set_finalizer(struct fy_emitter *emit, + void (*finalizer)(struct fy_emitter *emit)) +{ + if (!emit) + return; + emit->finalizer = finalizer; +} + +struct fy_emit_buffer_state { + char **bufp; + size_t *sizep; + char *buf; + size_t size; + size_t pos; + size_t need; + bool allocate_buffer; +}; + +static int do_buffer_output(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int leni, void *userdata) +{ + struct fy_emit_buffer_state *state = emit->xcfg.cfg.userdata; + size_t left, pagesize, size, len; + char *bufnew; + + /* convert to unsigned and use that */ + len = (size_t)leni; + + /* no funky business */ + if (len < 0) + return -1; + + state->need += len; + left = state->size - state->pos; + if (left < len) { + if (!state->allocate_buffer) + return 0; + + pagesize = sysconf(_SC_PAGESIZE); + size = state->need + pagesize - 1; + size = size - size % pagesize; + + bufnew = realloc(state->buf, size); + if (!bufnew) + return -1; + state->buf = bufnew; + state->size = size; + left = state->size - state->pos; + + } + + if (len > left) + len = left; + if (state->buf) + memcpy(state->buf + state->pos, str, len); + state->pos += len; + + return len; +} + +static void +fy_emitter_str_finalizer(struct fy_emitter *emit) +{ + struct fy_emit_buffer_state *state; + + if (!emit || !(state = emit->xcfg.cfg.userdata)) + return; + + /* if the buffer is allowed to allocate_buffer... */ + if (state->allocate_buffer && state->buf) + free(state->buf); + free(state); + + emit->xcfg.cfg.userdata = NULL; +} + +static struct fy_emitter * +fy_emitter_create_str_internal(enum fy_emitter_cfg_flags flags, char **bufp, size_t *sizep, bool allocate_buffer) +{ + struct fy_emitter *emit; + struct fy_emitter_cfg emit_cfg; + struct fy_emit_buffer_state *state; + + state = malloc(sizeof(*state)); + if (!state) + return NULL; + + /* if any of these NULL, it's a allocation case */ + if ((!bufp || !sizep) && !allocate_buffer) + return NULL; + + if (bufp && sizep) { + state->bufp = bufp; + state->buf = *bufp; + state->sizep = sizep; + state->size = *sizep; + } else { + state->bufp = NULL; + state->buf = NULL; + state->sizep = NULL; + state->size = 0; + } + state->pos = 0; + state->need = 0; + state->allocate_buffer = allocate_buffer; + + memset(&emit_cfg, 0, sizeof(emit_cfg)); + emit_cfg.output = do_buffer_output; + emit_cfg.userdata = state; + emit_cfg.flags = flags; + + emit = fy_emitter_create(&emit_cfg); + if (!emit) + goto err_out; + + /* set finalizer to cleanup */ + fy_emitter_set_finalizer(emit, fy_emitter_str_finalizer); + + return emit; + +err_out: + if (state) + free(state); + return NULL; +} + +static int +fy_emitter_collect_str_internal(struct fy_emitter *emit, char **bufp, size_t *sizep) +{ + struct fy_emit_buffer_state *state; + char *buf; + size_t size; + int rc; + + state = emit->xcfg.cfg.userdata; + assert(state); + + /* if NULL, then use the values stored on the state */ + if (!bufp) + bufp = state->bufp; + if (!sizep) + sizep = state->sizep; + + /* terminating zero */ + rc = do_buffer_output(emit, fyewt_terminating_zero, "\0", 1, state); + if (rc != 1) + goto err_out; + + state->size = state->need; + + if (state->allocate_buffer) { + /* resize */ + buf = realloc(state->buf, state->size); + /* very likely since we shrink the buffer, but make sure we don't error out */ + if (buf) + state->buf = buf; + } + + /* retreive the buffer and size */ + size = state->size; + if (size > 0 && state->buf[size-1] == '\0') + size--; + *sizep = size; + *bufp = state->buf; + + /* reset the buffer, ownership now to the caller */ + state->buf = NULL; + state->size = 0; + state->pos = 0; + state->bufp = NULL; + state->sizep = NULL; + + return 0; + +err_out: + *bufp = NULL; + *sizep = 0; + return -1; +} + +static int fy_emit_str_internal(struct fy_document *fyd, + enum fy_emitter_cfg_flags flags, + struct fy_node *fyn, char **bufp, size_t *sizep, + bool allocate_buffer) +{ + struct fy_emitter *emit = NULL; + int rc = -1; + + emit = fy_emitter_create_str_internal(flags, bufp, sizep, allocate_buffer); + if (!emit) + goto out_err; + + if (fyd) { + fy_emit_prepare_document_state(emit, fyd->fyds); + rc = 0; + if (fyd->root) + rc = fy_emit_node_check(emit, fyd->root); + if (!rc) + rc = fy_emit_document_no_check(emit, fyd); + } else { + rc = fy_emit_node_check(emit, fyn); + if (!rc) + rc = fy_emit_node_no_check(emit, fyn); + } + + if (rc) + goto out_err; + + rc = fy_emitter_collect_str_internal(emit, NULL, NULL); + if (rc) + goto out_err; + + /* OK, all done */ + +out_err: + fy_emitter_destroy(emit); + return rc; +} + +int fy_emit_document_to_buffer(struct fy_document *fyd, enum fy_emitter_cfg_flags flags, char *buf, size_t size) +{ + int rc; + + rc = fy_emit_str_internal(fyd, flags, NULL, &buf, &size, false); + if (rc != 0) + return -1; + return size; +} + +char *fy_emit_document_to_string(struct fy_document *fyd, enum fy_emitter_cfg_flags flags) +{ + char *buf; + size_t size; + int rc; + + buf = NULL; + size = 0; + rc = fy_emit_str_internal(fyd, flags, NULL, &buf, &size, true); + if (rc != 0) + return NULL; + return buf; +} + +struct fy_emitter * +fy_emit_to_buffer(enum fy_emitter_cfg_flags flags, char *buf, size_t size) +{ + if (!buf) + return NULL; + + return fy_emitter_create_str_internal(flags, &buf, &size, false); +} + +char * +fy_emit_to_buffer_collect(struct fy_emitter *emit, size_t *sizep) +{ + int rc; + char *buf; + + if (!emit || !sizep) + return NULL; + + rc = fy_emitter_collect_str_internal(emit, &buf, sizep); + if (rc) { + *sizep = 0; + return NULL; + } + return buf; +} + +struct fy_emitter * +fy_emit_to_string(enum fy_emitter_cfg_flags flags) +{ + return fy_emitter_create_str_internal(flags, NULL, NULL, true); +} + +char * +fy_emit_to_string_collect(struct fy_emitter *emit, size_t *sizep) +{ + int rc; + char *buf; + + if (!emit || !sizep) + return NULL; + + rc = fy_emitter_collect_str_internal(emit, &buf, sizep); + if (rc) { + *sizep = 0; + return NULL; + } + return buf; +} + +static int do_file_output(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int leni, void *userdata) +{ + FILE *fp = userdata; + size_t len; + + len = (size_t)leni; + + /* no funky stuff */ + if (len < 0) + return -1; + + return fwrite(str, 1, len, fp); +} + +int fy_emit_document_to_fp(struct fy_document *fyd, enum fy_emitter_cfg_flags flags, + FILE *fp) +{ + struct fy_emitter emit_state, *emit = &emit_state; + struct fy_emitter_cfg emit_cfg; + int rc; + + if (!fp) + return -1; + + memset(&emit_cfg, 0, sizeof(emit_cfg)); + emit_cfg.output = do_file_output; + emit_cfg.userdata = fp; + emit_cfg.flags = flags; + fy_emit_setup(emit, &emit_cfg); + + fy_emit_prepare_document_state(emit, fyd->fyds); + + rc = 0; + if (fyd->root) + rc = fy_emit_node_check(emit, fyd->root); + + rc = fy_emit_document_no_check(emit, fyd); + + fy_emit_cleanup(emit); + + return rc ? rc : 0; +} + +int fy_emit_document_to_file(struct fy_document *fyd, + enum fy_emitter_cfg_flags flags, + const char *filename) +{ + FILE *fp; + int rc; + + fp = filename ? fopen(filename, "wa") : stdout; + if (!fp) + return -1; + + rc = fy_emit_document_to_fp(fyd, flags, fp); + + if (fp != stdout) + fclose(fp); + + return rc ? rc : 0; +} + +static int do_fd_output(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int leni, void *userdata) +{ + size_t len; + int fd; + ssize_t wrn; + int total; + + len = (size_t)leni; + + /* no funky stuff */ + if (len < 0) + return -1; + + /* get the file descriptor */ + fd = (int)(uintptr_t)userdata; + if (fd < 0) + return -1; + + /* loop output to fd */ + total = 0; + while (len > 0) { + + do { + wrn = write(fd, str, len); + } while (wrn == -1 && errno == EAGAIN); + + if (wrn == -1) + return -1; + + if (wrn == 0) + return total; + + len -= wrn; + str += wrn; + total += wrn; + } + + return total; +} + +int fy_emit_document_to_fd(struct fy_document *fyd, enum fy_emitter_cfg_flags flags, int fd) +{ + struct fy_emitter emit_state, *emit = &emit_state; + struct fy_emitter_cfg emit_cfg; + int rc; + + if (fd < 0) + return -1; + + memset(&emit_cfg, 0, sizeof(emit_cfg)); + emit_cfg.output = do_fd_output; + emit_cfg.userdata = (void *)(uintptr_t)fd; + emit_cfg.flags = flags; + fy_emit_setup(emit, &emit_cfg); + + fy_emit_prepare_document_state(emit, fyd->fyds); + + rc = 0; + if (fyd->root) + rc = fy_emit_node_check(emit, fyd->root); + + rc = fy_emit_document_no_check(emit, fyd); + + fy_emit_cleanup(emit); + + return rc ? rc : 0; +} + +int fy_emit_node_to_buffer(struct fy_node *fyn, enum fy_emitter_cfg_flags flags, char *buf, size_t size) +{ + int rc; + + rc = fy_emit_str_internal(NULL, flags, fyn, &buf, &size, false); + if (rc != 0) + return -1; + return size; +} + +char *fy_emit_node_to_string(struct fy_node *fyn, enum fy_emitter_cfg_flags flags) +{ + char *buf; + size_t size; + int rc; + + buf = NULL; + size = 0; + rc = fy_emit_str_internal(NULL, flags, fyn, &buf, &size, true); + if (rc != 0) + return NULL; + return buf; +} + +/* we can consume events only if the lookaheads + * are satisfied. The lookaheads depends on the + * styling mode. Some styling modes do not require + * a lookahead at all but we do it anyway. + */ +static bool fy_emit_ready(struct fy_emitter *emit) +{ + struct fy_eventp *fyep; + int need, count; + + count = 0; + need = -1; + for (fyep = fy_eventp_list_head(&emit->queued_events); fyep; + fyep = fy_eventp_next(&emit->queued_events, fyep)) { + + count++; + switch (fyep->e.type) { + case FYET_DOCUMENT_START: + need++; + break; + + case FYET_SEQUENCE_START: + if (need < 0) + need = 2; + break; + + case FYET_MAPPING_START: + if (need < 0) + need = 2; + break; + + default: + need = 0; + break; + } + + if (count >= need) + return true; + + } + + return false; +} + +extern const char *fy_event_type_txt[]; + +const char *fy_emitter_state_txt[] = { + [FYES_NONE] = "NONE", + [FYES_STREAM_START] = "STREAM_START", + [FYES_FIRST_DOCUMENT_START] = "FIRST_DOCUMENT_START", + [FYES_DOCUMENT_START] = "DOCUMENT_START", + [FYES_DOCUMENT_CONTENT] = "DOCUMENT_CONTENT", + [FYES_DOCUMENT_END] = "DOCUMENT_END", + [FYES_SEQUENCE_FIRST_ITEM] = "SEQUENCE_FIRST_ITEM", + [FYES_SEQUENCE_ITEM] = "SEQUENCE_ITEM", + [FYES_MAPPING_FIRST_KEY] = "MAPPING_FIRST_KEY", + [FYES_MAPPING_KEY] = "MAPPING_KEY", + [FYES_MAPPING_SIMPLE_VALUE] = "MAPPING_SIMPLE_VALUE", + [FYES_MAPPING_VALUE] = "MAPPING_VALUE", + [FYES_END] = "END", +}; + +struct fy_eventp * +fy_emit_next_event(struct fy_emitter *emit) +{ + if (!fy_emit_ready(emit)) + return NULL; + + return fy_eventp_list_pop(&emit->queued_events); +} + +struct fy_eventp * +fy_emit_peek_next_event(struct fy_emitter *emit) +{ + return fy_eventp_list_head(&emit->queued_events); +} + +bool fy_emit_streaming_sequence_empty(struct fy_emitter *emit) +{ + struct fy_eventp *fyepn; + + fyepn = fy_emit_peek_next_event(emit); + + /* should never happen, check on debugging mode */ + assert(fyepn); + if (!fyepn) + return false; + + return fyepn->e.type == FYET_SEQUENCE_END; +} + +bool fy_emit_streaming_mapping_empty(struct fy_emitter *emit) +{ + struct fy_eventp *fyepn; + + fyepn = fy_emit_peek_next_event(emit); + + /* should never happen, check on debugging mode */ + assert(fyepn); + if (!fyepn) + return false; + + return fyepn->e.type == FYET_MAPPING_END; +} + +static void fy_emit_goto_state(struct fy_emitter *emit, enum fy_emitter_state state) +{ + if (emit->state == state) + return; + + emit->state = state; +} + +static int fy_emit_push_state(struct fy_emitter *emit, enum fy_emitter_state state) +{ + enum fy_emitter_state *states; + + if (emit->state_stack_top >= emit->state_stack_alloc) { + states = realloc(emit->state_stack == emit->state_stack_inplace ? NULL : emit->state_stack, + sizeof(emit->state_stack[0]) * emit->state_stack_alloc * 2); + if (!states) + return -1; + + if (emit->state_stack == emit->state_stack_inplace) + memcpy(states, emit->state_stack, sizeof(emit->state_stack[0]) * emit->state_stack_top); + emit->state_stack = states; + emit->state_stack_alloc *= 2; + } + emit->state_stack[emit->state_stack_top++] = state; + + return 0; +} + +enum fy_emitter_state fy_emit_pop_state(struct fy_emitter *emit) +{ + if (!emit->state_stack_top) + return FYES_NONE; + + return emit->state_stack[--emit->state_stack_top]; +} + +int fy_emit_push_sc(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) +{ + struct fy_emit_save_ctx *scs; + + if (emit->sc_stack_top >= emit->sc_stack_alloc) { + scs = realloc(emit->sc_stack == emit->sc_stack_inplace ? NULL : emit->sc_stack, + sizeof(emit->sc_stack[0]) * emit->sc_stack_alloc * 2); + if (!scs) + return -1; + + if (emit->sc_stack == emit->sc_stack_inplace) + memcpy(scs, emit->sc_stack, sizeof(emit->sc_stack[0]) * emit->sc_stack_top); + emit->sc_stack = scs; + emit->sc_stack_alloc *= 2; + } + emit->sc_stack[emit->sc_stack_top++] = *sc; + + return 0; +} + +int fy_emit_pop_sc(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) +{ + if (!emit->sc_stack_top) + return -1; + + *sc = emit->sc_stack[--emit->sc_stack_top]; + + return 0; +} + +static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, int flags) +{ + struct fy_event *fye = &fyep->e; + struct fy_emit_save_ctx *sc = &emit->s_sc; + enum fy_node_style style, xstyle; + int ret, s_flags, s_indent; + + if (fye->type != FYET_ALIAS && fye->type != FYET_SCALAR && + (emit->s_flags & DDNF_ROOT) && emit->column != 0) { + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; + } + + emit->s_flags = flags; + + switch (fye->type) { + case FYET_ALIAS: + fy_emit_token_write_alias(emit, fye->alias.anchor, emit->s_flags, emit->s_indent); + fy_emit_goto_state(emit, fy_emit_pop_state(emit)); + break; + + case FYET_SCALAR: + /* if we're pretty and at column 0 (meaning it's a single scalar document) output --- */ + if ((emit->s_flags & DDNF_ROOT) && fy_emit_is_pretty_mode(emit) && !emit->column && + !fy_emit_is_flow_mode(emit) && !(emit->s_flags & DDNF_FLOW)) { + fy_emit_document_start_indicator(emit); + emit->fyds->started_explicit = true; + } + fy_emit_common_node_preamble(emit, fye->scalar.value, fye->scalar.anchor, fye->scalar.tag, emit->s_flags, emit->s_indent); + style = fye->scalar.value ? + fy_node_style_from_scalar_style(fye->scalar.value->scalar.style) : + FYNS_PLAIN; + fy_emit_token_scalar(emit, fye->scalar.value, emit->s_flags, emit->s_indent, style, fye->scalar.tag); + fy_emit_goto_state(emit, fy_emit_pop_state(emit)); + break; + + case FYET_SEQUENCE_START: + + /* save this context */ + ret = fy_emit_push_sc(emit, sc); + if (ret) + return ret; + + s_flags = emit->s_flags; + s_indent = emit->s_indent; + + if (fye->sequence_start.sequence_start) + xstyle = fye->sequence_start.sequence_start->type == FYTT_BLOCK_SEQUENCE_START ? + FYNS_BLOCK : FYNS_FLOW; + else + xstyle = FYNS_ANY; + + fy_emit_common_node_preamble(emit, fye->sequence_start.sequence_start, + fye->sequence_start.anchor, fye->sequence_start.tag, + emit->s_flags, emit->s_indent); + + /* create new context */ + memset(sc, 0, sizeof(*sc)); + sc->flags = emit->s_flags & (DDNF_ROOT | DDNF_SEQ | DDNF_MAP); + sc->indent = emit->s_indent; + sc->empty = fy_emit_streaming_sequence_empty(emit); + sc->flow_token = xstyle == FYNS_FLOW; + sc->flow = !!(s_flags & DDNF_FLOW); + sc->xstyle = xstyle; + sc->old_indent = sc->indent; + sc->s_flags = s_flags; + sc->s_indent = s_indent; + + fy_emit_sequence_prolog(emit, sc); + sc->flags &= ~DDNF_MAP; + sc->flags |= DDNF_SEQ; + + emit->s_flags = sc->flags; + emit->s_indent = sc->indent; + + fy_emit_goto_state(emit, FYES_SEQUENCE_FIRST_ITEM); + break; + + case FYET_MAPPING_START: + /* save this context */ + ret = fy_emit_push_sc(emit, sc); + if (ret) + return ret; + + s_flags = emit->s_flags; + s_indent = emit->s_indent; + + if (fye->mapping_start.mapping_start) + xstyle = fye->mapping_start.mapping_start->type == FYTT_BLOCK_MAPPING_START ? + FYNS_BLOCK : FYNS_FLOW; + else + xstyle = FYNS_ANY; + + fy_emit_common_node_preamble(emit, fye->mapping_start.mapping_start, + fye->mapping_start.anchor, fye->mapping_start.tag, + emit->s_flags, emit->s_indent); + + /* create new context */ + memset(sc, 0, sizeof(*sc)); + sc->flags = emit->s_flags & (DDNF_ROOT | DDNF_SEQ | DDNF_MAP); + sc->indent = emit->s_indent; + sc->empty = fy_emit_streaming_mapping_empty(emit); + sc->flow_token = xstyle == FYNS_FLOW; + sc->flow = !!(s_flags & DDNF_FLOW); + sc->xstyle = xstyle; + sc->old_indent = sc->indent; + sc->s_flags = s_flags; + sc->s_indent = s_indent; + + fy_emit_mapping_prolog(emit, sc); + sc->flags &= ~DDNF_SEQ; + sc->flags |= DDNF_MAP; + + emit->s_flags = sc->flags; + emit->s_indent = sc->indent; + + fy_emit_goto_state(emit, FYES_MAPPING_FIRST_KEY); + break; + + default: + fy_error(emit->diag, "%s: expected ALIAS|SCALAR|SEQUENCE_START|MAPPING_START", __func__); + return -1; + } + + return 0; +} + +static int fy_emit_handle_stream_start(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_event *fye = &fyep->e; + + if (fye->type != FYET_STREAM_START) { + fy_error(emit->diag, "%s: expected FYET_STREAM_START", __func__); + return -1; + } + + fy_document_state_unref(emit->fyds); + emit->fyds = NULL; + + fy_emit_reset(emit, false); + + fy_emit_goto_state(emit, FYES_FIRST_DOCUMENT_START); + + return 0; +} + +static int fy_emit_handle_document_start(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool first) +{ + struct fy_event *fye = &fyep->e; + struct fy_document_state *fyds; + + if (fye->type != FYET_DOCUMENT_START && + fye->type != FYET_STREAM_END) { + fy_error(emit->diag, "%s: expected FYET_DOCUMENT_START|FYET_STREAM_END", __func__); + return -1; + } + + if (fye->type == FYET_STREAM_END) { + fy_emit_goto_state(emit, FYES_END); + return 0; + } + + /* transfer ownership to the emitter */ + fyds = fye->document_start.document_state; + // fye->document_start.document_state = NULL; + + /* prepare (i.e. adapt to the document state) */ + fy_emit_prepare_document_state(emit, fyds); + + fy_emit_common_document_start(emit, fyds, false); + + fy_emit_goto_state(emit, FYES_DOCUMENT_CONTENT); + + return 0; +} + +static int fy_emit_handle_document_end(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_event *fye = &fyep->e; + int ret; + + if (fye->type != FYET_DOCUMENT_END) { + fy_error(emit->diag, "%s: expected FYET_DOCUMENT_END", __func__); + return -1; + } + + ret = fy_emit_common_document_end(emit, true, fye->document_end.implicit); + if (ret) + return ret; + + fy_emit_reset(emit, false); + fy_emit_goto_state(emit, FYES_DOCUMENT_START); + emit->flags |= FYEF_HAD_DOCUMENT_END_OUTPUT; + return 0; +} + +static int fy_emit_handle_document_content(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_event *fye = &fyep->e; + int ret; + + /* empty document? */ + if (fye->type == FYET_DOCUMENT_END) + return fy_emit_handle_document_end(emit, fyp, fyep); + + ret = fy_emit_push_state(emit, FYES_DOCUMENT_END); + if (ret) + return ret; + + return fy_emit_streaming_node(emit, fyp, fyep, DDNF_ROOT); +} + +static int fy_emit_handle_sequence_item(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool first) +{ + struct fy_event *fye = &fyep->e; + struct fy_emit_save_ctx *sc = &emit->s_sc; + struct fy_token *fyt_item = NULL; + int ret; + + fy_emit_token_unref(emit, fyp, sc->fyt_last_value); + sc->fyt_last_value = NULL; + + switch (fye->type) { + case FYET_SEQUENCE_END: + fy_emit_sequence_item_epilog(emit, sc, true, sc->fyt_last_value); + + /* emit epilog */ + fy_emit_sequence_epilog(emit, sc); + + /* restore indent and flags */ + emit->s_indent = sc->s_indent; + emit->s_flags = sc->s_flags; + + /* pop state */ + ret = fy_emit_pop_sc(emit, sc); + assert(!ret); + + /* pop state */ + fy_emit_goto_state(emit, fy_emit_pop_state(emit)); + return ret; + + case FYET_ALIAS: + fyt_item = fye->alias.anchor; + break; + case FYET_SCALAR: + fyt_item = fye->scalar.value; + break; + case FYET_SEQUENCE_START: + fyt_item = fye->sequence_start.sequence_start; + break; + case FYET_MAPPING_START: + fyt_item = fye->mapping_start.mapping_start; + break; + default: + fy_error(emit->diag, "%s: expected SEQUENCE_END|ALIAS|SCALAR|SEQUENCE_START|MAPPING_START", __func__); + return -1; + } + + ret = fy_emit_push_state(emit, FYES_SEQUENCE_ITEM); + if (ret) + return ret; + + /* reset indent and flags for each item */ + emit->s_indent = sc->indent; + emit->s_flags = sc->flags; + + if (!first) + fy_emit_sequence_item_epilog(emit, sc, false, sc->fyt_last_value); + + sc->fyt_last_value = fyt_item; + + fy_emit_sequence_item_prolog(emit, sc, fyt_item); + + ret = fy_emit_streaming_node(emit, fyp, fyep, sc->flags); + + switch (fye->type) { + case FYET_ALIAS: + fye->alias.anchor = NULL; /* take ownership */ + break; + case FYET_SCALAR: + fye->scalar.value = NULL; /* take ownership */ + break; + case FYET_SEQUENCE_START: + fye->sequence_start.sequence_start = NULL; /* take ownership */ + break; + case FYET_MAPPING_START: + fye->mapping_start.mapping_start = NULL; /* take ownership */ + break; + default: + break; + } + + return ret; +} + +static int fy_emit_handle_mapping_key(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool first) +{ + struct fy_event *fye = &fyep->e; + struct fy_emit_save_ctx *sc = &emit->s_sc; + struct fy_token *fyt_key = NULL; + const struct fy_token_analysis *ta; + int ret; + bool simple_key; + + fy_emit_token_unref(emit, fyp, sc->fyt_last_key); + sc->fyt_last_key = NULL; + fy_emit_token_unref(emit, fyp, sc->fyt_last_value); + sc->fyt_last_value = NULL; + + simple_key = false; + + switch (fye->type) { + case FYET_MAPPING_END: + fy_emit_mapping_value_epilog(emit, sc, true, sc->fyt_last_value); + + /* emit epilog */ + fy_emit_mapping_epilog(emit, sc); + + /* restore indent and flags */ + emit->s_indent = sc->s_indent; + emit->s_flags = sc->s_flags; + + /* pop state */ + ret = fy_emit_pop_sc(emit, sc); + assert(!ret); + + /* pop state */ + fy_emit_goto_state(emit, fy_emit_pop_state(emit)); + return ret; + + case FYET_ALIAS: + fyt_key = fye->alias.anchor; + simple_key = true; + break; + case FYET_SCALAR: + fyt_key = fye->scalar.value; + ta = fy_token_text_analyze(fyt_key); + simple_key = !!(ta->flags & FYTTAF_CAN_BE_SIMPLE_KEY); + break; + case FYET_SEQUENCE_START: + fyt_key = fye->sequence_start.sequence_start; + simple_key = fy_emit_streaming_sequence_empty(emit); + break; + case FYET_MAPPING_START: + fyt_key = fye->mapping_start.mapping_start; + simple_key = fy_emit_streaming_mapping_empty(emit); + break; + default: + fy_error(emit->diag, "%s: expected MAPPING_END|ALIAS|SCALAR|SEQUENCE_START|MAPPING_START", __func__); + return -1; + } + + ret = fy_emit_push_state(emit, FYES_MAPPING_VALUE); + if (ret) + return ret; + + /* reset indent and flags for each key/value pair */ + emit->s_indent = sc->indent; + emit->s_flags = sc->flags; + + if (!first) + fy_emit_mapping_value_epilog(emit, sc, false, sc->fyt_last_value); + + sc->fyt_last_key = fyt_key; + + fy_emit_mapping_key_prolog(emit, sc, fyt_key, simple_key); + + ret = fy_emit_streaming_node(emit, fyp, fyep, sc->flags); + + switch (fye->type) { + case FYET_ALIAS: + fye->alias.anchor = NULL; /* take ownership */ + break; + case FYET_SCALAR: + fye->scalar.value = NULL; /* take ownership */ + break; + case FYET_SEQUENCE_START: + fye->sequence_start.sequence_start = NULL; /* take ownership */ + break; + case FYET_MAPPING_START: + fye->mapping_start.mapping_start = NULL; /* take ownership */ + break; + default: + break; + } + + return ret; +} + +static int fy_emit_handle_mapping_value(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep, bool simple) +{ + struct fy_event *fye = &fyep->e; + struct fy_emit_save_ctx *sc = &emit->s_sc; + struct fy_token *fyt_value = NULL; + int ret; + + switch (fye->type) { + case FYET_ALIAS: + fyt_value = fye->alias.anchor; + break; + case FYET_SCALAR: + fyt_value = fye->scalar.value; /* take ownership */ + break; + case FYET_SEQUENCE_START: + fyt_value = fye->sequence_start.sequence_start; + break; + case FYET_MAPPING_START: + fyt_value = fye->mapping_start.mapping_start; + break; + default: + fy_error(emit->diag, "%s: expected ALIAS|SCALAR|SEQUENCE_START|MAPPING_START", __func__); + return -1; + } + + ret = fy_emit_push_state(emit, FYES_MAPPING_KEY); + if (ret) + return ret; + + fy_emit_mapping_key_epilog(emit, sc, sc->fyt_last_key); + + sc->fyt_last_value = fyt_value; + + fy_emit_mapping_value_prolog(emit, sc, fyt_value); + + ret = fy_emit_streaming_node(emit, fyp, fyep, sc->flags); + + switch (fye->type) { + case FYET_ALIAS: + fye->alias.anchor = NULL; /* take ownership */ + break; + case FYET_SCALAR: + fye->scalar.value = NULL; /* take ownership */ + break; + case FYET_SEQUENCE_START: + fye->sequence_start.sequence_start = NULL; /* take ownership */ + break; + case FYET_MAPPING_START: + fye->mapping_start.mapping_start = NULL; /* take ownership */ + break; + default: + break; + } + + return ret; +} + +int fy_emit_event_from_parser(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_event *fye) +{ + struct fy_eventp *fyep; + int ret; + + if (!emit || !fye) + return -1; + + /* we're using the raw emitter interface, now mark first state */ + if (emit->state == FYES_NONE) + emit->state = FYES_STREAM_START; + + /* handle reset (parser error recovery) */ + if (fye->type == FYET_STREAM_START && emit->state != FYES_STREAM_START) { + if (emit->column != 0) + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + fy_emit_puts_simple(emit, fyewt_document_indicator, "..."); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->state = FYES_STREAM_START; + } + + fyep = container_of(fye, struct fy_eventp, e); + + fy_eventp_list_add_tail(&emit->queued_events, fyep); + + ret = 0; + while ((fyep = fy_emit_next_event(emit)) != NULL) { + + switch (emit->state) { + case FYES_STREAM_START: + ret = fy_emit_handle_stream_start(emit, fyp, fyep); + break; + + case FYES_FIRST_DOCUMENT_START: + case FYES_DOCUMENT_START: + ret = fy_emit_handle_document_start(emit, fyp, fyep, + emit->state == FYES_FIRST_DOCUMENT_START); + break; + + case FYES_DOCUMENT_CONTENT: + ret = fy_emit_handle_document_content(emit, fyp, fyep); + break; + + case FYES_DOCUMENT_END: + ret = fy_emit_handle_document_end(emit, fyp, fyep); + break; + + case FYES_SEQUENCE_FIRST_ITEM: + case FYES_SEQUENCE_ITEM: + ret = fy_emit_handle_sequence_item(emit, fyp, fyep, + emit->state == FYES_SEQUENCE_FIRST_ITEM); + break; + + case FYES_MAPPING_FIRST_KEY: + case FYES_MAPPING_KEY: + ret = fy_emit_handle_mapping_key(emit, fyp, fyep, + emit->state == FYES_MAPPING_FIRST_KEY); + break; + + case FYES_MAPPING_SIMPLE_VALUE: + case FYES_MAPPING_VALUE: + ret = fy_emit_handle_mapping_value(emit, fyp, fyep, + emit->state == FYES_MAPPING_SIMPLE_VALUE); + break; + + case FYES_END: + ret = -1; + break; + + default: + assert(1); /* Invalid state. */ + } + + /* always release the event */ + if (!fyp) + fy_eventp_release(fyep); + else + fy_parse_eventp_recycle(fyp, fyep); + + if (ret) + break; + } + + return ret; +} + +int fy_emit_event(struct fy_emitter *emit, struct fy_event *fye) +{ + return fy_emit_event_from_parser(emit, NULL, fye); +} + +struct fy_document_state * +fy_emitter_get_document_state(struct fy_emitter *emit) +{ + return emit ? emit->fyds : NULL; +} + +/* ANSI colors and escapes */ +#define A_RESET "\x1b[0m" +#define A_BLACK "\x1b[30m" +#define A_RED "\x1b[31m" +#define A_GREEN "\x1b[32m" +#define A_YELLOW "\x1b[33m" +#define A_BLUE "\x1b[34m" +#define A_MAGENTA "\x1b[35m" +#define A_CYAN "\x1b[36m" +#define A_LIGHT_GRAY "\x1b[37m" /* dark white is gray */ +#define A_GRAY "\x1b[1;30m" +#define A_BRIGHT_RED "\x1b[1;31m" +#define A_BRIGHT_GREEN "\x1b[1;32m" +#define A_BRIGHT_YELLOW "\x1b[1;33m" +#define A_BRIGHT_BLUE "\x1b[1;34m" +#define A_BRIGHT_MAGENTA "\x1b[1;35m" +#define A_BRIGHT_CYAN "\x1b[1;36m" +#define A_WHITE "\x1b[1;37m" + +static const char *default_colors[FYEWT_COUNT] = { + [fyewt_document_indicator] = A_CYAN, + [fyewt_tag_directive] = A_YELLOW, + [fyewt_version_directive] = A_YELLOW, + [fyewt_indent] = A_GREEN, // when visible only + [fyewt_indicator] = A_YELLOW, // in extended mode we produce more + [fyewt_whitespace] = A_GREEN, // when visible only + [fyewt_plain_scalar] = A_WHITE, + [fyewt_single_quoted_scalar] = A_YELLOW, + [fyewt_double_quoted_scalar] = A_YELLOW, + [fyewt_literal_scalar] = A_YELLOW, + [fyewt_folded_scalar] = A_YELLOW, + [fyewt_anchor] = A_BRIGHT_GREEN, + [fyewt_tag] = A_BRIGHT_GREEN, + [fyewt_linebreak] = A_GREEN, // when visible only + [fyewt_alias] = A_BRIGHT_GREEN, + [fyewt_terminating_zero] = A_BRIGHT_RED, // when visible only + [fyewt_plain_scalar_key] = A_BRIGHT_CYAN, + [fyewt_single_quoted_scalar_key] = A_BRIGHT_CYAN, + [fyewt_double_quoted_scalar_key] = A_BRIGHT_CYAN, + [fyewt_comment] = A_BRIGHT_BLUE, + // extended + [fyewt_indicator_question_mark] = A_MAGENTA, + [fyewt_indicator_colon] = A_MAGENTA, + [fyewt_indicator_dash] = A_MAGENTA, + [fyewt_indicator_left_bracket] = A_MAGENTA, + [fyewt_indicator_right_bracket] = A_MAGENTA, + [fyewt_indicator_left_brace] = A_MAGENTA, + [fyewt_indicator_right_brace] = A_MAGENTA, + [fyewt_indicator_comma] = A_MAGENTA, + [fyewt_indicator_bar] = A_MAGENTA, + [fyewt_indicator_greater] = A_MAGENTA, + [fyewt_indicator_single_quote_start] = A_YELLOW, + [fyewt_indicator_single_quote_end] = A_YELLOW, + [fyewt_indicator_double_quote_start] = A_YELLOW, + [fyewt_indicator_double_quote_end] = A_YELLOW, + [fyewt_indicator_ambersand] = A_BRIGHT_GREEN, + [fyewt_indicator_star] = A_MAGENTA, + [fyewt_indicator_chomp] = A_MAGENTA, + [fyewt_indicator_explicit_indent] = A_MAGENTA, +}; + +static void fy_emitter_fill_default_colors(struct fy_emitter *fye) +{ + unsigned int i; + + for (i = 0; i < FYEWT_COUNT; i++) { + if (!fye->xcfg.colors[i]) + fye->xcfg.colors[i] = default_colors[i]; + } +} + +static FILE *fy_emitter_get_output_fp(struct fy_emitter *fye) +{ + switch (fye->xcfg.xflags & (FYEXCF_OUTPUT_MASK << FYEXCF_OUTPUT_SHIFT)) { + case FYEXCF_OUTPUT_STDOUT: + return stdout; + case FYEXCF_OUTPUT_STDERR: + return stderr; + case FYEXCF_OUTPUT_FILE: + return fye->xcfg.output_fp; + case FYEXCF_OUTPUT_FILENAME: + if (!fye->owned_output_fp && fye->xcfg.output_filename) + fye->owned_output_fp = fopen(fye->xcfg.output_filename, "wb"); + return fye->owned_output_fp; + default: + break; + } + return NULL; +} + +static int fy_emitter_get_output_fd(struct fy_emitter *fye) +{ + switch (fye->xcfg.xflags & (FYEXCF_OUTPUT_MASK << FYEXCF_OUTPUT_SHIFT)) { + case FYEXCF_OUTPUT_STDOUT: + return STDOUT_FILENO; + case FYEXCF_OUTPUT_STDERR: + return STDERR_FILENO; + case FYEXCF_OUTPUT_FD: + return fye->xcfg.output_fd; + default: + break; + } + return -1; +} + + +static int fy_emitter_null_output(struct fy_emitter *fye, enum fy_emitter_write_type type, const char *str, int len, void *userdata) +{ + return len; +} + +static inline ssize_t raw_default_output(FILE *fp, int fd, const char *data, size_t len) +{ + ssize_t wrn, twrn; + + if (fp) + return fwrite(data, 1, len, fp); + + wrn = 0; + if (fd >= 0) { + while (len > 0) { + do { + twrn = write(fd, data, len); + } while (twrn == -1 && errno == EAGAIN); + if (twrn <= 0) + return wrn; + data += twrn; + len -= (size_t)twrn; + wrn += twrn; + } + } + return wrn; +} + +int fy_emitter_default_output(struct fy_emitter *fye, enum fy_emitter_write_type type, const char *str, int len, void *userdata) +{ + struct fy_emitter_default_output_data d_local, *d; + FILE *fp; + int fd; + int ret, w; + const char *color = NULL; + const char *s, *e; + const char *visible; + + d = userdata; + if (!d) { + d = &d_local; + fp = fye->output_fp; + fd = fye->output_fd; + d->colorize = fye->output_colorize; + d->visible = !!(fye->xcfg.xflags & FYEXCF_VISIBLE_WS); + } else { + fp = d->fp; + fd = -1; + } + + if (!fp && fd < 0) + return len; + + s = str; + e = str + len; + + if (d->colorize) { + + color = fye->xcfg.colors[type]; + + /* visible whitespace */ + switch (type) { + case fyewt_indent: + case fyewt_whitespace: + case fyewt_linebreak: + case fyewt_terminating_zero: + + /* if not visible don't bother */ + if (!d->visible) { + color = NULL; + break; + } + + if (color) + raw_default_output(fp, fd, color, strlen(color)); + + switch (type) { + case fyewt_indent: + /* open box - U+2423 */ + visible = "\xe2\x90\xa3"; + break; + case fyewt_whitespace: + /* symbol for space - U+2420 */ + /* symbol for interpunct - U+00B7 */ + visible = "\xc2\xb7"; + break; + case fyewt_linebreak: + /* down arrow - U+2193 */ + visible = "\xe2\x86\x93" "\n"; + break; + case fyewt_terminating_zero: + visible = "\\0"; + break; + default: + visible = ""; /* should never happen */ + break; + + } + + while (s < e && (w = fy_utf8_width_by_first_octet(((uint8_t)*s))) > 0) { + raw_default_output(fp, fd, visible, strlen(visible)); + s += w; + } + + if (color) + raw_default_output(fp, fd, A_RESET, strlen(A_RESET)); + return len; + + default: + break; + } + } else + color = NULL; + + /* don't output the terminating zero */ + if (type == fyewt_terminating_zero) + return len; + + if (color) + raw_default_output(fp, fd, color, strlen(color)); + + ret = raw_default_output(fp, fd, str, len); + + if (color) + raw_default_output(fp, fd, A_RESET, strlen(A_RESET)); + + return ret; +} + +int fy_document_default_emit_to_fp(struct fy_document *fyd, FILE *fp) +{ + struct fy_emitter emit_local, *emit = &emit_local; + struct fy_emitter_cfg ecfg_local, *ecfg = &ecfg_local; + struct fy_emitter_default_output_data d_local, *d = &d_local; + int rc; + + memset(d, 0, sizeof(*d)); + d->fp = fp; + d->colorize = isatty(fileno(fp)); + d->visible = false; + + memset(ecfg, 0, sizeof(*ecfg)); + ecfg->diag = fyd->diag; + ecfg->userdata = d; + + rc = fy_emit_setup(emit, ecfg); + if (rc) + goto err_setup; + + fy_emit_prepare_document_state(emit, fyd->fyds); + + rc = 0; + if (fyd->root) + rc = fy_emit_node_check(emit, fyd->root); + + rc = fy_emit_document_no_check(emit, fyd); + if (rc) + goto err_emit; + + fy_emit_cleanup(emit); + + return 0; + +err_emit: + fy_emit_cleanup(emit); +err_setup: + return -1; +} + +int fy_emit_body_node(struct fy_emitter *emit, struct fy_node *fyn) +{ + struct fy_document_iterator *fydi = NULL; + struct fy_event *fye; + struct fy_document_iterator_cfg cfg; + int rc; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYDICF_WANT_BODY_EVENTS; + cfg.fyd = fyn->fyd; + cfg.iterate_root = fyn; + + fydi = fy_document_iterator_create_cfg(&cfg); + if (!fydi) + goto err_out; + + while ((fye = fy_document_iterator_generate_next(fydi)) != NULL) { + rc = fy_emit_event(emit, fye); + if (rc) + goto err_out; + } + + fy_document_iterator_destroy(fydi); + return 0; + +err_out: + fy_document_iterator_destroy(fydi); + return -1; +} diff --git a/src/lib/fy-emit.h b/src/lib/fy-emit.h new file mode 100644 index 0000000..9269c62 --- /dev/null +++ b/src/lib/fy-emit.h @@ -0,0 +1,165 @@ +/* + * fy-emit.h - internal YAML emitter header + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_EMIT_H +#define FY_EMIT_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#include + +#include "fy-utf8.h" +#include "fy-event.h" +#include "fy-emit-accum.h" + +#define FYEF_WHITESPACE 0x0001 +#define FYEF_INDENTATION 0x0002 +#define FYEF_OPEN_ENDED 0x0004 +#define FYEF_HAD_DOCUMENT_START 0x0008 +#define FYEF_HAD_DOCUMENT_END 0x0010 +#define FYEF_HAD_DOCUMENT_OUTPUT 0x0020 +#define FYEF_HAD_DOCUMENT_END_OUTPUT 0x0040 + +struct fy_document; +struct fy_emitter; +struct fy_document_state; + +enum fy_emitter_state { + FYES_NONE, /* when not using the raw emitter interface */ + FYES_STREAM_START, + FYES_FIRST_DOCUMENT_START, + FYES_DOCUMENT_START, + FYES_DOCUMENT_CONTENT, + FYES_DOCUMENT_END, + FYES_SEQUENCE_FIRST_ITEM, + FYES_SEQUENCE_ITEM, + FYES_MAPPING_FIRST_KEY, + FYES_MAPPING_KEY, + FYES_MAPPING_SIMPLE_VALUE, + FYES_MAPPING_VALUE, + FYES_END, +}; + +struct fy_emit_save_ctx { + bool flow_token : 1; + bool flow : 1; + bool empty : 1; + enum fy_node_style xstyle; + int old_indent; + int flags; + int indent; + struct fy_token *fyt_last_key; + struct fy_token *fyt_last_value; + int s_flags; + int s_indent; +}; + +/* internal flags */ +#define DDNF_ROOT 0x0001 +#define DDNF_SEQ 0x0002 +#define DDNF_MAP 0x0004 +#define DDNF_SIMPLE 0x0008 +#define DDNF_FLOW 0x0010 +#define DDNF_INDENTLESS 0x0020 +#define DDNF_SIMPLE_SCALAR_KEY 0x0040 +#define DDNF_HANGING_INDENT 0x0080 + +struct fy_emitter { + int line; + int column; + int flow_level; + unsigned int flags; + bool output_error : 1; + bool source_json : 1; /* the source was json */ + bool force_json : 1; /* force JSON mode unconditionally */ + bool suppress_recycling_force : 1; + bool suppress_recycling : 1; + + /* current document */ + struct fy_emitter_xcfg xcfg; + FILE *output_fp; + int output_fd; + bool output_colorize; + FILE *owned_output_fp; + + struct fy_document *fyd; + struct fy_document_state *fyds; /* fyd->fyds when fyd != NULL */ + struct fy_emit_accum ea; + char ea_inplace_buf[256]; /* the in place accumulator buffer before allocating */ + struct fy_diag *diag; + + /* streaming event mode */ + enum fy_emitter_state state; + enum fy_emitter_state *state_stack; + unsigned int state_stack_alloc; + unsigned int state_stack_top; + enum fy_emitter_state state_stack_inplace[64]; + struct fy_eventp_list queued_events; + int s_indent; + int s_flags; + struct fy_emit_save_ctx s_sc; + struct fy_emit_save_ctx *sc_stack; + unsigned int sc_stack_alloc; + unsigned int sc_stack_top; + struct fy_emit_save_ctx sc_stack_inplace[16]; + + /* recycled */ + struct fy_eventp_list recycled_eventp; + struct fy_token_list recycled_token; + + struct fy_eventp_list *recycled_eventp_list; /* NULL when suppressing */ + struct fy_token_list *recycled_token_list; /* NULL when suppressing */ + + /* for special needs */ + void (*finalizer)(struct fy_emitter *emit); +}; + +int fy_emit_setup(struct fy_emitter *emit, const struct fy_emitter_cfg *cfg); +void fy_emit_cleanup(struct fy_emitter *emit); + +void fy_emit_write(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int len); + +static inline bool fy_emit_whitespace(struct fy_emitter *emit) +{ + return !!(emit->flags & FYEF_WHITESPACE); +} + +static inline bool fy_emit_indentation(struct fy_emitter *emit) +{ + return !!(emit->flags & FYEF_INDENTATION); +} + +static inline bool fy_emit_open_ended(struct fy_emitter *emit) +{ + return !!(emit->flags & FYEF_OPEN_ENDED); +} + +static inline void +fy_emit_output_accum(struct fy_emitter *emit, enum fy_emitter_write_type type, struct fy_emit_accum *ea) +{ + const char *text; + size_t len; + + text = fy_emit_accum_get(ea, &len); + if (text && len > 0) + fy_emit_write(emit, type, text, len); + fy_emit_accum_reset(ea); +} + +static inline void +fy_emit_output_col_sync(struct fy_emitter *emit, struct fy_emit_accum *ea) +{ + ea->col += emit->column; +} + +#endif diff --git a/src/lib/fy-event.c b/src/lib/fy-event.c new file mode 100644 index 0000000..f042713 --- /dev/null +++ b/src/lib/fy-event.c @@ -0,0 +1,1101 @@ +/* + * fy-event.c - YAML event methods + * + * Copyright (c) 2021 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" +#include "fy-emit.h" +#include "fy-doc.h" + +#include "fy-ctype.h" +#include "fy-utf8.h" +#include "fy-utils.h" + +#include "fy-event.h" + +struct fy_eventp *fy_eventp_alloc(void) +{ + struct fy_eventp *fyep; + + fyep = malloc(sizeof(*fyep)); + if (!fyep) + return NULL; + + fyep->e.type = FYET_NONE; + + return fyep; +} + +void fy_eventp_clean_rl(struct fy_token_list *fytl, struct fy_eventp *fyep) +{ + struct fy_event *fye; + + if (!fyep) + return; + + fye = &fyep->e; + switch (fye->type) { + case FYET_NONE: + break; + case FYET_STREAM_START: + fy_token_unref_rl(fytl, fye->stream_start.stream_start); + break; + case FYET_STREAM_END: + fy_token_unref_rl(fytl, fye->stream_end.stream_end); + break; + case FYET_DOCUMENT_START: + fy_token_unref_rl(fytl, fye->document_start.document_start); + fy_document_state_unref(fye->document_start.document_state); + break; + case FYET_DOCUMENT_END: + fy_token_unref_rl(fytl, fye->document_end.document_end); + break; + case FYET_MAPPING_START: + fy_token_unref_rl(fytl, fye->mapping_start.anchor); + fy_token_unref_rl(fytl, fye->mapping_start.tag); + fy_token_unref_rl(fytl, fye->mapping_start.mapping_start); + break; + case FYET_MAPPING_END: + fy_token_unref_rl(fytl, fye->mapping_end.mapping_end); + break; + case FYET_SEQUENCE_START: + fy_token_unref_rl(fytl, fye->sequence_start.anchor); + fy_token_unref_rl(fytl, fye->sequence_start.tag); + fy_token_unref_rl(fytl, fye->sequence_start.sequence_start); + break; + case FYET_SEQUENCE_END: + fy_token_unref_rl(fytl, fye->sequence_end.sequence_end); + break; + case FYET_SCALAR: + fy_token_unref_rl(fytl, fye->scalar.anchor); + fy_token_unref_rl(fytl, fye->scalar.tag); + fy_token_unref_rl(fytl, fye->scalar.value); + break; + case FYET_ALIAS: + fy_token_unref_rl(fytl, fye->alias.anchor); + break; + } + + fye->type = FYET_NONE; +} + +void fy_parse_eventp_clean(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + if (!fyp || !fyep) + return; + + fy_eventp_clean_rl(fyp->recycled_token_list, fyep); +} + +void fy_emit_eventp_clean(struct fy_emitter *emit, struct fy_eventp *fyep) +{ + if (!emit || !fyep) + return; + + fy_eventp_clean_rl(emit->recycled_token_list, fyep); +} + +void fy_eventp_free(struct fy_eventp *fyep) +{ + if (!fyep) + return; + + /* clean, safe to do */ + fy_eventp_clean_rl(NULL, fyep); + + free(fyep); +} + +void fy_eventp_release(struct fy_eventp *fyep) +{ + fy_eventp_free(fyep); +} + +struct fy_eventp *fy_parse_eventp_alloc(struct fy_parser *fyp) +{ + struct fy_eventp *fyep = NULL; + + if (!fyp) + return NULL; + + if (fyp->recycled_eventp_list) + fyep = fy_eventp_list_pop(fyp->recycled_eventp_list); + if (!fyep) + fyep = fy_eventp_alloc(); + if (!fyep) + return NULL; + + fyep->e.type = FYET_NONE; + + return fyep; +} + +void fy_parse_eventp_recycle(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + if (!fyp || !fyep) + return; + + /* clean, safe to do */ + fy_parse_eventp_clean(fyp, fyep); + + /* and push to the parser recycle list */ + if (fyp->recycled_eventp_list) + fy_eventp_list_push(fyp->recycled_eventp_list, fyep); + else + fy_eventp_free(fyep); +} + +void fy_parser_event_free(struct fy_parser *fyp, struct fy_event *fye) +{ + struct fy_eventp *fyep; + + if (!fyp || !fye) + return; + + if (fy_reader_generates_events(fyp->reader)) + return fy_reader_event_free(fyp->reader, fye); + + fyep = container_of(fye, struct fy_eventp, e); + + fy_parse_eventp_recycle(fyp, fyep); +} + +void fy_emit_eventp_recycle(struct fy_emitter *emit, struct fy_eventp *fyep) +{ + if (!emit || !fyep) + return; + + /* clean, safe to do */ + fy_emit_eventp_clean(emit, fyep); + + if (emit->recycled_eventp_list) + fy_eventp_list_push(emit->recycled_eventp_list, fyep); + else + fy_eventp_free(fyep); +} + +void fy_emit_event_free(struct fy_emitter *emit, struct fy_event *fye) +{ + struct fy_eventp *fyep; + + if (!emit || !fye) + return; + + fyep = container_of(fye, struct fy_eventp, e); + + fy_emit_eventp_recycle(emit, fyep); +} + +struct fy_eventp * +fy_eventp_vcreate_internal(struct fy_eventp_list *recycled_list, struct fy_diag *diag, + struct fy_document_state *fyds, enum fy_event_type type, va_list ap) +{ + struct fy_eventp *fyep = NULL; + struct fy_event *fye = NULL; + struct fy_document_state *fyds_new = NULL; + const struct fy_version *vers; + const struct fy_tag *tag, * const *tagp; + struct fy_token *fyt; + enum fy_token_type ttype; + int rc, tag_count = 0; + enum fy_node_style style; + enum fy_scalar_style sstyle; + struct fy_token **fyt_anchorp = NULL, **fyt_tagp = NULL; + struct fy_input *fyi = NULL; + struct fy_atom handle; + const char *value; + size_t len; + char *data = NULL; + struct fy_tag_scan_info info; + struct fy_token *fyt_td; + + /* try the recycled list first */ + if (recycled_list) + fyep = fy_eventp_list_pop(recycled_list); + /* if not there yet, allocate a fresh one */ + if (!fyep) + fyep = fy_eventp_alloc(); + if (!fyep) + return NULL; + + fye = &fyep->e; + + fye->type = type; + + switch (type) { + case FYET_NONE: + break; + case FYET_STREAM_START: + fye->stream_start.stream_start = NULL; + break; + case FYET_STREAM_END: + fye->stream_end.stream_end = NULL; + break; + case FYET_DOCUMENT_START: + fye->document_start.document_start = NULL; + fyds_new = fy_document_state_default(fy_document_state_version(fyds), NULL); /* start with the default state */ + if (!fyds_new) { + fy_error(diag, "fy_document_state_alloc() failed\n"); + goto err_out; + } + fye->document_start.implicit = va_arg(ap, int); + vers = va_arg(ap, const struct fy_version *); + if (vers) { + fyds_new->version = *vers; + fyds_new->version_explicit = true; + } + fyds_new->start_implicit = fye->document_start.implicit; + fyds_new->end_implicit = false; /* this is not used right now */ + tag_count = 0; + tagp = va_arg(ap, const struct fy_tag * const *); + if (tagp) { + while ((tag = tagp[tag_count]) != NULL) { + tag_count++; + rc = fy_document_state_append_tag(fyds_new, tag->handle, tag->prefix, false); + if (rc) { + fy_error(diag, "fy_document_state_append_tag() failed on handle='%s' prefix='%s'\n", + tag->handle, tag->prefix); + goto err_out; + } + } + } + if (tag_count) + fyds_new->tags_explicit = true; + fye->document_start.document_state = fyds_new; + fyds_new = NULL; + break; + case FYET_DOCUMENT_END: + fye->document_end.document_end = NULL; + fye->document_end.implicit = va_arg(ap, int); + break; + case FYET_MAPPING_START: + case FYET_SEQUENCE_START: + style = va_arg(ap, enum fy_node_style); + ttype = FYTT_NONE; + + if (style != FYNS_ANY && style != FYNS_FLOW && style != FYNS_BLOCK) { + fy_error(diag, "illegal style for %s_START\n", + type == FYET_MAPPING_START ? "MAPPING" : "SEQUENCE"); + goto err_out; + } + + if (style != FYNS_ANY) { + if (style == FYNS_FLOW) + ttype = type == FYET_MAPPING_START ? + FYTT_FLOW_MAPPING_START : + FYTT_FLOW_SEQUENCE_START; + else + ttype = type == FYET_MAPPING_START ? + FYTT_BLOCK_MAPPING_START : + FYTT_BLOCK_SEQUENCE_START; + fyt = fy_token_create(ttype, NULL); + if (!fyt) { + fy_error(diag, "fy_token_create() failed for %s_START\n", + type == FYET_MAPPING_START ? "MAPPING" : "SEQUENCE"); + goto err_out; + } + } else + fyt = NULL; + + if (type == FYET_MAPPING_START) { + fye->mapping_start.mapping_start = fyt; + fye->mapping_start.anchor = NULL; + fye->mapping_start.tag = NULL; + fyt_anchorp = &fye->mapping_start.anchor; + fyt_tagp = &fye->mapping_start.tag; + } else { + fye->sequence_start.sequence_start = fyt; + fye->sequence_start.anchor = NULL; + fye->sequence_start.tag = NULL; + fyt_anchorp = &fye->sequence_start.anchor; + fyt_tagp = &fye->sequence_start.tag; + } + fyt = NULL; + break; + case FYET_MAPPING_END: + fye->mapping_end.mapping_end = NULL; + break; + case FYET_SEQUENCE_END: + fye->sequence_end.sequence_end = NULL; + break; + + case FYET_SCALAR: + case FYET_ALIAS: + + if (type == FYET_SCALAR) { + sstyle = va_arg(ap, enum fy_scalar_style); + value = va_arg(ap, const char *); + len = va_arg(ap, size_t); + if (!value && len) { + fy_error(diag, "NULL value with len > 0, illegal SCALAR\n"); + goto err_out; + } + if (len == FY_NT) + len = strlen(value); + } else { + sstyle = FYSS_PLAIN; + value = va_arg(ap, const char *); + if (!value) { + fy_error(diag, "NULL value, illegal ALIAS\n"); + goto err_out; + } + len = strlen(value); + } + + fyt = NULL; + fyi = NULL; + + data = malloc(len + 1); + if (!data) { + fy_error(diag, "malloc() failed\n"); + goto err_out; + } + memcpy(data, value, len); + /* always NULL terminate */ + data[len] = '\0'; + fyi = fy_input_from_malloc_data(data, len, &handle, sstyle == FYSS_PLAIN); + if (!fyi) { + fy_error(diag, "fy_input_from_malloc_data() failed\n"); + goto err_out; + } + data = NULL; + + if (type == FYET_SCALAR) { + fyt = fy_token_create(FYTT_SCALAR, &handle, sstyle); + if (!fyt) { + fy_error(diag, "fy_token_create() failed for %s\n", + "SCALAR"); + goto err_out; + } + + fye->scalar.value = fyt; + fyt = NULL; + + fye->scalar.anchor = NULL; + fye->scalar.tag = NULL; + fyt_anchorp = &fye->scalar.anchor; + fyt_tagp = &fye->scalar.tag; + } else { + fyt = fy_token_create(FYTT_ALIAS, &handle, NULL); + if (!fyt) { + fy_error(diag, "fy_token_create() failed for %s\n", + "ALIAS"); + goto err_out; + } + + fye->alias.anchor = fyt; + fyt = NULL; + } + fy_input_unref(fyi); + fyi = NULL; + break; + } + + if (fyt_anchorp && (value = va_arg(ap, const char *)) != NULL) { + + len = strlen(value); + data = malloc(len + 1); + if (!data) { + fy_error(diag, "malloc() failed\n"); + goto err_out; + } + memcpy(data, value, len); + /* always NULL terminate */ + data[len] = '\0'; + + fyi = fy_input_from_malloc_data(data, len, &handle, true); + if (!fyi) { + fy_error(diag, "fy_input_from_malloc_data() failed\n"); + goto err_out; + } + data = NULL; + + /* make sure the input as valid as an anchor */ + if (!handle.valid_anchor) { + fy_error(diag, "input was not valid as anchor\n"); + goto err_out; + } + + fyt = fy_token_create(FYTT_ANCHOR, &handle); + if (!fyt) { + fy_error(diag, "fy_token_create() failed\n"); + goto err_out; + } + *fyt_anchorp = fyt; + fyt = NULL; + fy_input_unref(fyi); + fyi = NULL; + } + + if (fyt_tagp && (value = va_arg(ap, const char *)) != NULL) { + + len = strlen(value); + data = malloc(len + 1); + if (!data) { + fy_error(diag, "malloc() failed\n"); + goto err_out; + } + memcpy(data, value, len); + /* always NULL terminate */ + data[len] = '\0'; + + rc = fy_tag_scan(data, len, &info); + if (rc) { + fy_error(diag, "invalid tag %s (tag_scan)\n", value); + goto err_out; + } + + fyt_td = fy_document_state_lookup_tag_directive(fyds, data + info.prefix_length, info.handle_length); + if (!fyt_td) { + fy_error(diag, "invalid tag %s (lookup tag directive)\n", value); + goto err_out; + } + + fyi = fy_input_from_malloc_data(data, len, &handle, true); + if (!fyi) + goto err_out; + data = NULL; + + handle.style = FYAS_URI; + handle.direct_output = false; + handle.storage_hint = 0; + handle.storage_hint_valid = false; + + fyt = fy_token_create(FYTT_TAG, &handle, info.prefix_length, + info.handle_length, info.uri_length, fyt_td); + if (!fyt) { + fy_error(diag, "fy_token_create() failed\n"); + goto err_out; + } + *fyt_tagp = fyt; + fyt = NULL; + fy_input_unref(fyi); + fyi = NULL; + } + + return fyep; + +err_out: + fy_input_unref(fyi); + if (data) + free(data); + fy_document_state_unref(fyds_new); + /* don't bother with recycling on error */ + fy_eventp_free(fyep); + return NULL; +} + +struct fy_eventp * +fy_eventp_create_internal(struct fy_eventp_list *recycled_list, struct fy_diag *diag, + struct fy_document_state *fyds, + enum fy_event_type type, ...) +{ + struct fy_eventp *fyep; + va_list ap; + + va_start(ap, type); + fyep = fy_eventp_vcreate_internal(recycled_list, diag, fyds, type, ap); + va_end(ap); + + return fyep; +} + +struct fy_event * +fy_emit_event_vcreate(struct fy_emitter *emit, enum fy_event_type type, va_list ap) +{ + struct fy_eventp *fyep; + + if (!emit) + return NULL; + + fyep = fy_eventp_vcreate_internal(emit->recycled_eventp_list, emit->diag, emit->fyds, type, ap); + if (!fyep) + return NULL; + + return &fyep->e; +} + +struct fy_event * +fy_emit_event_create(struct fy_emitter *emit, enum fy_event_type type, ...) +{ + struct fy_event *fye; + va_list ap; + + va_start(ap, type); + fye = fy_emit_event_vcreate(emit, type, ap); + va_end(ap); + + return fye; +} + +int +fy_emit_vevent(struct fy_emitter *emit, enum fy_event_type type, va_list ap) +{ + struct fy_event *fye; + + if (!emit) + return -1; + + fye = fy_emit_event_vcreate(emit, type, ap); + if (!fye) + return -1; + + return fy_emit_event(emit, fye); +} + +int +fy_emit_eventf(struct fy_emitter *emit, enum fy_event_type type, ...) +{ + int rc; + va_list ap; + + va_start(ap, type); + rc = fy_emit_vevent(emit, type, ap); + va_end(ap); + + return rc; +} + +int +fy_emit_scalar_write(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *text, size_t len) +{ + return fy_emit_eventf(fye, FYET_SCALAR, style, text, len, anchor, tag); +} + +int +fy_emit_scalar_vprintf(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *fmt, va_list ap) +{ + char *buf; + size_t len; + int rc; + + rc = vasprintf(&buf, fmt, ap); + if (rc < 0) + return -1; + len = (size_t)rc; + + rc = fy_emit_scalar_write(fye, style, anchor, tag, buf, len); + free(buf); + + return rc; +} + +int +fy_emit_scalar_printf(struct fy_emitter *fye, enum fy_scalar_style style, + const char *anchor, const char *tag, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_emit_scalar_vprintf(fye, style, anchor, tag, fmt, ap); + va_end(ap); + + return rc; +} + +struct fy_event * +fy_parse_event_vcreate(struct fy_parser *fyp, enum fy_event_type type, va_list ap) +{ + struct fy_eventp *fyep; + + if (!fyp) + return NULL; + + fyep = fy_eventp_vcreate_internal(fyp->recycled_eventp_list, fyp->diag, fyp->current_document_state, type, ap); + if (!fyep) + return NULL; + + return &fyep->e; +} + +struct fy_event * +fy_parse_event_create(struct fy_parser *fyp, enum fy_event_type type, ...) +{ + struct fy_event *fye; + va_list ap; + + va_start(ap, type); + fye = fy_parse_event_vcreate(fyp, type, ap); + va_end(ap); + + return fye; +} + +bool fy_event_is_implicit(struct fy_event *fye) +{ + /* NULL event is implicit */ + if (!fye) + return true; + + switch (fye->type) { + + case FYET_DOCUMENT_START: + return fye->document_start.implicit; + + case FYET_DOCUMENT_END: + return fye->document_end.implicit; + + case FYET_MAPPING_START: + case FYET_MAPPING_END: + case FYET_SEQUENCE_START: + case FYET_SEQUENCE_END: + return fy_event_get_node_style(fye) == FYNS_BLOCK; + + default: + break; + } + + return false; +} + +bool fy_document_event_is_implicit(const struct fy_event *fye) +{ + if (fye->type == FYET_DOCUMENT_START) + return fye->document_start.implicit; + + if (fye->type == FYET_DOCUMENT_END) + return fye->document_end.implicit; + + return false; +} + +struct fy_token *fy_event_get_token(struct fy_event *fye) +{ + if (!fye) + return NULL; + + switch (fye->type) { + case FYET_NONE: + break; + + case FYET_STREAM_START: + return fye->stream_start.stream_start; + + case FYET_STREAM_END: + return fye->stream_end.stream_end; + + case FYET_DOCUMENT_START: + return fye->document_start.document_start; + + case FYET_DOCUMENT_END: + return fye->document_end.document_end; + + case FYET_MAPPING_START: + return fye->mapping_start.mapping_start; + + case FYET_MAPPING_END: + return fye->mapping_end.mapping_end; + + case FYET_SEQUENCE_START: + return fye->sequence_start.sequence_start; + + case FYET_SEQUENCE_END: + return fye->sequence_end.sequence_end; + + case FYET_SCALAR: + return fye->scalar.value; + + case FYET_ALIAS: + return fye->alias.anchor; + + } + + return NULL; +} + +struct fy_token *fy_event_get_anchor_token(struct fy_event *fye) +{ + if (!fye) + return NULL; + + switch (fye->type) { + case FYET_MAPPING_START: + return fye->mapping_start.anchor; + case FYET_SEQUENCE_START: + return fye->sequence_start.anchor; + case FYET_SCALAR: + return fye->scalar.anchor; + default: + break; + } + + return NULL; +} + +struct fy_token *fy_event_get_and_clear_anchor_token(struct fy_event *fye) +{ + struct fy_token *fyt = NULL; + + if (!fye) + return NULL; + + switch (fye->type) { + case FYET_MAPPING_START: + fyt = fye->mapping_start.anchor; + fye->mapping_start.anchor = NULL; + break; + + case FYET_SEQUENCE_START: + fyt = fye->sequence_start.anchor; + fye->sequence_start.anchor = NULL; + break; + + case FYET_SCALAR: + fyt = fye->scalar.anchor; + fye->scalar.anchor = NULL; + break; + + default: + break; + } + + return fyt; +} + +struct fy_token *fy_event_get_tag_token(struct fy_event *fye) +{ + if (!fye) + return NULL; + + switch (fye->type) { + case FYET_MAPPING_START: + return fye->mapping_start.tag; + case FYET_SEQUENCE_START: + return fye->sequence_start.tag; + case FYET_SCALAR: + return fye->scalar.tag; + default: + break; + } + + return NULL; +} + +const struct fy_mark *fy_event_start_mark(struct fy_event *fye) +{ + if (!fye) + return NULL; + + switch (fye->type) { + case FYET_NONE: + break; + + case FYET_STREAM_START: + return fy_token_start_mark(fye->stream_start.stream_start); + + case FYET_STREAM_END: + return fy_token_start_mark(fye->stream_end.stream_end); + + case FYET_DOCUMENT_START: + return fy_token_start_mark(fye->document_start.document_start); + + case FYET_DOCUMENT_END: + return fy_token_start_mark(fye->document_end.document_end); + + case FYET_MAPPING_START: + return fy_token_start_mark(fye->mapping_start.mapping_start); + + case FYET_MAPPING_END: + return fy_token_start_mark(fye->mapping_end.mapping_end); + + case FYET_SEQUENCE_START: + return fy_token_start_mark(fye->sequence_start.sequence_start); + + case FYET_SEQUENCE_END: + return fy_token_start_mark(fye->sequence_end.sequence_end); + + case FYET_SCALAR: + return fy_token_start_mark(fye->scalar.value); + + case FYET_ALIAS: + return fy_token_start_mark(fye->alias.anchor); + + } + + return NULL; +} + +const struct fy_mark *fy_event_end_mark(struct fy_event *fye) +{ + if (!fye) + return NULL; + + switch (fye->type) { + case FYET_NONE: + break; + + case FYET_STREAM_START: + return fy_token_end_mark(fye->stream_start.stream_start); + + case FYET_STREAM_END: + return fy_token_end_mark(fye->stream_end.stream_end); + + case FYET_DOCUMENT_START: + return fy_token_end_mark(fye->document_start.document_start); + + case FYET_DOCUMENT_END: + return fy_token_end_mark(fye->document_end.document_end); + + case FYET_MAPPING_START: + return fy_token_end_mark(fye->mapping_start.mapping_start); + + case FYET_MAPPING_END: + return fy_token_end_mark(fye->mapping_end.mapping_end); + + case FYET_SEQUENCE_START: + return fy_token_end_mark(fye->sequence_start.sequence_start); + + case FYET_SEQUENCE_END: + return fy_token_end_mark(fye->sequence_end.sequence_end); + + case FYET_SCALAR: + return fy_token_end_mark(fye->scalar.value); + + case FYET_ALIAS: + return fy_token_end_mark(fye->alias.anchor); + + } + + return NULL; +} + +enum fy_node_style +fy_event_get_node_style(struct fy_event *fye) +{ + struct fy_token *fyt; + + fyt = fy_event_get_token(fye); + if (!fyt) + return FYNS_ANY; + + switch (fye->type) { + /* unstyled events */ + case FYET_NONE: + case FYET_STREAM_START: + case FYET_STREAM_END: + case FYET_DOCUMENT_START: + case FYET_DOCUMENT_END: + return FYNS_ANY; + + case FYET_MAPPING_START: + return fyt && fyt->type == FYTT_FLOW_MAPPING_START ? FYNS_FLOW : FYNS_BLOCK; + + case FYET_MAPPING_END: + return fyt && fyt->type == FYTT_FLOW_MAPPING_END ? FYNS_FLOW : FYNS_BLOCK; + + case FYET_SEQUENCE_START: + return fyt && fyt->type == FYTT_FLOW_SEQUENCE_START ? FYNS_FLOW : FYNS_BLOCK; + + case FYET_SEQUENCE_END: + return fyt && fyt->type == FYTT_FLOW_SEQUENCE_END ? FYNS_FLOW : FYNS_BLOCK; + + case FYET_SCALAR: + return fyt ? fy_node_style_from_scalar_style(fyt->scalar.style) : FYNS_PLAIN; + + case FYET_ALIAS: + return FYNS_ALIAS; + + } + + return FYNS_ANY; +} + +struct fy_eventp *fy_parse_eventp_clone(struct fy_parser *fyp, struct fy_eventp *fyep_src, bool strip_anchors) +{ + struct fy_eventp *fyep = NULL; + struct fy_event *fye, *fye_src; + + if (!fyp || !fyep_src) + return NULL; + + if (fyp->recycled_eventp_list) + fyep = fy_eventp_list_pop(fyp->recycled_eventp_list); + if (!fyep) + fyep = fy_eventp_alloc(); + if (!fyep) + return NULL; + + fye_src = &fyep_src->e; + fye = &fyep->e; + + fye->type = fye_src->type; + switch (fye->type) { + case FYET_NONE: + break; + case FYET_STREAM_START: + fye->stream_start.stream_start = fy_token_ref(fye_src->stream_start.stream_start); + break; + case FYET_STREAM_END: + fye->stream_end.stream_end = fy_token_ref(fye_src->stream_end.stream_end); + break; + case FYET_DOCUMENT_START: + fye->document_start.document_start = fy_token_ref(fye_src->document_start.document_start); + fye->document_start.document_state = fy_document_state_ref(fye_src->document_start.document_state); + break; + case FYET_DOCUMENT_END: + fye->document_end.document_end = fy_token_ref(fye_src->document_end.document_end); + break; + case FYET_MAPPING_START: + if (!strip_anchors) + fye->mapping_start.anchor = fy_token_ref(fye_src->mapping_start.anchor); + else + fye->mapping_start.anchor = NULL; + fye->mapping_start.tag = fy_token_ref(fye_src->mapping_start.tag); + fye->mapping_start.mapping_start = fy_token_ref(fye_src->mapping_start.mapping_start); + break; + case FYET_MAPPING_END: + fye->mapping_end.mapping_end = fy_token_ref(fye_src->mapping_end.mapping_end); + break; + case FYET_SEQUENCE_START: + if (!strip_anchors) + fye->sequence_start.anchor = fy_token_ref(fye_src->sequence_start.anchor); + else + fye->sequence_start.anchor = NULL; + fye->sequence_start.tag = fy_token_ref(fye_src->sequence_start.tag); + fye->sequence_start.sequence_start = fy_token_ref(fye_src->sequence_start.sequence_start); + break; + case FYET_SEQUENCE_END: + fye->sequence_end.sequence_end = fy_token_ref(fye_src->sequence_end.sequence_end); + break; + case FYET_SCALAR: + if (!strip_anchors) + fye->scalar.anchor = fy_token_ref(fye_src->scalar.anchor); + else + fye->scalar.anchor = NULL; + fye->scalar.tag = fy_token_ref(fye_src->scalar.tag); + fye->scalar.value = fy_token_ref(fye_src->scalar.value); + break; + case FYET_ALIAS: + fye->alias.anchor = fy_token_ref(fye_src->alias.anchor); + break; + } + return fyep; +} + +const char *fy_event_get_anchor(struct fy_event *fye, size_t *anchor_lenp) +{ + struct fy_token *anchor; + + anchor = fy_event_get_anchor_token(fye); + if (!anchor) + return NULL; + + return fy_token_get_text(anchor, anchor_lenp); +} + +const struct fy_version * +fy_document_start_event_version(struct fy_event *fye) +{ + /* return the default if not set */ + if (!fye || fye->type != FYET_DOCUMENT_START) + return &fy_default_version; + return fy_document_state_version(fye->document_start.document_state); +} + +struct fy_eventp * +fy_document_iterator_eventp_alloc(struct fy_document_iterator *fydi) +{ + struct fy_eventp *fyep = NULL; + + if (!fydi) + return NULL; + + if (fydi->recycled_eventp_list) + fyep = fy_eventp_list_pop(fydi->recycled_eventp_list); + if (!fyep) + fyep = fy_eventp_alloc(); + if (!fyep) + return NULL; + + fyep->e.type = FYET_NONE; + + return fyep; +} + +void fy_document_iterator_eventp_clean(struct fy_document_iterator *fydi, struct fy_eventp *fyep) +{ + if (!fydi || !fyep) + return; + + fy_eventp_clean_rl(fydi->recycled_token_list, fyep); +} + +void fy_document_iterator_eventp_recycle(struct fy_document_iterator *fydi, struct fy_eventp *fyep) +{ + if (!fydi || !fyep) + return; + + /* clean, safe to do */ + fy_document_iterator_eventp_clean(fydi, fyep); + + if (fydi->recycled_eventp_list) + fy_eventp_list_push(fydi->recycled_eventp_list, fyep); + else + fy_eventp_free(fyep); +} + +struct fy_event * +fy_document_iterator_event_vcreate(struct fy_document_iterator *fydi, enum fy_event_type type, va_list ap) +{ + struct fy_eventp *fyep; + + if (!fydi) + return NULL; + + fyep = fy_eventp_vcreate_internal(fydi->recycled_eventp_list, + fydi->fyd ? fydi->fyd->diag : NULL, + fydi->fyd ? fydi->fyd->fyds : NULL, + type, ap); + if (!fyep) + return NULL; + + return &fyep->e; +} + +struct fy_event * +fy_document_iterator_event_create(struct fy_document_iterator *fydi, enum fy_event_type type, ...) +{ + struct fy_event *fye; + va_list ap; + + va_start(ap, type); + fye = fy_document_iterator_event_vcreate(fydi, type, ap); + va_end(ap); + + return fye; +} + +void fy_document_iterator_event_free(struct fy_document_iterator *fydi, struct fy_event *fye) +{ + struct fy_eventp *fyep; + + if (!fydi || !fye) + return; + + fyep = container_of(fye, struct fy_eventp, e); + + fy_document_iterator_eventp_recycle(fydi, fyep); +} diff --git a/src/lib/fy-event.h b/src/lib/fy-event.h new file mode 100644 index 0000000..90f8241 --- /dev/null +++ b/src/lib/fy-event.h @@ -0,0 +1,54 @@ +/* + * fy-event.h - YAML parser private event definition + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_EVENT_H +#define FY_EVENT_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include + +#include "fy-list.h" +#include "fy-typelist.h" + +/* private event type */ +FY_TYPE_FWD_DECL_LIST(eventp); +struct fy_eventp { + struct list_head node; + struct fy_event e; +}; +FY_TYPE_DECL_LIST(eventp); + +struct fy_eventp *fy_eventp_alloc(void); +void fy_eventp_free(struct fy_eventp *fyep); + +/* called from internal emitter */ +void fy_eventp_release(struct fy_eventp *fyep); + +struct fy_eventp *fy_parse_eventp_alloc(struct fy_parser *fyp); +void fy_parse_eventp_recycle(struct fy_parser *fyp, struct fy_eventp *fyep); + +struct fy_eventp *fy_emit_eventp_alloc(struct fy_emitter *fye); +void fy_emit_eventp_recycle(struct fy_emitter *emit, struct fy_eventp *fyep); + +struct fy_eventp *fy_parse_eventp_clone(struct fy_parser *fyp, struct fy_eventp *fyep_src, bool strip_anchors); +struct fy_token *fy_event_get_and_clear_anchor_token(struct fy_event *fye); +const char *fy_event_get_anchor(struct fy_event *fye, size_t *anchor_lenp); + +struct fy_document_iterator; + +struct fy_eventp *fy_document_iterator_eventp_alloc(struct fy_document_iterator *fydi); +void fy_document_iterator_eventp_recycle(struct fy_document_iterator *fydi, struct fy_eventp *fyep); +struct fy_event *fy_document_iterator_event_create(struct fy_document_iterator *document_iterator, enum fy_event_type type, ...); +struct fy_event *fy_document_iterator_event_vcreate(struct fy_document_iterator *document_iterator, enum fy_event_type type, va_list ap); +void fy_document_iterator_event_free(struct fy_document_iterator *document_iterator, struct fy_event *fye); + +#endif diff --git a/src/lib/fy-input-diag.c b/src/lib/fy-input-diag.c new file mode 100644 index 0000000..46baf03 --- /dev/null +++ b/src/lib/fy-input-diag.c @@ -0,0 +1,92 @@ +/* + * fy-input-diag.c - input and reader diagnostics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-diag.h" +#include "fy-input.h" + +int fy_reader_vdiag(struct fy_reader *fyr, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap) +{ + struct fy_diag_ctx fydc; + int fydc_level, fyd_level; + + if (!fyr || !fyr->diag || !fmt) + return -1; + + /* perform the enable tests early to avoid the overhead */ + fydc_level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; + fyd_level = fyr->diag->cfg.level; + + if (fydc_level < fyd_level) + return 0; + + /* fill in fy_diag_ctx */ + memset(&fydc, 0, sizeof(fydc)); + + fydc.level = fydc_level; + fydc.module = FYEM_SCAN; /* reader is always scanner */ + fydc.source_file = file; + fydc.source_line = line; + fydc.source_func = func; + fydc.line = fyr->line; + fydc.column = fyr->column; + + return fy_vdiag(fyr->diag, &fydc, fmt, ap); +} + +int fy_reader_diag(struct fy_reader *fyr, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_reader_vdiag(fyr, flags, file, line, func, fmt, ap); + va_end(ap); + + return rc; +} + +void fy_reader_diag_vreport(struct fy_reader *fyr, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap) +{ + if (!fyr || !fyr->diag || !fydrc || !fmt) + return; + + fy_diag_vreport(fyr->diag, fydrc, fmt, ap); +} + +void fy_reader_diag_report(struct fy_reader *fyr, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_reader_diag_vreport(fyr, fydrc, fmt, ap); + va_end(ap); +} diff --git a/src/lib/fy-input.c b/src/lib/fy-input.c new file mode 100644 index 0000000..da7665d --- /dev/null +++ b/src/lib/fy-input.c @@ -0,0 +1,1222 @@ +/* + * fy-input.c - YAML input methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" +#include "fy-ctype.h" + +#include "fy-input.h" + +/* amount of multiplication of page size for CHOP size + * for a 4K page this is 64K blocks + */ +#ifndef FYI_CHOP_MULT +#define FYI_CHOP_MULT 16 +#endif + +struct fy_input *fy_input_alloc(void) +{ + struct fy_input *fyi; + + fyi = malloc(sizeof(*fyi)); + if (!fyi) + return NULL; + memset(fyi, 0, sizeof(*fyi)); + + fyi->state = FYIS_NONE; + fyi->refs = 1; + + return fyi; +} + +void fy_input_free(struct fy_input *fyi) +{ + if (!fyi) + return; + + assert(fyi->refs == 1); + + switch (fyi->state) { + case FYIS_NONE: + case FYIS_QUEUED: + /* nothing to do */ + break; + case FYIS_PARSE_IN_PROGRESS: + case FYIS_PARSED: + fy_input_close(fyi); + break; + } + + /* always release the memory of the alloc memory */ + switch (fyi->cfg.type) { + case fyit_alloc: + free(fyi->cfg.alloc.data); + break; + + default: + break; + } + if (fyi->name) + free(fyi->name); + + free(fyi); +} + +const char *fy_input_get_filename(struct fy_input *fyi) +{ + if (!fyi) + return NULL; + + return fyi->name; +} + +static void fy_input_from_data_setup(struct fy_input *fyi, + struct fy_atom *handle, bool simple) +{ + const char *data; + size_t size; + unsigned int aflags; + + /* this is an internal method, you'd better to pass garbage */ + data = fy_input_start(fyi); + size = fy_input_size(fyi); + + fyi->buffer = NULL; + fyi->allocated = 0; + fyi->read = 0; + fyi->chunk = 0; + fyi->chop = 0; + fyi->fp = NULL; + + if (!handle) + goto out; + + memset(handle, 0, sizeof(*handle)); + + if (size > 0) + aflags = fy_analyze_scalar_content(data, size, + false, fylb_cr_nl, fyfws_space_tab); /* hardcoded yaml mode */ + else + aflags = FYACF_EMPTY | FYACF_FLOW_PLAIN | FYACF_BLOCK_PLAIN | FYACF_SIZE0; + + handle->start_mark.input_pos = 0; + handle->start_mark.line = 0; + handle->start_mark.column = 0; + handle->end_mark.input_pos = size; + handle->end_mark.line = 0; + handle->end_mark.column = fy_utf8_count(data, size); + /* if it's plain, all is good */ + if ((simple || (aflags & FYACF_FLOW_PLAIN)) && + !(aflags & (FYACF_LB | FYACF_ENDS_WITH_COLON))) { + handle->storage_hint = size; /* maximum */ + handle->storage_hint_valid = false; + handle->direct_output = !!(aflags & FYACF_JSON_ESCAPE); + handle->style = FYAS_PLAIN; + } else { + handle->storage_hint = 0; /* just calculate */ + handle->storage_hint_valid = false; + handle->direct_output = false; + handle->style = FYAS_DOUBLE_QUOTED_MANUAL; + } + handle->empty = !!(aflags & FYACF_EMPTY); + handle->has_lb = !!(aflags & FYACF_LB); + handle->has_ws = !!(aflags & FYACF_WS); + handle->starts_with_ws = !!(aflags & FYACF_STARTS_WITH_WS); + handle->starts_with_lb = !!(aflags & FYACF_STARTS_WITH_LB); + handle->ends_with_ws = !!(aflags & FYACF_ENDS_WITH_WS); + handle->ends_with_lb = !!(aflags & FYACF_ENDS_WITH_LB); + handle->trailing_lb = !!(aflags & FYACF_TRAILING_LB); + handle->size0 = !!(aflags & FYACF_SIZE0); + handle->valid_anchor = !!(aflags & FYACF_VALID_ANCHOR); + + handle->chomp = FYAC_STRIP; + handle->increment = 0; + handle->fyi = fyi; + handle->fyi_generation = fyi->generation; + handle->tabsize = 0; + handle->json_mode = false; /* XXX hardcoded */ + handle->lb_mode = fylb_cr_nl; + handle->fws_mode = fyfws_space_tab; + handle->directive0_mode = false; +out: + fyi->state = FYIS_PARSED; +} + +struct fy_input *fy_input_from_data(const char *data, size_t size, + struct fy_atom *handle, bool simple) +{ + struct fy_input *fyi; + + if (data && size == (size_t)-1) + size = strlen(data); + + fyi = fy_input_alloc(); + if (!fyi) + return NULL; + + fyi->cfg.type = fyit_memory; + fyi->cfg.userdata = NULL; + fyi->cfg.memory.data = data; + fyi->cfg.memory.size = size; + + fy_input_from_data_setup(fyi, handle, simple); + + return fyi; +} + +struct fy_input *fy_input_from_malloc_data(char *data, size_t size, + struct fy_atom *handle, bool simple) +{ + struct fy_input *fyi; + + if (data && size == (size_t)-1) + size = strlen(data); + + fyi = fy_input_alloc(); + if (!fyi) + return NULL; + + fyi->cfg.type = fyit_alloc; + fyi->cfg.userdata = NULL; + fyi->cfg.alloc.data = data; + fyi->cfg.alloc.size = size; + + fy_input_from_data_setup(fyi, handle, simple); + + return fyi; +} + +void fy_input_close(struct fy_input *fyi) +{ + if (!fyi) + return; + + switch (fyi->cfg.type) { + + case fyit_file: + case fyit_fd: + + if (fyi->addr) { + munmap(fyi->addr, fyi->length); + fyi->addr = NULL; + } + + if (fyi->fd != -1) { + if (!fyi->cfg.no_close_fd) + close(fyi->fd); + fyi->fd = -1; + } + + if (fyi->buffer) { + free(fyi->buffer); + fyi->buffer = NULL; + } + if (fyi->fp) { + if (!fyi->cfg.no_fclose_fp) + fclose(fyi->fp); + fyi->fp = NULL; + } + break; + + case fyit_stream: + case fyit_callback: + if (fyi->buffer) { + free(fyi->buffer); + fyi->buffer = NULL; + } + break; + + case fyit_memory: + /* nothing */ + break; + + case fyit_alloc: + /* nothing */ + break; + + case fyit_dociter: + /* nothing */ + break; + + default: + break; + } +} + +ssize_t fy_input_estimate_queued_size(const struct fy_input *fyi) +{ + struct stat sb; + int fd, rc; + + if (!fyi || fyi->state != FYIS_QUEUED) + return 0; + + memset(&sb, 0, sizeof(sb)); + + switch (fyi->cfg.type) { + case fyit_file: + rc = stat(fyi->cfg.file.filename, &sb); + if (rc) + return -1; + break; + + case fyit_stream: + fd = fileno(fyi->cfg.stream.fp); + if (fd < 0) + return -1; + rc = fstat(fd, &sb); + if (rc) + return -1; + break; + + case fyit_memory: + return (ssize_t)fyi->cfg.memory.size; + + case fyit_alloc: + return (ssize_t)fyi->cfg.alloc.size; + + case fyit_fd: + rc = fstat(fyi->cfg.fd.fd, &sb); + if (rc) + return -1; + break; + + case fyit_callback: + case fyit_dociter: + default: + return SSIZE_MAX; /* cannot determine */ + } + + /* only do it for regular files */ + if ((sb.st_mode & S_IFMT) != S_IFREG) + return SSIZE_MAX; + + /* check for very impossible roll-over */ + if ((size_t)sb.st_size > (size_t)SSIZE_MAX) + return SSIZE_MAX; + + /* ok, we did find it */ + return (ssize_t)sb.st_size; +} + +struct fy_diag *fy_reader_get_diag(struct fy_reader *fyr) +{ + if (fyr && fyr->ops && fyr->ops->get_diag) + return fyr->ops->get_diag(fyr); + + return NULL; +} + +int fy_reader_file_open(struct fy_reader *fyr, const char *filename) +{ + if (!fyr || !filename) + return -1; + + if (fyr->ops && fyr->ops->file_open) + return fyr->ops->file_open(fyr, filename); + + return open(filename, O_RDONLY); +} + +void fy_reader_reset(struct fy_reader *fyr) +{ + const struct fy_reader_ops *ops; + struct fy_diag *diag; + + if (!fyr) + return; + + ops = fyr->ops; + diag = fyr->diag; + + fy_input_unref(fyr->current_input); + + memset(fyr, 0, sizeof(*fyr)); + + /* by default we're always in yaml mode */ + fyr->mode = fyrm_yaml; + fyr->ops = ops; + fyr->diag = diag; +} + +void fy_reader_setup(struct fy_reader *fyr, const struct fy_reader_ops *ops) +{ + if (!fyr) + return; + + fyr->ops = ops; + fyr->diag = fy_reader_get_diag(fyr); + fyr->current_input = NULL; + fy_reader_reset(fyr); +} + +void fy_reader_cleanup(struct fy_reader *fyr) +{ + if (!fyr) + return; + + fy_input_unref(fyr->current_input); + fyr->current_input = NULL; + fy_reader_reset(fyr); +} + +void fy_reader_apply_mode(struct fy_reader *fyr) +{ + struct fy_input *fyi; + + assert(fyr); + + /* set input mode from the current reader settings */ + switch (fyr->mode) { + case fyrm_yaml: + fyr->json_mode = false; + fyr->lb_mode = fylb_cr_nl; + fyr->fws_mode = fyfws_space_tab; + fyr->directive0_mode = false; + break; + case fyrm_json: + fyr->json_mode = true; + fyr->lb_mode = fylb_cr_nl; + fyr->fws_mode = fyfws_space; + fyr->directive0_mode = false; + break; + case fyrm_yaml_1_1: + fyr->json_mode = false; + fyr->lb_mode = fylb_cr_nl_N_L_P; + fyr->fws_mode = fyfws_space_tab; + fyr->directive0_mode = true; + break; + } + fyi = fyr->current_input; + if (fyi) { + fyi->json_mode = fyr->json_mode; + fyi->lb_mode = fyr->lb_mode; + fyi->fws_mode = fyr->fws_mode; + fyi->directive0_mode = fyr->directive0_mode; + } +} + +int fy_reader_input_open(struct fy_reader *fyr, struct fy_input *fyi, const struct fy_reader_input_cfg *icfg) +{ + struct stat sb; + struct fy_document_iterator *fydi; + int rc; + + if (!fyi) + return -1; + + /* unref any previous input */ + fy_input_unref(fyr->current_input); + fyr->current_input = fy_input_ref(fyi); + + fy_reader_apply_mode(fyr); + + if (!icfg) + memset(&fyr->current_input_cfg, 0, sizeof(fyr->current_input_cfg)); + else + fyr->current_input_cfg = *icfg; + + /* reset common data */ + fyi->buffer = NULL; + fyi->allocated = 0; + fyi->read = 0; + fyi->chunk = 0; + fyi->chop = 0; + fyi->fp = NULL; + + switch (fyi->cfg.type) { + + case fyit_file: + case fyit_fd: + + switch (fyi->cfg.type) { + case fyit_file: + fyi->fd = fy_reader_file_open(fyr, fyi->cfg.file.filename); + fyr_error_check(fyr, fyi->fd != -1, err_out, + "failed to open %s", fyi->cfg.file.filename); + break; + + case fyit_fd: + fyi->fd = fyi->cfg.fd.fd; + fyr_error_check(fyr, fyi->fd >= 0, err_out, + "bad file.fd %d", fyi->cfg.fd.fd); + break; + default: + FY_IMPOSSIBLE_ABORT(); + } + + rc = fstat(fyi->fd, &sb); + fyr_error_check(fyr, rc != -1, err_out, + "failed to fstat %s", fyi->cfg.file.filename); + + fyi->length = sb.st_size; + + /* only map if not zero (and is not disabled) */ + if (sb.st_size > 0 && !fyr->current_input_cfg.disable_mmap_opt) { + fyi->addr = mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fyi->fd, 0); + + /* convert from MAP_FAILED to NULL */ + if (fyi->addr == MAP_FAILED) { + fyr_debug(fyr, "mmap failed for file %s", + fyi->cfg.file.filename); + fyi->addr = NULL; + } + } + /* if we've managed to mmap, we' good */ + if (fyi->addr) + break; + + /* if we're not ignoring stdio, open a FILE* using the fd */ + if (!fyi->cfg.ignore_stdio) { + fyi->fp = fdopen(fyi->fd, "r"); + fyr_error_check(fyr, rc != -1, err_out, "failed to fdopen %s", fyi->name); + } else + fyi->fp = NULL; + + break; + + case fyit_stream: + if (!fyi->cfg.ignore_stdio) + fyi->fp = fyi->cfg.stream.fp; + else + fyi->fd = fileno(fyi->cfg.stream.fp); + break; + + case fyit_memory: + /* nothing to do for memory */ + break; + + case fyit_alloc: + /* nothing to do for memory */ + break; + + case fyit_callback: + break; + + case fyit_dociter: + fydi = fyi->cfg.dociter.fydi; + fyr_error_check(fyr, fydi, err_out, + "missing document iterator"); + break; + + default: + FY_IMPOSSIBLE_ABORT(); + } + + switch (fyi->cfg.type) { + + /* those two need no memory */ + case fyit_memory: + case fyit_alloc: + case fyit_dociter: + break; + + /* all the rest need it */ + default: + /* if we're not in mmap mode */ + if (fyi->addr && !fyr->current_input_cfg.disable_mmap_opt) + break; + + fyi->chunk = fyi->cfg.chunk; + if (!fyi->chunk) + fyi->chunk = sysconf(_SC_PAGESIZE); + fyi->chop = fyi->chunk * FYI_CHOP_MULT; + fyi->buffer = malloc(fyi->chunk); + fyr_error_check(fyr, fyi->buffer, err_out, + "fy_alloc() failed"); + fyi->allocated = fyi->chunk; + break; + } + + fyr->this_input_start = 0; + fyr->line = 0; + fyr->column = 0; + fyr->current_ptr = NULL; + fyr->current_ptr_start = NULL; + fyr->current_ptr_end = NULL; + + fyi->state = FYIS_PARSE_IN_PROGRESS; + + return 0; + +err_out: + fy_input_close(fyi); + return -1; +} + +int fy_reader_input_done(struct fy_reader *fyr) +{ + struct fy_input *fyi; + void *buf; + size_t current_input_pos; + + if (!fyr) + return -1; + + fyi = fyr->current_input; + if (!fyi) + return 0; + + current_input_pos = fy_reader_current_input_pos(fyr); + + switch (fyi->cfg.type) { + case fyit_file: + case fyit_fd: + if (fyi->addr) + break; + + /* fall-through */ + + case fyit_stream: + case fyit_callback: + /* chop extra buffer */ + buf = realloc(fyi->buffer, current_input_pos); + fyr_error_check(fyr, buf || !current_input_pos, err_out, + "realloc() failed"); + + /* increate input generation; required for direct input to work */ + if (fyi->buffer != buf) { + fyi->buffer = buf; + fyi->generation++; + } + + fyi->allocated = current_input_pos; + + break; + + default: + break; + + } + + fyi->state = FYIS_PARSED; + fy_input_unref(fyi); + fyr->current_input = NULL; + + return 0; + +err_out: + return -1; +} + +int fy_reader_input_scan_token_mark_slow_path(struct fy_reader *fyr) +{ + struct fy_input *fyi, *fyi_new = NULL; + size_t current_input_pos; + + assert(fyr); + + if (!fy_reader_input_chop_active(fyr)) + return 0; + + fyi = fyr->current_input; + assert(fyi); + + fyi_new = fy_input_alloc(); + fyr_error_check(fyr, fyi_new, err_out, + "fy_input_alloc() failed\n"); + + /* copy the config over */ + fyi_new->cfg = fyi->cfg; + fyi_new->name = strdup(fyi->name); + fyr_error_check(fyr, fyi_new->name, err_out, + "strdup() failed\n"); + + fyi_new->chunk = fyi->chunk; + fyi_new->chop = fyi->chop; + fyi_new->buffer = malloc(fyi->chunk); + fyr_error_check(fyr, fyi_new->buffer, err_out, + "fy_alloc() failed"); + fyi_new->allocated = fyi->chunk; + fyi_new->fp = fyi->fp; + + fyi->fp = NULL; /* the file pointer now assigned to the new */ + + fyi_new->lb_mode = fyi->lb_mode; + fyi_new->fws_mode = fyi->fws_mode; + fyi_new->directive0_mode = fyi->directive0_mode; + + fyi_new->state = FYIS_PARSE_IN_PROGRESS; + + /* adjust and copy the left over reads */ + current_input_pos = fy_reader_current_input_pos(fyr); + assert(fyi->read >= current_input_pos); + fyi_new->read = fyi->read - current_input_pos; + if (fyi_new->read > 0) + memcpy(fyi_new->buffer, fyi->buffer + current_input_pos, fyi_new->read); + + fyr->this_input_start += current_input_pos; + + /* update the reader to point to the new input */ + fyr->current_input = fyi_new; + fyr->current_ptr = fyi_new->buffer; + fyr->current_ptr_start = fyi_new->buffer; + fyr->current_ptr_end = fyr->current_ptr_start + fyi_new->read; + + fyr_debug(fyr, "chop at this_input_start=%zu chop=%zu\n", fyr->this_input_start, fyi->chop); + + /* free the old input - while references to it exist it will hang around */ + fyi->state = FYIS_PARSED; + fy_input_unref(fyi); + fyi = NULL; + + return 0; +err_out: + fy_input_unref(fyi_new); + return -1; +} + +const void *fy_reader_ptr_slow_path(struct fy_reader *fyr, size_t *leftp) +{ + struct fy_input *fyi; + const void *p, *start; + size_t size, left, current_input_pos; + + if (fyr->current_ptr) { + if (leftp) + *leftp = fy_reader_current_left(fyr); + return fyr->current_ptr; + } + + fyi = fyr->current_input; + if (!fyi) + return NULL; + + /* tokens cannot cross boundaries */ + start = fy_input_start_size(fyi, &size); + + current_input_pos = fy_reader_current_input_pos(fyr); + left = size - current_input_pos; + assert(left <= size); + + p = start + current_input_pos; + + if (leftp) + *leftp = left; + + if (!fyr->current_ptr_start) { + fyr->current_ptr_start = start; + fyr->current_ptr_end = start + size; + } + + fyr->current_ptr = p; + + assert(current_input_pos <= size); + + return p; +} + +int fy_reader_peek_at_offset_width_slow_path(struct fy_reader *fyr, size_t offset, int *wp) +{ + const uint8_t *p; + size_t left; + int w; + + assert(fyr); + + /* ensure that the first octet at least is pulled in */ + p = fy_reader_ensure_lookahead(fyr, offset + 1, &left); + if (!p) + return FYUG_EOF; + + /* get width by first octet */ + w = fy_utf8_width_by_first_octet(p[offset]); + if (!w) + return FYUG_INV; + + /* make sure that there's enough to cover the utf8 width */ + if (offset + w > left) { + p = fy_reader_ensure_lookahead(fyr, offset + w, &left); + if (!p) + return FYUG_PARTIAL; + } + + return fy_utf8_get(p + offset, left - offset, wp); +} + +int64_t fy_reader_peek_at_offset_width_slow_path_64(struct fy_reader *fyr, size_t offset) +{ + const uint8_t *p; + size_t left; + int w; + + assert(fyr); + + /* ensure that the first octet at least is pulled in */ + p = fy_reader_ensure_lookahead(fyr, offset + 1, &left); + if (!p) + return FYUG_EOF; + + /* get width by first octet */ + w = fy_utf8_width_by_first_octet(p[offset]); + if (!w) + return FYUG_INV; + + /* make sure that there's enough to cover the utf8 width */ + if (offset + w > left) { + p = fy_reader_ensure_lookahead(fyr, offset + w, &left); + if (!p) + return FYUG_PARTIAL; + } + + return fy_utf8_get_64(p + offset, left - offset); +} + +const void *fy_reader_input_try_pull(struct fy_reader *fyr, struct fy_input *fyi, + size_t pull, size_t *leftp) +{ + const void *p; + size_t left, pos, size, nread, nreadreq, missing; + ssize_t snread, current_input_pos; + size_t space __FY_DEBUG_UNUSED__; + void *buf; + + if (!fyr || !fyi) { + if (leftp) + *leftp = 0; + return NULL; + } + + p = NULL; + left = 0; + + current_input_pos = fy_reader_current_input_pos(fyr); + pos = current_input_pos; + + switch (fyi->cfg.type) { + case fyit_file: + case fyit_fd: + + if (fyi->addr) { + assert(fyi->length >= (fyr->this_input_start + pos)); + + left = fyi->length - (fyr->this_input_start + pos); + if (!left) { + fyr_debug(fyr, "file input exhausted"); + break; + } + p = fyi->addr + pos; + break; + } + + /* fall-through */ + + case fyit_stream: + case fyit_callback: + + assert(fyi->read >= pos); + + left = fyi->read - pos; + p = fyi->buffer + pos; + + /* enough to satisfy directly */ + if (left >= pull) + break; + + /* no more */ + if (fyi->eof) { + if (!left) { + fyr_debug(fyr, "input exhausted (EOF)"); + p = NULL; + } + break; + } + + space = fyi->allocated - pos; + + /* if we're missing more than the buffer space */ + missing = pull - left; + + fyr_debug(fyr, "input: allocated=%zu read=%zu pos=%zu pull=%zu left=%zu space=%zu missing=%zu", + fyi->allocated, fyi->read, pos, pull, left, space, missing); + + if (pos + pull > fyi->allocated) { + + /* align size to chunk */ + size = fyi->allocated + missing + fyi->chunk - 1; + size = size - size % fyi->chunk; + + fyr_debug(fyr, "input buffer missing %zu bytes (pull=%zu)", missing, pull); + + buf = realloc(fyi->buffer, size); + if (!buf) { + fyr_error(fyr, "realloc() failed"); + goto err_out; + } + + fyr_debug(fyr, "input read allocated=%zu new-size=%zu", fyi->allocated, size); + + /* increase input generation; required for direct input to work */ + if (fyi->buffer != buf) { + fyi->buffer = buf; + fyi->generation++; + } + + fyi->allocated = size; + space = fyi->allocated - pos; + p = fyi->buffer + pos; + } + + /* always try to read up to the allocated space */ + do { + nreadreq = fyi->allocated - fyi->read; + assert(nreadreq > 0); + + if (fyi->cfg.type == fyit_callback) { + + fyr_debug(fyr, "performing callback request of %zu", nreadreq); + + nread = fyi->cfg.callback.input(fyi->cfg.userdata, fyi->buffer + fyi->read, nreadreq); + + fyr_debug(fyr, "callback returned %zu", nread); + + if (nread <= 0) { + if (!nread) { + fyi->eof = true; + fyr_debug(fyr, "callback got EOF"); + } else { + fyi->err = true; + fyr_debug(fyr, "callback got error"); + } + break; + } + + } else if (fyi->fp) { + + fyr_debug(fyr, "performing fread request of %zu", nreadreq); + + nread = fread(fyi->buffer + fyi->read, 1, nreadreq, fyi->fp); + + fyr_debug(fyr, "fread returned %zu", nread); + + if (nread <= 0) { + fyi->err = ferror(fyi->fp); + if (fyi->err) { + fyi->eof = true; + fyr_debug(fyr, "fread got ERROR"); + goto err_out; + } + + fyi->eof = feof(fyi->fp); + if (fyi->eof) + fyr_debug(fyr, "fread got EOF"); + nread = 0; + break; + } + + } else if (fyi->fd >= 0) { + + fyr_debug(fyr, "performing read request of %zu", nreadreq); + + do { + snread = read(fyi->fd, fyi->buffer + fyi->read, nreadreq); + } while (snread == -1 && errno == EAGAIN); + + fyr_debug(fyr, "read returned %zd", snread); + + if (snread == -1) { + fyi->err = true; + fyi->eof = true; + fyr_error(fyr, "read() failed: %s", strerror(errno)); + goto err_out; + } + + if (!snread) { + fyi->eof = true; + nread = 0; + break; + } + + nread = snread; + } else { + fyr_error(fyr, "No FILE* nor fd available?"); + fyi->eof = true; + nread = 0; + goto err_out; + } + + assert(nread > 0); + + fyi->read += nread; + left = fyi->read - pos; + + } while (left < pull); + + /* no more, move it to parsed input chunk list */ + if (!left) { + fyr_debug(fyr, "input exhausted"); + p = NULL; + } + break; + + case fyit_memory: + assert(fyi->cfg.memory.size >= pos); + + left = fyi->cfg.memory.size - pos; + if (!left) { + fyr_debug(fyr, "memory input exhausted"); + break; + } + p = fyi->cfg.memory.data + pos; + break; + + case fyit_alloc: + assert(fyi->cfg.alloc.size >= pos); + + left = fyi->cfg.alloc.size - pos; + if (!left) { + fyr_debug(fyr, "alloc input exhausted"); + break; + } + p = fyi->cfg.alloc.data + pos; + break; + + case fyit_dociter: + FY_IMPOSSIBLE_ABORT(); + + default: + FY_IMPOSSIBLE_ABORT(); + } + + if (leftp) + *leftp = left; + return p; + +err_out: + if (leftp) + *leftp = 0; + return NULL; +} + +void +fy_reader_advance_slow_path(struct fy_reader *fyr, int c) +{ + bool is_line_break = false; + + if (c < 0) + return; + + fy_reader_advance_octets(fyr, fy_utf8_width(c)); + + /* first check for CR/LF */ + if (c == '\r' && fy_reader_peek(fyr) == '\n') { + fy_reader_advance_octets(fyr, 1); + is_line_break = true; + } else if (fy_reader_is_lb(fyr, c)) + is_line_break = true; + + if (is_line_break) { + fyr->column = 0; + fyr->line++; + } else if (fyr->tabsize && fy_is_tab(c)) + fyr->column += (fyr->tabsize - (fyr->column % fyr->tabsize)); + else + fyr->column++; +} + +struct fy_input *fy_input_create(const struct fy_input_cfg *fyic) +{ + struct fy_input *fyi = NULL; + int ret; + + fyi = fy_input_alloc(); + if (!fyi) + return NULL; + fyi->cfg = *fyic; + + /* copy filename pointers and switch */ + switch (fyic->type) { + + case fyit_file: + fyi->name = strdup(fyic->file.filename); + break; + + case fyit_fd: + ret = asprintf(&fyi->name, "", fyic->fd.fd); + if (ret == -1) + fyi->name = NULL; + break; + + case fyit_stream: + if (fyic->stream.name) + fyi->name = strdup(fyic->stream.name); + else if (fyic->stream.fp == stdin) + fyi->name = strdup(""); + else { + ret = asprintf(&fyi->name, "", + fileno(fyic->stream.fp)); + if (ret == -1) + fyi->name = NULL; + } + break; + case fyit_memory: + ret = asprintf(&fyi->name, "", + fyic->memory.data, fyic->memory.data + fyic->memory.size - 1); + if (ret == -1) + fyi->name = NULL; + break; + case fyit_alloc: + ret = asprintf(&fyi->name, "", + fyic->memory.data, fyic->memory.data + fyic->memory.size - 1); + if (ret == -1) + fyi->name = NULL; + break; + case fyit_callback: + ret = asprintf(&fyi->name, ""); + if (ret == -1) + fyi->name = NULL; + break; + + case fyit_dociter: + ret = asprintf(&fyi->name, "", fyi->cfg.dociter.fydi); + if (ret == -1) + fyi->name = NULL; + break; + + default: + FY_IMPOSSIBLE_ABORT(); + } + if (!fyi->name) + goto err_out; + + fyi->buffer = NULL; + fyi->allocated = 0; + fyi->read = 0; + fyi->chunk = 0; + fyi->chop = 0; + fyi->fp = NULL; + fyi->fd = -1; + fyi->addr = NULL; + fyi->length = -1; + + /* default modes */ + fyi->lb_mode = fylb_cr_nl; + fyi->fws_mode = fyfws_space_tab; + fyi->directive0_mode = false; + + return fyi; + +err_out: + fy_input_unref(fyi); + return NULL; +} + +/* ensure that there are at least size octets available */ +const void *fy_reader_ensure_lookahead_slow_path(struct fy_reader *fyr, size_t size, size_t *leftp) +{ + const void *p, *new_start, *old_start; + size_t left, new_size, old_size; + + if (!leftp) + leftp = &left; + + p = fy_reader_ptr(fyr, leftp); + if (!p || *leftp < size) { + + if (!fyr->current_input) { + fyr_debug(fyr, "ensure lookahead size=%zd left=%zd no input left", + size, *leftp); + return NULL; + } + fyr_debug(fyr, "ensure lookahead size=%zd left=%zd (%s - %zu)", + size, *leftp, + fy_input_get_filename(fyr->current_input), + fy_reader_current_input_pos(fyr)); + + /* update with what is new */ + old_start = fy_input_start_size(fyr->current_input, &old_size); + + p = fy_reader_input_try_pull(fyr, fyr->current_input, size, leftp); + if (!p || *leftp < size) + return NULL; + + /* update with what is new */ + new_start = fy_input_start_size(fyr->current_input, &new_size); + + /* oops, input characteristics changed */ + if (old_start != new_start || old_size != new_size) { + + /* buffer just grew in place */ + if (old_start == new_start) { + fyr_debug(fyr, "buffer grew %zu -> %zu", + old_size, new_size); + } else { + fyr_debug(fyr, "buffer changed %p/%zu -> %p/%zu", + old_start, old_size, new_start, new_size); + + fyr->current_ptr_start = new_start; + } + + fyr->current_ptr_end = fyr->current_ptr_start + new_size; + } + + fyr->current_ptr = p; + } + return p; +} + +struct fy_event *fy_reader_generate_next_event(struct fy_reader *fyr) +{ + struct fy_input *fyi; + struct fy_document_iterator *fydi; + struct fy_event *fye; + + fyi = fy_reader_current_input(fyr); + if (!fyi || fyi->cfg.type != fyit_dociter) + return NULL; + + fydi = fyi->cfg.dociter.fydi; + + do { + fye = fy_document_iterator_generate_next(fydi); + if (!fye) + return NULL; + + /* remove if we don't want the event */ + if (((fye->type == FYET_STREAM_START || fye->type == FYET_STREAM_END) && + !(fyi->cfg.dociter.flags & FYPEGF_GENERATE_STREAM_EVENTS)) || + ((fye->type == FYET_DOCUMENT_START || fye->type == FYET_DOCUMENT_END) && + !(fyi->cfg.dociter.flags & FYPEGF_GENERATE_DOCUMENT_EVENTS))) { + fy_document_iterator_event_free(fydi, fye); + fye = NULL; + } + } while (!fye); + + return fye; +} + +void fy_reader_event_free(struct fy_reader *fyr, struct fy_event *fye) +{ + struct fy_input *fyi; + + fyi = fy_reader_current_input(fyr); + assert(fyi); + assert(fy_reader_generates_events(fyr)); + + assert (fyi->cfg.type == fyit_dociter); + + fy_document_iterator_event_free(fyi->cfg.dociter.fydi, fye); +} diff --git a/src/lib/fy-input.h b/src/lib/fy-input.h new file mode 100644 index 0000000..a401af2 --- /dev/null +++ b/src/lib/fy-input.h @@ -0,0 +1,983 @@ +/* + * fy-input.h - YAML input methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_INPUT_H +#define FY_INPUT_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include + +#include + +#include "fy-utils.h" +#include "fy-typelist.h" +#include "fy-ctype.h" + +#include "fy-atom.h" +#include "fy-diag.h" + +struct fy_atom; +struct fy_parser; + +enum fy_input_type { + fyit_file, + fyit_stream, + fyit_memory, + fyit_alloc, + fyit_callback, + fyit_fd, + fyit_dociter, +}; + +struct fy_input_cfg { + enum fy_input_type type; + void *userdata; + size_t chunk; + bool ignore_stdio : 1; + bool no_fclose_fp : 1; + bool no_close_fd : 1; + union { + struct { + const char *filename; + } file; + struct { + const char *name; + FILE *fp; + } stream; + struct { + const void *data; + size_t size; + } memory; + struct { + void *data; + size_t size; + } alloc; + struct { + /* negative return is error, 0 is EOF */ + ssize_t (*input)(void *user, void *buf, size_t count); + } callback; + struct { + int fd; + } fd; + struct { + enum fy_parser_event_generator_flags flags; + struct fy_document_iterator *fydi; + struct fy_document *fyd; + bool owns_iterator; + } dociter; + }; +}; + +enum fy_input_state { + FYIS_NONE, + FYIS_QUEUED, + FYIS_PARSE_IN_PROGRESS, + FYIS_PARSED, +}; + +FY_TYPE_FWD_DECL_LIST(input); +struct fy_input { + struct list_head node; + enum fy_input_state state; + struct fy_input_cfg cfg; + int refs; /* number of referers */ + char *name; + void *buffer; /* when the file can't be mmaped */ + uint64_t generation; + size_t allocated; + size_t read; + size_t chunk; + size_t chop; + FILE *fp; /* FILE* for the input if it exists */ + int fd; /* fd for file and stream */ + size_t length; /* length of file */ + void *addr; /* mmaped for files, allocated for streams */ + bool eof : 1; /* got EOF */ + bool err : 1; /* got an error */ + + /* propagated */ + bool json_mode; + enum fy_lb_mode lb_mode; + enum fy_flow_ws_mode fws_mode; + bool directive0_mode; +}; +FY_TYPE_DECL_LIST(input); + +static inline const void * +fy_input_start_size(const struct fy_input *fyi, size_t *sizep) +{ + const void *start; + size_t size; + + /* tokens cannot cross boundaries */ + switch (fyi->cfg.type) { + case fyit_file: + case fyit_fd: + if (fyi->addr) { + start = fyi->addr; + size = fyi->length; + break; + } + + /* fall-through */ + + case fyit_stream: + case fyit_callback: + start = fyi->buffer; + size = fyi->read; + break; + + case fyit_memory: + start = fyi->cfg.memory.data; + size = fyi->cfg.memory.size; + break; + + case fyit_alloc: + start = fyi->cfg.alloc.data; + size = fyi->cfg.alloc.size; + break; + + default: + start = NULL; + size = 0; + break; + } + + *sizep = size; + return start; +} + +static inline const void *fy_input_start(const struct fy_input *fyi) +{ + size_t size; + return fy_input_start_size(fyi, &size); +} + +static inline size_t fy_input_size(const struct fy_input *fyi) +{ + size_t size; + (void)fy_input_start_size(fyi, &size); + return size; +} + +struct fy_input *fy_input_alloc(void); +void fy_input_free(struct fy_input *fyi); + +static inline enum fy_input_state fy_input_get_state(struct fy_input *fyi) +{ + return fyi->state; +} + +struct fy_input *fy_input_create(const struct fy_input_cfg *fyic); + +const char *fy_input_get_filename(struct fy_input *fyi); + +struct fy_input *fy_input_from_data(const char *data, size_t size, + struct fy_atom *handle, bool simple); +struct fy_input *fy_input_from_malloc_data(char *data, size_t size, + struct fy_atom *handle, bool simple); + +void fy_input_close(struct fy_input *fyi); + +static inline struct fy_input * +fy_input_ref(struct fy_input *fyi) +{ + if (!fyi) + return NULL; + + + assert(fyi->refs + 1 > 0); + + fyi->refs++; + + return fyi; +} + +static inline void +fy_input_unref(struct fy_input *fyi) +{ + if (!fyi) + return; + + assert(fyi->refs > 0); + + if (fyi->refs == 1) + fy_input_free(fyi); + else + fyi->refs--; +} + +ssize_t fy_input_estimate_queued_size(const struct fy_input *fyi); + +struct fy_reader; + +enum fy_reader_mode { + fyrm_yaml, + fyrm_json, + fyrm_yaml_1_1, /* yaml 1.1 mode */ +}; + +struct fy_reader_ops { + struct fy_diag *(*get_diag)(struct fy_reader *fyr); + int (*file_open)(struct fy_reader *fyr, const char *filename); +}; + +struct fy_reader_input_cfg { + bool disable_mmap_opt; +}; + +struct fy_reader { + const struct fy_reader_ops *ops; + enum fy_reader_mode mode; + + struct fy_reader_input_cfg current_input_cfg; + struct fy_input *current_input; + + size_t this_input_start; /* this input start */ + const void *current_ptr; /* current pointer into the buffer */ + const void *current_ptr_start; /* the start of this input */ + const void *current_ptr_end; /* the end of this input */ + + int line; /* always on input */ + int column; + + int tabsize; /* very experimental tab size for indent purposes */ + + struct fy_diag *diag; + + /* decoded mode variables; update when changing modes */ + bool json_mode; + enum fy_lb_mode lb_mode; + enum fy_flow_ws_mode fws_mode; + bool directive0_mode; +}; + +void fy_reader_reset(struct fy_reader *fyr); +void fy_reader_setup(struct fy_reader *fyr, const struct fy_reader_ops *ops); +void fy_reader_cleanup(struct fy_reader *fyr); + +int fy_reader_input_open(struct fy_reader *fyr, struct fy_input *fyi, const struct fy_reader_input_cfg *icfg); +int fy_reader_input_done(struct fy_reader *fyr); +int fy_reader_input_scan_token_mark_slow_path(struct fy_reader *fyr); + +static FY_ALWAYS_INLINE inline size_t +fy_reader_current_input_pos(const struct fy_reader *fyr) +{ + return (size_t)(fyr->current_ptr - fyr->current_ptr_start); +} + +static FY_ALWAYS_INLINE inline size_t +fy_reader_current_left(const struct fy_reader *fyr) +{ + return (size_t)(fyr->current_ptr_end - fyr->current_ptr); +} + +static inline bool +fy_reader_input_chop_active(struct fy_reader *fyr) +{ + struct fy_input *fyi; + + assert(fyr); + + fyi = fyr->current_input; + assert(fyi); + + if (!fyi->chop) + return false; + + switch (fyi->cfg.type) { + case fyit_file: + return !fyi->addr && fyi->fp; /* non-mmap mode */ + + case fyit_stream: + case fyit_callback: + return true; + + default: + /* all the others do not support chop */ + break; + } + + return false; +} + +static inline int +fy_reader_input_scan_token_mark(struct fy_reader *fyr) +{ + /* don't chop until ready */ + if (!fy_reader_input_chop_active(fyr) || + fyr->current_input->chop > fy_reader_current_input_pos(fyr)) + return 0; + + return fy_reader_input_scan_token_mark_slow_path(fyr); +} + +const void *fy_reader_ptr_slow_path(struct fy_reader *fyr, size_t *leftp); +const void *fy_reader_ensure_lookahead_slow_path(struct fy_reader *fyr, size_t size, size_t *leftp); + +void fy_reader_apply_mode(struct fy_reader *fyr); + +static FY_ALWAYS_INLINE inline enum fy_reader_mode +fy_reader_get_mode(const struct fy_reader *fyr) +{ + assert(fyr); + return fyr->mode; +} + +static FY_ALWAYS_INLINE inline void +fy_reader_set_mode(struct fy_reader *fyr, enum fy_reader_mode mode) +{ + assert(fyr); + fyr->mode = mode; + fy_reader_apply_mode(fyr); +} + +static FY_ALWAYS_INLINE inline enum fy_reader_mode +fy_reader_calculate_mode(const struct fy_version *vers, bool json_mode) +{ + enum fy_reader_mode mode; + + if (!json_mode) + mode = fy_version_compare(vers, fy_version_make(1, 1)) <= 0 ? fyrm_yaml_1_1 : fyrm_yaml; + else + mode = fyrm_json; + + return mode; +} + +static FY_ALWAYS_INLINE inline struct fy_input * +fy_reader_current_input(const struct fy_reader *fyr) +{ + assert(fyr); + return fyr->current_input; +} + +static FY_ALWAYS_INLINE inline uint64_t +fy_reader_current_input_generation(const struct fy_reader *fyr) +{ + assert(fyr); + assert(fyr->current_input); + return fyr->current_input->generation; +} + +static FY_ALWAYS_INLINE inline int +fy_reader_column(const struct fy_reader *fyr) +{ + assert(fyr); + return fyr->column; +} + +static FY_ALWAYS_INLINE inline int +fy_reader_tabsize(const struct fy_reader *fyr) +{ + assert(fyr); + return fyr->tabsize; +} + +static FY_ALWAYS_INLINE inline int +fy_reader_line(const struct fy_reader *fyr) +{ + assert(fyr); + return fyr->line; +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_generates_events(const struct fy_reader *fyr) +{ + struct fy_input *fyi; + + fyi = fy_reader_current_input(fyr); + if (!fyi) + return false; + + return fyi->cfg.type == fyit_dociter; +} + +struct fy_event *fy_reader_generate_next_event(struct fy_reader *fyr); +void fy_reader_event_free(struct fy_reader *fyr, struct fy_event *fye); + +/* force new line at the end of stream */ +static inline void fy_reader_stream_end(struct fy_reader *fyr) +{ + assert(fyr); + + /* force new line */ + if (fyr->column) { + fyr->column = 0; + fyr->line++; + } +} + +static FY_ALWAYS_INLINE inline void +fy_reader_get_mark(struct fy_reader *fyr, struct fy_mark *fym) +{ + assert(fyr); + fym->input_pos = fy_reader_current_input_pos(fyr); + fym->line = fyr->line; + fym->column = fyr->column; +} + +static FY_ALWAYS_INLINE inline const void * +fy_reader_ptr(struct fy_reader *fyr, size_t *leftp) +{ + if (fyr->current_ptr) { + if (leftp) + *leftp = fy_reader_current_left(fyr); + return fyr->current_ptr; + } + + return fy_reader_ptr_slow_path(fyr, leftp); +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_json_mode(const struct fy_reader *fyr) +{ + assert(fyr); + return fyr->json_mode; +} + +static FY_ALWAYS_INLINE inline enum fy_lb_mode +fy_reader_lb_mode(const struct fy_reader *fyr) +{ + assert(fyr); + return fyr->lb_mode; +} + +static FY_ALWAYS_INLINE inline enum fy_flow_ws_mode +fy_reader_flow_ws_mode(const struct fy_reader *fyr) +{ + assert(fyr); + return fyr->fws_mode; +} + +static FY_ALWAYS_INLINE inline enum fy_flow_ws_mode +fy_reader_directive0_mode(const struct fy_reader *fyr) +{ + assert(fyr); + return fyr->directive0_mode; +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_is_lb(const struct fy_reader *fyr, int c) +{ + return fy_is_lb_m(c, fy_reader_lb_mode(fyr)); +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_is_lbz(const struct fy_reader *fyr, int c) +{ + return fy_is_lbz_m(c, fy_reader_lb_mode(fyr)); +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_is_blankz(const struct fy_reader *fyr, int c) +{ + return fy_is_blankz_m(c, fy_reader_lb_mode(fyr)); +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_is_generic_lb(const struct fy_reader *fyr, int c) +{ + return fy_is_generic_lb_m(c, fy_reader_lb_mode(fyr)); +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_is_generic_lbz(const struct fy_reader *fyr, int c) +{ + return fy_is_generic_lbz_m(c, fy_reader_lb_mode(fyr)); +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_is_generic_blankz(const struct fy_reader *fyr, int c) +{ + return fy_is_generic_blankz_m(c, fy_reader_lb_mode(fyr)); +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_is_flow_ws(const struct fy_reader *fyr, int c) +{ + return fy_is_flow_ws_m(c, fy_reader_flow_ws_mode(fyr)); +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_is_flow_blank(const struct fy_reader *fyr, int c) +{ + return fy_reader_is_flow_ws(fyr, c); /* same */ +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_is_flow_blankz(const struct fy_reader *fyr, int c) +{ + return fy_is_flow_ws_m(c, fy_reader_flow_ws_mode(fyr)) || + fy_is_generic_lbz_m(c, fy_reader_lb_mode(fyr)); +} + +static FY_ALWAYS_INLINE inline const void * +fy_reader_ensure_lookahead(struct fy_reader *fyr, size_t size, size_t *leftp) +{ + size_t current_left = fy_reader_current_left(fyr); + + if (current_left >= size) { + *leftp = current_left; + return fyr->current_ptr; + } + return fy_reader_ensure_lookahead_slow_path(fyr, size, leftp); +} + +/* compare string at the current point (n max) */ +static inline int +fy_reader_strncmp(struct fy_reader *fyr, const char *str, size_t n) +{ + const char *p; + size_t len; + int ret; + + assert(fyr); + p = fy_reader_ensure_lookahead(fyr, n, &len); + if (!p) + return -1; + + /* not enough? */ + if (n > len) + return 1; + + ret = strncmp(p, str, n); + return ret ? 1 : 0; +} + +int fy_reader_peek_at_offset_width_slow_path(struct fy_reader *fyr, size_t offset, int *wp); + +static FY_ALWAYS_INLINE inline int +fy_reader_peek_at_offset_width(struct fy_reader *fyr, size_t offset, int *wp) +{ + assert(fyr); + + if ((ssize_t)(fy_reader_current_left(fyr) - offset) >= 4) + return fy_utf8_get(fyr->current_ptr + offset, 4, wp); + + return fy_reader_peek_at_offset_width_slow_path(fyr, offset, wp); +} + +int64_t fy_reader_peek_at_offset_width_slow_path_64(struct fy_reader *fyr, size_t offset); + +static FY_ALWAYS_INLINE inline int64_t +fy_reader_peek_at_offset_width_64(struct fy_reader *fyr, size_t offset) +{ + assert(fyr); + + if ((ssize_t)(fy_reader_current_left(fyr) - offset) >= 4) + return fy_utf8_get_64(fyr->current_ptr + offset, 4); + + return fy_reader_peek_at_offset_width_slow_path_64(fyr, offset); +} + + +static FY_ALWAYS_INLINE inline int +fy_reader_peek_at_offset(struct fy_reader *fyr, size_t offset) +{ + int w; + + return fy_reader_peek_at_offset_width(fyr, offset, &w); +} + +static FY_ALWAYS_INLINE inline int +fy_reader_peek_at_width_internal(struct fy_reader *fyr, int pos, ssize_t *offsetp, int *wp) +{ + int i, c, w; + size_t offset; + + offset = (size_t)*offsetp; + if ((ssize_t)offset < 0) { + for (i = 0, offset = 0; i < pos; i++, offset += w) { + c = fy_reader_peek_at_offset_width(fyr, offset, &w); + if (c < 0) + return c; + } + } + c = fy_reader_peek_at_offset_width(fyr, offset, wp); + + *offsetp = offset + *wp; + return c; +} + +static FY_ALWAYS_INLINE inline int +fy_reader_peek_at_internal(struct fy_reader *fyr, int pos, ssize_t *offsetp) +{ + int w; + + return fy_reader_peek_at_width_internal(fyr, pos, offsetp, &w); +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_is_blank_at_offset(struct fy_reader *fyr, size_t offset) +{ + return fy_is_blank(fy_reader_peek_at_offset(fyr, offset)); +} + +static FY_ALWAYS_INLINE inline bool +fy_reader_is_blankz_at_offset(struct fy_reader *fyr, size_t offset) +{ + return fy_reader_is_blankz(fyr, fy_reader_peek_at_offset(fyr, offset)); +} + +static FY_ALWAYS_INLINE inline int +fy_reader_peek_at_width(struct fy_reader *fyr, int pos, int *wp) +{ + int i, c, w; + size_t offset; + + assert(fyr); + for (i = 0, offset = 0; i < pos; i++, offset += w) { + c = fy_reader_peek_at_offset_width(fyr, offset, &w); + if (c < 0) + return c; + } + + return fy_reader_peek_at_offset_width(fyr, offset, wp); +} + +static FY_ALWAYS_INLINE inline int +fy_reader_peek_at(struct fy_reader *fyr, int pos) +{ + int i, c, w; + size_t offset; + + assert(fyr); + for (i = 0, offset = 0; i < pos; i++, offset += w) { + c = fy_reader_peek_at_offset_width(fyr, offset, &w); + if (c < 0) + return c; + } + + return fy_reader_peek_at_offset(fyr, offset); +} + +static FY_ALWAYS_INLINE inline int +fy_reader_peek_width(struct fy_reader *fyr, int *wp) +{ + return fy_reader_peek_at_offset_width(fyr, 0, wp); +} + +static FY_ALWAYS_INLINE inline int +fy_reader_peek(struct fy_reader *fyr) +{ + int w; + return fy_reader_peek_width(fyr, &w); +} + +static FY_ALWAYS_INLINE inline const void * +fy_reader_peek_block(struct fy_reader *fyr, size_t *lenp) +{ + const void *p; + + /* try to pull at least one utf8 character usually */ + p = fy_reader_ensure_lookahead(fyr, 4, lenp); + + /* not a utf8 character available? try a single byte */ + if (!p) + p = fy_reader_ensure_lookahead(fyr, 1, lenp); + if (!*lenp) + p = NULL; + return p; +} + +static FY_ALWAYS_INLINE inline void +fy_reader_advance_octets(struct fy_reader *fyr, size_t advance) +{ + fyr->current_ptr += advance; +} + +void fy_reader_advance_slow_path(struct fy_reader *fyr, int c); + +static FY_ALWAYS_INLINE inline void +fy_reader_advance_printable_ascii(struct fy_reader *fyr, int c) +{ + assert(fyr); + fy_reader_advance_octets(fyr, 1); + fyr->column++; +} + +static FY_ALWAYS_INLINE inline void +fy_reader_update_state_lb_mode(struct fy_reader *fyr, const int c, const enum fy_lb_mode lb_mode) +{ + if (fy_is_lb_m(c, lb_mode)) { + fyr->column = 0; + fyr->line++; + } else if (fy_is_tab(c) && fyr->tabsize) + fyr->column += (fyr->tabsize - (fyr->column % fyr->tabsize)); + else + fyr->column++; +} + +static FY_ALWAYS_INLINE inline void +fy_reader_advance_lb_mode(struct fy_reader *fyr, const int c, const enum fy_lb_mode lb_mode) +{ + assert(fy_utf8_is_valid(c)); + fy_reader_advance_octets(fyr, fy_utf8_width(c)); + fy_reader_update_state_lb_mode(fyr, c, lb_mode); +} + +static FY_ALWAYS_INLINE inline void +fy_reader_advance(struct fy_reader *fyr, const int c) +{ + fy_reader_advance_lb_mode(fyr, c, fy_reader_lb_mode(fyr)); +} + +static FY_ALWAYS_INLINE inline void +fy_reader_advance_ws(struct fy_reader *fyr, int c) +{ + /* skip this character */ + fy_reader_advance_octets(fyr, fy_utf8_width(c)); + + if (fy_is_tab(c) && fyr->tabsize) + fyr->column += (fyr->tabsize - (fyr->column % fyr->tabsize)); + else + fyr->column++; +} + +static FY_ALWAYS_INLINE inline void +fy_reader_advance_space(struct fy_reader *fyr) +{ + fy_reader_advance_octets(fyr, 1); + fyr->column++; +} + +static FY_ALWAYS_INLINE inline int +fy_reader_get(struct fy_reader *fyr) +{ + int value; + + value = fy_reader_peek(fyr); + if (value < 0) + return value; + + fy_reader_advance(fyr, value); + + return value; +} + +static FY_ALWAYS_INLINE inline int +fy_reader_advance_by(struct fy_reader *fyr, int count) +{ + int i, c; + + for (i = 0; i < count; i++) { + c = fy_reader_get(fyr); + if (c < 0) + break; + } + return i ? i : -1; +} + +/* compare string at the current point */ +static inline bool +fy_reader_strcmp(struct fy_reader *fyr, const char *str) +{ + return fy_reader_strncmp(fyr, str, strlen(str)); +} + +/* atom */ + +static inline void +fy_reader_fill_atom_start(struct fy_reader *fyr, struct fy_atom *handle) +{ + /* start mark */ + fy_reader_get_mark(fyr, &handle->start_mark); + handle->fyi = fy_reader_current_input(fyr); + handle->fyi_generation = fy_reader_current_input_generation(fyr); + + handle->increment = 0; + handle->tozero = 0; + + /* note that handle->data may be zero for empty input */ +} + +static inline void +fy_reader_fill_atom_end_at(struct fy_reader *fyr, struct fy_atom *handle, struct fy_mark *end_mark) +{ + if (end_mark) + handle->end_mark = *end_mark; + else + fy_reader_get_mark(fyr, &handle->end_mark); + + /* default is plain, modify at return */ + handle->style = FYAS_PLAIN; + handle->chomp = FYAC_CLIP; + /* by default we don't do storage hints, it's the job of the caller */ + handle->storage_hint = 0; + handle->storage_hint_valid = false; + handle->tabsize = fy_reader_tabsize(fyr); + handle->json_mode = fy_reader_json_mode(fyr); + handle->lb_mode = fy_reader_lb_mode(fyr); + handle->fws_mode = fy_reader_flow_ws_mode(fyr); + handle->directive0_mode = fy_reader_directive0_mode(fyr); +} + +static inline void +fy_reader_fill_atom_end(struct fy_reader *fyr, struct fy_atom *handle) +{ + fy_reader_fill_atom_end_at(fyr, handle, NULL); +} + +/* diag */ + +static inline bool +fyr_debug_log_level_is_enabled(struct fy_reader *fyr) +{ + return fyr && fy_diag_log_level_is_enabled(fyr->diag, FYET_DEBUG, FYEM_SCAN); +} + +int fy_reader_vdiag(struct fy_reader *fyr, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap); + +int fy_reader_diag(struct fy_reader *fyr, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) + __attribute__((format(printf, 6, 7))); + +void fy_reader_diag_vreport(struct fy_reader *fyr, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap); +void fy_reader_diag_report(struct fy_reader *fyr, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +#ifdef FY_DEVMODE + +#define fyr_debug(_fyr, _fmt, ...) \ + do { \ + struct fy_reader *__fyr = (_fyr); \ + \ + if (fyr_debug_log_level_is_enabled(__fyr)) \ + fy_reader_diag(__fyr, FYET_DEBUG, \ + __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__); \ + } while(0) +#else + +#define fyr_debug(_fyr, _fmt, ...) \ + do { } while(0) + +#endif + +#define fyr_info(_fyr, _fmt, ...) \ + fy_reader_diag((_fyr), FYET_INFO, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyr_notice(_fyr, _fmt, ...) \ + fy_reader_diag((_fyr), FYET_NOTICE, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyr_warning(_fyr, _fmt, ...) \ + fy_reader_diag((_fyr), FYET_WARNING, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyr_error(_fyr, _fmt, ...) \ + fy_reader_diag((_fyr), FYET_ERROR, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) + +#define fyr_error_check(_fyr, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + fyr_error((_fyr), _fmt, ## __VA_ARGS__); \ + goto _label ; \ + } \ + } while(0) + +#define _FYR_TOKEN_DIAG(_fyr, _fyt, _type, _module, _fmt, ...) \ + do { \ + struct fy_diag_report_ctx _drc; \ + memset(&_drc, 0, sizeof(_drc)); \ + _drc.type = (_type); \ + _drc.module = (_module); \ + _drc.fyt = (_fyt); \ + fy_reader_diag_report((_fyr), &_drc, (_fmt) , ## __VA_ARGS__); \ + } while(0) + +#define FYR_TOKEN_DIAG(_fyr, _fyt, _type, _module, _fmt, ...) \ + _FYR_TOKEN_DIAG(_fyr, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYR_PARSE_DIAG(_fyr, _adv, _cnt, _type, _module, _fmt, ...) \ + _FYR_TOKEN_DIAG(_fyr, \ + fy_token_create(FYTT_INPUT_MARKER, \ + fy_reader_fill_atom_at((_fyr), (_adv), (_cnt), \ + alloca(sizeof(struct fy_atom)))), \ + _type, _module, _fmt, ## __VA_ARGS__) + +#define FYR_MARK_DIAG(_fyr, _sm, _em, _type, _module, _fmt, ...) \ + _FYR_TOKEN_DIAG(_fyr, \ + fy_token_create(FYTT_INPUT_MARKER, \ + fy_reader_fill_atom_mark(((_fyr)), (_sm), (_em), \ + alloca(sizeof(struct fy_atom)))), \ + _type, _module, _fmt, ## __VA_ARGS__) + +#define FYR_NODE_DIAG(_fyr, _fyn, _type, _module, _fmt, ...) \ + _FYR_TOKEN_DIAG(_fyr, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYR_TOKEN_ERROR(_fyr, _fyt, _module, _fmt, ...) \ + FYR_TOKEN_DIAG(_fyr, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYR_PARSE_ERROR(_fyr, _adv, _cnt, _module, _fmt, ...) \ + FYR_PARSE_DIAG(_fyr, _adv, _cnt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYR_MARK_ERROR(_fyr, _sm, _em, _module, _fmt, ...) \ + FYR_MARK_DIAG(_fyr, _sm, _em, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYR_NODE_ERROR(_fyr, _fyn, _module, _fmt, ...) \ + FYR_NODE_DIAG(_fyr, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYR_TOKEN_ERROR_CHECK(_fyr, _fyt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYR_TOKEN_ERROR(_fyr, _fyt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYR_PARSE_ERROR_CHECK(_fyr, _adv, _cnt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYR_PARSE_ERROR(_fyr, _adv, _cnt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYR_MARK_ERROR_CHECK(_fyr, _sm, _em, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYR_MARK_ERROR(_fyr, _sm, _em, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYR_NODE_ERROR_CHECK(_fyr, _fyn, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYR_NODE_ERROR(_fyr, _fyn, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYR_TOKEN_WARNING(_fyr, _fyt, _module, _fmt, ...) \ + FYR_TOKEN_DIAG(_fyr, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYR_PARSE_WARNING(_fyr, _adv, _cnt, _module, _fmt, ...) \ + FYR_PARSE_DIAG(_fyr, _adv, _cnt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYR_MARK_WARNING(_fyr, _sm, _em, _module, _fmt, ...) \ + FYR_MARK_DIAG(_fyr, _sm, _em, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYR_NODE_WARNING(_fyr, _fyn, _type, _module, _fmt, ...) \ + FYR_NODE_DIAG(_fyr, _fyn, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#endif diff --git a/src/lib/fy-parse-diag.c b/src/lib/fy-parse-diag.c new file mode 100644 index 0000000..a108312 --- /dev/null +++ b/src/lib/fy-parse-diag.c @@ -0,0 +1,162 @@ +/* + * fy-parse-diag.c - parser diagnostics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-diag.h" +#include "fy-parse.h" + +int fy_parser_vdiag(struct fy_parser *fyp, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap) +{ + enum fy_error_type level; + enum fy_error_module module; + struct fy_diag_ctx fydc; + int rc; + + if (!fyp || !fyp->diag || !fmt) + return -1; + + level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; + module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; + + if (!fy_diag_log_level_is_enabled(fyp->diag, level, module)) + return 0; + + /* fill in fy_diag_ctx */ + memset(&fydc, 0, sizeof(fydc)); + + fydc.level = level; + fydc.module = module; + fydc.source_file = file; + fydc.source_line = line; + fydc.source_func = func; + fydc.line = fyp_line(fyp); + fydc.column = fyp_column(fyp); + + rc = fy_vdiag(fyp->diag, &fydc, fmt, ap); + + if (fyp && !fyp->stream_error && fyp->diag->on_error) + fyp->stream_error = true; + + return rc; +} + +int fy_parser_diag(struct fy_parser *fyp, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_parser_vdiag(fyp, flags, file, line, func, fmt, ap); + va_end(ap); + + return rc; +} + +void fy_parser_diag_vreport(struct fy_parser *fyp, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap) +{ + struct fy_diag *diag; + + if (!fyp || !fyp->diag || !fydrc || !fmt) + return; + + diag = fyp->diag; + + fy_diag_vreport(diag, fydrc, fmt, ap); + + if (fyp && !fyp->stream_error && diag->on_error) + fyp->stream_error = true; +} + +void fy_parser_diag_report(struct fy_parser *fyp, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_parser_diag_vreport(fyp, fydrc, fmt, ap); + va_end(ap); +} + +void fy_parser_vlog(struct fy_parser *fyp, enum fy_error_type type, + const char *fmt, va_list ap) +{ + struct fy_diag_ctx ctx = { + .level = type, + .module = FYEM_UNKNOWN, + .source_func = "", + .source_file = "", + .source_line = 0, + .file = NULL, + .line = 0, + .column = 0, + }; + + if (!fyp || !fyp->diag) + return; + + (void)fy_vdiag(fyp->diag, &ctx, fmt, ap); +} + +void fy_parser_log(struct fy_parser *fyp, enum fy_error_type type, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_parser_vlog(fyp, type, fmt, ap); + va_end(ap); +} + +void fy_parser_vreport(struct fy_parser *fyp, enum fy_error_type type, + struct fy_token *fyt, const char *fmt, va_list ap) +{ + struct fy_diag_report_ctx fydrc_local, *fydrc = &fydrc_local; + + if (!fyp || !fyp->diag || !fyt) + return; + + memset(fydrc, 0, sizeof(*fydrc)); + fydrc->type = type; + fydrc->module = FYEM_UNKNOWN; + fydrc->fyt = fy_token_ref(fyt); + + fy_parser_diag_vreport(fyp, fydrc, fmt, ap); +} + +void fy_parser_report(struct fy_parser *fyp, enum fy_error_type type, + struct fy_token *fyt, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_parser_vreport(fyp, type, fyt, fmt, ap); + va_end(ap); +} + diff --git a/src/lib/fy-parse.c b/src/lib/fy-parse.c new file mode 100644 index 0000000..830e11b --- /dev/null +++ b/src/lib/fy-parse.c @@ -0,0 +1,9128 @@ +/* + * fy-parse.c - Internal parse interface + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" + +#include "fy-utils.h" + +/* only check atom sizes on debug */ +#ifndef NDEBUG +#define ATOM_SIZE_CHECK +#endif + +const char *fy_library_version(void) +{ +#ifndef VERSION +#warning No version defined + return "UNKNOWN"; +#else + return VERSION; +#endif +} + +int fy_parse_input_append(struct fy_parser *fyp, const struct fy_input_cfg *fyic) +{ + struct fy_input *fyi = NULL; + + fyi = fy_input_create(fyic); + fyp_error_check(fyp, fyi != NULL, err_out, + "fy_parse_input_create() failed!"); + + fyi->state = FYIS_QUEUED; + fy_input_list_add_tail(&fyp->queued_inputs, fyi); + + return 0; + +err_out: + fy_input_unref(fyi); + return -1; +} + +bool fy_parse_have_more_inputs(struct fy_parser *fyp) +{ + return !fy_input_list_empty(&fyp->queued_inputs); +} + +int fy_parse_get_next_input(struct fy_parser *fyp) +{ + const char *s; + struct fy_reader_input_cfg icfg; + struct fy_input *fyi; + int rc; + bool json_mode; + enum fy_reader_mode rdmode; + + assert(fyp); + + if (fy_reader_current_input(fyp->reader)) { + fyp_scan_debug(fyp, "get next input: already exists"); + return 1; + } + + /* get next queued input */ + fyi = fy_input_list_pop(&fyp->queued_inputs); + + /* none left? we're done */ + if (!fyi) { + fyp_scan_debug(fyp, "get next input: all inputs exhausted"); + return 0; + } + + json_mode = false; + if ((fyp->cfg.flags & (FYPCF_JSON_MASK << FYPCF_JSON_SHIFT)) == FYPCF_JSON_AUTO) { + /* detection only works for filenames (sucks) */ + if (fyi->cfg.type == fyit_file) { + s = fyi->cfg.file.filename; + if (s) + s = strrchr(s, '.'); + json_mode = s && !strcmp(s, ".json"); + } + } else if ((fyp->cfg.flags & (FYPCF_JSON_MASK << FYPCF_JSON_SHIFT)) == FYPCF_JSON_FORCE) + json_mode = true; + + /* set the initial reader mode according to json option and default version */ + rdmode = fy_reader_calculate_mode(&fyp->stream_version, json_mode); + fy_reader_set_mode(fyp->reader, rdmode); + + memset(&icfg, 0, sizeof(icfg)); + icfg.disable_mmap_opt = !!(fyp->cfg.flags & FYPCF_DISABLE_MMAP_OPT); + + rc = fy_reader_input_open(fyp->reader, fyi, &icfg); + fyp_error_check(fyp, !rc, err_out, + "failed to open input"); + + /* take off the reference; reader now owns */ + fy_input_unref(fyi); + + // inherit the JSON mode + if (fyp->current_document_state) + fyp->current_document_state->json_mode = fyp_json_mode(fyp); + + fyp_scan_debug(fyp, "get next input: new input - %s mode", json_mode ? "JSON" : "YAML"); + + return 1; + +err_out: + fy_input_unref(fyi); + return -1; +} + +ssize_t fy_parse_estimate_queued_input_size(struct fy_parser *fyp) +{ + struct fy_input *fyi; + ssize_t size, total; + + if (!fyp) + return 0; + + total = 0; + for (fyi = fy_input_list_head(&fyp->queued_inputs); fyi; fyi = fy_input_next(&fyp->queued_inputs, fyi)) { + if (fyi->state != FYIS_QUEUED) + continue; + + size = fy_input_estimate_queued_size(fyi); + if (size < 0) + return -1; + + /* something that's indetermined */ + if (size == SSIZE_MAX) + return SSIZE_MAX; + + total += size; + + /* roll-over check */ + if (total < 0) + return SSIZE_MAX; + } + + return total; +} + +static inline void +fy_token_queue_epilogue(struct fy_parser *fyp, struct fy_token *fyt) +{ + /* special handling for zero indented scalars */ + fyp->token_activity_counter++; + if (fyt->type == FYTT_DOCUMENT_START) + fyp->document_first_content_token = true; + else if (fyp->document_first_content_token && fy_token_type_is_content(fyt->type)) + fyp->document_first_content_token = false; +} + +static inline struct fy_token * +fy_token_queue_simple_internal(struct fy_parser *fyp, struct fy_token_list *fytl, enum fy_token_type type, int advance_octets) +{ + struct fy_reader *fyr = fyp->reader; + struct fy_token *fyt; + + /* allocate and copy in place */ + fyt = fy_token_alloc_rl(fyp->recycled_token_list); + if (!fyt) + return NULL; + + fyt->type = type; + + /* the advance is always octets */ + fy_reader_fill_atom_start(fyr, &fyt->handle); + if (advance_octets > 0) { + fy_reader_advance_octets(fyr, advance_octets); + fyr->column += advance_octets; + } + fy_reader_fill_atom_end(fyr, &fyt->handle); + + fy_input_ref(fyt->handle.fyi); + + fy_token_list_add_tail(fytl, fyt); + + return fyt; +} + +static inline struct fy_token * +fy_token_queue_simple(struct fy_parser *fyp, struct fy_token_list *fytl, enum fy_token_type type, int advance_octets) +{ + struct fy_token *fyt; + + fyt = fy_token_queue_simple_internal(fyp, fytl, type, advance_octets); + if (!fyt) + return NULL; + + fy_token_queue_epilogue(fyp, fyt); + return fyt; +} + +struct fy_token * +fy_token_vqueue_internal(struct fy_parser *fyp, struct fy_token_list *fytl, + enum fy_token_type type, va_list ap) +{ + struct fy_token *fyt; + + fyt = fy_token_vcreate_rl(fyp->recycled_token_list, type, ap); + if (!fyt) + return NULL; + fy_token_list_add_tail(fytl, fyt); + + fy_token_queue_epilogue(fyp, fyt); + return fyt; +} + +struct fy_token *fy_token_queue_internal(struct fy_parser *fyp, struct fy_token_list *fytl, + enum fy_token_type type, ...) +{ + va_list ap; + struct fy_token *fyt; + + va_start(ap, type); + fyt = fy_token_vqueue_internal(fyp, fytl, type, ap); + va_end(ap); + + return fyt; +} + +struct fy_token *fy_token_vqueue(struct fy_parser *fyp, enum fy_token_type type, va_list ap) +{ + struct fy_token *fyt; + + fyt = fy_token_vqueue_internal(fyp, &fyp->queued_tokens, type, ap); + if (fyt) + fyp->token_activity_counter++; + return fyt; +} + +struct fy_token *fy_token_queue(struct fy_parser *fyp, enum fy_token_type type, ...) +{ + va_list ap; + struct fy_token *fyt; + + va_start(ap, type); + fyt = fy_token_vqueue(fyp, type, ap); + va_end(ap); + + return fyt; +} + +const struct fy_version fy_default_version = { + .major = 1, + .minor = 2 +}; + +int fy_version_compare(const struct fy_version *va, const struct fy_version *vb) +{ + unsigned int vanum, vbnum; + + if (!va) + va = &fy_default_version; + if (!vb) + vb = &fy_default_version; + +#define FY_VERSION_UINT(_major, _minor) \ + ((((unsigned int)(_major) & 0xff) << 8) | ((unsigned int)((_minor) & 0xff))) + + vanum = FY_VERSION_UINT(va->major, va->minor); + vbnum = FY_VERSION_UINT(vb->major, vb->minor); + +#undef FY_VERSION_UINT + + return vanum == vbnum ? 0 : + vanum < vbnum ? -1 : 1; +} + +const struct fy_version * +fy_version_default(void) +{ + return &fy_default_version; +} + +static const struct fy_version * const fy_map_option_to_version[] = { + [FYPCF_DEFAULT_VERSION_AUTO >> FYPCF_DEFAULT_VERSION_SHIFT] = &fy_default_version, + [FYPCF_DEFAULT_VERSION_1_1 >> FYPCF_DEFAULT_VERSION_SHIFT] = fy_version_make(1, 1), + [FYPCF_DEFAULT_VERSION_1_2 >> FYPCF_DEFAULT_VERSION_SHIFT] = fy_version_make(1, 2), + [FYPCF_DEFAULT_VERSION_1_3 >> FYPCF_DEFAULT_VERSION_SHIFT] = fy_version_make(1, 3), +}; + +bool fy_version_is_supported(const struct fy_version *vers) +{ + unsigned int i; + const struct fy_version *vers_chk; + + if (!vers) + return true; /* NULL means default, which is supported */ + + for (i = 0; i < sizeof(fy_map_option_to_version)/sizeof(fy_map_option_to_version[0]); i++) { + + vers_chk = fy_map_option_to_version[i]; + if (!vers_chk) + continue; + + if (fy_version_compare(vers, vers_chk) == 0) + return true; + } + + return false; +} + +static const struct fy_version * +fy_parse_cfg_to_version(enum fy_parse_cfg_flags flags) +{ + unsigned int idx; + + idx = (flags >> FYPCF_DEFAULT_VERSION_SHIFT) & FYPCF_DEFAULT_VERSION_MASK; + + if (idx >= sizeof(fy_map_option_to_version)/sizeof(fy_map_option_to_version[0])) + return NULL; + + return fy_map_option_to_version[idx]; +} + +const struct fy_version *fy_version_supported_iterate(void **prevp) +{ + const struct fy_version * const *versp; + const struct fy_version *vers; + unsigned int idx; + + if (!prevp) + return NULL; + + versp = (const struct fy_version * const *)*prevp; + if (!versp) { + /* we skip over the first (which is the default) */ + versp = fy_map_option_to_version; + } + + versp++; + + idx = versp - fy_map_option_to_version; + if (idx >= sizeof(fy_map_option_to_version)/sizeof(fy_map_option_to_version[0])) + return NULL; + + vers = *versp; + *prevp = (void **)versp; + + return vers; +} + +const struct fy_tag * const fy_default_tags[] = { + &(struct fy_tag) { .handle = "!", .prefix = "!", }, + &(struct fy_tag) { .handle = "!!", .prefix = "tag:yaml.org,2002:", }, + &(struct fy_tag) { .handle = "", .prefix = "", }, + NULL +}; + +bool fy_tag_handle_is_default(const char *handle, size_t handle_size) +{ + int i; + const struct fy_tag *fytag; + + if (handle_size == (size_t)-1) + handle_size = strlen(handle); + + for (i = 0; (fytag = fy_default_tags[i]) != NULL; i++) { + + if (handle_size == strlen(fytag->handle) && + !memcmp(handle, fytag->handle, handle_size)) + return true; + + } + return false; +} + +bool fy_tag_is_default_internal(const char *handle, size_t handle_size, + const char *prefix, size_t prefix_size) +{ + int i; + const struct fy_tag *fytag; + + if (handle_size == (size_t)-1) + handle_size = strlen(handle); + + if (prefix_size == (size_t)-1) + prefix_size = strlen(prefix); + + for (i = 0; (fytag = fy_default_tags[i]) != NULL; i++) { + + if (handle_size == strlen(fytag->handle) && + !memcmp(handle, fytag->handle, handle_size) && + prefix_size == strlen(fytag->prefix) && + !memcmp(prefix, fytag->prefix, prefix_size)) + return true; + + } + return false; +} + +bool fy_document_state_tag_is_default(struct fy_document_state *fyds, const struct fy_tag *tag) +{ + struct fy_token *fyt_td; + + /* default tag, but it might be overriden */ + fyt_td = fy_document_state_lookup_tag_directive(fyds, tag->handle, strlen(tag->handle)); + if (!fyt_td) + return false; /* Huh? */ + + return fyt_td->tag_directive.is_default; +} + +bool fy_token_tag_directive_is_overridable(struct fy_token *fyt_td) +{ + const struct fy_tag *fytag; + const char *handle, *prefix; + size_t handle_size, prefix_size; + int i; + + if (!fyt_td) + return false; + + handle = fy_tag_directive_token_handle(fyt_td, &handle_size); + prefix = fy_tag_directive_token_prefix(fyt_td, &prefix_size); + if (!handle || !prefix) + return false; + + for (i = 0; (fytag = fy_default_tags[i]) != NULL; i++) { + + if (handle_size == strlen(fytag->handle) && + !memcmp(handle, fytag->handle, handle_size) && + prefix_size == strlen(fytag->prefix) && + !memcmp(prefix, fytag->prefix, prefix_size)) + return true; + + } + return false; +} + +int fy_reset_document_state(struct fy_parser *fyp) +{ + struct fy_document_state *fyds_new = NULL; + enum fy_reader_mode rdmode; + + fyp_scan_debug(fyp, "resetting document state"); + + if (!fyp->default_document_state) { + fyds_new = fy_document_state_default(&fyp->stream_version, NULL); + fyp_error_check(fyp, fyds_new, err_out, + "fy_document_state_default() failed"); + } else { + fyds_new = fy_document_state_copy(fyp->default_document_state); + fyp_error_check(fyp, fyds_new, err_out, + "fy_document_state_copy() failed"); + } + // inherit the JSON mode + fyds_new->json_mode = fyp_json_mode(fyp); + + if (fyp->current_document_state) + fy_document_state_unref(fyp->current_document_state); + fyp->current_document_state = fyds_new; + + /* TODO check when cleaning flow lists */ + fyp->flow_level = 0; + fyp->flow = FYFT_NONE; + fy_parse_flow_list_recycle_all(fyp, &fyp->flow_stack); + + /* and reset all aliases (TODO look for cross document aliases) */ + fy_parse_streaming_aliases_reset(fyp); + + rdmode = fy_reader_calculate_mode(&fyds_new->version, fyds_new->json_mode); + fy_reader_set_mode(fyp->reader, rdmode); + + return 0; + +err_out: + return -1; +} + +int fy_parser_set_default_document_state(struct fy_parser *fyp, + struct fy_document_state *fyds) +{ + if (!fyp) + return -1; + + /* only in a safe state */ + if (fyp->state != FYPS_NONE && fyp->state != FYPS_END) + return -1; + + if (fyp->default_document_state != fyds) { + if (fyp->default_document_state) { + fy_document_state_unref(fyp->default_document_state); + fyp->default_document_state = NULL; + } + + if (fyds) + fyp->default_document_state = fy_document_state_ref(fyds); + } + + fy_reset_document_state(fyp); + + return 0; +} + +void fy_parser_set_next_single_document(struct fy_parser *fyp) +{ + if (!fyp) + return; + + fyp->next_single_document = true; +} + +int fy_check_document_version(struct fy_parser *fyp) +{ + int major, minor; + + major = fyp->current_document_state->version.major; + minor = fyp->current_document_state->version.minor; + + /* we only support YAML version 1.x */ + if (major == 1) { + /* 1.1 is supported without warnings */ + if (minor == 1 || minor == 2) + goto ok; + + if (minor == 3) + goto experimental; + } + + return -1; + +experimental: + fyp_scan_debug(fyp, "Experimental support for version %d.%d", + major, minor); +ok: + return 0; +} + +int fy_parse_version_directive(struct fy_parser *fyp, struct fy_token *fyt, bool scan_mode) +{ + struct fy_document_state *fyds; + const char *vs; + size_t vs_len; + char *vs0; + char *s, *e; + long v; + int rc; + + fyp_error_check(fyp, fyt && fyt->type == FYTT_VERSION_DIRECTIVE, err_out, + "illegal token (or missing) version directive token"); + + fyds = fyp->current_document_state; + fyp_error_check(fyp, fyds, err_out, + "no current document state error"); + + if (!scan_mode) { + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !fyds->fyt_vd, err_out, + "duplicate version directive"); + } else { + /* in scan mode, we just override everything */ + fy_token_unref_rl(fyp->recycled_token_list, fyds->fyt_vd); + fyds->fyt_vd = NULL; + } + + /* version directive of the form: MAJ.MIN */ + vs = fy_token_get_text(fyt, &vs_len); + fyp_error_check(fyp, vs, err_out, + "fy_token_get_text() failed"); + vs0 = alloca(vs_len + 1); + memcpy(vs0, vs, vs_len); + vs0[vs_len] = '\0'; + + /* parse version numbers */ + v = strtol(vs0, &e, 10); + fyp_error_check(fyp, e > vs0 && v >= 0 && v <= INT_MAX, err_out, + "illegal major version number (%s)", vs0); + fyp_error_check(fyp, *e == '.', err_out, + "illegal version separator"); + fyds->version.major = (int)v; + + s = e + 1; + v = strtol(s, &e, 10); + fyp_error_check(fyp, e > s && v >= 0 && v <= INT_MAX, err_out, + "illegal minor version number"); + fyp_error_check(fyp, *e == '\0', err_out, + "garbage after version number"); + fyds->version.minor = (int)v; + + fyp_scan_debug(fyp, "document parsed YAML version: %d.%d", + fyds->version.major, + fyds->version.minor); + + rc = fy_check_document_version(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "unsupport version number %d.%d", + fyds->version.major, + fyds->version.minor); + + fyds->version_explicit = true; + fyds->fyt_vd = fyt; + + return 0; +err_out: + rc = -1; +err_out_rc: + fy_token_unref_rl(fyp->recycled_token_list, fyt); + return rc; +} + +int fy_parse_tag_directive(struct fy_parser *fyp, struct fy_token *fyt, bool scan_mode) +{ + struct fy_document_state *fyds; + struct fy_token *fyt_td; + const char *handle, *prefix; + size_t handle_size, prefix_size; + bool can_override; + + fyds = fyp->current_document_state; + fyp_error_check(fyp, fyds, err_out, + "no current document state error"); + + handle = fy_tag_directive_token_handle(fyt, &handle_size); + fyp_error_check(fyp, handle, err_out, + "bad tag directive token (handle)"); + + prefix = fy_tag_directive_token_prefix(fyt, &prefix_size); + fyp_error_check(fyp, prefix, err_out, + "bad tag directive token (prefix)"); + + fyt_td = fy_document_state_lookup_tag_directive(fyds, handle, handle_size); + + can_override = fyt_td && (fy_token_tag_directive_is_overridable(fyt_td) || scan_mode); + + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !fyt_td || can_override, err_out, + "duplicate tag directive"); + + if (fyt_td) { + /* fyp_notice(fyp, "overriding tag"); */ + fy_token_list_del(&fyds->fyt_td, fyt_td); + fy_token_unref_rl(fyp->recycled_token_list, fyt_td); + /* when we override a default tag the tags are explicit */ + fyds->tags_explicit = true; + } + + fy_token_list_add_tail(&fyds->fyt_td, fyt); + fyt = NULL; + + fyp_scan_debug(fyp, "document parsed tag directive with handle=%.*s", + (int)handle_size, handle); + + if (!fy_tag_is_default_internal(handle, handle_size, prefix, prefix_size)) + fyds->tags_explicit = true; + + return 0; +err_out: + fy_token_unref_rl(fyp->recycled_token_list, fyt); + return -1; +} + +static const struct fy_parse_cfg default_parse_cfg = { + .flags = FYPCF_DEFAULT_PARSE, +}; + +static struct fy_diag *fy_parser_reader_get_diag(struct fy_reader *fyr) +{ + struct fy_parser *fyp = container_of(fyr, struct fy_parser, builtin_reader); + return fyp->diag; +} + +static int fy_parser_reader_file_open(struct fy_reader *fyr, const char *name) +{ + struct fy_parser *fyp = container_of(fyr, struct fy_parser, builtin_reader); + char *sp, *s, *e, *t, *newp; + size_t len, maxlen; + int fd; + + if (!fyp || !name || name[0] == '\0') + return -1; + + /* for a full path, or no search path, open directly */ + if (name[0] == '/' || !fyp->cfg.search_path || !fyp->cfg.search_path[0]) { + fd = open(name, O_RDONLY); + if (fd == -1) + fyp_scan_debug(fyp, "failed to open file %s\n", name); + else + fyp_scan_debug(fyp, "opened file %s\n", name); + return fd; + } + + len = strlen(fyp->cfg.search_path); + sp = alloca(len + 1); + memcpy(sp, fyp->cfg.search_path, len + 1); + + /* allocate the maximum possible so that we don't deal with reallocations */ + maxlen = len + 1 + strlen(name); + newp = malloc(maxlen + 1); + if (!newp) + return -1; + + s = sp; + e = sp + strlen(s); + while (s < e) { + /* skip completely empty */ + if (*s == ':') { + s++; + continue; + } + + t = strchr(s, ':'); + if (t) + *t++ = '\0'; + else + t = e; + + len = strlen(s) + 1 + strlen(name) + 1; + snprintf(newp, maxlen, "%s/%s", s, name); + + /* try opening */ + fd = open(newp, O_RDONLY); + if (fd != -1) { + fyp_scan_debug(fyp, "opened file %s at %s", name, newp); + free(newp); + return fd; + } + + s = t; + } + + if (newp) + free(newp); + return -1; +} + +static const struct fy_reader_ops fy_parser_reader_ops = { + .get_diag = fy_parser_reader_get_diag, + .file_open = fy_parser_reader_file_open, +}; + +int fy_parse_setup(struct fy_parser *fyp, const struct fy_parse_cfg *cfg) +{ + struct fy_diag *diag; + struct fy_diag_cfg dcfg; + const struct fy_version *vers; + int rc; + + if (!fyp) + return -1; + + memset(fyp, 0, sizeof(*fyp)); + + diag = cfg ? cfg->diag : NULL; + fyp->cfg = cfg ? *cfg : default_parse_cfg; + + /* supported version? */ + vers = fy_parse_cfg_to_version(fyp->cfg.flags); + if (!vers) + return -1; + + if (!diag) { + fy_diag_cfg_default(&dcfg); + diag = fy_diag_create(&dcfg); + if (!diag) + return -1; + } else + fy_diag_ref(diag); + + fyp->diag = diag; + + fy_reader_setup(&fyp->builtin_reader, &fy_parser_reader_ops); + fyp->reader = &fyp->builtin_reader; + + fyp->default_version = *vers; + fyp->stream_version = fyp->default_version; + + fy_indent_list_init(&fyp->indent_stack); + fy_indent_list_init(&fyp->recycled_indent); + fyp->indent = -2; + fyp->indent_line = -1; + fyp->starting_indent = -1; + fyp->generated_block_map = false; + fyp->last_was_comma = false; + + fy_simple_key_list_init(&fyp->simple_keys); + fy_simple_key_list_init(&fyp->recycled_simple_key); + + fy_token_list_init(&fyp->queued_tokens); + + fy_input_list_init(&fyp->queued_inputs); + + fyp->state = FYPS_NONE; + fy_parse_state_log_list_init(&fyp->state_stack); + fy_parse_state_log_list_init(&fyp->recycled_parse_state_log); + + fy_eventp_list_init(&fyp->recycled_eventp); + fy_token_list_init(&fyp->recycled_token); + + fy_flow_list_init(&fyp->flow_stack); + fyp->flow = FYFT_NONE; + fy_flow_list_init(&fyp->recycled_flow); + + fyp->pending_complex_key_column = -1; + fyp->last_block_mapping_key_line = -1; + + fy_streaming_alias_list_init(&fyp->streaming_aliases); + fy_streaming_alias_list_init(&fyp->recycled_streaming_alias); + fy_eventp_list_init(&fyp->mks.args); + + fy_atom_reset(&fyp->last_comment); + + fyp->suppress_recycling = !!(fyp->cfg.flags & FYPCF_DISABLE_RECYCLING) || + (getenv("FY_VALGRIND") && + !getenv("FY_VALGRIND_RECYCLING")); + + if (fyp->suppress_recycling) + fyp_parse_debug(fyp, "Suppressing recycling"); + + if (!fyp->suppress_recycling) { + fyp->recycled_eventp_list = &fyp->recycled_eventp; + fyp->recycled_token_list = &fyp->recycled_token; + } else { + fyp->recycled_eventp_list = NULL; + fyp->recycled_token_list = NULL; + } + + fyp->current_document_state = NULL; + + rc = fy_reset_document_state(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_reset_document_state() failed"); + return 0; + +err_out_rc: + return rc; +} + +void fy_parse_cleanup(struct fy_parser *fyp) +{ + struct fy_input *fyi, *fyin; + struct fy_eventp *fyep; + struct fy_token *fyt; + + fy_parse_eventp_recycle(fyp, fyp->fyep_peek); + + fy_parse_streaming_aliases_reset(fyp); + + fy_input_unref(fyp->last_comment.fyi); + fy_atom_reset(&fyp->last_comment); + + fy_input_unref(fyp->last_event_handle.fyi); + fy_atom_reset(&fyp->last_event_handle); + + fy_composer_destroy(fyp->fyc); + fy_document_builder_destroy(fyp->fydb); + + fy_parse_indent_list_recycle_all(fyp, &fyp->indent_stack); + fy_parse_simple_key_list_recycle_all(fyp, &fyp->simple_keys); + fy_token_list_unref_all(&fyp->queued_tokens); + + fy_parse_parse_state_log_list_recycle_all(fyp, &fyp->state_stack); + fy_parse_flow_list_recycle_all(fyp, &fyp->flow_stack); + fy_parse_streaming_alias_list_recycle_all(fyp, &fyp->streaming_aliases); + + fy_token_unref_rl(fyp->recycled_token_list, fyp->stream_end_token); + + fy_document_state_unref(fyp->current_document_state); + fy_document_state_unref(fyp->default_document_state); + + for (fyi = fy_input_list_head(&fyp->queued_inputs); fyi; fyi = fyin) { + fyin = fy_input_next(&fyp->queued_inputs, fyi); + fy_input_unref(fyi); + } + + /* clean the builtin reader */ + fy_reader_cleanup(&fyp->builtin_reader); + + /* and vacuum (free everything) */ + fy_parse_indent_vacuum(fyp); + fy_parse_simple_key_vacuum(fyp); + fy_parse_parse_state_log_vacuum(fyp); + fy_parse_flow_vacuum(fyp); + fy_parse_streaming_alias_vacuum(fyp); + + /* free the recycled events */ + while ((fyep = fy_eventp_list_pop(&fyp->recycled_eventp)) != NULL) { + /* catch double recycles */ + /* assert(fy_eventp_list_head(&fyp->recycled_eventp)!= fyep); */ + fy_eventp_free(fyep); + } + + /* and the recycled tokens */ + while ((fyt = fy_token_list_pop(&fyp->recycled_token)) != NULL) + fy_token_free(fyt); + + fy_diag_unref(fyp->diag); +} + +static const char *state_txt[] __FY_DEBUG_UNUSED__ = { + [FYPS_NONE] = "NONE", + [FYPS_STREAM_START] = "STREAM_START", + [FYPS_IMPLICIT_DOCUMENT_START] = "IMPLICIT_DOCUMENT_START", + [FYPS_DOCUMENT_START] = "DOCUMENT_START", + [FYPS_DOCUMENT_CONTENT] = "DOCUMENT_CONTENT", + [FYPS_DOCUMENT_END] = "DOCUMENT_END", + [FYPS_BLOCK_NODE] = "BLOCK_NODE", + [FYPS_BLOCK_SEQUENCE_FIRST_ENTRY] = "BLOCK_SEQUENCE_FIRST_ENTRY", + [FYPS_BLOCK_SEQUENCE_ENTRY] = "BLOCK_SEQUENCE_ENTRY", + [FYPS_INDENTLESS_SEQUENCE_ENTRY] = "INDENTLESS_SEQUENCE_ENTRY", + [FYPS_BLOCK_MAPPING_FIRST_KEY] = "BLOCK_MAPPING_FIRST_KEY", + [FYPS_BLOCK_MAPPING_KEY] = "BLOCK_MAPPING_KEY", + [FYPS_BLOCK_MAPPING_VALUE] = "BLOCK_MAPPING_VALUE", + [FYPS_FLOW_SEQUENCE_FIRST_ENTRY] = "FLOW_SEQUENCE_FIRST_ENTRY", + [FYPS_FLOW_SEQUENCE_ENTRY] = "FLOW_SEQUENCE_ENTRY", + [FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_KEY] = "FLOW_SEQUENCE_ENTRY_MAPPING_KEY", + [FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE] = "FLOW_SEQUENCE_ENTRY_MAPPING_VALUE", + [FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_END] = "FLOW_SEQUENCE_ENTRY_MAPPING_END", + [FYPS_FLOW_MAPPING_FIRST_KEY] = "FLOW_MAPPING_FIRST_KEY", + [FYPS_FLOW_MAPPING_KEY] = "FLOW_MAPPING_KEY", + [FYPS_FLOW_MAPPING_VALUE] = "FLOW_MAPPING_VALUE", + [FYPS_FLOW_MAPPING_EMPTY_VALUE] = "FLOW_MAPPING_EMPTY_VALUE", + [FYPS_SINGLE_DOCUMENT_END] = "SINGLE_DOCUMENT_END", + [FYPS_END] = "END" +}; + +int fy_scan_comment(struct fy_parser *fyp, struct fy_atom *handle, bool single_line) +{ + int c, column, start_column, lines, scan_ahead; + bool has_ws; + + c = fy_parse_peek(fyp); + if (c != '#') + return -1; + + /* if it's no comment parsing is enabled just consume it */ + if (!(fyp->cfg.flags & FYPCF_PARSE_COMMENTS) || !handle) { + fy_advance(fyp, c); + while (!(fyp_is_lbz(fyp, c = fy_parse_peek(fyp)))) + fy_advance(fyp, c); + return 0; + } + + if (handle->fyi) + fy_input_unref(handle->fyi); + + memset(handle, 0, sizeof(*handle)); + + fy_fill_atom_start(fyp, handle); + + lines = 0; + start_column = fyp_column(fyp); + column = fyp_column(fyp); + scan_ahead = 0; + + has_ws = false; + + /* continuation must be a # on the same column */ + while (c == '#' && column == start_column) { + + lines++; + if (c == '#') { + /* chomp until line break */ + fy_advance(fyp, c); + while (!(fyp_is_lbz(fyp, c = fy_parse_peek(fyp)))) { + if (fy_is_ws(c)) + has_ws = true; + fy_advance(fyp, c); + } + + /* end of input break */ + if (fy_is_z(c)) + break; + } + + if (fy_is_ws(c)) + has_ws = true; + + if (!fyp_is_lb(fyp, c)) + break; + + column = 0; + + scan_ahead = 1; /* skipping over lb */ + while (fy_is_blank(c = fy_parse_peek_at(fyp, scan_ahead))) { + scan_ahead++; + column++; + } + + if (fy_is_z(c) || single_line) + break; + + if (c == '#' && column == start_column) { + fy_advance_by(fyp, scan_ahead); + c = fy_parse_peek(fyp); + } + } + + fy_fill_atom_end(fyp, handle); + handle->style = FYAS_COMMENT; + handle->direct_output = false; + handle->storage_hint = 0; + handle->storage_hint_valid = false; + handle->empty = false; + handle->has_lb = true; + handle->has_ws = has_ws; + handle->starts_with_ws = false; /* no-one cares for those */ + handle->starts_with_lb = false; + handle->ends_with_ws = false; + handle->ends_with_lb = false; + handle->trailing_lb = false; + handle->size0 = lines > 0; + handle->valid_anchor = false; + + /* and take the ref */ + fy_input_ref(handle->fyi); + + return 0; +} + +bool +fy_comment_atoms_seperated_by_ws(struct fy_parser *fyp, struct fy_atom *a, struct fy_atom *b) +{ + struct fy_atom *tmpatom; + struct fy_atom inbetween; + struct fy_atom_iter iter; + int c; + bool ws_or_lb; + + if (!a || !b) + return NULL; + + /* must be on same input */ + if (a->fyi != b->fyi) + return NULL; + + /* must be comments */ + if (a->style != FYAS_COMMENT || b->style != FYAS_COMMENT) + return NULL; + + /* a must be before */ + if (a->end_mark.input_pos > b->start_mark.input_pos) { + tmpatom = a; + a = b; + b = tmpatom; + } + + memcpy(&inbetween, a, sizeof(inbetween)); + inbetween.start_mark = a->end_mark; + inbetween.end_mark = b->start_mark; + + fy_atom_iter_start(&inbetween, &iter); + ws_or_lb = true; + while (ws_or_lb && (c = fy_atom_iter_utf8_get(&iter)) >= 0) + ws_or_lb = fy_is_ws(c) || fy_is_lb_m(c, a->lb_mode); + fy_atom_iter_finish(&iter); + return ws_or_lb; +} + +static inline bool +fy_reset_last_comment(struct fy_parser *fyp) +{ + if (!(fyp->cfg.flags & FYPCF_PARSE_COMMENTS)) + return false; + + if (!fy_atom_is_set(&fyp->last_comment)) + return false; + + fy_atom_reset(&fyp->last_comment); + return true; +} + +int fy_attach_comments_if_any(struct fy_parser *fyp, struct fy_token *fyt) +{ + struct fy_atom *handle; + struct fy_mark fym; + int c, rc; + + if (!fyp || !fyt) + return -1; + + if (!(fyp->cfg.flags & FYPCF_PARSE_COMMENTS)) + return 0; + + /* if a last comment exists and is valid */ + if (fy_atom_is_set(&fyp->last_comment) && + (handle = fy_token_comment_handle(fyt, fycp_top, true)) != NULL) { + + *handle = fyp->last_comment; + + fy_reset_last_comment(fyp); + } + + /* right hand comment */ + + /* skip white space */ + while (fy_is_ws(c = fy_parse_peek(fyp))) + fy_advance(fyp, c); + + if (c == '#') { + fy_get_mark(fyp, &fym); + + /* it's a right comment only if it's on the same line */ + if (fym.line == fyt->handle.end_mark.line) + handle = fy_token_comment_handle(fyt, fycp_right, true); + else + handle = &fyp->last_comment; /* otherwise, last comment */ + + rc = fy_scan_comment(fyp, handle, false); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_scan_comment() failed"); + } + return 0; + +err_out_rc: + return rc; +} + +int fy_scan_to_next_token(struct fy_parser *fyp) +{ + int c, c_after_ws, i, rc = 0; + bool tabs_allowed, sloppy_flow, no_indent; + ssize_t offset; + struct fy_atom this_comment; + struct fy_reader *fyr; + + fyr = fyp->reader; + + rc = fy_reader_input_scan_token_mark(fyr); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_reader_input_scan_token_mark() failed"); + + /* skip BOM at the start of the stream */ + if (fy_reader_current_input_pos(fyr) == 0 && (c = fy_parse_peek(fyp)) == FY_UTF8_BOM) { + + fy_advance(fyp, c); + /* reset column */ + fyr->column = 0; + } + + /* json does not have comments or tab handling... */ + if (fyp_json_mode(fyp)) { + fy_reader_skip_ws_cr_nl(fyr); + goto done; + } + + tabs_allowed = fyp->flow_level > 0 || !fyp->simple_key_allowed || fyp_tabsize(fyp) > 0; + sloppy_flow = fyp->flow_level > 0 && (fyp->cfg.flags & FYPCF_SLOPPY_FLOW_INDENTATION); + + for (;;) { + + /* skip white space, tabs are allowed in flow context */ + /* tabs also allowed in block context but not at start of line or after -?: */ + + /* if we're not in sloppy flow indent mode, a tab may not be used as indentation */ + if (!sloppy_flow) { + // fyp_notice(fyp, "not sloppy flow check c='%c' col=%d indent=%d\n", fy_parse_peek(fyp), fyp_column(fyp), fyp->indent); + c = -1; + while (fyp_column(fyp) <= fyp->indent && fy_is_ws(c = fy_parse_peek(fyp))) { + if (fy_is_tab(c)) + break; + fy_advance(fyp, c); + } + + /* it's an error, only if it is used for intentation */ + /* comments and empty lines are OK */ + if (fy_is_tab(c)) { + + /* skip all space and tabs */ + i = 0; + offset = -1; + while (fy_is_ws(c_after_ws = fy_parse_peek_at_internal(fyp, i, &offset))) + i++; + + no_indent = c_after_ws == '#' || fyp_is_lb(fyp, c_after_ws); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + no_indent, err_out, + "tab character may not be used as indentation"); + + /* advance by that amount */ + fy_advance_by(fyp, i); + } + } + + if (!tabs_allowed) { + /* skip space only */ + fy_reader_skip_space(fyr); + c = fy_parse_peek(fyp); + + /* it's a tab, here we go */ + if (fy_is_tab(c)) { + + /* we need to see if after ws follows a flow start marker */ + + /* skip all space and tabs */ + i = 0; + offset = -1; + while (fy_is_ws(c_after_ws = fy_parse_peek_at_internal(fyp, i, &offset))) + i++; + + /* flow start marker after spaces? allow tabs */ + if (c_after_ws == '{' || c_after_ws == '[') { + fy_advance_by(fyp, i); + c = fy_parse_peek(fyp); + } + } + } else { + fy_reader_skip_ws(fyr); + c = fy_parse_peek(fyp); + } + + /* comment? */ + if (c == '#') { + + if (!(fyp->cfg.flags & FYPCF_PARSE_COMMENTS)) { + rc = fy_scan_comment(fyp, NULL, false); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_scan_comment() failed"); + + + } else { + + memset(&this_comment, 0, sizeof(this_comment)); + fy_atom_reset(&this_comment); + rc = fy_scan_comment(fyp, &this_comment, false); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_scan_comment() failed"); + + if (fy_atom_is_set(&fyp->last_comment)) { + + /* merge consecutive comments */ + if (fy_comment_atoms_seperated_by_ws(fyp, &fyp->last_comment, &this_comment)) { + fyp->last_comment.end_mark = this_comment.end_mark; + fy_input_unref(this_comment.fyi); + fy_atom_reset(&this_comment); + } else { + fy_input_unref(fyp->last_comment.fyi); + fy_atom_reset(&fyp->last_comment); + fyp->last_comment = this_comment; + } + + } else + fyp->last_comment = this_comment; + } + + tabs_allowed = (fyp->flow_level || !fyp->simple_key_allowed) || fyp_tabsize(fyp); + } + + c = fy_parse_peek(fyp); + + /* not linebreak? we're done */ + if (!fyp_is_lb(fyp, c)) + goto done; + + /* line break */ + fy_advance(fyp, c); + + /* may start simple key (in block ctx) */ + if (!fyp->flow_level && !fyp->simple_key_allowed) { + fyp->simple_key_allowed = true; + fyp->tab_used_for_ws = false; + tabs_allowed = fyp->flow_level || !fyp->simple_key_allowed || fyp_tabsize(fyp); + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + } + } + + fyp_scan_debug(fyp, "%s: no-next-token", __func__); + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; + +done: + rc = fy_reader_input_scan_token_mark(fyr); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_reader_input_scan_token_mark() failed"); + + fyp_scan_debug(fyp, "%s: next token starts with c='%s'", __func__, + fy_utf8_format_a(fy_parse_peek(fyp), fyue_singlequote)); + return 0; +} + +static void fy_purge_required_simple_key_report(struct fy_parser *fyp, + struct fy_token *fyt, enum fy_token_type next_type) +{ + bool is_anchor, is_tag; + + is_anchor = fyt && fyt->type == FYTT_ANCHOR; + is_tag = fyt && fyt->type == FYTT_TAG; + + if (is_anchor || is_tag) { + if ((fyp->state == FYPS_BLOCK_MAPPING_VALUE || + fyp->state == FYPS_BLOCK_MAPPING_FIRST_KEY) && + next_type == FYTT_BLOCK_ENTRY) { + + FYP_TOKEN_ERROR(fyp, fyt, FYEM_SCAN, + "invalid %s indent for sequence", + is_anchor ? "anchor" : "tag"); + return; + } + + if (fyp->state == FYPS_BLOCK_MAPPING_VALUE && next_type == FYTT_SCALAR) { + FYP_TOKEN_ERROR(fyp, fyt, FYEM_SCAN, + "invalid %s indent for mapping", + is_anchor ? "anchor" : "tag"); + return; + } + } + + if (fyt) + FYP_TOKEN_ERROR(fyp, fyt, FYEM_SCAN, + "could not find expected ':'"); + else + FYP_PARSE_ERROR(fyp, 0, 1, FYEM_SCAN, + "could not find expected ':'"); +} + +static inline bool +fy_any_simple_keys(struct fy_parser *fyp) +{ + return !fy_simple_key_list_empty(&fyp->simple_keys); +} + +static int fy_purge_stale_simple_keys(struct fy_parser *fyp, bool *did_purgep, + enum fy_token_type next_type) +{ + struct fy_simple_key *fysk; + bool purge; + int line; + + *did_purgep = false; + while ((fysk = fy_simple_key_list_head(&fyp->simple_keys)) != NULL) { + +#ifdef FY_DEVMODE + fyp_scan_debug(fyp, "purge-check: flow_level=%d fysk->flow_level=%d fysk->mark.line=%d line=%d", + fyp->flow_level, fysk->flow_level, + fysk->mark.line, fyp_line(fyp)); + + fyp_debug_dump_simple_key(fyp, fysk, "purge-check: "); +#endif + + line = fysk->mark.line; + + /* in non-flow context we purge keys that are on different line */ + /* in flow context we purge only those with higher flow level */ + if (!fyp->flow_level) { + purge = fyp_line(fyp) > line; + } else { + purge = fyp->flow_level < fysk->flow_level; + /* also purge implicit complex keys on a different line */ + if (!purge && fysk->implicit_complex) { + purge = fyp_line(fyp) > line; + } + } + + if (!purge) + break; + + if (fysk->required) { + fy_purge_required_simple_key_report(fyp, fysk->token, next_type); + goto err_out; + } + +#ifdef FY_DEVMODE + fyp_debug_dump_simple_key(fyp, fysk, "purging: "); +#endif + + fy_simple_key_list_del(&fyp->simple_keys, fysk); + fy_parse_simple_key_recycle(fyp, fysk); + + *did_purgep = true; + } + + if (*did_purgep && fy_simple_key_list_empty(&fyp->simple_keys)) + fyp_scan_debug(fyp, "(purge) simple key list is now empty!"); + + return 0; + +err_out: + return -1; +} + +int fy_push_indent(struct fy_parser *fyp, int indent, bool generated_block_map, int indent_line) +{ + struct fy_indent *fyit; + + fyit = fy_parse_indent_alloc(fyp); + fyp_error_check(fyp, fyit != NULL, err_out, + "fy_indent_alloc() failed"); + + fyit->indent = fyp->indent; + fyit->indent_line = fyp->indent_line; + fyit->generated_block_map = fyp->generated_block_map; + + /* push */ + fy_indent_list_push(&fyp->indent_stack, fyit); + + /* update current state */ + fyp->parent_indent = fyp->indent; + fyp->indent = indent; + fyp->indent_line = indent_line; + if (fyp->parse_block_only && fyp->starting_indent < 0) + fyp->starting_indent = indent; + fyp->generated_block_map = generated_block_map; + + fyp_scan_debug(fyp, "push_indent %d -> %d - generated_block_map=%s\n", + fyp->parent_indent, fyp->indent, + fyp->generated_block_map ? "true" : "false"); + + return 0; + +err_out: + return -1; +} + +int fy_pop_indent(struct fy_parser *fyp) +{ + struct fy_indent *fyit; + int prev_indent __FY_DEBUG_UNUSED__; + + fyit = fy_indent_list_pop(&fyp->indent_stack); + if (!fyit) + return -1; + + prev_indent = fyp->indent; + + /* pop the indent and update */ + fyp->indent = fyit->indent; + fyp->generated_block_map = fyit->generated_block_map; + fyp->indent_line = fyit->indent_line; + + /* pop and recycle */ + fy_parse_indent_recycle(fyp, fyit); + + /* update the parent indent */ + fyit = fy_indent_list_head(&fyp->indent_stack); + fyp->parent_indent = fyit ? fyit->indent : -2; + + fyp_scan_debug(fyp, "pop indent %d -> %d (parent %d) - generated_block_map=%s\n", + prev_indent, fyp->indent, fyp->parent_indent, + fyp->generated_block_map ? "true" : "false"); + + return 0; +} + +int fy_parse_unroll_indent(struct fy_parser *fyp, int column) +{ + struct fy_token *fyt; + int rc; + + /* do nothing in flow context */ + if (fyp->flow_level) + return 0; + + /* pop while indentation level greater than argument */ + while (fyp->indent > column) { + + fyp_scan_debug(fyp, "unrolling: %d/%d", fyp->indent, column); + + /* create a block end token */ + fyt = fy_token_queue_simple(fyp, &fyp->queued_tokens, FYTT_BLOCK_END, 0); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple() failed"); + + rc = fy_pop_indent(fyp); + fyp_error_check(fyp, !rc, err_out, + "fy_pop_indent() failed"); + + /* the ident line has now moved */ + fyp->indent_line = fyp_line(fyp); + } + return 0; +err_out: + return -1; +} + +void fy_remove_all_simple_keys(struct fy_parser *fyp) +{ + struct fy_simple_key *fysk; + + fyp_scan_debug(fyp, "SK: removing all"); + + while ((fysk = fy_simple_key_list_pop(&fyp->simple_keys)) != NULL) + fy_parse_simple_key_recycle(fyp, fysk); + + fyp->simple_key_allowed = true; + fyp->tab_used_for_ws = false; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); +} + +struct fy_simple_key *fy_would_remove_required_simple_key(struct fy_parser *fyp) +{ + struct fy_simple_key *fysk; + + /* no simple key? */ + for (fysk = fy_simple_key_list_head(&fyp->simple_keys); + fysk && fysk->flow_level >= fyp->flow_level; + fysk = fy_simple_key_next(&fyp->simple_keys, fysk)) { + if (fysk->required) + return fysk; + } + + return NULL; +} + +int fy_remove_simple_key(struct fy_parser *fyp, enum fy_token_type next_type) +{ + struct fy_simple_key *fysk; + + /* no simple key? */ + while ((fysk = fy_simple_key_list_first(&fyp->simple_keys)) != NULL && + fysk->flow_level >= fyp->flow_level) { + +#ifdef FY_DEVMODE + fyp_debug_dump_simple_key(fyp, fysk, "removing: "); +#endif + + /* remove it from the list */ + fy_simple_key_list_del(&fyp->simple_keys, fysk); + + if (fysk->required) { + fy_purge_required_simple_key_report(fyp, fysk->token, next_type); + goto err_out; + } + + fy_parse_simple_key_recycle(fyp, fysk); + } + + return 0; + +err_out: + fy_parse_simple_key_recycle(fyp, fysk); + return -1; +} + +struct fy_simple_key *fy_simple_key_find(struct fy_parser *fyp, const struct fy_token *fyt) +{ + struct fy_simple_key *fysk; + + if (!fyt) + return NULL; + + /* no simple key? */ + for (fysk = fy_simple_key_list_head(&fyp->simple_keys); fysk; + fysk = fy_simple_key_next(&fyp->simple_keys, fysk)) + if (fysk->token == fyt) + return fysk; + + return NULL; +} + +int fy_save_simple_key(struct fy_parser *fyp, struct fy_mark *mark, struct fy_mark *end_mark, + struct fy_token *fyt, bool required, int flow_level, + enum fy_token_type next_type) +{ + struct fy_simple_key *fysk; + bool did_purge; + int rc; + + fyp_error_check(fyp, fyt && mark && end_mark, err_out, + "illegal arguments to fy_save_simple_key"); + + if (fy_any_simple_keys(fyp)) { + rc = fy_purge_stale_simple_keys(fyp, &did_purge, next_type); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_purge_stale_simple_keys() failed"); + } + + /* if no simple key is allowed, don't save */ + if (!fyp->simple_key_allowed) { + fyp_scan_debug(fyp, "not saving simple key; not allowed"); + return 0; + } + + /* remove pending complex key mark if in non flow context and a new line */ + if (!fyp->flow_level && fyp->pending_complex_key_column >= 0 && + mark->line > fyp->pending_complex_key_mark.line && + mark->column <= fyp->pending_complex_key_mark.column ) { + + fyp_scan_debug(fyp, "resetting pending_complex_key mark->line=%d line=%d\n", + mark->line, fyp->pending_complex_key_mark.line); + + fyp->pending_complex_key_column = -1; + fyp_scan_debug(fyp, "pending_complex_key_column -> %d", + fyp->pending_complex_key_column); + } + + fysk = fy_simple_key_list_head(&fyp->simple_keys); + + /* create new simple key if it does not exist or if has flow level less */ + if (!fysk || fysk->flow_level < fyp->flow_level) { + + fysk = fy_parse_simple_key_alloc(fyp); + fyp_error_check(fyp, fysk != NULL, err_out, + "fy_simple_key_alloc()"); + + fyp_scan_debug(fyp, "new simple key"); + + fy_simple_key_list_push(&fyp->simple_keys, fysk); + + } else { + fyp_error_check(fyp, !fysk->required, err_out, + "cannot save simple key, top is required"); + + if (fysk == fy_simple_key_list_tail(&fyp->simple_keys)) + fyp_scan_debug(fyp, "(reuse) simple key list is now empty!"); + + fyp_scan_debug(fyp, "reusing simple key"); + + } + + fysk->mark = *mark; + fysk->end_mark = *end_mark; + + fysk->required = required; + fysk->token = fyt; + fysk->flow_level = flow_level; + + /* if this is a an implicit flow collection key */ + fysk->implicit_complex = fyp->pending_complex_key_column < 0 && + (fyt->type == FYTT_FLOW_MAPPING_START || fyt->type == FYTT_FLOW_SEQUENCE_START); + + +#ifdef FY_DEVMODE + fyp_debug_dump_simple_key_list(fyp, &fyp->simple_keys, fysk, "fyp->simple_keys (saved): "); +#endif + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +struct fy_simple_key_mark { + struct fy_mark mark; + bool required; + int flow_level; +}; + +void fy_get_simple_key_mark(struct fy_parser *fyp, struct fy_simple_key_mark *fyskm) +{ + fy_get_mark(fyp, &fyskm->mark); + fyskm->flow_level = fyp->flow_level; + fyskm->required = !fyp->flow_level && fyp->indent == fyp_column(fyp); +} + +int fy_save_simple_key_mark(struct fy_parser *fyp, + struct fy_simple_key_mark *fyskm, + enum fy_token_type next_type, + struct fy_mark *end_markp) +{ + struct fy_mark end_mark; + + if (!end_markp) { + fy_get_mark(fyp, &end_mark); + end_markp = &end_mark; + } + + return fy_save_simple_key(fyp, &fyskm->mark, end_markp, + fy_token_list_last(&fyp->queued_tokens), + fyskm->required, fyskm->flow_level, + next_type); +} + +int fy_parse_flow_push(struct fy_parser *fyp) +{ + struct fy_flow *fyf; + + fyf = fy_parse_flow_alloc(fyp); + fyp_error_check(fyp, fyf != NULL, err_out, + "fy_flow_alloc() failed!"); + fyf->flow = fyp->flow; + + fyf->pending_complex_key_column = fyp->pending_complex_key_column; + fyf->pending_complex_key_mark = fyp->pending_complex_key_mark; + + fyp_scan_debug(fyp, "flow_push: flow=%d pending_complex_key_column=%d", + (int)fyf->flow, + fyf->pending_complex_key_column); + + fy_flow_list_push(&fyp->flow_stack, fyf); + + if (fyp->pending_complex_key_column >= 0) { + fyp->pending_complex_key_column = -1; + fyp_scan_debug(fyp, "pending_complex_key_column -> %d", + fyp->pending_complex_key_column); + } + + return 0; +err_out: + return -1; +} + +int fy_parse_flow_pop(struct fy_parser *fyp) +{ + struct fy_flow *fyf; + + fyf = fy_flow_list_pop(&fyp->flow_stack); + fyp_error_check(fyp, fyf, err_out, + "no flow to pop"); + + fyp->flow = fyf->flow; + fyp->pending_complex_key_column = fyf->pending_complex_key_column; + fyp->pending_complex_key_mark = fyf->pending_complex_key_mark; + + fy_parse_flow_recycle(fyp, fyf); + + fyp_scan_debug(fyp, "flow_pop: flow=%d pending_complex_key_column=%d", + (int)fyp->flow, + fyp->pending_complex_key_column); + + return 0; + +err_out: + return -1; +} + +/* special case for allowing whitespace (including tabs) after -?: */ +static int fy_ws_indentation_check(struct fy_parser *fyp, bool *found_tabp, struct fy_mark *tab_mark) +{ + int c, adv, tab_adv; + bool indentation, found_tab; + + found_tab = false; + + /* not meaning in flow mode */ + if (fyp->flow_level) + goto out; + + /* scan forward, keeping track if we found a tab */ + adv = 0; + tab_adv = -1; + if (tab_mark) + fy_get_mark(fyp, tab_mark); + while (fy_is_ws(c = fy_parse_peek_at(fyp, adv))) { + if (!found_tab && fy_is_tab(c)) { + found_tab = true; + tab_adv = adv; + /* XXX somewhat hacky, space is 1 char only so adjust */ + if (tab_mark) { + tab_mark->input_pos += adv; + tab_mark->column += adv; + } + } + adv++; + } + + if (found_tab) { + indentation = fy_utf8_strchr("?:|>", c) || + (c == '-' && fyp_is_blankz(fyp, fy_parse_peek_at(fyp, adv + 1))); + + /* any kind of block indentation is not allowed */ + FYP_PARSE_ERROR_CHECK(fyp, tab_adv, 1, FYEM_SCAN, + !indentation, err_out, + "cannot use tab for indentation of block entry"); + fy_advance_by(fyp, tab_adv + 1); + } + + /* now chomp spaces only afterwards */ + while (fy_is_space(c = fy_parse_peek(fyp))) + fy_advance(fyp, c); + +out: + if (found_tabp) + *found_tabp = found_tab; + + return 0; + +err_out: + return -1; +} + +int fy_fetch_stream_start(struct fy_parser *fyp) +{ + struct fy_token *fyt; + + /* simple key is allowed */ + fyp->simple_key_allowed = true; + fyp->tab_used_for_ws = false; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + fyt = fy_token_queue_simple(fyp, &fyp->queued_tokens, FYTT_STREAM_START, 0); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple() failed"); + return 0; + +err_out: + return -1; +} + +int fy_fetch_stream_end(struct fy_parser *fyp) +{ + struct fy_token *fyt; + int rc; + + /* only reset the stream in regular mode */ + if (!fyp->parse_flow_only && !fyp->parse_block_only) + fy_reader_stream_end(fyp->reader); + + fy_remove_all_simple_keys(fyp); + + if (fyp_block_mode(fyp)) { + rc = fy_parse_unroll_indent(fyp, -1); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_unroll_indent() failed"); + } + + fyt = fy_token_queue_simple(fyp, &fyp->queued_tokens, FYTT_STREAM_END, 0); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple() failed"); + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_scan_tag_uri_length(struct fy_parser *fyp, int start) +{ + int c, cn, length; + ssize_t offset, offset1; + + /* first find the utf8 length of the uri */ + length = 0; + offset = -1; + while (fy_is_uri(c = fy_parse_peek_at_internal(fyp, start + length, &offset))) { + + offset1 = offset; + cn = fy_parse_peek_at_internal(fyp, start + length + 1, &offset1); + + /* special handling for detecting URIs ending in ,}] */ + if (fyp_is_blankz(fyp, cn) && fy_utf8_strchr(",}]", c)) + break; + + length++; + } + + return length; +} + +bool fy_scan_tag_uri_is_valid(struct fy_parser *fyp, int start, int length) +{ + int i, j, k, width, c, w; + uint8_t octet, esc_octets[4]; + ssize_t offset; + + offset = -1; + for (i = 0; i < length; i++) { + c = fy_parse_peek_at_internal(fyp, start + i, &offset); + if (c != '%') + continue; + /* reset cursor */ + offset = -1; + + width = 0; + k = 0; + do { + /* % escape */ + FYP_PARSE_ERROR_CHECK(fyp, start + i, 1, FYEM_SCAN, + (length - i) >= 3, err_out, + "short URI escape"); + + if (width > 0) { + c = fy_parse_peek_at(fyp, start + i); + + FYP_PARSE_ERROR_CHECK(fyp, start + i, 1, FYEM_SCAN, + c == '%', err_out, + "missing URI escape"); + } + + octet = 0; + + for (j = 0; j < 2; j++) { + c = fy_parse_peek_at(fyp, start + i + 1 + j); + + FYP_PARSE_ERROR_CHECK(fyp, start + i + 1 + j, 1, FYEM_SCAN, + fy_is_hex(c), err_out, + "non hex URI escape"); + octet <<= 4; + if (c >= '0' && c <= '9') + octet |= c - '0'; + else if (c >= 'a' && c <= 'f') + octet |= 10 + c - 'a'; + else + octet |= 10 + c - 'A'; + } + if (!width) { + width = fy_utf8_width_by_first_octet(octet); + + FYP_PARSE_ERROR_CHECK(fyp, start + i + 1 + j, 1, FYEM_SCAN, + width >= 1 && width <= 4, err_out, + "bad width for hex URI escape"); + k = 0; + } + esc_octets[k++] = octet; + + /* skip over the 3 character escape */ + i += 3; + + } while (--width > 0); + + /* now convert to utf8 */ + c = fy_utf8_get(esc_octets, k, &w); + + FYP_PARSE_ERROR_CHECK(fyp, start + i, 1 + j, FYEM_SCAN, + c >= 0, err_out, + "bad utf8 URI escape"); + } + return true; +err_out: + return false; +} + +int fy_scan_tag_handle_length(struct fy_parser *fyp, int start) +{ + int c, length, i, width; + ssize_t offset; + uint8_t octet; + bool first, was_esc; + + length = 0; + + offset = -1; + c = fy_parse_peek_at_internal(fyp, start + length, &offset); + + FYP_PARSE_ERROR_CHECK(fyp, start + length, 1, FYEM_SCAN, + c == '!', err_out, + "invalid tag handle start"); + length++; + + /* get first character of the tag */ + c = fy_parse_peek_at_internal(fyp, start + length, &offset); + + if (fy_is_ws(c)) + return length; + + /* if first character is !, empty handle */ + if (c == '!') { + length++; + return length; + } + + first = true; + was_esc = false; + + /* now loop while it's alphanumeric */ + for (;;) { + if (c == '%') { + + octet = 0; + + for (i = 0; i < 2; i++) { + c = fy_parse_peek_at_internal(fyp, start + length + 1 + i, &offset); + FYP_PARSE_ERROR_CHECK(fyp, start + length + 1 + i, 1, FYEM_SCAN, + fy_is_hex(c), err_out, + "non hex URI escape"); + octet <<= 4; + if (c >= '0' && c <= '9') + octet |= c - '0'; + else if (c >= 'a' && c <= 'f') + octet |= 10 + c - 'a'; + else + octet |= 10 + c - 'A'; + } + + width = fy_utf8_width_by_first_octet(octet); + FYP_PARSE_ERROR_CHECK(fyp, start + length, 3, FYEM_SCAN, + width == 1, err_out, + "Illegal non 1 byte utf8 tag handle character"); + c = octet; + was_esc = true; + + } else + was_esc = false; + + if ((first && fy_is_first_alnum(c)) || (!first && fy_is_alnum(c))) + length += was_esc ? 3 : 1; + else + break; + + first = false; + c = fy_parse_peek_at_internal(fyp, start + length, &offset); + } + + /* if last character is !, copy it */ + if (c == '!') + length++; + + return length; + +err_out: + return -1; +} + +int fy_scan_yaml_version(struct fy_parser *fyp, struct fy_version *vers) +{ + int c, length, start_length, num; + ssize_t offset; + + memset(vers, 0, sizeof(*vers)); + + /* now loop while it's numeric */ + length = 0; + offset = -1; + num = 0; + while (fy_is_num(c = fy_parse_peek_at_internal(fyp, length, &offset))) { + length++; + num = num * 10; + num += c - '0'; + } + vers->major = num; + + FYP_PARSE_ERROR_CHECK(fyp, length, 1, FYEM_SCAN, + length > 0, err_out, + "version directive missing major number"); + + FYP_PARSE_ERROR_CHECK(fyp, length, 1, FYEM_SCAN, + c == '.', err_out, + "version directive missing dot separator"); + length++; + + start_length = length; + num = 0; + while (fy_is_num(c = fy_parse_peek_at_internal(fyp, length, &offset))) { + length++; + num = num * 10; + num += c - '0'; + } + vers->minor = num; + + /* note that the version is not checked for validity here */ + + FYP_PARSE_ERROR_CHECK(fyp, length, 1, FYEM_SCAN, + length > start_length, err_out, + "version directive missing minor number"); + + return length; + +err_out: + return -1; +} + +int fy_scan_tag_handle(struct fy_parser *fyp, bool is_directive, + struct fy_atom *handle) +{ + int length; + + length = fy_scan_tag_handle_length(fyp, 0); + fyp_error_check(fyp, length > 0, err_out, + "fy_scan_tag_handle_length() failed"); + + fy_fill_atom(fyp, length, handle); + + return 0; + +err_out: + return -1; +} + + +int fy_scan_tag_uri(struct fy_parser *fyp, bool is_directive, + struct fy_atom *handle) +{ + int length; + bool is_valid; + + length = fy_scan_tag_uri_length(fyp, 0); + fyp_error_check(fyp, length > 0, err_out, + "fy_scan_tag_uri_length() failed"); + + is_valid = fy_scan_tag_uri_is_valid(fyp, 0, length); + fyp_error_check(fyp, is_valid, err_out, + "tag URI is invalid"); + + fy_fill_atom(fyp, length, handle); + handle->style = FYAS_URI; /* this is a URI, need to handle URI escapes */ + + return 0; + +err_out: + return -1; +} + +int fy_scan_directive(struct fy_parser *fyp) +{ + int c, advance, version_length, tag_length, uri_length; + struct fy_version vers; + enum fy_reader_mode rdmode; + enum fy_token_type type = FYTT_NONE; + struct fy_atom handle; + bool is_uri_valid; + struct fy_token *fyt; + int i, lastc; + + if (!fy_parse_strcmp(fyp, "YAML") && fy_is_ws(fy_parse_peek_at(fyp, 4))) { + advance = 5; + type = FYTT_VERSION_DIRECTIVE; + } else if (!fy_parse_strcmp(fyp, "TAG") && fy_is_ws(fy_parse_peek_at(fyp, 3))) { + advance = 4; + type = FYTT_TAG_DIRECTIVE; + } else { + /* skip until linebreak (or #) */ + i = 0; + lastc = -1; + while ((c = fy_parse_peek_at(fyp, i)) >= 0 && !fyp_is_lb(fyp, c)) { + if (fy_is_ws(lastc) && c == '#') + break; + lastc = c; + i++; + } + + FYP_PARSE_WARNING(fyp, 0, i, FYEM_SCAN, + "Unsupported directive"); + + FYP_PARSE_ERROR_CHECK(fyp, i, 1, FYEM_SCAN, c >= 0 || c == FYUG_EOF, err_out, + "Bad UTF8 input while scanning unknown %%TAG directive"); + + if (fy_is_ws(lastc) && c == '#') { + while ((c = fy_parse_peek_at(fyp, i)) >= 0 && !fyp_is_lb(fyp, c)) + i++; + } + + fy_advance_by(fyp, i); + + /* skip over linebreak too */ + if (fyp_is_lb(fyp, c)) + fy_advance(fyp, c); + + /* bump activity counter */ + fyp->token_activity_counter++; + + return 0; + } + + fyp_error_check(fyp, type != FYTT_NONE, err_out, + "neither YAML|TAG found"); + + /* advance */ + fy_advance_by(fyp, advance); + + /* skip white space */ + while (fy_is_ws(c = fy_parse_peek(fyp))) + fy_advance(fyp, c); + + fy_fill_atom_start(fyp, &handle); + + /* for version directive, parse it */ + if (type == FYTT_VERSION_DIRECTIVE) { + + version_length = fy_scan_yaml_version(fyp, &vers); + fyp_error_check(fyp, version_length > 0, err_out, + "fy_scan_yaml_version() failed"); + + /* save the version per stream */ + fyp->stream_version = vers; + + /* set the reader's mode according to the version just scanned */ + rdmode = fy_reader_calculate_mode(&vers, false); + fy_reader_set_mode(fyp->reader, rdmode); + + fy_advance_by(fyp, version_length); + + fy_fill_atom_end(fyp, &handle); + + fyt = fy_token_queue(fyp, FYTT_VERSION_DIRECTIVE, &handle, &vers); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue() failed"); + + } else { + + tag_length = fy_scan_tag_handle_length(fyp, 0); + fyp_error_check(fyp, tag_length > 0, err_out, + "fy_scan_tag_handle_length() failed"); + + fy_advance_by(fyp, tag_length); + + c = fy_parse_peek(fyp); + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_is_ws(c), err_out, + "missing whitespace after TAG"); + + /* skip white space */ + while (fy_is_ws(c = fy_parse_peek(fyp))) + fy_advance(fyp, c); + + uri_length = fy_scan_tag_uri_length(fyp, 0); + fyp_error_check(fyp, uri_length > 0, err_out, + "fy_scan_tag_uri_length() failed"); + + is_uri_valid = fy_scan_tag_uri_is_valid(fyp, 0, uri_length); + fyp_error_check(fyp, is_uri_valid, err_out, + "tag URI is invalid"); + + fy_advance_by(fyp, uri_length); + + fy_fill_atom_end(fyp, &handle); + handle.style = FYAS_URI; + + fyt = fy_token_queue(fyp, FYTT_TAG_DIRECTIVE, &handle, tag_length, uri_length, false); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue() failed"); + } + + /* skip until linebreak (or #) */ + i = 0; + lastc = -1; + while ((c = fy_parse_peek_at(fyp, i)) >= 0 && !fyp_is_lb(fyp, c)) { + if (fy_is_ws(lastc) && c == '#') + break; + + FYP_PARSE_ERROR_CHECK(fyp, i, 1, FYEM_SCAN, + fy_is_ws(c) || fyp_is_lb(fyp, c), err_out, + "garbage after %s directive", + type == FYTT_VERSION_DIRECTIVE ? "version" : "tag"); + lastc = c; + i++; + } + + fy_advance_by(fyp, i); + + /* skip over linebreak */ + if (fyp_is_lb(fyp, c)) + fy_advance(fyp, c); + + return 0; +err_out: + return -1; +} + +int fy_fetch_directive(struct fy_parser *fyp) +{ + int rc; + + fy_remove_all_simple_keys(fyp); + + if (fyp_block_mode(fyp)) { + rc = fy_parse_unroll_indent(fyp, -1); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_unroll_indent() failed"); + } + + rc = fy_scan_directive(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_scan_directive() failed"); + + return 0; + +err_out_rc: + return rc; +} + +int fy_fetch_document_indicator(struct fy_parser *fyp, enum fy_token_type type) +{ + int rc, c; + struct fy_token *fyt; + + fy_remove_all_simple_keys(fyp); + + if (fyp_block_mode(fyp)) { + rc = fy_parse_unroll_indent(fyp, -1); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_unroll_indent() failed"); + } + + fyp->simple_key_allowed = false; + fyp->tab_used_for_ws = false; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + fyt = fy_token_queue_simple(fyp, &fyp->queued_tokens, type, 3); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple() failed"); + + /* skip whitespace after the indicator */ + while (fy_is_ws(c = fy_parse_peek(fyp))) + fy_advance(fyp, c); + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +static inline bool fy_flow_indent_check_internal(struct fy_parser *fyp, int column, int indent) +{ + return (!fyp->flow_level || column > indent) || + ((fyp->cfg.flags & FYPCF_SLOPPY_FLOW_INDENTATION) && fyp->flow_level); +} + +static inline bool fy_flow_indent_check(struct fy_parser *fyp) +{ + return fy_flow_indent_check_internal(fyp, fyp_column(fyp), fyp->indent); +} + +static inline bool fy_block_indent_check(struct fy_parser *fyp) +{ + return fyp->flow_level > 0 || fyp_column(fyp) > fyp->indent; +} + +int fy_fetch_flow_collection_mark_start(struct fy_parser *fyp, int c) +{ + enum fy_token_type type; + struct fy_simple_key_mark skm; + const char *typestr; + int rc = -1; + struct fy_token *fyt; + + if (c == '[') { + type = FYTT_FLOW_SEQUENCE_START; + typestr = "sequence"; + } else { + type = FYTT_FLOW_MAPPING_START; + typestr = "mapping"; + } + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_flow_indent_check(fyp), err_out, + "wrongly indented %s start in flow mode", typestr); + + fy_get_simple_key_mark(fyp, &skm); + + fyt = fy_token_queue_simple(fyp, &fyp->queued_tokens, type, 1); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple() failed"); + + /* json does not have complex keys */ + if (!fyp_json_mode(fyp)) { + rc = fy_save_simple_key_mark(fyp, &skm, type, NULL); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_save_simple_key_mark() failed"); + } + + /* increase flow level */ + fyp->flow_level++; + fyp_error_check(fyp, fyp->flow_level, err_out, + "overflow for the flow level counter"); + + /* push the current flow to the stack */ + rc = fy_parse_flow_push(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_flow_push() failed"); + /* set the current flow mode */ + fyp->flow = c == '[' ? FYFT_SEQUENCE : FYFT_MAP; + + fyp->simple_key_allowed = true; + fyp->tab_used_for_ws = false; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + /* the comment indicator must have at least a space */ + c = fy_parse_peek(fyp); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + c != '#', err_out, + "invalid comment after %s start", typestr); + return 0; + +err_out: + rc = -1; + +err_out_rc: + return rc; +} + +int fy_fetch_flow_collection_mark_end(struct fy_parser *fyp, int c) +{ + enum fy_token_type type = FYTT_NONE; + enum fy_flow_type flow; + const char *typestr, *markerstr; + int i, rc; + bool did_purge; + struct fy_mark mark; + struct fy_token *fyt; + + fy_get_mark(fyp, &mark); + + if (c == ']') { + flow = FYFT_SEQUENCE; + type = FYTT_FLOW_SEQUENCE_END; + typestr = "sequence"; + markerstr = "bracket"; + } else { + flow = FYFT_MAP; + type = FYTT_FLOW_MAPPING_END; + typestr = "mapping"; + markerstr = "brace"; + } + + FYP_MARK_ERROR_CHECK(fyp, &fyp->last_comma_mark, &fyp->last_comma_mark, FYEM_SCAN, + !fyp_json_mode(fyp) || !fyp->last_was_comma, err_out, + "JSON disallows trailing comma before closing %s", markerstr); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_flow_indent_check(fyp), err_out, + "wrongly indented %s end in flow mode", typestr); + + rc = fy_remove_simple_key(fyp, type); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_remove_simple_key() failed"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fyp->flow_level, err_out, + "flow %s with invalid extra closing %s", + typestr, markerstr); + + fyp->flow_level--; + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fyp->flow == flow, err_out, + "mismatched flow %s end", typestr); + + /* pop the flow type */ + rc = fy_parse_flow_pop(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_flow_pop() failed"); + + fyp->simple_key_allowed = false; + fyp->tab_used_for_ws = false; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", + fyp->simple_key_allowed ? "true" : "false"); + + fyt = fy_token_queue_simple(fyp, &fyp->queued_tokens, type, 1); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple() failed"); + + if (fyp->parse_flow_only && fyp->flow_level == 0) { + rc = fy_fetch_stream_end(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_stream_end() failed"); + return 0; + } + + /* the comment indicator must have at least a space */ + c = fy_parse_peek(fyp); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + c != '#', err_out, + "invalid comment after end of flow %s", typestr); + + /* due to the weirdness with simple keys and multiline flow keys scan forward + * until a linebreak, ';', or anything else */ + for (i = 0; ; i++) { + c = fy_parse_peek_at(fyp, i); + if (c < 0 || c == ':' || fyp_is_lb(fyp, c) || !fy_is_ws(c)) + break; + } + + /* we must be a key, purge */ + if (c == ':') { + if (fy_any_simple_keys(fyp)) { + rc = fy_purge_stale_simple_keys(fyp, &did_purge, type); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_purge_stale_simple_keys() failed"); + + /* if we did purge and the the list is now empty, we're hosed */ + if (did_purge && fy_simple_key_list_empty(&fyp->simple_keys)) { + FYP_PARSE_ERROR(fyp, 0, 1, FYEM_SCAN, + "invalid multiline flow %s key ", typestr); + goto err_out; + } + } + } + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_fetch_flow_collection_entry(struct fy_parser *fyp, int c) +{ + enum fy_token_type type = FYTT_NONE; + struct fy_token *fyt, *fyt_last; + struct fy_atom *handle; + int rc; + + type = FYTT_FLOW_ENTRY; + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_flow_indent_check(fyp), err_out, + "wrongly indented entry seperator in flow mode"); + + /* transform '? a,' to '? a: ,' */ + if (fyp->pending_complex_key_column >= 0) { + + fyt = fy_token_queue_simple(fyp, &fyp->queued_tokens, FYTT_VALUE, 0); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple() failed"); + + fyp->pending_complex_key_column = -1; + + } + + rc = fy_remove_simple_key(fyp, type); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_remove_simple_key() failed"); + + fyp->simple_key_allowed = true; + fyp->tab_used_for_ws = false; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + fyt_last = fy_token_list_tail(&fyp->queued_tokens); + + fyt = fy_token_queue_simple(fyp, &fyp->queued_tokens, type, 1); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple() failed"); + + /* the comment indicator must have at least a space */ + c = fy_parse_peek(fyp); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + c != '#', err_out, + "invalid comment after comma"); + + /* skip white space */ + while (fy_is_ws(c = fy_parse_peek(fyp))) + fy_advance(fyp, c); + + if (c == '#') { + if (fyt_last) + fyt = fyt_last; + + handle = NULL; + if (fyp->cfg.flags & FYPCF_PARSE_COMMENTS) + handle = fy_token_comment_handle(fyt, fycp_right, true); + + rc = fy_scan_comment(fyp, handle, true); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_scan_comment() failed"); + } + + return 0; +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_fetch_block_entry(struct fy_parser *fyp, int c) +{ + int rc; + struct fy_mark mark; + struct fy_simple_key *fysk; + struct fy_token *fyt; + + fyp_error_check(fyp, c == '-', err_out, + "illegal block entry"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + (!fyp->flow_level || (fyp_column(fyp) + 2) > fyp->indent) || + ((fyp->cfg.flags & FYPCF_SLOPPY_FLOW_INDENTATION) && fyp->flow_level), err_out, + "wrongly indented block sequence in flow mode"); + + if (!(fyp->flow_level || fyp->simple_key_allowed)) { + if (!fyp->simple_key_allowed && fyp->state == FYPS_BLOCK_MAPPING_VALUE) + FYP_PARSE_ERROR(fyp, 0, 1, FYEM_SCAN, + "block sequence on the same line as a mapping key"); + else if (fyp->state == FYPS_BLOCK_SEQUENCE_FIRST_ENTRY || + fyp->state == FYPS_BLOCK_SEQUENCE_ENTRY) + FYP_PARSE_ERROR(fyp, 0, 1, FYEM_SCAN, + "block sequence on the same line as a previous item"); + else + FYP_PARSE_ERROR(fyp, 0, 1, FYEM_SCAN, + "block sequence entries not allowed in this context"); + goto err_out; + } + + /* we have to save the start mark */ + fy_get_mark(fyp, &mark); + + if (fyp_block_mode(fyp) && fyp->indent < fyp_column(fyp)) { + + /* push the new indent level */ + rc = fy_push_indent(fyp, fyp_column(fyp), false, fyp_line(fyp)); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_push_indent() failed"); + + fyt = fy_token_queue_simple_internal(fyp, &fyp->queued_tokens, FYTT_BLOCK_SEQUENCE_START, 0); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple_internal() failed"); + } + + if (c == '-' && fyp->flow_level) { + /* this is an error, but we let the parser catch it */ + ; + } + + fysk = fy_would_remove_required_simple_key(fyp); + + if (fysk) { + if (fysk->token) { + if (fysk->token->type == FYTT_ANCHOR || fysk->token->type == FYTT_TAG) + FYP_TOKEN_ERROR(fyp, fysk->token, FYEM_SCAN, + "invalid %s indent for sequence", + fysk->token->type == FYTT_ANCHOR ? + "anchor" : "tag"); + else + FYP_TOKEN_ERROR(fyp, fysk->token, FYEM_SCAN, + "missing ':'"); + } else + FYP_PARSE_ERROR(fyp, 0, 1, FYEM_SCAN, + "missing ':'"); + goto err_out; + } + + rc = fy_remove_simple_key(fyp, FYTT_BLOCK_ENTRY); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_remove_simple_key() failed"); + + fyp->simple_key_allowed = true; + fyp->tab_used_for_ws = false; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + fyt = fy_token_queue_simple(fyp, &fyp->queued_tokens, FYTT_BLOCK_ENTRY, 1); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple() failed"); + + rc = fy_ws_indentation_check(fyp, NULL, NULL); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_ws_indentation_check() failed"); + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_fetch_key(struct fy_parser *fyp, int c) +{ + int rc; + struct fy_mark mark; + struct fy_simple_key_mark skm; + bool target_simple_key_allowed; + struct fy_token *fyt; + struct fy_atom *handle; + bool found_tab; + struct fy_mark tab_mark; + + fyp_error_check(fyp, c == '?', err_out, + "illegal block entry or key mark"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_flow_indent_check(fyp), err_out, + "wrongly indented mapping key in flow mode"); + + fy_get_simple_key_mark(fyp, &skm); + + /* we have to save the start mark */ + fy_get_mark(fyp, &mark); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fyp->flow_level || fyp->simple_key_allowed, err_out, + "invalid mapping key (not allowed in this context)"); + + if (fyp_block_mode(fyp) && fyp->indent < fyp_column(fyp)) { + + /* push the new indent level */ + rc = fy_push_indent(fyp, fyp_column(fyp), true, fyp_line(fyp)); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_push_indent() failed"); + + fyt = fy_token_queue_simple_internal(fyp, &fyp->queued_tokens, FYTT_BLOCK_MAPPING_START, 0); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple_internal() failed"); + } + + rc = fy_remove_simple_key(fyp, FYTT_KEY); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_remove_simple_key() failed"); + + target_simple_key_allowed = !fyp->flow_level; + + fyp->pending_complex_key_column = fyp_column(fyp); + fyp->pending_complex_key_mark = mark; + fyp_scan_debug(fyp, "pending_complex_key_column %d", + fyp->pending_complex_key_column); + + fyt = fy_token_queue_simple(fyp, &fyp->queued_tokens, FYTT_KEY, 1); + fyp_error_check(fyp, fyt, err_out_rc, + "fy_token_queue_simple() failed"); + /* extra KEY data */ + fyt->key.flow_level = fyp->flow_level; + + fyp->simple_key_allowed = target_simple_key_allowed; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + rc = fy_ws_indentation_check(fyp, &found_tab, &tab_mark); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_ws_indentation_check() failed"); + + /* record whether a tab was used for indentation */ + if (fyp->simple_key_allowed && found_tab) { + fyp->tab_used_for_ws = true; + fyp->last_tab_used_for_ws_mark = tab_mark; + } else + fyp->tab_used_for_ws = false; // XXX + + /* comment? */ + if (c == '#') { + + handle = NULL; + if (fyp->cfg.flags & FYPCF_PARSE_COMMENTS) + handle = fy_token_comment_handle(fyt, fycp_right, true); + + rc = fy_scan_comment(fyp, handle, false); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_scan_comment() failed"); + } + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_fetch_value(struct fy_parser *fyp, int c) +{ + struct fy_token_list sk_tl; + struct fy_simple_key *fysk = NULL; + struct fy_mark mark, mark_insert, mark_end_insert; + struct fy_token *fyt_insert, *fyt; + bool target_simple_key_allowed, is_complex, has_bmap; + bool push_bmap_start, push_key_only, did_purge, final_complex_key; + bool is_multiline __FY_DEBUG_UNUSED__; + struct fy_atom *chandle; + bool found_tab; + struct fy_mark tab_mark; + int rc; + + fyp_error_check(fyp, c == ':', err_out, + "illegal value mark"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + !fyp_json_mode(fyp) || fyp->flow == FYFT_MAP, err_out, + "JSON considers keys when not in mapping context invalid"); + + /* special handling for :: weirdness */ + if (!fyp_json_mode(fyp) && fyp->flow_level > 0) { + int adv, nextc, nextcol, tabsize, indent; + + /* this requires some explanation... + * we need to detect x::x, x: :x, and x:\n:x as the same + */ + adv = 1; + indent = fyp->indent; + nextcol = fyp_column(fyp) + 1; + tabsize = fyp_tabsize(fyp); + + while ((nextc = fy_parse_peek_at(fyp, adv)) > 0) { + + if (fyp_is_lb(fyp, nextc)) + nextcol = 0; + else if (fy_is_tab(nextc)) { + if (tabsize) + nextcol += tabsize - (nextcol % tabsize); + else + nextcol++; + } else if (fy_is_space(nextc)) + nextcol++; + else { + if (!fy_flow_indent_check_internal(fyp, nextcol, indent)) + nextc = -1; + break; + } + + adv++; + } + + fyp->colon_follows_colon = nextc == ':'; + } else + fyp->colon_follows_colon = false; + + fy_get_mark(fyp, &mark); + + fy_token_list_init(&sk_tl); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_flow_indent_check(fyp), err_out, + "wrongly indented mapping value in flow mode"); + + if (fy_any_simple_keys(fyp)) { + rc = fy_purge_stale_simple_keys(fyp, &did_purge, FYTT_VALUE); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_purge_stale_simple_keys() failed"); + } + + /* get the simple key (if available) for the value */ + fysk = fy_simple_key_list_head(&fyp->simple_keys); + if (fysk && fysk->flow_level == fyp->flow_level) + fy_simple_key_list_del(&fyp->simple_keys, fysk); + else + fysk = NULL; + + if (!fysk) { + fyp_scan_debug(fyp, "no simple key flow_level=%d", fyp->flow_level); + + fyt_insert = fy_token_list_tail(&fyp->queued_tokens); + mark_insert = mark; + mark_end_insert = mark; + } else { + assert(fysk->flow_level == fyp->flow_level); + fyt_insert = fysk->token; + mark_insert = fysk->mark; + mark_end_insert = fysk->end_mark; + + fyp_scan_debug(fyp, "have simple key flow_level=%d", fyp->flow_level); + } + + fyp_scan_debug(fyp, "flow_level=%d, column=%d parse_indent=%d", + fyp->flow_level, mark_insert.column, fyp->indent); + + is_complex = fyp->pending_complex_key_column >= 0; + final_complex_key = is_complex && (fyp->flow_level || fyp_column(fyp) <= fyp->pending_complex_key_mark.column); + is_multiline = mark_end_insert.line < fyp_line(fyp); + has_bmap = fyp->generated_block_map; + push_bmap_start = (!fyp->flow_level && mark_insert.column > fyp->indent); + push_key_only = (!is_complex && (fyp->flow_level || has_bmap)) || + (is_complex && !final_complex_key); + + fyp_scan_debug(fyp, "mark_insert.line=%d/%d mark_end_insert.line=%d/%d fyp->line=%d", + mark_insert.line, mark_insert.column, + mark_end_insert.line, mark_end_insert.column, + fyp_line(fyp)); + + fyp_scan_debug(fyp, "simple_key_allowed=%s is_complex=%s final_complex_key=%s is_multiline=%s has_bmap=%s push_bmap_start=%s push_key_only=%s", + fyp->simple_key_allowed ? "true" : "false", + is_complex ? "true" : "false", + final_complex_key ? "true" : "false", + is_multiline ? "true" : "false", + has_bmap ? "true" : "false", + push_bmap_start ? "true" : "false", + push_key_only ? "true" : "false"); + + if (!is_complex && is_multiline && (!fyp->flow_level || fyp->flow != FYFT_MAP)) { + FYP_PARSE_ERROR(fyp, 0, 1, FYEM_SCAN, "Illegal placement of ':' indicator"); + goto err_out; + } + + /* special handling for ?: */ + if (fyp->tab_used_for_ws) { + FYP_PARSE_ERROR(fyp, 0, 1, FYEM_SCAN, "Indentation used tabs for ':' indicator"); + goto err_out; + } + + if (push_bmap_start) { + + assert(!fyp->flow_level); + + fyp_scan_debug(fyp, "--- parse_roll"); + + /* push the new indent level */ + rc = fy_push_indent(fyp, mark_insert.column, true, mark_insert.line); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_push_indent() failed"); + + fyt = fy_token_queue_simple_internal(fyp, &sk_tl, FYTT_BLOCK_MAPPING_START, 0); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple_internal() failed"); + + /* update with this mark */ + fyt->handle.start_mark = fyt->handle.end_mark = mark_insert; + } + + if (push_bmap_start || push_key_only) { + + fyt = fy_token_queue_simple_internal(fyp, &sk_tl, FYTT_KEY, 0); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple_internal() failed"); + + /* update with the flow level */ + fyt->key.flow_level = fyp->flow_level; + } + +#ifdef FY_DEVMODE + fyp_debug_dump_token(fyp, fyt_insert, "insert-token: "); + fyp_debug_dump_token_list(fyp, &fyp->queued_tokens, fyt_insert, "fyp->queued_tokens (before): "); + fyp_debug_dump_token_list(fyp, &sk_tl, NULL, "sk_tl: "); +#endif + + if (fyt_insert) { + + if (fysk) + fy_token_list_splice_before(&fyp->queued_tokens, fyt_insert, &sk_tl); + else + fy_token_list_splice_after(&fyp->queued_tokens, fyt_insert, &sk_tl); + } else + fy_token_lists_splice(&fyp->queued_tokens, &sk_tl); + +#ifdef FY_DEVMODE + fyp_debug_dump_token_list(fyp, &fyp->queued_tokens, fyt_insert, "fyp->queued_tokens (after): "); +#endif + + target_simple_key_allowed = fysk ? false : !fyp->flow_level; + + fyt = fy_token_queue_simple(fyp, &fyp->queued_tokens, FYTT_VALUE, 1); + fyp_error_check(fyp, fyt, err_out, + "fy_token_queue_simple() failed"); + + if (fysk) { + fy_parse_simple_key_recycle(fyp, fysk); + fysk = NULL; + } + + + fyp->simple_key_allowed = target_simple_key_allowed; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + if (is_complex) { + rc = fy_ws_indentation_check(fyp, &found_tab, &tab_mark); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_ws_indentation_check() failed"); + + /* record whether a tab was used for indentation */ + if (fyp->simple_key_allowed && found_tab) { + fyp->tab_used_for_ws = true; + fyp->last_tab_used_for_ws_mark = tab_mark; + } else + fyp->tab_used_for_ws = false; // XXX + } else + fyp->tab_used_for_ws = false; + + if (final_complex_key) { + fyp->pending_complex_key_column = -1; + fyp_scan_debug(fyp, "pending_complex_key_column -> %d", + fyp->pending_complex_key_column); + } + + if (fyt_insert) { + /* eat whitespace */ + while (fy_is_blank(c = fy_parse_peek(fyp))) + fy_advance(fyp, c); + + /* comment? */ + if (c == '#') { + chandle = NULL; + if (fyp->cfg.flags & FYPCF_PARSE_COMMENTS) + chandle = fy_token_comment_handle(fyt_insert, fycp_right, true); + + rc = fy_scan_comment(fyp, chandle, false); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_scan_comment() failed"); + } + + } + + return 0; + +err_out: + rc = -1; +err_out_rc: + fy_parse_simple_key_recycle(fyp, fysk); + return rc; +} + +int fy_fetch_anchor_or_alias(struct fy_parser *fyp, int c) +{ + struct fy_atom handle; + enum fy_token_type type; + int i = 0, rc = -1, length; + struct fy_simple_key_mark skm; + struct fy_token *fyt; + const char *typestr; + + fyp_error_check(fyp, c == '*' || c == '&', err_out, + "illegal anchor mark (not '*' or '&')"); + + if (c == '*') { + type = FYTT_ALIAS; + typestr = "alias"; + } else { + type = FYTT_ANCHOR; + typestr = "anchor"; + } + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_flow_indent_check(fyp), err_out, + "wrongly indented %s in flow mode", typestr); + + /* we have to save the start mark (including the anchor/alias start) */ + fy_get_simple_key_mark(fyp, &skm); + + /* skip over the anchor mark */ + fy_advance(fyp, c); + + /* start mark */ + fy_fill_atom_start(fyp, &handle); + + length = 0; + + while ((c = fy_parse_peek(fyp)) >= 0) { + if (fyp_is_blankz(fyp, c) || fy_is_flow_indicator(c) || + fy_is_unicode_control(c) || fy_is_unicode_space(c)) + break; + fy_advance(fyp, c); + length++; + } + + if (!fyp_is_blankz(fyp, c) && !fy_is_flow_indicator(c)) { + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_is_unicode_control(c), err_out, + "illegal unicode control character in %s", typestr); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_is_unicode_space(c), err_out, + "illegal unicode space character in %s", typestr); + } + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + c != FYUG_INV, err_out, + "invalid character in %s", typestr); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + c != FYUG_PARTIAL, err_out, + "partial character in %s", typestr); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + length > 0, err_out, + "invalid %s detected", typestr); + + fy_fill_atom_end(fyp, &handle); + + handle.storage_hint = length; + handle.storage_hint_valid = true; + handle.direct_output = true; + handle.empty = false; + handle.has_lb = false; + handle.has_ws = false; + handle.starts_with_ws = false; + handle.starts_with_lb = false; + handle.ends_with_ws = false; + handle.ends_with_lb = false; + handle.trailing_lb = false; + handle.size0 = false; + handle.valid_anchor = true; + + if (type == FYTT_ALIAS) + fyt = fy_token_queue(fyp, type, &handle, NULL); + else + fyt = fy_token_queue(fyp, type, &handle); + fyp_error_check(fyp, fyt, err_out_rc, + "fy_token_queue() failed"); + + /* scan forward for '-' block sequence indicator */ + if (type == FYTT_ANCHOR && !fyp->flow_level) { + for (i = 0; ; i++) { + c = fy_parse_peek_at(fyp, i); + if (c < 0 || fyp_is_lb(fyp, c) || !fy_is_ws(c)) + break; + } + + /* if it's '-' followed by ws we have a problem */ + FYP_PARSE_ERROR_CHECK(fyp, i, 1, FYEM_SCAN, + !(c == '-' && fy_is_ws(fy_parse_peek_at(fyp, i + 1))), err_out, + "illegal block sequence on the same line as anchor"); + } + + + rc = fy_save_simple_key_mark(fyp, &skm, type, NULL); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_save_simple_key_mark() failed"); + + fyp->simple_key_allowed = false; + fyp->tab_used_for_ws = false; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_fetch_tag(struct fy_parser *fyp, int c) +{ + struct fy_atom handle; + int rc = -1, total_length, handle_length, uri_length, i, prefix_length, suffix_length; + const char *handlep; + bool is_valid; + struct fy_simple_key_mark skm; + struct fy_document_state *fyds; + struct fy_token *fyt_td; + struct fy_token *fyt; + + fyp_error_check(fyp, c == '!', err_out, + "illegal tag mark (not '!')"); + + FYP_PARSE_ERROR_CHECK(fyp, 0 ,1, FYEM_SCAN, + fy_flow_indent_check(fyp), err_out, + "wrongly indented tag in flow mode"); + + fyds = fyp->current_document_state; + + fy_get_simple_key_mark(fyp, &skm); + + if (fy_parse_peek_at(fyp, 1) == '<') { + /* skip over '!<' and '>' */ + prefix_length = 2; + suffix_length = 1; + } else + prefix_length = suffix_length = 0; + + if (prefix_length) + handle_length = 0; /* set the handle to '' */ + else { + /* either !suffix or !handle!suffix */ + /* we scan back to back, and split handle/suffix */ + handle_length = fy_scan_tag_handle_length(fyp, prefix_length); + fyp_error_check(fyp, handle_length > 0, err_out, + "fy_scan_tag_handle_length() failed"); + } + + uri_length = fy_scan_tag_uri_length(fyp, prefix_length + handle_length); + fyp_error_check(fyp, uri_length >= 0, err_out, + "fy_scan_tag_uri_length() failed"); + + /* a handle? */ + if (!prefix_length && (handle_length == 0 || fy_parse_peek_at(fyp, handle_length - 1) != '!')) { + /* special case, '!', handle set to '' and suffix to '!' */ + if (handle_length == 1 && uri_length == 0) { + handle_length = 0; + uri_length = 1; + } else { + uri_length = handle_length - 1 + uri_length; + handle_length = 1; + } + } + + is_valid = fy_scan_tag_uri_is_valid(fyp, prefix_length + handle_length, uri_length); + fyp_error_check(fyp, is_valid, err_out, + "tag URI is invalid"); + + if (suffix_length > 0) { + c = fy_parse_peek_at(fyp, prefix_length + handle_length + uri_length); + + FYP_PARSE_ERROR_CHECK(fyp, prefix_length + handle_length + uri_length, 1, FYEM_SCAN, + c == '>', err_out, + "missing '>' uri terminator"); + } + + total_length = prefix_length + handle_length + uri_length + suffix_length; + fy_fill_atom(fyp, total_length, &handle); + handle.style = FYAS_URI; /* this is a URI, need to handle URI escapes */ + + c = fy_parse_peek(fyp); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fyp_is_blankz(fyp, c) || fy_utf8_strchr(",}]", c), err_out, + "invalid tag terminator"); + + handlep = fy_atom_data(&handle) + prefix_length; + + fyt_td = fy_document_state_lookup_tag_directive(fyds, handlep, handle_length); + + FYP_MARK_ERROR_CHECK(fyp, &handle.start_mark, &handle.end_mark, FYEM_PARSE, + fyt_td, err_out, + "undefined tag prefix"); + + fyt = fy_token_queue(fyp, FYTT_TAG, &handle, prefix_length, handle_length, uri_length, fyt_td); + fyp_error_check(fyp, fyt, err_out_rc, + "fy_token_queue() failed"); + + /* scan forward for '-' block sequence indicator */ + if (!fyp->flow_level) { + for (i = 0; ; i++) { + c = fy_parse_peek_at(fyp, i); + if (c < 0 || fyp_is_lb(fyp, c) || !fy_is_ws(c)) + break; + } + + /* if it's '-' followed by ws we have a problem */ + FYP_PARSE_ERROR_CHECK(fyp, i ,1, FYEM_SCAN, + !(c == '-' && fy_is_ws(fy_parse_peek_at(fyp, i + 1))), err_out, + "illegal block sequence on the same line as the tag"); + } + + rc = fy_save_simple_key_mark(fyp, &skm, FYTT_TAG, NULL); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_save_simple_key_mark() failed"); + + fyp->simple_key_allowed = false; + fyp->tab_used_for_ws = false; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_scan_block_scalar_indent(struct fy_parser *fyp, int indent, int *breaks, int *breaks_length, + int *presentation_breaks_length, int *first_break_length, int *lastc) +{ + int c, max_indent = 0, min_indent, break_length; + + *breaks = 0; + *breaks_length = 0; + *presentation_breaks_length = 0; + *first_break_length = 0; + + /* minimum indent is 0 for zero indent scalars */ + min_indent = fyp->document_first_content_token ? 0 : 1; + + /* scan over the indentation spaces */ + /* we don't format content for display */ + for (;;) { + + /* skip over indentation */ + + if (!fyp_tabsize(fyp)) { + /* we must respect the enclosed indent */ + while (fyp_column(fyp) <= fyp->indent && fy_is_ws(c = fy_parse_peek(fyp))) { + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + !fy_is_tab(c), err_out, + "invalid tab character as indent instead of space"); + fy_advance(fyp, c); + } + + /* skip over spaces only */ + while ((c = fy_parse_peek(fyp)) == ' ' && + (!indent || fyp_column(fyp) < indent)) { + fy_advance(fyp, c); + } + } else { + while (fy_is_ws((c = fy_parse_peek(fyp))) && + (!indent || fyp_column(fyp) < indent)) + fy_advance(fyp, c); + } + + if (fyp_column(fyp) > max_indent) + max_indent = fyp_column(fyp); + + /* non-empty line or EOF */ + if (!fyp_is_lb(fyp, c)) { + *lastc = c; + break; + } + + fy_advance(fyp, c); + + break_length = fy_utf8_width(c); + + (*breaks)++; + (*breaks_length) += 1; + + if (fy_is_lb_LS_PS(c)) + (*presentation_breaks_length) += break_length; + + if (!*first_break_length) + *first_break_length = break_length; + } + + if (!indent) { + indent = max_indent; + if (indent < fyp->indent) + indent = fyp->indent; + if (indent < min_indent) + indent = min_indent; + } + + return indent; + +err_out: + return -1; +} + +int fy_fetch_block_scalar(struct fy_parser *fyp, bool is_literal, int c) +{ + struct fy_atom handle; + enum fy_atom_chomp chomp = FYAC_CLIP; /* default */ + int lastc, rc, increment = 0, current_indent, new_indent, indent = 0; + int breaks, breaks_length, presentation_breaks_length, first_break_length; + bool doc_start_end_detected, empty, empty_line, prev_empty_line, indented, prev_indented, first; + bool has_ws, has_lb, starts_with_ws, starts_with_lb, ends_with_ws, ends_with_lb, trailing_lb; + bool pending_nl, ends_with_eof, starts_with_eof, content_is_eof; + struct fy_token *fyt; + size_t length, line_length, trailing_ws, trailing_breaks_length; + size_t leading_ws; + size_t prefix_length, suffix_length; + unsigned int chomp_amt; + int actual_lb_length, pending_lb_length; + struct fy_mark indicator_mark; + bool generated_indent, final_lb; +#ifdef ATOM_SIZE_CHECK + size_t tlength; +#endif + + fyp_error_check(fyp, c == '|' || c == '>', err_out, + "bad start of block scalar ('%s')", + fy_utf8_format_a(c, fyue_singlequote)); + + fy_get_mark(fyp, &indicator_mark); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_flow_indent_check(fyp), err_out, + "wrongly indented block scalar in flow mode"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_block_indent_check(fyp), err_out, + "wrongly indented block scalar in block mode"); + + rc = fy_remove_simple_key(fyp, FYTT_SCALAR); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_remove_simple_key() failed"); + + fyp->simple_key_allowed = true; + fyp->tab_used_for_ws = false; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + /* skip over block scalar start */ + fy_advance(fyp, c); + + /* intentation indicator (either [-+] or [-+] */ + c = fy_parse_peek(fyp); + if (c == '+' || c == '-') { + + chomp = c == '+' ? FYAC_KEEP : FYAC_STRIP; + + fy_advance(fyp, c); + + c = fy_parse_peek(fyp); + if (fy_is_num(c)) { + increment = c - '0'; + fyp_error_check(fyp, increment != 0, err_out, + "indentation indicator 0"); + fy_advance(fyp, c); + } + } else if (fy_is_num(c)) { + + increment = c - '0'; + fyp_error_check(fyp, increment != 0, err_out, + "indentation indicator 0"); + fy_advance(fyp, c); + + c = fy_parse_peek(fyp); + if (c == '+' || c == '-') { + chomp = c == '+' ? FYAC_KEEP : FYAC_STRIP; + fy_advance(fyp, c); + } + } + + /* the comment indicator must have at least a space */ + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + c != '#', err_out, + "invalid comment without whitespace after block scalar indicator"); + + /* eat whitespace */ + while (fy_is_blank(c = fy_parse_peek(fyp))) + fy_advance(fyp, c); + + /* comment? */ + if (c == '#') { + /* XXX just ignore this one */ + rc = fy_scan_comment(fyp, NULL, true); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_scan_comment() failed"); + } + + c = fy_parse_peek(fyp); + + /* end of the line? */ + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fyp_is_lbz(fyp, c), err_out, + "block scalar no linebreak found"); + + /* if the block scalar indicator is on a different line we need a new indent */ + generated_indent = false; + if (!increment && indicator_mark.line != fyp->indent_line) { + fyp_scan_debug(fyp, "generating indent %d/%d\n", indicator_mark.line, fyp->indent_line); + rc = fy_push_indent(fyp, indicator_mark.column, false, indicator_mark.line); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_push_indent() failed"); + generated_indent = true; + } + + /* advance */ + if (fy_utf8_is_valid(c)) + fy_advance(fyp, c); + + fy_fill_atom_start(fyp, &handle); + + starts_with_eof = c < 0; + + c = fy_parse_peek(fyp); + content_is_eof = c < 0; /* any error or EOF */ + + current_indent = fyp->indent >= 0 ? fyp->indent : 0; + indent = increment ? current_indent + increment : 0; + + length = 0; + trailing_breaks_length = 0; + + empty = true; + has_ws = false; + has_lb = false; + starts_with_ws = false; + starts_with_lb = false; + ends_with_ws = false; + ends_with_lb = false; + trailing_lb = false; + + new_indent = fy_scan_block_scalar_indent(fyp, indent, &breaks, &breaks_length, &presentation_breaks_length, &first_break_length, &lastc); + fyp_error_check(fyp, new_indent >= 0, err_out, + "fy_scan_block_scalar_indent() failed"); + + length = breaks_length; + length += presentation_breaks_length; + indent = new_indent; + + doc_start_end_detected = false; + prev_empty_line = true; + + prefix_length = 0; + suffix_length = 0; + + prev_indented = false; + first = true; + pending_nl = false; + pending_lb_length = 0; + + chomp_amt = increment ? (unsigned int)(current_indent + increment) : (unsigned int)-1; + + actual_lb_length = 1; + while ((c = fy_parse_peek(fyp)) > 0 && fyp_column(fyp) >= indent) { + + lastc = c; + + if (first) { + if (fy_is_ws(c)) + starts_with_ws = true; + else if (fyp_is_lb(fyp, c)) + starts_with_lb = true; + } + + /* consume the list */ + line_length = 0; + trailing_ws = 0; + empty_line = true; + leading_ws = 0; + + indented = fy_is_ws(fy_parse_peek(fyp)); + + while (!(fyp_is_lbz(fyp, c = fy_parse_peek(fyp)))) { + + lastc = c; + + if (fyp_column(fyp) == 0 && + (!fy_parse_strncmp(fyp, "...", 3) || + !fy_parse_strncmp(fyp, "---", 3)) && + fy_is_blankz_at_offset(fyp, 3)) { + doc_start_end_detected = true; + break; + } + + if (!fy_is_space(c)) { + empty = false; + empty_line = false; + trailing_ws = 0; + if (chomp_amt == (unsigned int)-1) + chomp_amt = fyp_column(fyp); + } else { + has_ws = true; + if (empty_line) + leading_ws++; + trailing_ws++; + } + + fy_advance(fyp, c); + + line_length += fy_utf8_width(c); + } + /* keep the variables without warning */ + (void)trailing_ws; + (void)leading_ws; + + if (doc_start_end_detected) + break; + + if (!fy_is_z(c)) { + /* eat line break */ + actual_lb_length = fy_utf8_width(c); + fy_advance(fyp, c); + + has_lb = true; + new_indent = fy_scan_block_scalar_indent(fyp, indent, &breaks, &breaks_length, &presentation_breaks_length, &first_break_length, &lastc); + fyp_error_check(fyp, new_indent >= 0, err_out, + "fy_scan_block_scalar_indent() failed"); + if (fy_is_lb_LS_PS(c)) + presentation_breaks_length += actual_lb_length; + } else { + has_lb = false; + new_indent = indent; + // was chomp = FYAC_STRIP, very very wrong + + breaks = 0; + breaks_length = 0; + presentation_breaks_length = 0; + first_break_length = 0; + + actual_lb_length = 0; + + } + + if (is_literal) { + + prefix_length = 0; + + if (pending_nl) { + pending_nl = false; + prefix_length += pending_lb_length; + pending_lb_length = 0; + } + + prefix_length += trailing_breaks_length; + trailing_breaks_length = 0; + + suffix_length = 0; + + if (fy_is_lb_LS_PS(c)) { + trailing_breaks_length += breaks_length; + trailing_breaks_length += presentation_breaks_length; + + if (actual_lb_length > 1) + presentation_breaks_length -= actual_lb_length; + + pending_nl = true; + pending_lb_length = 0; + } else { + trailing_breaks_length += breaks_length; + trailing_breaks_length += presentation_breaks_length; + + pending_nl = !empty_line || indented; + pending_lb_length = pending_nl ? 1 : 0; + } + + } else { + + prefix_length = 0; + + if (!trailing_breaks_length) { + if (prev_indented || (prev_empty_line && !first) || indented) { + /* previous line was indented or empty, force output newline */ + if (pending_nl) { + pending_nl = false; + prefix_length += 1; // pending_lb_length; + pending_lb_length = 0; + } + } else if (!prev_empty_line && !prev_indented && !indented && !empty_line) { + /* previous line was not empty and not indented + * while this is not indented and not empty need sep */ + if (pending_nl) { + pending_nl = false; + prefix_length += 1; // pending_lb_length; + pending_lb_length = 0; + } + } + } else { + prefix_length += trailing_breaks_length; + if (prev_indented || indented) + prefix_length++; + } + pending_nl = true; + pending_lb_length = actual_lb_length; + + trailing_breaks_length = 0; + + suffix_length = 0; + trailing_breaks_length += breaks_length; + trailing_breaks_length += presentation_breaks_length; + } + + + length += prefix_length + line_length + suffix_length; + indent = new_indent; + + prev_empty_line = empty_line; + prev_indented = indented; + + prefix_length = 0; + suffix_length = 0; + + first = false; + } + + if (empty) { + trailing_breaks_length = breaks_length; + trailing_breaks_length += presentation_breaks_length; + length = 0; + } + + /* end... */ + fy_fill_atom_end(fyp, &handle); + + if (c == FYUG_INV || c == FYUG_PARTIAL) { + FYP_MARK_ERROR(fyp, &handle.start_mark, &handle.end_mark, FYEM_SCAN, + "block scalar is malformed UTF8"); + goto err_out; + } + + /* are we ended with EOF? */ + ends_with_eof = starts_with_eof || (c == FYUG_EOF && !fyp_is_lb(fyp, lastc) && !breaks); + + /* detect wrongly indented block scalar */ + if (c != FYUG_EOF && !(!empty || fyp_column(fyp) <= fyp->indent || c == '#' || doc_start_end_detected)) { + FYP_MARK_ERROR(fyp, &handle.start_mark, &handle.end_mark, FYEM_SCAN, + "block scalar with wrongly indented line after spaces only"); + goto err_out; + } + + if (empty && c == '#' && fyp_column(fyp) > fyp->indent) { + FYP_MARK_ERROR(fyp, &handle.start_mark, &handle.end_mark, FYEM_SCAN, + "empty block scalar with wrongly indented comment line after spaces only"); + goto err_out; + } + + if (chomp_amt == (unsigned int)-1) + chomp_amt = current_indent; + + /* whether the final character is an lb (or implied to be one) */ + final_lb = pending_nl || (!starts_with_eof && ends_with_eof); + trailing_lb = trailing_breaks_length > 0; + ends_with_ws = fy_is_ws(lastc); + + switch (chomp) { + case FYAC_CLIP: + case FYAC_KEEP: + if (chomp == FYAC_KEEP) + length += breaks + presentation_breaks_length; + + if (final_lb && !content_is_eof) + length += (actual_lb_length == 0 && pending_nl) ? 1 : actual_lb_length; + + ends_with_lb = final_lb || trailing_lb; + break; + case FYAC_STRIP: + ends_with_lb = false; + break; + } + + /* need to process to present */ + handle.style = is_literal ? FYAS_LITERAL : FYAS_FOLDED; + handle.chomp = chomp; + handle.increment = increment ? (unsigned int)(current_indent + increment) : chomp_amt; + + /* no point in trying to do direct output in a block scalar */ + /* TODO maybe revisit in the future */ + handle.direct_output = false; + handle.empty = empty; + handle.has_lb = has_lb; + handle.has_ws = has_ws; + handle.starts_with_ws = starts_with_ws; + handle.starts_with_lb = starts_with_lb; + handle.ends_with_ws = ends_with_ws; + handle.ends_with_lb = ends_with_lb; + handle.trailing_lb = trailing_lb; + handle.size0 = length == 0; + handle.valid_anchor = false; + handle.json_mode = fyp_json_mode(fyp); + handle.lb_mode = fyp_lb_mode(fyp); + handle.fws_mode = fyp_fws_mode(fyp); + handle.directive0_mode = fyp_directive0_mode(fyp); + handle.tabsize = fyp_tabsize(fyp); + handle.ends_with_eof = ends_with_eof; + handle.is_merge_key = false; + +#ifdef ATOM_SIZE_CHECK + tlength = fy_atom_format_text_length(&handle); + if (tlength != length) { + fyp_warning(fyp, "%s: storage hint calculation failed real %zu != hint %zu - \"%s\"", __func__, + tlength, length, + fy_utf8_format_text_a(fy_atom_data(&handle), fy_atom_size(&handle), fyue_doublequote)); + length = tlength; + } +#endif + + handle.storage_hint = length; + handle.storage_hint_valid = true; + + fyt = fy_token_queue(fyp, FYTT_SCALAR, &handle, is_literal ? FYSS_LITERAL : FYSS_FOLDED); + fyp_error_check(fyp, fyt, err_out_rc, + "fy_token_queue() failed"); + + if (fyp->cfg.flags & FYPCF_PARSE_COMMENTS) { + rc = fy_attach_comments_if_any(fyp, fyt); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_attach_right_hand_comment() failed"); + } + + if (generated_indent) { + rc = fy_pop_indent(fyp); + fyp_error_check(fyp, !rc, err_out, + "fy_pop_indent() failed"); + + /* the ident line has now moved */ + fyp->indent_line = fyp_line(fyp); + } + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_reader_fetch_flow_scalar_handle(struct fy_reader *fyr, int c, int indent, struct fy_atom *handle, bool sloppy_indent) +{ + size_t length; + int code_length, i = 0, j, end_c, last_line, lastc; + int breaks_found, blanks_found, break_run, total_code_length; + int breaks_found_length, first_break_length, value; + uint32_t hi_surrogate, lo_surrogate; + bool is_single, is_multiline, esc_lb, ws_lb_only, has_ws, has_lb, has_esc; + bool first, starts_with_ws, starts_with_lb, ends_with_ws, ends_with_lb, trailing_lb = false; + bool unicode_esc, is_json_unesc, has_json_esc; + int last_esc_lb, break_length, presentation_breaks_length; + struct fy_mark mark, mark2; + char escbuf[1 + FY_UTF8_FORMAT_BUFMIN]; + size_t escbuf_len; + enum fy_utf8_escape esc_mode; + const char *ep; +#ifdef ATOM_SIZE_CHECK + size_t tlength; +#endif + + (void)last_esc_lb; + + is_single = c == '\''; + end_c = c; + + fyr_error_check(fyr, c == '\'' || c == '"', err_out, + "bad start of flow scalar ('%s')", + fy_utf8_format_a(c, fyue_singlequote)); + + fy_reader_get_mark(fyr, &mark); + + /* skip over block scalar start */ + fy_reader_advance(fyr, c); + + fy_reader_fill_atom_start(fyr, handle); + + length = 0; + breaks_found = 0; + breaks_found_length = 0; + first_break_length = 0; + presentation_breaks_length = 0; + blanks_found = 0; + esc_lb = false; + last_esc_lb = -1; + ws_lb_only = true; + has_ws = false; + has_lb = false; + starts_with_ws = false; + starts_with_lb = false; + ends_with_ws = false; + ends_with_lb = false; + has_esc = false; + break_run = 0; + first = true; + has_json_esc = false; + + esc_mode = fy_reader_json_mode(fyr) ? fyue_doublequote_json : + fy_reader_lb_mode(fyr) == fylb_cr_nl ? fyue_doublequote : fyue_doublequote_yaml_1_1; + + last_line = -1; + lastc = -1; + for (;;) { + if (!fy_reader_json_mode(fyr)) { + /* no document indicators please */ + FYR_PARSE_ERROR_CHECK(fyr, 0, 3, FYEM_SCAN, + !(fy_reader_column(fyr) == 0 && + (!fy_reader_strncmp(fyr, "---", 3) || + !fy_reader_strncmp(fyr, "...", 3)) && + fy_reader_is_blankz_at_offset(fyr, 3)), err_out, + "invalid document-%s marker in %s scalar", + c == '-' ? "start" : "end", + is_single ? "single-quoted" : "double-quoted"); + } + + /* no EOF either */ + c = fy_reader_peek(fyr); + + if (c <= 0) { + fy_reader_get_mark(fyr, &mark); + + if (!c || c == FYUG_EOF) + FYR_MARK_ERROR(fyr, &handle->start_mark, &mark, FYEM_SCAN, + "%s scalar without closing quote", + is_single ? "single-quoted" : "double-quoted"); + else + FYR_MARK_ERROR(fyr, &handle->start_mark, &mark, FYEM_SCAN, + "%s scalar is malformed UTF8", + is_single ? "single-quoted" : "double-quoted"); + goto err_out; + } + + if (first) { + if (fy_reader_is_flow_ws(fyr, c)) + starts_with_ws = true; + else if (fy_reader_is_lb(fyr, c)) + starts_with_lb = true; + } + + while (!fy_reader_is_flow_blankz(fyr, c = fy_reader_peek(fyr))) { + + if (ws_lb_only && !(fy_reader_is_flow_ws(fyr, c) || fy_reader_is_lb(fyr, c)) && c != end_c) + ws_lb_only = false; + + esc_lb = false; + last_esc_lb = -1; + /* track line change (and first non blank) */ + if (last_line != fy_reader_line(fyr)) { + last_line = fy_reader_line(fyr); + + if ((indent >= 0 && fy_reader_column(fyr) <= indent) && !sloppy_indent) { + + fy_reader_advance(fyr, c); + fy_reader_get_mark(fyr, &mark2); + FYR_MARK_ERROR(fyr, &mark, &mark2, FYEM_SCAN, + "wrongly indented %s scalar", + is_single ? "single-quoted" : "double-quoted"); + goto err_out; + } + } + + if (breaks_found) { + length += breaks_found > 1 ? (breaks_found_length - first_break_length) : 1; + length += presentation_breaks_length; + breaks_found = 0; + blanks_found = 0; + presentation_breaks_length = 0; + } else if (blanks_found) { + length += blanks_found; + lastc = ' '; + blanks_found = 0; + } + + if (c >= 0 && c <= 0x7f && (fy_utf8_low_ascii_flags[c] & F_SIMPLE_SCALAR)) { + size_t len, consumed; + const char *p, *s, *e; + int8_t cc; + int run; + + run = 0; + while ((p = fy_reader_ensure_lookahead(fyr, 1, &len)) != NULL) { + + s = p; + e = s + len; + + while (s < e && (cc = (int8_t)*s) >= 0 && (fy_utf8_low_ascii_flags[cc] & F_SIMPLE_SCALAR)) + s++; + + consumed = s - p; + if (consumed) { + fy_reader_advance_octets(fyr, consumed); + fyr->column += consumed; + lastc = (int)cc; + } + run += consumed; + + /* we're done if stopped earlier */ + if (s < e) + break; + } + length += run; + break_run = 0; + continue; + } + + /* escaped single quote? */ + if (is_single && c == '\'' && fy_reader_peek_at(fyr, 1) == '\'') { + length++; + fy_reader_advance_by(fyr, 2); + break_run = 0; + lastc = '\''; + continue; + } + + /* right quote? */ + if (c == end_c) + break; + + /* escaped line break (any linebreak will do) */ + if (!is_single && c == '\\' && fy_reader_is_lb(fyr, fy_reader_peek_at(fyr, 1))) { + + esc_lb = true; + last_esc_lb = fy_reader_peek_at(fyr, 1); + + fy_reader_advance_by(fyr, 2); + + c = fy_reader_peek(fyr); + break_run = 0; + lastc = c; + + has_esc = true; + + break; + } + + /* escaped sequence? */ + if (!is_single && c == '\\') { + + /* note we don't generate formatted output */ + /* we are merely checking for validity */ + c = fy_reader_peek_at(fyr, 1); + + /* hex, unicode marks - json only supports u */ + unicode_esc = !fy_reader_json_mode(fyr) ? + (c == 'x' || c == 'u' || c == 'U') : + c == 'u'; + if (unicode_esc) { + + total_code_length = 0; + j = 0; + hi_surrogate = lo_surrogate = 0; + for (;;) { + total_code_length += 2; + + code_length = c == 'x' ? 2 : + c == 'u' ? 4 : 8; + value = 0; + for (i = 0; i < code_length; i++) { + c = fy_reader_peek_at(fyr, total_code_length + i); + + FYR_PARSE_ERROR_CHECK(fyr, 0, total_code_length + i + 1, FYEM_SCAN, + fy_is_hex(c), err_out, + "double-quoted scalar has invalid hex escape"); + + value <<= 4; + if (c >= '0' && c <= '9') + value |= c - '0'; + else if (c >= 'a' && c <= 'f') + value |= 10 + c - 'a'; + else + value |= 10 + c - 'A'; + } + + total_code_length += code_length; + j++; + + /* 0x10000 + (HI - 0xd800) * 0x400 + (LO - 0xdc00) */ + + /* high surrogate */ + if (j == 1 && code_length == 4 && value >= 0xd800 && value <= 0xdbff && + fy_reader_peek_at(fyr, total_code_length) == '\\' && + fy_reader_peek_at(fyr, total_code_length + 1) == 'u') { + hi_surrogate = value; + c = 'u'; + continue; + } + + if (j == 2 && code_length == 4 && hi_surrogate) { + + FYR_PARSE_ERROR_CHECK(fyr, total_code_length - 6, 6, FYEM_SCAN, + value >= 0xdc00 && value <= 0xdfff, err_out, + "Invalid low surrogate value"); + + lo_surrogate = value; + value = 0x10000 + (hi_surrogate - 0xd800) * 0x400 + (lo_surrogate - 0xdc00); + } + + break; + } + + /* check for validity */ + FYR_PARSE_ERROR_CHECK(fyr, 0, total_code_length, FYEM_SCAN, + !(value < 0 || (value >= 0xd800 && value <= 0xdfff) || + value > 0x10ffff), err_out, + "double-quoted scalar has invalid UTF8 escape"); + + fy_reader_advance_by(fyr, total_code_length); + + } else { + escbuf[0] = '\\'; + fy_utf8_put_unchecked(escbuf + 1, c); + escbuf_len = 1 + fy_utf8_width(c); + + ep = escbuf; + + value = fy_utf8_parse_escape(&ep, escbuf_len, esc_mode); + FYR_PARSE_ERROR_CHECK(fyr, 0, 2, FYEM_SCAN, + value >= 0, err_out, + "invalid escape '%s' in %s string", + fy_utf8_format_a(c, fyue_singlequote), + is_single ? "single-quoted" : "double-quoted"); + + fy_reader_advance_by(fyr, 2); + } + + length += fy_utf8_width(value); + + lastc = value; + + if (lastc == '\n') + break_run++; + + has_esc = true; + + continue; + } + + /* check whether we have a JSON unescaped character */ + is_json_unesc = fy_is_json_unescaped_range_only(c); + if (!is_json_unesc) + has_json_esc = true; + + if (!is_single && fy_reader_json_mode(fyr) && has_json_esc) { + FYR_PARSE_ERROR(fyr, 0, 2, FYEM_SCAN, + "Invalid JSON unescaped character"); + goto err_out; + } + + lastc = c; + + /* regular character */ + fy_reader_advance(fyr, c); + + length += fy_utf8_width(c); + break_run = 0; + } + + /* end of scalar */ + if (c == end_c) + break; + + /* consume blanks */ + breaks_found = 0; + breaks_found_length = 0; + blanks_found = 0; + while (fy_reader_is_flow_blank(fyr, c = fy_reader_peek(fyr)) || fy_reader_is_lb(fyr, c)) { + + if (!has_json_esc && !fy_is_json_unescaped(c)) + has_json_esc = true; + + break_run = 0; + + /* check for tab used as indentation */ + if (!fy_reader_tabsize(fyr) && fy_is_tab(c)) { + FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_SCAN, + fy_reader_column(fyr) > indent, err_out, + "invalid tab used as indentation"); + } + + fy_reader_advance(fyr, c); + + if (fy_reader_is_lb(fyr, c)) { + + if (!fy_is_lb_LS_PS(c)) { + break_length = 1; + } else { + break_length = fy_utf8_width(c); + presentation_breaks_length += break_length; + } + + has_lb = true; + if (!breaks_found) + first_break_length = break_length; + breaks_found++; + breaks_found_length += break_length; + blanks_found = 0; + esc_lb = false; + } else { + has_ws = true; + if (!esc_lb) + blanks_found++; + } + } + first = false; + } + + if (break_run > 0) + ends_with_lb = true; + else if (fy_reader_is_flow_ws(fyr, lastc)) + ends_with_ws = true; + trailing_lb = break_run > 1; + + /* end... */ + fy_reader_fill_atom_end(fyr, handle); + + is_multiline = handle->end_mark.line > handle->start_mark.line; + + /* need to process to present */ + handle->style = is_single ? FYAS_SINGLE_QUOTED : FYAS_DOUBLE_QUOTED; + handle->direct_output = !is_multiline && !has_esc && !has_json_esc && + fy_atom_size(handle) == length; + handle->empty = ws_lb_only; + handle->has_lb = has_lb; + handle->has_ws = has_ws; + handle->starts_with_ws = starts_with_ws; + handle->starts_with_lb = starts_with_lb; + handle->ends_with_ws = ends_with_ws; + handle->ends_with_lb = ends_with_lb; + handle->trailing_lb = trailing_lb; + handle->size0 = length == 0; + handle->valid_anchor = false; + handle->json_mode = fy_reader_json_mode(fyr); + handle->lb_mode = fy_reader_lb_mode(fyr); + handle->fws_mode = fy_reader_flow_ws_mode(fyr); + handle->directive0_mode = fy_reader_directive0_mode(fyr); + handle->tabsize = fy_reader_tabsize(fyr); + handle->ends_with_eof = false; /* flow scalars never end with EOF and be valid */ + handle->is_merge_key = false; + handle->simple_key_allowed = false; + + /* skip over block scalar end */ + fy_reader_advance_by(fyr, 1); + +#ifdef ATOM_SIZE_CHECK + tlength = fy_atom_format_text_length(handle); + if (tlength != length) { + fyr_warning(fyr, "%s: storage hint calculation failed real %zu != hint %zu - \"%s\"", __func__, + tlength, length, + fy_utf8_format_text_a(fy_atom_data(handle), fy_atom_size(handle), fyue_doublequote)); + length = tlength; + } +#endif + + handle->storage_hint = length; + handle->storage_hint_valid = true; + + FYR_MARK_ERROR_CHECK(fyr, &handle->start_mark, &handle->end_mark, FYEM_SCAN, + !fy_reader_json_mode(fyr) || !is_multiline, err_out, + "Multi line double quoted scalars not supported in JSON mode"); + + return 0; + +err_out: + return -1; +} + +struct fy_fetch_plain_state { + struct fy_mark last_mark; + size_t length; + int c; + int lastc; + bool has_lb; + bool has_ws; + bool has_json_esc; + bool has_high_ascii; + bool simple_key_allowed; + bool last_was_lb; +}; + +static FY_ALWAYS_INLINE inline int +fy_reader_fetch_plain_scalar_handle_inline(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + const bool directive0, + const enum fy_lb_mode lb_mode, + const bool json_mode, + struct fy_fetch_plain_state *state) +{ + int nextc, width; + size_t length, length_advance; + int blanks_found_length, breaks_found_length, first_break_length; + union { + struct { + bool run_has_lb : 1; + bool run_has_ws : 1; + bool has_json_esc : 1; + bool has_high_ascii : 1; + bool has_lb : 1; + bool has_ws : 1; + bool had_run : 1; + bool flow_level_gt_0 : 1; + bool flow_level_le_0_indent_ge_0 : 1; + bool check_tab : 1; + bool last_was_lb : 1; + }; + unsigned short flags; + } u; + size_t run, len; + const char *s; + + u.flags = 0; + + u.flow_level_gt_0 = flow_level > 0; + u.flow_level_le_0_indent_ge_0 = flow_level <= 0 && indent >= 0; + u.check_tab = fy_reader_tabsize(fyr) || indent < 0; + + length = 0; + length_advance = 0; + + for (;;) { + /* break for document indicators */ + if (fy_reader_column(fyr) == 0 && + ((!fy_reader_strncmp(fyr, "---", 3) || !fy_reader_strncmp(fyr, "...", 3)) && + fy_reader_is_blankz_at_offset(fyr, 3))) + break; + + if (c == '#') + break; + + /* for YAML 1.1 check % directive break */ + if (directive0 && fy_reader_column(fyr) == 0 && c == '%') + break; + + u.had_run = false; + u.last_was_lb = false; + while (!fy_is_blankz_m(c, lb_mode)) { + + if (fy_utf8_is_simple_scalar(c)) { + s = fy_reader_ensure_lookahead(fyr, 1, &len); + run = 0; + while (run < len && fy_utf8_is_simple_scalar_no_check((int8_t)s[run])) + run++; + + fyr->column += run; + nextc = -1; + } else { + + width = fy_utf8_width(c); + + if (c == ':') { + + nextc = fy_reader_peek_at_offset(fyr, width); + + /* ':' followed by space terminates */ + if (fy_is_blankz_m(nextc, lb_mode)) + break; + + /* in flow context ':' followed by flow markers */ + if (u.flow_level_gt_0 && fy_is_flow_indicator(nextc)) + break; + } else + nextc = -1; + + /* in flow context any or , [ ] { } */ + if (u.flow_level_gt_0 && fy_is_flow_indicator(c)) + break; + + /* check whether we have a JSON unescaped character */ + u.has_json_esc |= !fy_is_json_unescaped(c); + u.has_high_ascii |= c >= 0x80; + + fyr->column++; + + run = width; + } + + u.had_run = true; + fy_reader_advance_octets(fyr, run); + length += run; + + c = nextc >= 0 ? nextc : fy_reader_peek(fyr); + } + + /* save end mark if we processed more than one non-blank */ + if (u.had_run) { + length += length_advance; + + u.has_lb |= u.run_has_lb; + u.has_ws |= u.run_has_ws; + + fy_reader_get_mark(fyr, &state->last_mark); + } + + /* end? */ + if (!(fy_is_blank_lb_m(c, lb_mode))) + break; + + /* consume blanks */ + breaks_found_length = 0; + first_break_length = 0; + blanks_found_length = 0; + u.run_has_ws = u.run_has_lb = false; + + width = fy_utf8_width(c); + do { + fy_reader_advance_lb_mode(fyr, c, lb_mode); + + if (u.check_tab) { + FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_SCAN, + c != '\t' || fy_reader_column(fyr) >= (indent + 1), err_out, + "invalid tab used as indentation"); + } + + u.last_was_lb = !fy_is_blank(c); + + /* if it's a break */ + if (!u.last_was_lb) { + blanks_found_length += width; + u.run_has_ws = true; + } else { + /* first break, turn on leading blanks */ + if (!first_break_length) + first_break_length = width; + breaks_found_length += width; + u.run_has_lb = true; + } + + c = fy_reader_peek_width(fyr, &width); + + } while (fy_is_blank_lb_m(c, lb_mode)); + + /* break out if indentation is less */ + if (u.flow_level_le_0_indent_ge_0 && fy_reader_column(fyr) <= indent) + break; + + length_advance = u.run_has_lb ? + (breaks_found_length > first_break_length ? (breaks_found_length - first_break_length) : first_break_length) : + blanks_found_length; + } + + state->c = c; + state->length = length; + state->simple_key_allowed = u.run_has_lb; // simple key allowed if there was an lb + state->has_lb = u.has_lb; + state->has_ws = u.has_ws; + state->has_json_esc = u.has_json_esc; + state->has_high_ascii = u.has_high_ascii; + state->last_was_lb = u.last_was_lb; + + return 0; + +err_out: + return -1; +} + +int fy_reader_fetch_plain_scalar_handle_yaml(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + struct fy_fetch_plain_state *state) +{ + return fy_reader_fetch_plain_scalar_handle_inline(fyr, c, indent, flow_level, handle, + false, // directive0 + fylb_cr_nl, // lb_mode + false, // json + state); +} + +int fy_reader_fetch_plain_scalar_handle_json(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + struct fy_fetch_plain_state *state) +{ + return fy_reader_fetch_plain_scalar_handle_inline(fyr, c, indent, flow_level, handle, + false, // directive0 + fylb_cr_nl, // lb_mode + true, // json + state); +} + +int fy_reader_fetch_plain_scalar_handle_yaml_1_1(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + struct fy_fetch_plain_state *state) +{ + return fy_reader_fetch_plain_scalar_handle_inline(fyr, c, indent, flow_level, handle, + true, // directive0 + fylb_cr_nl_N_L_P, // lb_mode + false, // json + state); +} + +typedef int (*fy_reader_fetch_plain_scalar_handle_func)(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + struct fy_fetch_plain_state *state); + +static inline int +fy_reader_fetch_plain_scalar_handle_switch(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + struct fy_fetch_plain_state *state) +{ + + static const fy_reader_fetch_plain_scalar_handle_func funcs[] = { + [fyrm_yaml] = fy_reader_fetch_plain_scalar_handle_yaml, + [fyrm_json] = fy_reader_fetch_plain_scalar_handle_json, + [fyrm_yaml_1_1] = fy_reader_fetch_plain_scalar_handle_yaml_1_1, + }; + fy_reader_fetch_plain_scalar_handle_func func; + + assert((unsigned int)fyr->mode <= ARRAY_SIZE(funcs)); + func = funcs[fyr->mode]; + assert(func != NULL); + + return func(fyr, c, indent, flow_level, handle, state); +} + +int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, + const int indent, int flow_level, + struct fy_atom *handle) +{ + struct fy_fetch_plain_state state_local, *state = &state_local; + const enum fy_lb_mode lb_mode = fy_reader_lb_mode(fyr); + bool is_merge_key, ends_with_eof, is_multiline; + int nextc, rc; + + nextc = fy_reader_peek_at(fyr, 1); + + FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_SCAN, + !fy_is_blankz_m(c, lb_mode), err_out, + "plain scalar cannot start with blank or zero"); + + /* may not start with any of ,[]{}#&*!|>'\"%@` */ + FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_SCAN, + !fy_utf8_strchr(",[]{}#&*!|>'\"%@`", c), err_out, + "plain scalar cannot start with '%c'", c); + + /* may not start with - not followed by blankz */ + FYR_PARSE_ERROR_CHECK(fyr, 0, 2, FYEM_SCAN, + c != '-' || !fy_is_blank(nextc), err_out, + "plain scalar cannot start with '%c' followed by blank", c); + + /* may not start with -?: not followed by blankz (in block context) */ + FYR_PARSE_ERROR_CHECK(fyr, 0, 2, FYEM_SCAN, + flow_level > 0 || !((c == '?' || c == ':') && fy_is_blank(nextc)), err_out, + "plain scalar cannot start with '%c' followed by blank (in block context)", c); + + /* may not start with - followed by ",[]{}" in flow context */ + FYR_PARSE_ERROR_CHECK(fyr, 0, 2, FYEM_SCAN, + flow_level == 0 || !(c == '-' && fy_is_flow_indicator(nextc)), err_out, + "plain scalar cannot start with '%c' followed by ,[]{} (in flow context)", c); + + memset(state, 0, sizeof(*state)); + state->last_mark.input_pos = (size_t)-1; + + /* we can prime is_merge_key here */ + is_merge_key = c == '<' && nextc == '<'; + + fy_reader_fill_atom_start(fyr, handle); + + rc = fy_reader_fetch_plain_scalar_handle_switch(fyr, c, indent, flow_level, handle, state); + if (rc) + goto err_out; + + /* end... */ + if (state->last_mark.input_pos == (size_t)-1) + fy_reader_fill_atom_end(fyr, handle); + else + fy_reader_fill_atom_end_at(fyr, handle, &state->last_mark); + + c = state->c; + if (c == FYUG_INV || c == FYUG_PARTIAL) { + FYR_MARK_ERROR(fyr, &handle->start_mark, &handle->end_mark, FYEM_SCAN, + "plain scalar is malformed UTF8"); + goto err_out; + } + + ends_with_eof = c == FYUG_EOF && !state->last_was_lb; + + is_multiline = handle->end_mark.line > handle->start_mark.line; + + handle->style = FYAS_PLAIN; + handle->chomp = FYAC_STRIP; + handle->direct_output = !is_multiline && !state->has_json_esc && fy_atom_size(handle) == state->length; + handle->empty = false; + handle->has_lb = state->has_lb; + handle->has_ws = state->has_ws; + handle->starts_with_ws = false; + handle->starts_with_lb = false; + handle->ends_with_ws = false; + handle->ends_with_lb = false; + handle->trailing_lb = false; + handle->size0 = state->length == 0; + handle->valid_anchor = false; + handle->json_mode = fy_reader_json_mode(fyr); + handle->lb_mode = fy_reader_lb_mode(fyr); + handle->fws_mode = fy_reader_flow_ws_mode(fyr); + handle->directive0_mode = fy_reader_directive0_mode(fyr); + handle->tabsize = fy_reader_tabsize(fyr); + handle->ends_with_eof = ends_with_eof; + handle->is_merge_key = is_merge_key && state->length == 2; + handle->simple_key_allowed = state->simple_key_allowed; + handle->high_ascii = state->has_high_ascii; + +#ifdef ATOM_SIZE_CHECK + size_t tlength; + tlength = fy_atom_format_text_length(handle); + if (tlength != state->length) { + fyr_warning(fyr, "%s: storage hint calculation failed real %zu != hint %zu - \"%s\"", __func__, + tlength, state->length, + fy_utf8_format_text_a(fy_atom_data(handle), fy_atom_size(handle), fyue_doublequote)); + state->length = tlength; + } +#endif + + handle->storage_hint = state->length; + handle->storage_hint_valid = true; + + /* extra check in json mode */ + if (fy_reader_json_mode(fyr)) { + FYR_MARK_ERROR_CHECK(fyr, &handle->start_mark, &handle->end_mark, FYEM_SCAN, + !is_multiline, err_out, + "Multi line plain scalars not supported in JSON mode"); + + FYR_MARK_ERROR_CHECK(fyr, &handle->start_mark, &handle->end_mark, FYEM_SCAN, + !fy_atom_strcmp(handle, "false") || + !fy_atom_strcmp(handle, "true") || + !fy_atom_strcmp(handle, "null") || + fy_atom_is_number(handle), err_out, + "Invalid JSON plain scalar"); + } + + + return 0; + +err_out: + rc = -1; + return rc; +} + +int fy_fetch_flow_scalar(struct fy_parser *fyp, int c) +{ + struct fy_atom handle; + bool is_single, is_complex, is_multiline; + struct fy_mark mark; + struct fy_simple_key_mark skm; + struct fy_token *fyt; + int i = 0, rc = -1; + + is_single = c == '\''; + + fyp_error_check(fyp, c == '\'' || c == '"', err_out, + "bad start of flow scalar ('%s')", + fy_utf8_format_a(c, fyue_singlequote)); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_flow_indent_check(fyp), err_out, + "wrongly indented %s scalar in flow mode", + is_single ? "single-quoted" : "double-quoted"); + + fy_get_mark(fyp, &mark); + fy_get_simple_key_mark(fyp, &skm); + + /* errors are generated by reader */ + rc = fy_reader_fetch_flow_scalar_handle(fyp->reader, c, fyp->indent, &handle, !!(fyp->cfg.flags & FYPCF_SLOPPY_FLOW_INDENTATION)); + if (rc) { + fyp->stream_error = true; + goto err_out_rc; + } + + /* and we're done */ + fyt = fy_token_queue(fyp, FYTT_SCALAR, &handle, is_single ? FYSS_SINGLE_QUOTED : FYSS_DOUBLE_QUOTED); + fyp_error_check(fyp, fyt, err_out_rc, + "fy_token_queue() failed"); + + if (fyp->parse_flow_only && fyp->flow_level == 0) { + rc = fy_fetch_stream_end(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_stream_end() failed"); + return 0; + } + + is_complex = fyp->pending_complex_key_column >= 0; + is_multiline = handle.end_mark.line > handle.start_mark.line; + + if (!fyp->flow_level) { + /* due to the weirdness with simple keys scan forward + * until a linebreak, ';', or anything else */ + for (i = 0; ; i++) { + c = fy_parse_peek_at(fyp, i); + if (c < 0 || c == ':' || fyp_is_lb(fyp, c) || !fyp_is_flow_ws(fyp, c)) + break; + } + + /* if we're a multiline key that's bad */ + FYP_MARK_ERROR_CHECK(fyp, &mark, &mark, FYEM_SCAN, + !(is_multiline && !is_complex && c == ':'), err_out, + "invalid multiline %s scalar used as key", + is_single ? "single-quoted" : "double-quoted"); + + FYP_PARSE_ERROR_CHECK(fyp, i, 1, FYEM_SCAN, + c < 0 || c == ':' || c == '#' || fyp_is_lb(fyp, c), err_out, + "invalid trailing content after %s scalar", + is_single ? "single-quoted" : "double-quoted"); + } + + /* a plain scalar could be simple key */ + rc = fy_save_simple_key_mark(fyp, &skm, FYTT_SCALAR, &handle.end_mark); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_save_simple_key_mark() failed"); + + /* cannot follow a flow scalar */ + fyp->simple_key_allowed = false; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + /* make sure that no comment follows directly afterwards */ + c = fy_parse_peek(fyp); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + c != '#', err_out, + "invalid comment without whitespace after %s scalar", + is_single ? "single-quoted" : "double-quoted"); + + if (fyp->cfg.flags & FYPCF_PARSE_COMMENTS) { + rc = fy_attach_comments_if_any(fyp, fyt); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_attach_right_hand_comment() failed"); + } + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +void fy_reader_skip_ws_cr_nl(struct fy_reader *fyr) +{ + const char *p, *s, *e; + char cc; + size_t len; + int line, column; + + assert(fyr); + + column = fyr->column; + line = fyr->line; + while ((p = fy_reader_ensure_lookahead(fyr, 1, &len)) != NULL) { + + s = p; + e = s + len; + + while (s < e) { + cc = *s; + if (cc == ' ') { + column++; + } else if (cc == '\n') { + column = 0; + line++; + } else if (cc == '\t') { + if (fyr->tabsize) + column += (fyr->tabsize - (column % fyr->tabsize)); + else + column++; + } else if (cc == '\r') { + column = 0; + line++; + + if (s + 1 > e) { + /* we have a dangling cr at the end of a block */ + + /* advance up to the point here */ + fy_reader_advance_octets(fyr, s - p); + + /* try again (should return enough or NULL) */ + p = fy_reader_ensure_lookahead(fyr, 1, &len); + + /* if we couldn't pull enough we're done */ + if (!p || len < 1) + goto done; + + s = p; + e = s + len; + + if (*s == '\n') + s++; + } + /* \n followed, gulp it down */ + if (*s == '\n') + s++; + } else { + if (s > p) + fy_reader_advance_octets(fyr, s - p); + goto done; + } + + s++; + } + + fy_reader_advance_octets(fyr, s - p); + } + +done: + fyr->line = line; + fyr->column = column; +} + +void fy_reader_skip_ws(struct fy_reader *fyr) +{ + const char *p, *s, *e; + size_t len, consumed; + int column; + + assert(fyr); + + while ((p = fy_reader_ensure_lookahead(fyr, 1, &len)) != NULL) { + + s = p; + e = s + len; + + column = fyr->column; + if (!fyr->tabsize) { + while (s < e && fy_is_ws(*s)) { + column++; + s++; + } + } else { + while (s < e && fy_is_ws(*s)) { + if (fy_is_tab(*s)) + column += fyr->tabsize - (column % fyr->tabsize); + else + column++; + s++; + } + } + + consumed = s - p; + if (consumed) { + fy_reader_advance_octets(fyr, consumed); + fyr->column = column; + } + + /* we're done if stopped earlier */ + if (s < e) + break; + } +} + +void fy_reader_skip_space(struct fy_reader *fyr) +{ + const char *p, *s, *e; + size_t len, consumed; + + assert(fyr); + + while ((p = fy_reader_ensure_lookahead(fyr, 1, &len)) != NULL) { + + s = p; + e = s + len; + + while (s < e && fy_is_space(*s)) + s++; + + consumed = s - p; + if (consumed) { + fy_reader_advance_octets(fyr, consumed); + fyr->column += consumed; + } + + if (s < e) + break; + } +} + +void fy_reader_skip_ws_lb(struct fy_reader *fyr) +{ + const char *p, *s, *e; + size_t len, consumed; + int line, column, c, w; + bool dangling_cr; + enum fy_lb_mode lb_mode; + + assert(fyr); + + /* punt to json mode */ + lb_mode = fy_reader_lb_mode(fyr); + + if (fy_reader_json_mode(fyr) || lb_mode == fylb_cr_nl) { + fy_reader_skip_ws_cr_nl(fyr); + return; + } + + column = fyr->column; + line = fyr->line; + dangling_cr = false; + while ((p = fy_reader_ensure_lookahead(fyr, 1, &len)) != NULL) { + + s = p; + e = s + len; + + if (dangling_cr) { + if (*s == '\n') + s++; + dangling_cr = false; + } + + while (s < e) { + c = (int)*s; + + /* single byte utf8? */ + if (c < 0x80) { + if (c == ' ') { + column++; + } else if (c == '\n') { + column = 0; + line++; + } else if (c == '\t') { + if (fyr->tabsize) + column += (fyr->tabsize - (column % fyr->tabsize)); + else + column++; + } else if (c == '\r') { + column = 0; + line++; + /* check for '\n' following */ + if (s < e) { + if (*s == '\n') + s++; + } else { + /* we have a dangling cr at the end of a block */ + dangling_cr = true; + } + } else { + consumed = s - p; + if (consumed) + fy_reader_advance_octets(fyr, consumed); + goto done; + } + s++; + } else { + c = fy_utf8_get(s, (size_t)(e - s), &w); + + if (c == FYUG_PARTIAL) { + /* get the width (from the first octet */ + w = fy_utf8_width_by_first_octet((uint8_t)*s); + /* copy the partial utf8 in the buffer */ + + /* advance up to the point here */ + consumed = s - p; + if (consumed) + fy_reader_advance_octets(fyr, consumed); + + /* try again (should return enough or NULL) */ + p = fy_reader_ensure_lookahead(fyr, w, &len); + if (!p) + break; + + /* if we couldn't pull enough we're done */ + if (len < (size_t)w) + goto done; + + continue; + } + + if (lb_mode == fylb_cr_nl_N_L_P && fy_is_unicode_lb(c)) { + column = 0; + line++; + } else { + consumed = s - p; + if (consumed) + fy_reader_advance_octets(fyr, consumed); + goto done; + } + + s += w; + } + } + + consumed = s - p; + if (consumed) + fy_reader_advance_octets(fyr, consumed); + } + +done: + fyr->line = line; + fyr->column = column; +} + +int fy_fetch_plain_scalar(struct fy_parser *fyp, int c) +{ + struct fy_atom handle; + struct fy_simple_key_mark skm; + struct fy_token *fyt; + bool is_multiline, is_complex, is_tab_start = false; + struct fy_mark tab_mark; + int rc = -1, i; + + /* Extremely bad case, a tab... so, either an indentation or separation space in block mode */ + if (!fyp->flow && fy_is_tab(c)) { + + fy_get_mark(fyp, &tab_mark); + is_tab_start = true; + + /* skip all whitespace now */ + fy_reader_skip_ws(fyp->reader); + c = fy_parse_peek(fyp); + + /* if it's a linebreak or a comment start, just try again */ + if (fyp_is_lb(fyp, c) || c == '#') { + /* will need to scan more */ + fyp->token_activity_counter++; + return 0; + } + } + + /* check indentation */ + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + fy_flow_indent_check(fyp), err_out, + "wrongly indented flow %s", + fyp->flow == FYFT_SEQUENCE ? "sequence" : "mapping"); + + fy_get_simple_key_mark(fyp, &skm); + + rc = fy_reader_fetch_plain_scalar_handle(fyp->reader, c, fyp->indent, fyp->flow_level, &handle); + if (rc) { + fyp->stream_error = true; + goto err_out_rc; + } + + is_multiline = handle.end_mark.line > handle.start_mark.line; + is_complex = fyp->pending_complex_key_column >= 0; + + /* and we're done */ + fyt = fy_token_queue(fyp, FYTT_SCALAR, &handle, FYSS_PLAIN); + fyp_error_check(fyp, fyt, err_out_rc, + "fy_token_queue() failed"); + + if (fyp->parse_flow_only && fyp->flow_level == 0) { + rc = fy_fetch_stream_end(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_stream_end() failed"); + return 0; + } + + if (!fyp->flow_level && !is_complex && (is_multiline || is_tab_start)) { + /* due to the weirdness with simple keys scan forward + * until a linebreak, ':', or anything else */ + for (i = 0; ; i++) { + c = fy_parse_peek_at(fyp, i); + if (c < 0 || (c == ':' && fy_is_blankz_at_offset(fyp, i + 1)) || + fyp_is_lb(fyp, c) || !fy_is_ws(c)) + break; + } + + /* if we're a key, that's invalid */ + if (c == ':') { + + if (is_multiline) + FYP_MARK_ERROR(fyp, &handle.start_mark, &handle.end_mark, FYEM_SCAN, + "invalid multiline plain key"); + else + FYP_MARK_ERROR(fyp, &tab_mark, &tab_mark, FYEM_SCAN, + "invalid tab as indendation in a mapping"); + + goto err_out; + } + } + + rc = fy_save_simple_key_mark(fyp, &skm, FYTT_SCALAR, &handle.end_mark); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_save_simple_key_mark() failed"); + + fyp->simple_key_allowed = handle.simple_key_allowed; + fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); + + if (fyp->cfg.flags & FYPCF_PARSE_COMMENTS) { + rc = fy_attach_comments_if_any(fyp, fyt); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_attach_right_hand_comment() failed"); + } + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_fetch_tokens(struct fy_parser *fyp) +{ + struct fy_mark m; + bool was_double_colon; + int c, rc; + + /* do not fetch any more when stream end is reached */ + if (fyp->stream_end_reached) + return 0; + + if (!fyp->stream_start_produced) { + rc = fy_parse_get_next_input(fyp); + fyp_error_check(fyp, rc >= 0, err_out_rc, + "fy_parse_get_next_input() failed"); + + if (rc > 0) { + rc = fy_fetch_stream_start(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_stream_start() failed"); + } + return 0; + } + + fyp_scan_debug(fyp, "-------------------------------------------------"); + rc = fy_scan_to_next_token(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_scan_to_next_token() failed"); + + if (fyp_block_mode(fyp)) { + if (fyp->parse_block_only && fyp->starting_indent >= 0 && + fyp_column(fyp) < fyp->starting_indent) { + rc = fy_fetch_stream_end(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_stream_end() failed"); + return 0; + } + rc = fy_parse_unroll_indent(fyp, fyp_column(fyp)); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_unroll_indent() failed"); + } + + c = fy_parse_peek(fyp); + if (c < 0 || c == '\0') { + + fyp->stream_end_reached = true; + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + !fyp_json_mode(fyp) || c != '\0', err_out, + "JSON disallows '\\0' in the input stream"); + + if (c >= 0) + fy_advance(fyp, c); + rc = fy_fetch_stream_end(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_stream_end() failed"); + return 0; + } + + if (fyp_column(fyp) == 0 && c == '%') { + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + !fyp_json_mode(fyp), err_out, + "directives not supported in JSON mode"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + !fyp->bare_document_only, err_out, + "invalid directive in bare document mode"); + + fy_advance(fyp, c); + rc = fy_fetch_directive(fyp); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_directive() failed"); + goto out; + } + + /* probable document start/end indicator */ + if (fyp_column(fyp) == 0 && + (!fy_parse_strncmp(fyp, "---", 3) || + !fy_parse_strncmp(fyp, "...", 3)) && + fy_is_blankz_at_offset(fyp, 3)) { + + FYP_PARSE_ERROR_CHECK(fyp, 0, 3, FYEM_SCAN, + !fyp_json_mode(fyp), err_out, + "document %s indicator not supported in JSON mode", + c == '-' ? "start" : "end"); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 3, FYEM_SCAN, + !fyp->bare_document_only, err_out, + "invalid document %s indicator in bare document mode", + c == '-' ? "start" : "end"); + + rc = fy_fetch_document_indicator(fyp, + c == '-' ? FYTT_DOCUMENT_START : + FYTT_DOCUMENT_END); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_document_indicator() failed"); + + fyp->indent_line = fyp_line(fyp); + + /* for document end, nothing must follow except whitespace and comment */ + if (c == '.') { + c = fy_parse_peek(fyp); + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + c == -1 || c == '#' || fyp_is_lb(fyp, c), err_out, + "invalid content after document end marker"); + } + + goto out; + } + + fyp_scan_debug(fyp, "indent=%d, parent indent=%d\n", + fyp->indent, fyp->parent_indent); + + if (c == '[' || c == '{') { + + fyp->indent_line = fyp_line(fyp); + + fyp_scan_debug(fyp, "calling fy_fetch_flow_collection_mark_start(%c)", c); + rc = fy_fetch_flow_collection_mark_start(fyp, c); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_flow_collection_mark_start() failed"); + goto out; + } + + if (c == ']' || c == '}') { + + fyp->indent_line = fyp_line(fyp); + + fyp_scan_debug(fyp, "fy_fetch_flow_collection_mark_end(%c)", c); + rc = fy_fetch_flow_collection_mark_end(fyp, c); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_flow_collection_mark_end() failed"); + goto out; + } + + + if (c == ',') { + + fyp->indent_line = fyp_line(fyp); + + fy_get_mark(fyp, &m); + + fyp_scan_debug(fyp, "fy_fetch_flow_collection_entry(%c)", c); + rc = fy_fetch_flow_collection_entry(fyp, c); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_flow_collection_entry() failed"); + + fyp->last_was_comma = true; + fyp->last_comma_mark = m; + + goto out; + } + + if (c == '-' && fy_is_blankz_at_offset(fyp, 1)) { + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + !fyp_json_mode(fyp), err_out, + "block entries not supported in JSON mode"); + + fyp->indent_line = fyp_line(fyp); + + fyp_scan_debug(fyp, "fy_fetch_block_entry(%c)", c); + rc = fy_fetch_block_entry(fyp, c); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_block_entry() failed"); + goto out; + } + + if (c == '?' && fy_is_blankz_at_offset(fyp, 1)) { + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + !fyp_json_mode(fyp), err_out, + "complex keys not supported in JSON mode"); + + fyp->indent_line = fyp_line(fyp); + + fyp_scan_debug(fyp, "fy_fetch_key(%c)", c); + rc = fy_fetch_key(fyp, c); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_key() failed"); + goto out; + } + + if (c == ':') { + was_double_colon = c == ':' && fyp->colon_follows_colon && fyp->flow_level > 0; + fyp->colon_follows_colon = false; + + if (((fyp->flow_level && !fyp->simple_key_allowed) || fy_is_blankz_at_offset(fyp, 1)) && + !was_double_colon) { + + fyp->indent_line = fyp_line(fyp); + + fyp_scan_debug(fyp, "fy_fetch_value(%c)", c); + rc = fy_fetch_value(fyp, c); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_value() failed"); + goto out; + } + } + + if (c == '*' || c == '&') { + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + !fyp_json_mode(fyp), err_out, + "%s not supported in JSON mode", + c == '&' ? "anchor" : "alias"); + + fyp_scan_debug(fyp, "fy_fetch_anchor_or_alias(%c)", c); + rc = fy_fetch_anchor_or_alias(fyp, c); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_anchor_or_alias() failed"); + goto out; + } + + if (c == '!') { + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + !fyp_json_mode(fyp), err_out, + "tag not supported in JSON mode"); + + fyp_scan_debug(fyp, "fy_fetch_tag(%c)", c); + rc = fy_fetch_tag(fyp, c); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_tag() failed"); + goto out; + } + + if (!fyp->flow_level && (c == '|' || c == '>')) { + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + !fyp_json_mode(fyp), err_out, + "block scalars not supported in JSON mode"); + + fyp_scan_debug(fyp, "fy_fetch_block_scalar(%c)", c); + rc = fy_fetch_block_scalar(fyp, c == '|', c); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_block_scalar() failed"); + goto out; + } + + if (c == '\'' || c == '"') { + + FYP_PARSE_ERROR_CHECK(fyp, 0, 1, FYEM_SCAN, + c == '"' || !fyp_json_mode(fyp), err_out, + "single quoted scalars not supported in JSON mode"); + + fyp_scan_debug(fyp, "fy_fetch_flow_scalar(%c)", c); + rc = fy_fetch_flow_scalar(fyp, c); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_flow_scalar() failed"); + goto out; + } + + fyp_scan_debug(fyp, "fy_fetch_plain_scalar(%c)", c); + rc = fy_fetch_plain_scalar(fyp, c); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_fetch_plain_scalar() failed"); + +out: + if (c != ',' && fyp->last_was_comma) + fyp->last_was_comma = false; + + return 0; + +err_out: + rc = -1; +err_out_rc: + return rc; +} + +struct fy_token *fy_scan_peek(struct fy_parser *fyp) +{ + struct fy_token *fyt; + int rc, last_token_activity_counter; + bool have_simple_keys; + + /* do not try to peek */ + if (fy_reader_generates_events(fyp->reader)) + return NULL; + + /* nothing if stream end produced (and no stream end token in queue) */ + if (fyp->stream_end_produced) { + fyt = fy_token_list_head(&fyp->queued_tokens); + if (fyt && fyt->type == FYTT_STREAM_END) + return fyt; + + /* OK, we're done, flush everything */ + fy_token_list_unref_all(&fyp->queued_tokens); + + /* try to get the next input */ + rc = fy_parse_get_next_input(fyp); + fyp_error_check(fyp, rc >= 0, err_out, + "fy_parse_get_next_input() failed"); + + /* no more inputs */ + if (rc == 0) { + fyp_scan_debug(fyp, "token stream ends"); + return NULL; + } + + fyp_scan_debug(fyp, "starting new token stream"); + + fyp->stream_start_produced = false; + fyp->stream_end_produced = false; + fyp->stream_end_reached = false; + } + + /* we loop until we have a token and the simple key list is empty */ + for (;;) { + fyt = fy_token_list_head(&fyp->queued_tokens); + have_simple_keys = !fy_simple_key_list_empty(&fyp->simple_keys); + + /* we can produce a token when: + * a) one exists + * b) no simple keys exist at all + */ + if (fyt && !have_simple_keys) + break; + + /* on stream error we're done */ + if (fyp->stream_error) + return NULL; + + /* keep track of token activity, if it didn't change + * after the fetch tokens call, the state machine is stuck + */ + last_token_activity_counter = fyp->token_activity_counter; + + /* fetch more then */ + rc = fy_fetch_tokens(fyp); + fyp_error_check(fyp, !rc, err_out, + "fy_fetch_tokens() failed"); + + if (fy_reader_generates_events(fyp->reader)) + return NULL; + + fyp_error_check(fyp, last_token_activity_counter != fyp->token_activity_counter, err_out, + "out of tokens and failed to produce anymore"); + } + + switch (fyt->type) { + case FYTT_STREAM_START: + fyp_scan_debug(fyp, "setting stream_start_produced to true"); + fyp->stream_start_produced = true; + break; + case FYTT_STREAM_END: + fyp_scan_debug(fyp, "setting stream_end_produced to true"); + fyp->stream_end_produced = true; + + if (!fyp->parse_flow_only && !fyp->parse_block_only) { + rc = fy_reader_input_done(fyp->reader); + fyp_error_check(fyp, !rc, err_out, + "fy_parse_input_done() failed"); + } + break; + default: + break; + } + + return fyt; + +err_out: + return NULL; +} + +static inline struct fy_token * +fy_scan_remove(struct fy_parser *fyp, struct fy_token *fyt) +{ + if (!fyp || !fyt) + return NULL; + + fy_token_list_del(&fyp->queued_tokens, fyt); + + return fyt; +} + +static inline struct fy_token * +fy_scan_remove_peek(struct fy_parser *fyp, struct fy_token *fyt) +{ + if (fyt != NULL) { + (void)fy_scan_remove(fyp, fyt); + fy_token_unref_rl(fyp->recycled_token_list, fyt); + } + + return fy_scan_peek(fyp); +} + +struct fy_token *fy_scan(struct fy_parser *fyp) +{ + struct fy_token *fyt; + + fyt = fy_scan_remove(fyp, fy_scan_peek(fyp)); + + if (fyt && (fyt->type == FYTT_VERSION_DIRECTIVE || fyt->type == FYTT_TAG_DIRECTIVE)) { + + /* + * NOTE: we need to update the document state with the contents of + * directives, so that tags etc, work correctly. + * This is arguably a big hack, but so is using the scanner in such + * a low level. + * + * This is not very good because we don't keep track of parser state + * so tag directives in the middle of the document are AOK. + * But we don't really care, if you care about stream validity do + * a proper parse. + */ + + /* we take a reference because the parse methods take ownership */ + fy_token_ref(fyt); + + /* we ignore errors, because... they are parse errors, not scan errors */ + + if (fyt->type == FYTT_VERSION_DIRECTIVE) + (void)fy_parse_version_directive(fyp, fyt, true); + else + (void)fy_parse_tag_directive(fyp, fyt, true); + } + +#ifdef FY_DEVMODE + if (fyt) + fyp_debug_dump_token(fyp, fyt, "producing: "); +#endif + return fyt; +} + +void fy_scan_token_free(struct fy_parser *fyp, struct fy_token *fyt) +{ + fy_token_unref_rl(fyp->recycled_token_list, fyt); +} + +int fy_parse_state_push(struct fy_parser *fyp, enum fy_parser_state state) +{ + struct fy_parse_state_log *fypsl; + + fypsl = fy_parse_parse_state_log_alloc(fyp); + fyp_error_check(fyp, fypsl != NULL, err_out, + "fy_parse_state_log_alloc() failed!"); + fypsl->state = state; + fy_parse_state_log_list_push(&fyp->state_stack, fypsl); + + return 0; +err_out: + return -1; +} + +enum fy_parser_state fy_parse_state_pop(struct fy_parser *fyp) +{ + struct fy_parse_state_log *fypsl; + enum fy_parser_state state; + + fypsl = fy_parse_state_log_list_pop(&fyp->state_stack); + if (!fypsl) + return FYPS_NONE; + + state = fypsl->state; + + fy_parse_parse_state_log_recycle(fyp, fypsl); + + return state; +} + +void fy_parse_state_set(struct fy_parser *fyp, enum fy_parser_state state) +{ + fyp_parse_debug(fyp, "state %s -> %s\n", state_txt[fyp->state], state_txt[state]); + fyp->state = state; +} + +enum fy_parser_state fy_parse_state_get(struct fy_parser *fyp) +{ + return fyp->state; +} + +static struct fy_eventp * +fy_parse_node(struct fy_parser *fyp, struct fy_token *fyt, bool is_block) +{ + struct fy_eventp *fyep = NULL; + struct fy_event *fye = NULL; + struct fy_document_state *fyds = NULL; + struct fy_token *anchor = NULL, *tag = NULL; + const char *handle; + size_t handle_size; + struct fy_token *fyt_td; + struct fy_token *fytn; + struct fy_atom *ev_handle = NULL; + + fyds = fyp->current_document_state; + assert(fyds); + + fyp_parse_debug(fyp, "parse_node: is_block=%s - fyt %s", + is_block ? "true" : "false", + fy_token_type_txt[fyt->type]); + + if (fyt->type == FYTT_ALIAS) { + fy_parse_state_set(fyp, fy_parse_state_pop(fyp)); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_ALIAS; + fye->alias.anchor = fy_scan_remove(fyp, fyt); + + ev_handle = &fye->alias.anchor->handle; + goto return_ok; + } + + while ((!anchor && fyt->type == FYTT_ANCHOR) || (!tag && fyt->type == FYTT_TAG)) { + if (fyt->type == FYTT_ANCHOR) + anchor = fy_scan_remove(fyp, fyt); + else + tag = fy_scan_remove(fyp, fyt); + + fyt = fy_scan_peek(fyp); + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); + + fyp_parse_debug(fyp, "parse_node: ANCHOR|TAG got - fyt %s", + fy_token_type_txt[fyt->type]); + + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + fyt->type != FYTT_ALIAS, err_out, + "unexpected alias"); + } + + /* check tag prefix */ + if (tag && tag->tag.handle_length) { + handle = fy_atom_data(&tag->handle) + tag->tag.skip; + handle_size = tag->tag.handle_length; + + fyt_td = fy_document_state_lookup_tag_directive(fyds, handle, handle_size); + + FYP_TOKEN_ERROR_CHECK(fyp, tag, FYEM_PARSE, + fyt_td, err_out, + "undefined tag prefix '%.*s'", (int)handle_size, handle); + } + + if ((fyp->state == FYPS_BLOCK_MAPPING_VALUE || + fyp->state == FYPS_BLOCK_MAPPING_FIRST_KEY) + && fyt->type == FYTT_BLOCK_ENTRY) { + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_SEQUENCE_START; + fye->sequence_start.anchor = anchor; + fye->sequence_start.tag = tag; + + /* allocate and copy in place */ + fytn = fy_token_alloc_rl(fyp->recycled_token_list); + fyp_error_check(fyp, fytn, err_out, + "fy_token_alloc_rl() failed!"); + fytn->type = FYTT_BLOCK_SEQUENCE_START; + fytn->handle = fyt->handle; + fytn->handle.end_mark = fytn->handle.start_mark; /* no extent */ + fy_input_ref(fytn->handle.fyi); + + fye->sequence_start.sequence_start = fytn; + + fy_parse_state_set(fyp, FYPS_INDENTLESS_SEQUENCE_ENTRY); + + ev_handle = &fye->sequence_start.sequence_start->handle; + goto return_ok; + } + + if (fyt->type == FYTT_SCALAR) { + fy_parse_state_set(fyp, fy_parse_state_pop(fyp)); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_SCALAR; + fye->scalar.anchor = anchor; + fye->scalar.tag = tag; + fye->scalar.value = fy_scan_remove(fyp, fyt); + + ev_handle = &fye->scalar.value->handle; + goto return_ok; + } + + if (fyt->type == FYTT_FLOW_SEQUENCE_START) { + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_SEQUENCE_START; + fye->sequence_start.anchor = anchor; + fye->sequence_start.tag = tag; + fye->sequence_start.sequence_start = fy_scan_remove(fyp, fyt); + fy_parse_state_set(fyp, FYPS_FLOW_SEQUENCE_FIRST_ENTRY); + + ev_handle = &fye->sequence_start.sequence_start->handle; + goto return_ok; + } + + if (fyt->type == FYTT_FLOW_MAPPING_START) { + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_MAPPING_START; + fye->mapping_start.anchor = anchor; + fye->mapping_start.tag = tag; + fye->mapping_start.mapping_start = fy_scan_remove(fyp, fyt); + fy_parse_state_set(fyp, FYPS_FLOW_MAPPING_FIRST_KEY); + + ev_handle = &fye->mapping_start.mapping_start->handle; + goto return_ok; + } + + if (is_block && fyt->type == FYTT_BLOCK_SEQUENCE_START) { + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_SEQUENCE_START; + fye->sequence_start.anchor = anchor; + fye->sequence_start.tag = tag; + fye->sequence_start.sequence_start = fy_scan_remove(fyp, fyt); + fy_parse_state_set(fyp, FYPS_BLOCK_SEQUENCE_FIRST_ENTRY); + + ev_handle = &fye->sequence_start.sequence_start->handle; + goto return_ok; + } + + if (is_block && fyt->type == FYTT_BLOCK_MAPPING_START) { + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_MAPPING_START; + fye->mapping_start.anchor = anchor; + fye->mapping_start.tag = tag; + fye->mapping_start.mapping_start = fy_scan_remove(fyp, fyt); + fy_parse_state_set(fyp, FYPS_BLOCK_MAPPING_FIRST_KEY); + + ev_handle = &fye->mapping_start.mapping_start->handle; + goto return_ok; + } + + if (!anchor && !tag) { + + if (fyt->type == FYTT_FLOW_ENTRY && + (fyp->state == FYPS_FLOW_SEQUENCE_FIRST_ENTRY || + fyp->state == FYPS_FLOW_SEQUENCE_ENTRY)) + + FYP_TOKEN_ERROR(fyp, fyt, FYEM_PARSE, + "flow sequence with invalid %s", + fyp->state == FYPS_FLOW_SEQUENCE_FIRST_ENTRY ? + "comma in the beginning" : "extra comma"); + + else if ((fyt->type == FYTT_DOCUMENT_START || fyt->type == FYTT_DOCUMENT_END) && + (fyp->state == FYPS_FLOW_SEQUENCE_FIRST_ENTRY || + fyp->state == FYPS_FLOW_SEQUENCE_ENTRY)) + + FYP_TOKEN_ERROR(fyp, fyt, FYEM_PARSE, + "invalid document %s indicator in a flow sequence", + fyt->type == FYTT_DOCUMENT_START ? + "start" : "end"); + else + FYP_TOKEN_ERROR(fyp, fyt, FYEM_PARSE, + "did not find expected node content"); + + goto err_out; + } + + fyp_parse_debug(fyp, "parse_node: empty scalar..."); + + /* empty scalar */ + fy_parse_state_set(fyp, fy_parse_state_pop(fyp)); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_SCALAR; + fye->scalar.anchor = anchor; + fye->scalar.tag = tag; + + /* copy atom from the token and set to zero size at start */ + fye->scalar.value = fy_token_create_rl( + fyp->recycled_token_list, FYTT_SCALAR, &fyp->last_event_handle, FYSS_PLAIN); + fyp_error_check(fyp, fye->scalar.value, err_out, + "failed to allocate SCALAR token()"); + /* mark it as a special value */ + fye->scalar.value->scalar.is_null = true; + +return_ok: + if (ev_handle) { + fy_input_unref(fyp->last_event_handle.fyi); + fyp->last_event_handle = *ev_handle; + fyp->last_event_handle.start_mark = fyp->last_event_handle.end_mark; + fy_atom_reset_storage_hints(&fyp->last_event_handle); + fy_input_ref(fyp->last_event_handle.fyi); + } + + fyp_parse_debug(fyp, "parse_node: > %s", + fy_event_type_txt[fye->type]); + + return fyep; + +err_out: + fy_token_unref_rl(fyp->recycled_token_list, anchor); + fy_token_unref_rl(fyp->recycled_token_list, tag); + fy_parse_eventp_recycle(fyp, fyep); + + return NULL; +} + +static struct fy_eventp * +fy_parse_empty_scalar(struct fy_parser *fyp) +{ + struct fy_eventp *fyep; + struct fy_event *fye; + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_SCALAR; + fye->scalar.anchor = NULL; + fye->scalar.tag = NULL; + + /* for empty scalar the last event handle does not change, only */ + fye->scalar.value = fy_token_create_rl( + fyp->recycled_token_list, FYTT_SCALAR, &fyp->last_event_handle, FYSS_PLAIN); + fyp_error_check(fyp, fye->scalar.value, err_out, + "failed to allocate SCALAR token()"); + /* mark it as a special value */ + fye->scalar.value->scalar.is_null = true; + return fyep; + +err_out: + return NULL; +} + +int fy_parse_stream_start(struct fy_parser *fyp) +{ + fyp->indent = -2; + fyp->indent_line = -1; + fyp->starting_indent = -1; + fyp->generated_block_map = false; + fyp->last_was_comma = false; + fyp->flow = FYFT_NONE; + fyp->pending_complex_key_column = -1; + + fy_parse_indent_list_recycle_all(fyp, &fyp->indent_stack); + fy_parse_simple_key_list_recycle_all(fyp, &fyp->simple_keys); + fy_parse_parse_state_log_list_recycle_all(fyp, &fyp->state_stack); + fy_parse_flow_list_recycle_all(fyp, &fyp->flow_stack); + + fy_token_unref_rl(fyp->recycled_token_list, fyp->stream_end_token); + fyp->stream_end_token = NULL; + + return 0; +} + +int fy_parse_stream_end(struct fy_parser *fyp) +{ + fy_token_unref_rl(fyp->recycled_token_list, fyp->stream_end_token); + fyp->stream_end_token = NULL; + + return 0; +} + +static struct fy_eventp *fy_parse_internal(struct fy_parser *fyp) +{ + struct fy_eventp *fyep = NULL; + struct fy_event *fye = NULL; + struct fy_token *fyt = NULL; + struct fy_document_state *fyds = NULL; + bool is_block, is_seq, is_value, is_first, had_doc_end, had_directives; + enum fy_parser_state orig_state; + struct fy_token *version_directive; + struct fy_token_list tag_directives; + const struct fy_mark *fym; + struct fy_atom handle, *ev_handle; + struct fy_token *fytn; + char tbuf[16] __FY_DEBUG_UNUSED__; + int rc; + + version_directive = NULL; + fy_token_list_init(&tag_directives); + + /* are we done? */ + if (fyp->stream_error || fyp->state == FYPS_END) { + fy_parser_debug(fyp, "%s: NULL: fyp->stream_error=%s fyp->state=%s", __func__, + fyp->stream_error ? "true" : "false", state_txt[fyp->state]); + return NULL; + } + + fyt = fy_scan_peek(fyp); +#ifdef FY_DEVMODE + fy_parser_debug(fyp, "[%s]", state_txt[fyp->state]); + fyp_debug_dump_token(fyp, fyt, "scan_peek: "); +#endif + + if (!fyt && fy_reader_generates_events(fyp->reader)) { + fye = fy_reader_generate_next_event(fyp->reader); + if (!fye) { + rc = fy_reader_input_done(fyp->reader); + fyp_error_check(fyp, !rc, err_out, + "fy_parse_input_done() failed"); + + fy_parser_debug(fyp, "%s: NULL: !fyt && fy_reader_generates_events(fyp->reader)", __func__); + return NULL; + } + return container_of(fye, struct fy_eventp, e); + } + + /* special case without an error message for start */ + if (!fyt && fyp->state == FYPS_NONE) { + fy_parser_debug(fyp, "%s: NULL: !fyt && fyp->state==FYPS_NONE", __func__); + return NULL; + } + + /* keep a copy of stream end */ + if (fyt && fyt->type == FYTT_STREAM_END && !fyp->stream_end_token) { + fyp->stream_end_token = fy_token_ref(fyt); + fyp_parse_debug(fyp, "kept copy of STRM-"); + } + + /* keep on producing STREAM_END */ + if (!fyt && fyp->stream_end_token) { + fyt = fyp->stream_end_token; + fy_token_list_add_tail(&fyp->queued_tokens, fyt); + + fyp_parse_debug(fyp, "generated copy of STRM-"); + } + + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); + + assert(fyt->handle.fyi); + + fyp_parse_debug(fyp, "[%s] <- %s", state_txt[fyp->state], + fy_token_dump_format(fyt, tbuf, sizeof(tbuf))); + + is_first = false; + had_doc_end = false; + + fyep = NULL; + fye = NULL; + ev_handle = NULL; + + orig_state = fyp->state; + switch (fyp->state) { + case FYPS_NONE: + fy_parse_state_set(fyp, FYPS_STREAM_START); + /* fallthrough */ + + case FYPS_STREAM_START: + + fyp_error_check(fyp, fyt->type == FYTT_STREAM_START, err_out, + "failed to get valid stream start token"); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_STREAM_START; + + fye->stream_start.stream_start = fy_scan_remove(fyp, fyt); + + rc = fy_parse_stream_start(fyp); + fyp_error_check(fyp, !rc, err_out, + "stream start failed"); + + fy_parse_state_set(fyp, FYPS_IMPLICIT_DOCUMENT_START); + + fyp->stream_has_content = false; + + ev_handle = &fye->stream_start.stream_start->handle; + break; + + case FYPS_IMPLICIT_DOCUMENT_START: + + /* fallthrough */ + + case FYPS_DOCUMENT_START: + + had_doc_end = false; + + if (!fyp->stream_has_content && fyt->type != FYTT_STREAM_END) + fyp->stream_has_content = true; + + /* remove all extra document end indicators */ + while (fyt->type == FYTT_DOCUMENT_END) { + + /* reset document has content flag */ + fyp->document_has_content = false; + fyp->document_first_content_token = true; + + /* explicit end indicator, no more directives checking */ + fyp->had_directives = false; + + fyt = fy_scan_remove_peek(fyp, fyt); + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); + +#ifdef FY_DEVMODE + fyp_debug_dump_token(fyp, fyt, "next: "); +#endif + + had_doc_end = true; + } + + if (!fyp->current_document_state) { + + rc = fy_reset_document_state(fyp); + fyp_error_check(fyp, !rc, err_out, + "fy_reset_document_state() failed"); + } + + fyds = fyp->current_document_state; + fyp_error_check(fyp, fyds, err_out, + "no current document state error"); + + /* process directives */ + had_directives = false; + while (fyt->type == FYTT_VERSION_DIRECTIVE || + fyt->type == FYTT_TAG_DIRECTIVE) { + + had_directives = true; + fyp->had_directives = true; + + if (fyt->type == FYTT_VERSION_DIRECTIVE) { + + rc = fy_parse_version_directive(fyp, fy_scan_remove(fyp, fyt), false); + fyt = NULL; + fyp_error_check(fyp, !rc, err_out, + "failed to fy_parse_version_directive()"); + } else { + rc = fy_parse_tag_directive(fyp, fy_scan_remove(fyp, fyt), false); + fyt = NULL; + + fyp_error_check(fyp, !rc, err_out, + "failed to fy_parse_tag_directive()"); + } + + fyt = fy_scan_peek(fyp); + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); +#ifdef FY_DEVMODE + fyp_debug_dump_token(fyp, fyt, "next: "); +#endif + } + + /* the end */ + if (fyt->type == FYTT_STREAM_END) { + + /* empty content is not allowed in JSON mode */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !fyp_json_mode(fyp) || + fyp->stream_has_content, err_out, + "JSON does not allow empty root content"); + + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !fyp->had_directives || fyp->document_has_content || + !fyds->start_implicit, err_out, + "stream with directives without content"); + + rc = fy_parse_stream_end(fyp); + fyp_error_check(fyp, !rc, err_out, + "stream end failed"); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_STREAM_END; + fye->stream_end.stream_end = fy_scan_remove(fyp, fyt); + + fy_parse_state_set(fyp, + fy_parse_have_more_inputs(fyp) ? FYPS_NONE : FYPS_END); + + ev_handle = &fye->stream_end.stream_end->handle; + break; + } + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + /* document start */ + fye->type = FYET_DOCUMENT_START; + fye->document_start.document_start = NULL; + fye->document_start.document_state = NULL; + + if (!(fyp->state == FYPS_IMPLICIT_DOCUMENT_START || had_doc_end || fyt->type == FYTT_DOCUMENT_START)) { + fyds = fyp->current_document_state; + + /* not BLOCK_MAPPING_START */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + fyt->type == FYTT_BLOCK_MAPPING_START, err_out, + "missing document start"); + + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + fyds->start_implicit || + fyds->start_mark.line != fy_token_start_line(fyt), err_out, + "invalid mapping starting at --- line"); + + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + false, err_out, + "invalid mapping in plain multiline"); + } + + fym = fy_token_start_mark(fyt); + if (fym) + fyds->start_mark = *fym; + else + memset(&fyds->start_mark, 0, sizeof(fyds->start_mark)); + + if (fyt->type != FYTT_DOCUMENT_START) { + + /* copy atom from the token and set to zero size at start */ + handle = fyt->handle; + handle.end_mark = handle.start_mark; + fy_atom_reset_storage_hints(&handle); + + fye->document_start.document_start = fy_token_create_rl( + fyp->recycled_token_list, FYTT_DOCUMENT_START, &handle); + fyp_error_check(fyp, fye->document_start.document_start, err_out, + "failed to allocate DOCUMENT_START token()"); + + fyds->start_implicit = true; + fyp_parse_debug(fyp, "document_start_implicit=true"); + + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + fyt->type != FYTT_DOCUMENT_END || !had_directives, err_out, + "directive(s) without a document"); + + fy_parse_state_set(fyp, FYPS_BLOCK_NODE); + } else { + fye->document_start.document_start = fy_scan_remove(fyp, fyt); + + fyds->start_implicit = false; + fyp_parse_debug(fyp, "document_start_implicit=false"); + + fy_parse_state_set(fyp, FYPS_DOCUMENT_CONTENT); + } + + rc = fy_parse_state_push(fyp, FYPS_DOCUMENT_END); + fyp_error_check(fyp, !rc, err_out, + "failed to fy_parse_state_push()"); + + // update document state with json mode + fyds->json_mode = fyp_json_mode(fyp); + fye->document_start.document_state = fy_document_state_ref(fyds); + fye->document_start.implicit = fyds->start_implicit; + + ev_handle = &fye->document_start.document_start->handle; + break; + + case FYPS_DOCUMENT_END: + + fyds = fyp->current_document_state; + fyp_error_check(fyp, fyds, err_out, + "no current document state error"); + + if (fyt && (fyt->type == FYTT_VERSION_DIRECTIVE || + fyt->type == FYTT_TAG_DIRECTIVE)) { + int cmpval = fy_document_state_version_compare(fyds, fy_version_make(1, 1)); + + fyp_scan_debug(fyp, "version %d.%d %s %d.%d\n", + fyds->version.major, fyds->version.minor, + cmpval == 0 ? "=" : cmpval > 0 ? ">" : "<", + 1, 1); + + /* YAML 1.1 allows directives without document end */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + cmpval <= 0, err_out, + "missing explicit document end marker before directive(s)"); + + } + + fym = fy_token_end_mark(fyt); + if (fym) + fyds->end_mark = *fym; + else + memset(&fyds->end_mark, 0, sizeof(fyds->end_mark)); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + /* document end */ + fye->type = FYET_DOCUMENT_END; + if (fyt->type == FYTT_DOCUMENT_END) { + + handle = fyt->handle; + fy_atom_reset_storage_hints(&handle); + + fye->document_end.document_end = fy_token_create_rl( + fyp->recycled_token_list, FYTT_DOCUMENT_END, &handle); + fyp_error_check(fyp, fye->document_end.document_end, err_out, + "failed to allocate DOCUMENT_END token()"); + fyds->end_implicit = false; + + /* reset document has content flag */ + fyp->document_has_content = false; + fyp->document_first_content_token = true; + + /* reset directives */ + fyp->had_directives = false; + + } else { + + handle = fyt->handle; + handle.end_mark = handle.start_mark; + fy_atom_reset_storage_hints(&handle); + + fye->document_end.document_end = fy_token_create_rl( + fyp->recycled_token_list, FYTT_DOCUMENT_END, &handle); + fyp_error_check(fyp, fye->document_end.document_end, err_out, + "failed to allocate DOCUMENT_END token()"); + fyds->end_implicit = true; + } + + fye->document_end.implicit = fyds->end_implicit; + + if (!fyp->next_single_document) { + /* multi document mode */ + fy_parse_state_set(fyp, FYPS_DOCUMENT_START); + fyp->had_directives = false; + + /* and reset document state */ + rc = fy_reset_document_state(fyp); + fyp_error_check(fyp, !rc, err_out, + "fy_reset_document_state() failed"); + } else { + /* single document mode */ + fyp->next_single_document = false; + + fy_parse_state_set(fyp, FYPS_SINGLE_DOCUMENT_END); + } + + ev_handle = &fye->document_end.document_end->handle; + break; + + case FYPS_DOCUMENT_CONTENT: + + if (fyt->type == FYTT_VERSION_DIRECTIVE || + fyt->type == FYTT_TAG_DIRECTIVE || + fyt->type == FYTT_DOCUMENT_START || + fyt->type == FYTT_DOCUMENT_END || + fyt->type == FYTT_STREAM_END) { + + if (fyt->type == FYTT_DOCUMENT_START || + fyt->type == FYTT_DOCUMENT_END) { + fyp->document_has_content = false; + fyp->document_first_content_token = true; + fyp->had_directives = false; + } + + fy_parse_state_set(fyp, fy_parse_state_pop(fyp)); + + fyep = fy_parse_empty_scalar(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_empty_scalar() failed"); + break; + } + + fyp->document_has_content = true; + fyp_parse_debug(fyp, "document has content now"); + /* fallthrough */ + + case FYPS_BLOCK_NODE: + + fyep = fy_parse_node(fyp, fyt, + fyp->state == FYPS_BLOCK_NODE || + fyp->state == FYPS_DOCUMENT_CONTENT); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_node() failed"); + break; + + case FYPS_BLOCK_SEQUENCE_FIRST_ENTRY: + is_first = true; + /* fallthrough */ + + case FYPS_BLOCK_SEQUENCE_ENTRY: + case FYPS_INDENTLESS_SEQUENCE_ENTRY: + + if ((fyp->state == FYPS_BLOCK_SEQUENCE_ENTRY || + fyp->state == FYPS_BLOCK_SEQUENCE_FIRST_ENTRY) && + !(fyt->type == FYTT_BLOCK_ENTRY || + fyt->type == FYTT_BLOCK_END)) { + + + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !(fyt->type == FYTT_SCALAR), err_out, + "invalid scalar at the end of block sequence"); + + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !(fyt->type == FYTT_BLOCK_SEQUENCE_START), err_out, + "wrongly indented sequence item"); + + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + false, err_out, + "did not find expected '-' indicator"); + } + + if (fyt->type == FYTT_BLOCK_ENTRY) { + + /* BLOCK entry */ + fyt = fy_scan_remove_peek(fyp, fyt); + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); +#ifdef FY_DEVMODE + fyp_debug_dump_token(fyp, fyt, "next: "); +#endif + + /* check whether it's a sequence entry or not */ + is_seq = fyt->type != FYTT_BLOCK_ENTRY && fyt->type != FYTT_BLOCK_END; + if (!is_seq && fyp->state == FYPS_INDENTLESS_SEQUENCE_ENTRY) + is_seq = fyt->type != FYTT_KEY && fyt->type != FYTT_VALUE; + + if (is_seq) { + rc = fy_parse_state_push(fyp, fyp->state); + fyp_error_check(fyp, !rc, err_out, + "failed to push state"); + + fyep = fy_parse_node(fyp, fyt, true); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_node() failed"); + break; + } + fy_parse_state_set(fyp, FYPS_BLOCK_SEQUENCE_ENTRY); + + fyep = fy_parse_empty_scalar(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_empty_scalar() failed"); + break; + } + + /* FYTT_BLOCK_END */ + fy_parse_state_set(fyp, fy_parse_state_pop(fyp)); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_SEQUENCE_END; + if (orig_state == FYPS_INDENTLESS_SEQUENCE_ENTRY) { + + /* allocate and copy in place */ + fytn = fy_token_alloc_rl(fyp->recycled_token_list); + fyp_error_check(fyp, fytn, err_out, + "fy_token_alloc_rl() failed!"); + fytn->type = FYTT_BLOCK_END; + fytn->handle = fyt->handle; + fytn->handle.end_mark = fytn->handle.start_mark; /* no extent */ + fy_input_ref(fytn->handle.fyi); + + fye->sequence_end.sequence_end = fytn; + } else + fye->sequence_end.sequence_end = fy_scan_remove(fyp, fyt); + + ev_handle = &fye->sequence_end.sequence_end->handle; + break; + + case FYPS_BLOCK_MAPPING_FIRST_KEY: + is_first = true; + /* fallthrough */ + + case FYPS_BLOCK_MAPPING_KEY: + + if (!(fyt->type == FYTT_KEY || fyt->type == FYTT_BLOCK_END || fyt->type == FYTT_STREAM_END)) { + + if (fyt->type == FYTT_SCALAR) + FYP_TOKEN_ERROR(fyp, fyt, FYEM_PARSE, + !fyp->simple_key_allowed && !fyp->flow_level && fy_parse_peek(fyp) == ':' ? + "invalid block mapping key on same line as previous key" : + "invalid value after mapping"); + else if (fyt->type == FYTT_BLOCK_SEQUENCE_START) + FYP_TOKEN_ERROR(fyp, fyt, FYEM_PARSE, + "wrong indendation in sequence while in mapping"); + else if (fyt->type == FYTT_ANCHOR) + FYP_TOKEN_ERROR(fyp, fyt, FYEM_PARSE, + "two anchors for a single value while in mapping"); + else if (fyt->type == FYTT_BLOCK_MAPPING_START) + FYP_TOKEN_ERROR(fyp, fyt, FYEM_PARSE, + !fyp->flow_level && fyp->last_block_mapping_key_line == fy_token_start_line(fyt) ? + "invalid nested block mapping on the same line" : + "invalid indentation in mapping"); + else if (fyt->type == FYTT_ALIAS) + FYP_TOKEN_ERROR(fyp, fyt, FYEM_PARSE, + "invalid combination of anchor plus alias"); + else + FYP_TOKEN_ERROR(fyp, fyt, FYEM_PARSE, + "did not find expected key"); + goto err_out; + } + + if (fyt->type == FYTT_KEY) { + + fyp->last_block_mapping_key_line = fy_token_end_line(fyt); + + /* KEY entry */ + fyt = fy_scan_remove_peek(fyp, fyt); + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); +#ifdef FY_DEVMODE + fyp_debug_dump_token(fyp, fyt, "next: "); +#endif + + /* check whether it's a block entry or not */ + is_block = fyt->type != FYTT_KEY && fyt->type != FYTT_VALUE && + fyt->type != FYTT_BLOCK_END; + + if (is_block) { + rc = fy_parse_state_push(fyp, FYPS_BLOCK_MAPPING_VALUE); + fyp_error_check(fyp, !rc, err_out, + "failed to push state"); + + fyep = fy_parse_node(fyp, fyt, true); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_node() failed"); + break; + } + fy_parse_state_set(fyp, FYPS_BLOCK_MAPPING_VALUE); + + fyep = fy_parse_empty_scalar(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_empty_scalar() failed"); + break; + } + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + /* FYTT_BLOCK_END */ + fy_parse_state_set(fyp, fy_parse_state_pop(fyp)); + fye->type = FYET_MAPPING_END; + fye->mapping_end.mapping_end = fy_scan_remove(fyp, fyt); + + ev_handle = &fye->mapping_end.mapping_end->handle; + break; + + case FYPS_BLOCK_MAPPING_VALUE: + + if (fyt->type == FYTT_VALUE) { + + /* VALUE entry */ + fyt = fy_scan_remove_peek(fyp, fyt); + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); +#ifdef FY_DEVMODE + fyp_debug_dump_token(fyp, fyt, "next: "); +#endif + + /* check whether it's a block entry or not */ + is_value = fyt->type != FYTT_KEY && fyt->type != FYTT_VALUE && + fyt->type != FYTT_BLOCK_END; + + if (is_value) { + rc = fy_parse_state_push(fyp, FYPS_BLOCK_MAPPING_KEY); + fyp_error_check(fyp, !rc, err_out, + "failed to push state"); + + fyep = fy_parse_node(fyp, fyt, true); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_node() failed"); + break; + } + } + + fy_parse_state_set(fyp, FYPS_BLOCK_MAPPING_KEY); + + fyep = fy_parse_empty_scalar(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_empty_scalar() failed"); + break; + + case FYPS_FLOW_SEQUENCE_FIRST_ENTRY: + is_first = true; + /* fallthrough */ + + case FYPS_FLOW_SEQUENCE_ENTRY: + + if (fyt->type != FYTT_FLOW_SEQUENCE_END && + fyt->type != FYTT_STREAM_END) { + + if (!is_first) { + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + fyt->type == FYTT_FLOW_ENTRY, err_out, + "missing comma in flow %s", + fyp->state == FYPS_FLOW_SEQUENCE_ENTRY ? + "sequence" : "mapping"); + + fyt = fy_scan_remove_peek(fyp, fyt); + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); +#ifdef FY_DEVMODE + fyp_debug_dump_token(fyp, fyt, "next: "); +#endif + } + + if (fyt->type == FYTT_KEY) { + fy_parse_state_set(fyp, FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_KEY); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + /* convert KEY token to either block or flow mapping start */ + if (!fyt->key.flow_level) + fyt->type = FYTT_BLOCK_MAPPING_START; + else + fyt->type = FYTT_FLOW_MAPPING_START; + + fye->type = FYET_MAPPING_START; + fye->mapping_start.anchor = NULL; + fye->mapping_start.tag = NULL; + fye->mapping_start.mapping_start = fy_scan_remove(fyp, fyt); + + ev_handle = &fye->mapping_start.mapping_start->handle; + break; + } + + if (fyt->type != FYTT_FLOW_SEQUENCE_END) { + rc = fy_parse_state_push(fyp, FYPS_FLOW_SEQUENCE_ENTRY); + fyp_error_check(fyp, !rc, err_out, + "failed to push state"); + + fyep = fy_parse_node(fyp, fyt, false); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_node() failed"); + break; + } + } + + if (fyt->type == FYTT_STREAM_END && fyp->flow_level) { + FYP_TOKEN_ERROR(fyp, fyt, FYEM_PARSE, + "flow sequence without a closing bracket"); + goto err_out; + } + + /* FYTT_FLOW_SEQUENCE_END */ + fy_parse_state_set(fyp, fy_parse_state_pop(fyp)); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_SEQUENCE_END; + fye->sequence_end.sequence_end = fy_scan_remove(fyp, fyt); + + ev_handle = &fye->sequence_end.sequence_end->handle; + break; + + case FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_KEY: + if (fyt->type != FYTT_VALUE && fyt->type != FYTT_FLOW_ENTRY && + fyt->type != FYTT_FLOW_SEQUENCE_END) { + rc = fy_parse_state_push(fyp, FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE); + fyp_error_check(fyp, !rc, err_out, + "failed to push state"); + + fyep = fy_parse_node(fyp, fyt, false); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_node() failed"); + break; + } + + /* empty keys are not allowed in JSON mode */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !fyp_json_mode(fyp), err_out, + "JSON does not allow empty keys of a mapping"); + + fy_parse_state_set(fyp, FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE); + + fyep = fy_parse_empty_scalar(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_empty_scalar() failed"); + break; + + case FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE: + if (fyt->type == FYTT_VALUE) { + fyt = fy_scan_remove_peek(fyp, fyt); + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); +#ifdef FY_DEVMODE + fyp_debug_dump_token(fyp, fyt, "next: "); +#endif + + if (fyt->type != FYTT_FLOW_ENTRY && fyt->type != FYTT_FLOW_SEQUENCE_END) { + rc = fy_parse_state_push(fyp, FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_END); + fyp_error_check(fyp, !rc, err_out, + "failed to push state"); + + fyep = fy_parse_node(fyp, fyt, false); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_node() failed"); + break; + } + } + + /* empty values are not allowed in JSON mode */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !fyp_json_mode(fyp), err_out, + "JSON does not allow empty values in a mapping"); + + fy_parse_state_set(fyp, FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_END); + + fyep = fy_parse_empty_scalar(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_empty_scalar() failed"); + break; + + case FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_END: + fy_parse_state_set(fyp, FYPS_FLOW_SEQUENCE_ENTRY); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_MAPPING_END; + + /* allocate and copy in place */ + fytn = fy_token_alloc_rl(fyp->recycled_token_list); + fyp_error_check(fyp, fytn, err_out, + "fy_token_alloc_rl() failed!"); + fytn->type = FYTT_BLOCK_END; + fytn->handle = fyt->handle; + fytn->handle.end_mark = fytn->handle.start_mark; /* no extent */ + fy_input_ref(fytn->handle.fyi); + + fye->mapping_end.mapping_end = fytn; + + ev_handle = &fye->mapping_end.mapping_end->handle; + break; + + case FYPS_FLOW_MAPPING_FIRST_KEY: + is_first = true; + /* fallthrough */ + + case FYPS_FLOW_MAPPING_KEY: + if (fyt->type != FYTT_FLOW_MAPPING_END) { + + if (!is_first) { + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + fyt->type == FYTT_FLOW_ENTRY, err_out, + "missing comma in flow %s", + fyp->state == FYPS_FLOW_SEQUENCE_ENTRY ? + "sequence" : "mapping"); + + fyt = fy_scan_remove_peek(fyp, fyt); + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); +#ifdef FY_DEVMODE + fyp_debug_dump_token(fyp, fyt, "next: "); +#endif + } + + if (fyt->type == FYTT_KEY) { + /* next token */ + fyt = fy_scan_remove_peek(fyp, fyt); + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); +#ifdef FY_DEVMODE + fyp_debug_dump_token(fyp, fyt, "next: "); +#endif + + /* JSON key checks */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !fyp_json_mode(fyp) || fyt->type != FYTT_VALUE, + err_out, "JSON does not allow empty keys"); + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !fyp_json_mode(fyp) || + (fyt->type == FYTT_SCALAR && + fyt->scalar.style == FYSS_DOUBLE_QUOTED), + err_out, "JSON only allows double quoted scalar keys"); + + if (fyt->type != FYTT_VALUE && + fyt->type != FYTT_FLOW_ENTRY && + fyt->type != FYTT_FLOW_MAPPING_END) { + + rc = fy_parse_state_push(fyp, FYPS_FLOW_MAPPING_VALUE); + fyp_error_check(fyp, !rc, err_out, + "failed to push state"); + + fyep = fy_parse_node(fyp, fyt, false); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_node() failed"); + break; + } + + fy_parse_state_set(fyp, FYPS_FLOW_MAPPING_VALUE); + + fyep = fy_parse_empty_scalar(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_empty_scalar() failed"); + break; + } + + if (fyt->type != FYTT_FLOW_MAPPING_END) { + + /* empty values are not allowed in JSON mode */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !fyp_json_mode(fyp), err_out, + "JSON does not allow empty values in a mapping"); + + rc = fy_parse_state_push(fyp, FYPS_FLOW_MAPPING_EMPTY_VALUE); + fyp_error_check(fyp, !rc, err_out, + "failed to push state"); + + fyep = fy_parse_node(fyp, fyt, false); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_node() failed"); + break; + } + } + + /* FYTT_FLOW_MAPPING_END */ + fy_parse_state_set(fyp, fy_parse_state_pop(fyp)); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_MAPPING_END; + fye->mapping_end.mapping_end = fy_scan_remove(fyp, fyt); + + ev_handle = &fye->mapping_end.mapping_end->handle; + break; + + case FYPS_FLOW_MAPPING_VALUE: + if (fyt->type == FYTT_VALUE) { + /* next token */ + fyt = fy_scan_remove_peek(fyp, fyt); + fyp_error_check(fyp, fyt, err_out, + "failed to peek token"); +#ifdef FY_DEVMODE + fyp_debug_dump_token(fyp, fyt, "next: "); +#endif + + if (fyt->type != FYTT_FLOW_ENTRY && + fyt->type != FYTT_FLOW_MAPPING_END) { + + rc = fy_parse_state_push(fyp, FYPS_FLOW_MAPPING_KEY); + fyp_error_check(fyp, !rc, err_out, + "failed to push state"); + + fyep = fy_parse_node(fyp, fyt, false); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_node() failed"); + break; + } + } + + /* empty values are not allowed in JSON mode */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + !fyp_json_mode(fyp), err_out, + "JSON does not allow empty values in a mapping"); + + fy_parse_state_set(fyp, FYPS_FLOW_MAPPING_KEY); + + fyep = fy_parse_empty_scalar(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_empty_scalar() failed"); + break; + + case FYPS_FLOW_MAPPING_EMPTY_VALUE: + fy_parse_state_set(fyp, FYPS_FLOW_MAPPING_KEY); + + fyep = fy_parse_empty_scalar(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_parse_empty_scalar() failed"); + break; + + case FYPS_SINGLE_DOCUMENT_END: + + FYP_TOKEN_ERROR_CHECK(fyp, fyt, FYEM_PARSE, + fyt->type == FYTT_STREAM_END, err_out, + "Did not find expected stream end"); + + rc = fy_parse_stream_end(fyp); + fyp_error_check(fyp, !rc, err_out, + "stream end failed"); + + fyep = fy_parse_eventp_alloc(fyp); + fyp_error_check(fyp, fyep, err_out, + "fy_eventp_alloc() failed!"); + fye = &fyep->e; + + fye->type = FYET_STREAM_END; + fye->stream_end.stream_end = fy_scan_remove(fyp, fyt); + + fy_parse_state_set(fyp, + fy_parse_have_more_inputs(fyp) ? FYPS_NONE : FYPS_END); + + ev_handle = &fye->stream_end.stream_end->handle; + break; + + case FYPS_END: + FY_IMPOSSIBLE_ABORT(); + } + + assert(fyep); + + if (ev_handle) { + fy_input_unref(fyp->last_event_handle.fyi); + fyp->last_event_handle = *ev_handle; + fyp->last_event_handle.start_mark = fyp->last_event_handle.end_mark; + fy_atom_reset_storage_hints(&handle); + fy_input_ref(fyp->last_event_handle.fyi); + } + + return fyep; + +err_out: + fy_token_unref_rl(fyp->recycled_token_list, version_directive); + fy_token_list_unref_all_rl(fyp->recycled_token_list, &tag_directives); + fy_parse_eventp_recycle(fyp, fyep); + fyp->stream_error = true; + + fy_parser_debug(fyp, "%s: NULL: stream_error=true", __func__); + return NULL; +} + +const char *fy_event_type_txt[] = { + [FYET_NONE] = "NONE", + [FYET_STREAM_START] = "+STR", + [FYET_STREAM_END] = "-STR", + [FYET_DOCUMENT_START] = "+DOC", + [FYET_DOCUMENT_END] = "-DOC", + [FYET_MAPPING_START] = "+MAP", + [FYET_MAPPING_END] = "-MAP", + [FYET_SEQUENCE_START] = "+SEQ", + [FYET_SEQUENCE_END] = "-SEQ", + [FYET_SCALAR] = "=VAL", + [FYET_ALIAS] = "=ALI", +}; + +const char *fy_event_type_get_text(enum fy_event_type type) +{ + if ((unsigned int)type >= ARRAY_SIZE(fy_event_type_txt)) + return "*BAD"; + return fy_event_type_txt[type]; +} + +#ifndef NDEBUG + +static void fy_parse_dump_eventp(struct fy_parser *fyp, struct fy_eventp *fyep, const char *pfx) +{ + struct fy_event *fye = NULL; + FILE *fp; + char *mbuf = NULL; + const char *anchor = NULL; + const char *tag = NULL; + const char *text = NULL; + const char *alias = NULL; + size_t msize, anchor_len = 0, tag_len = 0, text_len = 0, alias_len = 0; + enum fy_scalar_style style; + const uint8_t *p; + int i, c, w; + + /* perform the enable tests early to avoid the overhead */ + if ((FYET_DEBUG >> FYDF_LEVEL_SHIFT) < fyp->diag->cfg.level) + return; + + fp = open_memstream(&mbuf, &msize); + if (!fp) + return; + + if (!fyep) { + fprintf(fp, ""); + goto out; + } + fye = &fyep->e; + + /* event type */ + switch (fye->type) { + case FYET_NONE: + fprintf(fp, "???"); + break; + case FYET_STREAM_START: + fprintf(fp, "+STR"); + break; + case FYET_STREAM_END: + fprintf(fp, "-STR"); + break; + case FYET_DOCUMENT_START: + fprintf(fp, "+DOC"); + break; + case FYET_DOCUMENT_END: + fprintf(fp, "-DOC"); + break; + case FYET_MAPPING_START: + fprintf(fp, "+MAP"); + if (fye->mapping_start.anchor) + anchor = fy_token_get_text(fye->mapping_start.anchor, &anchor_len); + if (fye->mapping_start.tag) + tag = fy_token_get_text(fye->mapping_start.tag, &tag_len); + if (fy_event_get_node_style(fye) == FYNS_FLOW) + fprintf(fp, " {}"); + break; + case FYET_MAPPING_END: + fprintf(fp, "-MAP"); + break; + case FYET_SEQUENCE_START: + fprintf(fp, "+SEQ"); + if (fye->sequence_start.anchor) + anchor = fy_token_get_text(fye->sequence_start.anchor, &anchor_len); + if (fye->sequence_start.tag) + tag = fy_token_get_text(fye->sequence_start.tag, &tag_len); + if (fy_event_get_node_style(fye) == FYNS_FLOW) + fprintf(fp, " []"); + break; + case FYET_SEQUENCE_END: + fprintf(fp, "-SEQ"); + break; + case FYET_SCALAR: + fprintf(fp, "=VAL"); + if (fye->scalar.anchor) + anchor = fy_token_get_text(fye->scalar.anchor, &anchor_len); + if (fye->scalar.tag) + tag = fy_token_get_text(fye->scalar.tag, &tag_len); + break; + case FYET_ALIAS: + fprintf(fp, "=ALI"); + break; + default: + break; + } + + /* (position) anchor and tag */ + if (anchor) + fprintf(fp, " &%.*s", (int)anchor_len, anchor); + if (tag) + fprintf(fp, " <%.*s>", (int)tag_len, tag); + + /* style hint */ + switch (fye->type) { + default: + break; + case FYET_DOCUMENT_START: + if (!fy_document_event_is_implicit(fye)) + fprintf(fp, " ---"); + break; + case FYET_DOCUMENT_END: + if (!fy_document_event_is_implicit(fye)) + fprintf(fp, " ..."); + break; + case FYET_MAPPING_START: + break; + case FYET_SEQUENCE_START: + break; + case FYET_SCALAR: + style = fy_token_scalar_style(fye->scalar.value); + switch (style) { + case FYSS_PLAIN: + fprintf(fp, " :"); + break; + case FYSS_SINGLE_QUOTED: + fprintf(fp, " '"); + break; + case FYSS_DOUBLE_QUOTED: + fprintf(fp, "\""); + break; + case FYSS_LITERAL: + fprintf(fp, " |"); + break; + case FYSS_FOLDED: + fprintf(fp, " >"); + break; + default: + break; + } + break; + case FYET_ALIAS: + break; + } + + /* content */ + switch (fye->type) { + default: + break; + case FYET_SCALAR: + text = fy_token_get_text(fye->scalar.value, &text_len); + for (p = (const uint8_t *)text; text_len > 0; p += w, text_len -= (size_t)w) { + + /* get width from the first octet */ + w = (p[0] & 0x80) == 0x00 ? 1 : + (p[0] & 0xe0) == 0xc0 ? 2 : + (p[0] & 0xf0) == 0xe0 ? 3 : + (p[0] & 0xf8) == 0xf0 ? 4 : 0; + + /* error, clip it */ + if ((size_t)w > text_len) + break; + + /* initial value */ + c = p[0] & (0xff >> w); + for (i = 1; i < w; i++) { + if ((p[i] & 0xc0) != 0x80) + break; + c = (c << 6) | (p[i] & 0x3f); + } + + /* check for validity */ + if ((w == 4 && c < 0x10000) || + (w == 3 && c < 0x800) || + (w == 2 && c < 0x80) || + (c >= 0xd800 && c <= 0xdfff) || c >= 0x110000) + break; + + switch (c) { + case '\\': + fprintf(fp, "\\\\"); + break; + case '\0': + fprintf(fp, "\\0"); + break; + case '\b': + fprintf(fp, "\\b"); + break; + case '\f': + fprintf(fp, "\\f"); + break; + case '\n': + fprintf(fp, "\\n"); + break; + case '\r': + fprintf(fp, "\\r"); + break; + case '\t': + fprintf(fp, "\\t"); + break; + case '\a': + fprintf(fp, "\\a"); + break; + case '\v': + fprintf(fp, "\\v"); + break; + case '\e': + fprintf(fp, "\\e"); + break; + case 0x85: + fprintf(fp, "\\N"); + break; + case 0xa0: + fprintf(fp, "\\_"); + break; + case 0x2028: + fprintf(fp, "\\L"); + break; + case 0x2029: + fprintf(fp, "\\P"); + break; + default: + if ((c >= 0x01 && c <= 0x1f) || c == 0x7f || /* C0 */ + (c >= 0x80 && c <= 0x9f)) /* C1 */ + fprintf(fp, "\\x%02x", c); + else + fprintf(fp, "%.*s", w, p); + break; + } + } + break; + case FYET_ALIAS: + alias = fy_token_get_text(fye->alias.anchor, &alias_len); + fprintf(fp, " %s%.*s", "*", (int)alias_len, alias); + break; + } + +out: + fclose(fp); + fyp_parse_debug(fyp, "%s%s", pfx, mbuf); + free(mbuf); +} + +#else +static inline void +fy_parse_dump_eventp(struct fy_parser *fyp, struct fy_eventp *fyep, const char *pfx) { } +#endif + +struct fy_eventp *fy_parse_private(struct fy_parser *fyp) +{ + struct fy_eventp *fyep = NULL; + + fyep = fy_parse_internal(fyp); + fy_parse_dump_eventp(fyp, fyep, "gen> "); + + return fyep; +} + +struct fy_parser *fy_parser_create(const struct fy_parse_cfg *cfg) +{ + struct fy_parser *fyp; + int rc; + + fyp = malloc(sizeof(*fyp)); + if (!fyp) + return NULL; + + rc = fy_parse_setup(fyp, cfg); + if (rc) { + free(fyp); + return NULL; + } + + return fyp; +} + +void fy_parser_destroy(struct fy_parser *fyp) +{ + if (!fyp) + return; + + fy_parse_cleanup(fyp); + + free(fyp); +} + +const struct fy_parse_cfg *fy_parser_get_cfg(struct fy_parser *fyp) +{ + if (!fyp) + return NULL; + return &fyp->cfg; +} + +struct fy_diag *fy_parser_get_diag(struct fy_parser *fyp) +{ + if (!fyp || !fyp->diag) + return NULL; + return fy_diag_ref(fyp->diag); +} + +int fy_parser_set_diag(struct fy_parser *fyp, struct fy_diag *diag) +{ + struct fy_diag_cfg dcfg; + + if (!fyp) + return -1; + + /* default? */ + if (!diag) { + fy_diag_cfg_default(&dcfg); + diag = fy_diag_create(&dcfg); + if (!diag) + return -1; + } + + fy_diag_unref(fyp->diag); + fyp->diag = fy_diag_ref(diag); + + return 0; +} + +static void fy_parse_input_reset(struct fy_parser *fyp) +{ + struct fy_input *fyi, *fyin; + + for (fyi = fy_input_list_head(&fyp->queued_inputs); fyi; fyi = fyin) { + fyin = fy_input_next(&fyp->queued_inputs, fyi); + fy_input_unref(fyi); + } + fy_input_list_init(&fyp->queued_inputs); + + fy_parse_parse_state_log_list_recycle_all(fyp, &fyp->state_stack); + + fyp->stream_start_produced = false; + fyp->stream_end_produced = false; + fyp->stream_end_reached = false; + fyp->state = FYPS_NONE; + + fyp->pending_complex_key_column = -1; + fyp->last_block_mapping_key_line = -1; + + fy_input_unref(fyp->last_event_handle.fyi); + fy_atom_reset(&fyp->last_event_handle); +} + +int fy_parser_set_input_file(struct fy_parser *fyp, const char *file) +{ + struct fy_input_cfg fyic; + int rc; + + if (!fyp || !file) + return -1; + + memset(&fyic, 0, sizeof(fyic)); + + if (!strcmp(file, "-")) { + fyic.type = fyit_stream; + fyic.stream.name = "stdin"; + fyic.stream.fp = stdin; + } else { + fyic.type = fyit_file; + fyic.file.filename = file; + } + fyic.ignore_stdio = !!(fyp->cfg.flags & FYPCF_DISABLE_BUFFERING); + + /* must not be in the middle of something */ + fyp_error_check(fyp, fyp->state == FYPS_NONE || fyp->state == FYPS_END, + err_out, "parser cannot be reset at state '%s'", + state_txt[fyp->state]); + + fy_parse_input_reset(fyp); + + rc = fy_parse_input_append(fyp, &fyic); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_input_append() failed"); + + return 0; +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_parser_set_string(struct fy_parser *fyp, const char *str, size_t len) +{ + struct fy_input_cfg fyic; + int rc; + + if (!fyp || !str) + return -1; + + if (len == (size_t)-1) + len = strlen(str); + + memset(&fyic, 0, sizeof(fyic)); + + fyic.type = fyit_memory; + fyic.memory.data = str; + fyic.memory.size = len; + + /* must not be in the middle of something */ + fyp_error_check(fyp, fyp->state == FYPS_NONE || fyp->state == FYPS_END, + err_out, "parser cannot be reset at state '%s'", + state_txt[fyp->state]); + + fy_parse_input_reset(fyp); + + rc = fy_parse_input_append(fyp, &fyic); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_input_append() failed"); + + return 0; +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_parser_set_malloc_string(struct fy_parser *fyp, char *str, size_t len) +{ + struct fy_input_cfg fyic; + int rc; + + if (!fyp || !str) + return -1; + + if (len == (size_t)-1) + len = strlen(str); + + memset(&fyic, 0, sizeof(fyic)); + + fyic.type = fyit_alloc; + fyic.alloc.data = str; + fyic.alloc.size = len; + + /* must not be in the middle of something */ + fyp_error_check(fyp, fyp->state == FYPS_NONE || fyp->state == FYPS_END, + err_out, "parser cannot be reset at state '%s'", + state_txt[fyp->state]); + + fy_parse_input_reset(fyp); + + rc = fy_parse_input_append(fyp, &fyic); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_input_append() failed"); + + return 0; +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_parser_set_input_fp(struct fy_parser *fyp, const char *name, FILE *fp) +{ + struct fy_input_cfg fyic; + int rc; + + if (!fyp || !fp) + return -1; + + memset(&fyic, 0, sizeof(fyic)); + + fyic.type = fyit_stream; + fyic.stream.name = name ? : ""; + fyic.stream.fp = fp; + fyic.ignore_stdio = !!(fyp->cfg.flags & FYPCF_DISABLE_BUFFERING); + + /* must not be in the middle of something */ + fyp_error_check(fyp, fyp->state == FYPS_NONE || fyp->state == FYPS_END, + err_out, "parser cannot be reset at state '%s'", + state_txt[fyp->state]); + + fy_parse_input_reset(fyp); + + rc = fy_parse_input_append(fyp, &fyic); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_input_append() failed"); + + return 0; +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_parser_set_input_callback(struct fy_parser *fyp, void *user, + ssize_t (*callback)(void *user, void *buf, size_t count)) +{ + struct fy_input_cfg fyic; + int rc; + + if (!fyp || !callback) + return -1; + + memset(&fyic, 0, sizeof(fyic)); + + fyic.type = fyit_callback; + fyic.userdata = user; + fyic.callback.input = callback; + fyic.ignore_stdio = !!(fyp->cfg.flags & FYPCF_DISABLE_BUFFERING); + + /* must not be in the middle of something */ + fyp_error_check(fyp, fyp->state == FYPS_NONE || fyp->state == FYPS_END, + err_out, "parser cannot be reset at state '%s'", + state_txt[fyp->state]); + + fy_parse_input_reset(fyp); + + rc = fy_parse_input_append(fyp, &fyic); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_input_append() failed"); + + return 0; +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_parser_set_input_fd(struct fy_parser *fyp, int fd) +{ + struct fy_input_cfg fyic; + int rc; + + if (!fyp || fd < 0) + return -1; + + memset(&fyic, 0, sizeof(fyic)); + + fyic.type = fyit_fd; + fyic.fd.fd = fd; + fyic.ignore_stdio = !!(fyp->cfg.flags & FYPCF_DISABLE_BUFFERING); + + /* must not be in the middle of something */ + fyp_error_check(fyp, fyp->state == FYPS_NONE || fyp->state == FYPS_END, + err_out, "parser cannot be reset at state '%s'", + state_txt[fyp->state]); + + fy_parse_input_reset(fyp); + + rc = fy_parse_input_append(fyp, &fyic); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_input_append() failed"); + + return 0; +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_parser_set_document_iterator(struct fy_parser *fyp, enum fy_parser_event_generator_flags flags, + struct fy_document_iterator *fydi) +{ + struct fy_input_cfg fyic; + int rc; + + if (!fyp || !fydi) + return -1; + + memset(&fyic, 0, sizeof(fyic)); + + fyic.type = fyit_dociter; + fyic.dociter.flags = flags; + fyic.dociter.fydi = fydi; + fyic.dociter.owns_iterator = false; + + /* must not be in the middle of something */ + fyp_error_check(fyp, fyp->state == FYPS_NONE || fyp->state == FYPS_END, + err_out, "parser cannot be reset at state '%s'", + state_txt[fyp->state]); + + fy_parse_input_reset(fyp); + + rc = fy_parse_input_append(fyp, &fyic); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_input_append() failed"); + + return 0; +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_parser_reset(struct fy_parser *fyp) +{ + if (!fyp) + return -1; + + fy_parse_input_reset(fyp); + + fy_reader_reset(fyp->reader); + + fyp->next_single_document = false; + fyp->stream_error = false; + fyp->generated_block_map = false; + fyp->last_was_comma = false; + fyp->document_has_content = false; + fyp->document_first_content_token = false; + fyp->bare_document_only = false; + fyp->stream_has_content = false; + fyp->had_directives = false; + + assert(fyp->diag); + fyp->diag->on_error = false; + + fy_document_state_unref(fyp->current_document_state); + fyp->current_document_state = NULL; + + fy_document_state_unref(fyp->default_document_state); + fyp->default_document_state = NULL; + + fy_parse_streaming_aliases_reset(fyp); + + fy_input_unref(fyp->last_event_handle.fyi); + fy_atom_reset(&fyp->last_event_handle); + + fy_parse_indent_list_recycle_all(fyp, &fyp->indent_stack); + fy_parse_simple_key_list_recycle_all(fyp, &fyp->simple_keys); + fy_token_list_unref_all(&fyp->queued_tokens); + + fy_parse_parse_state_log_list_recycle_all(fyp, &fyp->state_stack); + fy_parse_flow_list_recycle_all(fyp, &fyp->flow_stack); + fy_parse_streaming_alias_list_recycle_all(fyp, &fyp->streaming_aliases); + + fy_token_unref_rl(fyp->recycled_token_list, fyp->stream_end_token); + fyp->stream_end_token = NULL; + + return 0; +} + +struct fy_event *fy_parser_parse(struct fy_parser *fyp) +{ + struct fy_eventp *fyep, *fyep2; + enum fy_composer_return ret; + + if (!fyp) + return NULL; + + if (fyp->fyep_peek) { + fyep = fyp->fyep_peek; + fyp->fyep_peek = NULL; + fy_parse_dump_eventp(fyp, fyep, "peek> "); + return &fyep->e; + } + + if ((fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT) && fy_reader_get_mode(fyp->reader) != fyrm_json) { + fyep = fy_parser_parse_resolve_prolog(fyp); + if (fyep) + return &fyep->e; + } + + fyep = fy_parse_private(fyp); + if (!fyep) + return NULL; + + if ((fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT) && fy_reader_get_mode(fyp->reader) != fyrm_json) { + fyep2 = fy_parser_parse_resolve_epilog(fyp, fyep); + if (!fyep2) + return NULL; + fyep = fyep2; + } + + if (fyp->fyc) { + ret = fy_composer_process_event(fyp->fyc, &fyep->e); + if (ret == FYCR_ERROR) { + fyp->stream_error = true; + fy_parse_eventp_recycle(fyp, fyep); + return NULL; + } + /* note that the stop should be handled by + * an out of band mechanism */ + } + + return &fyep->e; +} + +const struct fy_event *fy_parser_parse_peek(struct fy_parser *fyp) +{ + struct fy_event *fye; + + if (!fyp->fyep_peek) { + fye = fy_parser_parse(fyp); + if (!fye) + return NULL; + + fyp->fyep_peek = container_of(fye, struct fy_eventp, e); + } + return &fyp->fyep_peek->e; +} + +int fy_parser_skip(struct fy_parser *fyp) +{ + struct fy_event *fye = 0; + enum fy_event_type start_type, end_type; + int nest, map_nest, seq_nest; + + /* check that we start on a scalar/seq/map */ + fye = fy_parser_parse(fyp); + if (!fye) + goto err_out; + + start_type = fye->type; + fy_parser_event_free(fyp, fye); + fye = NULL; + + if (start_type == FYET_SCALAR) + return 0; + + map_nest = 0; + seq_nest = 0; + if (start_type == FYET_SEQUENCE_START) { + end_type = FYET_SEQUENCE_END; + seq_nest = 1; + } else if (start_type == FYET_MAPPING_START) { + end_type = FYET_MAPPING_END; + map_nest = 1; + } else + goto err_out; + + nest = 1; + do { + fye = fy_parser_parse(fyp); + if (!fye) + goto err_out; + + switch (fye->type) { + case FYET_SEQUENCE_START: + seq_nest++; + break; + case FYET_MAPPING_START: + map_nest++; + break; + case FYET_SEQUENCE_END: + seq_nest--; + break; + case FYET_MAPPING_END: + map_nest--; + break; + default: + break; + } + + if (fye->type == start_type) + nest++; + else if (fye->type == end_type) + nest--; + + if (map_nest < 0 || seq_nest < 0) { + fy_event_report(fyp, fye, FYEP_VALUE, FYET_ERROR, + "unbalanced collection while skipping"); + goto err_out; + } + + fy_parser_event_free(fyp, fye); + fye = NULL; + + } while (nest > 0); + + return 0; + +err_out: + fy_parser_event_free(fyp, fye); + return -1; +} + +bool fy_parser_get_stream_error(struct fy_parser *fyp) +{ + if (!fyp) + return true; + + return fyp->stream_error; +} + +enum fy_parse_cfg_flags fy_parser_get_cfg_flags(const struct fy_parser *fyp) +{ + if (!fyp) + return 0; + + return fyp->cfg.flags; +} + +struct fy_document_state *fy_parser_get_document_state(struct fy_parser *fyp) +{ + return fyp ? fyp->current_document_state : NULL; +} + +static enum fy_composer_return +parse_process_event(struct fy_composer *fyc, struct fy_path *path, struct fy_event *fye) +{ + struct fy_parser *fyp = fy_composer_get_cfg_userdata(fyc); + + assert(fyp); + assert(fyp->fyc_cb); + return fyp->fyc_cb(fyp, fye, path, fyp->fyc_userdata); +} + +static struct fy_document_builder * +parse_create_document_builder(struct fy_composer *fyc) +{ + struct fy_parser *fyp = fy_composer_get_cfg_userdata(fyc); + struct fy_document_builder *fydb = NULL; + struct fy_document_builder_cfg cfg; + struct fy_document_state *fyds; + int rc; + + memset(&cfg, 0, sizeof(cfg)); + cfg.parse_cfg = fyp->cfg; + cfg.diag = fy_diag_ref(fyp->diag); + + fydb = fy_document_builder_create(&cfg); + fyp_error_check(fyp, fydb, err_out, + "fy_document_builder_create() failed\n"); + + /* start with this document state */ + fyds = fy_parser_get_document_state(fyp); + rc = fy_document_builder_set_in_document(fydb, fyds, true); + fyp_error_check(fyp, !rc, err_out, + "fy_document_builder_set_in_document() failed\n"); + + return fydb; + +err_out: + fy_document_builder_destroy(fydb); + return NULL; +} + +static const struct fy_composer_ops parser_composer_ops = { + .process_event = parse_process_event, + .create_document_builder = parse_create_document_builder, +}; + +int fy_parse_set_composer(struct fy_parser *fyp, fy_parse_composer_cb cb, void *userdata) +{ + struct fy_composer_cfg ccfg; + bool can_change_composer; + + if (!fyp) + return -1; + + can_change_composer = fyp->state == FYPS_NONE || + fyp->state == FYPS_END || + fyp->state == FYPS_DOCUMENT_START; + + /* must not be in the middle of something */ + fyp_error_check(fyp, can_change_composer, err_out, + "cannot change composer state at state '%s'", state_txt[fyp->state]); + + /* clear */ + if (!cb) { + if (fyp->fyc) { + fy_composer_destroy(fyp->fyc); + fyp->fyc = NULL; + } + fyp->fyc_cb = NULL; + fyp->fyc_userdata = NULL; + return 0; + } + + /* already exists */ + if (fyp->fyc) { + fyp->fyc_cb = cb; + fyp->fyc_userdata = userdata; + return 0; + } + + /* prepare the composer configuration */ + memset(&ccfg, 0, sizeof(ccfg)); + ccfg.ops = &parser_composer_ops; + ccfg.userdata = fyp; + ccfg.diag = fy_parser_get_diag(fyp); + fyp->fyc = fy_composer_create(&ccfg); + fyp_error_check(fyp, fyp->fyc, err_out, + "fy_composer_create() failed"); + + fyp->fyc_cb = cb; + fyp->fyc_userdata = userdata; + + return 0; +err_out: + return -1; +} + +static enum fy_composer_return fy_parse_compose_internal(struct fy_parser *fyp) +{ + struct fy_composer *fyc; + struct fy_document_iterator *fydi; + struct fy_event *fye; + struct fy_eventp *fyep; + struct fy_document *fyd = NULL; + enum fy_composer_return ret; + + assert(fyp); + + fyc = fyp->fyc; + assert(fyc); + + /* simple, without resolution */ + if (!(fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT)) { + + ret = FYCR_OK_STOP; + while ((fyep = fy_parse_private(fyp)) != NULL) { + ret = fy_composer_process_event(fyc, &fyep->e); + fy_parse_eventp_recycle(fyp, fyep); + if (ret != FYCR_OK_CONTINUE) + break; + } + if (fyp->stream_error) + fy_composer_halt(fyc, FYCR_ERROR); + + return ret; + } + + fydi = fy_document_iterator_create(); + fyp_error_check(fyp, fydi, err_out, + "fy_document_iterator_create() failed"); + + /* stream start event generation and processing */ + fye = fy_document_iterator_stream_start(fydi); + fyp_error_check(fyp, fye, err_out, + "fy_document_iterator_stream_start() failed"); + ret = fy_composer_process_event(fyc, fye); + fy_document_iterator_event_free(fydi, fye); + fye = NULL; + if (ret != FYCR_OK_CONTINUE) + goto out; + + /* convert to document and then process the generator event stream it */ + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + + /* document start event generation and processing */ + fye = fy_document_iterator_document_start(fydi, fyd); + fyp_error_check(fyp, fye, err_out, + "fy_document_iterator_document_start() failed"); + ret = fy_composer_process_event(fyc, fye); + fy_document_iterator_event_free(fydi, fye); + fye = NULL; + if (ret != FYCR_OK_CONTINUE) + goto out; + + /* and now process the body */ + ret = FYCR_OK_CONTINUE; + while ((fye = fy_document_iterator_body_next(fydi)) != NULL) { + ret = fy_composer_process_event(fyc, fye); + fy_document_iterator_event_free(fydi, fye); + fye = NULL; + if (ret != FYCR_OK_CONTINUE) + goto out; + } + + /* document end event generation and processing */ + fye = fy_document_iterator_document_end(fydi); + fyp_error_check(fyp, fye, err_out, + "fy_document_iterator_document_end() failed"); + ret = fy_composer_process_event(fyc, fye); + fy_document_iterator_event_free(fydi, fye); + fye = NULL; + if (ret != FYCR_OK_CONTINUE) + goto out; + + /* and destroy the document */ + fy_parse_document_destroy(fyp, fyd); + fyd = NULL; + } + + /* stream end event generation and processing */ + fye = fy_document_iterator_stream_end(fydi); + fyp_error_check(fyp, fye, err_out, + "fy_document_iterator_stream_end() failed"); + ret = fy_composer_process_event(fyc, fye); + fy_document_iterator_event_free(fydi, fye); + fye = NULL; + if (ret != FYCR_OK_CONTINUE) + goto out; + +out: + /* NULLs are OK */ + fy_parse_document_destroy(fyp, fyd); + fy_document_iterator_destroy(fydi); + return ret; + +err_out: + ret = FYCR_ERROR; + goto out; +} + +int fy_parse_compose(struct fy_parser *fyp, fy_parse_composer_cb cb, void *userdata) +{ + enum fy_composer_return ret; + int rc, rc_out; + + if (!fyp || !cb) + return -1; + + /* set the composer callback */ + rc = fy_parse_set_composer(fyp, cb, userdata); + fyp_error_check(fyp, !rc, err_out, + "fy_parse_set_composer() failed\n"); + + /* use the composer to parse */ + ret = fy_parse_compose_internal(fyp); + /* on error set the stream error */ + if (ret == FYCR_ERROR) { + fyp->stream_error = true; + rc_out = -1; + } else + rc_out = 0; + + /* reset the parser; the composer clear must always succeed */ + if (ret == FYCR_ERROR) + fy_parser_reset(fyp); + + /* clear composer */ + rc = fy_parse_set_composer(fyp, NULL, NULL); + fyp_error_check(fyp, !rc, err_out, + "fy_parse_set_composer() failed\n"); + + return rc_out; + +err_out: + return -1; +} + +struct fy_streaming_alias * +fy_parser_streaming_alias_lookup(struct fy_parser *fyp, struct fy_token *fyt_anchor) +{ + struct fy_streaming_alias *fysa; + + /* XXX todo hashing */ + for (fysa = fy_streaming_alias_list_head(&fyp->streaming_aliases); fysa != NULL; fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa)) { + if (fy_token_cmp(fyt_anchor, fysa->anchor) == 0) + return fysa; + } + return NULL; +} + +struct fy_streaming_alias * +fy_parser_streaming_alias_lookup_pivot(struct fy_parser *fyp, struct fy_streaming_alias *fysa_pivot, struct fy_token *fyt_anchor) +{ + struct fy_streaming_alias *fysa; + + assert(fysa_pivot); + /* at first do forwards starting at the pivot */ + for (fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa_pivot); fysa != NULL; fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa)) { + if (fysa->anchor && fy_token_cmp(fyt_anchor, fysa->anchor) == 0) + return fysa; + } + /* not found? do backwards (more recent) */ + for (fysa = fy_streaming_alias_prev(&fyp->streaming_aliases, fysa_pivot); fysa != NULL; fysa = fy_streaming_alias_prev(&fyp->streaming_aliases, fysa)) { + if (fysa->anchor && fy_token_cmp(fyt_anchor, fysa->anchor) == 0) + return fysa; + } + return NULL; +} + +struct fy_streaming_alias * +fy_parse_streaming_alias_create(struct fy_parser *fyp, struct fy_token *fyt_anchor) +{ + struct fy_streaming_alias *fysa; + + fysa = fy_parse_streaming_alias_alloc(fyp); + fyp_error_check(fyp, fysa != NULL, err_out, + "fy_parse_streaming_alias_alloc() failed!"); + fysa->anchor = fyt_anchor; + fysa->collecting = false; + fysa->mapping_nest = 0; + fysa->sequence_nest = 0; + fy_eventp_list_init(&fysa->events); + + return fysa; + +err_out: + return NULL; +} + +void +fy_parse_streaming_alias_clean(struct fy_parser *fyp, struct fy_streaming_alias *fysa) +{ + struct fy_eventp *fyep; + + if (!fyp || !fysa) + return; + + fy_token_unref(fysa->anchor); + fysa->anchor = NULL; + fysa->collecting = false; + fysa->mapping_nest = 0; + fysa->sequence_nest = 0; + while ((fyep = fy_eventp_list_pop(&fysa->events)) != NULL) + fy_parse_eventp_recycle(fyp, fyep); +} + +void fy_parse_streaming_aliases_reset(struct fy_parser *fyp) +{ + struct fy_streaming_alias *fysa; + struct fy_eventp *fyep; + + if (!fyp) + return; + + while ((fysa = fy_streaming_alias_list_pop(&fyp->streaming_aliases)) != NULL) { + fy_parse_streaming_alias_clean(fyp, fysa); + fy_parse_streaming_alias_recycle(fyp, fysa); + } + if (fyp->sas.stack != NULL && fyp->sas.stack != fyp->sas.local) + free(fyp->sas.stack); + memset(&fyp->sas, 0, sizeof(fyp->sas)); + if (fyp->cts.stack != NULL && fyp->cts.stack != fyp->cts.local) + free(fyp->cts.stack); + memset(&fyp->cts, 0, sizeof(fyp->cts)); + + while ((fyep = fy_eventp_list_pop(&fyp->mks.args)) != NULL) + fy_parse_eventp_recycle(fyp, fyep); + memset(&fyp->mks, 0, sizeof(fyp->mks)); + fy_eventp_list_init(&fyp->mks.args); +} + +struct fy_streaming_alias_state * +fy_parse_streaming_alias_state_push(struct fy_parser *fyp, struct fy_streaming_alias *fysa) +{ + struct fy_streaming_alias_state *new_stack; + struct fy_streaming_alias_state *fysas; + + /* if first one, just point to the local area */ + if (fyp->sas.stack == NULL) { + fyp->sas.stack = fyp->sas.local; + fyp->sas.top = 0; + fyp->sas.alloc = ARRAY_SIZE(fyp->sas.local); + } + /* grow the stack if required */ + if (fyp->sas.top >= fyp->sas.alloc) { + assert(fyp->sas.alloc > 0); + assert(fyp->sas.stack); + new_stack = malloc(sizeof(*new_stack) * fyp->sas.alloc * 2); + fyp_error_check(fyp, new_stack != NULL, err_out, + "realloc() failed!"); + memcpy(new_stack, fyp->sas.stack, sizeof(*new_stack) * fyp->sas.alloc); + fyp->sas.alloc *= 2; + if (fyp->sas.stack != fyp->sas.local) + free(fyp->sas.stack); + fyp->sas.stack = new_stack; + } + /* finally push it */ + fysas = &fyp->sas.stack[fyp->sas.top++]; + + fysas->fysa = fysa; + fysas->next = fy_eventp_list_head(&fysa->events); + + return fysas; +err_out: + return NULL; +} + +struct fy_streaming_alias_state * +fy_parse_streaming_alias_state_pop(struct fy_parser *fyp) +{ + struct fy_streaming_alias_state *fysas; + + if (fyp->sas.top <= 0) + return NULL; + + fysas = &fyp->sas.stack[--fyp->sas.top]; + fysas->fysa = NULL; + fysas->next = NULL; + if (fyp->sas.top <= 0) + return NULL; + return &fyp->sas.stack[fyp->sas.top - 1]; +} + +struct fy_streaming_alias_state * +fy_parse_streaming_alias_state_top(struct fy_parser *fyp) +{ + if (fyp->sas.top <= 0) + return NULL; + return &fyp->sas.stack[fyp->sas.top - 1]; +} + +void +fy_parse_streaming_alias_state_next_event(struct fy_parser *fyp) +{ + struct fy_streaming_alias_state *fysas; + + fysas = fy_parse_streaming_alias_state_top(fyp); + if (!fysas) + return; + + fysas->next = fy_eventp_next(&fysas->fysa->events, fysas->next); + while (!fysas->next) { + fysas = fy_parse_streaming_alias_state_pop(fyp); + if (!fysas) + break; + } +} + +static inline int +fy_parse_streaming_alias_collection_state_set_top(struct fy_parser *fyp, int cts) +{ + int pos, off, old_cts; + + if (fyp->cts.top < 2) + return -1; + + cts &= 3; /* keep 2 bits */ + + /* get position and offset */ + pos = (fyp->cts.top - 2) / (sizeof(*fyp->cts.stack) * 8); + off = (fyp->cts.top - 2) % (sizeof(*fyp->cts.stack) * 8); + + old_cts = (int)((fyp->cts.stack[pos] >> off) & 3); + + /* clear the bits first */ + fyp->cts.stack[pos] &= ~(3 << off); + /* set the state */ + fyp->cts.stack[pos] |= (cts & 3) << off; + + return old_cts; +} + +static inline int +fy_parse_streaming_alias_collection_state_top(struct fy_parser *fyp) +{ + int pos, off; + + if (fyp->cts.top < 2) + return -1; + + /* get position and offset */ + pos = (fyp->cts.top - 2) / (sizeof(*fyp->cts.stack) * 8); + off = (fyp->cts.top - 2) % (sizeof(*fyp->cts.stack) * 8); + + return (int)((fyp->cts.stack[pos] >> off) & 3); +} + +static inline bool +fy_parse_streaming_alias_collection_state_in_map_key(struct fy_parser *fyp) +{ + int cts = fy_parse_streaming_alias_collection_state_top(fyp); + return cts == FYCTS_MAP_KEY; +} + +static inline void +fy_parse_streaming_alias_collection_state_toggle_map(struct fy_parser *fyp) +{ + int cts = fy_parse_streaming_alias_collection_state_top(fyp); + if (cts >= 0 && (cts & FYCTS_MAP)) + fy_parse_streaming_alias_collection_state_set_top(fyp, cts ^ FYCTS_MAP_TOGGLE); +} + +int +fy_parse_streaming_alias_collection_state_push(struct fy_parser *fyp, int cts) +{ + uint32_t *new_stack; + int size; + + assert(cts >= 0); + + /* if first one, just point to the local area */ + if (fyp->cts.stack == NULL) { + fyp->cts.stack = fyp->cts.local; + fyp->cts.top = 0; + fyp->cts.alloc = sizeof(fyp->cts.local) * 8; /* in bits */ + } + /* grow the stack if required */ + if (fyp->cts.top + 2 >= fyp->cts.alloc) { + assert(fyp->cts.alloc > 0); + assert(fyp->cts.stack); + size = fyp->cts.alloc / 8; /* convert to bytes */ + new_stack = malloc(size * 2); + fyp_error_check(fyp, new_stack != NULL, err_out, + "realloc() failed!"); + memcpy(new_stack, fyp->cts.stack, size); + memset((uint8_t *)new_stack + size, 0, size); + fyp->cts.alloc *= 2; + if (fyp->cts.stack != fyp->cts.local) + free(fyp->cts.stack); + fyp->cts.stack = new_stack; + } + + fyp->cts.top += 2; + fy_parse_streaming_alias_collection_state_set_top(fyp, cts); + return cts; +err_out: + return -1; +} + +int +fy_parse_streaming_alias_collection_state_pop(struct fy_parser *fyp) +{ + int ret; + + ret = fy_parse_streaming_alias_collection_state_top(fyp); + if (ret >= 0) + fyp->cts.top -= 2; + return ret; +} + +static int +fy_parser_event_resolve_hook_collect(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_eventp *fyep_clone = NULL; + struct fy_streaming_alias *fysa; + long map_add, seq_add; + + if (!fyp || !fyep) + return -1; + + map_add = seq_add = 0; + switch (fyep->e.type) { + case FYET_MAPPING_START: + map_add = 1; + break; + case FYET_MAPPING_END: + map_add = -1; + break; + case FYET_SEQUENCE_START: + seq_add = 1; + break; + case FYET_SEQUENCE_END: + seq_add = -1; + break; + default: + break; + } + + /* for all streaming aliases that are collecting... */ + for (fysa = fy_streaming_alias_list_head(&fyp->streaming_aliases); fysa != NULL; fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa)) { + if (!fysa->collecting) + continue; + + /* clone event, stripping the anchors */ + fyep_clone = fy_parse_eventp_clone(fyp, fyep, true); + fyp_error_check(fyp, fyep_clone != NULL, err_out, + "fy_parse_eventp_clone() failed!"); + + /* add to the list */ + fy_eventp_list_add_tail(&fysa->events, fyep_clone); + fyep_clone = NULL; + + /* the scalar will be collected every time */ + fysa->mapping_nest += map_add; + fysa->sequence_nest += seq_add; + + if (fysa->mapping_nest == 0 && fysa->sequence_nest == 0) + fysa->collecting = false; + } + + return 0; + +err_out: + fy_parse_eventp_recycle(fyp, fyep_clone); + fyp->stream_error = true; + return -1; +} + +struct fy_eventp *fy_parser_event_resolve_hook_alias(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_token *fyt_anchor = NULL; + struct fy_streaming_alias *fysa; + struct fy_streaming_alias_state *fysas; + + if (!fyp || !fyep) + return NULL; + + /* if it's not an alias, continue normal processing */ + if (fyep->e.type != FYET_ALIAS) + return fyep; + + /* it's an alias, get the token and dispose the event */ + fyt_anchor = fyep->e.alias.anchor; + fyep->e.alias.anchor = NULL; + fy_parse_eventp_recycle(fyp, fyep); + fyep = NULL; + + /* start generation of alias events */ + fysa = fy_parser_streaming_alias_lookup(fyp, fyt_anchor); + + /* we don't support forward aliases for now like we do in document mode ... */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt_anchor, FYEM_PARSE, + fysa, err_out, + "alias not found; note streaming mode does not support forward alias references"); + + /* if this is a reference to a collecting alias, then it's recursive */ + FYP_TOKEN_ERROR_CHECK(fyp, fyt_anchor, FYEM_PARSE, + !fysa->collecting, err_out, + "Recursive alias reference detected"); + + fy_token_unref(fyt_anchor); + fyt_anchor = NULL; + + fysas = fy_parse_streaming_alias_state_push(fyp, fysa); + fyp_error_check(fyp, fysas, err_out, + "fy_parse_streaming_alias_push() failed!"); + assert(fysas->next); + + fyep = fy_parse_eventp_clone(fyp, fysas->next, true); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parse_eventp_clone() failed!"); + + /* advance event */ + fy_parse_streaming_alias_state_next_event(fyp); + + return fyep; + +err_out: + fy_token_unref(fyt_anchor); + fyp->stream_error = true; + return NULL; +} + +struct fy_eventp *fy_parser_event_resolve_hook_anchor_start(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_token *fyt_anchor = NULL; + struct fy_streaming_alias *fysa = NULL; + + if (!fyep) + return NULL; + + /* is there an anchor?, is so get it and clear it */ + fyt_anchor = fy_event_get_and_clear_anchor_token(&fyep->e); + if (!fyt_anchor) + return fyep; + + fysa = fy_parse_streaming_alias_create(fyp, fyt_anchor); + fyp_error_check(fyp, fysa != NULL, err_out, + "fy_parser_streaming_alias_create() failed!"); + fyt_anchor = NULL; + + /* we just mark that we're collecting + * the collection hook will pick up the first event + * including the scalar one + */ + fysa->collecting = true; + fysa->sequence_nest = 0; + fysa->mapping_nest = 0; + + /* always add to the head of the list (overrides what follows with the same name) */ + fy_streaming_alias_list_add(&fyp->streaming_aliases, fysa); + + return fyep; + +err_out: + fy_token_unref(fyt_anchor); + fy_parse_streaming_alias_clean(fyp, fysa); + fy_parse_streaming_alias_recycle(fyp, fysa); + fy_parse_eventp_recycle(fyp, fyep); + return NULL; +} + +static struct fy_document * +fy_parser_get_merge_key_argument(struct fy_parser *fyp, struct fy_document_builder *fydb, struct fy_node *fyn) +{ + struct fy_streaming_alias *fysa; + struct fy_document *fyd_new = NULL; + struct fy_document_state *fyds = NULL; + int rc; + + if (!fy_node_is_alias(fyn)) + return NULL; + + /* get the alias */ + fysa = fy_parser_streaming_alias_lookup(fyp, fyn->scalar); + FYP_TOKEN_ERROR_CHECK(fyp, fyn->scalar, FYEM_PARSE, + fysa, err_out, + "Unable to find merge key argument alias"); + + /* if this is a reference to a collecting alias, then it's recursive */ + FYP_TOKEN_ERROR_CHECK(fyp, fyn->scalar, FYEM_PARSE, + !fysa->collecting, err_out, + "merge key recursive alias reference detected"); + + fyds = fy_parser_get_document_state(fyp); + fyp_error_check(fyp, fyds, err_out, + "fy_parser_get_document_state() failed"); + + rc = fy_document_builder_set_in_document(fydb, fyds, true); + fyp_error_check(fyp, !rc, err_out, + "fy_document_builder_set_target_document() failed\n"); + + /* build a document from the events of the alias (must be the same) */ + fyd_new = fy_document_builder_event_document(fydb, &fysa->events); + fyp_error_check(fyp, fyd_new, err_out, + "fy_document_builder_event_document() failed\n"); + + FYP_TOKEN_ERROR_CHECK(fyp, fyn->scalar, FYEM_PARSE, + fy_node_is_mapping(fyd_new->root), err_out, + "alias argument does not refer to mapping"); + + return fyd_new; + +err_out: + fy_document_destroy(fyd_new); + return NULL; +} + +static struct fy_document * +fy_parser_get_merge_key_document(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_event *fye; + struct fy_document_builder_cfg cfg; + struct fy_document_builder *fydb = NULL; + struct fy_document *fyd = NULL, *fyd_arg = NULL; + struct fy_document_state *fyds; + struct fy_node *fyn = NULL, *fyn_src = NULL; + struct fy_node *new_root = NULL; + struct fy_node_pair *fynp, *fynp_exists; + void *iters, *iterm; + int rc; + + fye = &fyep->e; + + memset(&cfg, 0, sizeof(cfg)); + cfg.parse_cfg = fyp->cfg; + cfg.diag = fy_diag_ref(fyp->diag); + + fydb = fy_document_builder_create(&cfg); + fyp_error_check(fyp, fydb, err_out, + "fy_document_builder_create() failed\n"); + + fyds = fy_parser_get_document_state(fyp); + /* start with this document state */ + fyds = fy_parser_get_document_state(fyp); + rc = fy_document_builder_set_in_document(fydb, fyds, true); + fyp_error_check(fyp, !rc, err_out, + "fy_document_builder_set_in_document() failed\n"); + + fyd = fy_document_builder_load_document(fydb, fyp); + fyp_error_check(fyp, fyd, err_out, + "fy_document_builder_load_document() failed\n"); + + FYP_TOKEN_ERROR_CHECK(fyp, fy_event_get_token(fye), FYEM_PARSE, + fyd->root, err_out, + "merge key arguments must exist"); + + /* single alias, replace */ + if (fy_node_is_alias(fyd->root)) { + fyd_arg = fy_parser_get_merge_key_argument(fyp, fydb, fyd->root); + fyp_error_check(fyp, fyd_arg, err_out, + "fy_parser_get_merge_key_argument() failed\n"); + fy_node_free(fyd->root); + fyd->root = fy_node_copy(fyd, fyd_arg->root); + + fy_document_destroy(fyd_arg); + fyd_arg = NULL; + + } else if (fy_node_is_mapping(fyd->root)) { + + /* nothing, already a mapping */ + + } else if (fy_node_is_sequence(fyd->root)) { + + new_root = fy_node_create_mapping(fyd); + fyp_error_check(fyp, new_root, err_out, + "fy_node_create_mapping() failed\n"); + + iters = NULL; + while ((fyn_src = fy_node_sequence_iterate(fyd->root, &iters)) != NULL) { + + if (fy_node_is_alias(fyn_src)) { + fyd_arg = fy_parser_get_merge_key_argument(fyp, fydb, fyn_src); + fyp_error_check(fyp, fyd_arg, err_out, + "fy_parser_get_merge_key_argument() failed\n"); + + fyn = fy_node_copy(fyd, fyd_arg->root); + fyp_error_check(fyp, fyn, err_out, + "fy_node_copy() failed\n"); + fy_document_destroy(fyd_arg); + fyd_arg = NULL; + + } else { + FYP_NODE_ERROR_CHECK(fyp, fyn_src, FYEM_PARSE, + fy_node_is_mapping(fyn_src), err_out, + "merge key argument is not a mapping or an alias to such"); + fyn = fy_node_copy(fyd, fyn_src); + fyp_error_check(fyp, fyn, err_out, + "fy_node_copy() failed\n"); + } + + iterm = NULL; + while ((fynp = fy_node_mapping_iterate(fyn, &iterm)) != NULL) { + fynp_exists = fy_node_mapping_lookup_pair(new_root, fynp->key); + + /* do not override existing keys */ + if (fynp_exists) + continue; + + /* add it to the mapping */ + rc = fy_node_mapping_append(new_root, fy_node_copy(fyd, fynp->key), fy_node_copy(fyd, fynp->value)); + fyp_error_check(fyp, !rc, err_out, + "fy_node_mapping_append() failed\n"); + } + + /* and we're done with it */ + fy_node_free(fyn); + fyn = NULL; + } + fy_node_free(fyd->root); + fyd->root = new_root; + + } else { + FYP_TOKEN_ERROR(fyp, fy_event_get_token(fye), FYEM_PARSE, + "merge key argument is neither an alias or a mapping"); + goto err_out; + } + + fy_document_builder_destroy(fydb); + fydb = NULL; + + return fyd; + +err_out: + fy_node_free(new_root); + fy_document_destroy(fyd_arg); + fy_document_destroy(fyd); + fy_document_builder_destroy(fydb); + return NULL; +} + +struct fy_eventp *fy_parser_event_resolve_hook_merge_key_start(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + struct fy_document *fyd = NULL; + struct fy_document_iterator *fydi = NULL; + struct fy_eventp *fyep_next = NULL; + struct fy_event *fye; + struct fy_streaming_alias *fysa; + struct fy_streaming_alias_state *fysas; + + FYP_TOKEN_ERROR_CHECK(fyp, fyep->e.scalar.value, FYEM_PARSE, + !fyp->mks.active, err_out, + "No recursive merge key processing is supported"); + + fyd = fy_parser_get_merge_key_document(fyp, fyep); + FYP_TOKEN_ERROR_CHECK(fyp, fyep->e.scalar.value, FYEM_PARSE, + fyd, err_out, + "No merge key argument document"); + + fysa = fy_parse_streaming_alias_create(fyp, NULL); + fyp_error_check(fyp, fysa, err_out, + "fy_parse_streaming_alias_create() failed"); + + fydi = fy_document_iterator_create(); + fyp_error_check(fyp, fydi, err_out, + "fy_document_iterator_create() failed"); + + fy_document_iterator_node_start(fydi, fyd->root); + + /* now append all to the (fake) streaming anchor */ + while ((fye = fy_document_iterator_body_next(fydi)) != NULL) { + fyep_next = container_of(fye, struct fy_eventp, e); + fy_eventp_list_add_tail(&fysa->events, fyep_next); + } + + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); + fyd = NULL; + + /* and add it to the start of the list */ + fy_streaming_alias_list_add(&fyp->streaming_aliases, fysa); + + /* OK, trim mapping start and end */ + fyep_next = fy_eventp_list_head(&fysa->events); + fyp_error_check(fyp, fyep_next->e.type == FYET_MAPPING_START, err_out, + "not a mapping merge key start"); + fy_eventp_list_del(&fysa->events, fyep_next); + fy_eventp_free(fyep_next); + + fyep_next = fy_eventp_list_tail(&fysa->events); + fyp_error_check(fyp, fyep_next->e.type == FYET_MAPPING_END, err_out, + "not a mapping merge key end"); + fy_eventp_list_del(&fysa->events, fyep_next); + fy_eventp_free(fyep_next); + + /* OK, push the new state */ + fysas = fy_parse_streaming_alias_state_push(fyp, fysa); + fyp_error_check(fyp, fysas, err_out, + "fy_parse_streaming_alias_push() failed!"); + assert(fysas->next); + + /* dispose of the event */ + fy_parse_eventp_recycle(fyp, fyep); + fyep = NULL; + + fyep = fy_parse_eventp_clone(fyp, fysas->next, true); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parse_eventp_clone() failed!"); + + /* advance event */ + fy_parse_streaming_alias_state_next_event(fyp); + + return fyep; + +err_out: + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); + fy_parse_eventp_recycle(fyp, fyep_next); + fy_parse_eventp_recycle(fyp, fyep); + return NULL; +} + +struct fy_eventp *fy_parser_event_resolve_hook_merge_key(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + /* will toggle the map key bit if it is in a mapping */ + fy_parse_streaming_alias_collection_state_toggle_map(fyp); + switch (fyep->e.type) { + case FYET_SCALAR: + /* if a merge key is detected do the dance */ + if (fy_parse_streaming_alias_collection_state_in_map_key(fyp) && + fy_token_scalar_style(fyep->e.scalar.value) == FYSS_PLAIN && + fy_plain_atom_streq(fy_token_atom(fyep->e.scalar.value), "<<")) + return fy_parser_event_resolve_hook_merge_key_start(fyp, fyep); + break; + case FYET_ALIAS: + /* nothing for alias */ + break; + case FYET_MAPPING_START: + fy_parse_streaming_alias_collection_state_push(fyp, FYCTS_MAP_VAL); /* it will toggle to key at the first one */ + break; + case FYET_SEQUENCE_START: + fy_parse_streaming_alias_collection_state_push(fyp, FYCTS_SEQ); + break; + case FYET_MAPPING_END: + case FYET_SEQUENCE_END: + fy_parse_streaming_alias_collection_state_pop(fyp); + break; + default: + break; + } + + return fyep; +} + +/* NOTE: order is _very_ important, do not re-arrange if you're not sure */ +struct fy_eventp *fy_parser_event_resolve_hook(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + int rc; + + /* merge key processing for version 1.1 only! */ + if (fy_reader_get_mode(fyp->reader) == fyrm_yaml_1_1) { + fyep = fy_parser_event_resolve_hook_merge_key(fyp, fyep); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parser_event_resolve_hook_collection_tracking() failed!"); + } + + /* anchor marking */ + fyep = fy_parser_event_resolve_hook_anchor_start(fyp, fyep); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parser_event_resolve_hook_anchor_start() failed!"); + + /* collect afterwards */ + rc = fy_parser_event_resolve_hook_collect(fyp, fyep); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_event_resolve_hook_collect() failed!"); + + /* finally do first entry into alias processing */ + fyep = fy_parser_event_resolve_hook_alias(fyp, fyep); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parser_event_resolve_hook_alias() failed!"); + + return fyep; + +err_out: + return NULL; +} + +struct fy_eventp *fy_parser_parse_resolve_prolog(struct fy_parser *fyp) +{ + struct fy_streaming_alias *fysa; + struct fy_streaming_alias_state *fysas; + struct fy_eventp *fyep, *fyep_src; + bool was_merge_key; + int rc; + + if (!(fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT)) + return NULL; + + fysas = fy_parse_streaming_alias_state_top(fyp); + if (!fysas) + return NULL; + + was_merge_key = false; + for (;;) { + fysa = fysas->fysa; + assert(fysa); + + assert(fysas->next != NULL); + + /* keep track of it */ + fyep_src = fysas->next; + + /* merge key entries don't have an anchor */ + was_merge_key = fysa->anchor == NULL; + + /* advance event */ + fy_parse_streaming_alias_state_next_event(fyp); + + /* not an alias, continue normally */ + if (fyep_src->e.type != FYET_ALIAS) + break; + + /* get the anchor that was defined closest to this one */ + fysa = fy_parser_streaming_alias_lookup_pivot(fyp, fysa, fyep_src->e.alias.anchor); + fyp_error_check(fyp, fysa, err_out, + "fy_parser_streaming_alias_lookup_pivot() failed!"); + + /* and push */ + fysas = fy_parse_streaming_alias_state_push(fyp, fysa); + fyp_error_check(fyp, fysas, err_out, + "fy_parse_streaming_alias_state_push() failed!"); + } + + fyep = fy_parse_eventp_clone(fyp, fyep_src, true); + fyp_error_check(fyp, fyep != NULL, err_out, + "fy_parse_eventp_clone() failed!"); + + if (was_merge_key) { + rc = fy_parser_event_resolve_hook_collect(fyp, fyep_src); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_event_resolve_hook_collect() failed!"); + } + + return fyep; + +err_out: + fyp->stream_error = true; + return NULL; +} + +struct fy_eventp *fy_parser_parse_resolve_epilog(struct fy_parser *fyp, struct fy_eventp *fyep) +{ + if (!(fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT)) + return fyep; + + fyep = fy_parser_event_resolve_hook(fyp, fyep); + if (!fyep) + fyp->stream_error = true; + + return fyep; +} + +struct fy_parser_checkpoint *fy_parser_checkpoint_create(struct fy_parser *fyp) +{ + struct fy_parser_checkpoint *fypchk = NULL; + struct fy_parser *fypc = NULL; + struct fy_reader *fyr, *fyrc; + struct fy_streaming_alias *fysa, *fysac; + struct fy_eventp *fyep, *fyepc; + struct fy_indent *fyit, *fyitc; + struct fy_simple_key *fysk, *fyskc; + struct fy_parse_state_log *fypsl, *fypslc; + struct fy_flow *fyf, *fyfc; + struct fy_token *fyt; + struct fy_input *fyi; + int i; + + /* must be a parser, in a document and not a checkpoint of one, + * with no composer or builder, neither in a merge key state */ + if (!fyp || /* fyp->state != FYPS_DOCUMENT_CONTENT || */ + fyp->is_checkpoint || fyp->fydb || fyp->fyc || + fyp->mks.active) { + fyp_parse_debug(fyp, "cannot take checkpoint at this state"); + return NULL; + } + + /* no queued inputs */ + if (!fy_input_list_empty(&fyp->queued_inputs)) + return NULL; + + fyp_parse_debug(fyp, "taking checkpoint"); + + fypchk = malloc(sizeof(*fypchk)); + fyp_error_check(fyp, fypchk != NULL, err_out, + "fy_parse_input_create() failed!"); + memset(fypchk, 0, sizeof(*fypchk)); + fypc = &fypchk->fyp_checkpoint; + + /* copy everything over and then fill out the pointer */ + *fypc = *fyp; + + fypc->is_checkpoint = true; /* mark it as a checkpoint */ + + /* all recycled lists are initialized */ + fy_indent_list_init(&fypc->recycled_indent); + fy_parse_state_log_list_init(&fypc->recycled_parse_state_log); + fy_eventp_list_init(&fypc->recycled_eventp); + fy_token_list_init(&fypc->recycled_token); + fy_flow_list_init(&fypc->flow_stack); + fy_flow_list_init(&fypc->recycled_flow); + fy_streaming_alias_list_init(&fypc->recycled_streaming_alias); + fy_simple_key_list_init(&fypc->recycled_simple_key); + fy_eventp_list_init(&fypc->mks.args); + fypc->recycled_eventp_list = NULL; + fypc->recycled_token_list = NULL; + + fy_input_list_init(&fypc->queued_inputs); + fypc->stream_end_token = fy_token_ref(fyp->stream_end_token); + + fypc->last_comment.fyi = fy_input_ref(fyp->last_comment.fyi); + fypc->current_document_state = fy_document_state_ref(fyp->current_document_state); + fypc->default_document_state = fy_document_state_ref(fyp->default_document_state); + fypc->next_single_document = fyp->next_single_document; + fypc->diag = NULL; + fypc->last_event_handle.fyi = fy_input_ref(fyp->last_event_handle.fyi); + + if (fyp->fyep_peek) { + fypc->fyep_peek = fy_parse_eventp_clone(fyp, fyp->fyep_peek, false); + fyp_parse_debug(fyp, "checkpoint has a peeked event"); + } + + /* copy the current state of the reader to the builtin reader space */ + fyr = fyp->reader; + assert(fyr); + fyrc = &fypc->builtin_reader; + fypc->reader = fyrc; + + *fyrc = *fyr; + fyrc->current_input = fy_input_ref(fyr->current_input); + assert(fyrc->current_input); + fyrc->diag = NULL; /* no diag */ + + /* copy the streaming alias list */ + if (fyp->sas.stack) { + fyp_parse_debug(fyp, "checkpoint has a streaming alias list"); + if (fyp->sas.alloc <= (int)ARRAY_SIZE(fyp->sas.local)) + fypc->sas.stack = fypc->sas.local; + else { + fypc->sas.stack = malloc(sizeof(*fyp->sas.stack) * fyp->sas.alloc); + assert(fypc->sas.stack); + memcpy(fypc->sas.stack, fyp->sas.stack, sizeof(*fyp->sas.stack) * fyp->sas.top); + } + } + + fy_streaming_alias_list_init(&fypc->streaming_aliases); + for (fysa = fy_streaming_alias_list_head(&fyp->streaming_aliases); fysa != NULL; + fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa)) { + + fysac = fy_parse_streaming_alias_create(fyp, NULL); + assert(fysac); + fysac->anchor = fy_token_ref(fysa->anchor); + for (fyep = fy_eventp_list_head(&fysa->events); fyep; + fyep = fy_eventp_next(&fysa->events, fyep)) { + + fyepc = fy_parse_eventp_clone(fyp, fyep, false); + assert(fyepc); + fy_eventp_list_add_tail(&fysac->events, fyepc); + + /* scan stack and replace pointers */ + for (i = 0; i < fypc->sas.top; i++) { + if (fypc->sas.stack[i].fysa == fysa) + fypc->sas.stack[i].fysa = fysac; + if (fypc->sas.stack[i].next == fyep) + fypc->sas.stack[i].next = fyepc; + } + } + } + + if (fyp->cts.stack) { + fyp_parse_debug(fyp, "checkpoint has a cts stack"); + if (fyp->cts.alloc <= (int)ARRAY_SIZE(fyp->cts.local)) + fypc->cts.stack = fypc->cts.local; + else { + fypc->cts.stack = malloc(sizeof(*fyp->cts.stack) * fyp->cts.alloc); + assert(fypc->cts.stack); + memcpy(fypc->cts.stack, fyp->cts.stack, sizeof(*fyp->cts.stack) * fyp->cts.top); + } + } + + /* copy all remaining lists */ + fy_indent_list_init(&fypc->indent_stack); + for (fyit = fy_indent_list_head(&fyp->indent_stack); fyit != NULL; + fyit = fy_indent_next(&fyp->indent_stack, fyit)) { + + fyitc = fy_parse_indent_alloc(fyp); + assert(fyitc); + + *fyitc = *fyit; + + fy_indent_list_add_tail(&fypc->indent_stack, fyitc); + } + + fy_parse_state_log_list_init(&fypc->state_stack); + for (fypsl = fy_parse_state_log_list_head(&fyp->state_stack); fypsl != NULL; + fypsl = fy_parse_state_log_next(&fyp->state_stack, fypsl)) { + + fypslc = fy_parse_parse_state_log_alloc(fyp); + assert(fypslc); + + fypslc->state = fypsl->state; + + fyp_parse_debug(fyp, "checkpoint saving parse_state %s\n", state_txt[fypslc->state]); + + fy_parse_state_log_list_add_tail(&fypc->state_stack, fypslc); + } + + /* queued tokens and simple keys are 'special' */ + fy_token_list_init(&fypc->queued_tokens); + /* count, and take refs */ + for (fyt = fy_token_list_head(&fyp->queued_tokens), i = 0; fyt != NULL; + fyt = fy_token_next(&fyp->queued_tokens, fyt), i++) { } + fypchk->queued_token_count = i; + fypchk->queued_tokens = malloc(i * sizeof(*fypchk->queued_tokens)); + assert(fypchk->queued_tokens); + for (fyt = fy_token_list_head(&fyp->queued_tokens), i = 0; fyt != NULL; + fyt = fy_token_next(&fyp->queued_tokens, fyt), i++) { + fypchk->queued_tokens[i] = fy_token_ref(fyt); + } + fyp_parse_debug(fyp, "checkpoint has #%d queued tokens", fypchk->queued_token_count); + + fy_simple_key_list_init(&fypc->simple_keys); + + if (!fy_simple_key_list_empty(&fyp->simple_keys)) + fyp_parse_debug(fyp, "checkpoint has pending simple keys"); + + for (fysk = fy_simple_key_list_head(&fyp->simple_keys); fysk != NULL; + fysk = fy_simple_key_next(&fyp->simple_keys, fysk)) { + + fyskc = fy_parse_simple_key_alloc(fyp); + assert(fyskc); + + *fyskc = *fysk; + fyskc->token = fy_token_ref(fysk->token); + + fy_simple_key_list_add_tail(&fypc->simple_keys, fyskc); + } + + fy_input_list_init(&fypc->queued_inputs); + /* count, and take refs */ + for (fyi = fy_input_list_head(&fyp->queued_inputs), i = 0; fyi != NULL; + fyi = fy_input_next(&fyp->queued_inputs, fyi), i++) { } + fypchk->queued_input_count = i; + fypchk->queued_inputs = malloc(i * sizeof(*fypchk->queued_inputs)); + assert(fypchk->queued_inputs); + for (fyi = fy_input_list_head(&fyp->queued_inputs), i = 0; fyi != NULL; + fyi = fy_input_next(&fyp->queued_inputs, fyi), i++) { + fypchk->queued_inputs[i] = fy_input_ref(fyi); + } + + fyp_parse_debug(fyp, "checkpoint flow_level=%d", fypc->flow_level); + fypc->flow = fyp->flow; + if (!fy_flow_list_empty(&fyp->flow_stack)) + fyp_parse_debug(fyp, "checkpoint has flow state"); + + for (fyf = fy_flow_list_head(&fyp->flow_stack); fyf != NULL; + fyf = fy_flow_next(&fyp->flow_stack, fyf)) { + + fyfc = fy_parse_flow_alloc(fyp); + assert(fyfc); + + *fyfc = *fyf; + + fy_flow_list_add_tail(&fypc->flow_stack, fyfc); + } + + fypchk->diag_on_error = fyp->diag && fyp->diag->on_error; + + fyp_parse_debug(fyp, "took checkpoint"); + return fypchk; + +err_out: + if (fypchk) + free(fypchk); + return NULL; +} + +void fy_parser_checkpoint_destroy(struct fy_parser_checkpoint *fypchk) +{ + struct fy_parser *fypc; + int i; + + if (!fypchk) + return; + + if (fypchk->queued_inputs) { + for (i = 0; i < fypchk->queued_input_count; i++) + fy_input_unref(fypchk->queued_inputs[i]); + free(fypchk->queued_inputs); + } + + if (fypchk->queued_tokens) { + for (i = 0; i < fypchk->queued_token_count; i++) + fy_token_unref(fypchk->queued_tokens[i]); + free(fypchk->queued_tokens); + } + + fypc = &fypchk->fyp_checkpoint; + + fy_parse_cleanup(fypc); + + free(fypchk); +} + +int fy_parser_rollback(struct fy_parser *fyp, struct fy_parser_checkpoint *fypchk) +{ + struct fy_parser *fypc = NULL; + struct fy_reader *fyr, *fyrc; + struct fy_indent *fyit, *fyitc; + struct fy_flow *fyf, *fyfc; + struct fy_parse_state_log *fypsl, *fypslc; + struct fy_input *fyi, *fyic; + struct fy_token *fyt; + int i; + + if (!fypchk) + return -1; + + fyp_parse_debug(fyp, "rolling back checkpoint"); + + fypc = &fypchk->fyp_checkpoint; + + if (fyp->diag) + fyp->diag->on_error = fypchk->diag_on_error; + + /* reader restore */ + fyr = fyp->reader; + fyrc = fypc->reader; + + assert(fyr->ops == fyrc->ops); + fy_input_unref(fyr->current_input); + fyr->current_input = fy_input_ref(fyrc->current_input); + fyr->mode = fyrc->mode; + fyr->this_input_start = fyrc->this_input_start; + fyr->current_ptr = fyrc->current_ptr; + fyr->current_ptr_start =fyrc->current_ptr_start; + fyr->current_ptr_end =fyrc->current_ptr_end; + fyr->line = fyrc->line; + fyr->column = fyrc->column; + fyr->tabsize = fyrc->tabsize; + fyr->json_mode = fyrc->json_mode; + fyr->lb_mode = fyrc->lb_mode; + fyr->fws_mode = fyrc->fws_mode; + fyr->directive0_mode = fyrc->directive0_mode; + + fy_token_unref(fyp->stream_end_token); + fyp->stream_end_token = fy_token_ref(fypc->stream_end_token); + + /* inputs restore */ + while ((fyi = fy_input_list_pop(&fyp->queued_inputs)) != NULL) + fy_input_unref(fyi); + for (i = 0; i < fypchk->queued_input_count; i++) { + fyic = fypchk->queued_inputs[i]; + fy_input_list_add_tail(&fyp->queued_inputs, fy_input_ref(fyic)); + } + fyp->token_activity_counter = fypc->token_activity_counter; + + fy_input_unref(fyp->last_comment.fyi); + fyp->last_comment = fypc->last_comment; + fyp->last_comment.fyi = fy_input_ref(fypc->last_comment.fyi); + + /* indents */ + while ((fyit = fy_indent_list_pop(&fyp->indent_stack)) != NULL) + fy_parse_indent_recycle(fyp, fyit); + for (fyitc = fy_indent_list_head(&fypc->indent_stack); fyitc != NULL; + fyitc = fy_indent_next(&fypc->indent_stack, fyitc)) { + + fyit = fy_parse_indent_alloc(fyp); + assert(fyit); + + *fyit = *fyitc; + + fy_indent_list_add_tail(&fyp->indent_stack, fyit); + } + fyp->indent = fypc->indent; + fyp->parent_indent = fypc->parent_indent; + fyp->indent_line = fypc->indent_line; + fyp->starting_indent = fypc->starting_indent; + + /* parse state */ + fyp->state = fypc->state; + while ((fypsl = fy_parse_state_log_list_pop(&fyp->state_stack)) != NULL) + fy_parse_parse_state_log_recycle(fyp, fypsl); + for (fypslc = fy_parse_state_log_list_head(&fypc->state_stack); fypslc != NULL; + fypslc = fy_parse_state_log_next(&fypc->state_stack, fypslc)) { + + fypsl = fy_parse_parse_state_log_alloc(fyp); + assert(fypsl); + + fypsl->state = fypslc->state; + + fy_parse_state_log_list_add_tail(&fyp->state_stack, fypsl); + } + +#ifndef NDEBUG + for (fypsl = fy_parse_state_log_list_head(&fyp->state_stack); fypsl != NULL; + fypsl = fy_parse_state_log_next(&fyp->state_stack, fypsl)) { + fyp_parse_debug(fyp, "checkpoint restored parse_state %s\n", state_txt[fypsl->state]); + } +#endif + + while ((fyt = fy_token_list_pop(&fyp->queued_tokens)) != NULL) + fy_token_unref(fyt); + for (i = 0; i < fypchk->queued_token_count; i++) { + fyt = fypchk->queued_tokens[i]; + fy_token_list_add_tail(&fyp->queued_tokens, fy_token_ref(fyt)); + } + assert(fy_simple_key_list_empty(&fypc->simple_keys)); + assert(fy_streaming_alias_list_empty(&fypc->streaming_aliases)); + + fyp->default_version = fypc->default_version; + fyp->stream_version = fypc->stream_version; + fyp->parser_bitflags = fypc->parser_bitflags; + fyp->is_checkpoint = false; + + fyp_parse_debug(fyp, "checkpoint restoring flow_level=%d", fypc->flow_level); + + fyp->flow_level = fypc->flow_level; + fyp->flow = fypc->flow; + while ((fyf = fy_flow_list_pop(&fyp->flow_stack)) != NULL) + fy_parse_flow_recycle(fyp, fyf); + for (fyfc = fy_flow_list_head(&fypc->flow_stack); fyfc != NULL; + fyfc = fy_flow_next(&fypc->flow_stack, fyfc)) { + + fyf = fy_parse_flow_alloc(fyp); + assert(fyf); + + *fyf = *fyfc; + + fy_flow_list_add_tail(&fyp->flow_stack, fyf); + } + + fyp->pending_complex_key_column = fypc->pending_complex_key_column; + fyp->pending_complex_key_mark = fypc->pending_complex_key_mark; + fyp->last_block_mapping_key_line = fypc->last_block_mapping_key_line; + fyp->last_comma_mark = fypc->last_comma_mark; + fyp->last_tab_used_for_ws_mark = fypc->last_tab_used_for_ws_mark; + + fy_document_state_unref(fyp->current_document_state); + fyp->current_document_state = fy_document_state_ref(fypc->current_document_state); + fy_document_state_unref(fyp->default_document_state); + fyp->default_document_state = fy_document_state_ref(fypc->default_document_state); + + fyp->next_single_document = fypc->next_single_document; + fyp->last_event_handle = fypc->last_event_handle; + fy_input_unref(fyp->last_event_handle.fyi); + fyp->last_event_handle.fyi = fy_input_ref(fypc->last_event_handle.fyi); + + fy_parse_eventp_recycle(fyp, fyp->fyep_peek); + fyp->fyep_peek = NULL; + if (fypc->fyep_peek) { + fyp->fyep_peek = fy_parse_eventp_clone(fyp, fypc->fyep_peek, false); + assert(fypc->fyep_peek); + } + + assert(fyp->sas.top == 0); + + return 0; +} + +enum fy_parser_mode +fy_parser_get_mode(struct fy_parser *fyp) +{ + struct fy_version *versp; + int major, minor; + + if (!fyp) + return fypm_invalid; + + versp = fyp->current_document_state ? &fyp->current_document_state->version : &fyp->stream_version; + + switch (fy_reader_get_mode(fyp->reader)) { + case fyrm_yaml: + case fyrm_yaml_1_1: + major = versp->major; + minor = versp->minor; + /* 1.1, 1.2, 1.3 */ + if (major != 1 || minor < 1 || minor > 3) + break; + return fypm_yaml_1_1 + (minor - 1); + + case fyrm_json: + return fypm_json; + } + + return fypm_none; +} + +int fy_parser_count_sequence_items(struct fy_parser *fyp) +{ + const struct fy_event *fye_peek; + struct fy_parser_checkpoint *fypchk = NULL; + int rc, count; + + if (!fyp) + return -1; + + /* take a checkpoint and race ahead to find the size */ + fypchk = fy_parser_checkpoint_create(fyp); + fyp_error_check(fyp, fypchk != NULL, err_out, + "fy_parser_checkpoint_create() failed!"); + count = 0; + for (;;) { + fye_peek = fy_parser_parse_peek(fyp); + if (!fye_peek) + goto err_out; + if (fye_peek->type != FYET_SCALAR && + fye_peek->type != FYET_MAPPING_START && + fye_peek->type != FYET_SEQUENCE_START && + fye_peek->type != FYET_SEQUENCE_END) + goto err_out; + + if (fye_peek->type == FYET_SEQUENCE_END) + break; + + /* skip the item */ + rc = fy_parser_skip(fyp); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_skip() failed!"); + count++; + } + + rc = fy_parser_rollback(fyp, fypchk); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_rollback() failed!"); +out: + fy_parser_checkpoint_destroy(fypchk); + return count; + +err_out: + count = -1; + goto out; +} + +int fy_parser_count_mapping_items(struct fy_parser *fyp) +{ + const struct fy_event *fye_peek; + struct fy_parser_checkpoint *fypchk = NULL; + int rc, count; + + if (!fyp) + return -1; + + /* take a checkpoint and race ahead to find the size */ + fypchk = fy_parser_checkpoint_create(fyp); + fyp_error_check(fyp, fypchk != NULL, err_out, + "fy_parser_checkpoint_create() failed!"); + count = 0; + for (;;) { + fye_peek = fy_parser_parse_peek(fyp); + if (!fye_peek) + goto err_out; + if (fye_peek->type != FYET_SCALAR && + fye_peek->type != FYET_MAPPING_START && + fye_peek->type != FYET_SEQUENCE_START && + fye_peek->type != FYET_MAPPING_END) + goto err_out; + + if (fye_peek->type == FYET_MAPPING_END) + break; + + /* skip the key item */ + rc = fy_parser_skip(fyp); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_skip() failed!"); + + /* skip the value item */ + rc = fy_parser_skip(fyp); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_skip() failed!"); + count++; + } + + rc = fy_parser_rollback(fyp, fypchk); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_rollback() failed!"); +out: + fy_parser_checkpoint_destroy(fypchk); + return count; + +err_out: + count = -1; + goto out; +} + +// just for LLVMShutdown +#if defined(HAVE_LIBCLANG) && HAVE_LIBCLANG +#include +#endif + +void fy_shutdown(void) +{ +#if defined(HAVE_LIBCLANG) && HAVE_LIBCLANG + LLVMShutdown(); +#endif +} + +#ifdef FY_HAS_DESTRUCTOR +FY_DESTRUCTOR void fy_do_cleanup(void) +{ + fy_shutdown(); +} +#endif diff --git a/src/lib/fy-parse.h b/src/lib/fy-parse.h new file mode 100644 index 0000000..d269f58 --- /dev/null +++ b/src/lib/fy-parse.h @@ -0,0 +1,869 @@ +/* + * fy-parse.h - YAML parser internal header file + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_PARSE_H +#define FY_PARSE_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#include + +#include "fy-ctype.h" +#include "fy-utf8.h" +#include "fy-list.h" +#include "fy-typelist.h" +#include "fy-types.h" +#include "fy-diag.h" +#include "fy-dump.h" +#include "fy-atom.h" +#include "fy-input.h" +#include "fy-ctype.h" +#include "fy-token.h" +#include "fy-event.h" +#include "fy-docstate.h" +#include "fy-doc.h" +#include "fy-docbuilder.h" +#include "fy-emit.h" +#include "fy-accel.h" +#include "fy-emit-accum.h" +#include "fy-path.h" +#include "fy-composer.h" + +struct fy_parser; +struct fy_input; + +/* for the event */ +FY_PARSE_TYPE_DECL_ALLOC(eventp); + +enum fy_flow_type { + FYFT_NONE, + FYFT_MAP, + FYFT_SEQUENCE, +}; + +struct fy_flow { + struct list_head node; + enum fy_flow_type flow; + int pending_complex_key_column; + struct fy_mark pending_complex_key_mark; + int parent_indent; +}; +FY_PARSE_TYPE_DECL(flow); + +struct fy_indent { + struct list_head node; + int indent; + int indent_line; + bool generated_block_map : 1; +}; +FY_PARSE_TYPE_DECL(indent); + +struct fy_token; + +struct fy_simple_key { + struct list_head node; + struct fy_mark mark; + struct fy_mark end_mark; + struct fy_token *token; /* associated token */ + int flow_level; + bool required : 1; + bool implicit_complex : 1; +}; +FY_PARSE_TYPE_DECL(simple_key); + +struct fy_document_state; + +enum fy_parser_state { + /** none state */ + FYPS_NONE, + /** Expect STREAM-START. */ + FYPS_STREAM_START, + /** Expect the beginning of an implicit document. */ + FYPS_IMPLICIT_DOCUMENT_START, + /** Expect DOCUMENT-START. */ + FYPS_DOCUMENT_START, + /** Expect the content of a document. */ + FYPS_DOCUMENT_CONTENT, + /** Expect DOCUMENT-END. */ + FYPS_DOCUMENT_END, + /** Expect a block node. */ + FYPS_BLOCK_NODE, + /** Expect the first entry of a block sequence. */ + FYPS_BLOCK_SEQUENCE_FIRST_ENTRY, + /** Expect an entry of a block sequence. */ + FYPS_BLOCK_SEQUENCE_ENTRY, + /** Expect an entry of an indentless sequence. */ + FYPS_INDENTLESS_SEQUENCE_ENTRY, + /** Expect the first key of a block mapping. */ + FYPS_BLOCK_MAPPING_FIRST_KEY, + /** Expect a block mapping key. */ + FYPS_BLOCK_MAPPING_KEY, + /** Expect a block mapping value. */ + FYPS_BLOCK_MAPPING_VALUE, + /** Expect the first entry of a flow sequence. */ + FYPS_FLOW_SEQUENCE_FIRST_ENTRY, + /** Expect an entry of a flow sequence. */ + FYPS_FLOW_SEQUENCE_ENTRY, + /** Expect a key of an ordered mapping. */ + FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_KEY, + /** Expect a value of an ordered mapping. */ + FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_VALUE, + /** Expect the and of an ordered mapping entry. */ + FYPS_FLOW_SEQUENCE_ENTRY_MAPPING_END, + /** Expect the first key of a flow mapping. */ + FYPS_FLOW_MAPPING_FIRST_KEY, + /** Expect a key of a flow mapping. */ + FYPS_FLOW_MAPPING_KEY, + /** Expect a value of a flow mapping. */ + FYPS_FLOW_MAPPING_VALUE, + /** Expect an empty value of a flow mapping. */ + FYPS_FLOW_MAPPING_EMPTY_VALUE, + /** Expect only stream end */ + FYPS_SINGLE_DOCUMENT_END, + /** Expect nothing. */ + FYPS_END +}; + +struct fy_parse_state_log { + struct list_head node; + enum fy_parser_state state; +}; +FY_PARSE_TYPE_DECL(parse_state_log); + +struct fy_streaming_alias { + struct list_head node; + struct fy_token *anchor; + bool collecting; + long mapping_nest; + long sequence_nest; + struct fy_eventp_list events; +}; +FY_PARSE_TYPE_DECL(streaming_alias); + +struct fy_streaming_alias_state { + struct fy_streaming_alias *fysa; + struct fy_eventp *next; +}; + +/* 2 bits per collection tracking state + * pos: * 2 + * bit 0 - 0 map, 1 seq + * bit 1 - 0 key, 1 val (if map) + */ +#define FYCTS_SCALAR 0 +#define FYCTS_SEQ 1 +#define FYCTS_MAP 2 +#define FYCTS_MAP_KEY 2 +#define FYCTS_MAP_VAL 3 +#define FYCTS_MAP_TOGGLE 1 + +struct fy_parser { + struct fy_parse_cfg cfg; + + struct fy_input_list queued_inputs; /* all the inputs queued */ + struct fy_reader builtin_reader; /* the builtin reader */ + struct fy_reader *reader; /* the external reader, or ptr to builtin_reader */ + + struct fy_version default_version; + struct fy_version stream_version; /* last version set in stream */ + + union { + struct { + bool suppress_recycling : 1; + bool stream_start_produced : 1; + bool stream_end_produced : 1; + bool stream_end_reached : 1; + bool simple_key_allowed : 1; + bool tab_used_for_ws : 1; + bool stream_error : 1; + bool generated_block_map : 1; + bool last_was_comma : 1; + bool document_has_content : 1; + bool document_first_content_token : 1; + bool bare_document_only : 1; /* no document start indicators allowed, no directives */ + bool stream_has_content : 1; + bool parse_flow_only : 1; /* document is in flow form, and stop parsing at the end */ + bool parse_block_only : 1; /* document is in embedded block form, and stop parsing at the end */ + bool colon_follows_colon : 1; /* "foo"::bar -> "foo": :bar */ + bool had_directives : 1; /* document had directives */ + bool is_checkpoint : 1; /* a parser checkpoint (not a real parser) */ + }; + unsigned int parser_bitflags; + }; + + int flow_level; + int pending_complex_key_column; + struct fy_mark pending_complex_key_mark; + int last_block_mapping_key_line; + struct fy_mark last_comma_mark; + struct fy_mark last_tab_used_for_ws_mark; + + /* copy of stream_end token */ + struct fy_token *stream_end_token; + + /* produced tokens, but not yet consumed */ + struct fy_token_list queued_tokens; + int token_activity_counter; + + /* last comment */ + struct fy_atom last_comment; + + /* indent stack */ + struct fy_indent_list indent_stack; + int indent; + int parent_indent; + int indent_line; + int starting_indent; + /* simple key stack */ + struct fy_simple_key_list simple_keys; + /* state stack */ + enum fy_parser_state state; + struct fy_parse_state_log_list state_stack; + + /* current parse document */ + struct fy_document_state *current_document_state; + struct fy_document_state *default_document_state; + bool next_single_document; + + /* flow stack */ + enum fy_flow_type flow; + struct fy_flow_list flow_stack; + + /* recycling lists */ + struct fy_indent_list recycled_indent; + struct fy_simple_key_list recycled_simple_key; + struct fy_parse_state_log_list recycled_parse_state_log; + struct fy_flow_list recycled_flow; + struct fy_streaming_alias_list recycled_streaming_alias; + + struct fy_eventp_list recycled_eventp; + struct fy_token_list recycled_token; + + struct fy_eventp_list *recycled_eventp_list; + struct fy_token_list *recycled_token_list; + + /* the diagnostic object */ + struct fy_diag *diag; + + int err_term_width; + int err_term_height; + + /* for when using the built-in document builder */ + struct fy_document_builder *fydb; + + /* when using the composer interface */ + struct fy_composer *fyc; + fy_parse_composer_cb fyc_cb; + void *fyc_userdata; + + /* last generated event atom */ + struct fy_atom last_event_handle; + + struct fy_streaming_alias_list streaming_aliases; + /* streaming alias state */ + struct { + int alloc; + int top; + struct fy_streaming_alias_state *stack; + struct fy_streaming_alias_state local[8]; + } sas; + /* collection tracking state */ + struct { + int alloc; + int top; + uint32_t *stack; + uint32_t local[8]; /* 8 * 32 = 256 / 2 = 128 levels before needing dynamic allocation */ + } cts; + /* merge key state */ + struct { + bool active; + struct fy_eventp_list args; + struct fy_document *fyd; + } mks; + struct fy_eventp *fyep_peek; +}; + +static inline struct fy_input * +fyp_current_input(const struct fy_parser *fyp) +{ + assert(fyp); + return fy_reader_current_input(fyp->reader); +} + +static inline uint64_t +fyp_current_input_generation(const struct fy_parser *fyp) +{ + assert(fyp); + return fy_reader_current_input_generation(fyp->reader); +} + +static inline int fyp_column(const struct fy_parser *fyp) +{ + assert(fyp); + return fy_reader_column(fyp->reader); +} + +static inline int fyp_line(const struct fy_parser *fyp) +{ + return fy_reader_line(fyp->reader); +} + +static inline int fyp_tabsize(const struct fy_parser *fyp) +{ + return fy_reader_tabsize(fyp->reader); +} + +static inline bool fyp_json_mode(const struct fy_parser *fyp) +{ + assert(fyp); + return fy_reader_json_mode(fyp->reader); +} + +static inline enum fy_lb_mode fyp_lb_mode(const struct fy_parser *fyp) +{ + assert(fyp); + return fy_reader_lb_mode(fyp->reader); +} + +static inline enum fy_flow_ws_mode fyp_fws_mode(const struct fy_parser *fyp) +{ + assert(fyp); + return fy_reader_flow_ws_mode(fyp->reader); +} + +static inline bool fyp_directive0_mode(const struct fy_parser *fyp) +{ + assert(fyp); + return fy_reader_directive0_mode(fyp->reader); +} + +static inline bool fyp_block_mode(struct fy_parser *fyp) +{ + return !fyp_json_mode(fyp) && !fyp->flow_level; +} + +static inline bool fyp_is_lb(const struct fy_parser *fyp, int c) +{ + assert(fyp); + return fy_reader_is_lb(fyp->reader, c); +} + +static inline bool fyp_is_lbz(const struct fy_parser *fyp, int c) +{ + assert(fyp); + return fy_reader_is_lbz(fyp->reader, c); +} + +static inline bool fyp_is_blankz(const struct fy_parser *fyp, int c) +{ + assert(fyp); + return fy_reader_is_blankz(fyp->reader, c); +} + +static inline bool fyp_is_generic_lb(const struct fy_parser *fyp, int c) +{ + assert(fyp); + return fy_reader_is_generic_lb(fyp->reader, c); +} + +static inline bool fyp_is_generic_lbz(const struct fy_parser *fyp, int c) +{ + assert(fyp); + return fy_reader_is_generic_lbz(fyp->reader, c); +} + +static inline bool fyp_is_generic_blankz(const struct fy_parser *fyp, int c) +{ + assert(fyp); + return fy_reader_is_generic_blankz(fyp->reader, c); +} + +static inline bool fyp_is_flow_ws(const struct fy_parser *fyp, int c) +{ + assert(fyp); + return fy_reader_is_flow_ws(fyp->reader, c); +} + +static inline bool fyp_is_flow_blank(const struct fy_parser *fyp, int c) +{ + assert(fyp); + return fy_reader_is_flow_blank(fyp->reader, c); +} + +static inline bool fyp_is_flow_blankz(const struct fy_parser *fyp, int c) +{ + assert(fyp); + return fy_reader_is_flow_blankz(fyp->reader, c); +} + +static inline const void * +fy_ensure_lookahead_slow_path(struct fy_parser *fyp, size_t size, size_t *leftp) +{ + assert(fyp); + return fy_reader_ensure_lookahead_slow_path(fyp->reader, size, leftp); +} + +/* only allowed if input does not update */ +static inline void +fy_get_mark(struct fy_parser *fyp, struct fy_mark *fym) +{ + assert(fyp); + return fy_reader_get_mark(fyp->reader, fym); +} + +static inline const void * +fy_ptr(struct fy_parser *fyp, size_t *leftp) +{ + assert(fyp); + return fy_reader_ptr(fyp->reader, leftp); +} + +static inline const void * +fy_ensure_lookahead(struct fy_parser *fyp, size_t size, size_t *leftp) +{ + assert(fyp); + return fy_reader_ensure_lookahead(fyp->reader, size, leftp); +} + +/* advance the given number of ascii characters, not utf8 */ +static inline void +fy_advance_octets(struct fy_parser *fyp, size_t advance) +{ + assert(fyp); + return fy_reader_advance_octets(fyp->reader, advance); +} + +/* compare string at the current point (n max) */ +static inline int +fy_parse_strncmp(struct fy_parser *fyp, const char *str, size_t n) +{ + assert(fyp); + return fy_reader_strncmp(fyp->reader, str, n); +} + +static FY_ALWAYS_INLINE inline int +fy_parse_peek_at_offset_width(struct fy_parser *fyp, size_t offset, int *wp) +{ + assert(fyp); + return fy_reader_peek_at_offset_width(fyp->reader, offset, wp); +} + +static FY_ALWAYS_INLINE inline int +fy_parse_peek_at_offset(struct fy_parser *fyp, size_t offset) +{ + assert(fyp); + return fy_reader_peek_at_offset(fyp->reader, offset); +} + +static FY_ALWAYS_INLINE inline int +fy_parse_peek_at_width_internal(struct fy_parser *fyp, int pos, ssize_t *offsetp, int *wp) +{ + assert(fyp); + return fy_reader_peek_at_width_internal(fyp->reader, pos, offsetp, wp); +} + +static FY_ALWAYS_INLINE inline int +fy_parse_peek_at_internal(struct fy_parser *fyp, int pos, ssize_t *offsetp) +{ + assert(fyp); + return fy_reader_peek_at_internal(fyp->reader, pos, offsetp); +} + +static inline bool +fy_is_blank_at_offset(struct fy_parser *fyp, size_t offset) +{ + assert(fyp); + return fy_is_blank(fy_reader_peek_at_offset(fyp->reader, offset)); +} + +static inline bool +fy_is_blankz_at_offset(struct fy_parser *fyp, size_t offset) +{ + assert(fyp); + return fy_reader_is_blankz(fyp->reader, fy_reader_peek_at_offset(fyp->reader, offset)); +} + +static inline bool +fy_is_generic_blankz_at_offset(struct fy_parser *fyp, size_t offset) +{ + assert(fyp); + return fy_reader_is_generic_blankz(fyp->reader, fy_reader_peek_at_offset(fyp->reader, offset)); +} + +static FY_ALWAYS_INLINE inline int +fy_parse_peek_at_width(struct fy_parser *fyp, int pos, int *wp) +{ + assert(fyp); + return fy_reader_peek_at_width(fyp->reader, pos, wp); +} + +static FY_ALWAYS_INLINE inline int +fy_parse_peek_at(struct fy_parser *fyp, int pos) +{ + assert(fyp); + return fy_reader_peek_at(fyp->reader, pos); +} + +static FY_ALWAYS_INLINE inline int +fy_parse_peek_width(struct fy_parser *fyp, int *wp) +{ + assert(fyp); + return fy_reader_peek_width(fyp->reader, wp); +} + +static FY_ALWAYS_INLINE inline int +fy_parse_peek(struct fy_parser *fyp) +{ + assert(fyp); + return fy_reader_peek(fyp->reader); +} + +static FY_ALWAYS_INLINE inline void +fy_advance(struct fy_parser *fyp, int c) +{ + assert(fyp); + fy_reader_advance(fyp->reader, c); +} + +static FY_ALWAYS_INLINE inline void +fy_advance_ws(struct fy_parser *fyp, int c) +{ + assert(fyp); + fy_reader_advance_ws(fyp->reader, c); +} + +static FY_ALWAYS_INLINE inline void +fy_advance_space(struct fy_parser *fyp) +{ + assert(fyp); + fy_reader_advance_space(fyp->reader); +} + +static FY_ALWAYS_INLINE inline int +fy_parse_get(struct fy_parser *fyp) +{ + assert(fyp); + return fy_reader_get(fyp->reader); +} + +static FY_ALWAYS_INLINE inline int +fy_advance_by(struct fy_parser *fyp, int count) +{ + assert(fyp); + return fy_reader_advance_by(fyp->reader, count); +} + +/* compare string at the current point */ +static inline bool +fy_parse_strcmp(struct fy_parser *fyp, const char *str) +{ + assert(fyp); + return fy_reader_strcmp(fyp->reader, str); +} + +static inline void +fy_fill_atom_start(struct fy_parser *fyp, struct fy_atom *handle) +{ + assert(fyp); + fy_reader_fill_atom_start(fyp->reader, handle); +} + +static inline void +fy_fill_atom_end_at(struct fy_parser *fyp, struct fy_atom *handle, struct fy_mark *end_mark) +{ + assert(fyp); + fy_reader_fill_atom_end_at(fyp->reader, handle, end_mark); +} + +static inline void +fy_fill_atom_end(struct fy_parser *fyp, struct fy_atom *handle) +{ + assert(fyp); + fy_reader_fill_atom_end(fyp->reader, handle); +} + +static inline struct fy_atom * +fy_fill_atom(struct fy_parser *fyp, int advance, struct fy_atom *handle) +{ + assert(fyp); + return fy_reader_fill_atom(fyp->reader, advance, handle); +} + +static inline struct fy_atom * +fy_fill_atom_mark(struct fy_parser *fyp, const struct fy_mark *start_mark, + const struct fy_mark *end_mark, struct fy_atom *handle) +{ + assert(fyp); + return fy_reader_fill_atom_mark(fyp->reader, start_mark, end_mark, handle); +} + +static inline struct fy_atom * +fy_fill_atom_at(struct fy_parser *fyp, int advance, int count, struct fy_atom *handle) +{ + assert(fyp); + return fy_reader_fill_atom_at(fyp->reader, advance, count, handle); +} + +static inline void +fy_parser_set_reader(struct fy_parser *fyp, struct fy_reader *fyr) +{ + if (!fyp) + return; + fyp->reader = fyr ? : &fyp->builtin_reader; +} + +static inline void +fy_parser_set_flow_only_mode(struct fy_parser *fyp, bool flow_only_mode) +{ + fyp->parse_flow_only = flow_only_mode; +} + +static inline void +fy_parser_set_block_only_mode(struct fy_parser *fyp, bool block_only_mode) +{ + fyp->parse_block_only = block_only_mode; +} + +#define fy_fill_atom_a(_fyp, _advance) \ + fy_fill_atom((_fyp), (_advance), alloca(sizeof(struct fy_atom))) + +struct fy_token *fy_token_vqueue(struct fy_parser *fyp, enum fy_token_type type, va_list ap); +struct fy_token *fy_token_queue(struct fy_parser *fyp, enum fy_token_type type, ...); + +struct fy_token * +fy_token_vqueue_internal(struct fy_parser *fyp, struct fy_token_list *fytl, + enum fy_token_type type, va_list ap); +struct fy_token * +fy_token_queue_internal(struct fy_parser *fyp, struct fy_token_list *fytl, + enum fy_token_type type, ...); + +int fy_parse_setup(struct fy_parser *fyp, const struct fy_parse_cfg *cfg); +void fy_parse_cleanup(struct fy_parser *fyp); + +int fy_parse_input_append(struct fy_parser *fyp, const struct fy_input_cfg *fyic); +ssize_t fy_parse_estimate_queued_input_size(struct fy_parser *fyp); + +struct fy_eventp *fy_parse_private(struct fy_parser *fyp); + +extern const char *fy_event_type_txt[]; + +enum fy_parse_cfg_flags fy_parser_get_cfg_flags(const struct fy_parser *fyp); + +extern const struct fy_tag * const fy_default_tags[]; + +extern const struct fy_version fy_default_version; /* usually highest stable */ + +bool fy_tag_handle_is_default(const char *handle, size_t handle_size); +bool fy_tag_is_default_internal(const char *handle, size_t handle_size, + const char *prefix, size_t prefix_size); +bool fy_token_tag_directive_is_overridable(struct fy_token *fyt_td); + +int fy_parser_set_default_document_state(struct fy_parser *fyp, + struct fy_document_state *fyds); +void fy_parser_set_next_single_document(struct fy_parser *fyp); + +void *fy_alloc_default(void *userdata, size_t size); +void fy_free_default(void *userdata, void *ptr); +void *fy_realloc_default(void *userdata, void *ptr, size_t size); + +int fy_reader_fetch_flow_scalar_handle(struct fy_reader *fyr, int c, int indent, struct fy_atom *handle, bool sloppy_indent); +int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, int indent, int flow_level, struct fy_atom *handle); + +void fy_reader_skip_ws_cr_nl(struct fy_reader *fyr); + +void fy_reader_skip_ws(struct fy_reader *fyr); +void fy_reader_skip_space(struct fy_reader *fyr); + +static inline int fy_document_state_version_compare(struct fy_document_state *fyds, const struct fy_version *vb) +{ + return fy_version_compare(fy_document_state_version(fyds), vb); +} + +int fy_parse_set_composer(struct fy_parser *fyp, fy_parse_composer_cb cb, void *userdata); + +struct fy_streaming_alias * +fy_parse_streaming_alias_create(struct fy_parser *fyp, struct fy_token *fyt_anchor); +void fy_parse_streaming_alias_clean(struct fy_parser *fyp, struct fy_streaming_alias *fysa); +void fy_parse_streaming_aliases_reset(struct fy_parser *fyp); + +struct fy_eventp *fy_parser_parse_resolve_prolog(struct fy_parser *fyp); +struct fy_eventp *fy_parser_parse_resolve_epilog(struct fy_parser *fyp, struct fy_eventp *fyep); + +struct fy_parser_checkpoint { + struct fy_parser fyp_checkpoint; + int queued_token_count; + struct fy_token **queued_tokens; + int queued_input_count; + struct fy_input **queued_inputs; + bool diag_on_error; +}; + +/* diagnostics */ + +static inline bool +fyp_debug_log_level_is_enabled(struct fy_parser *fyp, enum fy_error_module module) +{ + return fyp && fy_diag_log_level_is_enabled(fyp->diag, FYET_DEBUG, module); +} + +int fy_parser_vdiag(struct fy_parser *fyp, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap); + +int fy_parser_diag(struct fy_parser *fyp, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) + __attribute__((format(printf, 6, 7))); + +void fy_parser_diag_vreport(struct fy_parser *fyp, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap); +void fy_parser_diag_report(struct fy_parser *fyp, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +#ifdef FY_DEVMODE + +#define fyp_debug(_fyp, _module, _fmt, ...) \ + do { \ + struct fy_parser *__fyp = (_fyp); \ + enum fy_error_module __module = (_module); \ + \ + if (fyp_debug_log_level_is_enabled(__fyp, __module)) \ + fy_parser_diag(__fyp, FYET_DEBUG | FYDF_MODULE(__module), \ + __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__); \ + } while(0) +#else + +#define fyp_debug(_fyp, _module, _fmt, ...) \ + do { } while(0) + +#endif + +#define fyp_info(_fyp, _fmt, ...) \ + fy_parser_diag((_fyp), FYET_INFO, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyp_notice(_fyp, _fmt, ...) \ + fy_parser_diag((_fyp), FYET_NOTICE, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyp_warning(_fyp, _fmt, ...) \ + fy_parser_diag((_fyp), FYET_WARNING, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyp_error(_fyp, _fmt, ...) \ + fy_parser_diag((_fyp), FYET_ERROR, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) + +#define fyp_scan_debug(_fyp, _fmt, ...) \ + fyp_debug((_fyp), FYEM_SCAN, (_fmt) , ## __VA_ARGS__) +#define fyp_parse_debug(_fyp, _fmt, ...) \ + fyp_debug((_fyp), FYEM_PARSE, (_fmt) , ## __VA_ARGS__) +#define fyp_doc_debug(_fyp, _fmt, ...) \ + fyp_debug((_fyp), FYEM_DOC, (_fmt) , ## __VA_ARGS__) + +#define fyp_error_check(_fyp, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + fyp_error((_fyp), _fmt, ## __VA_ARGS__); \ + goto _label ; \ + } \ + } while(0) + +#define _FYP_TOKEN_DIAG(_fyp, _fyt, _type, _module, _fmt, ...) \ + do { \ + struct fy_diag_report_ctx _drc; \ + memset(&_drc, 0, sizeof(_drc)); \ + _drc.type = (_type); \ + _drc.module = (_module); \ + _drc.fyt = (_fyt); \ + fy_parser_diag_report((_fyp), &_drc, (_fmt) , ## __VA_ARGS__); \ + } while(0) + +#define FYP_TOKEN_DIAG(_fyp, _fyt, _type, _module, _fmt, ...) \ + _FYP_TOKEN_DIAG(_fyp, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYP_PARSE_DIAG(_fyp, _adv, _cnt, _type, _module, _fmt, ...) \ + _FYP_TOKEN_DIAG(_fyp, \ + fy_token_create(FYTT_INPUT_MARKER, \ + fy_fill_atom_at((_fyp), (_adv), (_cnt), \ + alloca(sizeof(struct fy_atom)))), \ + _type, _module, _fmt, ## __VA_ARGS__) + +#define FYP_MARK_DIAG(_fyp, _sm, _em, _type, _module, _fmt, ...) \ + _FYP_TOKEN_DIAG(_fyp, \ + fy_token_create(FYTT_INPUT_MARKER, \ + fy_fill_atom_mark(((_fyp)), (_sm), (_em), \ + alloca(sizeof(struct fy_atom)))), \ + _type, _module, _fmt, ## __VA_ARGS__) + +#define FYP_NODE_DIAG(_fyp, _fyn, _type, _module, _fmt, ...) \ + _FYP_TOKEN_DIAG(_fyp, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYP_TOKEN_ERROR(_fyp, _fyt, _module, _fmt, ...) \ + FYP_TOKEN_DIAG(_fyp, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYP_PARSE_ERROR(_fyp, _adv, _cnt, _module, _fmt, ...) \ + FYP_PARSE_DIAG(_fyp, _adv, _cnt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYP_MARK_ERROR(_fyp, _sm, _em, _module, _fmt, ...) \ + FYP_MARK_DIAG(_fyp, _sm, _em, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYP_NODE_ERROR(_fyp, _fyn, _module, _fmt, ...) \ + FYP_NODE_DIAG(_fyp, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYP_TOKEN_ERROR_CHECK(_fyp, _fyt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYP_TOKEN_ERROR(_fyp, _fyt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYP_PARSE_ERROR_CHECK(_fyp, _adv, _cnt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYP_PARSE_ERROR(_fyp, _adv, _cnt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYP_MARK_ERROR_CHECK(_fyp, _sm, _em, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYP_MARK_ERROR(_fyp, _sm, _em, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYP_NODE_ERROR_CHECK(_fyp, _fyn, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYP_NODE_ERROR(_fyp, _fyn, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYP_TOKEN_WARNING(_fyp, _fyt, _module, _fmt, ...) \ + FYP_TOKEN_DIAG(_fyp, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYP_PARSE_WARNING(_fyp, _adv, _cnt, _module, _fmt, ...) \ + FYP_PARSE_DIAG(_fyp, _adv, _cnt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYP_MARK_WARNING(_fyp, _sm, _em, _module, _fmt, ...) \ + FYP_MARK_DIAG(_fyp, _sm, _em, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYP_NODE_WARNING(_fyp, _fyn, _type, _module, _fmt, ...) \ + FYP_NODE_DIAG(_fyp, _fyn, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#endif diff --git a/src/lib/fy-path.c b/src/lib/fy-path.c new file mode 100644 index 0000000..b770698 --- /dev/null +++ b/src/lib/fy-path.c @@ -0,0 +1,717 @@ +/* + * fy-path.c - Internal ypath support + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" +#include "fy-doc.h" + +#include "fy-utils.h" + +#undef DBG +// #define DBG fyp_notice +#define DBG fyp_scan_debug + +static int fy_path_setup(struct fy_path *fypp) +{ + memset(fypp, 0, sizeof(*fypp)); + + fy_path_component_list_init(&fypp->recycled_component); + fy_path_component_list_init(&fypp->components); + + return 0; +} + +static void fy_path_cleanup(struct fy_path *fypp) +{ + struct fy_path_component *fypc; + + if (!fypp) + return; + + if (fypp->fydb) { + fy_document_builder_destroy(fypp->fydb); + fypp->fydb = NULL; + } + + while ((fypc = fy_path_component_list_pop(&fypp->components)) != NULL) + fy_path_component_free(fypc); + + while ((fypc = fy_path_component_list_pop(&fypp->recycled_component)) != NULL) + fy_path_component_free(fypc); +} + +struct fy_path *fy_path_create(void) +{ + struct fy_path *fypp; + int rc; + + fypp = malloc(sizeof(*fypp)); + if (!fypp) + return NULL; + + rc = fy_path_setup(fypp); + if (rc) + return NULL; + + return fypp; +} + +void fy_path_destroy(struct fy_path *fypp) +{ + if (!fypp) + return; + + fy_path_cleanup(fypp); + + free(fypp); +} + +void fy_path_reset(struct fy_path *fypp) +{ + struct fy_path_component *fypc; + + if (!fypp) + return; + + while ((fypc = fy_path_component_list_pop(&fypp->components)) != NULL) + fy_path_component_free(fypc); +} + +struct fy_path_component *fy_path_component_alloc(struct fy_path *fypp) +{ + struct fy_path_component *fypc; + + if (!fypp) + return NULL; + + fypc = fy_path_component_list_pop(&fypp->recycled_component); + if (!fypc) { + fypc = malloc(sizeof(*fypc)); + if (!fypc) + return NULL; + memset(fypc, 0, sizeof(*fypc)); + } + + /* not yet instantiated */ + fypc->type = FYPCT_NONE; + + return fypc; +} + +void fy_path_component_clear_state(struct fy_path_component *fypc) +{ + if (!fypc) + return; + + switch (fypc->type) { + case FYPCT_NONE: + /* nothing */ + break; + + case FYPCT_MAP: + if (fypc->map.has_key) { + if (fypc->map.is_complex_key) { + if (fypc->map.complex_key_complete) + fy_document_destroy(fypc->map.complex_key); + fypc->map.complex_key = NULL; + } else { + fy_token_unref(fypc->map.scalar.tag); + fy_token_unref(fypc->map.scalar.key); + fypc->map.scalar.tag = NULL; + fypc->map.scalar.key = NULL; + } + } + fypc->map.root = true; + fypc->map.has_key = false; + fypc->map.await_key = true; + fypc->map.is_complex_key = false; + fypc->map.accumulating_complex_key = false; + fypc->map.complex_key_complete = false; + break; + + case FYPCT_SEQ: + fypc->seq.idx = -1; + break; + } +} + +void fy_path_component_cleanup(struct fy_path_component *fypc) +{ + if (!fypc) + return; + + fy_path_component_clear_state(fypc); + fypc->type = FYPCT_NONE; +} + +void fy_path_component_free(struct fy_path_component *fypc) +{ + if (!fypc) + return; + + fy_path_component_cleanup(fypc); + free(fypc); +} + +void fy_path_component_destroy(struct fy_path_component *fypc) +{ + if (!fypc) + return; + + fy_path_component_cleanup(fypc); + fy_path_component_free(fypc); +} + +void fy_path_component_recycle(struct fy_path *fypp, struct fy_path_component *fypc) +{ + if (!fypc) + return; + + fy_path_component_cleanup(fypc); + + if (!fypp) + fy_path_component_free(fypc); + else + fy_path_component_list_push(&fypp->recycled_component, fypc); +} + +struct fy_path_component *fy_path_component_create_mapping(struct fy_path *fypp) +{ + struct fy_path_component *fypc; + + if (!fypp) + return NULL; + + fypc = fy_path_component_alloc(fypp); + if (!fypc) + return NULL; + + fypc->type = FYPCT_MAP; + + fypc->map.root = true; + fypc->map.await_key = true; + fypc->map.is_complex_key = false; + fypc->map.accumulating_complex_key = false; + fypc->map.complex_key_complete = false; + + return fypc; +} + +struct fy_path_component *fy_path_component_create_sequence(struct fy_path *fypp) +{ + struct fy_path_component *fypc; + + if (!fypp) + return NULL; + + fypc = fy_path_component_alloc(fypp); + if (!fypc) + return NULL; + + fypc->type = FYPCT_SEQ; + + fypc->seq.idx = -1; + + return fypc; +} + +bool fy_path_component_is_mapping(struct fy_path_component *fypc) +{ + return fypc && fypc->type == FYPCT_MAP; +} + +int fy_path_component_sequence_get_index(struct fy_path_component *fypc) +{ + return fypc && fypc->type == FYPCT_SEQ ? fypc->seq.idx : -1; +} + +struct fy_token *fy_path_component_mapping_get_scalar_key(struct fy_path_component *fypc) +{ + return fypc && fypc->type == FYPCT_MAP && + fypc->map.has_key && !fypc->map.is_complex_key ? fypc->map.scalar.key : NULL; +} + +struct fy_token *fy_path_component_mapping_get_scalar_key_tag(struct fy_path_component *fypc) +{ + return fypc && fypc->type == FYPCT_MAP && + fypc->map.has_key && !fypc->map.is_complex_key ? fypc->map.scalar.tag : NULL; +} + +struct fy_document *fy_path_component_mapping_get_complex_key(struct fy_path_component *fypc) +{ + return fypc && fypc->type == FYPCT_MAP && + fypc->map.has_key && fypc->map.is_complex_key ? fypc->map.complex_key : NULL; +} + +bool fy_path_component_is_sequence(struct fy_path_component *fypc) +{ + return fypc && fypc->type == FYPCT_SEQ; +} + +static int fy_path_component_get_text_internal(struct fy_emit_accum *ea, struct fy_path_component *fypc) +{ + char *doctxt; + const char *text; + size_t len; + + switch (fypc->type) { + case FYPCT_NONE: + FY_IMPOSSIBLE_ABORT(); + + case FYPCT_MAP: + + /* we don't handle transitionals */ + if (!fypc->map.has_key || fypc->map.await_key || fypc->map.root) + return -1; + + if (!fypc->map.is_complex_key && fypc->map.scalar.key) { + text = fy_token_get_text(fypc->map.scalar.key, &len); + if (!text) + return -1; + + if (fypc->map.scalar.key->type == FYTT_ALIAS) + fy_emit_accum_utf8_put_raw(ea, '*'); + fy_emit_accum_utf8_write_raw(ea, text, len); + + } else if (fypc->map.complex_key) { + /* complex key */ + doctxt = fy_emit_document_to_string(fypc->map.complex_key, + FYECF_WIDTH_INF | FYECF_INDENT_DEFAULT | + FYECF_MODE_FLOW_ONELINE | FYECF_NO_ENDING_NEWLINE); + fy_emit_accum_utf8_write_raw(ea, doctxt, strlen(doctxt)); + free(doctxt); + } + break; + + case FYPCT_SEQ: + + /* not started filling yet */ + if (fypc->seq.idx < 0) + return -1; + + fy_emit_accum_utf8_printf_raw(ea, "%d", fypc->seq.idx); + break; + } + + return 0; +} + +static int fy_path_get_text_internal(struct fy_emit_accum *ea, struct fy_path *fypp) +{ + struct fy_path_component *fypc; + struct fy_document *fyd; + char *doctxt; + const char *text; + size_t len; + bool local_key = false; + int rc, count; + + if (fypp->parent) { + rc = fy_path_get_text_internal(ea, fypp->parent); + assert(!rc); + if (rc) + return -1; + } + + /* OK, we have to iterate and rebuild the paths */ + for (fypc = fy_path_component_list_head(&fypp->components), count = 0; fypc; + fypc = fy_path_component_next(&fypp->components, fypc), count++) { + + fy_emit_accum_utf8_put_raw(ea, '/'); + + switch (fypc->type) { + case FYPCT_NONE: + FY_IMPOSSIBLE_ABORT(); + + case FYPCT_MAP: + + if (!fypc->map.has_key || fypc->map.root) + break; + + /* key reference ? wrap in .key(X)*/ + local_key = false; + if (fypc->map.await_key) + local_key = true; + + if (local_key) + fy_emit_accum_utf8_write_raw(ea, ".key(", 5); + + if (!fypc->map.is_complex_key) { + + if (fypc->map.scalar.key) { + text = fy_token_get_text(fypc->map.scalar.key, &len); + assert(text); + if (!text) + return -1; + if (fypc->map.scalar.key->type == FYTT_ALIAS) + fy_emit_accum_utf8_put_raw(ea, '*'); + fy_emit_accum_utf8_write_raw(ea, text, len); + } else { + fy_emit_accum_utf8_write_raw(ea, ".null()", 7); + } + } else { + if (fypc->map.complex_key) + fyd = fypc->map.complex_key; + else + fyd = fy_document_builder_peek_document(fypp->fydb); + + /* complex key */ + if (fyd) { + doctxt = fy_emit_document_to_string(fyd, + FYECF_WIDTH_INF | FYECF_INDENT_DEFAULT | + FYECF_MODE_FLOW_ONELINE | FYECF_NO_ENDING_NEWLINE); + } else + doctxt = NULL; + + if (doctxt) { + fy_emit_accum_utf8_write_raw(ea, doctxt, strlen(doctxt)); + free(doctxt); + } else { + fy_emit_accum_utf8_write_raw(ea, "", 3); + } + } + + if (local_key) + fy_emit_accum_utf8_put_raw(ea, ')'); + + break; + + case FYPCT_SEQ: + + /* not started filling yet */ + if (fypc->seq.idx < 0) + break; + + fy_emit_accum_utf8_printf_raw(ea, "%d", fypc->seq.idx); + break; + } + } + + return 0; +} + +char *fy_path_get_text(struct fy_path *fypp) +{ + struct fy_emit_accum ea; /* use an emit accumulator */ + char *path = NULL; + size_t len; + int rc; + + /* no inplace buffer; we will need the malloc'ed contents anyway */ + fy_emit_accum_init(&ea, NULL, 0, 0, fylb_cr_nl); + + fy_emit_accum_start(&ea, 0, fylb_cr_nl); + + rc = fy_path_get_text_internal(&ea, fypp); + if (rc) + goto err_out; + + if (fy_emit_accum_empty(&ea)) + fy_emit_accum_utf8_printf_raw(&ea, "/"); + + fy_emit_accum_make_0_terminated(&ea); + + path = fy_emit_accum_steal(&ea, &len); + +err_out: + fy_emit_accum_cleanup(&ea); + + return path; +} + +char *fy_path_component_get_text(struct fy_path_component *fypc) +{ + struct fy_emit_accum ea; /* use an emit accumulator */ + char *text = NULL; + size_t len; + int rc; + + /* no inplace buffer; we will need the malloc'ed contents anyway */ + fy_emit_accum_init(&ea, NULL, 0, 0, fylb_cr_nl); + + fy_emit_accum_start(&ea, 0, fylb_cr_nl); + + rc = fy_path_component_get_text_internal(&ea, fypc); + if (rc) + goto err_out; + + fy_emit_accum_make_0_terminated(&ea); + + text = fy_emit_accum_steal(&ea, &len); + +err_out: + fy_emit_accum_cleanup(&ea); + + return text; +} + +int fy_path_depth(struct fy_path *fypp) +{ + struct fy_path_component *fypc; + int depth; + + if (!fypp) + return 0; + + depth = fy_path_depth(fypp->parent); + for (fypc = fy_path_component_list_head(&fypp->components); fypc; + fypc = fy_path_component_next(&fypp->components, fypc)) { + + depth++; + } + + return depth; +} + +struct fy_path *fy_path_parent(struct fy_path *fypp) +{ + if (!fypp) + return NULL; + return fypp->parent; +} + +struct fy_path_component * +fy_path_last_component(struct fy_path *fypp) +{ + return fy_path_component_list_tail(&fypp->components); +} + +struct fy_path_component * +fy_path_last_not_collection_root_component(struct fy_path *fypp) +{ + struct fy_path_component *fypc_last; + + fypc_last = fy_path_component_list_tail(&fypp->components); + if (!fypc_last) + return NULL; + + if (!fy_path_component_is_collection_root(fypc_last)) + return fypc_last; + + fypc_last = fy_path_component_prev(&fypp->components, fypc_last); + if (fypc_last) + return fypc_last; + + if (fypp->parent) + return fy_path_component_list_tail(&fypp->parent->components); + + return NULL; +} + +bool fy_path_in_root(struct fy_path *fypp) +{ + struct fy_path_component *fypc_last; + + if (!fypp) + return true; + + fypc_last = fy_path_last_not_collection_root_component(fypp); + return fypc_last == NULL; +} + + +bool fy_path_in_mapping(struct fy_path *fypp) +{ + struct fy_path_component *fypc_last; + + if (!fypp) + return false; + + fypc_last = fy_path_last_not_collection_root_component(fypp); + if (!fypc_last) + return false; + + return fypc_last->type == FYPCT_MAP; +} + +bool fy_path_in_sequence(struct fy_path *fypp) +{ + struct fy_path_component *fypc_last; + + if (!fypp) + return false; + + fypc_last = fy_path_last_not_collection_root_component(fypp); + if (!fypc_last) + return false; + + return fypc_last->type == FYPCT_SEQ; +} + +bool fy_path_in_mapping_key(struct fy_path *fypp) +{ + struct fy_path_component *fypc_last; + + if (!fypp) + return false; + + fypc_last = fy_path_last_not_collection_root_component(fypp); + if (!fypc_last) + return false; + + return fypc_last->type == FYPCT_MAP && fypc_last->map.await_key; +} + +bool fy_path_in_mapping_value(struct fy_path *fypp) +{ + struct fy_path_component *fypc_last; + + if (!fypp) + return false; + + fypc_last = fy_path_last_not_collection_root_component(fypp); + if (!fypc_last) + return false; + + return fypc_last->type == FYPCT_MAP && !fypc_last->map.await_key; +} + +bool fy_path_in_collection_root(struct fy_path *fypp) +{ + struct fy_path_component *fypc_last; + + if (!fypp) + return false; + + fypc_last = fy_path_component_list_tail(&fypp->components); + if (!fypc_last) + return false; + + return fy_path_component_is_collection_root(fypc_last); +} + +void *fy_path_get_root_user_data(struct fy_path *fypp) +{ + if (!fypp) + return NULL; + + if (!fypp->parent) + return fypp->user_data; + + return fy_path_get_root_user_data(fypp->parent); +} + +void fy_path_set_root_user_data(struct fy_path *fypp, void *data) +{ + if (!fypp) + return; + + if (!fypp->parent) { + fypp->user_data = data; + return; + } + + fy_path_set_root_user_data(fypp->parent, data); +} + +void *fy_path_component_get_mapping_user_data(struct fy_path_component *fypc) +{ + return fypc && fypc->type == FYPCT_MAP ? fypc->user_data : NULL; +} + +void *fy_path_component_get_mapping_key_user_data(struct fy_path_component *fypc) +{ + return fypc && fypc->type == FYPCT_MAP ? fypc->map.key_user_data : NULL; +} + +void *fy_path_component_get_sequence_user_data(struct fy_path_component *fypc) +{ + return fypc && fypc->type == FYPCT_SEQ ? fypc->user_data : NULL; +} + +void fy_path_component_set_mapping_user_data(struct fy_path_component *fypc, void *data) +{ + if (!fypc || fypc->type != FYPCT_MAP) + return; + + fypc->user_data = data; +} + +void fy_path_component_set_mapping_key_user_data(struct fy_path_component *fypc, void *data) +{ + if (!fypc || fypc->type != FYPCT_MAP) + return; + + fypc->map.key_user_data = data; +} + +void fy_path_component_set_sequence_user_data(struct fy_path_component *fypc, void *data) +{ + if (!fypc || fypc->type != FYPCT_SEQ) + return; + + fypc->user_data = data; +} + +void *fy_path_get_parent_user_data(struct fy_path *fypp) +{ + struct fy_path_component *parent; + + if (fy_path_in_root(fypp)) + return fy_path_get_root_user_data(fypp); + + parent = fy_path_last_not_collection_root_component(fypp); + if (fy_path_in_sequence(fypp)) + return fy_path_component_get_sequence_user_data(parent); + else + return fy_path_component_get_mapping_user_data(parent); +} + +void fy_path_set_parent_user_data(struct fy_path *fypp, void *data) +{ + struct fy_path_component *parent; + + if (fy_path_in_root(fypp)) { + fy_path_set_root_user_data(fypp, data); + } else { + parent = fy_path_last_not_collection_root_component(fypp); + if (fy_path_in_sequence(fypp)) + fy_path_component_set_sequence_user_data(parent, data); + else + fy_path_component_set_mapping_user_data(parent, data); + } +} + +void *fy_path_get_last_user_data(struct fy_path *fypp) +{ + struct fy_path_component *last; + + last = fy_path_last_component(fypp); + if (!last) + return NULL; + + if (last->type == FYPCT_SEQ) + return fy_path_component_get_sequence_user_data(last); + else + return fy_path_component_get_mapping_user_data(last); +} + +void fy_path_set_last_user_data(struct fy_path *fypp, void *data) +{ + struct fy_path_component *last; + + last = fy_path_last_component(fypp); + if (!last) + return; + + if (last->type == FYPCT_SEQ) + fy_path_component_set_sequence_user_data(last, data); + else + fy_path_component_set_mapping_user_data(last, data); +} diff --git a/src/lib/fy-path.h b/src/lib/fy-path.h new file mode 100644 index 0000000..22dde9c --- /dev/null +++ b/src/lib/fy-path.h @@ -0,0 +1,114 @@ +/* + * fy-path.h - YAML parser private path definitions + * + * Copyright (c) 2021 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_PATH_H +#define FY_PATH_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include + +#include "fy-list.h" +#include "fy-typelist.h" + +#include "fy-emit-accum.h" + +FY_TYPE_FWD_DECL_LIST(path_component); + +enum fy_path_component_type { + FYPCT_NONE, /* not yet instantiated */ + FYPCT_MAP, /* it's a mapping */ + FYPCT_SEQ, /* it's a sequence */ +}; + +/* fwd declaration */ +struct fy_document; +struct fy_document_builder; + +#define FY_PATH_MAPPING_SHORT_KEY 32 + +struct fy_path_mapping_state { + bool root : 1; /* no keys, values yet */ + bool await_key : 1; + bool accumulating_complex_key : 1; + bool has_key : 1; /* has a key */ + bool is_complex_key : 1; + bool complex_key_complete : 1; + union { + struct { + struct fy_token *tag; + struct fy_token *key; + } scalar; + struct fy_document *complex_key; + }; + void *key_user_data; +}; + +struct fy_path_sequence_state { + int idx; +}; + +struct fy_path_component { + struct list_head node; + enum fy_path_component_type type; + union { + struct fy_path_mapping_state map; + struct fy_path_sequence_state seq; + }; + void *user_data; +}; +FY_TYPE_DECL_LIST(path_component); + +static inline bool +fy_path_component_is_collection_root(struct fy_path_component *fypc) +{ + if (!fypc) + return false; + + switch (fypc->type) { + case FYPCT_NONE: + break; + case FYPCT_SEQ: + return fypc->seq.idx < 0; + case FYPCT_MAP: + return fypc->map.root; + } + + return false; +} + +FY_TYPE_FWD_DECL_LIST(path); +struct fy_path { + struct list_head node; + struct fy_path_component_list recycled_component; + struct fy_path_component_list components; + struct fy_document_builder *fydb; /* for complex keys */ + struct fy_path *parent; /* when we have a parent */ + void *user_data; +}; +FY_TYPE_DECL_LIST(path); + +struct fy_path *fy_path_create(void); +void fy_path_destroy(struct fy_path *fypp); + +void fy_path_reset(struct fy_path *fypp); + +struct fy_path_component *fy_path_component_alloc(struct fy_path *fypp); +void fy_path_component_cleanup(struct fy_path_component *fypc); +void fy_path_component_free(struct fy_path_component *fypc); +void fy_path_component_destroy(struct fy_path_component *fypc); +void fy_path_component_recycle(struct fy_path *fypp, struct fy_path_component *fypc); +void fy_path_component_clear_state(struct fy_path_component *fypc); + +struct fy_path_component *fy_path_component_create_mapping(struct fy_path *fypp); +struct fy_path_component *fy_path_component_create_sequence(struct fy_path *fypp); + +#endif diff --git a/src/lib/fy-token.c b/src/lib/fy-token.c new file mode 100644 index 0000000..58f8315 --- /dev/null +++ b/src/lib/fy-token.c @@ -0,0 +1,2054 @@ +/* + * fy-token.c - YAML token methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" + +#include "fy-ctype.h" +#include "fy-utf8.h" +#include "fy-emit-accum.h" + +#include "fy-walk.h" + +#include "fy-token.h" + +enum fy_scalar_style fy_token_scalar_style(struct fy_token *fyt) +{ + return fy_token_scalar_style_inline(fyt); +} + +enum fy_token_type fy_token_get_type(struct fy_token *fyt) +{ + return fy_token_get_type_inline(fyt); +} + +void fy_token_clean_rl(struct fy_token_list *fytl, struct fy_token *fyt) +{ + int i; + + if (!fyt) + return; + + /* release reference */ + fy_input_unref(fyt->handle.fyi); + fyt->handle.fyi = NULL; + + /* release comment references */ + if (fyt->comment) { + for (i = 0; i < fycp_max; i++) + fy_input_unref(fyt->comment[i].fyi); + free(fyt->comment); + fyt->comment = NULL; + } + + switch (fyt->type) { + case FYTT_TAG: + fy_token_unref(fyt->tag.fyt_td); + fyt->tag.fyt_td = NULL; + if (fyt->tag.handle0) { + free(fyt->tag.handle0); + fyt->tag.handle0 = NULL; + } + if (fyt->tag.suffix0) { + free(fyt->tag.suffix0); + fyt->tag.suffix0 = NULL; + } + if (fyt->tag.short0) { + free(fyt->tag.short0); + fyt->tag.short0 = NULL; + } + break; + + case FYTT_TAG_DIRECTIVE: + if (fyt->tag_directive.prefix0) { + free(fyt->tag_directive.prefix0); + fyt->tag_directive.prefix0 = NULL; + } + if (fyt->tag_directive.handle0) { + free(fyt->tag_directive.handle0); + fyt->tag_directive.handle0 = NULL; + } + break; + + case FYTT_PE_MAP_KEY: + fy_document_destroy(fyt->map_key.fyd); + fyt->map_key.fyd = NULL; + break; + + case FYTT_SCALAR: + if (fyt->scalar.path_key_storage) { + free(fyt->scalar.path_key_storage); + fyt->scalar.path_key_storage = NULL; + } + break; + + case FYTT_ALIAS: + if (fyt->alias.expr) { + fy_path_expr_free(fyt->alias.expr); + fyt->alias.expr = NULL; + } + break; + + default: + break; + } + + if (fyt->text0) { + free(fyt->text0); + fyt->text0 = NULL; + } + + fyt->type = FYTT_NONE; + memset(&fyt->analysis, 0, sizeof(fyt->analysis)); + fyt->text_len = 0; + fyt->text = NULL; +} + +void fy_token_list_unref_all_rl(struct fy_token_list *fytl, struct fy_token_list *fytl_tofree) +{ + struct fy_token *fyt; + + while ((fyt = fy_token_list_pop(fytl_tofree)) != NULL) + fy_token_unref_rl(fytl, fyt); +} + +static bool fy_token_text_needs_rebuild(struct fy_token *fyt) +{ + const struct fy_atom *fya; + + if (!fy_token_text_is_direct(fyt)) + return false; + + fya = fy_token_atom(fyt); + if (!fya || !fya->fyi) + return false; + + return fya->fyi_generation != fya->fyi->generation; +} + +static int fy_tag_token_format_internal(const struct fy_token *fyt, void *out, size_t *outszp) +{ + char *o = NULL, *oe = NULL; + size_t outsz; + const char *handle, *suffix; + size_t handle_size, suffix_size; + int len, code_length, rlen; + uint8_t code[4]; + const char *t, *s, *e; + + if (!fyt || fyt->type != FYTT_TAG) + return 0; + + if (out && *outszp <= 0) + return 0; + + if (out) { + outsz = *outszp; + o = out; + oe = out + outsz; + } + + if (!fyt->tag.fyt_td) + return -1; + + handle = fy_tag_directive_token_prefix(fyt->tag.fyt_td, &handle_size); + if (!handle) + return -1; + + suffix = fy_atom_data(&fyt->handle) + fyt->tag.skip + fyt->tag.handle_length; + suffix_size = fyt->tag.suffix_length; + +#define O_CPY(_src, _len) \ + do { \ + int _l = (_len); \ + if (o && _l) { \ + int _cl = _l; \ + if (_cl > (oe - o)) \ + _cl = oe - o; \ + memcpy(o, (_src), _cl); \ + o += _cl; \ + } \ + len += _l; \ + } while(0) + + len = 0; + O_CPY(handle, handle_size); + + /* escape suffix as a URI */ + s = suffix; + e = s + suffix_size; + while (s < e) { + /* find next escape */ + t = memchr(s, '%', e - s); + rlen = (t ? t : e) - s; + O_CPY(s, rlen); + + /* end of string */ + if (!t) + break; + s = t; + + code_length = sizeof(code); + t = fy_uri_esc(s, e - s, code, &code_length); + if (!t) + break; + + /* output escaped utf8 */ + O_CPY(code, code_length); + s = t; + } + +#undef O_CPY + return len; + +} + +int fy_tag_token_format_text_length(const struct fy_token *fyt) +{ + return fy_tag_token_format_internal(fyt, NULL, NULL); +} + +const char *fy_tag_token_format_text(const struct fy_token *fyt, char *buf, size_t maxsz) +{ + fy_tag_token_format_internal(fyt, buf, &maxsz); + return buf; +} + +static int fy_tag_directive_token_format_internal(const struct fy_token *fyt, + void *out, size_t *outszp) +{ + char *o = NULL, *oe = NULL; + size_t outsz; + int len; + const char *handle, *prefix; + size_t handle_size, prefix_size; + + if (!fyt || fyt->type != FYTT_TAG_DIRECTIVE) + return 0; + + if (out && *outszp <= 0) + return 0; + + if (out) { + outsz = *outszp; + o = out; + oe = out + outsz; + } + +#define O_CPY(_src, _len) \ + do { \ + int _l = (_len); \ + if (o && _l) { \ + int _cl = _l; \ + if (_cl > (oe - o)) \ + _cl = oe - o; \ + memcpy(o, (_src), _cl); \ + o += _cl; \ + } \ + len += _l; \ + } while(0) + + len = 0; + + handle = fy_atom_data(&fyt->handle); + handle_size = fy_atom_size(&fyt->handle); + + prefix = handle + handle_size - fyt->tag_directive.uri_length; + prefix_size = fyt->tag_directive.uri_length; + handle_size = fyt->tag_directive.tag_length; + + if (handle_size) + O_CPY(handle, handle_size); + else + O_CPY("!<", 2); + O_CPY(prefix, prefix_size); + if (!handle_size) + O_CPY(">", 1); + +#undef O_CPY + return len; + +} + +int fy_tag_directive_token_format_text_length(const struct fy_token *fyt) +{ + return fy_tag_directive_token_format_internal(fyt, NULL, NULL); +} + +const char *fy_tag_directive_token_format_text(const struct fy_token *fyt, char *buf, size_t maxsz) +{ + fy_tag_directive_token_format_internal(fyt, buf, &maxsz); + return buf; +} + +const char *fy_tag_directive_token_prefix(struct fy_token *fyt, size_t *lenp) +{ + const char *ptr; + size_t len; + + if (!fyt || fyt->type != FYTT_TAG_DIRECTIVE) { + *lenp = 0; + return NULL; + } + ptr = fy_atom_data(&fyt->handle); + len = fy_atom_size(&fyt->handle); + ptr = ptr + len - fyt->tag_directive.uri_length; + *lenp = fyt->tag_directive.uri_length; + + return ptr; +} + +const char *fy_tag_directive_token_prefix0(struct fy_token *fyt) +{ + char *text0; + const char *text; + size_t len; + + if (!fyt || fyt->type != FYTT_TAG_DIRECTIVE) + return NULL; + + /* use the cache if it's there (and doesn't need a rebuild) */ + if (fyt->tag_directive.prefix0 && !fy_token_text_needs_rebuild(fyt)) + return fyt->tag_directive.prefix0; + + if (fyt->tag_directive.prefix0) { + free(fyt->tag_directive.prefix0); + fyt->tag_directive.prefix0 = NULL; + } + + text = fy_tag_directive_token_prefix(fyt, &len); + if (!text) + return NULL; + + text0 = malloc(len + 1); + if (!text0) + return NULL; + memcpy(text0, text, len); + text0[len] = '\0'; + + fyt->tag_directive.prefix0 = text0; + + return fyt->tag_directive.prefix0; +} + +const char *fy_tag_directive_token_handle(struct fy_token *fyt, size_t *lenp) +{ + const char *ptr; + + if (!fyt || fyt->type != FYTT_TAG_DIRECTIVE) { + *lenp = 0; + return NULL; + } + ptr = fy_atom_data(&fyt->handle); + *lenp = fyt->tag_directive.tag_length; + return ptr; +} + +const char *fy_tag_directive_token_handle0(struct fy_token *fyt) +{ + char *text0; + const char *text; + size_t len; + + if (!fyt || fyt->type != FYTT_TAG_DIRECTIVE) + return NULL; + + /* use the cache if it's there (and doesn't need a rebuild) */ + if (fyt->tag_directive.handle0 && !fy_token_text_needs_rebuild(fyt)) + return fyt->tag_directive.handle0; + + if (fyt->tag_directive.handle0) { + free(fyt->tag_directive.handle0); + fyt->tag_directive.handle0 = NULL; + } + + text = fy_tag_directive_token_handle(fyt, &len); + if (!text) + return NULL; + + text0 = malloc(len + 1); + if (!text0) + return NULL; + memcpy(text0, text, len); + text0[len] = '\0'; + + fyt->tag_directive.handle0 = text0; + + return fyt->tag_directive.handle0; +} + +struct fy_token *fy_token_vcreate_rl(struct fy_token_list *fytl, enum fy_token_type type, va_list ap) +{ + struct fy_token *fyt = NULL; + struct fy_atom *handle; + struct fy_token *fyt_td; + + if ((unsigned int)type >= FYTT_COUNT) + goto err_out; + + fyt = fy_token_alloc_rl(fytl); + if (!fyt) + goto err_out; + fyt->type = type; + + handle = va_arg(ap, struct fy_atom *); + if (handle) + fyt->handle = *handle; + else + fy_atom_reset(&fyt->handle); + + switch (fyt->type) { + case FYTT_TAG_DIRECTIVE: + fyt->tag_directive.tag_length = va_arg(ap, unsigned int); + fyt->tag_directive.uri_length = va_arg(ap, unsigned int); + fyt->tag_directive.is_default = va_arg(ap, int) ? true : false; + fyt->tag_directive.prefix0 = NULL; + fyt->tag_directive.handle0 = NULL; + break; + case FYTT_SCALAR: + fyt->scalar.style = va_arg(ap, enum fy_scalar_style); + if (fyt->scalar.style != FYSS_ANY && (unsigned int)fyt->scalar.style >= FYSS_MAX) + goto err_out; + fyt->scalar.path_key = NULL; + fyt->scalar.path_key_len = 0; + fyt->scalar.path_key_storage = NULL; + fyt->scalar.is_null = false; /* by default the scalar is not NULL */ + break; + case FYTT_TAG: + fyt->tag.skip = va_arg(ap, unsigned int); + fyt->tag.handle_length = va_arg(ap, unsigned int); + fyt->tag.suffix_length = va_arg(ap, unsigned int); + + fyt_td = va_arg(ap, struct fy_token *); + if (!fyt_td) + goto err_out; + fyt->tag.fyt_td = fy_token_ref(fyt_td); + fyt->tag.handle0 = NULL; + fyt->tag.suffix0 = NULL; + fyt->tag.short0 = NULL; + break; + + case FYTT_VERSION_DIRECTIVE: + fyt->version_directive.vers = *va_arg(ap, struct fy_version *); + break; + + case FYTT_ALIAS: + fyt->alias.expr = va_arg(ap, struct fy_path_expr *); + break; + + case FYTT_KEY: + fyt->key.flow_level = va_arg(ap, int); + break; + + case FYTT_PE_MAP_KEY: + fyt->map_key.fyd = va_arg(ap, struct fy_document *); + break; + + case FYTT_PE_SEQ_INDEX: + fyt->seq_index.index = va_arg(ap, int); + break; + + case FYTT_PE_SEQ_SLICE: + fyt->seq_slice.start_index = va_arg(ap, int); + fyt->seq_slice.end_index = va_arg(ap, int); + break; + + case FYTT_NONE: + goto err_out; + + default: + break; + } + + if (fyt->handle.fyi) + fy_input_ref(fyt->handle.fyi); + + return fyt; + +err_out: + fy_token_unref(fyt); + + return NULL; +} + +struct fy_token *fy_token_create_rl(struct fy_token_list *fytl, enum fy_token_type type, ...) +{ + struct fy_token *fyt; + va_list ap; + + va_start(ap, type); + fyt = fy_token_vcreate_rl(fytl, type, ap); + va_end(ap); + + return fyt; +} + +struct fy_token *fy_token_vcreate(enum fy_token_type type, va_list ap) +{ + return fy_token_vcreate_rl(NULL, type, ap); +} + +struct fy_token *fy_token_create(enum fy_token_type type, ...) +{ + struct fy_token *fyt; + va_list ap; + + va_start(ap, type); + fyt = fy_token_vcreate_rl(NULL, type, ap); + va_end(ap); + + return fyt; +} + +struct fy_token *fy_parse_token_create(struct fy_parser *fyp, enum fy_token_type type, ...) +{ + struct fy_token *fyt; + va_list ap; + + if (!fyp) + return NULL; + + va_start(ap, type); + fyt = fy_token_vcreate_rl(fyp->recycled_token_list, type, ap); + va_end(ap); + + return fyt; +} + +int fy_token_format_text_length(struct fy_token *fyt) +{ + int length; + + if (!fyt) + return 0; + + switch (fyt->type) { + + case FYTT_TAG: + return fy_tag_token_format_text_length(fyt); + + case FYTT_TAG_DIRECTIVE: + return fy_tag_directive_token_format_text_length(fyt); + + default: + break; + } + + length = fy_atom_format_text_length(&fyt->handle); + + return length; +} + +const char *fy_token_format_text(struct fy_token *fyt, char *buf, size_t maxsz) +{ + const char *str; + + if (maxsz == 0) + return buf; + + if (!fyt) { + if (maxsz > 0) + buf[0] = '\0'; + return buf; + } + + switch (fyt->type) { + + case FYTT_TAG: + return fy_tag_token_format_text(fyt, buf, maxsz); + + case FYTT_TAG_DIRECTIVE: + return fy_tag_directive_token_format_text(fyt, buf, maxsz); + + default: + break; + } + + str = fy_atom_format_text(&fyt->handle, buf, maxsz); + + return str; +} + +int fy_token_format_utf8_length(struct fy_token *fyt) +{ + const char *str; + size_t len; + + if (!fyt) + return 0; + + switch (fyt->type) { + + case FYTT_TAG: + case FYTT_TAG_DIRECTIVE: + str = fy_token_get_text(fyt, &len); + if (!str) + return 0; + return fy_utf8_count(str, len); + + default: + break; + } + + return fy_atom_format_utf8_length(&fyt->handle); +} + + +struct fy_atom *fy_token_atom(struct fy_token *fyt) +{ + return fyt ? &fyt->handle : NULL; +} + +const struct fy_mark *fy_token_start_mark(struct fy_token *fyt) +{ + const struct fy_atom *atom; + + atom = fy_token_atom(fyt); + if (atom) + return &atom->start_mark; + + /* something we don't track */ + return NULL; +} + +const struct fy_mark *fy_token_end_mark(struct fy_token *fyt) +{ + const struct fy_atom *atom; + + atom = fy_token_atom(fyt); + if (atom) + return &atom->end_mark; + + /* something we don't track */ + return NULL; +} + +const struct fy_token_analysis * +fy_token_text_analyze(struct fy_token *fyt) +{ + static const struct fy_token_analysis null_analysis = { + .flags = FYTTAF_CAN_BE_SIMPLE_KEY | FYTTAF_DIRECT_OUTPUT | + FYTTAF_EMPTY | FYTTAF_CAN_BE_DOUBLE_QUOTED | + FYTTAF_ANALYZED, + .maxspan = 0, + .maxcol = 0, + }; + struct fy_atom_iter iter; + enum fy_atom_style style; + int c, cn, cnn, cp, col; + uint8_t col0si, col0ei; /* mask for --- ... at indent 0 */ + int flags, span, maxspan, maxcol; + + if (!fyt) + return &null_analysis; + + if (fyt->analysis.flags & FYTTAF_ANALYZED) + return &fyt->analysis; + + /* only tokens that can generate text */ + if (fyt->type != FYTT_SCALAR && + fyt->type != FYTT_TAG && + fyt->type != FYTT_ANCHOR && + fyt->type != FYTT_ALIAS) { + flags = FYTTAF_NO_TEXT_TOKEN; + + fyt->analysis.flags = flags | FYTTAF_ANALYZED; + fyt->analysis.maxspan = 0; + fyt->analysis.maxcol = 0; + + return &fyt->analysis; + } + + flags = FYTTAF_TEXT_TOKEN; + + style = fy_token_atom_style(fyt); + + /* hardwired and fast for regular plain scalars */ + if (style == FYAS_PLAIN && fyt->handle.storage_hint_valid && + fyt->handle.direct_output && !fyt->handle.high_ascii && + !fyt->handle.has_lb && !fyt->handle.has_ws && !fyt->handle.empty) { + + flags |= FYTTAF_DIRECT_OUTPUT; + + maxcol = fyt->handle.storage_hint; + maxspan = maxcol - 1; + flags |= + FYTTAF_DIRECT_OUTPUT | + FYTTAF_CAN_BE_SIMPLE_KEY | + FYTTAF_CAN_BE_PLAIN | + FYTTAF_CAN_BE_SINGLE_QUOTED | + FYTTAF_CAN_BE_DOUBLE_QUOTED | + FYTTAF_CAN_BE_LITERAL | + FYTTAF_CAN_BE_PLAIN_FLOW | + FYTTAF_CAN_BE_UNQUOTED_PATH_KEY; + + goto done; + } + + /* can this token be a simple key initial condition */ + if (!fy_atom_style_is_block(style) && style != FYAS_URI) + flags |= FYTTAF_CAN_BE_SIMPLE_KEY; + + /* can this token be directly output initial condition */ + if (!fy_atom_style_is_block(style)) + flags |= FYTTAF_DIRECT_OUTPUT; + + fy_atom_iter_start(&fyt->handle, &iter); + + col = 0; + maxcol = 0; + maxspan = 0; + span = 0; + + /* get first character */ + cn = fy_atom_iter_utf8_get(&iter); + if (cn < 0) { + /* empty? */ + flags |= FYTTAF_EMPTY | FYTTAF_CAN_BE_DOUBLE_QUOTED | FYTTAF_CAN_BE_UNQUOTED_PATH_KEY | FYTTAF_CAN_BE_SIMPLE_KEY; + goto out; + } + + flags |= FYTTAF_CAN_BE_PLAIN | + FYTTAF_CAN_BE_SINGLE_QUOTED | + FYTTAF_CAN_BE_DOUBLE_QUOTED | + FYTTAF_CAN_BE_LITERAL | + FYTTAF_CAN_BE_FOLDED | + FYTTAF_CAN_BE_PLAIN_FLOW | + FYTTAF_CAN_BE_UNQUOTED_PATH_KEY; + + col0si = col0ei = 0; + + /* disable folded right off the bat, it's a pain */ + flags &= ~FYTTAF_CAN_BE_FOLDED; + + /* plain scalars can't start with any indicator (or space/lb) */ + if ((flags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_CAN_BE_PLAIN_FLOW))) { + if (fy_is_start_indicator(cn) || fy_token_is_lb(fyt, cn) || fy_is_ws(cn)) + flags &= ~(FYTTAF_CAN_BE_PLAIN | FYTTAF_CAN_BE_PLAIN_FLOW); + } + + /* plain scalars in flow mode can't start with a flow indicator */ + if ((flags & FYTTAF_CAN_BE_PLAIN_FLOW) && + fy_is_flow_indicator(cn)) + flags &= ~FYTTAF_CAN_BE_PLAIN_FLOW; + + if ((flags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_CAN_BE_PLAIN_FLOW))) { + cnn = fy_atom_iter_utf8_peek(&iter); + if (fy_is_blankz_m(cnn, fy_token_atom_lb_mode(fyt)) && fy_is_indicator_before_space(cn)) + flags &= ~(FYTTAF_CAN_BE_PLAIN | FYTTAF_CAN_BE_PLAIN_FLOW); + } + + /* plain unquoted path keys can only start with [a-zA-Z_] */ + if ((flags & FYTTAF_CAN_BE_UNQUOTED_PATH_KEY) && + !fy_is_first_alpha(cn)) + flags &= ~FYTTAF_CAN_BE_UNQUOTED_PATH_KEY; + + cp = -1; + for (c = cn; c >= 0; cp = c, c = cn) { + + if (col <= 2) { + if (cn == '-') { + col0si |= (uint8_t)1 << col; + if (col0si == 7) + flags |= FYTTAF_HAS_START_IND | FYTTAF_QUOTE_AT_0; + } else if (cn == '.') { + col0ei |= (uint8_t)1 << col; + if (col0ei == 7) + flags |= FYTTAF_HAS_END_IND | FYTTAF_QUOTE_AT_0; + } + } + + /* can be -1 on end */ + cn = fy_atom_iter_utf8_get(&iter); + + /* zero can't be output, only in double quoted mode */ + if (c == 0) { + flags &= ~(FYTTAF_DIRECT_OUTPUT | + FYTTAF_CAN_BE_PLAIN | + FYTTAF_CAN_BE_SINGLE_QUOTED | + FYTTAF_CAN_BE_LITERAL | + FYTTAF_CAN_BE_FOLDED | + FYTTAF_CAN_BE_PLAIN_FLOW | + FYTTAF_CAN_BE_UNQUOTED_PATH_KEY); + flags |= FYTTAF_CAN_BE_DOUBLE_QUOTED; + + } else if (fy_is_ws(c)) { + + flags |= FYTTAF_HAS_WS; + if (fy_is_ws(cn)) + flags |= FYTTAF_HAS_CONSECUTIVE_WS; + + } else if (fy_token_is_lb(fyt, c)) { + + flags |= FYTTAF_HAS_LB; + if (fy_token_is_lb(fyt, cn)) + flags |= FYTTAF_HAS_CONSECUTIVE_LB; + + /* only non linebreaks can be simple keys */ + flags &= ~FYTTAF_CAN_BE_SIMPLE_KEY; + + /* anything with linebreaks, can't be direct */ + flags &= ~FYTTAF_DIRECT_OUTPUT; + } + + if ((flags & FYTTAF_CAN_BE_UNQUOTED_PATH_KEY) && !fy_is_alnum(c)) + flags &= ~FYTTAF_CAN_BE_UNQUOTED_PATH_KEY; + + /* illegal plain combination */ + if ((flags & FYTTAF_CAN_BE_PLAIN) && + ((c == ':' && fy_is_blankz_m(cn, fy_token_atom_lb_mode(fyt))) || + (fy_is_blankz_m(c, fy_token_atom_lb_mode(fyt)) && cn == '#') || + (cp < 0 && c == '#' && cn < 0) || + !fy_is_print(c))) { + flags &= ~(FYTTAF_CAN_BE_PLAIN | + FYTTAF_CAN_BE_PLAIN_FLOW); + } + + /* illegal plain flow combination */ + if ((flags & FYTTAF_CAN_BE_PLAIN_FLOW) && + (fy_is_flow_indicator(c) || (c == ':' && fy_is_flow_indicator(cn)))) + flags &= ~FYTTAF_CAN_BE_PLAIN_FLOW; + + /* non printable characters, turn off these styles */ + if (!fy_is_print(c)) { + flags &= ~(FYTTAF_CAN_BE_SINGLE_QUOTED | FYTTAF_CAN_BE_LITERAL | + FYTTAF_CAN_BE_FOLDED); + flags |= FYTTAF_HAS_NON_PRINT; + } + + /* if there's an escape, it can't be direct */ + if ((flags & FYTTAF_DIRECT_OUTPUT) && + ((style == FYAS_URI && c == '%') || + (style == FYAS_SINGLE_QUOTED && c == '\'') || + (style == FYAS_DOUBLE_QUOTED && c == '\\'))) + flags &= ~FYTTAF_DIRECT_OUTPUT; + + if (cn < 0 || !c || fy_token_is_lb(fyt, c) || fy_is_ws(c)) { + if (span > maxspan) + maxspan = span; + span = 0; + } else + span++; + + if (fy_token_is_lb(fyt, c)) { + if (col > maxcol) + maxcol = col; + col = 0; + col0si = col0ei = 0; + } else + col++; + + if (fy_is_any_lb(c)) + flags |= FYTTAF_HAS_ANY_LB; + + /* last character */ + if (cn < 0) { + /* if ends with whitespace or linebreak, or : can't be plain */ + if (fy_is_ws(c) || fy_token_is_lb(fyt, c) || c == ':') { + flags &= ~(FYTTAF_CAN_BE_PLAIN | + FYTTAF_CAN_BE_PLAIN_FLOW); + if (c == ':') + flags |= FYTTAF_ENDS_WITH_COLON; + } + } + } + + if (col > maxcol) + maxcol = col; + if (span > maxspan) + maxspan = span; +out: + fy_atom_iter_finish(&iter); + +done: + fyt->analysis.flags = flags | FYTTAF_ANALYZED; + fyt->analysis.maxspan = maxspan; + fyt->analysis.maxcol = maxcol; + return &fyt->analysis; +} + +const char *fy_tag_token_get_directive_handle(struct fy_token *fyt, size_t *td_handle_sizep) +{ + if (!fyt || fyt->type != FYTT_TAG || !fyt->tag.fyt_td) + return NULL; + + return fy_tag_directive_token_handle(fyt->tag.fyt_td, td_handle_sizep); +} + +const char *fy_tag_token_get_directive_prefix(struct fy_token *fyt, size_t *td_prefix_sizep) +{ + if (!fyt || fyt->type != FYTT_TAG || !fyt->tag.fyt_td) + return NULL; + + return fy_tag_directive_token_prefix(fyt->tag.fyt_td, td_prefix_sizep); +} + +const char *fy_token_get_direct_output(struct fy_token *fyt, size_t *sizep) +{ + const struct fy_atom *fya; + + fya = fy_token_atom(fyt); + if (!fya || !fya->direct_output || + (fyt->type == FYTT_TAG || fyt->type == FYTT_TAG_DIRECTIVE) ) { + *sizep = 0; + return NULL; + } + *sizep = fy_atom_size(fya); + return fy_atom_data(fya); +} + +const char *fy_token_get_direct_simple_output(struct fy_token *fyt, size_t *sizep) +{ + const struct fy_atom *handle; + + handle = fy_token_atom(fyt); + + if (!(handle->style == FYAS_PLAIN && handle->storage_hint_valid && + handle->direct_output && !handle->high_ascii && + !handle->has_lb && !handle->has_ws && !handle->empty)) { + *sizep = 0; + return NULL; + } + + *sizep = fy_atom_size(handle); + return fy_atom_data(handle); +} + +const char *fy_tag_token_handle(struct fy_token *fyt, size_t *lenp) +{ + return fy_tag_token_get_directive_handle(fyt, lenp); +} + +const char *fy_tag_token_suffix(struct fy_token *fyt, size_t *lenp) +{ + const char *tag, *prefix, *handle, *suffix; + size_t tag_len, prefix_len, handle_len, suffix_len; + + if (!fyt || fyt->type != FYTT_TAG) { + *lenp = 0; + return NULL; + } + + tag = fy_token_get_text(fyt, &tag_len); + if (!tag) + return NULL; + prefix = fy_tag_token_get_directive_prefix(fyt, &prefix_len); + if (!prefix) + return NULL; + handle = fy_tag_token_handle(fyt, &handle_len); + if (!handle || !handle_len) { + suffix = tag; + suffix_len = tag_len; + } else { + assert(prefix_len <= tag_len); + assert(tag_len >= prefix_len); + suffix = tag + prefix_len; + suffix_len = tag_len - prefix_len; + } + *lenp = suffix_len; + return suffix; +} + +const char *fy_tag_token_handle0(struct fy_token *fyt) +{ + char *text0; + const char *text; + size_t len; + + if (!fyt || fyt->type != FYTT_TAG) + return NULL; + + /* use the cache if it's there (and doesn't need a rebuild) */ + if (fyt->tag.handle0 && !fy_token_text_needs_rebuild(fyt)) + return fyt->tag.handle0; + + if (fyt->tag.handle0) { + free(fyt->tag.handle0); + fyt->tag.handle0 = NULL; + } + + text = fy_tag_token_handle(fyt, &len); + if (!text) + return NULL; + + text0 = malloc(len + 1); + if (!text0) + return NULL; + memcpy(text0, text, len); + text0[len] = '\0'; + + fyt->tag.handle0 = text0; + + return fyt->tag.handle0; +} + +const char *fy_tag_token_suffix0(struct fy_token *fyt) +{ + char *text0; + const char *text; + size_t len; + + if (!fyt || fyt->type != FYTT_TAG) + return NULL; + + /* use the cache if it's there (and doesn't need a rebuild) */ + if (fyt->tag.suffix0 && !fy_token_text_needs_rebuild(fyt)) + return fyt->tag.suffix0; + + if (fyt->tag.suffix0) { + free(fyt->tag.suffix0); + fyt->tag.suffix0 = NULL; + } + + text = fy_tag_token_suffix(fyt, &len); + if (!text) + return NULL; + + text0 = malloc(len + 1); + if (!text0) + return NULL; + memcpy(text0, text, len); + text0[len] = '\0'; + + fyt->tag.suffix0 = text0; + + return fyt->tag.suffix0; +} + +const char *fy_tag_token_short(struct fy_token *fyt, size_t *lenp) +{ + const char *handle, *suffix; + size_t handle_len, suffix_len; + char *text0; + + if (!fyt || fyt->type != FYTT_TAG) + return NULL; + + /* use the cache if it's there (and doesn't need a rebuild) */ + if (fyt->tag.short0 && !fy_token_text_needs_rebuild(fyt)) + return fyt->tag.short0; + + if (fyt->tag.short0) { + free(fyt->tag.short0); + fyt->tag.short0 = NULL; + } + + handle = fy_tag_token_handle(fyt, &handle_len); + if (!handle) + return NULL; + + suffix = fy_tag_token_suffix(fyt, &suffix_len); + if (!suffix) + return NULL; + + text0 = malloc(handle_len + suffix_len + 1); + if (!text0) + return NULL; + memcpy(text0, handle, handle_len); + memcpy(text0 + handle_len, suffix, suffix_len); + text0[handle_len + suffix_len] = '\0'; + + fyt->tag.short0 = text0; + fyt->tag.short_length = handle_len + suffix_len; + + *lenp = fyt->tag.short_length; + + return fyt->tag.short0; +} + +const char *fy_tag_token_short0(struct fy_token *fyt) +{ + const char *text; + size_t len; + + text = fy_tag_token_short(fyt, &len); + if (!text) + return NULL; + return text; +} + +const struct fy_version * fy_version_directive_token_version(struct fy_token *fyt) +{ + if (!fyt || fyt->type != FYTT_VERSION_DIRECTIVE) + return NULL; + return &fyt->version_directive.vers; +} + +static void fy_token_prepare_text(struct fy_token *fyt) +{ + int ret; + + assert(fyt); + + /* get text length of this token */ + ret = fy_token_format_text_length(fyt); + + /* no text on this token? */ + if (ret == -1) { + fyt->text_len = 0; + fyt->text = fyt->text0 = strdup(""); + return; + } + + fyt->text0 = malloc(ret + 1); + if (!fyt->text0) { + fyt->text_len = 0; + fyt->text = fyt->text0 = strdup(""); + return; + } + + fyt->text0[0] = '\0'; + + fyt->text_len = ret; + + fy_token_format_text(fyt, fyt->text0, ret + 1); + fyt->text0[ret] = '\0'; + + fyt->text_len = ret; + fyt->text = fyt->text0; +} + +const char *fy_token_get_text(struct fy_token *fyt, size_t *lenp) +{ + /* return empty */ + if (!fyt) { + *lenp = 0; + return ""; + } + + /* already found something */ + if (fyt->text && !fy_token_text_needs_rebuild(fyt)) { + *lenp = fyt->text_len; + return fyt->text; + } + + /* try direct output first */ + fyt->text = fy_token_get_direct_output(fyt, &fyt->text_len); + if (!fyt->text) + fy_token_prepare_text(fyt); + + *lenp = fyt->text_len; + return fyt->text; +} + +const char *fy_token_get_text0(struct fy_token *fyt) +{ + /* return empty */ + if (!fyt) + return ""; + + /* created text is always zero terminated */ + if (!fyt->text0) + fy_token_prepare_text(fyt); + + return fyt->text0; +} + +size_t fy_token_get_text_length(struct fy_token *fyt) +{ + return fy_token_format_text_length(fyt); +} + +enum comment_out_state { + cos_normal, + cos_lastnl, + cos_lastnlhash, + cos_lastnlhashspc, +}; + +const char *fy_token_get_comment(struct fy_token *fyt, char *buf, size_t maxsz, + enum fy_comment_placement which) +{ + struct fy_atom *handle; + struct fy_atom_iter iter; + const struct fy_iter_chunk *ic; + char *s, *e; + const char *ss, *ee; + int c, w, ret; + enum comment_out_state state; + bool output; + + if (!buf || maxsz == 0 || (unsigned int)which >= fycp_max) + return NULL; + + /* return empty? */ + handle = fy_token_comment_handle(fyt, which, false); + if (!handle || !fy_atom_is_set(handle)) + return NULL; + + /* start expecting # */ + state = cos_lastnl; + + s = buf; + e = s + maxsz; + fy_atom_iter_start(handle, &iter); + ic = NULL; + while ((ic = fy_atom_iter_chunk_next(&iter, ic, &ret)) != NULL) { + ss = ic->str; + ee = ss + ic->len; + + while ((c = fy_utf8_get(ss, ee - ss, &w)) > 0) { + + output = true; + switch (state) { + case cos_normal: + if (fy_is_lb_m(c, handle->lb_mode)) + state = cos_lastnl; + break; + + case cos_lastnl: + if (c == '#') { + state = cos_lastnlhash; + output = false; + break; + } + state = cos_normal; + break; + + case cos_lastnlhash: + if (c == ' ') { + state = cos_lastnlhashspc; + output = false; + break; + } + state = cos_normal; + break; + + case cos_lastnlhashspc: + state = cos_normal; + break; + } + + if (output) { + s = fy_utf8_put(s, (size_t)(e - s), c); + if (!s) + return NULL; + } + + ss += w; + } + } + fy_atom_iter_finish(&iter); + + if (ret != 0 || s >= e) + return NULL; + *s = '\0'; + + return buf; +} + +const char *fy_token_get_scalar_path_key(struct fy_token *fyt, size_t *lenp) +{ + struct fy_atom *atom; + struct fy_atom_iter iter; + struct fy_emit_accum ea; /* use an emit accumulator */ + uint8_t non_utf8[4]; + size_t non_utf8_len, k; + int c, i, w, digit; + const struct fy_token_analysis *ta; + + if (!fyt || fyt->type != FYTT_SCALAR) { + *lenp = 0; + return NULL; + } + + /* was it cached? return */ + if (fyt->scalar.path_key) { + *lenp = fyt->scalar.path_key_len; + return fyt->scalar.path_key; + } + + /* analyze the token */ + ta = fy_token_text_analyze(fyt); + + /* simple one? perfect */ + if ((ta->flags & FYTTAF_CAN_BE_UNQUOTED_PATH_KEY) == FYTTAF_CAN_BE_UNQUOTED_PATH_KEY) { + fyt->scalar.path_key = fy_token_get_text(fyt, &fyt->scalar.path_key_len); + *lenp = fyt->scalar.path_key_len; + return fyt->scalar.path_key; + } + + /* not possible, need to quote (and escape) */ + + /* no atom? i.e. empty */ + atom = fy_token_atom(fyt); + if (!atom) { + fyt->scalar.path_key = ""; + fyt->scalar.path_key_len = 0; + *lenp = 0; + return fyt->scalar.path_key; + } + + /* no inplace buffer; we will need the malloc'ed contents anyway */ + fy_emit_accum_init(&ea, NULL, 0, 0, fylb_cr_nl); + + fy_atom_iter_start(atom, &iter); + fy_emit_accum_start(&ea, 0, fy_token_atom_lb_mode(fyt)); + + /* output in quoted form */ + fy_emit_accum_utf8_put(&ea, '"'); + + for (;;) { + non_utf8_len = sizeof(non_utf8); + c = fy_atom_iter_utf8_quoted_get(&iter, &non_utf8_len, non_utf8); + if (c < 0) + break; + + if (c == 0 && non_utf8_len > 0) { + for (k = 0; k < non_utf8_len; k++) { + c = (int)non_utf8[k] & 0xff; + fy_emit_accum_utf8_put(&ea, '\\'); + fy_emit_accum_utf8_put(&ea, 'x'); + digit = ((unsigned int)c >> 4) & 15; + fy_emit_accum_utf8_put(&ea, + digit <= 9 ? ('0' + digit) : ('A' + digit - 10)); + digit = (unsigned int)c & 15; + fy_emit_accum_utf8_put(&ea, + digit <= 9 ? ('0' + digit) : ('A' + digit - 10)); + } + continue; + } + + if (!fy_is_printq(c) || c == '"' || c == '\\') { + + fy_emit_accum_utf8_put(&ea, '\\'); + + switch (c) { + + /* common YAML & JSON escapes */ + case '\b': + fy_emit_accum_utf8_put(&ea, 'b'); + break; + case '\f': + fy_emit_accum_utf8_put(&ea, 'f'); + break; + case '\n': + fy_emit_accum_utf8_put(&ea, 'n'); + break; + case '\r': + fy_emit_accum_utf8_put(&ea, 'r'); + break; + case '\t': + fy_emit_accum_utf8_put(&ea, 't'); + break; + case '"': + fy_emit_accum_utf8_put(&ea, '"'); + break; + case '\\': + fy_emit_accum_utf8_put(&ea, '\\'); + break; + + /* YAML only escapes */ + case '\0': + fy_emit_accum_utf8_put(&ea, '0'); + break; + case '\a': + fy_emit_accum_utf8_put(&ea, 'a'); + break; + case '\v': + fy_emit_accum_utf8_put(&ea, 'v'); + break; + case '\e': + fy_emit_accum_utf8_put(&ea, 'e'); + break; + case 0x85: + fy_emit_accum_utf8_put(&ea, 'N'); + break; + case 0xa0: + fy_emit_accum_utf8_put(&ea, '_'); + break; + case 0x2028: + fy_emit_accum_utf8_put(&ea, 'L'); + break; + case 0x2029: + fy_emit_accum_utf8_put(&ea, 'P'); + break; + + default: + /* any kind of binary value */ + if ((unsigned int)c <= 0xff) { + fy_emit_accum_utf8_put(&ea, 'x'); + w = 2; + } else if ((unsigned int)c <= 0xffff) { + fy_emit_accum_utf8_put(&ea, 'u'); + w = 4; + } else if ((unsigned int)c <= 0xffffffff) { + fy_emit_accum_utf8_put(&ea, 'U'); + w = 8; + } + + for (i = w - 1; i >= 0; i--) { + digit = ((unsigned int)c >> (i * 4)) & 15; + fy_emit_accum_utf8_put(&ea, + digit <= 9 ? ('0' + digit) : ('A' + digit - 10)); + } + break; + } + + continue; + } + + /* regular character */ + fy_emit_accum_utf8_put(&ea, c); + } + + fy_atom_iter_finish(&iter); + + /* closing quote */ + fy_emit_accum_utf8_put(&ea, '"'); + + fy_emit_accum_make_0_terminated(&ea); + + /* get the output (note it's now NULL terminated) */ + fyt->scalar.path_key_storage = fy_emit_accum_steal(&ea, &fyt->scalar.path_key_len); + fyt->scalar.path_key = fyt->scalar.path_key_storage; + fy_emit_accum_cleanup(&ea); + + *lenp = fyt->scalar.path_key_len; + + return fyt->scalar.path_key; +} + +size_t fy_token_get_scalar_path_key_length(struct fy_token *fyt) +{ + const char *text; + size_t len; + + text = fy_token_get_scalar_path_key(fyt, &len); + if (!text) + return 0; + return len; +} + +const char *fy_token_get_scalar_path_key0(struct fy_token *fyt) +{ + const char *text; + size_t len; + + if (!fyt || fyt->type != FYTT_SCALAR) { + return NULL; + } + + /* storage is \0 terminated */ + if (fyt->scalar.path_key_storage) + return fyt->scalar.path_key_storage; + + text = fyt->scalar.path_key; + len = fyt->scalar.path_key_len; + if (!text) + text = fy_token_get_scalar_path_key(fyt, &len); + + /* something is catastrophically wrong */ + if (!text) + return NULL; + + if (fyt->scalar.path_key_storage) + return fyt->scalar.path_key_storage; + + fyt->scalar.path_key_storage = malloc(len + 1); + if (!fyt->scalar.path_key_storage) + return NULL; + + memcpy(fyt->scalar.path_key_storage, text, len); + fyt->scalar.path_key_storage[len] = '\0'; + + return fyt->scalar.path_key_storage; +} + +unsigned int fy_analyze_scalar_content(const char *data, size_t size, + bool json_mode, enum fy_lb_mode lb_mode, enum fy_flow_ws_mode fws_mode) +{ + const char *s, *e; + int c, lastc, nextc, w, ww, col, break_run; + unsigned int flags; + bool first; + + flags = FYACF_EMPTY | FYACF_BLOCK_PLAIN | FYACF_FLOW_PLAIN | + FYACF_PRINTABLE | FYACF_SINGLE_QUOTED | FYACF_DOUBLE_QUOTED | + FYACF_SIZE0 | FYACF_VALID_ANCHOR; + + s = data; + e = data + size; + + /* if it ends in : can't be a plain */ + if (e > s && e[-1] == ':') { + flags &= ~(FYACF_BLOCK_PLAIN | FYACF_FLOW_PLAIN); + flags |= FYACF_ENDS_WITH_COLON; + } + + col = 0; + first = true; + lastc = -1; + break_run = 0; + while (s < e && (c = fy_utf8_get(s, e - s, &w)) >= 0) { + + flags &= ~FYACF_SIZE0; + + lastc = c; + + if (first) { + if (fy_is_ws(c)) + flags |= FYACF_STARTS_WITH_WS; + else if (fy_is_generic_lb_m(c, lb_mode)) + flags |= FYACF_STARTS_WITH_LB; + /* scalars starting with & or * must be quoted */ + if (c == '&' || c == '*') + flags &= ~(FYACF_BLOCK_PLAIN | FYACF_FLOW_PLAIN); + first = false; + } + nextc = fy_utf8_get(s + w, e - (s + w), &ww); + + /* anything other than white space or linebreak */ + if ((flags & FYACF_EMPTY) && + !fy_is_ws(c) && !fy_is_generic_lb_m(c, lb_mode)) + flags &= ~FYACF_EMPTY; + + if ((flags & FYACF_VALID_ANCHOR) && + (fy_utf8_strchr(",[]{}&*:", c) || fy_is_ws(c) || + fy_is_any_lb(c) || fy_is_unicode_control(c) || + fy_is_unicode_space(c))) + flags &= ~FYACF_VALID_ANCHOR; + + /* linebreak */ + if (fy_is_generic_lb_m(c, lb_mode)) { + flags |= FYACF_LB; + if (!(flags & FYACF_CONSECUTIVE_LB) && + fy_is_generic_lb_m(nextc, lb_mode)) + flags |= FYACF_CONSECUTIVE_LB; + break_run++; + } else + break_run = 0; + + /* white space */ + if (!(flags & FYACF_WS) && fy_is_ws(c)) { + flags |= FYACF_WS; + flags &= ~FYACF_VALID_ANCHOR; + } + + /* anything not printable (or \r, \n) */ + if ((flags & FYACF_PRINTABLE) && + !fy_is_printq(c)) { + flags &= ~FYACF_PRINTABLE; + flags &= ~(FYACF_BLOCK_PLAIN | FYACF_FLOW_PLAIN | + FYACF_SINGLE_QUOTED | FYACF_VALID_ANCHOR); + } + + /* check for document indicators (at column 0) */ + if (!(flags & FYACF_DOC_IND) && + ((col == 0 && (e - s) >= 3 && + (!strncmp(s, "---", 3) || !strncmp(s, "...", 3))))) { + flags |= FYACF_DOC_IND; + flags &= ~(FYACF_BLOCK_PLAIN | FYACF_FLOW_PLAIN | FYACF_VALID_ANCHOR); + } + + /* comment indicator can't be present after a space or lb */ + /* : followed by blank can't be any plain */ + if ((flags & (FYACF_BLOCK_PLAIN | FYACF_FLOW_PLAIN)) && + (((fy_is_blank(c) || fy_is_generic_lb_m(c, lb_mode)) && nextc == '#') || + (c == ':' && fy_is_blankz_m(nextc, lb_mode)))) + flags &= ~(FYACF_BLOCK_PLAIN | FYACF_FLOW_PLAIN); + + /* : followed by flow markers can't be a plain in flow context */ + if ((flags & FYACF_FLOW_PLAIN) && + (fy_utf8_strchr(",[]{}", c) || (c == ':' && fy_utf8_strchr(",[]{}", nextc)))) + flags &= ~FYACF_FLOW_PLAIN; + + if (!(flags & FYACF_JSON_ESCAPE) && !fy_is_json_unescaped(c)) + flags |= FYACF_JSON_ESCAPE; + + if (fy_is_generic_lb_m(c, lb_mode)) + col = 0; + else + col++; + + s += w; + } + + /* this contains arbitrary binany values, mark it as such */ + if (s < e) + return FYACF_DOUBLE_QUOTED; + + if (fy_is_ws(lastc)) + flags |= FYACF_ENDS_WITH_WS; + else if (fy_is_generic_lb_m(lastc, lb_mode)) + flags |= FYACF_ENDS_WITH_LB; + + if (break_run > 1) + flags |= FYACF_TRAILING_LB; + + if ((flags & FYACF_STARTS_WITH_WS) || + (flags & FYACF_STARTS_WITH_LB) || + (flags & FYACF_ENDS_WITH_WS) || + (flags & FYACF_ENDS_WITH_LB)) + flags &= ~FYACF_FLOW_PLAIN; + + return flags; +} + +char *fy_token_debug_text(struct fy_token *fyt) +{ + const char *typetxt; + const char *text; + char *buf; + size_t length; + int wlen; + int rc __FY_DEBUG_UNUSED__; + + if (!fyt || !fy_token_type_is_valid(fyt->type)) { + typetxt = ""; + goto out; + } + typetxt = fy_token_type_txt[fyt->type]; + + /* should never happen really */ + assert(typetxt); + +out: + text = fy_token_get_text(fyt, &length); + + wlen = length > 8 ? 8 : length; + + rc = asprintf(&buf, "%s:%.*s%s", typetxt, wlen, text, wlen < (int)length ? "..." : ""); + assert(rc != -1); + + return buf; +} + +int fy_token_memcmp(struct fy_token *fyt, const void *ptr, size_t len) +{ + const char *value = NULL; + size_t tlen = 0; + + /* special zero length handling */ + if (len == 0 && fyt && fy_token_get_text_length(fyt) == 0) + return 0; + + /* handle NULL cases */ + if (!fyt && (!ptr || !len)) + return 0; + + if (!fyt && (ptr || len)) + return -1; + + if (fyt && (!ptr || !len)) + return 1; + + /* those two are special */ + if (fyt->type == FYTT_TAG || fyt->type == FYTT_TAG_DIRECTIVE) { + value = fy_token_get_text(fyt, &tlen); + if (!value) + return -1; + return tlen == len ? memcmp(value, ptr, tlen) : tlen < len ? -1 : 1; + } + + return fy_atom_memcmp(fy_token_atom(fyt), ptr, len); +} + +int fy_token_strcmp(struct fy_token *fyt, const char *str) +{ + size_t len; + + len = str ? strlen(str) : 0; + + return fy_token_memcmp(fyt, str, len); +} + +int fy_token_cmp(struct fy_token *fyt1, struct fy_token *fyt2) +{ + const char *t1, *t2; + size_t l1, l2, l; + int ret; + bool aoa; /* anchor or alias */ + + /* handles both NULL */ + if (fyt1 == fyt2) + return 0; + + /* fyt1 is null, 2 wins */ + if (!fyt1 && fyt2) + return -1; + + /* fyt2 is null, 1 wins */ + if (fyt1 && !fyt2) + return 1; + + /* special case for comparing anchors and aliases */ + aoa = (fyt1->type == FYTT_ANCHOR || fyt1->type == FYTT_ALIAS) && + (fyt2->type == FYTT_ANCHOR || fyt2->type == FYTT_ALIAS); + + /* tokens with different types can't be equal */ + if (!aoa && fyt1->type != fyt2->type) + return fyt2->type > fyt1->type ? -1 : 1; + + /* special case, these can't use the atom comparisons */ + if (fyt1->type == FYTT_TAG || fyt1->type == FYTT_TAG_DIRECTIVE) { + t1 = fy_token_get_text(fyt1, &l1); + t2 = fy_token_get_text(fyt2, &l2); + l = l1 > l2 ? l2 : l1; + ret = memcmp(t1, t2, l); + if (ret) + return ret; + return l1 == l2 ? 0 : l2 > l1 ? -1 : 1; + } + + /* just pass it to the atom comparison methods */ + return fy_atom_cmp(fy_token_atom(fyt1), fy_token_atom(fyt2)); +} + +void fy_token_iter_start(struct fy_token *fyt, struct fy_token_iter *iter) +{ + if (!iter) + return; + + memset(iter, 0, sizeof(*iter)); + + iter->unget_c = -1; + + if (!fyt) + return; + + iter->fyt = fyt; + + /* TAG or TAG_DIRECTIVE may only work by getting the text */ + if (fyt->type == FYTT_TAG || fyt->type == FYTT_TAG_DIRECTIVE) + iter->ic.str = fy_token_get_text(fyt, &iter->ic.len); + else /* try the direct output next */ + iter->ic.str = fy_token_get_direct_output(fyt, &iter->ic.len); + + /* got it */ + if (iter->ic.str) { + memset(&iter->atom_iter, 0, sizeof(iter->atom_iter)); + return; + } + + assert(fyt->type != FYTT_TAG && fyt->type != FYTT_TAG_DIRECTIVE); + + /* fall back to the atom iterator */ + fy_atom_iter_start(fy_token_atom(fyt), &iter->atom_iter); +} + +void fy_token_iter_finish(struct fy_token_iter *iter) +{ + if (!iter) + return; + + if (!iter->ic.str) + fy_atom_iter_finish(&iter->atom_iter); +} + +struct fy_token_iter * +fy_token_iter_create(struct fy_token *fyt) +{ + struct fy_token_iter *iter; + + iter = malloc(sizeof(*iter)); + if (!iter) + return NULL; + fy_token_iter_start(fyt, iter); + return iter; +} + +void fy_token_iter_destroy(struct fy_token_iter *iter) +{ + if (!iter) + return; + + fy_token_iter_finish(iter); + free(iter); +} + +const struct fy_iter_chunk *fy_token_iter_peek_chunk(struct fy_token_iter *iter) +{ + if (!iter) + return NULL; + + /* direct mode? */ + if (iter->ic.str) + return &iter->ic; + + /* fallback to the atom iterator */ + return fy_atom_iter_peek_chunk(&iter->atom_iter); +} + +void fy_token_iter_advance(struct fy_token_iter *iter, size_t len) +{ + if (!iter) + return; + + /* direct mode? */ + if (iter->ic.str) { + if (len > iter->ic.len) + len = iter->ic.len; + iter->ic.str += len; + iter->ic.len -= len; + return; + } + + /* fallback to the atom iterator */ + fy_atom_iter_advance(&iter->atom_iter, len); +} + +const struct fy_iter_chunk * +fy_token_iter_chunk_next(struct fy_token_iter *iter, const struct fy_iter_chunk *curr, int *errp) +{ + if (!iter) + return NULL; + + if (errp) + *errp = 0; + + /* first time in */ + if (!curr) { + if (iter->ic.str) + return iter->ic.len ? &iter->ic : NULL; + return fy_atom_iter_chunk_next(&iter->atom_iter, NULL, errp); + } + + /* direct, all consumed */ + if (curr == &iter->ic) { + iter->ic.str += iter->ic.len; + iter->ic.len = 0; + return NULL; + } + + /* fallback */ + return fy_atom_iter_chunk_next(&iter->atom_iter, curr, errp); +} + +ssize_t fy_token_iter_read(struct fy_token_iter *iter, void *buf, size_t count) +{ + if (!iter || !buf) + return -1; + + /* direct mode */ + if (iter->ic.str) { + if (count > iter->ic.len) + count = iter->ic.len; + memcpy(buf, iter->ic.str, count); + iter->ic.str += count; + iter->ic.len -= count; + return count; + } + + return fy_atom_iter_read(&iter->atom_iter, buf, count); +} + +int fy_token_iter_getc(struct fy_token_iter *iter) +{ + int c; + + if (!iter) + return -1; + + /* first try the pushed ungetc */ + if (iter->unget_c >= 0) { + c = iter->unget_c; + /* unmatched getc/ungetc */ + if (fy_utf8_width(c) != 1) + return -1; + iter->unget_c = -1; + return c; + } + + /* direct mode */ + if (iter->ic.str) { + if (!iter->ic.len) + return -1; + c = *iter->ic.str++; + iter->ic.len--; + return c; + } + + return fy_atom_iter_getc(&iter->atom_iter); +} + +int fy_token_iter_ungetc(struct fy_token_iter *iter, int c) +{ + if (!iter || c >= 0x80) + return -1; + + if (iter->unget_c >= 0) + return -1; + + if (c < 0) { + iter->unget_c = -1; + return 0; + } + iter->unget_c = c; + return c; +} + +int fy_token_iter_peekc(struct fy_token_iter *iter) +{ + int c; + + c = fy_token_iter_getc(iter); + if (c == -1) + return -1; + + return fy_token_iter_ungetc(iter, c); +} + +int fy_token_iter_utf8_get(struct fy_token_iter *iter) +{ + int c, w, w1; + + if (!iter) + return -1; + + /* first try the pushed ungetc */ + if (iter->unget_c >= 0) { + c = iter->unget_c; + iter->unget_c = -1; + return c; + } + + /* direct */ + if (iter->ic.str) { + + /* not even 1 octet */ + if (!iter->ic.len) + return -1; + + /* get width by the first octet */ + w = fy_utf8_width_by_first_octet((uint8_t)*iter->ic.str); + if (!w || (unsigned int)w > iter->ic.len) + return -1; + + /* get the next character */ + c = fy_utf8_get(iter->ic.str, w, &w1); + + iter->ic.str += w; + iter->ic.len -= w; + + return c; + } + + return fy_atom_iter_utf8_get(&iter->atom_iter); +} + +int fy_token_iter_utf8_unget(struct fy_token_iter *iter, int c) +{ + if (!iter) + return -1; + + if (iter->unget_c >= 0) + return -1; + + if (c < 0) { + iter->unget_c = -1; + return 0; + } + if (!fy_utf8_is_valid(c)) + return -1; + + iter->unget_c = c; + return c; +} + +int fy_token_iter_utf8_peek(struct fy_token_iter *iter) +{ + int c; + + c = fy_token_iter_utf8_get(iter); + if (c == -1) + return -1; + + return fy_token_iter_utf8_unget(iter, c); +} + +enum fy_scalar_style +fy_scalar_token_get_style(struct fy_token *fyt) +{ + if (!fyt || fyt->type != FYTT_SCALAR) + return FYSS_ANY; + return fyt->scalar.style; +} + +const struct fy_tag *fy_tag_token_tag(struct fy_token *fyt) +{ + if (!fyt || fyt->type != FYTT_TAG) + return NULL; + + /* always refresh, should be relatively infrequent */ + fyt->tag.tag.handle = fy_tag_token_handle0(fyt); + fyt->tag.tag.prefix = fy_tag_token_suffix0(fyt); + + return &fyt->tag.tag; +} + +const struct fy_tag * +fy_tag_directive_token_tag(struct fy_token *fyt) +{ + if (!fyt || fyt->type != FYTT_TAG_DIRECTIVE) + return NULL; + + /* always refresh, should be relatively infrequent */ + fyt->tag_directive.tag.handle = fy_tag_directive_token_handle0(fyt); + fyt->tag_directive.tag.prefix = fy_tag_directive_token_prefix0(fyt); + + return &fyt->tag_directive.tag; +} + +struct fy_atom *fy_token_comment_handle(struct fy_token *fyt, enum fy_comment_placement placement, bool alloc) +{ + struct fy_atom *handle; + size_t size; + + if (!fyt || (unsigned int)placement >= fycp_max) + return NULL; + + if (!fyt->comment) { + if (!alloc) + return NULL; + + size = sizeof(*fyt->comment) * fycp_max; + fyt->comment = malloc(size); + if (!fyt->comment) + return NULL; + memset(fyt->comment, 0, size); + } + handle = &fyt->comment[placement]; + + return handle; +} + +bool fy_token_has_any_comment(struct fy_token *fyt) +{ + struct fy_atom *handle; + enum fy_comment_placement placement; + + if (!fyt || !fyt->comment) + return false; + + for (placement = fycp_top; placement <= fycp_bottom; placement++) { + handle = &fyt->comment[placement]; + if (fy_atom_is_set(handle)) + return true; + } + return false; +} + +bool +fy_token_scalar_is_null(struct fy_token *fyt) +{ + return !fyt || fyt->type != FYTT_SCALAR || fyt->scalar.is_null; +} diff --git a/src/lib/fy-token.h b/src/lib/fy-token.h new file mode 100644 index 0000000..c0a76f5 --- /dev/null +++ b/src/lib/fy-token.h @@ -0,0 +1,541 @@ +/* + * fy-token.h - YAML token methods header + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_TOKEN_H +#define FY_TOKEN_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include + +#include "fy-typelist.h" +#include "fy-utils.h" +#include "fy-atom.h" + +extern const char *fy_token_type_txt[FYTT_COUNT]; + +struct fy_document; +struct fy_path_expr; + +static inline bool fy_token_type_is_sequence_start(enum fy_token_type type) +{ + return type == FYTT_BLOCK_SEQUENCE_START || type == FYTT_FLOW_SEQUENCE_START; +} + +static inline bool fy_token_type_is_sequence_end(enum fy_token_type type) +{ + return type == FYTT_BLOCK_SEQUENCE_START || type == FYTT_FLOW_SEQUENCE_START; +} + +static inline bool fy_token_type_is_sequence_marker(enum fy_token_type type) +{ + return fy_token_type_is_sequence_start(type) || fy_token_type_is_sequence_end(type); +} + +static inline bool fy_token_type_is_mapping_start(enum fy_token_type type) +{ + return type == FYTT_BLOCK_MAPPING_START || type == FYTT_FLOW_MAPPING_START; +} + +static inline bool fy_token_type_is_mapping_end(enum fy_token_type type) +{ + return type == FYTT_BLOCK_MAPPING_START || type == FYTT_FLOW_MAPPING_START; +} + +static inline bool fy_token_type_is_mapping_marker(enum fy_token_type type) +{ + return fy_token_type_is_mapping_start(type) || fy_token_type_is_mapping_end(type); +} + +/* analyze content flags */ +#define FYACF_EMPTY 0x000001 /* is empty (only ws & lb) */ +#define FYACF_LB 0x000002 /* has a linebreak */ +#define FYACF_BLOCK_PLAIN 0x000004 /* can be a plain scalar in block context */ +#define FYACF_FLOW_PLAIN 0x000008 /* can be a plain scalar in flow context */ +#define FYACF_PRINTABLE 0x000010 /* every character is printable */ +#define FYACF_SINGLE_QUOTED 0x000020 /* can be a single quoted scalar */ +#define FYACF_DOUBLE_QUOTED 0x000040 /* can be a double quoted scalar */ +#define FYACF_CONTAINS_ZERO 0x000080 /* contains a zero */ +#define FYACF_DOC_IND 0x000100 /* contains document indicators */ +#define FYACF_CONSECUTIVE_LB 0x000200 /* has consecutive linebreaks */ +#define FYACF_SIMPLE_KEY 0x000400 /* can be a simple key */ +#define FYACF_WS 0x000800 /* has at least one whitespace */ +#define FYACF_STARTS_WITH_WS 0x001000 /* starts with whitespace */ +#define FYACF_STARTS_WITH_LB 0x002000 /* starts with whitespace */ +#define FYACF_ENDS_WITH_WS 0x004000 /* ends with whitespace */ +#define FYACF_ENDS_WITH_LB 0x008000 /* ends with linebreak */ +#define FYACF_TRAILING_LB 0x010000 /* ends with trailing lb > 1 */ +#define FYACF_SIZE0 0x020000 /* contains absolutely nothing */ +#define FYACF_VALID_ANCHOR 0x040000 /* contains valid anchor (without & prefix) */ +#define FYACF_JSON_ESCAPE 0x080000 /* contains a character that JSON escapes */ +#define FYACF_ENDS_WITH_COLON 0x100000 /* ends with : */ + +/* analysis flags */ +#define FYTTAF_HAS_LB FY_BIT(0) +#define FYTTAF_HAS_WS FY_BIT(1) +#define FYTTAF_HAS_CONSECUTIVE_LB FY_BIT(2) +#define FYTTAF_HAS_CONSECUTIVE_WS FY_BIT(4) +#define FYTTAF_EMPTY FY_BIT(5) +#define FYTTAF_CAN_BE_SIMPLE_KEY FY_BIT(6) +#define FYTTAF_DIRECT_OUTPUT FY_BIT(7) +#define FYTTAF_NO_TEXT_TOKEN FY_BIT(8) +#define FYTTAF_TEXT_TOKEN FY_BIT(9) +#define FYTTAF_CAN_BE_PLAIN FY_BIT(10) +#define FYTTAF_CAN_BE_SINGLE_QUOTED FY_BIT(11) +#define FYTTAF_CAN_BE_DOUBLE_QUOTED FY_BIT(12) +#define FYTTAF_CAN_BE_LITERAL FY_BIT(13) +#define FYTTAF_CAN_BE_FOLDED FY_BIT(14) +#define FYTTAF_CAN_BE_PLAIN_FLOW FY_BIT(15) +#define FYTTAF_QUOTE_AT_0 FY_BIT(16) +#define FYTTAF_CAN_BE_UNQUOTED_PATH_KEY FY_BIT(17) +#define FYTTAF_HAS_ANY_LB FY_BIT(18) /* any LB including unicode, not per input */ +#define FYTTAF_HAS_START_IND FY_BIT(19) /* has --- at col 0 */ +#define FYTTAF_HAS_END_IND FY_BIT(20) /* has ... at col 0 */ +#define FYTTAF_HAS_NON_PRINT FY_BIT(21) /* has any non printable utf8 */ +#define FYTTAF_ENDS_WITH_COLON FY_BIT(22) /* ends with a colon */ + +#define FYTTAF_ANALYZED FY_BIT(31) /* analyzed mark */ + +struct fy_token_analysis { + unsigned int flags; + int maxspan; + int maxcol; +}; + +FY_TYPE_FWD_DECL_LIST(token); +struct fy_token { + struct list_head node; + enum fy_token_type type; + int refs; /* when on document, we switch to reference counting */ + struct fy_token_analysis analysis; + size_t text_len; + const char *text; + char *text0; /* this is allocated */ + struct fy_atom handle; + struct fy_atom *comment; /* only when enabled */ + union { + struct { + unsigned int tag_length; /* from start */ + unsigned int uri_length; /* from end */ + char *prefix0; + char *handle0; + struct fy_tag tag; + bool is_default; /* true when default */ + } tag_directive; + struct { + enum fy_scalar_style style; + /* path key (if requested only) */ + const char *path_key; + size_t path_key_len; + char *path_key_storage; /* if this is not null, it's \0 terminated */ + bool is_null; /* special case; the scalar was NULL */ + } scalar; + struct { + unsigned int skip; + unsigned int handle_length; + unsigned int suffix_length; + struct fy_token *fyt_td; + char *handle0; /* zero terminated and allocated, only used by binding */ + char *suffix0; + unsigned int short_length; + char *short0; /* zero terminated and allocated for when the short tag is requested */ + struct fy_tag tag; /* prefix is now suffix */ + } tag; + struct { + struct fy_version vers; /* parsed version number */ + } version_directive; + /* path expressions */ + struct { + struct fy_document *fyd; /* when key is complex */ + } map_key; + struct { + int index; + } seq_index; + struct { + int start_index; + int end_index; + } seq_slice; + struct { + struct fy_path_expr *expr; + } alias; + struct { + int flow_level; + } key; + }; +}; +FY_TYPE_DECL_LIST(token); + +static inline bool fy_token_text_is_direct(struct fy_token *fyt) +{ + if (!fyt || !fyt->text) + return false; + return fyt->text && fyt->text != fyt->text0; +} + +void fy_token_clean_rl(struct fy_token_list *fytl, struct fy_token *fyt); +void fy_token_list_unref_all_rl(struct fy_token_list *fytl, struct fy_token_list *fytl_tofree); + +static inline FY_ALWAYS_INLINE struct fy_token * +fy_token_alloc_rl(struct fy_token_list *fytl) +{ + struct fy_token *fyt; + + fyt = NULL; + if (fytl) + fyt = fy_token_list_pop(fytl); + if (!fyt) { + fyt = malloc(sizeof(*fyt)); + if (!fyt) + return NULL; + } + + fyt->type = FYTT_NONE; + fyt->refs = 1; + memset(&fyt->analysis, 0, sizeof(fyt->analysis)); + fyt->text_len = 0; + fyt->text = NULL; + fyt->text0 = NULL; + fyt->handle.fyi = NULL; + fyt->comment = NULL; + + return fyt; +} + +static inline FY_ALWAYS_INLINE void +fy_token_free_rl(struct fy_token_list *fytl, struct fy_token *fyt) +{ + if (!fyt) + return; + + fy_token_clean_rl(fytl, fyt); + + if (fytl) + fy_token_list_push(fytl, fyt); + else + free(fyt); +} + +static inline FY_ALWAYS_INLINE void +fy_token_unref_rl(struct fy_token_list *fytl, struct fy_token *fyt) +{ + if (!fyt) + return; + + assert(fyt->refs > 0); + + if (--fyt->refs == 0) + fy_token_free_rl(fytl, fyt); +} + +static inline FY_ALWAYS_INLINE struct fy_token * +fy_token_alloc(void) +{ + return fy_token_alloc_rl(NULL); +} + +static inline FY_ALWAYS_INLINE void +fy_token_clean(struct fy_token *fyt) +{ + return fy_token_clean_rl(NULL, fyt); +} + +static inline FY_ALWAYS_INLINE void +fy_token_free(struct fy_token *fyt) +{ + return fy_token_free_rl(NULL, fyt); +} + +static inline FY_ALWAYS_INLINE struct fy_token * +fy_token_ref(struct fy_token *fyt) +{ + /* take care of overflow */ + if (!fyt) + return NULL; + assert(fyt->refs + 1 > 0); + fyt->refs++; + + return fyt; +} + +static inline FY_ALWAYS_INLINE void +fy_token_unref(struct fy_token *fyt) +{ + return fy_token_unref_rl(NULL, fyt); +} + +static inline void +fy_token_list_unref_all(struct fy_token_list *fytl_tofree) +{ + return fy_token_list_unref_all_rl(NULL, fytl_tofree); +} + +/* recycling aware */ +struct fy_token *fy_token_vcreate_rl(struct fy_token_list *fytl, enum fy_token_type type, va_list ap); +struct fy_token *fy_token_create_rl(struct fy_token_list *fytl, enum fy_token_type type, ...); + +struct fy_token *fy_token_vcreate(enum fy_token_type type, va_list ap); +struct fy_token *fy_token_create(enum fy_token_type type, ...); + +static inline struct fy_token * +fy_token_list_vqueue(struct fy_token_list *fytl, enum fy_token_type type, va_list ap) +{ + struct fy_token *fyt; + + fyt = fy_token_vcreate(type, ap); + if (!fyt) + return NULL; + fy_token_list_add_tail(fytl, fyt); + return fyt; +} + +static inline struct fy_token * +fy_token_list_queue(struct fy_token_list *fytl, enum fy_token_type type, ...) +{ + va_list ap; + struct fy_token *fyt; + + va_start(ap, type); + fyt = fy_token_list_vqueue(fytl, type, ap); + va_end(ap); + + return fyt; +} + +int fy_tag_token_format_text_length(const struct fy_token *fyt); +const char *fy_tag_token_format_text(const struct fy_token *fyt, char *buf, size_t maxsz); +int fy_token_format_utf8_length(struct fy_token *fyt); + +int fy_token_format_text_length(struct fy_token *fyt); +const char *fy_token_format_text(struct fy_token *fyt, char *buf, size_t maxsz); + +/* non-parser token methods */ +struct fy_atom *fy_token_atom(struct fy_token *fyt); + +static inline size_t fy_token_start_pos(struct fy_token *fyt) +{ + const struct fy_mark *start_mark; + + if (!fyt) + return (size_t)-1; + + start_mark = fy_token_start_mark(fyt); + return start_mark ? start_mark->input_pos : (size_t)-1; +} + +static inline size_t fy_token_end_pos(struct fy_token *fyt) +{ + const struct fy_mark *end_mark; + + if (!fyt) + return (size_t)-1; + + end_mark = fy_token_end_mark(fyt); + return end_mark ? end_mark->input_pos : (size_t)-1; +} + +static inline int fy_token_start_line(struct fy_token *fyt) +{ + const struct fy_mark *start_mark; + + if (!fyt) + return -1; + + start_mark = fy_token_start_mark(fyt); + return start_mark ? start_mark->line : -1; +} + +static inline int fy_token_start_column(struct fy_token *fyt) +{ + const struct fy_mark *start_mark; + + if (!fyt) + return -1; + + start_mark = fy_token_start_mark(fyt); + return start_mark ? start_mark->column : -1; +} + +static inline int fy_token_end_line(struct fy_token *fyt) +{ + const struct fy_mark *end_mark; + + if (!fyt) + return -1; + + end_mark = fy_token_end_mark(fyt); + return end_mark ? end_mark->line : -1; +} + +static inline int fy_token_end_column(struct fy_token *fyt) +{ + const struct fy_mark *end_mark; + + if (!fyt) + return -1; + + end_mark = fy_token_end_mark(fyt); + return end_mark ? end_mark->column : -1; +} + +static inline bool fy_token_is_multiline(struct fy_token *fyt) +{ + const struct fy_mark *start_mark, *end_mark; + + if (!fyt) + return false; + + start_mark = fy_token_start_mark(fyt); + end_mark = fy_token_end_mark(fyt); + return start_mark && end_mark ? end_mark->line > start_mark->line : false; +} + +const char *fy_token_get_direct_output(struct fy_token *fyt, size_t *sizep); +const char *fy_token_get_direct_simple_output(struct fy_token *fyt, size_t *sizep); + +static inline struct fy_input *fy_token_get_input(struct fy_token *fyt) +{ + return fyt ? fyt->handle.fyi : NULL; +} + +static inline enum fy_atom_style fy_token_atom_style(struct fy_token *fyt) +{ + if (!fyt) + return FYAS_PLAIN; + + if (fyt->type == FYTT_TAG) + return FYAS_URI; + + return fyt->handle.style; +} + +static inline bool fy_token_atom_json_mode(struct fy_token *fyt) +{ + if (!fyt) + return false; + + return fy_atom_json_mode(&fyt->handle); +} + +static inline enum fy_lb_mode fy_token_atom_lb_mode(struct fy_token *fyt) +{ + if (!fyt) + return fylb_cr_nl; + + return fy_atom_lb_mode(&fyt->handle); +} + +static inline enum fy_flow_ws_mode fy_token_atom_flow_ws_mode(struct fy_token *fyt) +{ + if (!fyt) + return fyfws_space_tab; + + return fy_atom_flow_ws_mode(&fyt->handle); +} + +static inline bool fy_token_is_lb(struct fy_token *fyt, int c) +{ + if (!fyt) + return false; + + return fy_atom_is_lb(&fyt->handle, c); +} + +static inline bool fy_token_is_flow_ws(struct fy_token *fyt, int c) +{ + if (!fyt) + return false; + + return fy_atom_is_flow_ws(&fyt->handle, c); +} + +const struct fy_token_analysis *fy_token_text_analyze(struct fy_token *fyt); + +unsigned int fy_analyze_scalar_content(const char *data, size_t size, + bool json_mode, enum fy_lb_mode lb_mode, enum fy_flow_ws_mode fws_mode); + +/* must be freed */ +char *fy_token_debug_text(struct fy_token *fyt); + +#define fy_token_debug_text_a(_fyt) \ + ({ \ + struct fy_token *__fyt = (_fyt); \ + char *_buf, *_rbuf = ""; \ + size_t _len; \ + _buf = fy_token_debug_text(__fyt); \ + if (_buf) { \ + _len = strlen(_buf); \ + _rbuf = alloca(_len + 1); \ + memcpy(_rbuf, _buf, _len + 1); \ + free(_buf); \ + } \ + _rbuf; \ + }) + +int fy_token_memcmp(struct fy_token *fyt, const void *ptr, size_t len); +int fy_token_strcmp(struct fy_token *fyt, const char *str); +int fy_token_cmp(struct fy_token *fyt1, struct fy_token *fyt2); + +struct fy_token_iter { + struct fy_token *fyt; + struct fy_iter_chunk ic; /* direct mode */ + struct fy_atom_iter atom_iter; + int unget_c; +}; + +void fy_token_iter_start(struct fy_token *fyt, struct fy_token_iter *iter); +void fy_token_iter_finish(struct fy_token_iter *iter); + +const char *fy_tag_token_get_directive_handle(struct fy_token *fyt, size_t *td_handle_sizep); +const char *fy_tag_token_get_directive_prefix(struct fy_token *fyt, size_t *td_prefix_sizep); + +static inline bool fy_token_is_number(struct fy_token *fyt) +{ + struct fy_atom *atom; + + if (!fyt || fyt->type != FYTT_SCALAR || fyt->scalar.style != FYSS_PLAIN) + return false; + atom = fy_token_atom(fyt); + if (!atom) + return false; + return fy_atom_is_number(atom); +} + +struct fy_atom *fy_token_comment_handle(struct fy_token *fyt, enum fy_comment_placement placement, bool alloc); +bool fy_token_has_any_comment(struct fy_token *fyt); + +const char *fy_token_get_scalar_path_key(struct fy_token *fyt, size_t *lenp); +size_t fy_token_get_scalar_path_key_length(struct fy_token *fyt); +const char *fy_token_get_scalar_path_key0(struct fy_token *fyt); + +struct fy_atom *fy_token_comment_handle(struct fy_token *fyt, enum fy_comment_placement placement, bool alloc); + +static inline FY_ALWAYS_INLINE enum fy_scalar_style +fy_token_scalar_style_inline(struct fy_token *fyt) +{ + if (!fyt || fyt->type != FYTT_SCALAR) + return FYSS_PLAIN; + + if (fyt->type == FYTT_SCALAR) + return fyt->scalar.style; + + return FYSS_PLAIN; +} + +static inline FY_ALWAYS_INLINE enum fy_token_type +fy_token_get_type_inline(struct fy_token *fyt) +{ + return fyt ? fyt->type : FYTT_NONE; +} + +#endif diff --git a/src/lib/fy-types.c b/src/lib/fy-types.c new file mode 100644 index 0000000..1649e04 --- /dev/null +++ b/src/lib/fy-types.c @@ -0,0 +1,34 @@ +/* + * fy-types.c - types definition + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" + +/* parse only types */ +FY_PARSE_TYPE_DEFINE_SIMPLE(indent); +FY_PARSE_TYPE_DEFINE_SIMPLE(simple_key); +FY_PARSE_TYPE_DEFINE_SIMPLE(parse_state_log); +FY_PARSE_TYPE_DEFINE_SIMPLE(flow); +FY_PARSE_TYPE_DEFINE_SIMPLE(streaming_alias); diff --git a/src/lib/fy-types.h b/src/lib/fy-types.h new file mode 100644 index 0000000..24a2b26 --- /dev/null +++ b/src/lib/fy-types.h @@ -0,0 +1,143 @@ +/* + * fy-types.h - common types builder + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_TYPES_H +#define FY_TYPES_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#include + +#include "fy-list.h" + +struct fy_parser; + +/* define type methods */ +#define FY_ALLOC_TYPE_DEFINE(_type) \ +\ +struct fy_ ## _type *fy_ ## _type ## _alloc_simple_internal( \ + struct fy_ ## _type ## _list *_rl) \ +{ \ + struct fy_ ## _type *_n; \ + \ + _n = fy_ ## _type ## _list_pop(_rl); \ + if (_n) \ + return _n; \ + _n = malloc(sizeof(*_n)); \ + if (_n) \ + list_init(&_n->node); \ + return _n; \ +} \ +\ +void fy_ ## _type ## _recycle_internal(struct fy_ ## _type ## _list *_rl, \ + struct fy_ ## _type *_n) \ +{ \ + if (_n) \ + fy_ ## _type ## _list_push(_rl, _n); \ +} \ +\ +void fy_ ## _type ## _vacuum_internal(struct fy_ ## _type ## _list *_rl) \ +{ \ + struct fy_ ## _type *_n; \ + \ + while ((_n = fy_ ## _type ## _list_pop(_rl)) != NULL) \ + free(_n); \ +} \ +\ +struct __useless_struct_to_allow_semicolon + +/* declarations for alloc */ + +#define FY_ALLOC_TYPE_ALLOC(_type) \ +struct fy_ ## _type *fy_ ## _type ## _alloc_simple_internal( \ + struct fy_ ## _type ## _list *_rl); \ +void fy_ ## _type ## _recycle_internal(struct fy_ ## _type ## _list *_rl, \ + struct fy_ ## _type *_n); \ +void fy_ ## _type ## _vacuum_internal(struct fy_ ## _type ## _list *_rl); \ +struct __useless_struct_to_allow_semicolon + +/* parser type methods */ +#define FY_PARSE_TYPE_DECL_ALLOC(_type) \ +\ +struct fy_ ## _type *fy_parse_ ## _type ## _alloc(struct fy_parser *fyp); \ +void fy_parse_ ## _type ## _vacuum(struct fy_parser *fyp); \ +void fy_parse_ ## _type ## _recycle(struct fy_parser *fyp, struct fy_ ## _type *_n); \ +void fy_parse_ ## _type ## _list_recycle_all(struct fy_parser *fyp, struct fy_ ## _type ## _list *_l); \ +\ +struct __useless_struct_to_allow_semicolon + +#define FY_PARSE_TYPE_DECL(_type) \ +FY_TYPE_FWD_DECL_LIST(_type); \ +FY_TYPE_DECL_LIST(_type); \ +FY_PARSE_TYPE_DECL_ALLOC(_type); \ +struct __useless_struct_to_allow_semicolon + +#define FY_PARSE_TYPE_DECL_AFTER_FWD(_type) \ +FY_TYPE_DECL_LIST(_type); \ +FY_PARSE_TYPE_DECL_ALLOC(_type); \ +struct __useless_struct_to_allow_semicolon + +/* define type methods */ +#define FY_PARSE_TYPE_DEFINE(_type) \ +\ +struct fy_ ## _type *fy_parse_ ## _type ## _alloc_simple(struct fy_parser *fyp) \ +{ \ + return fy_ ## _type ## _alloc_simple_internal(&fyp->recycled_ ## _type); \ +} \ +\ +void fy_parse_ ## _type ## _vacuum(struct fy_parser *fyp) \ +{ \ + fy_ ## _type ## _vacuum_internal(&fyp->recycled_ ## _type); \ +} \ +\ +void fy_parse_ ## _type ## _list_recycle_all(struct fy_parser *fyp, struct fy_ ## _type ## _list *_l) \ +{ \ + struct fy_ ## _type *_n; \ + \ + while ((_n = fy_ ## _type ## _list_pop(_l)) != NULL) \ + fy_parse_ ## _type ## _recycle(fyp, _n); \ +} \ +\ +void fy_parse_ ## _type ## _recycle_simple(struct fy_parser *fyp, struct fy_ ## _type *_n) \ +{ \ + if (!fyp->suppress_recycling) \ + fy_ ## _type ## _recycle_internal(&fyp->recycled_ ## _type, _n); \ + else \ + free(_n); \ +} \ +\ +struct __useless_struct_to_allow_semicolon + +#define FY_PARSE_TYPE_DEFINE_ALLOC_SIMPLE(_type) \ +struct fy_ ## _type *fy_parse_ ## _type ## _alloc(struct fy_parser *_fyp) \ +{ \ + return fy_parse_ ## _type ## _alloc_simple(_fyp); \ +} \ +\ +void fy_parse_ ## _type ## _recycle(struct fy_parser *_fyp, struct fy_ ## _type *_n) \ +{ \ + if (_n) \ + fy_parse_ ## _type ## _recycle_simple(_fyp, _n); \ +} \ +\ +struct __useless_struct_to_allow_semicolon + +#define FY_PARSE_TYPE_DEFINE_SIMPLE(_type) \ +\ +FY_ALLOC_TYPE_DEFINE(_type); \ +FY_PARSE_TYPE_DEFINE(_type); \ +FY_PARSE_TYPE_DEFINE_ALLOC_SIMPLE(_type); \ +\ +struct __useless_struct_to_allow_semicolon + +#endif diff --git a/src/lib/fy-walk.c b/src/lib/fy-walk.c new file mode 100644 index 0000000..8c83d9c --- /dev/null +++ b/src/lib/fy-walk.c @@ -0,0 +1,5588 @@ +/* + * fy-walk.c - path walker + * + * Copyright (c) 2021 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-parse.h" +#include "fy-doc.h" +#include "fy-walk.h" + +#include "fy-utils.h" + +#undef DEBUG_EXPR +// #define DEBUG_EXPR + +const char *fy_walk_result_type_txt[FWRT_COUNT] = { + [fwrt_none] = "none", + [fwrt_node_ref] = "node-ref", + [fwrt_number] = "number", + [fwrt_string] = "string", + [fwrt_doc] = "doc", + [fwrt_refs] = "refs", +}; + +void fy_walk_result_dump(struct fy_walk_result *fwr, struct fy_diag *diag, enum fy_error_type errlevel, int level, + const char *fmt, ...); + +void fy_walk_result_vdump(struct fy_walk_result *fwr, struct fy_diag *diag, enum fy_error_type errlevel, int level, + const char *fmt, va_list ap) +{ + struct fy_walk_result *fwr2; + char *banner; + char *texta = NULL; + const char *text = ""; + size_t len; + bool save_on_error; + char buf[30]; + int rc __FY_DEBUG_UNUSED__; + + if (!diag) + return; + + if (errlevel < diag->cfg.level) + return; + + save_on_error = diag->on_error; + diag->on_error = true; + + if (fmt) { + banner = NULL; + rc = vasprintf(&banner, fmt, ap); + assert(rc != -1); + assert(banner); + fy_diag_diag(diag, errlevel, "%-*s%s", level*2, "", banner); + free(banner); + } + if (!fwr) + goto out; + + switch (fwr->type) { + case fwrt_none: + text=""; + break; + case fwrt_node_ref: + texta = fy_node_get_path(fwr->fyn); + assert(texta); + text = texta; + break; + case fwrt_number: + snprintf(buf, sizeof(buf), "%f", fwr->number); + text = buf; + break; + case fwrt_string: + text = fwr->string; + break; + case fwrt_doc: + texta = fy_emit_document_to_string(fwr->fyd, FYECF_WIDTH_INF | FYECF_INDENT_DEFAULT | FYECF_MODE_FLOW_ONELINE); + assert(texta); + text = texta; + break; + case fwrt_refs: + text=""; + break; + } + + len = strlen(text); + + fy_diag_diag(diag, errlevel, "%-*s%s%s%.*s", + (level + 1) * 2, "", + fy_walk_result_type_txt[fwr->type], + len ? " " : "", + (int)len, text); + + if (texta) + free(texta); + + if (fwr->type == fwrt_refs) { + for (fwr2 = fy_walk_result_list_head(&fwr->refs); fwr2; fwr2 = fy_walk_result_next(&fwr->refs, fwr2)) + fy_walk_result_dump(fwr2, diag, errlevel, level + 1, NULL); + } +out: + diag->on_error = save_on_error; +} + +void fy_walk_result_dump(struct fy_walk_result *fwr, struct fy_diag *diag, enum fy_error_type errlevel, int level, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_walk_result_vdump(fwr, diag, errlevel, level, fmt, ap); + va_end(ap); +} + +/* NOTE that walk results do not take references and it is invalid to + * use _any_ call that modifies the document structure + */ +struct fy_walk_result *fy_walk_result_alloc_rl(struct fy_walk_result_list *fwrl) +{ + struct fy_walk_result *fwr = NULL; + + if (fwrl) + fwr = fy_walk_result_list_pop(fwrl); + + if (!fwr) { + fwr = malloc(sizeof(*fwr)); + if (!fwr) + return NULL; + memset(fwr, 0, sizeof(*fwr)); + } else + memset(&fwr->_clean, 0, sizeof(fwr->_clean)); + fwr->type = fwrt_none; + return fwr; +} + +struct fy_walk_result *fy_walk_result_clone_rl(struct fy_walk_result_list *fwrl, struct fy_walk_result *fwr, bool deep) +{ + struct fy_walk_result *fwrn = NULL, *fwrn2 = NULL, *fwrn3; + + if (!fwr) + return NULL; + + fwrn = fy_walk_result_alloc_rl(fwrl); + if (!fwrn) + return NULL; + + fwrn->type = fwr->type; + switch (fwr->type) { + case fwrt_none: + break; + case fwrt_node_ref: + if (!deep) { + fwrn->fyn = fwr->fyn; + fwrn->fyn_source = NULL; + } else { + fwrn->fyn = fy_node_copy(fy_node_document(fwr->fyn), fwr->fyn); + if (!fwrn->fyn) + goto err_out; + fwrn->fyn_source = fwr->fyn; + } + break; + case fwrt_number: + fwrn->number = fwr->number; + break; + case fwrt_string: + fwrn->string = strdup(fwr->string); + if (!fwrn->string) + goto err_out; + break; + case fwrt_doc: + fwrn->fyd = fy_document_clone(fwr->fyd); + if (!fwrn->fyd) + goto err_out; + break; + case fwrt_refs: + + fy_walk_result_list_init(&fwrn->refs); + + for (fwrn2 = fy_walk_result_list_head(&fwr->refs); fwrn2; + fwrn2 = fy_walk_result_next(&fwr->refs, fwrn2)) { + + fwrn3 = fy_walk_result_clone_rl(fwrl, fwrn2, deep); + if (!fwrn3) + goto err_out; + + fy_walk_result_list_add_tail(&fwrn->refs, fwrn3); + } + break; + } + return fwrn; +err_out: + if (fwrn) + fy_walk_result_free_rl(fwrl, fwrn); + return NULL; +} + +struct fy_walk_result *fy_walk_result_clone(struct fy_walk_result *fwr) +{ + struct fy_walk_result_list *fwrl; + + if (!fwr) + return NULL; + + fwrl = fy_path_exec_walk_result_rl(fwr->fypx); + return fy_walk_result_clone_rl(fwrl, fwr, false); +} + +struct fy_walk_result *fy_walk_result_clone_deep(struct fy_walk_result *fwr) +{ + struct fy_walk_result_list *fwrl; + + if (!fwr) + return NULL; + + fwrl = fy_path_exec_walk_result_rl(fwr->fypx); + return fy_walk_result_clone_rl(fwrl, fwr, true); +} + +void fy_walk_result_clean_rl(struct fy_walk_result_list *fwrl, struct fy_walk_result *fwr) +{ + struct fy_walk_result *fwrn; + + if (!fwr) + return; + + switch (fwr->type) { + case fwrt_none: + break; + case fwrt_node_ref: + /* if deep copy, free */ + if (fwr->fyn_source) + fy_node_free(fwr->fyn); + break; + case fwrt_number: + break; + case fwrt_string: + if (fwr->string) + free(fwr->string); + break; + case fwrt_doc: + if (fwr->fyd) + fy_document_destroy(fwr->fyd); + break; + case fwrt_refs: + while ((fwrn = fy_walk_result_list_pop(&fwr->refs)) != NULL) + fy_walk_result_free_rl(fwrl, fwrn); + break; + } + + fwr->type = fwrt_none; + memset(&fwr->_clean, 0, sizeof(fwr->_clean)); +} + +void fy_walk_result_clean(struct fy_walk_result *fwr) +{ + struct fy_walk_result_list *fwrl; + + if (!fwr) + return; + + fwrl = fy_path_exec_walk_result_rl(fwr->fypx); + fy_walk_result_clean_rl(fwrl, fwr); +} + +void fy_walk_result_free_rl(struct fy_walk_result_list *fwrl, struct fy_walk_result *fwr) +{ + struct fy_path_exec *fypx; + + if (!fwr) + return; + + fypx = fwr->fypx; + + fy_walk_result_clean_rl(fwrl, fwr); + + if (fwrl) + fy_walk_result_list_push(fwrl, fwr); + else + free(fwr); + + fy_path_exec_unref(fypx); /* NULL is OK */ +} + +void fy_walk_result_free(struct fy_walk_result *fwr) +{ + struct fy_walk_result_list *fwrl; + + if (!fwr) + return; + + fwrl = fy_path_exec_walk_result_rl(fwr->fypx); + fy_walk_result_free_rl(fwrl, fwr); +} + +void fy_walk_result_list_free_rl(struct fy_walk_result_list *fwrl, struct fy_walk_result_list *results) +{ + struct fy_walk_result *fwr; + + while ((fwr = fy_walk_result_list_pop(results)) != NULL) + fy_walk_result_free_rl(fwrl, fwr); +} + +struct fy_walk_result *fy_walk_result_vcreate_rl(struct fy_walk_result_list *fwrl, enum fy_walk_result_type type, va_list ap) +{ + struct fy_walk_result *fwr = NULL; + + if ((unsigned int)type >= FWRT_COUNT) + goto err_out; + + fwr = fy_walk_result_alloc_rl(fwrl); + if (!fwr) + goto err_out; + + fwr->type = type; + + switch (fwr->type) { + case fwrt_none: + break; + case fwrt_node_ref: + fwr->fyn = va_arg(ap, struct fy_node *); + break; + case fwrt_number: + fwr->number = va_arg(ap, double); + break; + case fwrt_string: + fwr->string = strdup(va_arg(ap, const char *)); + if (!fwr->string) + goto err_out; + break; + case fwrt_doc: + fwr->fyd = va_arg(ap, struct fy_document *); + break; + case fwrt_refs: + fy_walk_result_list_init(&fwr->refs); + break; + } + + return fwr; + +err_out: + fy_walk_result_free_rl(fwrl, fwr); + + return NULL; +} + +struct fy_walk_result *fy_walk_result_create_rl(struct fy_walk_result_list *fwrl, enum fy_walk_result_type type, ...) +{ + struct fy_walk_result *fwr; + va_list ap; + + va_start(ap, type); + fwr = fy_walk_result_vcreate_rl(fwrl, type, ap); + va_end(ap); + + return fwr; +} + +void fy_walk_result_flatten_internal(struct fy_walk_result *fwr, struct fy_walk_result *fwrf) +{ + struct fy_walk_result *fwr2, *fwr2n; + + if (!fwr || !fwrf || fwr->type != fwrt_refs) + return; + + for (fwr2 = fy_walk_result_list_head(&fwr->refs); fwr2; fwr2 = fwr2n) { + + fwr2n = fy_walk_result_next(&fwr->refs, fwr2); + + if (fwr2->type != fwrt_refs) { + fy_walk_result_list_del(&fwr->refs, fwr2); + fy_walk_result_list_add_tail(&fwrf->refs, fwr2); + continue; + } + fy_walk_result_flatten_internal(fwr2, fwrf); + } +} + +bool fy_walk_result_has_leaves_only(struct fy_walk_result *fwr) +{ + struct fy_walk_result *fwrn; + + if (!fwr || fwr->type != fwrt_refs) + return false; + + if (fy_walk_result_list_empty(&fwr->refs)) + return false; + + for (fwrn = fy_walk_result_list_head(&fwr->refs); fwrn; + fwrn = fy_walk_result_next(&fwr->refs, fwrn)) { + + if (fwrn->type == fwrt_refs) + return false; + } + + return true; +} + +struct fy_walk_result * +fy_walk_result_flatten_rl(struct fy_walk_result_list *fwrl, struct fy_walk_result *fwr) +{ + struct fy_walk_result *fwrf; + + if (!fwr) + return NULL; + + fwrf = fy_walk_result_create_rl(fwrl, fwrt_refs); + assert(fwrf); + + fy_walk_result_flatten_internal(fwr, fwrf); + fy_walk_result_free_rl(fwrl, fwr); + return fwrf; +} + +struct fy_walk_result * +fy_walk_result_flatten(struct fy_walk_result *fwr) +{ + struct fy_walk_result_list *fwrl; + + if (!fwr) + return NULL; + + fwrl = fy_path_exec_walk_result_rl(fwr->fypx); + return fy_walk_result_flatten_rl(fwrl, fwr); +} + +struct fy_node * +fy_walk_result_node_iterate(struct fy_walk_result *fwr, void **prevp) +{ + struct fy_walk_result *fwrn; + + if (!fwr || !prevp) + return NULL; + + switch (fwr->type) { + case fwrt_node_ref: + if (!*prevp) { + *prevp = fwr; + return fwr->fyn; + } + *prevp = NULL; + return NULL; + + case fwrt_refs: + if (!*prevp) + fwrn = fy_walk_result_list_head(&fwr->refs); + else + fwrn = fy_walk_result_next(&fwr->refs, *prevp); + + /* skip over any non node refs */ + while (fwrn && fwrn->type != fwrt_node_ref) + fwrn = fy_walk_result_next(&fwr->refs, fwrn); + *prevp = fwrn; + return fwrn ? fwrn->fyn : NULL; + + default: + break; + } + + return NULL; +} + +const char *fy_path_expr_type_txt[FPET_COUNT] = { + [fpet_none] = "none", + /* */ + [fpet_root] = "root", + [fpet_this] = "this", + [fpet_parent] = "parent", + [fpet_every_child] = "every-child", + [fpet_every_child_r] = "every-child-recursive", + [fpet_filter_collection] = "filter-collection", + [fpet_filter_scalar] = "filter-scalar", + [fpet_filter_sequence] = "filter-sequence", + [fpet_filter_mapping] = "filter-mapping", + [fpet_filter_unique] = "filter-unique", + [fpet_seq_index] = "seq-index", + [fpet_seq_slice] = "seq-slice", + [fpet_alias] = "alias", + + [fpet_map_key] = "map-key", + + [fpet_multi] = "multi", + [fpet_chain] = "chain", + [fpet_logical_or] = "logical-or", + [fpet_logical_and] = "logical-and", + [fpet_select] = "select", + [fpet_unselect] = "unselect", + + [fpet_eq] = "equals", + [fpet_neq] = "not-equals", + [fpet_lt] = "less-than", + [fpet_gt] = "greater-than", + [fpet_lte] = "less-or-equal-than", + [fpet_gte] = "greater-or-equal-than", + + [fpet_scalar] = "scalar", + + [fpet_plus] = "plus", + [fpet_minus] = "minus", + [fpet_mult] = "multiply", + [fpet_div] = "divide", + + [fpet_lparen] = "left-parentheses", + [fpet_rparen] = "right-parentheses", + [fpet_method] = "method", + + [fpet_scalar_expr] = "scalar-expression", + [fpet_path_expr] = "path-expression", + [fpet_arg_separator] = "argument-separator", +}; + +struct fy_path_expr *fy_path_expr_alloc(void) +{ + struct fy_path_expr *expr = NULL; + + expr = malloc(sizeof(*expr)); + if (!expr) + return NULL; + memset(expr, 0, sizeof(*expr)); + fy_path_expr_list_init(&expr->children); + + return expr; +} + +void fy_path_expr_free(struct fy_path_expr *expr) +{ + struct fy_path_expr *exprn; + + if (!expr) + return; + + while ((exprn = fy_path_expr_list_pop(&expr->children)) != NULL) + fy_path_expr_free(exprn); + + fy_token_unref(expr->fyt); + + free(expr); +} + +struct fy_path_expr *fy_path_expr_alloc_recycle(struct fy_path_parser *fypp) +{ + struct fy_path_expr *expr = NULL; + + if (!fypp || fypp->suppress_recycling) + expr = fy_path_expr_alloc(); + + if (!expr) { + expr = fy_path_expr_list_pop(&fypp->expr_recycle); + if (expr) { + memset(expr, 0, sizeof(*expr)); + fy_path_expr_list_init(&expr->children); + } else + expr = fy_path_expr_alloc(); + } + + if (!expr) + return NULL; + + expr->expr_mode = fypp->expr_mode; + + return expr; +} + +void fy_path_expr_free_recycle(struct fy_path_parser *fypp, struct fy_path_expr *expr) +{ + struct fy_path_expr *exprn; + + if (!fypp || fypp->suppress_recycling) { + fy_path_expr_free(expr); + return; + } + + while ((exprn = fy_path_expr_list_pop(&expr->children)) != NULL) + fy_path_expr_free_recycle(fypp, exprn); + + if (expr->fyt) { + fy_token_unref(expr->fyt); + expr->fyt = NULL; + } + fy_path_expr_list_add_tail(&fypp->expr_recycle, expr); +} + +void fy_expr_stack_setup(struct fy_expr_stack *stack) +{ + if (!stack) + return; + + memset(stack, 0, sizeof(*stack)); + stack->items = stack->items_static; + stack->alloc = ARRAY_SIZE(stack->items_static); +} + +void fy_expr_stack_cleanup(struct fy_expr_stack *stack) +{ + if (!stack) + return; + + while (stack->top > 0) + fy_path_expr_free(stack->items[--stack->top]); + + if (stack->items != stack->items_static) + free(stack->items); + stack->items = stack->items_static; + stack->alloc = ARRAY_SIZE(stack->items_static); +} + +void fy_expr_stack_dump(struct fy_diag *diag, struct fy_expr_stack *stack) +{ + struct fy_path_expr *expr; + unsigned int i; + + if (!stack) + return; + + if (!stack->top) + return; + + i = stack->top; + do { + expr = stack->items[--i]; + fy_path_expr_dump(expr, diag, FYET_NOTICE, 0, NULL); + } while (i > 0); +} + +int fy_expr_stack_size(struct fy_expr_stack *stack) +{ + if (!stack || stack->top >= (unsigned int)INT_MAX) + return -1; + return (int)stack->top; +} + +int fy_expr_stack_push(struct fy_expr_stack *stack, struct fy_path_expr *expr) +{ + struct fy_path_expr **items_new; + unsigned int alloc; + size_t size; + + if (!stack || !expr) + return -1; + + assert(stack->items); + assert(stack->alloc > 0); + + assert(expr->fyt); + + /* grow the stack if required */ + if (stack->top >= stack->alloc) { + alloc = stack->alloc; + size = alloc * sizeof(*items_new); + + if (stack->items == stack->items_static) { + items_new = malloc(size * 2); + if (items_new) + memcpy(items_new, stack->items_static, size); + } else + items_new = realloc(stack->items, size * 2); + + if (!items_new) + return -1; + + stack->alloc = alloc * 2; + stack->items = items_new; + } + + stack->items[stack->top++] = expr; + + return 0; +} + +struct fy_path_expr *fy_expr_stack_peek_at(struct fy_expr_stack *stack, unsigned int pos) +{ + if (!stack || stack->top <= pos) + return NULL; + return stack->items[stack->top - 1 - pos]; +} + +struct fy_path_expr *fy_expr_stack_peek(struct fy_expr_stack *stack) +{ + return fy_expr_stack_peek_at(stack, 0); +} + +struct fy_path_expr *fy_expr_stack_pop(struct fy_expr_stack *stack) +{ + if (!stack || !stack->top) + return NULL; + + return stack->items[--stack->top]; +} + +bool fy_token_type_can_be_path_expr(enum fy_token_type type) +{ + return type == FYTT_NONE || + type == FYTT_PE_LPAREN || + type == FYTT_PE_RPAREN || + type == FYTT_PE_EQEQ || + type == FYTT_PE_NOTEQ || + type == FYTT_PE_GT || + type == FYTT_PE_LT || + type == FYTT_PE_GTE || + type == FYTT_PE_LTE || + type == FYTT_PE_METHOD; +} + +bool fy_token_type_can_be_before_negative_number(enum fy_token_type type) +{ + return type == FYTT_NONE || + type == FYTT_PE_LPAREN || + type == FYTT_PE_RPAREN || + type == FYTT_PE_EQEQ || + type == FYTT_PE_NOTEQ || + type == FYTT_PE_GT || + type == FYTT_PE_LT || + type == FYTT_PE_GTE || + type == FYTT_PE_LTE || + type == FYTT_SE_PLUS || + type == FYTT_SE_MINUS || + type == FYTT_SE_MULT || + type == FYTT_SE_DIV || + + type == FYTT_SE_METHOD; +} + +const char *fy_expr_mode_txt[FYEM_COUNT] = { + [fyem_none] = "none", + [fyem_path] = "path", + [fyem_scalar] = "scalar", +}; + +static struct fy_diag *fy_path_parser_reader_get_diag(struct fy_reader *fyr) +{ + struct fy_path_parser *fypp = container_of(fyr, struct fy_path_parser, reader); + return fypp->cfg.diag; +} + +static const struct fy_reader_ops fy_path_parser_reader_ops = { + .get_diag = fy_path_parser_reader_get_diag, +}; + +void fy_path_parser_setup(struct fy_path_parser *fypp, const struct fy_path_parse_cfg *pcfg) +{ + struct fy_diag_cfg dcfg; + + if (!fypp) + return; + + memset(fypp, 0, sizeof(*fypp)); + if (pcfg) + fypp->cfg = *pcfg; + if (!fypp->cfg.diag) { + fy_diag_cfg_default(&dcfg); + fypp->cfg.diag = fy_diag_create(&dcfg); + fypp->owns_diag = true; + } + fy_reader_setup(&fypp->reader, &fy_path_parser_reader_ops); + fy_token_list_init(&fypp->queued_tokens); + fypp->last_queued_token_type = FYTT_NONE; + + fy_expr_stack_setup(&fypp->operators); + fy_expr_stack_setup(&fypp->operands); + + fy_path_expr_list_init(&fypp->expr_recycle); + fypp->suppress_recycling = (fypp->cfg.flags & FYPPCF_DISABLE_RECYCLING) || getenv("FY_VALGRIND"); + + fypp->expr_mode = fyem_path; + fypp->paren_nest_level = 0; +} + +int fy_path_parser_reset(struct fy_path_parser *fypp) +{ + struct fy_path_expr *expr; + + if (!fypp) + return -1; + + fy_expr_stack_cleanup(&fypp->operands); + fy_expr_stack_cleanup(&fypp->operators); + + fy_reader_cleanup(&fypp->reader); + fy_token_list_unref_all(&fypp->queued_tokens); + + while ((expr = fy_path_expr_list_pop(&fypp->expr_recycle)) != NULL) + fy_path_expr_free(expr); + + fypp->last_queued_token_type = FYTT_NONE; + fypp->stream_start_produced = false; + fypp->stream_end_produced = false; + fypp->stream_error = false; + fypp->token_activity_counter = 0; + fypp->paren_nest_level = 0; + + return 0; +} + +void fy_path_parser_cleanup(struct fy_path_parser *fypp) +{ + if (!fypp) + return; + + fy_path_parser_reset(fypp); + if (fypp->owns_diag && fypp->cfg.diag) + fy_diag_unref(fypp->cfg.diag); + memset(fypp, 0, sizeof(*fypp)); +} + +int fy_path_parser_open(struct fy_path_parser *fypp, + struct fy_input *fyi, const struct fy_reader_input_cfg *icfg) +{ + int ret; + if (!fypp) + return -1; + + ret = fy_reader_input_open(&fypp->reader, fyi, icfg); + if (ret) + return ret; + /* take a reference to the input */ + fypp->fyi = fy_input_ref(fyi); + return 0; +} + +void fy_path_parser_close(struct fy_path_parser *fypp) +{ + if (!fypp) + return; + + fy_input_unref(fypp->fyi); + + fy_reader_input_done(&fypp->reader); +} + +struct fy_token *fy_path_token_vqueue(struct fy_path_parser *fypp, enum fy_token_type type, va_list ap) +{ + struct fy_token *fyt; + + fyt = fy_token_list_vqueue(&fypp->queued_tokens, type, ap); + if (fyt) { + fypp->token_activity_counter++; + fypp->last_queued_token_type = type; + } + return fyt; +} + +struct fy_token *fy_path_token_queue(struct fy_path_parser *fypp, enum fy_token_type type, ...) +{ + va_list ap; + struct fy_token *fyt; + + va_start(ap, type); + fyt = fy_path_token_vqueue(fypp, type, ap); + va_end(ap); + + return fyt; +} + +int fy_path_fetch_seq_index_or_slice(struct fy_path_parser *fypp, int c) +{ + struct fy_reader *fyr; + struct fy_token *fyt; + bool neg; + int i, j, val, nval, digits, indices[2]; + + fyr = &fypp->reader; + + /* verify that the called context is correct */ + assert(fy_is_num(c) || (c == '-' && fy_is_num(fy_reader_peek_at(fyr, 1)))); + + i = 0; + indices[0] = indices[1] = -1; + + j = 0; + while (j < 2) { + + neg = false; + if (c == '-') { + neg = true; + i++; + } + + digits = 0; + val = 0; + while (fy_is_num((c = fy_reader_peek_at(fyr, i)))) { + nval = (val * 10) | (c - '0'); + FYR_PARSE_ERROR_CHECK(fyr, 0, i, FYEM_SCAN, + nval >= val && nval >= 0, err_out, + "illegal sequence index (overflow)"); + val = nval; + i++; + digits++; + } + FYR_PARSE_ERROR_CHECK(fyr, 0, i, FYEM_SCAN, + (val == 0 && digits == 1) || (val > 0), err_out, + "bad number"); + if (neg) + val = -val; + + indices[j] = val; + + /* continue only on slice : */ + if (c == ':') { + c = fy_reader_peek_at(fyr, i + 1); + if (fy_is_num(c) || (c == '-' && fy_is_num(fy_reader_peek_at(fyr, i + 2)))) { + i++; + j++; + continue; + } + } + + break; + } + + if (j >= 1) + fyt = fy_path_token_queue(fypp, FYTT_PE_SEQ_SLICE, fy_reader_fill_atom_a(fyr, i), indices[0], indices[1]); + else + fyt = fy_path_token_queue(fypp, FYTT_PE_SEQ_INDEX, fy_reader_fill_atom_a(fyr, i), indices[0]); + + fyr_error_check(fyr, fyt, err_out, "fy_path_token_queue() failed\n"); + + return 0; + +err_out: + fypp->stream_error = true; + return -1; +} + +int fy_path_fetch_plain_or_method(struct fy_path_parser *fypp, int c, + enum fy_token_type fytt_plain, + enum fy_token_type fytt_method) +{ + struct fy_reader *fyr; + struct fy_token *fyt; + struct fy_atom *handlep; + int i; + enum fy_token_type type; + + fyr = &fypp->reader; + + assert(fy_is_first_alpha(c)); + + type = fytt_plain; + + i = 1; + while (fy_is_alnum(fy_reader_peek_at(fyr, i))) + i++; + + if (fy_reader_peek_at(fyr, i) == '(') + type = fytt_method; + + handlep = fy_reader_fill_atom_a(fyr, i); + if (type == FYTT_SCALAR) { + fyt = fy_path_token_queue(fypp, FYTT_SCALAR, handlep, FYSS_PLAIN, NULL); + fyr_error_check(fyr, fyt, err_out, "fy_path_token_queue() failed\n"); + } else { + fyt = fy_path_token_queue(fypp, type, handlep, NULL); + fyr_error_check(fyr, fyt, err_out, "fy_path_token_queue() failed\n"); + } + + return 0; + +err_out: + fypp->stream_error = true; + return -1; +} + +int fy_path_fetch_dot_method(struct fy_path_parser *fypp, int c, enum fy_token_type fytt) +{ + struct fy_reader *fyr; + struct fy_token *fyt; + struct fy_atom *handlep; + int i; + + fyr = &fypp->reader; + + assert(c == '.'); + fy_reader_advance(fyr, c); + c = fy_reader_peek(fyr); + assert(fy_is_first_alpha(c)); + + /* verify that the called context is correct */ + i = 1; + while (fy_is_alnum(fy_reader_peek_at(fyr, i))) + i++; + + handlep = fy_reader_fill_atom_a(fyr, i); + + fyt = fy_path_token_queue(fypp, fytt, handlep, NULL); + fyr_error_check(fyr, fyt, err_out, "fy_path_token_queue() failed\n"); + + return 0; + +err_out: + fypp->stream_error = true; + return -1; +} + +int fy_path_fetch_flow_document(struct fy_path_parser *fypp, int c, enum fy_token_type fytt) +{ + struct fy_reader *fyr; + struct fy_token *fyt; + struct fy_document *fyd; + struct fy_atom handle; + struct fy_parser fyp_data, *fyp = &fyp_data; + struct fy_parse_cfg cfg_data, *cfg = NULL; + int rc; + + fyr = &fypp->reader; + + /* verify that the called context is correct */ + assert(fy_is_path_flow_key_start(c)); + + fy_reader_fill_atom_start(fyr, &handle); + + cfg = &cfg_data; + memset(cfg, 0, sizeof(*cfg)); + cfg->flags = FYPCF_DEFAULT_PARSE; + cfg->diag = fypp->cfg.diag; + + rc = fy_parse_setup(fyp, cfg); + fyr_error_check(fyr, !rc, err_out, "fy_parse_setup() failed\n"); + + /* associate with reader and set flow mode */ + fy_parser_set_reader(fyp, fyr); + fy_parser_set_flow_only_mode(fyp, true); + + fyd = fy_parse_load_document(fyp); + + /* cleanup the parser no matter what */ + fy_parse_cleanup(fyp); + + fyr_error_check(fyr, fyd, err_out, "fy_parse_load_document() failed\n"); + + fy_reader_fill_atom_end(fyr, &handle); + + /* document is NULL, is a simple key */ + fyt = fy_path_token_queue(fypp, fytt, &handle, fyd); + fyr_error_check(fyr, fyt, err_out, "fy_path_token_queue() failed\n"); + + return 0; + +err_out: + fypp->stream_error = true; + return -1; +} + +int fy_path_fetch_flow_map_key(struct fy_path_parser *fypp, int c) +{ + return fy_path_fetch_flow_document(fypp, c, FYTT_PE_MAP_KEY); +} + +int fy_path_fetch_flow_scalar(struct fy_path_parser *fypp, int c) +{ + struct fy_reader *fyr; + struct fy_token *fyt; + struct fy_atom handle; + bool is_single; + int rc = -1; + + fyr = &fypp->reader; + + /* verify that the called context is correct */ + assert(fy_is_path_flow_scalar_start(c)); + + is_single = c == '\''; + + rc = fy_reader_fetch_flow_scalar_handle(fyr, c, 0, &handle, false); + if (rc) + goto err_out_rc; + + /* document is NULL, is a simple key */ + fyt = fy_path_token_queue(fypp, FYTT_SCALAR, &handle, is_single ? FYSS_SINGLE_QUOTED : FYSS_DOUBLE_QUOTED); + fyr_error_check(fyr, fyt, err_out, "fy_path_token_queue() failed\n"); + + return 0; + +err_out: + rc = -1; +err_out_rc: + fypp->stream_error = true; + return rc; +} + +int fy_path_fetch_number(struct fy_path_parser *fypp, int c) +{ + struct fy_reader *fyr; + struct fy_token *fyt; + int i, digits; + + fyr = &fypp->reader; + + /* verify that the called context is correct */ + assert(fy_is_num(c) || (c == '-' && fy_is_num(fy_reader_peek_at(fyr, 1)))); + + i = 0; + if (c == '-') + i++; + + digits = 0; + while (fy_is_num((c = fy_reader_peek_at(fyr, i)))) { + i++; + digits++; + } + FYR_PARSE_ERROR_CHECK(fyr, 0, i, FYEM_SCAN, + digits > 0, err_out, + "bad number"); + + fyt = fy_path_token_queue(fypp, FYTT_SCALAR, fy_reader_fill_atom_a(fyr, i), FYSS_PLAIN); + fyr_error_check(fyr, fyt, err_out, "fy_path_token_queue() failed\n"); + + return 0; + +err_out: + fypp->stream_error = true; + return -1; +} + +int fy_path_fetch_tokens(struct fy_path_parser *fypp) +{ + enum fy_token_type type; + struct fy_token *fyt = NULL; + struct fy_reader *fyr; + int c, cn, rc, simple_token_count; + + fyr = &fypp->reader; + if (!fypp->stream_start_produced) { + + fyt = fy_path_token_queue(fypp, FYTT_STREAM_START, fy_reader_fill_atom_a(fyr, 0)); + fyr_error_check(fyr, fyt, err_out, "fy_path_token_queue() failed\n"); + fyt = NULL; + + fypp->stream_start_produced = true; + return 0; + } + + /* XXX scan to next token? */ + + c = fy_reader_peek(fyr); + + if (fy_is_z(c)) { + + if (c >= 0) + fy_reader_advance(fyr, c); + + /* produce stream end continuously */ + fyt = fy_path_token_queue(fypp, FYTT_STREAM_END, fy_reader_fill_atom_a(fyr, 0)); + fyr_error_check(fyr, fyt, err_out, "fy_path_token_queue() failed\n"); + fyt = NULL; + + return 0; + } + + fyt = NULL; + type = FYTT_NONE; + simple_token_count = 0; + + /* first do the common tokens */ + switch (c) { + case ',': + type = FYTT_PE_COMMA; + simple_token_count = 1; + break; + + case '|': + if (fy_reader_peek_at(fyr, 1) == '|') { + type = FYTT_PE_BARBAR; + simple_token_count = 2; + break; + } + break; + + case '&': + if (fy_reader_peek_at(fyr, 1) == '&') { + type = FYTT_PE_AMPAMP; + simple_token_count = 2; + break; + } + break; + + case '(': + type = FYTT_PE_LPAREN; + simple_token_count = 1; + break; + + case ')': + type = FYTT_PE_RPAREN; + simple_token_count = 1; + break; + + case '=': + cn = fy_reader_peek_at(fyr, 1); + if (cn == '=') { + type = FYTT_PE_EQEQ; + simple_token_count = 2; + break; + } + break; + + case '>': + cn = fy_reader_peek_at(fyr, 1); + if (cn == '=') { + type = FYTT_PE_GTE; + simple_token_count = 2; + break; + } + type = FYTT_PE_GT; + simple_token_count = 1; + break; + + case '<': + cn = fy_reader_peek_at(fyr, 1); + if (cn == '=') { + type = FYTT_PE_LTE; + simple_token_count = 2; + break; + } + type = FYTT_PE_LT; + simple_token_count = 1; + break; + + case '!': + cn = fy_reader_peek_at(fyr, 1); + if (cn == '=') { + type = FYTT_PE_NOTEQ; + simple_token_count = 2; + break; + } + if (cn == '/' || cn == '(' || fy_is_alnum(cn) || cn == '!') { + /* bang bang */ + type = FYTT_PE_BANG; + simple_token_count = 1; + } + break; + + case '@': + type = FYTT_PE_AT; + simple_token_count = 1; + break; + + default: + break; + } + + if (type != FYTT_NONE) + goto do_token; + +again: + + switch (fypp->expr_mode) { + case fyem_none: + FY_IMPOSSIBLE_ABORT(); + + case fyem_path: + + switch (c) { + case '/': + type = FYTT_PE_SLASH; + simple_token_count = 1; + break; + + case '^': + type = FYTT_PE_ROOT; + simple_token_count = 1; + break; + + case ':': + type = FYTT_PE_SIBLING; + simple_token_count = 1; + break; + + case '$': + type = FYTT_PE_SCALAR_FILTER; + simple_token_count = 1; + break; + + case '%': + type = FYTT_PE_COLLECTION_FILTER; + simple_token_count = 1; + break; + + case '[': + if (fy_reader_peek_at(fyr, 1) == ']') { + type = FYTT_PE_SEQ_FILTER; + simple_token_count = 2; + } + break; + + case '{': + if (fy_reader_peek_at(fyr, 1) == '}') { + type = FYTT_PE_MAP_FILTER; + simple_token_count = 2; + } + break; + + + case '.': + cn = fy_reader_peek_at(fyr, 1); + if (cn == '.') { + type = FYTT_PE_PARENT; + simple_token_count = 2; + } else if (!fy_is_first_alpha(cn)) { + type = FYTT_PE_THIS; + simple_token_count = 1; + } + break; + + case '*': + if (fy_reader_peek_at(fyr, 1) == '*') { + type = FYTT_PE_EVERY_CHILD_R; + simple_token_count = 2; + } else if (!fy_is_first_alpha(fy_reader_peek_at(fyr, 1))) { + type = FYTT_PE_EVERY_CHILD; + simple_token_count = 1; + } else { + type = FYTT_PE_ALIAS; + simple_token_count = 2; + while (fy_is_alnum(fy_reader_peek_at(fyr, simple_token_count))) + simple_token_count++; + } + break; + + case '!': + cn = fy_reader_peek_at(fyr, 1); + if (cn == '=') { + type = FYTT_PE_NOTEQ; + simple_token_count = 2; + break; + } + type = FYTT_PE_UNIQUE_FILTER; + simple_token_count = 1; + break; + + default: + break; + } + break; + + case fyem_scalar: + + + /* it is possible for the expression to be a path + * we only detect a few cases (doing all too complex) + * (/ , (./ , (* + */ + if (fy_token_type_can_be_path_expr(fypp->last_queued_token_type)) { + cn = fy_reader_peek_at(fyr, 1); + if (c == '/' || + (c == '.' && (cn == '/' || cn == ')' || cn == '>' || cn == '<' || cn == '!' || cn == '='))) { + fypp->expr_mode = fyem_path; +#ifdef DEBUG_EXPR + fyr_notice(fyr, "switching to path expr\n"); +#endif + goto again; + } + } + + switch (c) { + case '+': + type = FYTT_SE_PLUS; + simple_token_count = 1; + break; + + case '-': + cn = fy_reader_peek_at(fyr, 1); + if (fy_is_num(cn) && + fy_token_type_can_be_before_negative_number(fypp->last_queued_token_type)) + break; + + type = FYTT_SE_MINUS; + simple_token_count = 1; + break; + + case '*': + type = FYTT_SE_MULT; + simple_token_count = 1; + break; + + case '/': + type = FYTT_SE_DIV; + simple_token_count = 1; + break; + + default: + break; + } + break; + } + +do_token: + /* simple tokens */ + if (simple_token_count > 0) { + fyt = fy_path_token_queue(fypp, type, fy_reader_fill_atom_a(fyr, simple_token_count)); + fyr_error_check(fyr, fyt, err_out, "fy_path_token_queue() failed\n"); + fyt = NULL; + + return 0; + } + + switch (fypp->expr_mode) { + case fyem_none: + FY_IMPOSSIBLE_ABORT(); + + case fyem_path: + if (fy_is_first_alpha(c)) + return fy_path_fetch_plain_or_method(fypp, c, FYTT_PE_MAP_KEY, FYTT_PE_METHOD); + + if (fy_is_path_flow_key_start(c)) + return fy_path_fetch_flow_map_key(fypp, c); + + if (fy_is_num(c) || (c == '-' && fy_is_num(fy_reader_peek_at(fyr, 1)))) + return fy_path_fetch_seq_index_or_slice(fypp, c); + + if (c == '.' && fy_is_first_alpha(fy_reader_peek_at(fyr, 1))) + return fy_path_fetch_dot_method(fypp, c, FYTT_PE_METHOD); + + break; + + case fyem_scalar: + + if (fy_is_first_alpha(c)) + return fy_path_fetch_plain_or_method(fypp, c, FYTT_SCALAR, FYTT_SE_METHOD); + + if (fy_is_path_flow_scalar_start(c)) + return fy_path_fetch_flow_scalar(fypp, c); + + if (fy_is_num(c) || (c == '-' && fy_is_num(fy_reader_peek_at(fyr, 1)))) + return fy_path_fetch_number(fypp, c); + +#if 0 + if (c == '.' && fy_is_first_alpha(fy_reader_peek_at(fyr, 1))) + return fy_path_fetch_dot_method(fypp, c, FYTT_SE_METHOD); +#endif + + break; + } + + FYR_PARSE_ERROR(fyr, 0, 1, FYEM_SCAN, "bad path expression starts here c=%d", c); + +err_out: + fypp->stream_error = true; + rc = -1; + return rc; +} + +struct fy_token *fy_path_scan_peek(struct fy_path_parser *fypp, struct fy_token *fyt_prev) +{ + struct fy_token *fyt; + struct fy_reader *fyr; + int rc, last_token_activity_counter; + + fyr = &fypp->reader; + + /* nothing if stream end produced (and no stream end token in queue) */ + if (!fyt_prev && fypp->stream_end_produced && fy_token_list_empty(&fypp->queued_tokens)) { + + fyt = fy_token_list_head(&fypp->queued_tokens); + if (fyt && fyt->type == FYTT_STREAM_END) + return fyt; + + return NULL; + } + + for (;;) { + if (!fyt_prev) + fyt = fy_token_list_head(&fypp->queued_tokens); + else + fyt = fy_token_next(&fypp->queued_tokens, fyt_prev); + if (fyt) + break; + + /* on stream error we're done */ + if (fypp->stream_error) + return NULL; + + /* keep track of token activity, if it didn't change + * after the fetch tokens call, the state machine is stuck + */ + last_token_activity_counter = fypp->token_activity_counter; + + /* fetch more then */ + rc = fy_path_fetch_tokens(fypp); + if (rc) { + fy_error(fypp->cfg.diag, "fy_path_fetch_tokens() failed\n"); + goto err_out; + } + if (last_token_activity_counter == fypp->token_activity_counter) { + fy_error(fypp->cfg.diag, "out of tokens and failed to produce anymore"); + goto err_out; + } + } + + switch (fyt->type) { + case FYTT_STREAM_START: + fypp->stream_start_produced = true; + break; + case FYTT_STREAM_END: + fypp->stream_end_produced = true; + + rc = fy_reader_input_done(fyr); + if (rc) { + fy_error(fypp->cfg.diag, "fy_parse_input_done() failed"); + goto err_out; + } + break; + default: + break; + } + + return fyt; + +err_out: + fypp->stream_error = true; + return NULL; +} + +struct fy_token *fy_path_scan_remove(struct fy_path_parser *fypp, struct fy_token *fyt) +{ + if (!fypp || !fyt) + return NULL; + + fy_token_list_del(&fypp->queued_tokens, fyt); + + return fyt; +} + +struct fy_token *fy_path_scan_remove_peek(struct fy_path_parser *fypp, struct fy_token *fyt) +{ + fy_token_unref(fy_path_scan_remove(fypp, fyt)); + + return fy_path_scan_peek(fypp, NULL); +} + +struct fy_token *fy_path_scan(struct fy_path_parser *fypp) +{ + return fy_path_scan_remove(fypp, fy_path_scan_peek(fypp, NULL)); +} + +void fy_path_expr_dump(struct fy_path_expr *expr, struct fy_diag *diag, enum fy_error_type errlevel, int level, const char *banner) +{ + struct fy_path_expr *expr2; + const char *style = ""; + const char *text; + size_t len; + bool save_on_error; + + if (errlevel < diag->cfg.level) + return; + + save_on_error = diag->on_error; + diag->on_error = true; + + if (banner) + fy_diag_diag(diag, errlevel, "%-*s%s", level*2, "", banner); + + text = fy_token_get_text(expr->fyt, &len); + + style = ""; + if (expr->type == fpet_scalar) { + switch (fy_scalar_token_get_style(expr->fyt)) { + case FYSS_SINGLE_QUOTED: + style = "'"; + break; + case FYSS_DOUBLE_QUOTED: + style = "\""; + break; + default: + style = ""; + break; + } + } + + fy_diag_diag(diag, errlevel, "> %-*s%s:%s %s%.*s%s", + level*2, "", + fy_path_expr_type_txt[expr->type], + fy_expr_mode_txt[expr->expr_mode], + style, (int)len, text, style); + + for (expr2 = fy_path_expr_list_head(&expr->children); expr2; expr2 = fy_path_expr_next(&expr->children, expr2)) + fy_path_expr_dump(expr2, diag, errlevel, level + 1, NULL); + + diag->on_error = save_on_error; +} + +static struct fy_node * +fy_path_expr_to_node_internal(struct fy_document *fyd, struct fy_path_expr *expr) +{ + struct fy_path_expr *expr2; + const char *style = ""; + const char *text; + size_t len; + struct fy_node *fyn = NULL, *fyn2, *fyn_seq = NULL; + int rc; + + text = fy_token_get_text(expr->fyt, &len); + + /* by default use double quoted style */ + style = "\""; + switch (expr->type) { + case fpet_scalar: + switch (fy_scalar_token_get_style(expr->fyt)) { + case FYSS_SINGLE_QUOTED: + style = "'"; + break; + case FYSS_DOUBLE_QUOTED: + style = "\""; + break; + default: + style = ""; + break; + } + break; + + case fpet_map_key: + /* no styles for complex map keys */ + if (expr->fyt->map_key.fyd) + style = ""; + break; + + default: + break; + } + + /* list is empty this is a terminal */ + if (fy_path_expr_list_empty(&expr->children) && + expr->type != fpet_method) { + + fyn = fy_node_buildf(fyd, "%s: %s%.*s%s", + fy_path_expr_type_txt[expr->type], + style, (int)len, text, style); + if (!fyn) + return NULL; + + return fyn; + } + + fyn = fy_node_create_mapping(fyd); + if (!fyn) + goto err_out; + + fyn_seq = fy_node_create_sequence(fyd); + if (!fyn_seq) + goto err_out; + + for (expr2 = fy_path_expr_list_head(&expr->children); expr2; expr2 = fy_path_expr_next(&expr->children, expr2)) { + fyn2 = fy_path_expr_to_node_internal(fyd, expr2); + if (!fyn2) + goto err_out; + rc = fy_node_sequence_append(fyn_seq, fyn2); + if (rc) + goto err_out; + } + + if (expr->type != fpet_method) { + rc = fy_node_mapping_append(fyn, + fy_node_create_scalar(fyd, fy_path_expr_type_txt[expr->type], FY_NT), + fyn_seq); + } else { + rc = fy_node_mapping_append(fyn, + fy_node_create_scalarf(fyd, "%s()", expr->fym->name), + fyn_seq); + } + if (rc) + goto err_out; + + return fyn; + +err_out: + fy_node_free(fyn_seq); + fy_node_free(fyn); + return NULL; +} + +struct fy_document *fy_path_expr_to_document(struct fy_path_expr *expr) +{ + struct fy_document *fyd = NULL; + + if (!expr) + return NULL; + + fyd = fy_document_create(NULL); + if (!fyd) + return NULL; + + fyd->root = fy_path_expr_to_node_internal(fyd, expr); + if (!fyd->root) + goto err_out; + + return fyd; + +err_out: + fy_document_destroy(fyd); + return NULL; +} + +enum fy_path_expr_type fy_map_token_to_path_expr_type(enum fy_token_type type, enum fy_expr_mode mode) +{ + switch (type) { + case FYTT_PE_ROOT: + return fpet_root; + case FYTT_PE_THIS: + return fpet_this; + case FYTT_PE_PARENT: + case FYTT_PE_SIBLING: /* sibling maps to a chain of fpet_parent */ + return fpet_parent; + case FYTT_PE_MAP_KEY: + return fpet_map_key; + case FYTT_PE_SEQ_INDEX: + return fpet_seq_index; + case FYTT_PE_SEQ_SLICE: + return fpet_seq_slice; + case FYTT_PE_EVERY_CHILD: + return fpet_every_child; + case FYTT_PE_EVERY_CHILD_R: + return fpet_every_child_r; + case FYTT_PE_ALIAS: + return fpet_alias; + case FYTT_PE_SCALAR_FILTER: + return fpet_filter_scalar; + case FYTT_PE_COLLECTION_FILTER: + return fpet_filter_collection; + case FYTT_PE_SEQ_FILTER: + return fpet_filter_sequence; + case FYTT_PE_MAP_FILTER: + return fpet_filter_mapping; + case FYTT_PE_UNIQUE_FILTER: + return fpet_filter_unique; + case FYTT_PE_COMMA: + return mode == fyem_path ? fpet_multi : fpet_arg_separator; + case FYTT_PE_SLASH: + return fpet_chain; + case FYTT_PE_BARBAR: + return fpet_logical_or; + case FYTT_PE_AMPAMP: + return fpet_logical_and; + + case FYTT_PE_EQEQ: + return fpet_eq; + case FYTT_PE_NOTEQ: + return fpet_neq; + case FYTT_PE_LT: + return fpet_lt; + case FYTT_PE_GT: + return fpet_gt; + case FYTT_PE_LTE: + return fpet_lte; + case FYTT_PE_GTE: + return fpet_gte; + + case FYTT_SCALAR: + return fpet_scalar; + + case FYTT_SE_PLUS: + return fpet_plus; + case FYTT_SE_MINUS: + return fpet_minus; + case FYTT_SE_MULT: + return fpet_mult; + case FYTT_SE_DIV: + return fpet_div; + + case FYTT_PE_LPAREN: + return fpet_lparen; + case FYTT_PE_RPAREN: + return fpet_rparen; + + case FYTT_SE_METHOD: + case FYTT_PE_METHOD: + return fpet_method; + + case FYTT_PE_BANG: + return fpet_unselect; + + case FYTT_PE_AT: + return fpet_select; + + default: + FY_IMPOSSIBLE_ABORT(); + } + return fpet_none; +} + +bool fy_token_type_is_operand(enum fy_token_type type) +{ + return type == FYTT_PE_ROOT || + type == FYTT_PE_THIS || + type == FYTT_PE_PARENT || + type == FYTT_PE_MAP_KEY || + type == FYTT_PE_SEQ_INDEX || + type == FYTT_PE_SEQ_SLICE || + type == FYTT_PE_EVERY_CHILD || + type == FYTT_PE_EVERY_CHILD_R || + type == FYTT_PE_ALIAS || + + type == FYTT_SCALAR; +} + +bool fy_token_type_is_operator(enum fy_token_type type) +{ + return type == FYTT_PE_SLASH || + type == FYTT_PE_SCALAR_FILTER || + type == FYTT_PE_COLLECTION_FILTER || + type == FYTT_PE_SEQ_FILTER || + type == FYTT_PE_MAP_FILTER || + type == FYTT_PE_UNIQUE_FILTER || + type == FYTT_PE_SIBLING || + type == FYTT_PE_COMMA || + type == FYTT_PE_BARBAR || + type == FYTT_PE_AMPAMP || + type == FYTT_PE_LPAREN || + type == FYTT_PE_RPAREN || + + type == FYTT_PE_EQEQ || + type == FYTT_PE_NOTEQ || + type == FYTT_PE_LT || + type == FYTT_PE_GT || + type == FYTT_PE_LTE || + type == FYTT_PE_GTE || + + type == FYTT_SE_PLUS || + type == FYTT_SE_MINUS || + type == FYTT_SE_MULT || + type == FYTT_SE_DIV || + + type == FYTT_PE_BANG; +} + +bool fy_token_type_is_operand_or_operator(enum fy_token_type type) +{ + return fy_token_type_is_operand(type) || + fy_token_type_is_operator(type); +} + +int fy_path_expr_type_prec(enum fy_path_expr_type type) +{ + switch (type) { + default: + return -1; /* terminals */ + case fpet_filter_collection: + case fpet_filter_scalar: + case fpet_filter_sequence: + case fpet_filter_mapping: + case fpet_filter_unique: + return 5; + case fpet_logical_or: + case fpet_logical_and: + return 4; + case fpet_multi: + return 21; + case fpet_eq: + case fpet_neq: + case fpet_lt: + case fpet_gt: + case fpet_lte: + case fpet_gte: + return 7; + case fpet_mult: + case fpet_div: + return 9; + case fpet_plus: + case fpet_minus: + return 8; + case fpet_chain: + return 20; + case fpet_select: + case fpet_unselect: + return 10; /* must be less than chain */ + case fpet_lparen: + case fpet_rparen: + case fpet_method: + return 1000; + case fpet_arg_separator: + return 1; /* lowest */ + } + return -1; +} + +static inline FY_UNUSED void +dump_operand_stack(struct fy_path_parser *fypp) +{ + return fy_expr_stack_dump(fypp->cfg.diag, &fypp->operands); +} + +static inline int +push_operand(struct fy_path_parser *fypp, struct fy_path_expr *expr) +{ + return fy_expr_stack_push(&fypp->operands, expr); +} + +static inline FY_UNUSED struct fy_path_expr * +peek_operand_at(struct fy_path_parser *fypp, unsigned int pos) +{ + return fy_expr_stack_peek_at(&fypp->operands, pos); +} + +static inline FY_UNUSED struct fy_path_expr * +peek_operand(struct fy_path_parser *fypp) +{ + return fy_expr_stack_peek(&fypp->operands); +} + +static inline FY_UNUSED struct fy_path_expr * +pop_operand(struct fy_path_parser *fypp) +{ + return fy_expr_stack_pop(&fypp->operands); +} + +#define PREFIX 0 +#define INFIX 1 +#define POSTFIX 2 + +int fy_token_type_operator_placement(enum fy_token_type type) +{ + switch (type) { + case FYTT_PE_SLASH: /* SLASH is special at the start of the expression */ + case FYTT_PE_COMMA: + case FYTT_PE_BARBAR: + case FYTT_PE_AMPAMP: + case FYTT_PE_EQEQ: + case FYTT_PE_NOTEQ: + case FYTT_PE_LT: + case FYTT_PE_GT: + case FYTT_PE_LTE: + case FYTT_PE_GTE: + case FYTT_SE_PLUS: + case FYTT_SE_MINUS: + case FYTT_SE_MULT: + case FYTT_SE_DIV: + return INFIX; + case FYTT_PE_SCALAR_FILTER: + case FYTT_PE_COLLECTION_FILTER: + case FYTT_PE_SEQ_FILTER: + case FYTT_PE_MAP_FILTER: + case FYTT_PE_UNIQUE_FILTER: + return POSTFIX; + case FYTT_PE_SIBLING: + case FYTT_PE_BANG: + return PREFIX; + default: + break; + } + return -1; +} + +int fy_path_expr_type_assoc(enum fy_path_expr_type type) +{ + switch (type) { + case fpet_root: + case fpet_select: + case fpet_unselect: + return PREFIX; + + case fpet_this: + case fpet_parent: + case fpet_every_child: + case fpet_every_child_r: + case fpet_filter_collection: + case fpet_filter_scalar: + case fpet_filter_sequence: + case fpet_filter_mapping: + case fpet_filter_unique: + case fpet_seq_index: + case fpet_map_key: + case fpet_seq_slice: + case fpet_alias: + return POSTFIX; + + case fpet_logical_or: + case fpet_logical_and: + case fpet_eq: + case fpet_neq: + case fpet_lt: + case fpet_gt: + case fpet_lte: + case fpet_gte: + case fpet_plus: + case fpet_minus: + case fpet_mult: + case fpet_div: + case fpet_arg_separator: /* argument separator (comma in scalar mode) */ + return INFIX; + + case fpet_scalar: + case fpet_lparen: + case fpet_rparen: + case fpet_method: + case fpet_scalar_expr: + case fpet_path_expr: + default: + break; + } + return -1; +} + +const struct fy_mark *fy_path_expr_start_mark(struct fy_path_expr *expr) +{ + if (!expr) + return NULL; + + return fy_token_start_mark(expr->fyt); +} + +const struct fy_mark *fy_path_expr_end_mark(struct fy_path_expr *expr) +{ + if (!expr) + return NULL; + + return fy_token_end_mark(expr->fyt); +} + +struct fy_token * +expr_to_token_mark(struct fy_path_expr *expr, struct fy_input *fyi) +{ + const struct fy_mark *ms, *me; + struct fy_atom handle; + + if (!expr || !fyi) + return NULL; + + ms = fy_path_expr_start_mark(expr); + assert(ms); + me = fy_path_expr_end_mark(expr); + assert(me); + + memset(&handle, 0, sizeof(handle)); + handle.start_mark = *ms; + handle.end_mark = *me; + handle.fyi = fyi; + handle.style = FYAS_PLAIN; + handle.chomp = FYAC_CLIP; + + return fy_token_create(FYTT_INPUT_MARKER, &handle); +} + +struct fy_token * +expr_lr_to_token_mark(struct fy_path_expr *exprl, struct fy_path_expr *exprr, struct fy_input *fyi) +{ + const struct fy_mark *ms, *me; + struct fy_atom handle; + + if (!exprl || !exprr || !fyi) + return NULL; + + ms = fy_path_expr_start_mark(exprl); + assert(ms); + me = fy_path_expr_end_mark(exprr); + assert(me); + + memset(&handle, 0, sizeof(handle)); + handle.start_mark = *ms; + handle.end_mark = *me; + handle.fyi = fyi; + handle.style = FYAS_PLAIN; + handle.chomp = FYAC_CLIP; + + return fy_token_create(FYTT_INPUT_MARKER, &handle); +} + +int +fy_path_expr_order(struct fy_path_expr *expr1, struct fy_path_expr *expr2) +{ + const struct fy_mark *m1 = NULL, *m2 = NULL; + + if (expr1) + m1 = fy_path_expr_start_mark(expr1); + + if (expr2) + m2 = fy_path_expr_start_mark(expr2); + + if (m1 == m2) + return 0; + + if (!m1) + return -1; + + if (!m2) + return 1; + + return m1->input_pos == m2->input_pos ? 0 : + m1->input_pos < m2->input_pos ? -1 : 1; +} + +int push_operand_lr(struct fy_path_parser *fypp, + enum fy_path_expr_type type, + struct fy_path_expr *exprl, struct fy_path_expr *exprr, + bool optimize) +{ + struct fy_reader *fyr; + struct fy_path_expr *expr = NULL, *exprt; + const struct fy_mark *ms = NULL, *me = NULL; + struct fy_atom handle; + int ret; + + optimize = false; + assert(exprl || exprr); + + fyr = &fypp->reader; + +#if 0 + fyr_notice(fyr, ">>> %s <%s> l=<%s> r=<%s>\n", __func__, + fy_path_expr_type_txt[type], + exprl ?fy_path_expr_type_txt[exprl->type] : "NULL", + exprr ?fy_path_expr_type_txt[exprr->type] : "NULL"); +#endif + + expr = fy_path_expr_alloc_recycle(fypp); + fyr_error_check(fyr, expr, err_out, + "fy_path_expr_alloc_recycle() failed\n"); + + expr->type = type; + expr->fyt = NULL; + + if (exprl) { + assert(exprl->fyt); + ms = fy_token_start_mark(exprl->fyt); + assert(ms); + } else { + ms = fy_token_start_mark(exprr->fyt); + assert(ms); + } + + if (exprr) { + assert(exprr->fyt); + me = fy_token_end_mark(exprr->fyt); + assert(me); + } else { + me = fy_token_end_mark(exprl->fyt); + assert(me); + } + + assert(ms && me); + + memset(&handle, 0, sizeof(handle)); + handle.start_mark = *ms; + handle.end_mark = *me; + handle.fyi = fypp->fyi; + handle.style = FYAS_PLAIN; + handle.chomp = FYAC_CLIP; + + if (exprl) { + if (type == exprl->type && fy_path_expr_type_is_mergeable(type)) { + while ((exprt = fy_path_expr_list_pop(&exprl->children)) != NULL) { + fy_path_expr_list_add_tail(&expr->children, exprt); + exprt->parent = expr; + } + fy_path_expr_free_recycle(fypp, exprl); + } else { + fy_path_expr_list_add_tail(&expr->children, exprl); + exprl->parent = expr; + } + exprl = NULL; + } + + if (exprr) { + if (type == exprr->type && fy_path_expr_type_is_mergeable(type)) { + while ((exprt = fy_path_expr_list_pop(&exprr->children)) != NULL) { + fy_path_expr_list_add_tail(&expr->children, exprt); + exprt->parent = expr; + } + fy_path_expr_free_recycle(fypp, exprr); + } else { + fy_path_expr_list_add_tail(&expr->children, exprr); + exprr->parent = expr; + } + exprr = NULL; + } + + expr->fyt = fy_token_create(FYTT_INPUT_MARKER, &handle); + fyr_error_check(fyr, expr->fyt, err_out, + "expr_to_token_mark() failed\n"); + + ret = push_operand(fypp, expr); + fyr_error_check(fyr, !ret, err_out, + "push_operand() failed\n"); + +#ifdef DEBUG_EXPR + FYR_TOKEN_DIAG(fyr, expr->fyt, + FYDF_NOTICE, FYEM_PARSE, "pushed operand lr"); +#endif + + return 0; +err_out: + fy_path_expr_free(expr); + fy_path_expr_free(exprl); + fy_path_expr_free(exprr); + return -1; +} + +enum fy_method_idx { + fymi_test, + fymi_sum, + fymi_this, + fymi_parent, + fymi_root, + fymi_any, + fymi_all, + fymi_select, + fymi_key, + fymi_value, + fymi_index, + fymi_null, +}; + +static struct fy_walk_result * +common_builtin_ref_exec(const struct fy_method *fym, + struct fy_path_exec *fypx, int level, + struct fy_path_expr *expr, + struct fy_walk_result *input, + struct fy_walk_result **args, int nargs); + +static struct fy_walk_result * +common_builtin_collection_exec(const struct fy_method *fym, + struct fy_path_exec *fypx, int level, + struct fy_path_expr *expr, + struct fy_walk_result *input, + struct fy_walk_result **args, int nargs); + + +static struct fy_walk_result * +test_exec(const struct fy_method *fym, + struct fy_path_exec *fypx, int level, + struct fy_path_expr *expr, + struct fy_walk_result *input, + struct fy_walk_result **args, int nargs); + +static struct fy_walk_result * +sum_exec(const struct fy_method *fym, + struct fy_path_exec *fypx, int level, + struct fy_path_expr *expr, + struct fy_walk_result *input, + struct fy_walk_result **args, int nargs); + +static const struct fy_method fy_methods[] = { + [fymi_test] = { + .name = "test", + .len = 4, + .mode = fyem_scalar, + .nargs = 1, + .exec = test_exec, + }, + [fymi_sum] = { + .name = "sum", + .len = 3, + .mode = fyem_scalar, + .nargs = 2, + .exec = sum_exec, + }, + [fymi_this] = { + .name = "this", + .len = 4, + .mode = fyem_path, + .nargs = 0, + .exec = common_builtin_ref_exec, + }, + [fymi_parent] = { + .name = "parent", + .len = 6, + .mode = fyem_path, + .nargs = 0, + .exec = common_builtin_ref_exec, + }, + [fymi_root] = { + .name = "root", + .len = 4, + .mode = fyem_path, + .nargs = 0, + .exec = common_builtin_ref_exec, + }, + [fymi_any] = { + .name = "any", + .len = 3, + .mode = fyem_path, + .nargs = 1, + .exec = common_builtin_collection_exec, + }, + [fymi_all] = { + .name = "all", + .len = 3, + .mode = fyem_path, + .nargs = 1, + .exec = common_builtin_collection_exec, + }, + [fymi_select] = { + .name = "select", + .len = 6, + .mode = fyem_path, + .nargs = 1, + .exec = common_builtin_collection_exec, + }, + [fymi_key] = { + .name = "key", + .len = 3, + .mode = fyem_path, + .nargs = 1, + .exec = common_builtin_ref_exec, + }, + [fymi_value] = { + .name = "value", + .len = 5, + .mode = fyem_path, + .nargs = 1, + .exec = common_builtin_ref_exec, + }, + [fymi_index] = { + .name = "index", + .len = 5, + .mode = fyem_path, + .nargs = 1, + .exec = common_builtin_ref_exec, + }, + [fymi_null] = { + .name = "null", + .len = 4, + .mode = fyem_path, + .nargs = 0, + .exec = common_builtin_ref_exec, + }, +}; + +static inline int fy_method_to_builtin_idx(const struct fy_method *fym) +{ + if (!fym || fym < fy_methods || fym >= &fy_methods[ARRAY_SIZE(fy_methods)]) + return -1; + return fym - fy_methods; +} + +static struct fy_walk_result * +common_builtin_ref_exec(const struct fy_method *fym, + struct fy_path_exec *fypx, int level, + struct fy_path_expr *expr, + struct fy_walk_result *input, + struct fy_walk_result **args, int nargs) +{ + enum fy_method_idx midx; + struct fy_walk_result *output = NULL; + struct fy_walk_result *fwr, *fwrn; + struct fy_node *fyn, *fynt; + int i; + + if (!fypx || !input) + goto out; + + i = fy_method_to_builtin_idx(fym); + if (i < 0) + goto out; + midx = (enum fy_method_idx)i; + + switch (midx) { + case fymi_key: + case fymi_value: + if (!args || nargs != 1 || !args[0] || args[0]->type != fwrt_string) + goto out; + break; + case fymi_index: + if (!args || nargs != 1 || !args[0] || args[0]->type != fwrt_number) + goto out; + break; + case fymi_root: + case fymi_parent: + case fymi_this: + case fymi_null: + if (nargs != 0) + goto out; + break; + default: + goto out; + } + + output = fy_path_exec_walk_result_create(fypx, fwrt_refs); + assert(output); + + for (fwr = fy_walk_result_iter_start(input); fwr; + fwr = fy_walk_result_iter_next(input, fwr)) { + + if (fwr->type != fwrt_node_ref || !fwr->fyn) + continue; + + fynt = fwr->fyn; + + switch (midx) { + case fymi_key: + case fymi_value: + case fymi_index: + case fymi_this: + case fymi_null: + /* dereference alias */ + if (fy_node_is_alias(fynt)) { + // fprintf(stderr, "%s: %s calling fy_node_alias_resolve_by_ypath()\n", __func__, fy_node_get_path_alloca(fyn)); + fynt = fy_node_alias_resolve_by_ypath(fynt); + } + break; + default: + break; + } + + fyn = NULL; + switch (midx) { + case fymi_key: + if (!fy_node_is_mapping(fynt)) + break; + fyn = fy_node_mapping_lookup_key_by_string(fynt, args[0]->string, FY_NT); + break; + + case fymi_value: + if (!fy_node_is_mapping(fynt)) + break; + fyn = fy_node_mapping_lookup_by_string(fynt, args[0]->string, FY_NT); + break; + + case fymi_index: + if (!fy_node_is_sequence(fynt)) + break; + fyn = fy_node_sequence_get_by_index(fynt, (int)args[0]->number); + break; + + case fymi_root: + fyn = fy_document_root(fy_node_document(fynt)); + break; + + case fymi_parent: + fyn = fy_node_get_parent(fynt); + break; + + case fymi_null: + if (!fy_node_is_mapping(fynt)) + break; + fyn = fy_node_mapping_lookup_value_by_null_key(fynt); + break; + + case fymi_this: + fyn = fynt; + break; + + default: + break; + } + + if (!fyn) + continue; + + fwrn = fy_path_exec_walk_result_create(fypx, fwrt_node_ref, fyn); + assert(fwrn); + + fy_walk_result_list_add_tail(&output->refs, fwrn); + } + + /* output convert zero to NULL, singular to node_ref */ + if (output && output->type == fwrt_refs) { + if (fy_walk_result_list_empty(&output->refs)) { + fy_walk_result_free(output); + output = NULL; + } else if (fy_walk_result_list_is_singular(&output->refs)) { + fwr = fy_walk_result_list_pop(&output->refs); + assert(fwr); + fy_walk_result_free(output); + output = fwr; + } + } + +out: + fy_walk_result_free(input); + if (args) { + for (i = 0; i < nargs; i++) + fy_walk_result_free(args[i]); + } + return output; +} + +static struct fy_walk_result * +common_builtin_collection_exec(const struct fy_method *fym, + struct fy_path_exec *fypx, int level, + struct fy_path_expr *expr, + struct fy_walk_result *input, + struct fy_walk_result **args, int nargs) +{ + enum fy_method_idx midx; + struct fy_walk_result *output = NULL; + struct fy_walk_result *fwr, *fwrn, *fwrt; + struct fy_path_expr *expr_arg; + bool match, done; + int input_count, match_count; + int i; + + if (!fypx || !input) + goto out; + + i = fy_method_to_builtin_idx(fym); + if (i < 0) + goto out; + midx = (enum fy_method_idx)i; + + switch (midx) { + case fymi_any: + case fymi_all: + case fymi_select: + if (!args || nargs != 1 || !args[0]) + goto out; + break; + default: + goto out; + } + + expr_arg = fy_path_expr_list_head(&expr->children); + assert(expr_arg); + + /* only handle inputs of node and refs */ + if (input->type != fwrt_node_ref && input->type != fwrt_refs) + goto out; + + output = NULL; + + switch (midx) { + case fymi_select: + output = fy_path_exec_walk_result_create(fypx, fwrt_refs); + assert(output); + break; + default: + break; + } + + + done = false; + match_count = input_count = 0; + for (fwr = fy_walk_result_iter_start(input); fwr && !done; fwr = fy_walk_result_iter_next(input, fwr)) { + + input_count++; + + fwrt = fy_walk_result_clone(fwr); + assert(fwrt); + + fwrn = fy_path_expr_execute(fypx, level + 1, expr_arg, fwrt, expr->type); + + match = fwrn != NULL; + if (match) + match_count++; + + switch (midx) { + case fymi_any: + /* on any match, we're done */ + if (match) + done = true; + break; + + case fymi_all: + /* on any non match, we're done */ + if (!match) + done = true; + break; + + case fymi_select: + /* select only works on node refs */ + if (fwr->type != fwrt_node_ref) + break; + if (match) { + fwrt = fy_walk_result_clone(fwr); + assert(fwrt); + fy_walk_result_list_add_tail(&output->refs, fwrt); + } + break; + + default: + break; + } + + if (fwrn) + fy_walk_result_free(fwrn); + } + + switch (midx) { + case fymi_any: + if (input_count > 0 && match_count <= input_count) { + output = input; + input = NULL; + } + break; + case fymi_all: + if (input_count > 0 && match_count == input_count) { + output = input; + input = NULL; + } + break; + + default: + break; + } + + /* output convert zero to NULL, singular to node_ref */ + if (output && output->type == fwrt_refs) { + if (fy_walk_result_list_empty(&output->refs)) { + fy_walk_result_free(output); + output = NULL; + } else if (fy_walk_result_list_is_singular(&output->refs)) { + fwr = fy_walk_result_list_pop(&output->refs); + assert(fwr); + fy_walk_result_free(output); + output = fwr; + } + } + +out: + if (input) + fy_walk_result_free(input); + if (args) { + for (i = 0; i < nargs; i++) + fy_walk_result_free(args[i]); + } + + return output; +} + +static struct fy_walk_result * +test_exec(const struct fy_method *fym, + struct fy_path_exec *fypx, int level, + struct fy_path_expr *expr, + struct fy_walk_result *input, + struct fy_walk_result **args, int nargs) +{ + int i; + struct fy_walk_result *output = NULL; + + if (!fypx || !args || nargs != 1) + goto out; + + /* require a single number argument */ + if (!args[0] || args[0]->type != fwrt_number) + goto out; + + /* reuse argument */ + output = args[0]; + args[0] = NULL; + + /* add 1 to the number */ + output->number += 1; + +out: + fy_walk_result_free(input); + if (args) { + for (i = 0; i < nargs; i++) + fy_walk_result_free(args[i]); + } + return output; +} + +static struct fy_walk_result * +sum_exec(const struct fy_method *fym, + struct fy_path_exec *fypx, int level, + struct fy_path_expr *expr, + struct fy_walk_result *input, + struct fy_walk_result **args, int nargs) +{ + int i; + struct fy_walk_result *output = NULL; + + if (!fypx || !args || nargs != 2) + goto out; + + /* require two number argument */ + if (!args[0] || args[0]->type != fwrt_number || + !args[1] || args[1]->type != fwrt_number) + goto out; + + /* reuse argument */ + output = args[0]; + args[0] = NULL; + + /* add 1 to the number */ + output->number += args[1]->number; + +out: + fy_walk_result_free(input); + if (args) { + for (i = 0; i < nargs; i++) + fy_walk_result_free(args[i]); + } + return output; +} + +int evaluate_method(struct fy_path_parser *fypp, struct fy_path_expr *exprm, + struct fy_path_expr *exprl, struct fy_path_expr *exprr) +{ + struct fy_reader *fyr; + struct fy_path_expr *exprt; + struct fy_token *fyt; + const char *text; + size_t len; + const struct fy_method *fym; + unsigned int i, count; + int ret; + + fyr = &fypp->reader; + +#ifdef DEBUG_EXPR + FYR_TOKEN_DIAG(fyr, exprm->fyt, + FYDF_NOTICE, FYEM_PARSE, "evaluating method"); +#endif + + text = fy_token_get_text(exprm->fyt, &len); + fyr_error_check(fyr, text, err_out, + "fy_token_get_text() failed\n"); + + for (i = 0, fym = fy_methods; i < ARRAY_SIZE(fy_methods); i++, fym++) { + if (fym->len == len && !memcmp(text, fym->name, len)) + break; + } + + FYR_TOKEN_ERROR_CHECK(fyr, exprm->fyt, FYEM_PARSE, + i < ARRAY_SIZE(fy_methods), err_out, + "invalid method %.*s\n", (int)len, text); + + /* reuse exprm */ + count = 0; + while ((exprt = fy_expr_stack_peek(&fypp->operands)) != NULL && + fy_path_expr_order(exprm, exprt) < 0) { + + exprt = fy_expr_stack_pop(&fypp->operands); + assert(exprt); + +#ifdef DEBUG_EXPR + FYR_TOKEN_DIAG(fyr, exprt->fyt, + FYDF_NOTICE, FYEM_PARSE, "poped argument %d", count); +#endif + + /* add in reverse order */ + fy_path_expr_list_add(&exprm->children, exprt); + exprt->parent = exprm; + count++; + + } + + if (exprr) { + fyt = expr_lr_to_token_mark(exprm, exprr, fypp->fyi); + fyr_error_check(fyr, fyt, err_out, + "expr_lr_to_token_mark() failed\n"); + + fy_token_unref(exprm->fyt); + exprm->fyt = fyt; + } + + FYR_TOKEN_ERROR_CHECK(fyr, exprm->fyt, FYEM_PARSE, + fym->nargs == count, err_out, + "too %s argument for method %s, expected %d, got %d\n", + fym->nargs < count ? "many" : "few", + fym->name, fym->nargs, count); + + exprm->fym = fym; + + if (exprl) + fy_path_expr_free_recycle(fypp, exprl); + if (exprr) + fy_path_expr_free_recycle(fypp, exprr); + + /* and push as an operand */ + ret = push_operand(fypp, exprm); + fyr_error_check(fyr, !ret, err_out, + "push_operand() failed\n"); + +#ifdef DEBUG_EXPR + FYR_TOKEN_DIAG(fyr, exprm->fyt, + FYDF_NOTICE, FYEM_PARSE, "pushed operand evaluate_method"); +#endif + return 0; + +err_out: + /* we don't need the parentheses operators */ + fy_path_expr_free_recycle(fypp, exprm); + if (exprl) + fy_path_expr_free_recycle(fypp, exprl); + if (exprr) + fy_path_expr_free_recycle(fypp, exprr); + + return -1; +} + +int evaluate_new(struct fy_path_parser *fypp) +{ + struct fy_reader *fyr; + struct fy_path_expr *expr = NULL, *expr_peek, *exprt; + struct fy_path_expr *exprl = NULL, *exprr = NULL, *chain = NULL, *exprm = NULL; + struct fy_path_expr *parent = NULL; + struct fy_token *fyt; + enum fy_path_expr_type type, etype; + int ret; + + fyr = &fypp->reader; + + expr = fy_expr_stack_pop(&fypp->operators); + fyr_error_check(fyr, expr, err_out, + "pop_operator() failed to find token operator to evaluate\n"); + + assert(expr->fyt); + +#ifdef DEBUG_EXPR + FYR_TOKEN_DIAG(fyr, expr->fyt, + FYDF_NOTICE, FYEM_PARSE, "poped operator expression"); + fy_path_expr_dump(expr, fypp->cfg.diag, FYET_NOTICE, 0, "poped expr: "); +#endif + + exprl = NULL; + exprr = NULL; + type = expr->type; + switch (type) { + + case fpet_chain: + + /* dump_operand_stack(fypp); */ + /* dump_operator_stack(fypp); */ + + /* peek the next operator */ + expr_peek = fy_expr_stack_peek(&fypp->operators); + + /* pop the top in either case */ + exprr = fy_expr_stack_pop(&fypp->operands); + if (!exprr) { + // fyr_notice(fyr, "ROOT value (with no arguments)\n"); + + /* conver to root and push to operands */ + expr->type = fpet_root; + + ret = push_operand(fypp, expr); + fyr_error_check(fyr, !ret, err_out, + "push_operand() failed\n"); + return 0; + } + +#ifdef DEBUG_EXPR + FYR_TOKEN_DIAG(fyr, exprr->fyt, + FYDF_NOTICE, FYEM_PARSE, "exprr"); +#endif + + /* expression is to the left, that means it's a root chain */ + if (fy_path_expr_order(expr, exprr) < 0 && + (!(exprl = fy_expr_stack_peek(&fypp->operands)) || + (expr_peek && fy_path_expr_order(exprl, expr_peek) <= 0))) { + + // fyr_notice(fyr, "ROOT operator (with arguments)\n"); + + exprl = fy_path_expr_alloc_recycle(fypp); + fyr_error_check(fyr, exprl, err_out, + "fy_path_expr_alloc_recycle() failed\n"); + exprl->type = fpet_root; + + /* move token to the root */ + exprl->fyt = expr->fyt; + expr->fyt = NULL; + + } else if (!(exprl = fy_expr_stack_pop(&fypp->operands))) { + + // fyr_notice(fyr, "COLLECTION operator\n"); + + exprl = exprr; + + exprr = fy_path_expr_alloc_recycle(fypp); + fyr_error_check(fyr, exprr, err_out, + "fy_path_expr_alloc_recycle() failed\n"); + exprr->type = fpet_filter_collection; + + /* move token to the filter collection */ + exprr->fyt = expr->fyt; + expr->fyt = NULL; + + } else { + assert(exprr && exprl); + + // fyr_notice(fyr, "CHAIN operator\n"); + } + + /* we don't need the chain operator now */ + fy_path_expr_free_recycle(fypp, expr); + expr = NULL; + + ret = push_operand_lr(fypp, fpet_chain, exprl, exprr, true); + fyr_error_check(fyr, !ret, err_out, + "push_operand_lr() failed\n"); + return 0; + + case fpet_multi: + case fpet_logical_or: + case fpet_logical_and: + + case fpet_eq: + case fpet_neq: + case fpet_lt: + case fpet_gt: + case fpet_lte: + case fpet_gte: + + case fpet_plus: + case fpet_minus: + case fpet_mult: + case fpet_div: + + exprl = NULL; + exprr = NULL; + exprt = fy_expr_stack_peek(&fypp->operators); + + // fyr_error(fyr, "mode=%s top-mode=%s\n", + // fy_expr_mode_txt[fypp->expr_mode], + // exprt ? fy_expr_mode_txt[exprt->expr_mode] : ""); + +#if 0 + exprr = fy_expr_stack_peek(&fypp->operands); + if (exprr && exprt && fy_path_expr_order(exprr, exprt) <= 0) + exprr = NULL; + else +#endif + exprr = fy_expr_stack_pop(&fypp->operands); + fyr_error_check(fyr, exprr, err_out, + "fy_expr_stack_pop() failed for exprr\n"); + +#if 0 + exprl = fy_expr_stack_peek_at(&fypp->operands, 1); + if (exprl && exprt && fy_path_expr_order(exprl, exprt) <= 0) + exprl = NULL; + else +#endif + exprl = fy_expr_stack_pop(&fypp->operands); + fyr_error_check(fyr, exprl, err_out, + "fy_expr_stack_pop() failed for exprl\n"); + + /* we don't need the operator now */ + fy_path_expr_free_recycle(fypp, expr); + expr = NULL; + + ret = push_operand_lr(fypp, type, exprl, exprr, true); + fyr_error_check(fyr, !ret, err_out, + "push_operand_lr() failed\n"); + + break; + + case fpet_filter_collection: + case fpet_filter_scalar: + case fpet_filter_sequence: + case fpet_filter_mapping: + case fpet_filter_unique: + + exprl = fy_expr_stack_pop(&fypp->operands); + FYR_TOKEN_ERROR_CHECK(fyr, expr->fyt, FYEM_PARSE, + exprl, err_out, + "filter operator without argument"); + + exprr = fy_path_expr_alloc_recycle(fypp); + fyr_error_check(fyr, exprr, err_out, + "fy_path_expr_alloc_recycle() failed\n"); + exprr->type = type; + + /* move token to the filter collection */ + exprr->fyt = expr->fyt; + expr->fyt = NULL; + + /* we don't need the operator now */ + fy_path_expr_free_recycle(fypp, expr); + expr = NULL; + + /* push as a chain */ + ret = push_operand_lr(fypp, fpet_chain, exprl, exprr, true); + fyr_error_check(fyr, !ret, err_out, + "push_operand_lr() failed\n"); + + break; + + case fpet_lparen: +#ifdef DEBUG_EXPR + FYR_TOKEN_DIAG(fyr, expr->fyt, + FYDF_NOTICE, FYEM_PARSE, "("); +#endif + + return 0; + + case fpet_arg_separator: + + /* separator is right hand side of the expression now */ + exprr = expr; + expr = NULL; + + /* evaluate until we hit a match to the rparen */ + exprl = fy_expr_stack_peek(&fypp->operators); + if (!exprl) + goto err_out; + + if (!fy_path_expr_type_is_lparen(exprl->type)) { + exprl = NULL; + ret = evaluate_new(fypp); + if (ret) + goto err_out; + } + exprl = NULL; + + fy_path_expr_free_recycle(fypp, exprr); + exprr = NULL; + + break; + + case fpet_rparen: + + /* rparen is right hand side of the expression now */ + exprr = expr; + expr = NULL; + + /* evaluate until we hit a match to the rparen */ + while ((exprl = fy_expr_stack_peek(&fypp->operators)) != NULL) { + + if (fy_path_expr_type_is_lparen(exprl->type)) + break; + + exprl = NULL; + ret = evaluate_new(fypp); + if (ret) + goto err_out; + } + + FYR_TOKEN_ERROR_CHECK(fyr, exprr->fyt, FYEM_PARSE, + exprl, err_out, + "missing matching left parentheses"); + + exprl = fy_expr_stack_pop(&fypp->operators); + if (!exprl) + goto err_out; + + exprt = fy_expr_stack_peek(&fypp->operands); + + etype = exprl->expr_mode == fyem_scalar ? fpet_scalar_expr : fpet_path_expr; + + /* already is an expression, reuse */ + if (exprt && exprt->type == etype) { + + fyt = expr_lr_to_token_mark(exprl, exprr, fypp->fyi); + fyr_error_check(fyr, fyt, err_out, + "expr_lr_to_token_mark() failed\n"); + fy_token_unref(exprt->fyt); + exprt->fyt = fyt; + exprt->expr_mode = exprl->expr_mode; + + /* we don't need the parentheses operators */ + fy_path_expr_free_recycle(fypp, exprl); + exprl = NULL; + fy_path_expr_free_recycle(fypp, exprr); + exprr = NULL; + + return 0; + } + + /* if it's method, evaluate */ + exprm = fy_expr_stack_peek(&fypp->operators); + if (exprm && exprm->type == fpet_method) { + + exprm = fy_expr_stack_pop(&fypp->operators); + assert(exprm); + + return evaluate_method(fypp, exprm, exprl, exprr); + } + + expr = fy_path_expr_alloc_recycle(fypp); + fyr_error_check(fyr, expr, err_out, + "fy_path_expr_alloc_recycle() failed\n"); + expr->type = etype; + expr->expr_mode = exprl->expr_mode; + + expr->fyt = expr_lr_to_token_mark(exprl, exprr, fypp->fyi); + + exprt = fy_expr_stack_pop(&fypp->operands); + if (!exprt) + goto err_out; + + FYR_TOKEN_ERROR_CHECK(fyr, exprr->fyt, FYEM_PARSE, + exprt, err_out, + "empty expression in parentheses"); + + fy_path_expr_list_add_tail(&expr->children, exprt); + exprt->parent = expr; + + /* pop all operands that after exprl */ + while ((exprt = fy_expr_stack_peek(&fypp->operands)) != NULL && + fy_path_expr_order(exprt, exprl) >= 0) { +#ifdef DEBUG_EXPR + FYR_TOKEN_DIAG(fyr, exprt->fyt, + FYDF_NOTICE, FYEM_PARSE, "discarding argument"); +#endif + fy_path_expr_free_recycle(fypp, fy_expr_stack_pop(&fypp->operands)); + } + + if (exprl->expr_mode != fyem_none) { + fypp->expr_mode = exprl->expr_mode; +#ifdef DEBUG_EXPR + fyr_notice(fyr, "poping expr_mode %s\n", fy_expr_mode_txt[fypp->expr_mode]); +#endif + } + + /* we don't need the parentheses operators */ + fy_path_expr_free_recycle(fypp, exprl); + exprl = NULL; + fy_path_expr_free_recycle(fypp, exprr); + exprr = NULL; + + ret = push_operand(fypp, expr); + fyr_error_check(fyr, !ret, err_out, + "push_operand() failed\n"); + +#ifdef DEBUG_EXPR + FYR_TOKEN_DIAG(fyr, expr->fyt, + FYDF_NOTICE, FYEM_PARSE, "pushed operand evaluate_new"); +#endif + return 0; + + case fpet_method: + return evaluate_method(fypp, expr, NULL, NULL); + + case fpet_select: + case fpet_unselect: + + /* we don't need the not operator now */ + fy_path_expr_free_recycle(fypp, expr); + expr = NULL; + + exprl = fy_expr_stack_pop(&fypp->operands); + fyr_error_check(fyr, exprl, err_out, + "fy_expr_stack_pop() failed for exprl\n"); + exprr = NULL; + + ret = push_operand_lr(fypp, type, exprl, exprr, true); + fyr_error_check(fyr, !ret, err_out, + "push_operand_lr() failed\n"); + + return 0; + + /* shoud never */ + case fpet_scalar_expr: + case fpet_path_expr: + FY_IMPOSSIBLE_ABORT(); + + default: + fyr_error(fyr, "Unknown expression %s\n", fy_path_expr_type_txt[expr->type]); + goto err_out; + } + + return 0; + +err_out: + +#ifdef DEBUG_EXPR + if (expr) + fy_path_expr_dump(expr, fypp->cfg.diag, FYET_NOTICE, 0, "expr:"); + if (exprl) + fy_path_expr_dump(exprl, fypp->cfg.diag, FYET_NOTICE, 0, "exprl:"); + if (exprr) + fy_path_expr_dump(exprr, fypp->cfg.diag, FYET_NOTICE, 0, "exprr:"); + if (chain) + fy_path_expr_dump(chain, fypp->cfg.diag, FYET_NOTICE, 0, "chain:"); + if (parent) + fy_path_expr_dump(parent, fypp->cfg.diag, FYET_NOTICE, 0, "parent:"); + + fy_notice(fypp->cfg.diag, "operator stack\n"); + fy_expr_stack_dump(fypp->cfg.diag, &fypp->operators); + fy_notice(fypp->cfg.diag, "operand stack\n"); + fy_expr_stack_dump(fypp->cfg.diag, &fypp->operands); +#endif + + fy_path_expr_free(expr); + fy_path_expr_free(exprl); + fy_path_expr_free(exprr); + fy_path_expr_free(chain); + fy_path_expr_free(parent); + + return -1; +} + +int fy_path_check_expression_alias(struct fy_path_parser *fypp, struct fy_path_expr *expr) +{ + struct fy_reader *fyr; + struct fy_path_expr *exprn; + int rc; + + if (!expr) + return 0; + + fyr = &fypp->reader; + + /* an alias with a parent.. must be the first one */ + if (expr->type == fpet_alias && expr->parent) { + + exprn = fy_path_expr_list_head(&expr->parent->children); + + /* an alias may only be the first of a path expression */ + FYR_TOKEN_ERROR_CHECK(fyr, expr->fyt, FYEM_PARSE, + expr == exprn, err_out, + "alias is not first in the path expresion"); + } + + for (exprn = fy_path_expr_list_head(&expr->children); exprn; + exprn = fy_path_expr_next(&expr->children, exprn)) { + + rc = fy_path_check_expression_alias(fypp, exprn); + if (rc) + return rc; + } + + return 0; + +err_out: + return -1; +} + +/* check expression for validity */ +int fy_path_check_expression(struct fy_path_parser *fypp, struct fy_path_expr *expr) +{ + int rc; + + rc = fy_path_check_expression_alias(fypp, expr); + if (rc) + return rc; + + return 0; +} + +struct fy_path_expr * +fy_path_parse_expression(struct fy_path_parser *fypp) +{ + struct fy_reader *fyr; + struct fy_token *fyt = NULL; + enum fy_token_type fytt; + struct fy_path_expr *expr = NULL, *expr_top, *exprt; + enum fy_expr_mode old_scan_mode, prev_scan_mode; + int ret, rc; + + /* the parser must be in the correct state */ + if (!fypp || fy_expr_stack_size(&fypp->operators) > 0 || fy_expr_stack_size(&fypp->operands) > 0) + return NULL; + + fyr = &fypp->reader; + + /* find stream start */ + fyt = fy_path_scan_peek(fypp, NULL); + FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_PARSE, + fyt && fyt->type == FYTT_STREAM_START, err_out, + "no tokens available or start without stream start"); + + /* remove stream start */ + fy_token_unref(fy_path_scan_remove(fypp, fyt)); + fyt = NULL; + + prev_scan_mode = fypp->expr_mode; + + while ((fyt = fy_path_scan_peek(fypp, NULL)) != NULL) { + + if (fyt->type == FYTT_STREAM_END) + break; + +#ifdef DEBUG_EXPR + FYR_TOKEN_DIAG(fyr, fyt, FYET_NOTICE, FYEM_PARSE, "next token %s", fy_token_debug_text_a(fyt)); +#endif + fytt = fyt->type; + + /* create an expression in either operator/operand case */ + expr = fy_path_expr_alloc_recycle(fypp); + fyr_error_check(fyr, expr, err_out, + "fy_path_expr_alloc_recycle() failed\n"); + + expr->fyt = fy_path_scan_remove(fypp, fyt); + /* this it the first attempt, it might not be the final one */ + expr->type = fy_map_token_to_path_expr_type(fyt->type, fypp->expr_mode); + fyt = NULL; + +#ifdef DEBUG_EXPR + fy_path_expr_dump(expr, fypp->cfg.diag, FYET_NOTICE, 0, "-> expr"); +#endif + + if (prev_scan_mode != fypp->expr_mode) { +#ifdef DEBUG_EXPR + fyr_warning(fyr, "switched expr_mode %s -> %s\n", + fy_expr_mode_txt[prev_scan_mode], + fy_expr_mode_txt[fypp->expr_mode]); +#endif + expr_top = fy_expr_stack_peek(&fypp->operators); + +#ifdef DEBUG_EXPR + if (expr_top) + fy_path_expr_dump(expr_top, fypp->cfg.diag, FYET_NOTICE, 0, NULL); +#endif + + if (expr_top && fy_path_expr_type_is_lparen(expr_top->type) && + expr_top->expr_mode != fypp->expr_mode) { +#ifdef DEBUG_EXPR + fyr_warning(fyr, "switched top lparen expr_mode %s -> %s\n", + fy_expr_mode_txt[expr_top->expr_mode], + fy_expr_mode_txt[fypp->expr_mode]); + expr_top->expr_mode = fypp->expr_mode; +#endif + } + } + + prev_scan_mode = fypp->expr_mode; + + /* if it's an operand convert it to expression and push */ + if (fy_token_type_is_operand(fytt)) { + + ret = fy_expr_stack_push(&fypp->operands, expr); + fyr_error_check(fyr, !ret, err_out, "push_operand() failed\n"); + expr = NULL; + + continue; + } + + /* specials for SLASH */ + + if (expr->fyt->type == FYTT_PE_SLASH) { + + /* try to get next token */ + fyt = fy_path_scan_peek(fypp, NULL); + if (!fyt) { + if (!fypp->stream_error) { + (void)fy_path_fetch_tokens(fypp); + fyt = fy_path_scan_peek(fypp, NULL); + } + } + + /* last token, it means it's a collection filter (or a root) */ + if (!fyt || fyt->type == FYTT_STREAM_END || fyt->type == FYTT_PE_RPAREN) { + + exprt = fy_expr_stack_peek(&fypp->operands); + + /* if no argument exists it's a root */ + if (!exprt) { + expr->type = fpet_root; + + ret = fy_expr_stack_push(&fypp->operands, expr); + fyr_error_check(fyr, !ret, err_out, "push_operand() failed\n"); + expr = NULL; + continue; + } + + expr->type = fpet_filter_collection; + } + } + +#ifdef DEBUG_EXPR + fy_notice(fypp->cfg.diag, "operator stack (before)\n"); + fy_expr_stack_dump(fypp->cfg.diag, &fypp->operators); + fy_notice(fypp->cfg.diag, "operator stack (before) ends\n"); + fy_notice(fypp->cfg.diag, "operand stack (before)\n"); + fy_expr_stack_dump(fypp->cfg.diag, &fypp->operands); + fy_notice(fypp->cfg.diag, "operand stack (before) ends\n"); +#endif + + old_scan_mode = fypp->expr_mode; + + /* for rparen, need to push before */ + if (expr->type == fpet_rparen) { + + FYR_TOKEN_ERROR_CHECK(fyr, expr->fyt, FYEM_PARSE, + fypp->paren_nest_level > 0, err_out, + "Mismatched right parentheses"); + + fypp->paren_nest_level--; + + ret = fy_expr_stack_push(&fypp->operators, expr); + fyr_error_check(fyr, !ret, err_out, "push_operator() failed\n"); + expr = NULL; + + ret = evaluate_new(fypp); + /* evaluate will print diagnostic on error */ + if (ret < 0) + goto err_out; + + } else if (fy_path_expr_type_is_lparen(expr->type)) { + + expr->expr_mode = fypp->expr_mode; + fypp->expr_mode = fyem_scalar; + + fypp->paren_nest_level++; + + /* push the operator */ + ret = fy_expr_stack_push(&fypp->operators, expr); + fyr_error_check(fyr, !ret, err_out, "push_operator() failed\n"); + expr = NULL; + +#ifdef DEBUG_EXPR + if (old_scan_mode != fypp->expr_mode) + fyr_notice(fyr, "expr_mode %s -> %s\n", + fy_expr_mode_txt[old_scan_mode], + fy_expr_mode_txt[fypp->expr_mode]); +#endif + } else { + switch (fypp->expr_mode) { + case fyem_none: + FY_IMPOSSIBLE_ABORT(); + + case fyem_path: + if (fy_path_expr_type_is_conditional(expr->type)) { + /* switch to scalar mode */ + fypp->expr_mode = fyem_scalar; + break; + } + break; + case fyem_scalar: + if (expr->type == fpet_root) { + fypp->expr_mode = fyem_path; + break; + } + + /* div out of parentheses, it's a chain */ + if (expr->type == fpet_div && fypp->paren_nest_level == 0) { + expr->type = fpet_chain; + fypp->expr_mode = fyem_path; + + /* mode change means evaluate */ + ret = evaluate_new(fypp); + /* evaluate will print diagnostic on error */ + if (ret < 0) + goto err_out; + + break; + } + break; + + } + + if (old_scan_mode != fypp->expr_mode) { +#ifdef DEBUG_EXPR + fyr_notice(fyr, "expr_mode %s -> %s\n", + fy_expr_mode_txt[old_scan_mode], + fy_expr_mode_txt[fypp->expr_mode]); +#endif + } + + ret = -1; + for (;;) { + expr_top = fy_expr_stack_peek(&fypp->operators); + if (!expr_top) + break; + + if (fy_path_expr_type_prec(expr->type) > fy_path_expr_type_prec(expr_top->type)) + break; + + if (fy_path_expr_type_is_lparen(expr_top->type)) + break; + + /* XXX same priority and prefix operator (have no arguments yet) */ + if (fy_path_expr_type_prec(expr->type) == fy_path_expr_type_prec(expr_top->type)) { + if (fy_path_expr_type_assoc(expr_top->type) == PREFIX) + break; + } + + ret = evaluate_new(fypp); + /* evaluate will print diagnostic on error */ + if (ret < 0) + goto err_out; + } + + /* push the operator */ + ret = fy_expr_stack_push(&fypp->operators, expr); + fyr_error_check(fyr, !ret, err_out, "push_operator() failed\n"); + expr = NULL; + } + +#ifdef DEBUG_EXPR + fy_notice(fypp->cfg.diag, "operator stack (after)\n"); + fy_expr_stack_dump(fypp->cfg.diag, &fypp->operators); + fy_notice(fypp->cfg.diag, "operator stack (after) ends\n"); + fy_notice(fypp->cfg.diag, "operand stack (after)\n"); + fy_expr_stack_dump(fypp->cfg.diag, &fypp->operands); + fy_notice(fypp->cfg.diag, "operand stack (after) ends\n"); +#endif + + prev_scan_mode = fypp->expr_mode; + } + + if (fypp->stream_error) + goto err_out; + + FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_PARSE, + fypp->stream_error || (fyt && fyt->type == FYTT_STREAM_END), err_out, + "stream ended without STREAM_END"); + + /* remove stream end */ + fy_token_unref(fy_path_scan_remove(fypp, fyt)); + fyt = NULL; + +#if 0 + FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_PARSE, + fypp->paren_nest_level == 0, err_out, + "Missing right parenthesis"); +#endif + + /* drain */ + while ((expr_top = fy_expr_stack_peek(&fypp->operators)) != NULL && + !fy_path_expr_type_is_lparen(expr_top->type)) { + + ret = evaluate_new(fypp); + /* evaluate will print diagnostic on error */ + if (ret < 0) + goto err_out; + + } + +#ifdef DEBUG_EXPR + expr_top = fy_expr_stack_peek(&fypp->operators); + if (expr_top) + fy_path_expr_dump(expr_top, fypp->cfg.diag, FYET_NOTICE, 0, "operator top left"); +#endif + + expr = fy_expr_stack_pop(&fypp->operands); + + FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_PARSE, + expr != NULL, err_out, + "No operands left on operand stack"); + + if (fy_expr_stack_size(&fypp->operands) != 0) { + FYR_TOKEN_ERROR(fyr, expr->fyt, FYEM_PARSE, + "Operand stack contains more than 1 value at end"); + fy_path_expr_free(expr); + expr = NULL; + goto err_out; + } + + /* check the expression for validity */ + rc = fy_path_check_expression(fypp, expr); + if (rc) { + fy_path_expr_free(expr); + expr = NULL; + } + + return expr; + +err_out: + if (expr) + fy_path_expr_free(expr); +#ifdef DEBUG_EXPR + fy_notice(fypp->cfg.diag, "operator stack (error)\n"); + fy_expr_stack_dump(fypp->cfg.diag, &fypp->operators); + fy_notice(fypp->cfg.diag, "operand stack (error)\n"); + fy_expr_stack_dump(fypp->cfg.diag, &fypp->operands); +#endif + fypp->stream_error = true; + return NULL; +} + +static struct fy_node * +fy_path_expr_execute_single_result(struct fy_diag *diag, struct fy_path_expr *expr, struct fy_node *fyn) +{ + struct fy_token *fyt; + struct fy_anchor *fya; + const char *text; + size_t len; + + assert(expr); + + switch (expr->type) { + case fpet_root: + return fyn->fyd->root; + + case fpet_this: + if (fy_node_is_alias(fyn)) { + // fprintf(stderr, "%s:%d %s calling fy_node_alias_resolve_by_ypath()\n", __func__, __LINE__, fy_node_get_path_alloca(fyn)); + fyn = fy_node_alias_resolve_by_ypath(fyn); + } + + return fyn; + + case fpet_parent: + return fy_node_get_parent(fyn); + + case fpet_alias: + fyt = expr->fyt; + assert(fyt); + assert(fyt->type == FYTT_PE_ALIAS); + + text = fy_token_get_text(fyt, &len); + if (!text || len < 1) + break; + + if (*text == '*') { + text++; + len--; + } + fya = fy_document_lookup_anchor(fyn->fyd, text, len); + if (!fya) + break; + return fya->fyn; + + case fpet_seq_index: + fyt = expr->fyt; + assert(fyt); + assert(fyt->type == FYTT_PE_SEQ_INDEX); + + if (fy_node_is_alias(fyn)) { + // fprintf(stderr, "%s:%d %s calling fy_node_alias_resolve_by_ypath()\n", __func__, __LINE__, fy_node_get_path_alloca(fyn)); + fyn = fy_node_alias_resolve_by_ypath(fyn); + } + + /* only on sequence */ + if (!fy_node_is_sequence(fyn)) + break; + + return fy_node_sequence_get_by_index(fyn, fyt->seq_index.index); + + case fpet_map_key: + fyt = expr->fyt; + assert(fyt); + assert(fyt->type == FYTT_PE_MAP_KEY); + + if (fy_node_is_alias(fyn)) { + // fprintf(stderr, "%s:%d %s calling fy_node_alias_resolve_by_ypath()\n", __func__, __LINE__, fy_node_get_path_alloca(fyn)); + fyn = fy_node_alias_resolve_by_ypath(fyn); + } + + if (!fy_node_is_mapping(fyn)) + break; + + if (!fyt->map_key.fyd) { + /* simple key */ + text = fy_token_get_text(fyt, &len); + if (!text || len < 1) + break; + return fy_node_mapping_lookup_value_by_simple_key(fyn, text, len); + } + + return fy_node_mapping_lookup_value_by_key(fyn, fyt->map_key.fyd->root); + + case fpet_filter_scalar: + if (!(fy_node_is_scalar(fyn) || fy_node_is_alias(fyn))) + break; + return fyn; + + case fpet_filter_collection: + + if (fy_node_is_alias(fyn)) { + // fprintf(stderr, "%s:%d %s calling fy_node_alias_resolve_by_ypath()\n", __func__, __LINE__, fy_node_get_path_alloca(fyn)); + fyn = fy_node_alias_resolve_by_ypath(fyn); + } + + if (!(fy_node_is_mapping(fyn) || fy_node_is_sequence(fyn))) + break; + return fyn; + + case fpet_filter_sequence: + + if (fy_node_is_alias(fyn)) { + // fprintf(stderr, "%s:%d %s calling fy_node_alias_resolve_by_ypath()\n", __func__, __LINE__, fy_node_get_path_alloca(fyn)); + fyn = fy_node_alias_resolve_by_ypath(fyn); + } + + if (!fy_node_is_sequence(fyn)) + break; + return fyn; + + case fpet_filter_mapping: + + if (fy_node_is_alias(fyn)) { + // fprintf(stderr, "%s:%d %s calling fy_node_alias_resolve_by_ypath()\n", __func__, __LINE__, fy_node_get_path_alloca(fyn)); + fyn = fy_node_alias_resolve_by_ypath(fyn); + } + + if (!fy_node_is_mapping(fyn)) + break; + return fyn; + + default: + break; + } + + return NULL; +} + +static double +token_number(struct fy_token *fyt) +{ + const char *value; + + if (!fyt || fyt->type != FYTT_SCALAR || (value = fy_token_get_text0(fyt)) == NULL) + return NAN; + return strtod(value, NULL); +} + +void fy_path_exec_cleanup(struct fy_path_exec *fypx) +{ + if (!fypx) + return; + + fy_walk_result_free(fypx->result); + fypx->result = NULL; + fypx->fyn_start = NULL; +} + +/* publicly exported methods */ +struct fy_path_parser *fy_path_parser_create(const struct fy_path_parse_cfg *pcfg) +{ + struct fy_path_parser *fypp; + + fypp = malloc(sizeof(*fypp)); + if (!fypp) + return NULL; + fy_path_parser_setup(fypp, pcfg); + return fypp; +} + +void fy_path_parser_destroy(struct fy_path_parser *fypp) +{ + if (!fypp) + return; + fy_path_parser_cleanup(fypp); + free(fypp); +} + +struct fy_path_expr * +fy_path_parse_expr_from_string(struct fy_path_parser *fypp, + const char *str, size_t len) +{ + struct fy_path_expr *expr = NULL; + struct fy_input *fyi = NULL; + int rc; + + if (!fypp || !str || !len) + return NULL; + + fy_path_parser_reset(fypp); + + fyi = fy_input_from_data(str, len, NULL, false); + if (!fyi) { + fy_error(fypp->cfg.diag, "failed to create ypath input from %.*s\n", + (int)len, str); + goto err_out; + } + + rc = fy_path_parser_open(fypp, fyi, NULL); + if (rc) { + fy_error(fypp->cfg.diag, "failed to open path parser input from %.*s\n", + (int)len, str); + goto err_out; + } + + expr = fy_path_parse_expression(fypp); + if (!expr) { + fy_error(fypp->cfg.diag, "failed to parse path expression %.*s\n", + (int)len, str); + goto err_out; + } + + fy_path_parser_close(fypp); + + fy_input_unref(fyi); + + return expr; + +err_out: + fy_path_expr_free(expr); + fy_path_parser_close(fypp); + fy_input_unref(fyi); + return NULL; +} + +struct fy_path_expr * +fy_path_expr_build_from_string(const struct fy_path_parse_cfg *pcfg, + const char *str, size_t len) +{ + struct fy_path_parser fypp_data, *fypp = &fypp_data; + struct fy_path_expr *expr = NULL; + + if (!str) + return NULL; + + fy_path_parser_setup(fypp, pcfg); + expr = fy_path_parse_expr_from_string(fypp, str, len); + fy_path_parser_cleanup(fypp); + + return expr; +} + +struct fy_path_exec *fy_path_exec_create(const struct fy_path_exec_cfg *xcfg) +{ + struct fy_path_exec *fypx; + + fypx = malloc(sizeof(*fypx)); + if (!fypx) + return NULL; + + memset(fypx, 0, sizeof(*fypx)); + if (xcfg) + fypx->cfg = *xcfg; + fypx->fwr_recycle = NULL; /* initially no recycling list */ + fypx->refs = 1; + + fypx->supress_recycling = !!(fypx->cfg.flags & FYPXCF_DISABLE_RECYCLING) || + (getenv("FY_VALGRIND") && + !getenv("FY_VALGRIND_RECYCLING")); + return fypx; +} + +struct fy_path_exec *fy_path_exec_create_on_document(struct fy_document *fyd) +{ + struct fy_path_exec_cfg xcfg_local, *xcfg = &xcfg_local; + struct fy_path_exec *fypx; + + memset(xcfg, 0, sizeof(*xcfg)); + xcfg->diag = fyd ? fyd->diag : NULL; + + xcfg->flags = (fyd->parse_cfg.flags & FYPCF_DISABLE_RECYCLING) ? + FYPXCF_DISABLE_RECYCLING : 0; + + fypx = fy_path_exec_create(xcfg); + if (!fypx) + return NULL; + return fypx; +} + +void fy_path_exec_destroy(struct fy_path_exec *fypx) +{ + if (!fypx) + return; + fy_path_exec_cleanup(fypx); + free(fypx); +} + +int fy_path_exec_reset(struct fy_path_exec *fypx) +{ + if (!fypx) + return -1; + fy_path_exec_cleanup(fypx); + return 0; +} + +struct fy_walk_result *fy_walk_result_simplify(struct fy_walk_result *fwr) +{ + struct fy_walk_result *fwr2; +#if 0 + struct fy_walk_result *fwrf; + bool recursive; +#endif + + /* no fwr */ + if (!fwr) + return NULL; + + /* non recursive */ + if (fwr->type != fwrt_refs) + return fwr; + + /* refs, if empty, return NULL */ + if (fy_walk_result_list_empty(&fwr->refs)) { + fy_walk_result_free(fwr); + return NULL; + } + + /* single element, switch it out */ + if (fy_walk_result_list_is_singular(&fwr->refs)) { + fwr2 = fy_walk_result_list_pop(&fwr->refs); + assert(fwr2); + + fy_walk_result_free(fwr); + fwr = fwr2; + } + + return fwr; +#if 0 + /* non recursive return immediately */ + if (fwr->type != fwrt_refs) + return fwr; + + /* flatten if recursive */ + recursive = false; + for (fwr2 = fy_walk_result_list_head(&fwr->refs); fwr2; + fwr2 = fy_walk_result_next(&fwr->refs, fwr2)) { + + /* refs, recursive */ + if (fwr2->type == fwrt_refs) { + recursive = true; + break; + } + } + + if (!recursive) + return fwr; + + fwrf = fy_path_exec_walk_result_create(fypx, fwrt_refs); + assert(fwrf); + + fy_walk_result_flatten_internal(fwr, fwrf); + + fy_walk_result_free(fwr); + return fwrf; +#endif + +} + +int fy_walk_result_all_children_recursive_internal(struct fy_path_exec *fypx, struct fy_node *fyn, struct fy_walk_result *output) +{ + struct fy_node *fyni; + struct fy_walk_result *fwr; + void *prevp; + int ret; + + if (!fyn) + return 0; + + assert(output); + assert(output->type == fwrt_refs); + + /* this node */ + fwr = fy_path_exec_walk_result_create(fypx, fwrt_node_ref, fyn); + if (!fwr) + return -1; + fy_walk_result_list_add_tail(&output->refs, fwr); + + if (fy_node_is_scalar(fyn)) + return 0; + + prevp = NULL; + while ((fyni = fy_node_collection_iterate(fyn, &prevp)) != NULL) { + ret = fy_walk_result_all_children_recursive_internal(fypx, fyni, output); + if (ret) + return ret; + } + + return 0; +} + +bool +fy_walk_result_compare_simple(struct fy_path_exec *fypx, enum fy_path_expr_type type, + struct fy_walk_result *fwrl, struct fy_walk_result *fwrr) +{ + struct fy_token *fyt; + struct fy_walk_result *fwrt; + const char *str; + bool match; + + /* both NULL */ + if (!fwrl && !fwrr) { + switch (type) { + case fpet_eq: + return true; + default: + break; + } + return false; + } + + /* any NULL */ + if (!fwrl || !fwrr) { + switch (type) { + case fpet_neq: + return true; + default: + break; + } + return false; + } + + /* both are non NULL */ + + /* none should be multiple */ + assert(fwrl->type != fwrt_refs && fwrr->type != fwrt_refs); + + /* both are the same type */ + if (fwrl->type == fwrr->type) { + + switch (fwrl->type) { + case fwrt_none: + FY_IMPOSSIBLE_ABORT(); + + case fwrt_node_ref: + switch (type) { + case fpet_eq: + /* simple and fast direct node comparison */ + if (fwrl->fyn == fwrr->fyn) + return true; + return fy_node_compare(fwrl->fyn, fwrr->fyn); + case fpet_neq: + /* simple and fast direct node comparison */ + if (fwrl->fyn != fwrr->fyn) + return true; + return !fy_node_compare(fwrl->fyn, fwrr->fyn); + default: + break; + } + break; + + case fwrt_refs: + FY_IMPOSSIBLE_ABORT(); + + case fwrt_doc: + switch (type) { + case fpet_eq: + case fpet_neq: + match = false; + if (fwrl->fyd == fwrr->fyd) + match = true; + else if (!fwrl->fyd || !fwrr->fyd) + match = false; + else + match = fy_node_compare(fwrl->fyd->root, fwrr->fyd->root); + if (type == fpet_neq) + match = !match; + return match; + default: + break; + } + break; + + case fwrt_number: + switch (type) { + case fpet_eq: + return fwrl->number == fwrr->number; + case fpet_neq: + return fwrl->number != fwrr->number; + case fpet_lt: + return fwrl->number < fwrr->number; + case fpet_gt: + return fwrl->number > fwrr->number; + case fpet_lte: + return fwrl->number <= fwrr->number; + case fpet_gte: + return fwrl->number >= fwrr->number; + default: + break; + } + break; + + case fwrt_string: + switch (type) { + case fpet_eq: + return strcmp(fwrl->string, fwrr->string) == 0; + case fpet_neq: + return strcmp(fwrl->string, fwrr->string) != 0; + case fpet_lt: + return strcmp(fwrl->string, fwrr->string) < 0; + case fpet_gt: + return strcmp(fwrl->string, fwrr->string) > 0; + case fpet_lte: + return strcmp(fwrl->string, fwrr->string) <= 0; + case fpet_gte: + return strcmp(fwrl->string, fwrr->string) >= 0; + default: + break; + } + break; + } + return false; + } + + /* only handle the node refs at the left */ + if (fwrr->type == fwrt_node_ref) { + switch (type) { + case fpet_lt: + type = fpet_gte; + break; + case fpet_gt: + type = fpet_lte; + break; + case fpet_lte: + type = fpet_gt; + break; + case fpet_gte: + type = fpet_lt; + break; + default: + break; + } + + /* swap left with right */ + return fy_walk_result_compare_simple(fypx, type, fwrr, fwrl); + } + + switch (fwrl->type) { + case fwrt_node_ref: + + /* non scalar mode, only returns true for non-eq */ + if (!fy_node_is_scalar(fwrl->fyn)) { + /* XXX case of rhs being a document not handled */ + return type == fpet_neq; + } + + fyt = fy_node_get_scalar_token(fwrl->fyn); + assert(fyt); + + str = fy_token_get_text0(fyt); + assert(str); + + fwrt = NULL; + /* node ref against */ + switch (fwrr->type) { + case fwrt_string: + /* create a new temporary walk result */ + fwrt = fy_path_exec_walk_result_create(fypx, fwrt_string, str); + assert(fwrt); + + break; + + case fwrt_number: + /* if it's not a number return true only for non-eq */ + if (!fy_token_is_number(fyt)) + return type == fpet_neq; + + /* create a new temporary walk result */ + fwrt = fy_path_exec_walk_result_create(fypx, fwrt_number, strtod(str, NULL)); + assert(fwrt); + break; + + default: + break; + } + + if (!fwrt) + return false; + + match = fy_walk_result_compare_simple(fypx, type, fwrt, fwrr); + + /* free the temporary result */ + fy_walk_result_free(fwrt); + + return match; + + default: + break; + } + + return false; +} + +struct fy_walk_result * +fy_walk_result_arithmetic_simple(struct fy_path_exec *fypx, + struct fy_path_expr *expr, + struct fy_path_expr *exprl, struct fy_walk_result *fwrl, + struct fy_path_expr *exprr, struct fy_walk_result *fwrr) +{ + struct fy_diag *diag; + struct fy_walk_result *output = NULL; + char *str; + size_t len, len1, len2; + + if (!fwrl || !fwrr) + goto out; + + diag = fypx->cfg.diag; + + /* node refs are not handled yet */ + if (fwrl->type == fwrt_node_ref || fwrr->type == fwrt_node_ref) + goto out; + + /* same type */ + if (fwrl->type == fwrr->type) { + + switch (fwrl->type) { + + case fwrt_string: + /* for strings, only concatenation */ + if (expr->type != fpet_plus) + break; + len1 = strlen(fwrl->string); + len2 = strlen(fwrr->string); + len = len1 + len2; + str = malloc(len + 1); + assert(str); + memcpy(str, fwrl->string, len1); + memcpy(str + len1, fwrr->string, len2); + str[len] = '\0'; + + free(fwrl->string); + fwrl->string = str; + + /* reuse */ + output = fwrl; + fwrl = NULL; + + break; + + case fwrt_number: + /* reuse fwrl */ + output = fwrl; + switch (expr->type) { + case fpet_plus: + output->number = fwrl->number + fwrr->number; + break; + case fpet_minus: + output->number = fwrl->number - fwrr->number; + break; + case fpet_mult: + output->number = fwrl->number * fwrr->number; + break; + case fpet_div: + output->number = fwrr->number ? (fwrl->number / fwrr->number) : INFINITY; + break; + default: + FY_IMPOSSIBLE_ABORT(); + } + fwrl = NULL; + break; + + default: + fy_error(diag, "fwrl->type=%s\n", fy_walk_result_type_txt[fwrl->type]); + FY_IMPOSSIBLE_ABORT(); + } + } + +out: + fy_walk_result_free(fwrl); + fy_walk_result_free(fwrr); + return output; +} + +struct fy_walk_result * +fy_walk_result_conditional_simple(struct fy_path_exec *fypx, + struct fy_path_expr *expr, + struct fy_path_expr *exprl, struct fy_walk_result *fwrl, + struct fy_path_expr *exprr, struct fy_walk_result *fwrr) +{ + bool match; + + match = fy_walk_result_compare_simple(fypx, expr->type, fwrl, fwrr); + + if (!match) { + fy_walk_result_free(fwrl); + fy_walk_result_free(fwrr); + return NULL; + } + + /* path expr, return left hand side result */ + fy_walk_result_free(fwrr); + return fwrl; +} + +struct fy_walk_result * +fy_walk_result_lhs_rhs(struct fy_path_exec *fypx, + struct fy_path_expr *expr, + struct fy_path_expr *exprl, struct fy_walk_result *fwrl, + struct fy_path_expr *exprr, struct fy_walk_result *fwrr) +{ + struct fy_walk_result *output = NULL, *fwr, *fwrrt, *fwrlt, *fwrlc, *fwrrc; + struct fy_walk_result *outputl = NULL, *outputr = NULL; + + assert(expr); + assert(exprl); + assert(exprr); + + /* only supports those */ + if (!fy_path_expr_type_is_conditional(expr->type) && + !fy_path_expr_type_is_arithmetic(expr->type)) + goto out; + + /* both NULL */ + if (!fwrl && !fwrr) + goto out; + + /* any NULL */ + if (!fwrl || !fwrr) { + if (expr->type == fpet_neq) { + output = fwrl; + fwrl = NULL; + } + goto out; + } + + output = fy_path_exec_walk_result_create(fypx, fwrt_refs); + assert(output); + + for (fwrlt = fy_walk_result_iter_start(fwrl); fwrlt; + fwrlt = fy_walk_result_iter_next(fwrl, fwrlt)) { + + /* for recursive ones */ + if (fwrlt->type == fwrt_refs) { + + fwrlc = fy_walk_result_clone(fwrlt); + assert(fwrlc); + fwrrc = fy_walk_result_clone(fwrr); + assert(fwrrc); + + outputl = fy_walk_result_lhs_rhs(fypx, expr, exprl, fwrlc, exprr, fwrrc); + if (outputl) + fy_walk_result_list_add_tail(&output->refs, outputl); + else + fy_walk_result_free(outputl); + continue; + } + + /* non-recursive case */ + for (fwrrt = fy_walk_result_iter_start(fwrr); fwrrt; + fwrrt = fy_walk_result_iter_next(fwrr, fwrrt)) { + + /* for recursive ones */ + if (fwrrt->type == fwrt_refs) { + + fwrlc = fy_walk_result_clone(fwrlt); + assert(fwrlc); + fwrrc = fy_walk_result_clone(fwrrt); + assert(fwrrc); + + outputr = fy_walk_result_lhs_rhs(fypx, expr, exprl, fwrlc, exprr, fwrrc); + if (outputr) + fy_walk_result_list_add_tail(&output->refs, outputr); + else + fy_walk_result_free(outputr); + continue; + } + + + fwrlc = fy_walk_result_clone(fwrlt); + assert(fwrlc); + fwrrc = fy_walk_result_clone(fwrrt); + assert(fwrrc); + + fwr = NULL; + + if (fy_path_expr_type_is_conditional(expr->type)) + fwr = fy_walk_result_conditional_simple(fypx, expr, exprl, fwrlc, exprr, fwrrc); + else if (fy_path_expr_type_is_arithmetic(expr->type)) + fwr = fy_walk_result_arithmetic_simple(fypx, expr, exprl, fwrlc, exprr, fwrrc); + else + FY_IMPOSSIBLE_ABORT(); + + fwrlc = NULL; + fwrrc = NULL; + + if (fwr) + fy_walk_result_list_add_tail(&output->refs, fwr); + } + } + +out: + fy_walk_result_free(fwrl); + fy_walk_result_free(fwrr); + + return fy_walk_result_simplify(output); +} + +struct fy_path_expr * +fy_scalar_walk_result_to_expr(struct fy_path_exec *fypx, struct fy_walk_result *fwr, enum fy_path_expr_type ptype) +{ + struct fy_input *fyit = NULL; + struct fy_path_expr *exprt = NULL; + struct fy_atom handle; + bool collection_addressing; + char *buf; + int rc __FY_DEBUG_UNUSED__; + + exprt = NULL; + + if (!fwr) + return NULL; + + collection_addressing = ptype == fpet_chain || ptype == fpet_multi; + + switch (fwr->type) { + case fwrt_string: + fyit = fy_input_from_malloc_data(fwr->string, FY_NT, &handle, true); + assert(fyit); + + fwr->string = NULL; + fy_walk_result_free(fwr); + fwr = NULL; + + exprt = fy_path_expr_alloc(); + assert(exprt); + if (collection_addressing) { + exprt->type = fpet_map_key; + exprt->fyt = fy_token_create(FYTT_PE_MAP_KEY, &handle, NULL); + assert(exprt->fyt); + } else { + exprt->type = fpet_scalar; + exprt->fyt = fy_token_create(FYTT_SCALAR, &handle, FYSS_PLAIN, NULL); + assert(exprt->fyt); + } + break; + + case fwrt_number: + + rc = asprintf(&buf, "%d", (int)fwr->number); + assert(rc != -1); + + fyit = fy_input_from_malloc_data(buf, FY_NT, &handle, true); + assert(fyit); + + exprt = fy_path_expr_alloc(); + assert(exprt); + if (collection_addressing) { + exprt->type = fpet_seq_index; + exprt->fyt = fy_token_create(FYTT_PE_SEQ_INDEX, &handle, (int)fwr->number); + assert(exprt->fyt); + } else { + exprt->type = fpet_scalar; + exprt->fyt = fy_token_create(FYTT_SCALAR, &handle, FYSS_PLAIN, NULL); + assert(exprt->fyt); + } + + break; + + default: + break; + } + + fy_walk_result_free(fwr); + fy_input_unref(fyit); + + return exprt; +} + +enum fy_walk_result_set_op { + FYWRSO_SELECT, + FYWRSO_UNSELECT, +}; + +static void fy_node_delete_non_marked(struct fy_node *fyn) +{ + struct fy_node *fyni, *fynin; + struct fy_node_pair *fynp, *fynpn; + + if (!fyn) + return; + + if (!(fyn->marks & FY_BIT(FYNWF_INSET_MARKER))) { + fy_node_delete(fyn); + return; + } + + switch (fyn->type) { + case FYNT_SCALAR: + break; + + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn->sequence); fyni; fyni = fynin) { + fynin = fy_node_next(&fyn->sequence, fyni); + + fy_node_delete_non_marked(fyni); + } + break; + + case FYNT_MAPPING: + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; fynp = fynpn) { + fynpn = fy_node_pair_next(&fyn->mapping, fynp); + + /* the mark is on the value */ + fy_node_delete_non_marked(fynp->value); + } + break; + } +} + +struct fy_walk_result * +fy_walk_result_perform_set_op(struct fy_path_exec *fypx, struct fy_walk_result *input, + struct fy_walk_result *set, enum fy_walk_result_set_op op) +{ + struct fy_walk_result *fwr, *fwrrm, *fwrin; + struct fy_walk_result *output = NULL; + char *relpath = NULL; + struct fy_node *fyn2, *fynt; + int rc; + + /* no input? return it */ + if (!input) + return NULL; + + if (!set) { + /* on unselect, return input, on select return NULL */ + if (op == FYWRSO_UNSELECT) + return input; + + fy_walk_result_free(input); + return NULL; + } + + assert(input->type == fwrt_node_ref || input->type == fwrt_refs); + +#ifdef DEBUG_EXPR + fy_walk_result_dump(input, fypx->cfg.diag, FYET_WARNING, 0, "set-op: input\n"); + fy_walk_result_dump(input, fypx->cfg.diag, FYET_WARNING, 0, "set-op: set\n"); +#endif + + for (fwrin = fy_walk_result_iter_start(input); fwrin; fwrin = fy_walk_result_iter_next(input, fwrin)) { + +#ifdef DEBUG_EXPR + fy_walk_result_dump(input, fypx->cfg.diag, FYET_WARNING, 0, "set-op: fwrin\n"); +#endif + + fwr = NULL; + + for (fwrrm = fy_walk_result_iter_start(set); fwrrm; fwrrm = fy_walk_result_iter_next(set, fwrrm)) { + +#ifdef DEBUG_EXPR + fy_walk_result_dump(fwrrm, fypx->cfg.diag, FYET_WARNING, 0, "set-op: fwrrm\n"); +#endif + + if (!output) { + output = fy_path_exec_walk_result_create(fypx, fwrt_refs); + if (!output) + goto err_out; + } + + /* if this fails, it means we don't have any output */ + if (fwrin->fyn == fwrrm->fyn || + (fwrrm->type == fwrt_node_ref && (relpath = fy_node_get_path_relative_to(fwrin->fyn, fwrrm->fyn)) == NULL)) + continue; +#ifdef DEBUG_EXPR + fy_diag_diag(fypx->cfg.diag, FYDF_WARNING, "relpath: %s", relpath); +#endif + + /* ok, do a deep copy now */ + if (!fwr) { + fwr = fy_walk_result_clone_deep(fwrin); + if (!fwr) + goto err_out; + } + + /* get the node at the relative path of the copy */ + fyn2 = fy_node_by_path(fwr->fyn, relpath, FY_NT, FYNWF_DONT_FOLLOW); + if (!fyn2) + goto err_out; + + free(relpath); + relpath = NULL; + + if (op == FYWRSO_UNSELECT) { + /* and remove it */ + rc = fy_node_delete(fyn2); + if (rc) + goto err_out; + } else { + + fynt = fyn2; + while (fynt && !(fynt->marks & FY_BIT(FYNWF_INSET_MARKER))) { + fynt->marks |= FY_BIT(FYNWF_INSET_MARKER); +#ifdef DEBUG_EXPR + fy_diag_diag(fypx->cfg.diag, FYDF_WARNING, "marking: %s", fy_node_get_path_alloca(fynt)); +#endif + fynt = fynt->parent; + } + } + } + + if (output && fwr) { + fy_walk_result_list_add_tail(&output->refs, fwr); + } + fwr = NULL; + } + + if (!output) { + /* nothing? nothing was removed or nothing was selected */ + if (op == FYWRSO_UNSELECT) + return input; + + fy_walk_result_free(input); + return NULL; + } + + /* simplify (might remove everything if empty) */ + output = fy_walk_result_simplify(output); + + /* for all marked nodes */ + if (output && op == FYWRSO_SELECT) { + + /* delete all not marked */ + for (fwr = fy_walk_result_iter_start(output); fwr; fwr = fy_walk_result_iter_next(output, fwr)) { +#ifdef DEBUG_EXPR + fy_diag_diag(fypx->cfg.diag, FYDF_WARNING, "deleting non marked: %s", fy_node_get_path_alloca(fwr->fyn)); +#endif + fy_node_delete_non_marked(fwr->fyn); + fy_node_clear_system_marks(fwr->fyn); + } + + } + + /* get rid of input */ + fy_walk_result_free(input); + return output; + +err_out: + fy_walk_result_free(input); + fy_walk_result_free(output); + if (relpath) + free(relpath); + return NULL; +} + +struct fy_walk_result * +fy_path_expr_execute(struct fy_path_exec *fypx, int level, struct fy_path_expr *expr, + struct fy_walk_result *input, enum fy_path_expr_type ptype) +{ + struct fy_diag *diag; + struct fy_walk_result *fwr, *fwrn, *fwrt, *fwrtn; + struct fy_walk_result *output = NULL, *input1, *output1, *input2, *output2; + struct fy_path_expr *exprn, *exprl, *exprr; + struct fy_node *fyn, *fynn, *fyni; + struct fy_token *fyt; + int start, end, count, i; + bool match; + struct fy_path_expr *exprt; + unsigned int nargs; + struct fy_walk_result **fwr_args; + void *prevp; + int rc __FY_DEBUG_UNUSED__; + + /* error */ + if (!fypx || !expr) + goto out; + + diag = fypx->cfg.diag; + +#ifdef DEBUG_EXPR + if (input) + fy_walk_result_dump(input, diag, FYET_NOTICE, level, "input %s\n", fy_path_expr_type_txt[expr->type]); +#endif + + /* recursive */ + if (input && input->type == fwrt_refs && !fy_path_expr_type_handles_refs(expr->type)) { + + output = fy_path_exec_walk_result_create(fypx, fwrt_refs); + assert(output); + + while ((fwr = fy_walk_result_list_pop(&input->refs)) != NULL) { + + fwrn = fy_path_expr_execute(fypx, level + 1, expr, fwr, ptype); + if (fwrn) + fy_walk_result_list_add_tail(&output->refs, fwrn); + } + fy_walk_result_free(input); + input = NULL; + goto out; + } + + + /* single result case is common enough to optimize */ + if (fy_path_expr_type_is_single_result(expr->type)) { + + if (input && input->type == fwrt_node_ref) { + + fynn = fy_path_expr_execute_single_result(diag, expr, input->fyn); + if (!fynn) + goto out; + + fy_walk_result_clean(input); + output = input; + output->type = fwrt_node_ref; + output->fyn = fynn; + input = NULL; + } + + goto out; + } + + /* handle the remaining multi result cases */ + switch (expr->type) { + + case fpet_chain: + + if (!input) + goto out; + + /* iterate over each chain item */ + output = input; + input = NULL; + for (exprn = fy_path_expr_list_head(&expr->children); exprn; + exprn = fy_path_expr_next(&expr->children, exprn)) { + + output = fy_path_expr_execute(fypx, level + 1, exprn, output, expr->type); + if (!output) + break; + } + + break; + + case fpet_multi: + + if (!input) + goto out; + + /* allocate a refs output result */ + output = fy_path_exec_walk_result_create(fypx, fwrt_refs); + assert(output); + + /* iterate over each multi item */ + for (exprn = fy_path_expr_list_head(&expr->children); exprn; + exprn = fy_path_expr_next(&expr->children, exprn)) { + + input2 = fy_walk_result_clone(input); + assert(input2); + + output2 = fy_path_expr_execute(fypx, level + 1, exprn, input2, expr->type); + if (!output2) + continue; + + fy_walk_result_list_add_tail(&output->refs, output2); + } + fy_walk_result_free(input); + input = NULL; + break; + + case fpet_every_child: + + if (!input) + goto out; + + /* only valid for node ref */ + if (input->type != fwrt_node_ref) + break; + + fyn = input->fyn; + + /* every scalar/alias is a single result (although it should not happen) */ + if (fy_node_is_scalar(fyn) || fy_node_is_alias(fyn)) { + output = input; + input = NULL; + break; + + } + + /* re-use input for output root */ + fy_walk_result_clean(input); + output = input; + input = NULL; + + output->type = fwrt_refs; + fy_walk_result_list_init(&output->refs); + + prevp = NULL; + while ((fyni = fy_node_collection_iterate(fyn, &prevp)) != NULL) { + fwr = fy_path_exec_walk_result_create(fypx, fwrt_node_ref, fyni); + assert(fwr); + + fy_walk_result_list_add_tail(&output->refs, fwr); + } + + break; + + case fpet_every_child_r: + + if (!input) + goto out; + + /* only valid for node ref */ + if (input->type != fwrt_node_ref) + break; + + fyn = input->fyn; + + /* re-use input for output root */ + fy_walk_result_clean(input); + output = input; + input = NULL; + + output->type = fwrt_refs; + fy_walk_result_list_init(&output->refs); + + rc = fy_walk_result_all_children_recursive_internal(fypx, fyn, output); + assert(!rc); + + break; + + case fpet_seq_slice: + + if (!input) + goto out; + + /* only valid for node ref on a sequence */ + if (input->type != fwrt_node_ref || !fy_node_is_sequence(input->fyn)) { + break; + } + fyn = input->fyn; + + fyt = expr->fyt; + assert(fyt); + assert(fyt->type == FYTT_PE_SEQ_SLICE); + + start = fyt->seq_slice.start_index; + end = fyt->seq_slice.end_index; + count = fy_node_sequence_item_count(fyn); + + /* don't handle negative slices yet */ + if (start < 0 || end < 1 || start >= end) + break; + + if (count < end) + end = count; + + /* re-use input for output root */ + fy_walk_result_clean(input); + output = input; + input = NULL; + + output->type = fwrt_refs; + fy_walk_result_list_init(&output->refs); + + for (i = start; i < end; i++) { + + fynn = fy_node_sequence_get_by_index(fyn, i); + if (!fynn) + continue; + + fwr = fy_path_exec_walk_result_create(fypx, fwrt_node_ref, fynn); + assert(fwr); + + fy_walk_result_list_add_tail(&output->refs, fwr); + } + + break; + + case fpet_eq: + case fpet_neq: + case fpet_lt: + case fpet_gt: + case fpet_lte: + case fpet_gte: + case fpet_plus: + case fpet_minus: + case fpet_mult: + case fpet_div: + + exprl = fy_path_expr_lhs(expr); + assert(exprl); + + exprr = fy_path_expr_rhs(expr); + assert(exprr); + + if (input) { + input1 = fy_walk_result_clone(input); + assert(input1); + + input2 = input; + input = NULL; + + } else { + input1 = NULL; + input2 = NULL; + } + + /* execute LHS and RHS */ + output1 = fy_path_expr_execute(fypx, level + 1, exprl, input1, expr->type); + output2 = fy_path_expr_execute(fypx, level + 1, exprr, input2, expr->type); + + output = fy_walk_result_lhs_rhs(fypx, expr, exprl, output1, exprr, output2); + + break; + + case fpet_scalar: + + /* duck typing! */ + if (fy_token_is_number(expr->fyt)) { + output = fy_path_exec_walk_result_create(fypx, fwrt_number, token_number(expr->fyt)); + assert(output); + } else { + output = fy_path_exec_walk_result_create(fypx, fwrt_string, fy_token_get_text0(expr->fyt)); + assert(output); + } + + fy_walk_result_free(input); + input = NULL; + + break; + + case fpet_logical_or: + + /* return the first that is not NULL */ + for (exprn = fy_path_expr_list_head(&expr->children); exprn; + exprn = fy_path_expr_next(&expr->children, exprn)) { + + if (input) { + input1 = fy_walk_result_clone(input); + assert(input1); + } else { + input1 = NULL; + } + + output = fy_path_expr_execute(fypx, level + 1, exprn, input1, expr->type); + if (output) + break; + } + break; + + case fpet_logical_and: + output = NULL; + + /* return the last that was not NULL */ + for (exprn = fy_path_expr_list_head(&expr->children); exprn; + exprn = fy_path_expr_next(&expr->children, exprn)) { + + if (input) { + input1 = fy_walk_result_clone(input); + assert(input1); + } else { + input1 = NULL; + } + + output1 = fy_path_expr_execute(fypx, level + 1, exprn, input1, expr->type); + if (output1) { + fy_walk_result_free(output); + output = output1; + } else + break; + } + break; + + case fpet_filter_unique: + + if (!input) + goto out; + + /* for non refs, return input */ + if (input->type != fwrt_refs) { + output = input; + input = NULL; + break; + } + + /* nothing in there? */ + if (fy_walk_result_list_empty(&input->refs)) { + output = NULL; + break; + } + + /* remove duplicates filter */ + for (fwr = fy_walk_result_list_head(&input->refs); fwr; + fwr = fy_walk_result_next(&input->refs, fwr)) { + + /* do not check recursively */ + if (fwr->type == fwrt_refs) { + continue; + } + + /* check the entries from this point forward */ + for (fwrt = fy_walk_result_next(&input->refs, fwr); fwrt; fwrt = fwrtn) { + + fwrtn = fy_walk_result_next(&input->refs, fwrt); + + /* do not check recursively (or the same result) */ + if (fwrt->type == fwrt_refs) + continue; + + assert(fwrt != fwr); + + match = fy_walk_result_compare_simple(fypx, fpet_eq, fwr, fwrt); + + if (match) { + fy_walk_result_list_del(&input->refs, fwrt); + fy_walk_result_free(fwrt); + } + } + } + output = input; + input = NULL; + + break; + + case fpet_scalar_expr: + + exprl = fy_path_expr_list_head(&expr->children); + if (!exprl) { + fy_warning(diag, "%s:%d\n", __FILE__, __LINE__); + goto out; + } + + output = fy_path_expr_execute(fypx, level + 1, exprl, NULL, ptype); + if (!output) { + fy_warning(diag, "%s:%d\n", __FILE__, __LINE__); + goto out; + } + + exprt = fy_scalar_walk_result_to_expr(fypx, output, ptype); + output = NULL; + if (!exprt) { + fy_warning(diag, "%s:%d\n", __FILE__, __LINE__); + break; + } + + output = fy_path_expr_execute(fypx, level + 1, exprt, input, ptype); + if (!output) { + fy_warning(diag, "%s:%d\n", __FILE__, __LINE__); + } + input = NULL; + + fy_path_expr_free(exprt); + break; + + case fpet_path_expr: + exprl = fy_path_expr_list_head(&expr->children); + if (!exprl) + goto out; + + output = fy_path_expr_execute(fypx, level + 1, exprl, input, ptype); + input = NULL; + break; + + case fpet_method: + + assert(expr->fym); + + /* execute the arguments */ + nargs = expr->fym->nargs; + if (nargs > 0) { + fwr_args = alloca(sizeof(*fwr_args) * nargs); + memset(fwr_args, 0, sizeof(*fwr_args) * nargs); + for (i = 0, exprt = fy_path_expr_list_head(&expr->children); exprt; + exprt = fy_path_expr_next(&expr->children, exprt), i++) { + + if (input) { + input1 = fy_walk_result_clone(input); + assert(input1); + } else + input1 = NULL; + + fwr_args[i] = fy_path_expr_execute(fypx, level + 1, exprt, input1, expr->type); + } + } else + fwr_args = NULL; + + output = expr->fym->exec(expr->fym, fypx, level + 1, expr, input, fwr_args, nargs); + input = NULL; + + break; + + case fpet_select: + case fpet_unselect: + + /* pop the top in either case */ + assert(input); + if (!input) + goto out; + + /* only handle inputs of node and refs */ + if (input->type != fwrt_node_ref && input->type != fwrt_refs) + goto out; + + /* execute the expression */ + exprn = fy_path_expr_list_head(&expr->children); + assert(exprn); + if (!exprn) + goto out; + + fwrt = fy_walk_result_clone(input); + assert(fwrt); + if (!fwrt) + goto out; + + fwrn = fy_path_expr_execute(fypx, level + 1, exprn, fwrt, expr->type); + + output = fy_walk_result_perform_set_op(fypx, input, fwrn, expr->type == fpet_unselect ? FYWRSO_UNSELECT : FYWRSO_SELECT); + input = NULL; + + fy_walk_result_free(fwrn); + fwrn = NULL; + break; + + default: + fy_error(diag, "%s\n", fy_path_expr_type_txt[expr->type]); + FY_IMPOSSIBLE_ABORT(); + } + +out: + fy_walk_result_free(input); + output = fy_walk_result_simplify(output); + +#ifdef DEBUG_EXPR + if (output) + fy_walk_result_dump(output, diag, FYET_NOTICE, level, "output %s\n", fy_path_expr_type_txt[expr->type]); +#endif + return output; +} + +static int fy_path_exec_execute_internal(struct fy_path_exec *fypx, + struct fy_path_expr *expr, struct fy_node *fyn_start) +{ + struct fy_walk_result *fwr; + + if (!fypx || !expr || !fyn_start) + return -1; + + fy_walk_result_free(fypx->result); + fypx->result = NULL; + + fwr = fy_path_exec_walk_result_create(fypx, fwrt_node_ref, fyn_start); + assert(fwr); + + fwr = fy_path_expr_execute(fypx, 0, expr, fwr, fpet_none); + if (!fwr) + return 0; + + /* flatten results */ + if (fwr->type == fwrt_refs) { + fwr = fy_walk_result_flatten(fwr); + if (!fwr) + return -1; + } + fypx->result = fwr; + + return 0; +} + +int fy_path_exec_execute(struct fy_path_exec *fypx, struct fy_path_expr *expr, struct fy_node *fyn_start) +{ + if (!fypx || !expr || !fyn_start) + return -1; + + fypx->fyn_start = fyn_start; + return fy_path_exec_execute_internal(fypx, expr, fypx->fyn_start); +} + +struct fy_node * +fy_path_exec_results_iterate(struct fy_path_exec *fypx, void **prevp) +{ + struct fy_walk_result *fwr; + + if (!fypx || !prevp) + return NULL; + + if (!fypx->result) + return NULL; + + if (fypx->result->type != fwrt_refs) { + fwr = fypx->result; + + if (fwr->type != fwrt_node_ref) + return NULL; + + if (!*prevp) { + *prevp = fwr; + return fwr->fyn; + } + *prevp = NULL; + return NULL; + } + + /* loop over non node refs for now */ + do { + if (!*prevp) + fwr = fy_walk_result_list_head(&fypx->result->refs); + else + fwr = fy_walk_result_next(&fypx->result->refs, *prevp); + *prevp = fwr; + } while (fwr && fwr->type != fwrt_node_ref); + + return fwr ? fwr->fyn : NULL; +} + +struct fy_walk_result * +fy_path_exec_take_results(struct fy_path_exec *fypx) +{ + struct fy_walk_result *fwr; + + if (!fypx || !fypx->result) + return NULL; + fwr = fypx->result; + fypx->result = NULL; + return fwr; +} + +struct fy_walk_result * +fy_path_exec_walk_result_vcreate(struct fy_path_exec *fypx, enum fy_walk_result_type type, va_list ap) +{ + struct fy_walk_result_list *fwrl; + + if (!fypx) + return NULL; + + fwrl = fy_path_exec_walk_result_rl(fypx); + return fy_walk_result_vcreate_rl(fwrl, type, ap); +} + +struct fy_walk_result * +fy_path_exec_walk_result_create(struct fy_path_exec *fypx, enum fy_walk_result_type type, ...) +{ + struct fy_walk_result_list *fwrl; + struct fy_walk_result *fwr; + va_list ap; + + if (!fypx) + return NULL; + + fwrl = fy_path_exec_walk_result_rl(fypx); + + va_start(ap, type); + fwr = fy_walk_result_vcreate_rl(fwrl, type, ap); + va_end(ap); + + if (!fwr) + return NULL; + + fwr->fypx = fy_path_exec_ref(fypx); + + return fwr; +} + +void +fy_path_exec_walk_result_free(struct fy_path_exec *fypx, struct fy_walk_result *fwr) +{ + struct fy_walk_result_list *fwrl; + + fwrl = fypx ? fy_path_exec_walk_result_rl(fypx) : NULL; + fy_walk_result_free_rl(fwrl, fwr); +} + +int fy_document_setup_path_expr_data(struct fy_document *fyd) +{ + struct fy_path_parse_cfg pcfg_local, *pcfg = &pcfg_local; + struct fy_path_expr_document_data *pxdd; + + if (!fyd || fyd->pxdd) + return 0; + + pxdd = malloc(sizeof(*pxdd)); + if (!pxdd) + goto err_no_mem; + + memset(pxdd, 0, sizeof(*pxdd)); + + fy_walk_result_list_init(&pxdd->fwr_recycle); + + memset(pcfg, 0, sizeof(*pcfg)); + pcfg->diag = fyd->diag; + pxdd->fypp = fy_path_parser_create(pcfg); + if (!pxdd->fypp) + goto err_no_fypp; + + fyd->pxdd = pxdd; + + return 0; + +err_no_fypp: + free(pxdd); +err_no_mem: + return -1; +} + +void fy_document_cleanup_path_expr_data(struct fy_document *fyd) +{ + struct fy_path_expr_document_data *pxdd; + struct fy_walk_result *fwr; + + if (!fyd || !fyd->pxdd) + return; + + pxdd = fyd->pxdd; + + fy_path_parser_destroy(pxdd->fypp); + + while ((fwr = fy_walk_result_list_pop(&pxdd->fwr_recycle)) != NULL) + free(fwr); + + free(fyd->pxdd); + fyd->pxdd = NULL; +} + +int fy_node_setup_path_expr_data(struct fy_node *fyn) +{ + struct fy_path_expr_document_data *pxdd; + struct fy_path_expr_node_data *pxnd; + const char *text; + size_t len; + char *alloc = NULL; + int rc; + + if (!fyn || fyn->pxnd) + return 0; + + /* only on alias nodes */ + if (!fy_node_is_alias(fyn)) + return 0; + + /* a document must exist */ + if (!fyn->fyd) + return -1; + + if (!fyn->fyd->pxdd) { + rc = fy_document_setup_path_expr_data(fyn->fyd); + if (rc) + return rc; + } + pxdd = fyn->fyd->pxdd; + assert(pxdd); + + pxnd = malloc(sizeof(*pxnd)); + if (!pxnd) + goto err_no_mem; + memset(pxnd, 0, sizeof(*pxnd)); + + text = fy_token_get_text(fyn->scalar, &len); + if (!text) + goto err_no_text; + + if (!fy_is_first_alpha(*text)) { + pxnd->fyi = fy_input_from_data(text, len, NULL, false); + if (!pxnd->fyi) + goto err_no_input; + } else { + alloc = malloc(len + 2); + if (!alloc) + goto err_no_input; + alloc[0] = '*'; + memcpy(alloc + 1, text, len); + alloc[len + 1] = '\0'; + + pxnd->fyi = fy_input_from_malloc_data(alloc, len + 1, NULL, false); + if (!pxnd->fyi) + goto err_no_input; + alloc = NULL; + } + + fy_path_parser_reset(pxdd->fypp); + + rc = fy_path_parser_open(pxdd->fypp, pxnd->fyi, NULL); + if (rc) + goto err_no_open; + + pxnd->expr = fy_path_parse_expression(pxdd->fypp); + if (!pxnd->expr) + goto err_parse; + + fy_path_parser_close(pxdd->fypp); + + fyn->pxnd = pxnd; + + return 0; +err_parse: + fy_path_parser_close(pxdd->fypp); +err_no_open: + fy_input_unref(pxnd->fyi); +err_no_input: + if (alloc) + free(alloc); +err_no_text: + free(pxnd); +err_no_mem: + return -1; +} + +void fy_node_cleanup_path_expr_data(struct fy_node *fyn) +{ + struct fy_path_expr_node_data *pxnd; + + if (!fyn || !fyn->pxnd) + return; + + pxnd = fyn->pxnd; + + if (pxnd->expr) + fy_path_expr_free(pxnd->expr); + + if (pxnd->fyi) + fy_input_unref(pxnd->fyi); + + free(pxnd); + fyn->pxnd = NULL; +} + +struct fy_walk_result * +fy_node_alias_resolve_by_ypath_result(struct fy_node *fyn) +{ + struct fy_document *fyd; + struct fy_path_expr_document_data *pxdd = NULL; + struct fy_path_expr_node_data *pxnd = NULL; + struct fy_walk_result *fwr; + struct fy_anchor *fya; + struct fy_path_exec *fypx = NULL; + int rc; + + if (!fyn || !fy_node_is_alias(fyn)) + return NULL; + + fyd = fyn->fyd; + if (!fyd) + return NULL; + + /* simple */ + fya = fy_document_lookup_anchor_by_token(fyd, fyn->scalar); + if (fya) { + + fwr = fy_path_exec_walk_result_create(fypx, fwrt_node_ref, fya->fyn); + fyd_error_check(fyd, fwr, err_out, + "fy_walk_result_alloc_rl() failed"); + + return fwr; + } + + /* ok, complex, setup the node data */ + rc = fy_node_setup_path_expr_data(fyn); + fyd_error_check(fyd, !rc, err_out, + "fy_node_setup_path_expr_data() failed"); + + pxnd = fyn->pxnd; + assert(pxnd); + + pxdd = fyd->pxdd; + assert(pxdd); + + if (pxnd->traversals++ > 0) { + FYD_NODE_ERROR(fyd, fyn, FYEM_DOC, + "recursive reference detected at %s\n", + fy_node_get_path_alloca(fyn)); + pxnd->traversals--; + return NULL; + } + + fypx = fy_path_exec_create_on_document(fyd); + fyd_error_check(fyd, !rc, err_out, + "fy_path_exec_create_on_document() failed"); + + fy_path_exec_set_result_recycle_list(fypx, &pxdd->fwr_recycle); + +#if 0 + { + struct fy_document *fyd_pe; + const char *text; + size_t len; + + text = fy_token_get_text(fyn->scalar, &len); + if (text) { + fyd_pe = fy_path_expr_to_document(pxnd->expr); + if (fyd_pe) { + fprintf(stderr, "%s: %.*s\n", __func__, (int)len, text); + fy_document_default_emit_to_fp(fyd_pe, stderr); + fy_document_destroy(fyd_pe); + } + } + } +#endif + + // fprintf(stderr, "%s: %s 2\n", __func__, fy_node_get_path_alloca(fyn)); + + /* execute, starting at this */ + rc = fy_path_exec_execute(fypx, pxnd->expr, fyn); + fyd_error_check(fyd, !rc, err_out, + "fy_path_exec_execute() failed"); + + // fprintf(stderr, "%s: %s 3\n", __func__, fy_node_get_path_alloca(fyn)); + + fwr = fy_path_exec_take_results(fypx); + + fy_path_exec_unref(fypx); + + pxnd->traversals--; + + if (!fwr) + return NULL; + + // fprintf(stderr, "%s: %s 4\n", __func__, fy_node_get_path_alloca(fyn)); + + return fwr; + +err_out: + if (pxnd) + pxnd->traversals--; + fy_path_exec_unref(fypx); /* NULL OK */ + return NULL; +} + +struct fy_node *fy_node_alias_resolve_by_ypath(struct fy_node *fyn) +{ + struct fy_anchor *fya; + struct fy_walk_result *fwr; + void *iterp; + + if (!fyn || !fy_node_is_alias(fyn)) + return NULL; + + /* simple and common enough to do it now */ + fya = fy_document_lookup_anchor_by_token(fyn->fyd, fyn->scalar); + if (fya) + return fya->fyn; + + fwr = fy_node_alias_resolve_by_ypath_result(fyn); + if (!fwr) + return NULL; + + iterp = NULL; + fyn = fy_walk_result_node_iterate(fwr, &iterp); + + fy_walk_result_free(fwr); + + return fyn; +} + +struct fy_walk_result * +fy_node_by_ypath_result(struct fy_node *fyn, const char *path, size_t len) +{ + struct fy_path_expr_document_data *pxdd; + struct fy_document *fyd; + struct fy_walk_result *fwr; + struct fy_anchor *fya; + struct fy_input *fyi; + struct fy_path_expr *expr; + struct fy_path_exec *fypx = NULL; + int rc; + + if (!fyn || !path || !len) + return NULL; + + fyd = fyn->fyd; + if (!fyd) + return NULL; + + if (len == FY_NT) + len = strlen(path); + + /* simple */ + fya = fy_document_lookup_anchor(fyn->fyd, path, len); + if (fya) { + fwr = fy_path_exec_walk_result_create(fypx, fwrt_node_ref, fya->fyn); + fyd_error_check(fyd, fwr, err_out, + "fy_walk_result_alloc_rl() failed"); + return fwr; + } + + /* ok, complex, setup the document data */ + rc = fy_document_setup_path_expr_data(fyd); + fyd_error_check(fyd, !rc, err_setup, + "fy_node_setup_path_expr_data() failed"); + + pxdd = fyd->pxdd; + assert(pxdd); + + fyi = fy_input_from_data(path, len, NULL, false); + fyd_error_check(fyd, fyi, err_no_input, + "fy_input_from_data() failed"); + + fy_path_parser_reset(pxdd->fypp); + + rc = fy_path_parser_open(pxdd->fypp, fyi, NULL); + fyd_error_check(fyd, !rc, err_no_open, + "fy_path_parser_open() failed"); + + expr = fy_path_parse_expression(pxdd->fypp); + fyd_error_check(fyd, expr, err_parse, + "fy_path_parse_expression() failed"); + + fy_path_parser_close(pxdd->fypp); + + fypx = fy_path_exec_create_on_document(fyd); + fyd_error_check(fyd, !rc, err_no_fypx, + "fy_path_exec_create_on_document() failed"); + + /* execute, starting at this */ + rc = fy_path_exec_execute(fypx, expr, fyn); + fyd_error_check(fyd, !rc, err_exec, + "fy_path_parse_expression() failed"); + + fwr = fy_path_exec_take_results(fypx); + + fy_path_exec_unref(fypx); + + fy_path_expr_free(expr); + fy_input_unref(fyi); + + return fwr; + +err_exec: + fy_path_expr_free(expr); +err_no_fypx: + fy_path_exec_unref(fypx); +err_parse: + fy_path_parser_close(pxdd->fypp); + +err_no_open: + fy_input_unref(fyi); + +err_no_input: +err_setup: +err_out: + return NULL; +} + +struct fy_node *fy_node_by_ypath(struct fy_node *fyn, const char *path, size_t len) +{ + struct fy_walk_result *fwr; + struct fy_anchor *fya; + void *iterp; + + if (!fyn || !path || !len) + return NULL; + + /* simple */ + fya = fy_document_lookup_anchor(fyn->fyd, path, len); + if (fya) + return fya->fyn; + + fwr = fy_node_by_ypath_result(fyn, path, len); + if (!fwr) + return NULL; + + iterp = NULL; + fyn = fy_walk_result_node_iterate(fwr, &iterp); + + fy_walk_result_free(fwr); + + return fyn; +} diff --git a/src/lib/fy-walk.h b/src/lib/fy-walk.h new file mode 100644 index 0000000..3697e9c --- /dev/null +++ b/src/lib/fy-walk.h @@ -0,0 +1,448 @@ +/* + * fy-walk.h - walker internal header file + * + * Copyright (c) 2021 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_WALK_H +#define FY_WALK_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include + +#include + +#include "fy-ctype.h" +#include "fy-utf8.h" +#include "fy-list.h" +#include "fy-typelist.h" +#include "fy-types.h" +#include "fy-diag.h" +#include "fy-dump.h" +#include "fy-docstate.h" +#include "fy-accel.h" +#include "fy-token.h" +#include "fy-input.h" + +struct fy_document; + +enum fy_walk_result_type { + fwrt_none, + fwrt_node_ref, + fwrt_number, + fwrt_string, + fwrt_doc, + fwrt_refs, +}; + +#define FWRT_COUNT (fwrt_refs + 1) +extern const char *fy_walk_result_type_txt[FWRT_COUNT]; + +struct fy_path_exec; + +FY_TYPE_FWD_DECL_LIST(walk_result); +struct fy_walk_result { + struct list_head node; + struct fy_path_exec *fypx; + enum fy_walk_result_type type; + union { + uintptr_t _clean[2]; + struct { + struct fy_node *fyn; + struct fy_node *fyn_source; /* when deep */ + }; + double number; + char *string; + struct fy_walk_result_list refs; + struct fy_document *fyd; + }; +}; +FY_TYPE_DECL_LIST(walk_result); + +struct fy_walk_result *fy_walk_result_alloc_rl(struct fy_walk_result_list *fwrl); +void fy_walk_result_free_rl(struct fy_walk_result_list *fwrl, struct fy_walk_result *fwr); +void fy_walk_result_list_free_rl(struct fy_walk_result_list *fwrl, struct fy_walk_result_list *results); + +void fy_walk_result_free(struct fy_walk_result *fwr); + +struct fy_walk_result *fy_walk_result_vcreate_rl(struct fy_walk_result_list *fwrl, enum fy_walk_result_type type, va_list ap); +struct fy_walk_result *fy_walk_result_create_rl(struct fy_walk_result_list *fwrl, enum fy_walk_result_type type, ...); + +static inline struct fy_walk_result * +fy_walk_result_iter_start(struct fy_walk_result *fwr) +{ + struct fy_walk_result *fwri; + + if (!fwr) + return NULL; + if (fwr->type != fwrt_refs) + return fwr; + fwri = fy_walk_result_list_head(&fwr->refs); + if (!fwri) + return NULL; + return fwri; +} + +static inline struct fy_walk_result * +fy_walk_result_iter_next(struct fy_walk_result *fwr, struct fy_walk_result *fwri) +{ + if (!fwr || !fwri || fwr->type != fwrt_refs) + return NULL; + fwri = fy_walk_result_next(&fwr->refs, fwri); + if (!fwri) + return NULL; + return fwri; +} + +struct fy_node * +fy_walk_result_node_iterate(struct fy_walk_result *fwr, void **prevp); + +enum fy_path_expr_type { + fpet_none, + /* ypath */ + fpet_root, /* /^ or / at the beginning of the expr */ + fpet_this, /* /. */ + fpet_parent, /* /.. */ + fpet_every_child, // /* every immediate child + fpet_every_child_r, // /** every recursive child + fpet_filter_collection, /* match only collection (at the end only) */ + fpet_filter_scalar, /* match only scalars (leaves) */ + fpet_filter_sequence, /* match only sequences */ + fpet_filter_mapping, /* match only mappings */ + fpet_filter_unique, /* removes duplicates */ + fpet_seq_index, + fpet_map_key, /* complex map key (quoted, flow seq or map) */ + fpet_seq_slice, + fpet_alias, + + fpet_multi, /* merge results of children */ + fpet_chain, /* children move in sequence */ + fpet_logical_or, /* first non null result set */ + fpet_logical_and, /* the last non null result set */ + + fpet_select, /* set selection */ + fpet_unselect, /* negation of set selection */ + + fpet_eq, /* equal expression */ + fpet_neq, /* not equal */ + fpet_lt, /* less than */ + fpet_gt, /* greater than */ + fpet_lte, /* less or equal than */ + fpet_gte, /* greater or equal than */ + + fpet_scalar, /* scalar */ + + fpet_plus, /* add */ + fpet_minus, /* subtract */ + fpet_mult, /* multiply */ + fpet_div, /* divide */ + + fpet_lparen, /* left paren (they do not appear in final expression) */ + fpet_rparen, /* right parent */ + fpet_method, /* method (or parentheses) */ + + fpet_scalar_expr, /* non-eval phase scalar expression */ + fpet_path_expr, /* non-eval phase path expression */ + fpet_arg_separator, /* argument separator (comma in scalar mode) */ +}; + +#define FPET_COUNT (fpet_arg_separator + 1) + +extern const char *path_expr_type_txt[FPET_COUNT]; + +static inline bool fy_path_expr_type_is_valid(enum fy_path_expr_type type) +{ + return type >= fpet_root && type < FPET_COUNT; +} + +/* XXX do neg and reg check */ +static inline bool fy_path_expr_type_is_single_result(enum fy_path_expr_type type) +{ + return type == fpet_root || + type == fpet_this || + type == fpet_parent || + type == fpet_map_key || + type == fpet_seq_index || + type == fpet_alias || + type == fpet_filter_collection || + type == fpet_filter_scalar || + type == fpet_filter_sequence || + type == fpet_filter_mapping; +} + +static inline bool fy_path_expr_type_is_parent(enum fy_path_expr_type type) +{ + return type == fpet_multi || + type == fpet_chain || + type == fpet_logical_or || + type == fpet_logical_and || + type == fpet_eq || + type == fpet_method || + type == fpet_scalar_expr || + type == fpet_path_expr; +} + +static inline bool fy_path_expr_type_is_mergeable(enum fy_path_expr_type type) +{ + return type == fpet_multi || + type == fpet_chain || + type == fpet_logical_or || + type == fpet_logical_and; +} + +/* type handles refs by itself */ +static inline bool fy_path_expr_type_handles_refs(enum fy_path_expr_type type) +{ + return type == fpet_filter_unique || + type == fpet_method; +} + +static inline bool fy_path_expr_type_is_parent_lhs_rhs(enum fy_path_expr_type type) +{ + return type == fpet_eq || + type == fpet_neq || + type == fpet_lt || + type == fpet_gt || + type == fpet_lte || + type == fpet_gte || + + type == fpet_plus || + type == fpet_minus || + type == fpet_mult || + type == fpet_div; +} + +static inline bool +fy_path_expr_type_is_conditional(enum fy_path_expr_type type) +{ + return type == fpet_eq || + type == fpet_neq || + type == fpet_lt || + type == fpet_gt || + type == fpet_lte || + type == fpet_gte; +} + +static inline bool +fy_path_expr_type_is_arithmetic(enum fy_path_expr_type type) +{ + return type == fpet_plus || + type == fpet_minus || + type == fpet_mult || + type == fpet_div; +} + +static inline bool +fy_path_expr_type_is_lparen(enum fy_path_expr_type type) +{ + return type == fpet_lparen /* || + type == fpet_method */ ; +} + +enum fy_expr_mode { + fyem_none, /* invalid mode */ + fyem_path, /* expression is path */ + fyem_scalar, /* expression is scalar */ +}; + +#define FYEM_COUNT (fyem_scalar + 1) + +extern const char *fy_expr_mode_txt[FYEM_COUNT]; + +struct fy_path_expr; + +struct fy_method { + const char *name; + size_t len; + enum fy_expr_mode mode; + unsigned int nargs; + struct fy_walk_result *(*exec)(const struct fy_method *fym, + struct fy_path_exec *fypx, int level, + struct fy_path_expr *expr, + struct fy_walk_result *input, + struct fy_walk_result **args, int nargs); +}; + +FY_TYPE_FWD_DECL_LIST(path_expr); +struct fy_path_expr { + struct list_head node; + struct fy_path_expr *parent; + enum fy_path_expr_type type; + struct fy_token *fyt; + struct fy_path_expr_list children; + enum fy_expr_mode expr_mode; /* for parens */ + const struct fy_method *fym; +}; +FY_TYPE_DECL_LIST(path_expr); + +static inline struct fy_path_expr * +fy_path_expr_lhs(struct fy_path_expr *expr) +{ + if (!expr || !fy_path_expr_type_is_parent_lhs_rhs(expr->type)) + return NULL; + return fy_path_expr_list_head(&expr->children); +} + +static inline struct fy_path_expr * +fy_path_expr_rhs(struct fy_path_expr *expr) +{ + if (!expr || !fy_path_expr_type_is_parent_lhs_rhs(expr->type)) + return NULL; + return fy_path_expr_list_tail(&expr->children); +} + +const struct fy_mark *fy_path_expr_start_mark(struct fy_path_expr *expr); +const struct fy_mark *fy_path_expr_end_mark(struct fy_path_expr *expr); + +struct fy_expr_stack { + unsigned int top; + unsigned int alloc; + struct fy_path_expr **items; + struct fy_path_expr *items_static[32]; +}; + +void fy_expr_stack_setup(struct fy_expr_stack *stack); +void fy_expr_stack_cleanup(struct fy_expr_stack *stack); +void fy_expr_stack_dump(struct fy_diag *diag, struct fy_expr_stack *stack); +int fy_expr_stack_push(struct fy_expr_stack *stack, struct fy_path_expr *expr); +struct fy_path_expr *fy_expr_stack_peek_at(struct fy_expr_stack *stack, unsigned int pos); +struct fy_path_expr *fy_expr_stack_peek(struct fy_expr_stack *stack); +struct fy_path_expr *fy_expr_stack_pop(struct fy_expr_stack *stack); + +struct fy_path_parser { + struct fy_path_parse_cfg cfg; + struct fy_reader reader; + struct fy_token_list queued_tokens; + enum fy_token_type last_queued_token_type; + bool stream_start_produced; + bool stream_end_produced; + bool stream_error; + bool owns_diag; + int token_activity_counter; + + struct fy_input *fyi; + struct fy_expr_stack operators; + struct fy_expr_stack operands; + + /* to avoid allocating */ + struct fy_path_expr_list expr_recycle; + bool suppress_recycling; + + enum fy_expr_mode expr_mode; + int paren_nest_level; + +}; + +struct fy_path_expr *fy_path_expr_alloc(void); +/* fy_path_expr_free is declared in libfyaml.h */ +// void fy_path_expr_free(struct fy_path_expr *expr); + +void fy_path_parser_setup(struct fy_path_parser *fypp, const struct fy_path_parse_cfg *pcfg); +void fy_path_parser_cleanup(struct fy_path_parser *fypp); +int fy_path_parser_open(struct fy_path_parser *fypp, + struct fy_input *fyi, const struct fy_reader_input_cfg *icfg); +void fy_path_parser_close(struct fy_path_parser *fypp); + +struct fy_token *fy_path_scan(struct fy_path_parser *fypp); + +struct fy_path_expr *fy_path_parse_expression(struct fy_path_parser *fypp); + +void fy_path_expr_dump(struct fy_path_expr *expr, struct fy_diag *diag, enum fy_error_type errlevel, int level, const char *banner); + +struct fy_path_exec { + struct fy_path_exec_cfg cfg; + struct fy_node *fyn_start; + struct fy_walk_result *result; + struct fy_walk_result_list *fwr_recycle; + int refs; + bool supress_recycling; +}; + +struct fy_path_exec *fy_path_exec_create(const struct fy_path_exec_cfg *xcfg); +struct fy_path_exec *fy_path_exec_create_on_document(struct fy_document *fyd); +void fy_path_exec_destroy(struct fy_path_exec *fypx); +void fy_path_exec_cleanup(struct fy_path_exec *fypx); + +static inline struct fy_path_exec * +fy_path_exec_ref(struct fy_path_exec *fypx) +{ + /* take care of overflow */ + if (!fypx) + return NULL; + assert(fypx->refs + 1 > 0); + fypx->refs++; + + return fypx; +} + +static inline void +fy_path_exec_unref(struct fy_path_exec *fypx) +{ + if (!fypx) + return; + + assert(fypx->refs > 0); + + if (--fypx->refs == 0) + fy_path_exec_destroy(fypx); +} + +struct fy_walk_result * +fy_path_expr_execute(struct fy_path_exec *fypx, int level, struct fy_path_expr *expr, + struct fy_walk_result *input, enum fy_path_expr_type ptype); + +static inline struct fy_walk_result_list * +fy_path_exec_walk_result_rl(struct fy_path_exec *fypx) +{ + return fypx && !fypx->supress_recycling ? fypx->fwr_recycle : NULL; +} + +static inline void +fy_path_exec_set_result_recycle_list(struct fy_path_exec *fypx, + struct fy_walk_result_list *fwrl) +{ + if (!fypx) + return; + fypx->fwr_recycle = fwrl; +} + +struct fy_walk_result * +fy_path_exec_walk_result_create(struct fy_path_exec *fypx, enum fy_walk_result_type type, ...); + +void +fy_path_exec_walk_result_free(struct fy_path_exec *fypx, struct fy_walk_result *fwr); + +struct fy_path_expr_document_data { + struct fy_path_parser *fypp; + struct fy_walk_result_list fwr_recycle; +}; + +struct fy_path_expr_node_data { + struct fy_input *fyi; + struct fy_path_expr *expr; + struct fy_node *fyn_target; + int traversals; +}; + +int fy_document_setup_path_expr_data(struct fy_document *fyd); +void fy_document_cleanup_path_expr_data(struct fy_document *fyd); +int fy_node_setup_path_expr_data(struct fy_node *fyn); +void fy_node_cleanup_path_expr_data(struct fy_node *fyn); + +struct fy_walk_result * +fy_node_alias_resolve_by_ypath_result(struct fy_node *fyn); +struct fy_node *fy_node_alias_resolve_by_ypath(struct fy_node *fyn); + +struct fy_walk_result * +fy_node_by_ypath_result(struct fy_node *fyn, const char *path, size_t len); +struct fy_node *fy_node_by_ypath(struct fy_node *fyn, const char *path, size_t len); + +#endif diff --git a/src/thread/fy-thread.c b/src/thread/fy-thread.c new file mode 100644 index 0000000..08b7771 --- /dev/null +++ b/src/thread/fy-thread.c @@ -0,0 +1,1160 @@ +/* + * fy-thread.c - Lighting fast thread pool implementation + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#include +#endif + +#include "fy-diag.h" +#include "fy-utils.h" +#include "fy-thread.h" + +#undef WORK_SHUTDOWN +#define WORK_SHUTDOWN ((struct fy_thread_work *)(void *)-1) + +#ifdef FY_THREAD_DEBUG +#define TDBG(_fmt, ...) \ + do { fprintf(stderr, (_fmt) __VA_OPT__(,) __VA_ARGS__); } while(0) +#else +#define TDBG(_fmt, ...) \ + do { /* nothing */ } while(0) +#endif + +static void *fy_worker_thread_standard(void *arg); +static void *fy_worker_thread_steal(void *arg); + +static void fy_thread_work_join_standard(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn); +static void fy_thread_work_join_steal(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn); +static void fy_thread_work_join_steal_2(struct fy_thread_pool *tp, struct fy_thread_work works[2], fy_work_check_fn check_fn); + +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + +/* linux pedal to the metal implementation */ +static inline int futex(FY_ATOMIC(uint32_t) *uaddr, int futex_op, uint32_t val, const struct timespec *timeout, uint32_t *uaddr2, uint32_t val3) +{ + return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3); +} + +static inline int fwait(FY_ATOMIC(uint32_t) *futexp) +{ + long s; + uint32_t one = 1; + + while (!fy_atomic_compare_exchange_strong(futexp, &one, 0)) { + s = futex(futexp, FUTEX_WAIT_PRIVATE, 0, NULL, NULL, 0); + if (s == -1 && errno != EAGAIN) + return -1; + } + return 0; +} + +static inline int fpost(FY_ATOMIC(uint32_t) *futexp) +{ + long s; + uint32_t zero = 0; + + if (fy_atomic_compare_exchange_strong(futexp, &zero, 1)) { + s = futex(futexp, FUTEX_WAKE_PRIVATE, 1, NULL, NULL, 0); + if (s == -1) + return -1; + } + return 0; +} + +static inline void fy_thread_init_sync(struct fy_thread *t) +{ + /* nothing more needed for futexes */ + fy_atomic_store(&t->submit, 0); + fy_atomic_store(&t->done, 0); +} + +static inline struct fy_thread_work *fy_worker_wait_for_work(struct fy_thread *t) +{ + struct fy_thread_work *w; + int rc __FY_DEBUG_UNUSED__; + + while ((w = fy_atomic_load(&t->work)) == NULL) { + rc = fwait(&t->submit); + assert(!rc); + } + return w; +} + +static inline void fy_worker_signal_work_done(struct fy_thread *t, struct fy_thread_work *work) +{ + struct fy_thread_work *exp_work; + bool ok __FY_DEBUG_UNUSED__; + int rc __FY_DEBUG_UNUSED__; + + /* note that the work won't be replaced if it's a shutdown */ + exp_work = work; + ok = fy_atomic_compare_exchange_strong(&t->work, &exp_work, NULL); + assert(ok); + + rc = fpost(&t->done); + assert(!rc); +} + +static inline int fy_thread_submit_work_internal(struct fy_thread *t, struct fy_thread_work *work) +{ + struct fy_thread_work *exp_work; + int rc __FY_DEBUG_UNUSED__; + + /* atomically update the work */ + exp_work = NULL; + if (!fy_atomic_compare_exchange_strong(&t->work, &exp_work, work)) + return -1; + + rc = fpost(&t->submit); + assert(!rc); + + return 0; +} + +static inline int fy_thread_wait_work_internal(struct fy_thread *t) +{ + const struct fy_thread_work *work; + + while ((work = fy_atomic_load(&t->work)) != NULL) + fwait(&t->done); + + fy_atomic_store(&t->done, 0); + + return 0; +} + +void fy_worker_thread_shutdown(struct fy_thread *t) +{ + int rc __FY_DEBUG_UNUSED__; + + fy_atomic_store(&t->work, WORK_SHUTDOWN); + rc = fpost(&t->submit); + assert(!rc); + + rc = pthread_join(t->tid, NULL); + assert(!rc); + + if (!(t->tp->cfg.flags & FYTPCF_STEAL_MODE)) + fy_atomic_store(&t->work, NULL); +} + +#else + +/* portable pthread implementation */ + +static inline void fy_thread_init_sync(struct fy_thread *t) +{ + pthread_mutex_init(&t->lock, NULL); + pthread_cond_init(&t->cond, NULL); + + pthread_mutex_init(&t->wait_lock, NULL); + pthread_cond_init(&t->wait_cond, NULL); +} + +static inline struct fy_thread_work *fy_worker_wait_for_work(struct fy_thread *t) +{ + struct fy_thread_work *work; + + pthread_mutex_lock(&t->lock); + while ((work = fy_atomic_load(&t->work)) == NULL) + pthread_cond_wait(&t->cond, &t->lock); + pthread_mutex_unlock(&t->lock); + + return work; +} + +static inline void fy_worker_signal_work_done(struct fy_thread *t, struct fy_thread_work *work) +{ + struct fy_thread_work *exp_work; + + /* clear the work, so that the user knows we're done */ + pthread_mutex_lock(&t->wait_lock); + + /* note that the work won't be replaced if it's a shutdown */ + exp_work = work; + if (!fy_atomic_compare_exchange_strong(&t->work, &exp_work, NULL)) + assert(exp_work == WORK_SHUTDOWN); + pthread_cond_signal(&t->wait_cond); + pthread_mutex_unlock(&t->wait_lock); +} + +static inline int fy_thread_submit_work_internal(struct fy_thread *t, struct fy_thread_work *work) +{ + struct fy_thread_work *exp_work; + int ret; + + /* atomically update the work */ + + assert(t); + assert(work); + + pthread_mutex_lock(&t->lock); + exp_work = NULL; + if (!fy_atomic_compare_exchange_strong(&t->work, &exp_work, work)) { + assert(exp_work == WORK_SHUTDOWN); + ret = -1; + } else { + pthread_cond_signal(&t->cond); + ret = 0; + } + pthread_mutex_unlock(&t->lock); + + return ret; +} + +static inline int fy_thread_wait_work_internal(struct fy_thread *t) +{ + const struct fy_thread_work *work; + + pthread_mutex_lock(&t->wait_lock); + while ((work = fy_atomic_load(&t->work)) != NULL) + pthread_cond_wait(&t->wait_cond, &t->wait_lock); + pthread_mutex_unlock(&t->wait_lock); + + return 0; +} + +void fy_worker_thread_shutdown(struct fy_thread *t) +{ + pthread_mutex_lock(&t->lock); + fy_atomic_store(&t->work, WORK_SHUTDOWN); + pthread_cond_signal(&t->cond); + pthread_mutex_unlock(&t->lock); + pthread_join(t->tid, NULL); +} + +#endif + +static inline struct fy_thread *fy_thread_reserve_internal(struct fy_thread_pool *tp) +{ + struct fy_thread *t; + unsigned int slot; + FY_ATOMIC(uint64_t) *free; + uint64_t exp, v; + unsigned int i, num_threads_words; + + t = NULL; + + num_threads_words = FY_BIT64_COUNT(tp->num_threads); + for (i = 0, free = tp->freep; i < num_threads_words; i++, free++) { + v = fy_atomic_load(free); + while (v) { + slot = FY_BIT64_LOWEST(v); + assert(v & FY_BIT64(slot)); + exp = v; /* expecting the previous value */ + v &= ~FY_BIT64(slot); /* clear this bit */ + if (fy_atomic_compare_exchange_strong(free, &exp, v)) { + slot += i * 64; + t = tp->threads + slot; + assert(slot == t->id); + return t; + } + v = exp; + } + } + + return NULL; +} + +static inline void fy_thread_unreserve_internal(struct fy_thread *t) +{ + struct fy_thread_pool *tp; + FY_ATOMIC(uint64_t) *free; + + tp = t->tp; + free = tp->freep + (unsigned int)(t->id / 64); + fy_atomic_fetch_or(free, FY_BIT64(t->id & 63)); +} + +static inline bool fy_thread_is_reserved_internal(struct fy_thread *t) +{ + struct fy_thread_pool *tp = t->tp; + FY_ATOMIC(uint64_t) *free; + + free = tp->freep + (unsigned int)(t->id / 64); + return !(fy_atomic_load(free) & FY_BIT64(t->id & 63)); +} + +static inline bool fy_thread_pool_are_all_reserved_internal(struct fy_thread_pool *tp) +{ + FY_ATOMIC(uint64_t) *free; + uint64_t v, m; + unsigned int i, num_threads_words; + + num_threads_words = FY_BIT64_COUNT(tp->num_threads); + for (i = 0, free = tp->freep; i < num_threads_words - 1; i++, free++) { + v = fy_atomic_load(free); + if (v != (uint64_t)-1) + return false; + } + v = fy_atomic_load(free); + m = FY_BIT64(tp->num_threads & 63) - 1; + return (v & m) == m; +} + +static inline bool fy_thread_pool_is_any_reserved_internal(struct fy_thread_pool *tp) +{ + FY_ATOMIC(uint64_t) *free; + uint64_t v, m; + unsigned int i, num_threads_words; + + num_threads_words = FY_BIT64_COUNT(tp->num_threads); + for (i = 0, free = tp->freep; i < num_threads_words - 1; i++, free++) { + v = fy_atomic_load(free); + if (!v) + return true; + } + v = fy_atomic_load(free); + m = FY_BIT64(tp->num_threads & 63) - 1; + return !(v & m); +} + +struct fy_thread *fy_thread_reserve(struct fy_thread_pool *tp) +{ + if (!tp) + return NULL; + + /* only valid for non-work stealing thread pools */ + if (tp->cfg.flags & FYTPCF_STEAL_MODE) + return NULL; + + return fy_thread_reserve_internal(tp); +} + +void fy_thread_unreserve(struct fy_thread *t) +{ + struct fy_thread_pool *tp; + + if (!t) + return; + + tp = t->tp; + assert(tp); + + /* only valid for non-work stealing thread pools */ + if (tp->cfg.flags & FYTPCF_STEAL_MODE) + return; + + fy_thread_unreserve_internal(t); +} + +bool fy_thread_is_reserved(struct fy_thread *t) +{ + if (!t) + return false; + + return fy_thread_is_reserved_internal(t); +} + +bool fy_thread_pool_are_all_reserved(struct fy_thread_pool *tp) +{ + return fy_thread_pool_are_all_reserved_internal(tp); +} + +bool fy_thread_pool_is_any_reserved(struct fy_thread_pool *tp) +{ + return fy_thread_pool_is_any_reserved_internal(tp); +} + +int fy_thread_submit_work(struct fy_thread *t, struct fy_thread_work *work) +{ + if (!t || !work) + return -1; + + if (t->tp->cfg.flags & FYTPCF_STEAL_MODE) + return -1; + + return fy_thread_submit_work_internal(t, work); +} + +int fy_thread_wait_work(struct fy_thread *t) +{ + if (!t) + return -1; + + if (t->tp->cfg.flags & FYTPCF_STEAL_MODE) + return -1; + + return fy_thread_wait_work_internal(t); +} + +void fy_thread_pool_cleanup(struct fy_thread_pool *tp) +{ + struct fy_thread *t; + unsigned int i; + + if (!tp) + return; + + if (tp->threads) { + /* get out of steal mode */ + for (i = 0, t = tp->threads; i < tp->num_threads; i++, t++) { + fy_worker_thread_shutdown(t); + } + + fy_cacheline_free(tp->threads); + } + + memset(tp, 0, sizeof(*tp)); +} + +int fy_thread_pool_setup(struct fy_thread_pool *tp, const struct fy_thread_pool_cfg *cfg) +{ + struct fy_thread *t; + unsigned int i, num_threads, num_threads_words; + size_t size, free_offset, loot_offset, thread_bitmask_size; + void *(*start_routine)(void *); + long scval; + int rc __FY_DEBUG_UNUSED__; + + assert(tp); + + memset(tp, 0, sizeof(*tp)); + + if (!cfg) { + tp->cfg.flags = 0; + tp->cfg.num_threads = 0; + } else + tp->cfg = *cfg; + + if (!tp->cfg.num_threads) { + scval = sysconf(_SC_NPROCESSORS_ONLN); + assert(scval > 0); + num_threads = (unsigned int)scval; + } else + num_threads = tp->cfg.num_threads; + + tp->num_threads = num_threads; + + num_threads_words = FY_BIT64_COUNT(tp->num_threads); + thread_bitmask_size = FY_BIT64_SIZE(tp->num_threads); + + /* the size of the threads, aligned to a cache line */ + size = FY_CACHELINE_SIZE_ALIGN(sizeof(*tp->threads) * tp->num_threads); + + /* the free bitmask array offset */ + free_offset = size; + size = FY_CACHELINE_SIZE_ALIGN(size + thread_bitmask_size); + + /* the loot bitmask array offset, aligned to a 64 byte cacheline */ + loot_offset = size; + size = FY_CACHELINE_SIZE_ALIGN(size + thread_bitmask_size); + + /* allocate everything in one go */ + tp->threads = fy_cacheline_alloc(size); + if (!tp->threads) + goto err_out; + + memset(tp->threads, 0, size); + + rc = pthread_key_create(&tp->key, NULL); + assert(!rc); + + tp->freep = (void *)tp->threads + free_offset; + tp->lootp = (void *)tp->threads + loot_offset; + + /* prime the thread free */ + for (i = 0; i < num_threads_words - 1; i++) + tp->freep[i] = (uint64_t)-1; + tp->freep[i] = FY_BIT64(tp->num_threads & 63) - 1; + + /* the lootp's are zero */ + + for (i = 0, t = tp->threads; i < tp->num_threads; i++, t++) { + + t->tp = tp; + t->id = i; + + fy_thread_init_sync(t); + } + + start_routine = !(tp->cfg.flags & FYTPCF_STEAL_MODE) ? + fy_worker_thread_standard : + fy_worker_thread_steal; + + for (i = 0, t = tp->threads; i < tp->num_threads; i++, t++) { + rc = pthread_create(&t->tid, NULL, start_routine, t); + if (rc) + goto err_out; + } + + return 0; + +err_out: + fy_thread_pool_cleanup(tp); + return -1; +} + +struct fy_thread_pool *fy_thread_pool_create(const struct fy_thread_pool_cfg *cfg) +{ + struct fy_thread_pool *tp; + int rc; + + tp = malloc(sizeof(*tp)); + if (!tp) + return NULL; + + rc = fy_thread_pool_setup(tp, cfg); + if (rc) { + free(tp); + return NULL; + } + + return tp; +} + +void fy_thread_pool_destroy(struct fy_thread_pool *tp) +{ + if (!tp) + return; + + fy_thread_pool_cleanup(tp); + free(tp); +} + +int fy_thread_pool_get_num_threads(struct fy_thread_pool *tp) +{ + if (!tp) + return -1; + + return (int)tp->num_threads; +} + +const struct fy_thread_pool_cfg *fy_thread_pool_get_cfg(struct fy_thread_pool *tp) +{ + if (!tp) + return NULL; + return &tp->cfg; +} + +void fy_thread_work_join(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn) +{ + if (!(tp->cfg.flags & FYTPCF_STEAL_MODE)) + fy_thread_work_join_standard(tp, works, work_count, check_fn); + else if (work_count == 2) + fy_thread_work_join_steal_2(tp, works, check_fn); + else + fy_thread_work_join_steal(tp, works, work_count, check_fn); +} + +void fy_thread_args_join(struct fy_thread_pool *tp, fy_work_exec_fn fn, fy_work_check_fn check_fn, void **args, size_t count) +{ + struct fy_thread_work *works; + size_t i; + + if (!count) + return; + + works = alloca(sizeof(*works) * count); + memset(works, 0, sizeof(*works) * count); + for (i = 0; i < count; i++) { + works[i].fn = fn; + works[i].arg = args ? args[i] : NULL; + } + + fy_thread_work_join(tp, works, count, check_fn); +} + +void fy_thread_arg_array_join(struct fy_thread_pool *tp, fy_work_exec_fn fn, fy_work_check_fn check_fn, void *args, size_t argsize, size_t count) +{ + struct fy_thread_work *works; + size_t i; + + if (!count) + return; + + works = alloca(sizeof(*works) * count); + memset(works, 0, sizeof(*works) * count); + for (i = 0; i < count; i++) { + works[i].fn = fn; + works[i].arg = args; + args += argsize; + } + + fy_thread_work_join(tp, works, count, check_fn); +} + +void fy_thread_arg_join(struct fy_thread_pool *tp, fy_work_exec_fn fn, fy_work_check_fn check_fn, void *arg, size_t count) +{ + struct fy_thread_work *works; + size_t i; + + if (!count) + return; + + works = alloca(sizeof(*works) * count); + memset(works, 0, sizeof(*works) * count); + for (i = 0; i < count; i++) { + works[i].fn = fn; + works[i].arg = arg; + } + + fy_thread_work_join(tp, works, count, check_fn); +} + +/* + * the standard (non-stealing implementation) + */ +static void *fy_worker_thread_standard(void *arg) +{ + struct fy_thread *t = arg; + struct fy_thread_pool *tp; + struct fy_thread_work *work; + + tp = t->tp; + + /* store per thread info */ + pthread_setspecific(tp->key, t); + + while ((work = fy_worker_wait_for_work(t)) != WORK_SHUTDOWN) { + work->fn(work->arg); + fy_worker_signal_work_done(t, work); + } + + return NULL; +} + +static void fy_thread_work_join_standard(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn) +{ + struct fy_thread_work **direct_work, **thread_work, *w; + struct fy_thread **threads, *t; + size_t i, direct_work_count, thread_work_count; + int rc; + + /* just a single (or no) work, or no threads? execute directly */ + if (work_count <= 1 || !tp || !tp->num_threads) { + for (i = 0, w = works; i < work_count; i++, w++) + w->fn(w->arg); + return; + } + + /* allocate the keeper of direct work */ + direct_work = alloca(work_count * sizeof(*direct_work)); + direct_work_count = 0; + + threads = alloca(work_count * sizeof(*threads)); + thread_work = alloca(work_count * sizeof(*thread_work)); + thread_work_count = 0; + + for (i = 0, w = works; i < work_count; i++, w++) { + + t = NULL; + if (!check_fn || check_fn(w->arg)) + t = fy_thread_reserve_internal(tp); + + if (t) { + threads[thread_work_count] = t; + thread_work[thread_work_count++] = w; + } else + direct_work[direct_work_count++] = w; + } + + /* if we don't have any direct_work, steal the last threaded work as direct */ + if (!direct_work_count) { + assert(thread_work_count > 0); + t = threads[thread_work_count - 1]; + w = thread_work[thread_work_count - 1]; + thread_work_count--; + + /* unreserve this */ + fy_thread_unreserve_internal(t); + direct_work[direct_work_count++] = w; + } + + /* submit the threaded work */ + for (i = 0; i < thread_work_count; i++) { + t = threads[i]; + w = thread_work[i]; + rc = fy_thread_submit_work_internal(t, w); + if (rc) { + /* unable to submit? remove work, and move to direct */ + threads[i] = NULL; + thread_work[i] = NULL; + fy_thread_unreserve_internal(t); + direct_work[direct_work_count++] = w; + } + } + + /* now perform the direct work while the threaded work is being performed in parallel */ + for (i = 0; i < direct_work_count; i++) { + w = direct_work[i]; + w->fn(w->arg); + } + + /* finally wait for all threaded work to complete */ + for (i = 0; i < thread_work_count; i++) { + t = threads[i]; + assert(t); + fy_thread_wait_work_internal(t); + fy_thread_unreserve_internal(t); + } +} + +/* + * the stealing implementation + */ +static inline struct fy_work_pool *fy_work_pool_init(struct fy_work_pool *wp, size_t work_count) +{ +#if !(defined(__linux__) && !defined(FY_THREAD_PORTABLE)) + int rc __FY_DEBUG_UNUSED__; +#endif + + if (!wp) + return NULL; + + fy_atomic_store(&wp->work_left, work_count); +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + fy_atomic_store(&wp->done, !work_count); +#elif defined(__APPLE__) + wp->sem = dispatch_semaphore_create(!work_count); + assert(wp->sem != NULL); + (void)rc; +#else + rc = sem_init(&wp->sem, 0, !work_count); + assert(!rc); +#endif + return wp; +} + +static inline void fy_work_pool_cleanup(struct fy_work_pool *wp) +{ +#if !(defined(__linux__) && !defined(FY_THREAD_PORTABLE)) + int rc __FY_DEBUG_UNUSED__; +#endif + + if (!wp) + return; + +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + /* nothing */ +#elif defined(__APPLE__) + /* nothing */ + (void)rc; +#else + rc = sem_destroy(&wp->sem); + assert(!rc); +#endif +} + +static inline bool fy_work_pool_signal(struct fy_work_pool *wp) +{ + size_t prev_work_left; + int rc __FY_DEBUG_UNUSED__; + + if (!wp) + return false; + + prev_work_left = fy_atomic_fetch_sub(&wp->work_left, 1); + if (prev_work_left == 1) { +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + rc = fpost(&wp->done); + assert(!rc); +#elif defined(__APPLE__) + dispatch_semaphore_signal(wp->sem); + (void)rc; +#else + rc = sem_post(&wp->sem); + assert(!rc); +#endif + return true; + } + return false; +} + +static inline void fy_work_pool_wait(struct fy_work_pool *wp) +{ + int rc __FY_DEBUG_UNUSED__; + + if (!wp) + return; + + /* if there's any work left, wait for it */ + while (fy_atomic_load(&wp->work_left) > 0) { +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + rc = fwait(&wp->done); +#elif defined(__APPLE__) + dispatch_semaphore_wait(wp->sem, DISPATCH_TIME_FOREVER); + rc = 0; +#else + rc = sem_wait(&wp->sem); +#endif + assert(!rc || (rc == -1 && errno == EAGAIN)); + } +} + +static inline void fy_worker_thread_steal_execute(struct fy_thread *t, struct fy_thread_work *w) +{ + struct fy_work_pool *wp; + bool signalled __FY_DEBUG_UNUSED__; + + TDBG("%s: T#%u worker executing W:%p\n", __func__, t->id, w); + wp = w->wp; + assert(wp); + w->fn(w->arg); + TDBG("%s: T#%u worker executed W:%p\n", __func__, t->id, w); + + signalled = fy_work_pool_signal(wp); + (void)signalled; + TDBG("%s: T#%u W:%p WP:%p signalled=%s\n", __func__, + t->id, w, wp, signalled ? "true" : "false"); +} + +static inline struct fy_thread_work *fy_worker_thread_steal_work(struct fy_thread_pool *tp, struct fy_thread *t_thief) +{ + struct fy_thread *t; + unsigned int slot; + FY_ATOMIC(uint64_t) *loot; + uint64_t exp, v; + unsigned int i, num_threads_words; + struct fy_thread_work *w, *w_exp; + + /* the threads that have work to steal, have the loot bit set */ + t = NULL; + + num_threads_words = FY_BIT64_COUNT(tp->num_threads); + for (i = 0, loot = tp->lootp; i < num_threads_words; i++, loot++) { + v = fy_atomic_load(loot); + while (v) { + slot = FY_BIT64_LOWEST(v); + assert(v & FY_BIT64(slot)); + exp = v; /* expecting the previous value */ + v &= ~FY_BIT64(slot); /* clear this bit */ + + if (fy_atomic_compare_exchange_strong(loot, &exp, v)) { + slot += i * 64; + t = tp->threads + slot; + if ((w = fy_atomic_load(&t->next_work)) != NULL) { + w_exp = w; + if (fy_atomic_compare_exchange_strong(&t->next_work, &w_exp, NULL)) + return w; + } + } + v = exp; + } + } + + return NULL; +} + +static void *fy_worker_thread_steal(void *arg) +{ + struct fy_thread *t = arg; + struct fy_thread_pool *tp; + struct fy_thread_work *w, *w_exp, *w_stolen, *w_last; + + tp = t->tp; + + /* store per thread info */ + pthread_setspecific(tp->key, t); + + TDBG("%s: T#%u in steal mode\n", __func__, t->id); + + while ((w = fy_worker_wait_for_work(t)) != WORK_SHUTDOWN) { + + assert(fy_thread_is_reserved_internal(t)); + + for (;;) { + fy_worker_thread_steal_execute(t, w); + w_last = w; + w_stolen = fy_worker_thread_steal_work(tp, t); + w = NULL; + if (!w_stolen) + break; + + TDBG("%s: T#%u stole W:%p\n", __func__, t->id, w_stolen); + w_exp = w_last; + if (!fy_atomic_compare_exchange_strong(&t->work, &w_exp, w_stolen)) { + assert(w_exp != WORK_SHUTDOWN); + TDBG("%s: T#%u t->work:%p w:%p w_stolen:%p\n", __func__, t->id, fy_atomic_load(&t->work), w, w_stolen); + FY_IMPOSSIBLE_ABORT(); + } + w = w_stolen; + } + + /* unreserve first */ + fy_thread_unreserve_internal(t); + + w_exp = w_last; + if (!fy_atomic_compare_exchange_strong(&t->work, &w_exp, NULL)) { + assert(w_exp != WORK_SHUTDOWN); + break; + } + } + TDBG("%s: T#%u leaving steal mode\n", __func__, t->id); + + return NULL; +} + +static void fy_thread_work_join_steal(struct fy_thread_pool *tp, struct fy_thread_work *works, size_t work_count, fy_work_check_fn check_fn) +{ + struct fy_work_pool wp_local, *wp; + struct fy_thread_work *dw, *expw; + struct fy_thread *t, *tw; + bool has_loot, resolved_t; + int rc __FY_DEBUG_UNUSED__; + int tid __FY_DEBUG_UNUSED__; + + t = NULL; + resolved_t = false; + tid = -1; + (void)tid; + +#ifdef FY_THREAD_DEBUG + t = pthread_getspecific(tp->key); + tid = t ? (int)t->id : -1; + resolved_t = true; +#else + t = NULL; + tid = -1; + resolved_t = false; +#endif + + dw = NULL; + wp = NULL; + + while (work_count > 0) { + if (!dw) { + dw = works++; + work_count--; + + TDBG("%s: T#%d sdir W:%p\n", __func__, tid, dw); + + continue; + } + + has_loot = false; + if (work_count > 0 && (!check_fn || check_fn(works->arg))) { + while (work_count > 0 && (tw = fy_thread_reserve_internal(tp)) != NULL) { + + assert(!works->wp); + + if (!wp) + wp = fy_work_pool_init(&wp_local, work_count + !!dw); + + works->wp = wp; + expw = NULL; + + rc = fy_thread_submit_work_internal(tw, works); + assert(!rc); + + TDBG("%s: T#%d post W:%p to T#%u\n", __func__, tid, works, tw->id); + + works++; + work_count--; + } + + if (work_count > 0) { + + if (!resolved_t) { + t = pthread_getspecific(tp->key); + tid = t ? (int)t->id : -1; + resolved_t = true; + } + + if (t && fy_atomic_load(&t->next_work) == NULL) { + TDBG("%s: T#%d could not post, available to steal W:%p\n", __func__, tid, works); + assert(works->wp == NULL); + + if (!wp) + wp = fy_work_pool_init(&wp_local, work_count + !!dw); + + works->wp = wp; + + expw = NULL; + if (!fy_atomic_compare_exchange_strong(&t->next_work, &expw, works)) { + TDBG("%s: T#%d could not update next_work: W:%p\n", __func__, tid, works); + abort(); + } + + /* set the has loot bit */ + fy_atomic_fetch_or(tp->lootp + (unsigned int)(t->id / 64), FY_BIT64(t->id & 63)); + + has_loot = true; + } + } + } + + if (dw) { + TDBG("%s: T#%d exec W:%p\n", __func__, tid, dw); + + /* execute the direct work */ + dw->fn(dw->arg); + dw = NULL; + + fy_work_pool_signal(wp); + } + + if (has_loot) { + assert(t); + expw = works; + + /* clear the has loot bit unconditionally */ + fy_atomic_fetch_and(tp->lootp + (unsigned int)(t->id / 64), ~FY_BIT64(t->id & 63)); + + if (!fy_atomic_compare_exchange_strong(&t->next_work, &expw, NULL)) { + TDBG("%s: T#%d had W:%p stolen, good\n", __func__, tid, works); + work_count--; + works++; + } else { + TDBG("%s: T#%d had W:%p not stolen, bad\n", __func__, tid, works); + } + } + } + + /* last out and direct work */ + if (dw) { + TDBG("%s: T#%d executing final direct W:%p\n", __func__, tid, dw); + + dw->fn(dw->arg); + dw = NULL; + + fy_work_pool_signal(wp); + } + + TDBG("%s: T#%d wait WP:%p\n", __func__, tid, wp); + + fy_work_pool_wait(wp); + fy_work_pool_cleanup(wp); + + TDBG("%s: T#%d done WP:%p\n", __func__, tid, wp); +} + +static void fy_thread_work_join_steal_2(struct fy_thread_pool *tp, struct fy_thread_work works[2], fy_work_check_fn check_fn) +{ + struct fy_work_pool wp_local, *wp; + struct fy_thread_work *expw; + struct fy_thread *t, *tw; + bool has_loot, pushed, resolved_t; + int rc __FY_DEBUG_UNUSED__; + int tid __FY_DEBUG_UNUSED__; + + t = NULL; + resolved_t = false; + tid = -1; + + (void)tid; + +#ifdef FY_THREAD_DEBUG + t = pthread_getspecific(tp->key); + tid = t ? (int)t->id : -1; + resolved_t = true; +#else + t = NULL; + tid = -1; + resolved_t = false; +#endif + + wp = NULL; + + pushed = false; + has_loot = false; + if (!check_fn || check_fn(works->arg)) { + if ((tw = fy_thread_reserve_internal(tp)) != NULL) { + + assert(!works[1].wp); + + if (!wp) + wp = fy_work_pool_init(&wp_local, 1); + + works[1].wp = wp; + expw = NULL; + + rc = fy_thread_submit_work_internal(tw, &works[1]); + assert(!rc); + + TDBG("%s: T#%d post W:%p to T#%u\n", __func__, tid, &works[1], tw->id); + + pushed = true; + + } else { + if (!resolved_t) { + t = pthread_getspecific(tp->key); + tid = t ? (int)t->id : -1; + resolved_t = true; + } + + if (t && fy_atomic_load(&t->next_work) == NULL) { + TDBG("%s: T#%d could not post, available to steal W:%p\n", __func__, tid, &works[1]); + assert(works[1].wp == NULL); + + if (!wp) + wp = fy_work_pool_init(&wp_local, 1); + + works[1].wp = wp; + + expw = NULL; + if (!fy_atomic_compare_exchange_strong(&t->next_work, &expw, &works[1])) { + TDBG("%s: T#%d could not update next_work: W:%p\n", __func__, tid, &works[1]); + abort(); + } + + /* set the has loot bit */ + fy_atomic_fetch_or(tp->lootp + (unsigned int)(t->id / 64), FY_BIT64(t->id & 63)); + + has_loot = true; + } + } + } + + TDBG("%s: T#%d exec W:%p (left)\n", __func__, tid, &works[0]); + + /* execute the direct work */ + works[0].fn(works[0].arg); + + if (has_loot) { + assert(t); + expw = &works[1]; + + /* clear the has loot bit unconditionally */ + fy_atomic_fetch_and(tp->lootp + (unsigned int)(t->id / 64), ~FY_BIT64(t->id & 63)); + + if (!fy_atomic_compare_exchange_strong(&t->next_work, &expw, NULL)) { + TDBG("%s: T#%d had W:%p stolen\n", __func__, tid, &works[1]); + } else { + TDBG("%s: T#%d had W:%p not stolen\n", __func__, tid, &works[1]); + + TDBG("%s: T#%d exec W:%p (right)\n", __func__, tid, &works[1]); + /* execute the direct work */ + works[1].fn(works[1].arg); + fy_work_pool_signal(wp); + } + + } else if (!pushed) { + + TDBG("%s: T#%d exec W:%p (right)\n", __func__, tid, &works[1]); + /* execute the direct work */ + works[1].fn(works[1].arg); + fy_work_pool_signal(wp); + } + + TDBG("%s: T#%d wait WP:%p\n", __func__, tid, wp); + + fy_work_pool_wait(wp); + fy_work_pool_cleanup(wp); + + TDBG("%s: T#%d done WP:%p\n", __func__, tid, wp); +} diff --git a/src/thread/fy-thread.h b/src/thread/fy-thread.h new file mode 100644 index 0000000..8b18025 --- /dev/null +++ b/src/thread/fy-thread.h @@ -0,0 +1,77 @@ +/* + * fy-thread.h - Lighting fast thread pool implementation + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_THREAD_H +#define FY_THREAD_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include + +#ifndef __APPLE__ +#include +#else +#include +#endif + +#include "fy-bit64.h" +#include "fy-align.h" +#include "fy-atomics.h" + +#include + +// #define FY_THREAD_DEBUG /* define to enable debugging information to stderr */ +// #define FY_THREAD_PORTABLE /* define to use the portable implementation even on linux */ + +struct fy_work_pool { + FY_ATOMIC(size_t) work_left; +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + FY_ATOMIC(uint32_t) done; +#elif defined(__APPLE__) + dispatch_semaphore_t sem; +#else + sem_t sem; +#endif +}; + +struct fy_thread { + struct fy_thread_pool *tp; + unsigned int id; + pthread_t tid; + FY_ATOMIC(struct fy_thread_work *)work; + FY_ATOMIC(struct fy_thread_work *)next_work; +#if defined(__linux__) && !defined(FY_THREAD_PORTABLE) + FY_ATOMIC(uint32_t) submit; + FY_ATOMIC(uint32_t) done; +#else + pthread_mutex_t lock; + pthread_cond_t cond; + pthread_mutex_t wait_lock; + pthread_cond_t wait_cond; +#endif +}; + +struct fy_thread_pool { + struct fy_thread_pool_cfg cfg; + unsigned int num_threads; + struct fy_thread *threads; + FY_ATOMIC(uint64_t) *freep; + FY_ATOMIC(uint64_t) *lootp; + pthread_key_t key; +}; + +/* those are internal only */ +int fy_thread_pool_setup(struct fy_thread_pool *tp, const struct fy_thread_pool_cfg *cfg); +void fy_thread_pool_cleanup(struct fy_thread_pool *tp); + +#endif diff --git a/src/tool/fy-tool-dump.c b/src/tool/fy-tool-dump.c new file mode 100644 index 0000000..a66a4f4 --- /dev/null +++ b/src/tool/fy-tool-dump.c @@ -0,0 +1,767 @@ +/* + * fy-tool-dump.c - tool utils for dumping + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-tool-util.h" + +void print_escaped(const char *str, size_t length) +{ + const uint8_t *p; + int i, c, w; + + for (p = (const uint8_t *)str; length > 0; p += w, length -= (size_t)w) { + + /* get width from the first octet */ + w = (p[0] & 0x80) == 0x00 ? 1 : + (p[0] & 0xe0) == 0xc0 ? 2 : + (p[0] & 0xf0) == 0xe0 ? 3 : + (p[0] & 0xf8) == 0xf0 ? 4 : 0; + + /* error, clip it */ + if ((size_t)w > length) + goto err_out; + + /* initial value */ + c = p[0] & (0xff >> w); + for (i = 1; i < w; i++) { + if ((p[i] & 0xc0) != 0x80) + goto err_out; + c = (c << 6) | (p[i] & 0x3f); + } + + /* check for validity */ + if ((w == 4 && c < 0x10000) || + (w == 3 && c < 0x800) || + (w == 2 && c < 0x80) || + (c >= 0xd800 && c <= 0xdfff) || c >= 0x110000) + goto err_out; + + switch (c) { + case '\\': + printf("\\\\"); + break; + case '\0': + printf("\\0"); + break; + case '\b': + printf("\\b"); + break; + case '\f': + printf("\\f"); + break; + case '\n': + printf("\\n"); + break; + case '\r': + printf("\\r"); + break; + case '\t': + printf("\\t"); + break; + case '\a': + printf("\\a"); + break; + case '\v': + printf("\\v"); + break; + case '\e': + printf("\\e"); + break; + case 0x85: + printf("\\N"); + break; + case 0xa0: + printf("\\_"); + break; + case 0x2028: + printf("\\L"); + break; + case 0x2029: + printf("\\P"); + break; + default: + if ((c >= 0x01 && c <= 0x1f) || c == 0x7f || /* C0 */ + (c >= 0x80 && c <= 0x9f)) /* C1 */ + printf("\\x%02x", c); + else + printf("%.*s", w, p); + break; + } + } + + return; +err_out: + fprintf(stderr, "escape input error\n"); + abort(); +} + +void dump_token_comments(struct fy_token *fyt, bool colorize, const char *banner) +{ + static const char *placement_txt[] = { + [fycp_top] = "top", + [fycp_right] = "right", + [fycp_bottom] = "bottom", + }; + enum fy_comment_placement placement; + char buf[4096]; + const char *str; + + if (!fyt) + return; + + for (placement = fycp_top; placement < fycp_max; placement++) { + str = fy_token_get_comment(fyt, buf, sizeof(buf), placement); + if (!str) + continue; + fputs("\n", stdout); + if (colorize) + fputs(A_RED, stdout); + printf("\t%s %6s: ", banner, placement_txt[placement]); + print_escaped(str, strlen(str)); + if (colorize) + fputs(A_RESET, stdout); + } +} + +void dump_testsuite_event(struct fy_event *fye, enum dump_testsuite_event_flags dump_flags) +{ + bool colorize, disable_flow_markers, disable_doc_markers, disable_scalar_styles, tsv_format; + const char *anchor = NULL; + const char *tag = NULL; + const char *text = NULL; + const char *alias = NULL; + size_t anchor_len = 0, tag_len = 0, text_len = 0, alias_len = 0; + enum fy_scalar_style style; + const struct fy_mark *sm, *em = NULL; + char separator; + size_t spos, epos; + int sline, eline, scolumn, ecolumn; + + colorize = !!(dump_flags & DTEF_COLORIZE); + disable_flow_markers = !!(dump_flags & DTEF_DISABLE_FLOW_MARKERS); + disable_doc_markers = !!(dump_flags & DTEF_DISABLE_DOC_MARKERS); + disable_scalar_styles = !!(dump_flags & DTEF_DISABLE_SCALAR_STYLES); + tsv_format = !!(dump_flags & DTEF_TSV_FORMAT); + + if (!tsv_format) { + separator = ' '; + spos = epos = (size_t)-1; + sline = eline = -1; + scolumn = ecolumn = -1; + } else { + sm = fy_event_start_mark(fye); + if (sm) { + spos = sm->input_pos; + sline = sm->line + 1; + scolumn = sm->column + 1; + } else { + spos = (size_t)-1; + sline = -1; + scolumn = -1; + } + + + em = fy_event_end_mark(fye); + if (em) { + epos = em->input_pos; + eline = em->line + 1; + ecolumn = em->column + 1; + } else { + epos = (size_t)-1; + eline = -1; + ecolumn = -1; + } + separator = '\t'; + /* no colors for TSV */ + colorize = false; + /* no flow or doc markers for TSV */ + disable_flow_markers = true; + disable_doc_markers = true; + disable_scalar_styles = true; + } + + /* event type */ + switch (fye->type) { + case FYET_NONE: + if (colorize) + fputs(A_BRIGHT_RED, stdout); + printf("???"); + break; + case FYET_STREAM_START: + if (colorize) + fputs(A_CYAN, stdout); + printf("+%s", !tsv_format ? "STR" : "str"); + break; + case FYET_STREAM_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("-%s", !tsv_format ? "STR" : "str"); + break; + case FYET_DOCUMENT_START: + if (colorize) + fputs(A_CYAN, stdout); + printf("+%s", !tsv_format ? "DOC" : "doc"); + break; + case FYET_DOCUMENT_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("-%s", !tsv_format ? "DOC" : "doc"); + break; + case FYET_MAPPING_START: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("+%s", !tsv_format ? "MAP" : "map"); + if (fye->mapping_start.anchor) + anchor = fy_token_get_text(fye->mapping_start.anchor, &anchor_len); + if (fye->mapping_start.tag) + tag = fy_token_get_text(fye->mapping_start.tag, &tag_len); + if (!disable_flow_markers && fy_event_get_node_style(fye) == FYNS_FLOW) + printf("%c{}", separator); + break; + case FYET_MAPPING_END: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("-%s", !tsv_format ? "MAP" : "map"); + break; + case FYET_SEQUENCE_START: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("+%s", !tsv_format ? "SEQ" : "seq"); + if (fye->sequence_start.anchor) + anchor = fy_token_get_text(fye->sequence_start.anchor, &anchor_len); + if (fye->sequence_start.tag) + tag = fy_token_get_text(fye->sequence_start.tag, &tag_len); + if (!disable_flow_markers && fy_event_get_node_style(fye) == FYNS_FLOW) + printf("%c[]", separator); + break; + case FYET_SEQUENCE_END: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("-%s", !tsv_format ? "SEQ" : "seq"); + break; + case FYET_SCALAR: + if (colorize) + fputs(A_WHITE, stdout); + printf("=%s", !tsv_format ? "VAL" : "val"); + if (fye->scalar.anchor) + anchor = fy_token_get_text(fye->scalar.anchor, &anchor_len); + if (fye->scalar.tag) + tag = fy_token_get_text(fye->scalar.tag, &tag_len); + break; + case FYET_ALIAS: + if (colorize) + fputs(A_GREEN, stdout); + printf("=%s", !tsv_format ? "ALI" : "ali"); + break; + default: + abort(); + break; + } + + /* (position) anchor and tag */ + if (!tsv_format) { + if (anchor) { + if (colorize) + fputs(A_GREEN, stdout); + printf("%c&%.*s", separator, (int)anchor_len, anchor); + } + if (tag) { + if (colorize) + fputs(A_GREEN, stdout); + printf("%c<%.*s>", separator, (int)tag_len, tag); + } + } else { + if (!anchor) { + anchor = "-"; + anchor_len = 1; + } + if (!tag) { + tag = "-"; + tag_len = 1; + } + printf("%c%zd%c%d%c%d", separator, (ssize_t)spos, + separator, sline, separator, scolumn); + printf("%c%zd%c%d%c%d", separator, (ssize_t)epos, + separator, eline, separator, ecolumn); + printf("%c%.*s", separator, (int)anchor_len, anchor); + printf("%c%.*s", separator, (int)tag_len, tag); + } + + /* style hint */ + switch (fye->type) { + default: + break; + case FYET_DOCUMENT_START: + if (!fy_document_event_is_implicit(fye) && !disable_doc_markers) + printf("%c---", separator); + break; + case FYET_DOCUMENT_END: + if (!fy_document_event_is_implicit(fye) && !disable_doc_markers) + printf("%c...", separator); + break; + case FYET_MAPPING_START: + if (!tsv_format) + break; + printf("%c%s", separator, fy_event_get_node_style(fye) == FYNS_FLOW ? "{}" : ""); + break; + case FYET_SEQUENCE_START: + if (!tsv_format) + break; + printf("%c%s", separator, fy_event_get_node_style(fye) == FYNS_FLOW ? "[]" : ""); + break; + case FYET_SCALAR: + if (!disable_scalar_styles) + style = fy_token_scalar_style(fye->scalar.value); + else + style = FYSS_DOUBLE_QUOTED; /* double quoted can handle anything */ + switch (style) { + case FYSS_PLAIN: + if (colorize) + fputs(A_WHITE, stdout); + printf("%c:", separator); + break; + case FYSS_SINGLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf("%c'", separator); + break; + case FYSS_DOUBLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf("%c\"", separator); + break; + case FYSS_LITERAL: + if (colorize) + fputs(A_YELLOW, stdout); + printf("%c|", separator); + break; + case FYSS_FOLDED: + if (colorize) + fputs(A_YELLOW, stdout); + printf("%c>", separator); + break; + default: + abort(); + } + break; + case FYET_ALIAS: + if (tsv_format) + printf("%c*", separator); + break; + } + + /* content */ + switch (fye->type) { + default: + break; + case FYET_SCALAR: + if (tsv_format) + printf("%c", separator); + text = fy_token_get_text(fye->scalar.value, &text_len); + if (text && text_len > 0) + print_escaped(text, text_len); + break; + case FYET_ALIAS: + alias = fy_token_get_text(fye->alias.anchor, &alias_len); + printf("%c%s%.*s", separator, !tsv_format ? "*" : "", (int)alias_len, alias); + break; + } + + if (colorize) + fputs(A_RESET, stdout); + fputs("\n", stdout); +} + +void dump_parse_event(struct fy_parser *fyp, struct fy_event *fye, bool colorize) +{ + struct fy_token *fyt_tag = NULL, *fyt_anchor = NULL; + const char *anchor = NULL; + const char *tag = NULL; + const char *value = NULL; + size_t anchor_len = 0, tag_len = 0, len = 0; + enum fy_scalar_style style; + const struct fy_version *vers; + const struct fy_tag *tagp = NULL; + void *iterp; + struct fy_document_state *fyds; + + fyt_anchor = fy_event_get_anchor_token(fye); + if (fyt_anchor) { + anchor = fy_token_get_text(fyt_anchor, &anchor_len); + assert(anchor); + } + + fyt_tag = fy_event_get_tag_token(fye); + if (fyt_tag) { + tag = fy_token_get_text(fyt_tag, &tag_len); + assert(tag); + tagp = fy_tag_token_tag(fyt_tag); + assert(tagp); + } + + switch (fye->type) { + case FYET_NONE: + if (colorize) + fputs(A_BRIGHT_RED, stdout); + printf("???"); + break; + case FYET_STREAM_START: + if (colorize) + fputs(A_CYAN, stdout); + printf("STREAM_START"); + dump_token_comments(fye->stream_start.stream_start, colorize, ""); + break; + case FYET_STREAM_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("STREAM_END"); + dump_token_comments(fye->stream_end.stream_end, colorize, ""); + break; + case FYET_DOCUMENT_START: + if (colorize) + fputs(A_CYAN, stdout); + + printf("DOCUMENT_START implicit=%s", + fye->document_start.implicit ? "true" : "false"); + + fyds = fye->document_start.document_state; + assert(fyds); + vers = fy_document_state_version(fyds); + assert(vers); + printf("( V=%d.%d VE=%s TE=%s", vers->major, vers->minor, + fy_document_state_version_explicit(fyds) ? "true" : "false", + fy_document_state_tags_explicit(fyds) ? "true" : "false"); + iterp = NULL; + if ((tagp = fy_document_state_tag_directive_iterate(fyds, &iterp)) != NULL) { + printf(" TDs: ["); + do { + printf(" \"%s\",\"%s\"", tagp->handle, tagp->prefix); + } while ((tagp = fy_document_state_tag_directive_iterate(fyds, &iterp)) != NULL); + printf(" ]"); + } + printf(" )"); + dump_token_comments(fye->document_start.document_start, colorize, ""); + break; + + case FYET_DOCUMENT_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("DOCUMENT_END implicit=%s", + fye->document_end.implicit ? "true" : "false"); + dump_token_comments(fye->document_end.document_end, colorize, ""); + break; + case FYET_MAPPING_START: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("MAPPING_START"); + if (anchor) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" &%.*s", (int)anchor_len, anchor); + } + if (tag) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" <%.*s> (\"%s\",\"%s\")", + (int)tag_len, tag, + tagp->handle, tagp->prefix); + } + dump_token_comments(fye->mapping_start.mapping_start, colorize, ""); + break; + case FYET_MAPPING_END: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("MAPPING_END"); + dump_token_comments(fye->mapping_end.mapping_end, colorize, ""); + break; + case FYET_SEQUENCE_START: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("SEQUENCE_START"); + if (anchor) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" &%.*s", (int)anchor_len, anchor); + } + if (tag) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" <%.*s> (\"%s\",\"%s\")", + (int)tag_len, tag, + tagp->handle, tagp->prefix); + } + dump_token_comments(fye->sequence_start.sequence_start, colorize, ""); + break; + case FYET_SEQUENCE_END: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("SEQUENCE_END"); + dump_token_comments(fye->sequence_end.sequence_end, colorize, ""); + break; + case FYET_SCALAR: + if (colorize) + fputs(A_WHITE, stdout); + printf("SCALAR"); + if (anchor) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" &%.*s", (int)anchor_len, anchor); + } + if (tag) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" <%.*s> (\"%s\",\"%s\")", + (int)tag_len, tag, + tagp->handle, tagp->prefix); + } + + style = fy_token_scalar_style(fye->scalar.value); + switch (style) { + case FYSS_PLAIN: + if (colorize) + fputs(A_WHITE, stdout); + printf(" "); + break; + case FYSS_SINGLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" '"); + break; + case FYSS_DOUBLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" \""); + break; + case FYSS_LITERAL: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" |"); + break; + case FYSS_FOLDED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" >"); + break; + default: + abort(); + } + value = fy_token_get_text(fye->scalar.value, &len); + if (value && len > 0) + print_escaped(value, len); + dump_token_comments(fye->scalar.value, colorize, ""); + break; + case FYET_ALIAS: + anchor = fy_token_get_text(fye->alias.anchor, &anchor_len); + if (colorize) + fputs(A_GREEN, stdout); + printf("ALIAS *%.*s", (int)anchor_len, anchor); + dump_token_comments(fye->alias.anchor, colorize, ""); + break; + default: + /* ignored */ + break; + } + if (colorize) + fputs(A_RESET, stdout); + fputs("\n", stdout); +} + +void dump_scan_token(struct fy_parser *fyp, struct fy_token *fyt, bool colorize) +{ + const char *anchor = NULL, *value = NULL; + size_t anchor_len = 0, len = 0; + enum fy_scalar_style style; + const struct fy_version *vers; + const struct fy_tag *tag; + + switch (fy_token_get_type(fyt)) { + case FYTT_NONE: + if (colorize) + fputs(A_BRIGHT_RED, stdout); + printf("NONE"); + break; + case FYTT_STREAM_START: + if (colorize) + fputs(A_CYAN, stdout); + printf("STREAM_START"); + break; + case FYTT_STREAM_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("STREAM_END"); + break; + case FYTT_VERSION_DIRECTIVE: + if (colorize) + fputs(A_CYAN, stdout); + vers = fy_version_directive_token_version(fyt); + assert(vers); + printf("VERSION_DIRECTIVE major=%d minor=%d", vers->major, vers->minor); + break; + case FYTT_TAG_DIRECTIVE: + if (colorize) + fputs(A_CYAN, stdout); + tag = fy_tag_directive_token_tag(fyt); + assert(tag); + printf("TAG_DIRECTIVE handle=\"%s\" prefix=\"%s\"", tag->handle, tag->prefix); + break; + case FYTT_DOCUMENT_START: + if (colorize) + fputs(A_CYAN, stdout); + printf("DOCUMENT_START"); + break; + case FYTT_DOCUMENT_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("DOCUMENT_END"); + break; + case FYTT_BLOCK_SEQUENCE_START: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("BLOCK_SEQUENCE_START"); + break; + case FYTT_BLOCK_MAPPING_START: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("BLOCK_MAPPING_START"); + break; + case FYTT_BLOCK_END: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("BLOCK_END"); + break; + case FYTT_FLOW_SEQUENCE_START: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("FLOW_SEQUENCE_START"); + break; + case FYTT_FLOW_SEQUENCE_END: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("FLOW_SEQUENCE_END"); + break; + case FYTT_FLOW_MAPPING_START: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("FLOW_MAPPING_START"); + break; + case FYTT_FLOW_MAPPING_END: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("FLOW_MAPPING_END"); + break; + case FYTT_BLOCK_ENTRY: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("BLOCK_ENTRY"); + break; + case FYTT_FLOW_ENTRY: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("BLOCK_ENTRY"); + break; + case FYTT_KEY: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("KEY"); + break; + case FYTT_VALUE: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("KEY"); + break; + case FYTT_ALIAS: + anchor = fy_token_get_text(fyt, &anchor_len); + assert(anchor); + if (colorize) + fputs(A_GREEN, stdout); + printf("ALIAS *%.*s", (int)anchor_len, anchor); + break; + case FYTT_ANCHOR: + anchor = fy_token_get_text(fyt, &anchor_len); + assert(anchor); + if (colorize) + fputs(A_GREEN, stdout); + printf("ANCHOR &%.*s", (int)anchor_len, anchor); + break; + case FYTT_TAG: + tag = fy_tag_token_tag(fyt); + assert(tag); + if (colorize) + fputs(A_GREEN, stdout); + /* prefix is a suffix for tag */ + printf("TAG handle=\"%s\" suffix=\"%s\"", tag->handle, tag->prefix); + break; + case FYTT_SCALAR: + if (colorize) + fputs(A_WHITE, stdout); + + printf("SCALAR "); + value = fy_token_get_text(fyt, &len); + assert(value); + style = fy_token_scalar_style(fyt); + switch (style) { + case FYSS_PLAIN: + if (colorize) + fputs(A_WHITE, stdout); + printf(" "); + break; + case FYSS_SINGLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" '"); + break; + case FYSS_DOUBLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" \""); + break; + case FYSS_LITERAL: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" |"); + break; + case FYSS_FOLDED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" >"); + break; + default: + abort(); + } + printf("%.*s", (int)len, value); + break; + default: + /* not handled; should not be produced by scan */ + break; + } + if (colorize) + fputs(A_RESET, stdout); + fputs("\n", stdout); +} diff --git a/src/tool/fy-tool-util.c b/src/tool/fy-tool-util.c new file mode 100644 index 0000000..a3eac5d --- /dev/null +++ b/src/tool/fy-tool-util.c @@ -0,0 +1,251 @@ +/* + * fy-tool-util.c - tool utils + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-tool-util.h" + +uintmax_t load_le(const void *ptr, size_t width, bool is_signed) +{ + uintmax_t v; + const uint8_t *p; + size_t off; + + assert(width <= sizeof(uintmax_t)); + + switch (width) { + case sizeof(uint8_t): + v = (uintmax_t)*(uint8_t *)ptr; + break; + case sizeof(uint16_t): + v = (uintmax_t)*(uint16_t *)ptr; + break; + case sizeof(uint32_t): + v = (uintmax_t)*(uint32_t *)ptr; + break; + case sizeof(uint64_t): + v = (uintmax_t)*(uint64_t *)ptr; + break; + default: + for (v = 0, p = ptr, off = 0; off < width; off++) + v |= (uintmax_t)p[off] << off; + break; + } + + /* sign extension? */ + if (is_signed && width < sizeof(uintmax_t) && (v & ((uintmax_t)1 << (width * 8 - 1)))) + v |= (uintmax_t)-1 << (width * 8); + + return v; +} + +void store_le(void *ptr, size_t width, uintmax_t v) +{ + uint8_t *p; + size_t off; + + switch (width) { + case sizeof(uint8_t): + *(uint8_t *)ptr = (uint8_t)v; + break; + case sizeof(uint16_t): + *(uint16_t *)ptr = (uint16_t)v; + break; + case sizeof(uint32_t): + *(uint32_t *)ptr = (uint32_t)v; + break; + case sizeof(uint64_t): + *(uint64_t *)ptr = (uint64_t)v; + break; + default: + for (p = ptr, off = 0; off < width; off++) + p[off] = (uint8_t)(v >> (8 * off)); + break; + } +} + +uintmax_t load_bitfield_le(const void *ptr, size_t bit_offset, size_t bit_width, bool is_signed) +{ + const uint8_t *p; + size_t off, width, space, use; + uint8_t bmask; + uintmax_t v; + + v = 0; + + width = bit_width; + p = ptr + bit_offset / 8; + off = bit_offset & 7; + if (off) { + space = 8 - off; + use = width > space ? space : width; + + bmask = (((uint8_t)1 << use) - 1) << off; + width -= use; + + v = (*p++ & bmask) >> off; + off = use; + + // fprintf(stderr, "%s: 0. [%02x] use=%zu v=%jx\n", __func__, p[-1] & 0xff, use, v); + } + while (width >= 8) { + v |= (uintmax_t)*p++ << off; + width -= 8; + off += 8; + // fprintf(stderr, "%s: 1. [%02x] v=%jx\n", __func__, p[-1] & 0xff, v); + } + if (width) { + v |= (uintmax_t)(*p & ((1 << width) - 1)) << off; + // fprintf(stderr, "%s: 2. [%02x] off=%zu v=%jx\n", __func__, p[0], off, v); + } + + /* sign extension? */ + if (bit_width < sizeof(uintmax_t) * 8) { + if (is_signed) { + if (v & ((uintmax_t)1 << (bit_width - 1))) + v |= (uintmax_t)-1 << bit_width; + } else { + v &= ~((uintmax_t)-1 << bit_width); + } + } + + return v; +} + +void store_bitfield_le(void *ptr, size_t bit_offset, size_t bit_width, uintmax_t v) +{ + uint8_t *p; + size_t off, width, space, use; + uint8_t bmask; + + width = bit_width; + p = ptr + bit_offset / 8; + off = bit_offset & 7; + if (off) { + space = 8 - off; + use = width > space ? space : width; + + bmask = (((uint8_t)1 << use) - 1) << off; + + *p = (*p & ~bmask) | ((uint8_t)(v << off) & bmask); + p++; + + // fprintf(stderr, "%s: 0. [%02x] bmask=%02x off=%zu %02x v=%jx\n", __func__, p[-1] & 0xff, bmask, off, (uint8_t)(v << off) & 0xff, v); + + v >>= use; + width -= use; + } + while (width >= 8) { + *p++ = (uint8_t)v; + + // fprintf(stderr, "%s: 1. [%02x] v=%jx\n", __func__, p[-1] & 0xff, v); + + v >>= 8; + width -= 8; + } + if (width) { + bmask = (1 << width) - 1; + *p = (*p & ~bmask) | ((uint8_t)v & bmask); + + // fprintf(stderr, "%s: 1. [%02x] v=%jx\n", __func__, p[0] & 0xff, v); + } +} + +#define STRTOXF_IS_UNSIGNED BIT(0) +#define STRTOXF_IS_SIGNED BIT(1) +#define STRTOXF_SKIP_UNDERSCORE BIT(2) +#define STRTOXF_ALLOW_BASE2 BIT(3) +#define STRTOXF_ALLOW_BASE8 BIT(4) +#define STRTOXF_ALLOW_BASE16 BIT(5) +#define STRTOXF_SINGLE_ZERO BIT(6) + +#define STRTOXF_YAML (STRTOXF_ALLOW_BASE8 | STRTOXF_ALLOW_BASE16) +#define STRTOXF_YAML_1_1 (STRTOXF_ALLOW_BASE2 | STRTOXF_ALLOW_BASE8 | \ + STRTOXF_ALLOW_BASE16 | STRTOXF_SKIP_UNDERSCORE) +#define STRTOXF_JSON (STRTOXF_SINGLE_ZERO) + +#define STRTOX_DECLARE(_type, _typename, _flags) \ +int str_to_ ## _typename (const char *str, _type *res) \ +{ \ + int base, dv; \ + bool negative; \ + char c; \ + _type v; \ + \ + negative = false; \ + if (*str == '+' || *str == '-') { \ + negative = *str == '-'; \ + str++; \ + if (!((_flags) & STRTOXF_IS_SIGNED) && negative) \ + return -EINVAL; \ + } \ + base = 10; \ + if (*str == '0') { \ + if (((_flags) & STRTOXF_ALLOW_BASE16) && str[1] == 'x') { \ + str += 2; \ + base = 16; \ + } else if (((_flags) & STRTOXF_ALLOW_BASE8) && str[1] == 'o') { \ + str += 2; \ + base = 8; \ + } else if (((_flags) & STRTOXF_ALLOW_BASE2) && str[1] == 'b') { \ + str += 2; \ + base = 2; \ + } else if (((_flags) & STRTOXF_SINGLE_ZERO) && str[1] == '\0') \ + return -EINVAL; \ + } else if (*str == '\0') \ + return -EINVAL; /* empty number without digits */ \ + v = 0; \ + while ((c = *str++) != '\0') { \ + if (((_flags) & STRTOXF_SKIP_UNDERSCORE) && c == '_') \ + continue; \ + dv = c >= '0' && c <= '9' ? (c - '0') : \ + c >= 'a' && c <= 'z' ? (10 + c - 'a') : \ + c >= 'A' && c <= 'Z' ? (10 + c - 'A') : -1; \ + if (dv < 0 || dv >= (int)base) \ + return -EINVAL; \ + if (MUL_OVERFLOW(v, (_type)base, &v) || \ + ADD_OVERFLOW(v, (_type)(!negative ? dv : -dv), &v)) \ + return -ERANGE; \ + } \ + *res = v; \ + return 0; \ +} \ +struct __useless_struct_to_allow_semicolon + +typedef int (*strtox_intmax_func)(const char *str, intmax_t *res); +typedef int (*strtox_uintmax_func)(const char *str, uintmax_t *res); + +STRTOX_DECLARE(intmax_t, intmax, STRTOXF_IS_SIGNED | STRTOXF_YAML); +STRTOX_DECLARE(uintmax_t, uintmax, STRTOXF_IS_UNSIGNED | STRTOXF_YAML); + +STRTOX_DECLARE(intmax_t, intmax_1_1, STRTOXF_IS_SIGNED | STRTOXF_YAML_1_1); +STRTOX_DECLARE(uintmax_t, uintmax_1_1, STRTOXF_IS_UNSIGNED | STRTOXF_YAML_1_1); + +STRTOX_DECLARE(intmax_t, intmax_json, STRTOXF_IS_SIGNED | STRTOXF_JSON); +STRTOX_DECLARE(uintmax_t, uintmax_json, STRTOXF_IS_UNSIGNED | STRTOXF_JSON); diff --git a/src/tool/fy-tool-util.h b/src/tool/fy-tool-util.h new file mode 100644 index 0000000..ca34287 --- /dev/null +++ b/src/tool/fy-tool-util.h @@ -0,0 +1,255 @@ +/* + * fy-tool-utils.h - internal utilities header file + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_TOOL_UTILS_H +#define FY_TOOL_UTILS_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#endif + +#ifndef BIT +#define BIT(x) (1U << (x)) +#endif + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) ((sizeof(x)/sizeof((x)[0]))) +#endif + +#if defined(NDEBUG) && (defined(__GNUC__) && __GNUC__ >= 4) +#define ALWAYS_INLINE __attribute__((always_inline)) +#else +#define ALWAYS_INLINE /* nothing */ +#endif + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define UNUSED __attribute__((unused)) +#else +#define UNUSED /* nothing */ +#endif + +#if defined(NDEBUG) && defined(__GNUC__) && __GNUC__ >= 4 +#define DEBUG_UNUSED __attribute__((unused)) +#else +#define DEBUG_UNUSED /* nothing */ +#endif + +/* if expression is zero, then the build will break */ +#define COMPILE_ERROR_ON_ZERO(_e) ((void)(sizeof(char[1 - 2*!!(_e)]))) + +/* true, if types are the same, false otherwise (depends on builtin_types_compatible_p) */ +#if defined(__has_builtin) && __has_builtin(__builtin_types_compatible_p) +#define SAME_TYPE(_a, _b) __builtin_types_compatible_p(__typeof__(_a), __typeof__(_b)) +#else +#define SAME_TYPE(_a, _b) true +#endif + +/* compile error if types are not the same */ +#define CHECK_SAME_TYPE(_a, _b) \ + COMPILE_ERROR_ON_ZERO(!SAME_TYPE(_a, _b)) + +/* type safe add overflow */ +#if defined(__has_builtin) && __has_builtin(__builtin_add_overflow) +#define ADD_OVERFLOW __builtin_add_overflow +#else +#define ADD_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + CHECK_SAME_TYPE(__a, __b); \ + \ + __res = __a + __b; \ + /* overflow when signs of a, b same, but results different */ \ + __overflow = ((__a ^ __result) & (__b & __result)) < 0; \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +/* type safe sub overflow */ +#if defined(__has_builtin) && __has_builtin(__builtin_sub_overflow) +#define SUB_OVERFLOW __builtin_sub_overflow +#else +#define SUB_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + CHECK_SAME_TYPE(__a, __b); \ + \ + __res = __a - __b; \ + /* overflow when signs of a, b differ, but results different from minuend */ \ + __overflow = ((__a ^ __b) & (__a & __result)) < 0; \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +/* type safe multiply overflow */ +#if defined(__has_builtin) && __has_builtin(__builtin_mul_overflow) +#define MUL_OVERFLOW __builtin_mul_overflow +#else +#define MUL_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + CHECK_SAME_TYPE(__a, __b); \ + \ + if (!__a || !__b) { \ + __overflow = false; \ + __res = 0; \ + } else { \ + __res = __a * __b; \ + /* overflow when division of the result differs */ \ + __overflow = (__res / __a) != __b; \ + } \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +/* ANSI colors and escapes */ +#define A_RESET "\x1b[0m" +#define A_BLACK "\x1b[30m" +#define A_RED "\x1b[31m" +#define A_GREEN "\x1b[32m" +#define A_YELLOW "\x1b[33m" +#define A_BLUE "\x1b[34m" +#define A_MAGENTA "\x1b[35m" +#define A_CYAN "\x1b[36m" +#define A_LIGHT_GRAY "\x1b[37m" /* dark white is gray */ +#define A_GRAY "\x1b[1;30m" +#define A_BRIGHT_RED "\x1b[1;31m" +#define A_BRIGHT_GREEN "\x1b[1;32m" +#define A_BRIGHT_YELLOW "\x1b[1;33m" +#define A_BRIGHT_BLUE "\x1b[1;34m" +#define A_BRIGHT_MAGENTA "\x1b[1;35m" +#define A_BRIGHT_CYAN "\x1b[1;36m" +#define A_WHITE "\x1b[1;37m" + +static inline bool +memiszero(const void *ptr, size_t size) +{ + const uint8_t *p; + size_t i; + + for (i = 0, p = ptr; i < size; i++) { + if (p[i]) + return false; + } + return true; +} + +/* in fy-tool-dump.c */ +void print_escaped(const char *str, size_t length); + +enum dump_testsuite_event_flags { + DTEF_COLORIZE = FY_BIT(0), + DTEF_DISABLE_FLOW_MARKERS = FY_BIT(1), + DTEF_DISABLE_DOC_MARKERS = FY_BIT(2), + DTEF_DISABLE_SCALAR_STYLES = FY_BIT(3), + DTEF_TSV_FORMAT = FY_BIT(4), +}; + +void dump_token_comments(struct fy_token *fyt, bool colorize, const char *banner); +void dump_testsuite_event(struct fy_event *fye, enum dump_testsuite_event_flags dump_flags); +void dump_parse_event(struct fy_parser *fyp, struct fy_event *fye, bool colorize); +void dump_scan_token(struct fy_parser *fyp, struct fy_token *fyt, bool colorize); + +/* */ + +static inline int parse_match_value(const char *text0, const char **check_vp) +{ + const char *check; + int i; + + for (i = 0; (check = *check_vp++) != NULL; i++) { + if (!strcmp(check, text0)) + return i; + } + return -1; +} + +union integer_scalar { + intmax_t sval; + uintmax_t uval; +}; + +union float_scalar { + float f; + double d; + long double ld; +}; + +uintmax_t load_le(const void *ptr, size_t width, bool is_signed); +void store_le(void *ptr, size_t width, uintmax_t v); +uintmax_t load_bitfield_le(const void *ptr, size_t bit_offset, size_t bit_width, bool is_signed); +void store_bitfield_le(void *ptr, size_t bit_offset, size_t bit_width, uintmax_t v); + +static inline intmax_t signed_integer_max_from_bit_width(int bit_width) +{ + assert((size_t)bit_width <= sizeof(intmax_t) * 8); + return INTMAX_MAX >> (sizeof(intmax_t) * 8 - bit_width); +} + +static inline intmax_t signed_integer_min_from_bit_width(int bit_width) +{ + assert((size_t)bit_width <= sizeof(intmax_t) * 8); + return INTMAX_MIN >> (sizeof(intmax_t) * 8 - bit_width); +} + +static inline uintmax_t unsigned_integer_max_from_bit_width(int bit_width) +{ + assert((size_t)bit_width <= sizeof(uintmax_t) * 8); + return UINTMAX_MAX >> (sizeof(uintmax_t) * 8 - bit_width); +} + +static inline intmax_t +signed_integer_min_from_size(size_t size) +{ + return signed_integer_min_from_bit_width(size * 8); +} + +static inline intmax_t +signed_integer_max_from_size(size_t size) +{ + return signed_integer_max_from_bit_width(size * 8); +} + +static inline uintmax_t +unsigned_integer_max_from_size(size_t size) +{ + return unsigned_integer_max_from_bit_width(size * 8); +} + +static inline bool str_null_eq(const char *s1, const char *s2) +{ + if (s1 == s2) + return true; + if (!s1 || !s2) + return false; + return !strcmp(s1, s2); +} + +#endif diff --git a/src/tool/fy-tool.c b/src/tool/fy-tool.c new file mode 100644 index 0000000..3529e4f --- /dev/null +++ b/src/tool/fy-tool.c @@ -0,0 +1,2192 @@ +/* + * fy-tool.c - libfyaml YAML manipulation/dumping utility + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-valgrind.h" +#include "fy-tool-util.h" + +#define QUIET_DEFAULT false +#define INCLUDE_DEFAULT "" +#define DEBUG_LEVEL_DEFAULT 3 +#define COLOR_DEFAULT "auto" +#define INDENT_DEFAULT 2 +#define WIDTH_DEFAULT 80 +#define RESOLVE_DEFAULT false +#define SORT_DEFAULT false +#define COMMENT_DEFAULT false +#define VISIBLE_DEFAULT false +#define MODE_DEFAULT "original" +#define TO_DEFAULT "/" +#define FROM_DEFAULT "/" +#define TRIM_DEFAULT "/" +#define FOLLOW_DEFAULT false +#define STRIP_LABELS_DEFAULT false +#define STRIP_TAGS_DEFAULT false +#define STRIP_DOC_DEFAULT false +#define STREAMING_DEFAULT true +#define RECREATING_DEFAULT false +#define JSON_DEFAULT "auto" +#define DISABLE_ACCEL_DEFAULT false +#define DISABLE_BUFFERING_DEFAULT false +#define DISABLE_DEPTH_LIMIT_DEFAULT false +#define SLOPPY_FLOW_INDENTATION_DEFAULT false +#define PREFER_RECURSIVE_DEFAULT false +#define YPATH_ALIASES_DEFAULT false +#define DISABLE_FLOW_MARKERS_DEFAULT false +#define DISABLE_DOC_MARKERS_DEFAULT false +#define DISABLE_SCALAR_STYLES_DEFAULT false +#define DUMP_PATH_DEFAULT false +#define DOCUMENT_EVENT_STREAM_DEFAULT false +#define COLLECT_ERRORS_DEFAULT false +#define ALLOW_DUPLICATE_KEYS_DEFAULT false +#define STRIP_EMPTY_KV_DEFAULT false +#define TSV_FORMAT_DEFAULT false +#define NO_ENDING_NEWLINE_DEFAULT false + +#define OPT_DUMP 1000 +#define OPT_TESTSUITE 1001 +#define OPT_FILTER 1002 +#define OPT_JOIN 1003 +#define OPT_TOOL 1004 +#define OPT_YPATH 1005 +#define OPT_SCAN_DUMP 1006 +#define OPT_PARSE_DUMP 1007 +#define OPT_YAML_VERSION_DUMP 1008 +#define OPT_COMPOSE 1009 +#define OPT_B3SUM 1010 +#define OPT_REFLECT 1011 + +#define OPT_STRIP_LABELS 2000 +#define OPT_STRIP_TAGS 2001 +#define OPT_STRIP_DOC 2002 +#define OPT_STREAMING 2003 +#define OPT_RECREATING 2004 +#define OPT_DISABLE_ACCEL 2005 +#define OPT_DISABLE_BUFFERING 2006 +#define OPT_DISABLE_DEPTH_LIMIT 2007 +#define OPT_SLOPPY_FLOW_INDENTATION 2008 +#define OPT_PREFER_RECURSIVE 2009 +#define OPT_DUMP_PATHEXPR 2010 +#define OPT_NOEXEC 2011 +#define OPT_NULL_OUTPUT 2012 +#define OPT_YPATH_ALIASES 2013 +#define OPT_DISABLE_FLOW_MARKERS 2014 +#define OPT_DUMP_PATH 2015 +#define OPT_DOCUMENT_EVENT_STREAM 2016 +#define OPT_COLLECT_ERRORS 2017 +#define OPT_ALLOW_DUPLICATE_KEYS 2018 +#define OPT_STRIP_EMPTY_KV 2019 +#define OPT_DISABLE_MMAP 2020 +#define OPT_TSV_FORMAT 2021 +#define OPT_DISABLE_DOC_MARKERS 2022 +#define OPT_DISABLE_SCALAR_STYLES 2023 +#define OPT_NO_STREAMING 2024 +#define OPT_NO_ENDING_NEWLINE 2025 + +#define OPT_DISABLE_DIAG 3000 +#define OPT_ENABLE_DIAG 3001 +#define OPT_SHOW_DIAG 3002 +#define OPT_HIDE_DIAG 3003 + +#define OPT_YAML_1_1 4000 +#define OPT_YAML_1_2 4001 +#define OPT_YAML_1_3 4002 + +/* b3sum options */ +#define OPT_CHECK 5000 +#define OPT_DERIVE_KEY 5001 +#define OPT_NO_NAMES 5002 +#define OPT_RAW 5003 +#define OPT_KEYED 5005 +#define OPT_LENGTH 5006 +#define OPT_LIST_BACKENDS 5007 +#define OPT_BACKEND 5008 +#define OPT_NUM_THREADS 5009 +#define OPT_FILE_BUFFER 5010 +#define OPT_MMAP_MIN_CHUNK 5011 +#define OPT_MMAP_MAX_CHUNK 5012 + +static struct option lopts[] = { + {"include", required_argument, 0, 'I' }, + {"debug-level", required_argument, 0, 'd' }, + {"indent", required_argument, 0, 'i' }, + {"width", required_argument, 0, 'w' }, + {"resolve", no_argument, 0, 'r' }, + {"sort", no_argument, 0, 's' }, + {"comment", no_argument, 0, 'c' }, + {"color", required_argument, 0, 'C' }, + {"visible", no_argument, 0, 'V' }, + {"mode", required_argument, 0, 'm' }, + {"json", required_argument, 0, 'j' }, + {"file", required_argument, 0, 'f' }, + {"trim", required_argument, 0, 't' }, + {"follow", no_argument, 0, 'l' }, + {"dump", no_argument, 0, OPT_DUMP }, + {"testsuite", no_argument, 0, OPT_TESTSUITE }, + {"filter", no_argument, 0, OPT_FILTER }, + {"join", no_argument, 0, OPT_JOIN }, + {"ypath", no_argument, 0, OPT_YPATH }, + {"scan-dump", no_argument, 0, OPT_SCAN_DUMP }, + {"parse-dump", no_argument, 0, OPT_PARSE_DUMP }, + {"compose", no_argument, 0, OPT_COMPOSE }, + {"dump-path", no_argument, 0, OPT_DUMP_PATH }, + {"yaml-version-dump", no_argument, 0, OPT_YAML_VERSION_DUMP }, + {"b3sum", no_argument, 0, OPT_B3SUM }, + {"strip-labels", no_argument, 0, OPT_STRIP_LABELS }, + {"strip-tags", no_argument, 0, OPT_STRIP_TAGS }, + {"strip-doc", no_argument, 0, OPT_STRIP_DOC }, + {"streaming", no_argument, 0, OPT_STREAMING }, + {"no-streaming", no_argument, 0, OPT_NO_STREAMING }, + {"recreating", no_argument, 0, OPT_RECREATING }, + {"disable-accel", no_argument, 0, OPT_DISABLE_ACCEL }, + {"disable-buffering", no_argument, 0, OPT_DISABLE_BUFFERING }, + {"disable-depth-limit", no_argument, 0, OPT_DISABLE_DEPTH_LIMIT }, + {"disable-mmap", no_argument, 0, OPT_DISABLE_MMAP }, + {"disable-diag", required_argument, 0, OPT_DISABLE_DIAG }, + {"enable-diag", required_argument, 0, OPT_ENABLE_DIAG }, + {"show-diag", required_argument, 0, OPT_SHOW_DIAG }, + {"hide-diag", required_argument, 0, OPT_HIDE_DIAG }, + {"yaml-1.1", no_argument, 0, OPT_YAML_1_1 }, + {"yaml-1.2", no_argument, 0, OPT_YAML_1_2 }, + {"yaml-1.3", no_argument, 0, OPT_YAML_1_3 }, + {"sloppy-flow-indentation", no_argument, 0, OPT_SLOPPY_FLOW_INDENTATION }, + {"prefer-recursive", no_argument, 0, OPT_PREFER_RECURSIVE }, + {"ypath-aliases", no_argument, 0, OPT_YPATH_ALIASES }, + {"disable-flow-markers",no_argument, 0, OPT_DISABLE_FLOW_MARKERS }, + {"disable-doc-markers", no_argument, 0, OPT_DISABLE_DOC_MARKERS }, + {"disable-scalar-styles",no_argument, 0, OPT_DISABLE_SCALAR_STYLES }, + {"dump-pathexpr", no_argument, 0, OPT_DUMP_PATHEXPR }, + {"document-event-stream",no_argument, 0, OPT_DOCUMENT_EVENT_STREAM }, + {"noexec", no_argument, 0, OPT_NOEXEC }, + {"null-output", no_argument, 0, OPT_NULL_OUTPUT }, + {"no-ending-newline", no_argument, 0, OPT_NO_ENDING_NEWLINE }, + {"collect-errors", no_argument, 0, OPT_COLLECT_ERRORS }, + {"allow-duplicate-keys",no_argument, 0, OPT_ALLOW_DUPLICATE_KEYS }, + {"strip-empty-kv", no_argument, 0, OPT_STRIP_EMPTY_KV }, + {"tsv-format", no_argument, 0, OPT_TSV_FORMAT }, + {"to", required_argument, 0, 'T' }, + {"from", required_argument, 0, 'F' }, + {"quiet", no_argument, 0, 'q' }, + + {"check", no_argument, 0, OPT_CHECK }, + {"derive-key", required_argument, 0, OPT_DERIVE_KEY }, + {"no-names", no_argument, 0, OPT_NO_NAMES }, + {"raw", no_argument, 0, OPT_RAW }, + {"length", required_argument, 0, OPT_LENGTH }, + {"keyed", no_argument, 0, OPT_KEYED }, + {"list-backends", no_argument, 0, OPT_LIST_BACKENDS }, + {"backend", required_argument, 0, OPT_BACKEND }, + {"num-threads", required_argument, 0, OPT_NUM_THREADS }, + {"file-buffer", required_argument, 0, OPT_FILE_BUFFER }, + {"mmap-min-chunk", required_argument, 0, OPT_MMAP_MIN_CHUNK }, + {"mmap-max-chunk", required_argument, 0, OPT_MMAP_MAX_CHUNK }, + + {"help", no_argument, 0, 'h' }, + {"version", no_argument, 0, 'v' }, + {0, 0, 0, 0 }, +}; + +static void display_usage(FILE *fp, char *progname, int tool_mode) +{ + fprintf(fp, "Usage: %s [options] [args]\n", progname); + fprintf(fp, "\nOptions:\n\n"); + fprintf(fp, "\t--include, -I : Add directory to include path " + "(default path \"%s\")\n", + INCLUDE_DEFAULT); + fprintf(fp, "\t--debug-level, -d : Set debug level to " + "(default level %d)\n", + DEBUG_LEVEL_DEFAULT); + fprintf(fp, "\t--disable-diag : Disable diag error module \n"); + fprintf(fp, "\t--enable-diag : Enable diag error module \n"); + fprintf(fp, "\t--show-diag : Show diag option (source, position, type, module)\n"); + fprintf(fp, "\t--hide-diag : Hide diag optione (source, position, type, module)\n"); + + fprintf(fp, "\t--indent, -i : Set dump indent to " + " (default indent %d)\n", + INDENT_DEFAULT); + fprintf(fp, "\t--width, -w : Set dump width to - inf for infinite" + " (default width %d)\n", + WIDTH_DEFAULT); + fprintf(fp, "\t--resolve, -r : Perform anchor and merge key resolution" + " (default %s)\n", + RESOLVE_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--color, -C : Color output can be one of on, off, auto" + " (default %s)\n", + COLOR_DEFAULT); + fprintf(fp, "\t--visible, -V : Make all whitespace and linebreaks visible" + " (default %s)\n", + VISIBLE_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--follow, -l : Follow aliases when using paths" + " (default %s)\n", + FOLLOW_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--strip-labels : Strip labels when emitting" + " (default %s)\n", + STRIP_LABELS_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--strip-tags : Strip tags when emitting" + " (default %s)\n", + STRIP_TAGS_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--strip-doc : Strip document headers and indicators when emitting" + " (default %s)\n", + STRIP_DOC_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--disable-accel : Disable access accelerators (slower but uses less memory)" + " (default %s)\n", + DISABLE_ACCEL_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--disable-buffering : Disable buffering (i.e. no stdio file reads, unix fd instead)" + " (default %s)\n", + DISABLE_BUFFERING_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--disable-depth-limit : Disable depth limit" + " (default %s)\n", + DISABLE_DEPTH_LIMIT_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--json, -j : JSON input mode (no | force | auto)" + " (default %s)\n", + JSON_DEFAULT); + fprintf(fp, "\t--yaml-1.1 : Enable YAML 1.1 version instead of the library's default\n"); + fprintf(fp, "\t--yaml-1.2 : Enable YAML 1.2 version instead of the library's default\n"); + fprintf(fp, "\t--yaml-1.3 : Enable YAML 1.3 version instead of the library's default\n"); + fprintf(fp, "\t--sloppy-flow-indentation: Enable sloppy indentation in flow mode)" + " (default %s)\n", + SLOPPY_FLOW_INDENTATION_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--prefer-recursive : Prefer recursive instead of iterative algorighms" + " (default %s)\n", + PREFER_RECURSIVE_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--ypath-aliases : Use YPATH aliases (default %s)\n", + YPATH_ALIASES_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--null-output : Do not generate output (for scanner profiling)\n"); + fprintf(fp, "\t--no-ending-newline : Do not generate a final newline\n"); + fprintf(fp, "\t--collect-errors : Collect errors instead of outputting directly" + " (default %s)\n", + COLLECT_ERRORS_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--allow-duplicate-keys : Allow duplicate keys" + " (default %s)\n", + ALLOW_DUPLICATE_KEYS_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--strip-empty-kv : Strip keys with empty values when emitting (not available in streaming mode)" + " (default %s)\n", + STRIP_EMPTY_KV_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--quiet, -q : Quiet operation, do not " + "output messages (default %s)\n", + QUIET_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--dry-run : Do not parse/emit\n"); + fprintf(fp, "\t--version, -v : Display libfyaml version\n"); + fprintf(fp, "\t--help, -h : Display help message\n"); + + if (tool_mode == OPT_TOOL || tool_mode != OPT_TESTSUITE) { + fprintf(fp, "\t--sort, -s : Perform mapping key sort (valid for dump)" + " (default %s)\n", + SORT_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--comment, -c : Output comments (experimental)" + " (default %s)\n", + COMMENT_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--mode, -m : Output mode can be one of original, block, flow, flow-oneline, json, json-tp, json-oneline, dejson, pretty|yamlfmt, flow-compact, json-compact" + " (default %s)\n", + MODE_DEFAULT); + fprintf(fp, "\t--disable-flow-markers : Disable testsuite's flow-markers" + " (default %s)\n", + DISABLE_FLOW_MARKERS_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--disable-doc-markers : Disable testsuite's document-markers" + " (default %s)\n", + DISABLE_DOC_MARKERS_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--disable-scalar-styles : Disable testsuite's scalar styles (all are double quoted)" + " (default %s)\n", + DISABLE_SCALAR_STYLES_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--document-event-stream : Generate a document and then produce the event stream" + " (default %s)\n", + DOCUMENT_EVENT_STREAM_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--tsv-format : Display testsuite in TSV format" + " (default %s)\n", + TSV_FORMAT_DEFAULT ? "true" : "false"); + if (tool_mode == OPT_TOOL || tool_mode == OPT_DUMP) { + fprintf(fp, "\t--streaming : Use streaming output mode (default)"); + fprintf(fp, "\t--no-streaming : Don't use streaming output mode"); + fprintf(fp, "\t--recreating : Recreate streaming events" + " (default %s)\n", + RECREATING_DEFAULT ? "true" : "false"); + } + } + + if (tool_mode == OPT_TOOL || (tool_mode != OPT_DUMP && tool_mode != OPT_TESTSUITE)) { + fprintf(fp, "\t--file, -f : Use given file instead of \n" + "\t Note that using a string with a leading '>' is equivalent to a file with the trailing content\n" + "\t --file \">foo: bar\" is as --file file.yaml with file.yaml \"foo: bar\"\n"); + } + + if (tool_mode == OPT_TOOL || tool_mode == OPT_JOIN) { + fprintf(fp, "\t--to, -T : Join to (default %s)\n", + TO_DEFAULT); + fprintf(fp, "\t--from, -F : Join from (default %s)\n", + FROM_DEFAULT); + fprintf(fp, "\t--trim, -t : Output given path (default %s)\n", + TRIM_DEFAULT); + } + + if (tool_mode == OPT_TOOL || tool_mode == OPT_YPATH) { + fprintf(fp, "\t--from, -F : Start from (default %s)\n", + FROM_DEFAULT); + fprintf(fp, "\t--dump-pathexpr : Dump the path expresion before the results\n"); + fprintf(fp, "\t--noexec : Do not execute the expression\n"); + } + + if (tool_mode == OPT_TOOL || tool_mode == OPT_COMPOSE) { + fprintf(fp, "\t--dump-path : Dump the path while composing\n"); + } + + if (tool_mode == OPT_TOOL) { + fprintf(fp, "\t--dump : Dump mode, [arguments] are file names\n"); + fprintf(fp, "\t--testsuite : Testsuite mode, [arguments] are s to output parse events\n"); + fprintf(fp, "\t--filter : Filter mode, is input, [arguments] are s, outputs to stdout\n"); + fprintf(fp, "\t--join : Join mode, [arguments] are s, outputs to stdout\n"); + fprintf(fp, "\t--ypath : YPATH mode, [arguments] are s, file names, outputs to stdout\n"); + fprintf(fp, "\t--scan-dump : scan-dump mode, [arguments] are file names\n"); + fprintf(fp, "\t--parse-dump : parse-dump mode, [arguments] are file names\n"); + fprintf(fp, "\t--compose : composer driver dump mode, [arguments] are file names\n"); + fprintf(fp, "\t--yaml-version : Information about supported libfyaml's YAML versions\n"); + } + + fprintf(fp, "\n"); + + switch (tool_mode) { + case OPT_TOOL: + default: + break; + case OPT_TESTSUITE: + fprintf(fp, "\tParse and dump test-suite event format\n"); + fprintf(fp, "\t$ %s input.yaml\n\t...\n", progname); + fprintf(fp, "\n"); + fprintf(fp, "\tParse and dump of event example\n"); + fprintf(fp, "\t$ echo \"foo: bar\" | %s -\n", progname); + fprintf(fp, "\t+STR\n\t+DOC\n\t+MAP\n\t=VAL :foo\n\t=VAL :bar\n\t-MAP\n\t-DOC\n\t-STR\n"); + break; + case OPT_DUMP: + fprintf(fp, "\tParse and dump generated YAML document tree in the original YAML form\n"); + fprintf(fp, "\t$ %s input.yaml\n\t...\n", progname); + fprintf(fp, "\n"); + fprintf(fp, "\tParse and dump generated YAML document tree in block YAML form (and make whitespace visible)\n"); + fprintf(fp, "\t$ %s -V -mblock input.yaml\n\t...\n", progname); + fprintf(fp, "\n"); + fprintf(fp, "\tParse and dump generated YAML document from the input string\n"); + fprintf(fp, "\t$ %s -mjson \">foo: bar\"\n", progname); + fprintf(fp, "\t{\n\t \"foo\": \"bar\"\n\t}\n"); + break; + case OPT_FILTER: + fprintf(fp, "\tParse and filter YAML document tree starting from the '/foo' path followed by the '/bar' path\n"); + fprintf(fp, "\t$ %s --file input.yaml /foo /bar\n\t...\n", progname); + fprintf(fp, "\n"); + fprintf(fp, "\tParse and filter for two paths (note how a multi-document stream is produced)\n"); + fprintf(fp, "\t$ %s --file -mblock --filter --file \">{ foo: bar, baz: [ frooz, whee ] }\" /foo /baz\n", progname); + fprintf(fp, "\tbar\n\t---\n\t- frooz\n\t- whee\n"); + fprintf(fp, "\n"); + fprintf(fp, "\tParse and filter YAML document in stdin (note how the key may be complex)\n"); + fprintf(fp, "\t$ echo \"{ foo: bar }: baz\" | %s \"/{foo: bar}/\"\n", progname); + fprintf(fp, "\tbaz\n"); + break; + case OPT_JOIN: + fprintf(fp, "\tParse and join two YAML files\n"); + fprintf(fp, "\t$ %s file1.yaml file2.yaml\n\t...\n", progname); + fprintf(fp, "\n"); + fprintf(fp, "\tParse and join two YAML maps\n"); + fprintf(fp, "\t$ %s \">foo: bar\" \">baz: frooz\"\n", progname); + fprintf(fp, "\tfoo: bar\n\tbaz: frooz\n"); + fprintf(fp, "\n"); + fprintf(fp, "\tParse and join two YAML sequences\n"); + fprintf(fp, "\t$ %s -mblock \">[ foo ]\" \">[ bar ]\"\n", progname); + fprintf(fp, "\t- foo\n\t- bar\n"); + fprintf(fp, "\n"); + break; + case OPT_YPATH: + fprintf(fp, "\tParse and filter YAML with the ypath expression that results to /foo followed by /bar\n"); + fprintf(fp, "\t$ %s --ypath /foo,bar input.yaml\n\t...\n", progname); + fprintf(fp, "\n"); + break; + case OPT_SCAN_DUMP: + fprintf(fp, "\tParse and dump YAML scanner tokens (internal)\n"); + fprintf(fp, "\n"); + break; + case OPT_PARSE_DUMP: + fprintf(fp, "\tParse and dump YAML parser events (internal)\n"); + fprintf(fp, "\n"); + break; + case OPT_COMPOSE: + fprintf(fp, "\tParse and dump generated YAML document tree using the composer api\n"); + fprintf(fp, "\t$ %s input.yaml\n\t...\n", progname); + fprintf(fp, "\n"); + fprintf(fp, "\tParse and dump generated YAML document tree in block YAML form (and make whitespace visible)\n"); + fprintf(fp, "\t$ %s --compose -V -mblock input.yaml\n\t...\n", progname); + fprintf(fp, "\n"); + fprintf(fp, "\tParse and dump generated YAML document from the input string\n"); + fprintf(fp, "\t$ %s --compose -mjson \">foo: bar\"\n", progname); + fprintf(fp, "\t{\n\t \"foo\": \"bar\"\n\t}\n"); + break; + case OPT_YAML_VERSION_DUMP: + fprintf(fp, "\tDisplay information about the YAML versions libfyaml supports\n"); + fprintf(fp, "\n"); + break; + + case OPT_B3SUM: + fprintf(fp, "\tBLAKE3 hash b3sum utility\n"); + fprintf(fp, "\t--derive-key : Key derivation mode, with the given context string\n"); + fprintf(fp, "\t--no-names : Omit filenames\n"); + fprintf(fp, "\t--raw : Output result in raw bytes (single input allowed)\n"); + fprintf(fp, "\t--length : Output only this amount of bytes per output (max %u)\n", FY_BLAKE3_OUT_LEN); + fprintf(fp, "\t--check : Read files with BLAKE3 checksums and check files\n"); + fprintf(fp, "\t--keyed : Keyed mode with secret key read from (32 raw bytes)\n"); + fprintf(fp, "\t--backend : Select a BLAKE3 backend instead of the default\n"); + fprintf(fp, "\t--list-backends : Print out a list of available backends\n"); + fprintf(fp, "\t--num-threads : Number of threads, -1 disable, 0 let system decide, >= 1 explicit\n"); + fprintf(fp, "\t--file-buffer : Size of file I/O buffer (non-mmap case), 0 let system decide\n"); + fprintf(fp, "\t--mmap-min-chunk : Size of minimum mmap chunk, 0 let system decide\n"); + fprintf(fp, "\t--mmap-max-chunk : Size of maximum mmap chunk, 0 let system decide\n"); + fprintf(fp, "\n"); + break; + } +} + +static int apply_mode_flags(const char *what, enum fy_emitter_cfg_flags *flagsp) +{ + static const struct { + const char *name; + unsigned int value; + } mf[] = { + { .name = "original", .value = FYECF_MODE_ORIGINAL }, + { .name = "block", .value = FYECF_MODE_BLOCK }, + { .name = "flow", .value = FYECF_MODE_FLOW }, + { .name = "flow-oneline", .value = FYECF_MODE_FLOW_ONELINE }, + { .name = "json", .value = FYECF_MODE_JSON }, + { .name = "json-tp", .value = FYECF_MODE_JSON_TP }, + { .name = "json-oneline", .value = FYECF_MODE_JSON_ONELINE }, + { .name = "dejson", .value = FYECF_MODE_DEJSON }, + { .name = "pretty", .value = FYECF_MODE_PRETTY }, + { .name = "yamlfmt", .value = FYECF_MODE_PRETTY }, /* alias for pretty */ + { .name = "flow-compact", .value = FYECF_MODE_FLOW_COMPACT }, + { .name = "json-compact", .value = FYECF_MODE_JSON_COMPACT }, + }; + unsigned int i; + + if (!what || !flagsp) + return -1; + + if (!strcmp(what, "default")) + what = MODE_DEFAULT; + + for (i = 0; i < sizeof(mf)/sizeof(mf[0]); i++) { + if (!strcmp(what, mf[i].name)) { + *flagsp &= ~FYECF_MODE(FYECF_MODE_MASK); + *flagsp |= mf[i].value; + return 0; + } + } + + return -1; +} + +int apply_flags_option(const char *arg, unsigned int *flagsp, + int (*modify_flags)(const char *what, unsigned int *flagsp)) +{ + const char *s, *e, *sn; + char *targ; + int len, rc; + + if (!arg || !flagsp || !modify_flags) + return -1; + + s = arg; + e = arg + strlen(s); + while (s < e) { + sn = strchr(s, ','); + if (!sn) + sn = e; + + len = sn - s; + targ = alloca(len + 1); + memcpy(targ, s, len); + targ[len] = '\0'; + + rc = modify_flags(targ, flagsp); + if (rc) + return rc; + + s = sn < e ? (sn + 1) : sn; + } + + return 0; +} + +static int set_parser_input(struct fy_parser *fyp, const char *what, + bool default_string) +{ + int rc; + + if (!strcmp(what, "-")) + rc = fy_parser_set_input_fp(fyp, "stdin", stdin); + else if (*what == '<') + rc = fy_parser_set_input_file(fyp, what + 1); + else if (*what == '>') + rc = fy_parser_set_string(fyp, what + 1, FY_NT); + else + rc = fy_parser_set_input_file(fyp, what); + + return rc; +} + +static void no_diag_output_fn(struct fy_diag *diag, void *user, + const char *buf, size_t len) +{ + /* nothing */ +} + +struct composer_data { + struct fy_parser *fyp; + struct fy_document *fyd; + struct fy_emitter *emit; + bool null_output; + bool document_ready; + bool verbose; + bool single_document; +}; + +static enum fy_composer_return +compose_process_event(struct fy_parser *fyp, struct fy_event *fye, struct fy_path *path, void *userdata) +{ + struct composer_data *cd = userdata; + struct fy_document *fyd; + struct fy_path_component *parent, *last; + struct fy_node *fyn, *fyn_parent; + struct fy_node_pair *fynp; + int rc; + + if (cd->verbose) { + fy_parser_info(fyp, "%s: %c%c%c%c%c %3d - %-32s\n", + fy_event_type_get_text(fye->type), + fy_path_in_root(path) ? 'R' : '-', + fy_path_in_sequence(path) ? 'S' : '-', + fy_path_in_mapping(path) ? 'M' : '-', + fy_path_in_mapping_key(path) ? 'K' : + fy_path_in_mapping_value(path) ? 'V' : '-', + fy_path_in_collection_root(path) ? '/' : '-', + fy_path_depth(path), + fy_path_get_text_alloca(path)); + } + + switch (fye->type) { + /* nothing to do for those */ + case FYET_NONE: + case FYET_STREAM_START: + case FYET_STREAM_END: + break; + + case FYET_DOCUMENT_START: + if (cd->fyd) { + fy_document_destroy(cd->fyd); + cd->fyd = NULL; + } + cd->document_ready = false; + + cd->fyd = fy_document_create_from_event(fyp, fye); + assert(cd->fyd); + + break; + + case FYET_DOCUMENT_END: + rc = fy_document_update_from_event(cd->fyd, fyp, fye); + assert(!rc); + + cd->document_ready = true; + + if (!cd->null_output && cd->fyd) + fy_emit_document(cd->emit, cd->fyd); + + fy_document_destroy(cd->fyd); + cd->fyd = NULL; + + /* on single document mode we stop here */ + if (cd->single_document) + return FYCR_OK_STOP; + break; + + + case FYET_SCALAR: + case FYET_ALIAS: + case FYET_MAPPING_START: + case FYET_SEQUENCE_START: + + fyd = cd->fyd; + assert(fyd); + + fyn = fy_node_create_from_event(fyd, fyp, fye); + assert(fyn); + + switch (fye->type) { + default: + /* XXX should now happen */ + break; + + case FYET_SCALAR: + case FYET_ALIAS: + last = NULL; + break; + + case FYET_MAPPING_START: + last = fy_path_last_component(path); + assert(last); + + fy_path_component_set_mapping_user_data(last, fyn); + fy_path_component_set_mapping_key_user_data(last, NULL); + + break; + + case FYET_SEQUENCE_START: + + last = fy_path_last_component(path); + assert(last); + + fy_path_component_set_sequence_user_data(last, fyn); + break; + } + + /* parent */ + parent = fy_path_last_not_collection_root_component(path); + + if (fy_path_in_root(path)) { + + rc = fy_document_set_root(cd->fyd, fyn); + assert(!rc); + + } else if (fy_path_in_sequence(path)) { + + assert(parent); + fyn_parent = fy_path_component_get_sequence_user_data(parent); + assert(fyn_parent); + assert(fy_node_is_sequence(fyn_parent)); + + rc = fy_node_sequence_add_item(fyn_parent, fyn); + assert(!rc); + + } else { + /* only thing left */ + assert(fy_path_in_mapping(path)); + + assert(parent); + fyn_parent = fy_path_component_get_mapping_user_data(parent); + assert(fyn_parent); + assert(fy_node_is_mapping(fyn_parent)); + + if (fy_path_in_mapping_key(path)) { + + fynp = fy_node_pair_create_with_key(fyd, fyn_parent, fyn); + assert(fynp); + + fy_path_component_set_mapping_key_user_data(parent, fynp); + + } else { + + assert(fy_path_in_mapping_value(path)); + + fynp = fy_path_component_get_mapping_key_user_data(parent); + assert(fynp); + + rc = fy_node_pair_update_with_value(fynp, fyn); + if (rc) /* this may happen normally */ + goto err_out; + + fy_path_component_set_mapping_key_user_data(parent, NULL); + } + } + + break; + + case FYET_MAPPING_END: + last = fy_path_last_component(path); + assert(last); + + fyn = fy_path_component_get_mapping_user_data(last); + assert(fyn); + assert(fy_node_is_mapping(fyn)); + + rc = fy_node_update_from_event(fyn, fyp, fye); + assert(!rc); + + break; + + case FYET_SEQUENCE_END: + last = fy_path_last_component(path); + assert(last); + + fyn = fy_path_component_get_sequence_user_data(last); + assert(fyn); + assert(fy_node_is_sequence(fyn)); + + rc = fy_node_update_from_event(fyn, fyp, fye); + assert(!rc); + + break; + } + + return FYCR_OK_CONTINUE; + +err_out: + return FYCR_ERROR; +} + +struct b3sum_config { + bool no_names : 1, + raw : 1, + keyed : 1, + check : 1, + derive_key : 1, + quiet : 1, + list_backends : 1, + no_mmap : 1; + size_t file_buffer; + size_t mmap_min_chunk; + size_t mmap_max_chunk; + unsigned int length; + const char *context; + size_t context_len; + const char *backend; + unsigned int num_threads; +}; + +struct b3sum_config default_b3sum_cfg = { + .length = FY_BLAKE3_OUT_LEN, +}; + +static int do_b3sum_hash_file(struct fy_blake3_hasher *hasher, const char *filename, bool no_names, bool raw, unsigned int length) +{ + static const char *hexb = "0123456789abcdef"; + const uint8_t *output; + size_t filename_sz, line_sz, outsz; + ssize_t wrn; + uint8_t v; + const void *outp; + char *line, *s; + unsigned int i; + + filename_sz = strlen(filename); + + output = fy_blake3_hash_file(hasher, filename); + if (!output) { + fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno)); + return -1; + } + + if (!raw) { + /* output line (optimized) */ + line_sz = (length * 2); /* the hex output */ + if (!no_names) + line_sz += 2 + filename_sz; /* 2 spaces + filename */ + line_sz++; /* '\n' */ + line = alloca(line_sz + 1); + s = line; + for (i = 0; i < length; i++) { + v = output[i]; + *s++ = hexb[v >> 4]; + *s++ = hexb[v & 15]; + } + if (!no_names) { + *s++ = ' '; + *s++ = ' '; + memcpy(s, filename, filename_sz); + s += filename_sz; + } + *s++ = '\n'; + *s = '\0'; + outp = line; + outsz = (size_t)(s - line); + } else { + outp = output; + outsz = length; + } + + wrn = fwrite(outp, 1, outsz, stdout); + if ((size_t)wrn != outsz) { + fprintf(stderr, "Unable to write to stdout! error: %s\n", strerror(errno)); + return -1; + } + + return 0; +} + +static int do_b3sum_check_file(struct fy_blake3_hasher *hasher, const char *check_filename, bool quiet) +{ + char *hash, *filename; + FILE *fp = NULL; + char linebuf[8192]; /* maximum size for a line is 8K, should be enough (PATH_MAX is 4K at linux) */ + uint8_t read_hash[FY_BLAKE3_OUT_LEN], v; + const uint8_t *computed_hash; + char *s; + char c; + unsigned int i, j, length; + size_t linesz; + int line, exit_code; + + if (check_filename && strcmp(check_filename, "-")) { + fp = fopen(check_filename, "ra"); + if (!fp) { + fprintf(stderr, "Failed to open check file: \"%s\", error: %s\n", check_filename, strerror(errno)); + goto err_out; + } + } else { + fp = stdin; + } + + /* default error code if all is fine */ + exit_code = 0; + + line = 0; + while (fgets(linebuf, sizeof(linebuf), fp)) { + /* '\0' terminate always */ + linebuf[(sizeof(linebuf)/sizeof(linebuf[0]))-1] = '\0'; + + linesz = strlen(linebuf); + while (linesz > 0 && linebuf[linesz-1] == '\n') + linesz--; + + if (!linesz) { + fprintf(stderr, "Empty line found at file \"%s\" line #%d\n", check_filename, line); + goto err_out; + } + linebuf[linesz] = '\0'; + + length = 0; + s = linebuf; + while (isxdigit((unsigned char)*s)) + s++; + + length = s - linebuf; + + if (length == 0 || length > (FY_BLAKE3_OUT_LEN * 2) || (length % 1) || !isspace((unsigned char)*s)) { + fprintf(stderr, "Bad line found at file \"%s\" line #%d\n", check_filename, line); + fprintf(stderr, "%s\n", linebuf); + goto err_out; + } + + *s++ = '\0'; + + while (isspace((unsigned char)*s)) + s++; + + length >>= 1; /* to bytes */ + hash = linebuf; + filename = s; + + for (i = 0, s = hash; i < length; i++) { + v = 0; + for (j = 0; j < 2; j++) { + v <<= 4; + c = *s++; + if (c >= '0' && c <= '9') + v |= c - '0'; + else if (c >= 'a' && c <= 'f') + v |= c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + v |= c - 'A' + 10; + else + v = 0; + } + read_hash[i] = v; + } + + computed_hash = fy_blake3_hash_file(hasher, filename); + if (!computed_hash) { + fprintf(stderr, "Failed to hash file: \"%s\", error: %s\n", filename, strerror(errno)); + goto err_out; + } + + /* constant time comparison */ + v = 0; + for (i = 0; i < length; i++) + v |= (read_hash[i] ^ computed_hash[i]); + + if (v) { + printf("%s: FAILED\n", filename); + exit_code = -1; + } else if (!quiet) + printf("%s: OK\n", filename); + } + +out: + if (fp && fp != stdin) + fclose(fp); + + return exit_code; + +err_out: + exit_code = -1; + goto out; +} + +static int +do_b3sum(int argc, char *argv[], int optind, const struct b3sum_config *cfg) +{ + struct fy_blake3_hasher_cfg hcfg; + struct fy_blake3_hasher *hasher; + uint8_t key[FY_BLAKE3_OUT_LEN]; + const char *filename; + int rc, num_inputs, num_ok, i; + size_t rdn; + const char *backend, *prev; + + if (cfg->list_backends) { + prev = NULL; + while ((backend = fy_blake3_backend_iterate(&prev)) != NULL) + printf("%s\n", backend); + return 0; + } + + if (cfg->quiet && !cfg->check) { + fprintf(stderr, "Error: --quiet may only be used together with --check\n\n"); + return 1; + } + + if (cfg->keyed && cfg->derive_key) { + fprintf(stderr, "Error: --keyed and --derive-key may not be used together\n\n"); + return 1; + } + + if (cfg->check && cfg->length != FY_BLAKE3_OUT_LEN) { + fprintf(stderr, "Error: --check and --length may not be used together\n\n"); + return 1; + } + + if (cfg->keyed) { + rdn = fread(key, 1, FY_BLAKE3_KEY_LEN, stdin); + if (rdn != FY_BLAKE3_KEY_LEN) { + if (rdn >= 0 && rdn < FY_BLAKE3_KEY_LEN) + fprintf(stderr, "Error: could not read secret key from : short key\n\n"); + else + fprintf(stderr, "Error: could not read secret key from : error %s\n\n", strerror(errno)); + return 1; + } + rc = fgetc(stdin); + if (rc != EOF) { + fprintf(stderr, "Error: garbage trailing secret key from \n\n"); + return -1; + } + } + + num_inputs = argc - optind; + if (num_inputs <= 0) + num_inputs = 1; /* stdin mode */ + + if (cfg->raw && num_inputs > 1) { + fprintf(stderr, "Error: Raw output mode is only supported with a single input\n\n"); + return 1; + } + + /* we can't handle '-' in keyed mode */ + if (cfg->keyed) { + for (i = optind; i < argc; i++) { + if (!strcmp(argv[i], "-")) { + fprintf(stderr, "Cannot use in keyed mode\n"); + return 1; + } + } + } + + memset(&hcfg, 0, sizeof(hcfg)); + hcfg.key = cfg->keyed ? key : NULL; + hcfg.context = cfg->derive_key ? cfg->context : NULL; + hcfg.context_len = cfg->derive_key ? cfg->context_len : 0; + hcfg.backend = cfg->backend; + hcfg.no_mmap = cfg->no_mmap; + hcfg.file_buffer = cfg->file_buffer; + hcfg.mmap_min_chunk = cfg->mmap_min_chunk; + hcfg.mmap_max_chunk = cfg->mmap_max_chunk; + hcfg.num_threads = cfg->num_threads; + hasher = fy_blake3_hasher_create(&hcfg); + if (!hasher) { + fprintf(stderr, "unable to create blake3 hasher\n"); + return -1; + } + + /* we will get in the loop even when no arguments (we'll do stdin instead) */ + num_ok = 0; + i = optind; + do { + /* if no arguments, use stdin */ + filename = i < argc ? argv[i] : "-"; + + if (!cfg->check) + rc = do_b3sum_hash_file(hasher, filename, cfg->no_names, cfg->raw, cfg->length); + else + rc = do_b3sum_check_file(hasher, filename, cfg->quiet); + if (!rc) + num_ok++; + + } while (++i < argc); + + fy_blake3_hasher_destroy(hasher); + + return num_inputs == num_ok ? 0 : -1; +} + +int main(int argc, char *argv[]) +{ + struct fy_parse_cfg cfg = { + .search_path = INCLUDE_DEFAULT, + .flags = + (QUIET_DEFAULT ? FYPCF_QUIET : 0) | + (RESOLVE_DEFAULT ? FYPCF_RESOLVE_DOCUMENT : 0) | + (DISABLE_ACCEL_DEFAULT ? FYPCF_DISABLE_ACCELERATORS : 0) | + (DISABLE_BUFFERING_DEFAULT ? FYPCF_DISABLE_BUFFERING : 0) | + (DISABLE_DEPTH_LIMIT_DEFAULT ? FYPCF_DISABLE_DEPTH_LIMIT : 0) | + (SLOPPY_FLOW_INDENTATION_DEFAULT ? FYPCF_SLOPPY_FLOW_INDENTATION : 0) | + (PREFER_RECURSIVE_DEFAULT ? FYPCF_PREFER_RECURSIVE : 0) | + (YPATH_ALIASES_DEFAULT ? FYPCF_YPATH_ALIASES : 0), + }; + struct fy_emitter_xcfg emit_xcfg; + struct fy_parser *fyp = NULL; + struct fy_emitter *emit = NULL; + int rc, exitcode = EXIT_FAILURE, opt, lidx, i, j, step = 1; + enum fy_error_module errmod; + unsigned int errmod_mask; + bool show; + int indent = INDENT_DEFAULT; + int width = WIDTH_DEFAULT; + bool manual_width = false; + bool follow = FOLLOW_DEFAULT; + const char *to = TO_DEFAULT; + const char *from = FROM_DEFAULT; + const char *color = COLOR_DEFAULT; + const char *file = NULL, *trim = TRIM_DEFAULT; + char *tmp, *s, *progname; + struct fy_document *fyd, *fyd_join = NULL; + enum fy_emitter_cfg_flags emit_flags = 0; + enum fy_emitter_xcfg_flags emit_xflags = 0; + struct fy_node *fyn, *fyn_emit, *fyn_to, *fyn_from; + int count_ins = 0; + struct fy_document **fyd_ins = NULL; + int tool_mode = OPT_TOOL; + struct fy_event *fyev; + struct fy_event *fyeev; + const char *eevtext; + size_t eevlen; + struct fy_tag **tags; + struct fy_token *fyt; + bool join_resolve = RESOLVE_DEFAULT; + struct fy_token_iter *iter; + bool streaming = STREAMING_DEFAULT; + bool recreating = RECREATING_DEFAULT; + struct fy_diag_cfg dcfg; + struct fy_diag *diag = NULL; + struct fy_path_parse_cfg pcfg; + struct fy_path_expr *expr = NULL; + struct fy_path_exec_cfg xcfg; + struct fy_path_exec *fypx = NULL; + struct fy_node *fyn_start; + bool dump_pathexpr = false; + bool noexec = false; + bool null_output = false; + bool stdin_input; + void *res_iter; + bool disable_flow_markers = DISABLE_FLOW_MARKERS_DEFAULT; + bool disable_doc_markers = DISABLE_DOC_MARKERS_DEFAULT; + bool disable_scalar_styles = DISABLE_SCALAR_STYLES_DEFAULT; + bool document_event_stream = DOCUMENT_EVENT_STREAM_DEFAULT; + bool collect_errors = COLLECT_ERRORS_DEFAULT; + bool allow_duplicate_keys = ALLOW_DUPLICATE_KEYS_DEFAULT; + bool tsv_format = TSV_FORMAT_DEFAULT; + struct composer_data cd; + bool dump_path = DUMP_PATH_DEFAULT; + const char *input_arg; + enum dump_testsuite_event_flags dump_flags; + /* b3sum */ + int opti; + struct b3sum_config b3cfg = default_b3sum_cfg; + + fy_valgrind_check(&argc, &argv); + + /* select the appropriate tool mode */ + progname = strrchr(argv[0], '/'); + if (!progname) + progname = argv[0]; + else + progname++; + + /* default mode is dump */ + if (!strcmp(progname, "fy-filter")) + tool_mode = OPT_FILTER; + else if (!strcmp(progname, "fy-testsuite")) + tool_mode = OPT_TESTSUITE; + else if (!strcmp(progname, "fy-dump")) + tool_mode = OPT_DUMP; + else if (!strcmp(progname, "fy-join")) + tool_mode = OPT_JOIN; + else if (!strcmp(progname, "fy-ypath")) + tool_mode = OPT_YPATH; + else if (!strcmp(progname, "fy-scan-dump")) + tool_mode = OPT_SCAN_DUMP; + else if (!strcmp(progname, "fy-parse-dump")) + tool_mode = OPT_PARSE_DUMP; + else if (!strcmp(progname, "fy-compose")) + tool_mode = OPT_COMPOSE; + else if (!strcmp(progname, "fy-yaml-version-dump")) + tool_mode = OPT_YAML_VERSION_DUMP; + else if (!strcmp(progname, "fy-b3sum")) + tool_mode = OPT_B3SUM; + else + tool_mode = OPT_TOOL; + + fy_diag_cfg_default(&dcfg); + /* XXX remember to modify this if you change COLOR_DEFAULT */ + + emit_flags = (SORT_DEFAULT ? FYECF_SORT_KEYS : 0) | + (COMMENT_DEFAULT ? FYECF_OUTPUT_COMMENTS : 0) | + (STRIP_LABELS_DEFAULT ? FYECF_STRIP_LABELS : 0) | + (STRIP_TAGS_DEFAULT ? FYECF_STRIP_TAGS : 0) | + (STRIP_DOC_DEFAULT ? FYECF_STRIP_DOC : 0); + apply_mode_flags(MODE_DEFAULT, &emit_flags); + + emit_xflags = (VISIBLE_DEFAULT ? FYEXCF_VISIBLE_WS : 0) | + (!strcmp(COLOR_DEFAULT, "auto") ? FYEXCF_COLOR_AUTO : + !strcmp(COLOR_DEFAULT, "on") ? FYEXCF_COLOR_FORCE : + FYEXCF_COLOR_NONE) | + FYEXCF_OUTPUT_STDOUT; + + while ((opt = getopt_long(argc, argv, + "I:" "d:" "i:" "w:" "rsc" "C:" "m:" "V" "f:" "t:" "T:F:" "j:" "qhvl", + lopts, &lidx)) != -1) { + switch (opt) { + case 'I': + tmp = alloca(strlen(cfg.search_path) + 1 + strlen(optarg) + 1); + s = tmp; + strcpy(s, cfg.search_path); + if (cfg.search_path && cfg.search_path[0]) { + s += strlen(cfg.search_path); + *s++ = ':'; + } + strcpy(s, optarg); + s += strlen(optarg); + *s = '\0'; + cfg.search_path = tmp; + break; + case 'i': + indent = atoi(optarg); + if (indent <= 0 || indent > FYECF_INDENT_MASK) { + fprintf(stderr, "bad indent option %s\n", optarg); + goto err_out_usage; + } + + break; + case 'w': + if (!strcmp(optarg, "inf")) { + width = 0; /* infinite */ + } else { + width = atoi(optarg); + if (width <= 8 || width > FYECF_WIDTH_MASK) { /* it should fit %YAML 1.3 at least */ + fprintf(stderr, "bad width option %s\n", optarg); + goto err_out_usage; + } + } + manual_width = true; + break; + case 'd': + dcfg.level = fy_string_to_error_type(optarg); + if (dcfg.level == FYET_MAX) { + fprintf(stderr, "bad debug level option %s\n", optarg); + goto err_out_usage; + } + break; + case OPT_DISABLE_DIAG: + case OPT_ENABLE_DIAG: + if (!strcmp(optarg, "all")) { + errmod_mask = FY_BIT(FYEM_MAX) - 1; + } else { + errmod = fy_string_to_error_module(optarg); + if (errmod == FYEM_MAX) { + fprintf(stderr, "bad error module option %s\n", optarg); + goto err_out_usage; + } + errmod_mask = FY_BIT(errmod); + } + if (opt == OPT_DISABLE_DIAG) + dcfg.module_mask &= ~errmod_mask; + else + dcfg.module_mask |= errmod_mask; + break; + + case OPT_SHOW_DIAG: + case OPT_HIDE_DIAG: + show = opt == OPT_SHOW_DIAG; + if (!strcmp(optarg, "source")) { + dcfg.show_source = show; + } else if (!strcmp(optarg, "position")) { + dcfg.show_position = show; + } else if (!strcmp(optarg, "type")) { + dcfg.show_type = show; + } else if (!strcmp(optarg, "module")) { + dcfg.show_module = show; + } else { + fprintf(stderr, "bad %s option %s\n", + show ? "show" : "hide", optarg); + goto err_out_usage; + } + break; + + case 'r': + cfg.flags |= FYPCF_RESOLVE_DOCUMENT; + break; + case 's': + emit_flags |= FYECF_SORT_KEYS; + break; + case 'c': + cfg.flags |= FYPCF_PARSE_COMMENTS; + emit_flags |= FYECF_OUTPUT_COMMENTS; + break; + case 'C': + color = optarg; + if (!strcmp(color, "auto")) { + dcfg.colorize = isatty(fileno(stderr)) == 1; + emit_xflags &= ~(FYEXCF_COLOR_MASK << FYEXCF_COLOR_SHIFT); + emit_xflags |= FYEXCF_COLOR_AUTO; + } + else if (!strcmp(color, "yes") || !strcmp(color, "1") || !strcmp(color, "on")) { + dcfg.colorize = true; + emit_xflags &= ~(FYEXCF_COLOR_MASK << FYEXCF_COLOR_SHIFT); + emit_xflags |= FYEXCF_COLOR_FORCE; + } else if (!strcmp(color, "no") || !strcmp(color, "0") || !strcmp(color, "off")) { + dcfg.colorize = false; + emit_xflags &= ~(FYEXCF_COLOR_MASK << FYEXCF_COLOR_SHIFT); + emit_xflags |= FYEXCF_COLOR_NONE; + } else { + fprintf(stderr, "bad color option %s\n", optarg); + goto err_out_usage; + } + break; + case 'm': + rc = apply_mode_flags(optarg, &emit_flags); + if (rc) { + fprintf(stderr, "bad mode option %s\n", optarg); + goto err_out_usage; + } + break; + case 'V': + emit_xflags |= FYEXCF_VISIBLE_WS; + break; + case 'l': + follow = true; + break; + case 'q': + cfg.flags |= FYPCF_QUIET; + dcfg.output_fn = no_diag_output_fn; + dcfg.fp = NULL; + dcfg.colorize = false; + b3cfg.quiet = true; + break; + case 'f': + file = optarg; + break; + case 't': + trim = optarg; + break; + case 'T': + to = optarg; + break; + case 'F': + from = optarg; + break; + case OPT_TESTSUITE: + case OPT_FILTER: + case OPT_DUMP: + case OPT_JOIN: + case OPT_TOOL: + case OPT_YPATH: + case OPT_SCAN_DUMP: + case OPT_PARSE_DUMP: + case OPT_COMPOSE: + case OPT_YAML_VERSION_DUMP: + case OPT_B3SUM: + tool_mode = opt; + break; + case OPT_STRIP_LABELS: + emit_flags |= FYECF_STRIP_LABELS; + break; + case OPT_STRIP_TAGS: + emit_flags |= FYECF_STRIP_TAGS; + break; + case OPT_STRIP_DOC: + emit_flags |= FYECF_STRIP_DOC; + break; + case OPT_STREAMING: + streaming = true; + break; + case OPT_NO_STREAMING: + streaming = false; + break; + case OPT_RECREATING: + recreating = true; + break; + case OPT_DUMP_PATH: + dump_path = true; + break; + case 'j': + cfg.flags &= ~(FYPCF_JSON_MASK << FYPCF_JSON_SHIFT); + if (!strcmp(optarg, "no")) + cfg.flags |= FYPCF_JSON_NONE; + else if (!strcmp(optarg, "auto")) + cfg.flags |= FYPCF_JSON_AUTO; + else if (!strcmp(optarg, "force")) + cfg.flags |= FYPCF_JSON_FORCE; + else { + fprintf(stderr, "bad json option %s\n", optarg); + goto err_out_usage; + } + break; + case OPT_DISABLE_ACCEL: + cfg.flags |= FYPCF_DISABLE_ACCELERATORS; + break; + case OPT_DISABLE_BUFFERING: + cfg.flags |= FYPCF_DISABLE_BUFFERING; + break; + case OPT_DISABLE_DEPTH_LIMIT: + cfg.flags |= FYPCF_DISABLE_DEPTH_LIMIT; + break; + case OPT_DISABLE_MMAP: + cfg.flags |= FYPCF_DISABLE_MMAP_OPT; + b3cfg.no_mmap = true; + break; + case OPT_DUMP_PATHEXPR: + dump_pathexpr = true; + break; + case OPT_NOEXEC: + noexec = true; + break; + case OPT_NULL_OUTPUT: + null_output = true; + emit_xflags |= FYEXCF_NULL_OUTPUT; + break; + case OPT_NO_ENDING_NEWLINE: + emit_flags |= FYECF_NO_ENDING_NEWLINE; + break; + case OPT_YAML_1_1: + cfg.flags &= ~(FYPCF_DEFAULT_VERSION_MASK << FYPCF_DEFAULT_VERSION_SHIFT); + cfg.flags |= FYPCF_DEFAULT_VERSION_1_1; + break; + case OPT_YAML_1_2: + cfg.flags &= ~(FYPCF_DEFAULT_VERSION_MASK << FYPCF_DEFAULT_VERSION_SHIFT); + cfg.flags |= FYPCF_DEFAULT_VERSION_1_2; + break; + case OPT_YAML_1_3: + cfg.flags &= ~(FYPCF_DEFAULT_VERSION_MASK << FYPCF_DEFAULT_VERSION_SHIFT); + cfg.flags |= FYPCF_DEFAULT_VERSION_1_3; + break; + case OPT_SLOPPY_FLOW_INDENTATION: + cfg.flags |= FYPCF_SLOPPY_FLOW_INDENTATION; + break; + case OPT_PREFER_RECURSIVE: + cfg.flags |= FYPCF_PREFER_RECURSIVE; + break; + case OPT_YPATH_ALIASES: + cfg.flags |= FYPCF_YPATH_ALIASES; + break; + case OPT_DISABLE_FLOW_MARKERS: + disable_flow_markers = true; + break; + case OPT_DISABLE_DOC_MARKERS: + disable_doc_markers = true; + break; + case OPT_DISABLE_SCALAR_STYLES: + disable_scalar_styles = true; + break; + case OPT_DOCUMENT_EVENT_STREAM: + document_event_stream = true; + break; + case OPT_COLLECT_ERRORS: + collect_errors = true; + break; + case OPT_ALLOW_DUPLICATE_KEYS: + allow_duplicate_keys = true; + break; + case OPT_STRIP_EMPTY_KV: + emit_flags |= FYECF_STRIP_EMPTY_KV; + break; + case OPT_TSV_FORMAT: + tsv_format = true; + break; + + case OPT_DERIVE_KEY: + b3cfg.derive_key = true; + b3cfg.context = optarg; + b3cfg.context_len = strlen(optarg); + break; + case OPT_NO_NAMES: + b3cfg.no_names = true; + break; + case OPT_RAW: + b3cfg.raw = true; + break; + case OPT_CHECK: + b3cfg.check = true; + break; + case OPT_KEYED: + b3cfg.keyed = true; + break; + + case OPT_LENGTH: + opti = atoi(optarg); + if (opti <= 0 || opti > FY_BLAKE3_OUT_LEN) { + fprintf(stderr, "Error: bad length=%d (must be > 0 and <= %u)\n\n", opti, FY_BLAKE3_OUT_LEN); + goto err_out_usage; + } + b3cfg.length = (unsigned int)opti; + break; + + case OPT_LIST_BACKENDS: + b3cfg.list_backends = true; + break; + + case OPT_BACKEND: + b3cfg.backend = optarg; + break; + + case OPT_NUM_THREADS: + b3cfg.num_threads = atoi(optarg); + break; + + case OPT_FILE_BUFFER: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad file-buffer=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + b3cfg.file_buffer = (size_t)opti; + break; + + case OPT_MMAP_MIN_CHUNK: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad mmap-min-chunk=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + b3cfg.mmap_min_chunk = (size_t)opti; + break; + + case OPT_MMAP_MAX_CHUNK: + opti = atoi(optarg); + if (opti < 0) { + fprintf(stderr, "Error: bad mmap-max-chunk=%d (must be >= 0)\n\n", opti); + goto err_out_usage; + } + b3cfg.mmap_max_chunk = (size_t)opti; + break; + + + case 'h' : + default: + if (opt != 'h') + fprintf(stderr, "Unknown option '%c' %d\n", opt, opt); + display_usage(opt == 'h' ? stdout : stderr, progname, tool_mode); + return opt == 'h' ? EXIT_SUCCESS : EXIT_FAILURE; + case 'v': + printf("%s\n", fy_library_version()); + return EXIT_SUCCESS; + } + } + + if (tool_mode == OPT_B3SUM) { + + rc = do_b3sum(argc, argv, optind, &b3cfg); + if (rc == 1) { + /* display usage */ + goto err_out_usage; + } + exitcode = !rc ? EXIT_SUCCESS : EXIT_FAILURE; + goto cleanup; + } + + if (tool_mode == OPT_YAML_VERSION_DUMP) { + const struct fy_version *vers; + void *iter; + + vers = fy_version_default(); + printf("Default version : %d.%d\n", vers->major, vers->minor); + + printf("Supported versions :"); + iter = NULL; + while ((vers = fy_version_supported_iterate(&iter)) != NULL) + printf(" %d.%d", vers->major, vers->minor); + printf("\n"); + } + + /* if we're still in tool mode, switch to dump */ + if (tool_mode == OPT_TOOL) + tool_mode = OPT_DUMP; + + /* as a special case for join, we resolve the document once */ + if (tool_mode == OPT_JOIN) { + join_resolve = !!(cfg.flags & FYPCF_RESOLVE_DOCUMENT); + cfg.flags &= ~FYPCF_RESOLVE_DOCUMENT; + } + + /* create common diagnostic object */ + diag = fy_diag_create(&dcfg); + if (!diag) { + fprintf(stderr, "fy_diag_create() failed\n"); + goto cleanup; + } + + /* collect errors, instead of outputting directly */ + if (collect_errors) + fy_diag_set_collect_errors(diag, true); + + if (allow_duplicate_keys) + cfg.flags |= FYPCF_ALLOW_DUPLICATE_KEYS; + + /* all set, use fy_diag for error reporting, debugging now */ + + cfg.diag = diag; + fyp = fy_parser_create(&cfg); + if (!fyp) { + fprintf(stderr, "fy_parser_create() failed\n"); + goto cleanup; + } + + exitcode = EXIT_FAILURE; + + if (tool_mode != OPT_TESTSUITE) { + enum fy_emitter_cfg_flags emit_width_flags; + + /* if we're dumping to a non tty stdout width is infinite */ + if (tool_mode == OPT_DUMP && !isatty(fileno(stdout)) && !manual_width) + emit_width_flags = FYECF_WIDTH_INF; + else if (width > 0) + emit_width_flags = FYECF_WIDTH(width); + else + emit_width_flags = FYECF_WIDTH_INF; + + memset(&emit_xcfg, 0, sizeof(emit_xcfg)); + emit_xcfg.cfg.flags = emit_flags | emit_width_flags | + FYECF_INDENT(indent) | + FYECF_EXTENDED_CFG; // use extended config + + /* unconditionally turn on document start markers for ypath */ + if (tool_mode == OPT_YPATH) + emit_xcfg.cfg.flags |= FYECF_DOC_START_MARK_ON; + + emit_xcfg.xflags = emit_xflags; + + emit = fy_emitter_create(&emit_xcfg.cfg); + if (!emit) { + fprintf(stderr, "fy_emitter_create() failed\n"); + goto cleanup; + } + + } + + switch (tool_mode) { + + case OPT_TESTSUITE: + if (optind >= argc || !strcmp(argv[optind], "-")) + rc = fy_parser_set_input_fp(fyp, "stdin", stdin); + else + rc = fy_parser_set_input_file(fyp, argv[optind]); + if (rc) { + fprintf(stderr, "failed to set testsuite input\n"); + goto cleanup; + } + + iter = fy_token_iter_create(NULL); + if (!iter) { + fprintf(stderr, "failed to create token iterator\n"); + goto cleanup; + } + + dump_flags = (dcfg.colorize && isatty(fileno(stdout)) ? DTEF_COLORIZE : 0) | + (disable_flow_markers ? DTEF_DISABLE_FLOW_MARKERS : 0) | + (disable_doc_markers ? DTEF_DISABLE_DOC_MARKERS : 0) | + (disable_scalar_styles ? DTEF_DISABLE_SCALAR_STYLES : 0) | + (tsv_format ? DTEF_TSV_FORMAT : 0); + + if (!document_event_stream) { + /* regular test suite */ + while ((fyev = fy_parser_parse(fyp)) != NULL) { + dump_testsuite_event(fyev, dump_flags); + fy_parser_event_free(fyp, fyev); + } + } else { + struct fy_document_iterator *fydi; + + fydi = fy_document_iterator_create(); + assert(fydi); + + fyev = fy_document_iterator_stream_start(fydi); + if (!fyev) { + fprintf(stderr, "failed to create document iterator's stream start event\n"); + goto cleanup; + } + dump_testsuite_event(fyev, dump_flags); + fy_document_iterator_event_free(fydi, fyev); + + /* convert to document and then process the generator event stream it */ + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + + fyev = fy_document_iterator_document_start(fydi, fyd); + if (!fyev) { + fprintf(stderr, "failed to create document iterator's document start event\n"); + goto cleanup; + } + dump_testsuite_event(fyev, dump_flags); + fy_document_iterator_event_free(fydi, fyev); + + while ((fyev = fy_document_iterator_body_next(fydi)) != NULL) { + dump_testsuite_event(fyev, dump_flags); + fy_document_iterator_event_free(fydi, fyev); + } + + fyev = fy_document_iterator_document_end(fydi); + if (!fyev) { + fprintf(stderr, "failed to create document iterator's stream document end\n"); + goto cleanup; + } + dump_testsuite_event(fyev, dump_flags); + fy_document_iterator_event_free(fydi, fyev); + + fy_parse_document_destroy(fyp, fyd); + if (rc) + break; + + } + + fyev = fy_document_iterator_stream_end(fydi); + if (!fyev) { + fprintf(stderr, "failed to create document iterator's stream end event\n"); + goto cleanup; + } + dump_testsuite_event(fyev, dump_flags); + fy_document_iterator_event_free(fydi, fyev); + + fy_document_iterator_destroy(fydi); + fydi = NULL; + } + + fy_token_iter_destroy(iter); + iter = NULL; + + if (fy_parser_get_stream_error(fyp)) + goto cleanup; + break; + + case OPT_DUMP: + for (i = optind; ; i++) { + + if (optind < argc) { + if (i >= argc) + break; + input_arg = argv[i]; + } else { + if (i >= argc + 1) + break; + input_arg = "-"; + } + + rc = set_parser_input(fyp, input_arg, false); + if (rc) { + fprintf(stderr, "failed to set parser input to '%s' for dump\n", input_arg); + goto cleanup; + } + + if (!streaming) { + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + if (!null_output) + rc = fy_emit_document(emit, fyd); + else + rc = 0; + fy_parse_document_destroy(fyp, fyd); + if (rc) + goto cleanup; + + } + } else { + while ((fyev = fy_parser_parse(fyp)) != NULL) { + if (!null_output) { + if (recreating) { + fyeev = NULL; + switch (fyev->type) { + case FYET_STREAM_START: + case FYET_STREAM_END: + case FYET_MAPPING_END: + case FYET_SEQUENCE_END: + fyeev = fy_emit_event_create(emit, fyev->type); + break; + case FYET_DOCUMENT_START: + tags = fy_document_state_tag_directives(fyev->document_start.document_state); + fyeev = fy_emit_event_create(emit, FYET_DOCUMENT_START, + fyev->document_start.implicit, + fy_document_state_version_explicit(fyev->document_start.document_state) + ? fy_document_state_version(fyev->document_start.document_state) + : NULL, + fy_document_state_tags_explicit(fyev->document_start.document_state) + ? tags + : NULL); + if (tags) + free(tags); + break; + case FYET_DOCUMENT_END: + fyeev = fy_emit_event_create(emit, FYET_DOCUMENT_END, + fyev->document_end.implicit); + break; + case FYET_MAPPING_START: + case FYET_SEQUENCE_START: + fyeev = fy_emit_event_create(emit, fyev->type, + fy_event_get_node_style(fyev), + fy_event_get_anchor_token(fyev) + ? fy_token_get_text0(fy_event_get_anchor_token(fyev)) + : NULL, + fy_event_get_tag_token(fyev) + ? fy_tag_token_short0(fy_event_get_tag_token(fyev)) + : NULL); + break; + case FYET_SCALAR: + eevlen = 0; + eevtext = fy_token_get_text(fy_event_get_token(fyev), &eevlen); + if (!eevtext) + goto cleanup; + fyeev = fy_emit_event_create(emit, FYET_SCALAR, + fy_scalar_token_get_style(fy_event_get_token(fyev)), + eevtext, eevlen, + fy_event_get_anchor_token(fyev) + ? fy_token_get_text0(fy_event_get_anchor_token(fyev)) + : NULL, + fy_event_get_tag_token(fyev) + ? fy_tag_token_short0(fy_event_get_tag_token(fyev)) + : NULL); + break; + case FYET_ALIAS: + fyeev = fy_emit_event_create(emit, FYET_ALIAS, + fy_token_get_text0(fy_event_get_token(fyev))); + break; + default: + goto cleanup; + } + fy_parser_event_free(fyp, fyev); + if (fyeev == NULL) { + goto cleanup; + } + + rc = fy_emit_event(emit, fyeev); + } else { + rc = fy_emit_event_from_parser(emit, fyp, fyev); + } + + if (rc) + goto cleanup; + } else { + fy_parser_event_free(fyp, fyev); + } + } + } + + if (fy_parser_get_stream_error(fyp)) + goto cleanup; + } + break; + + case OPT_FILTER: + step = 1; + if (optind >= argc || (argc - optind) % step) { + fprintf(stderr, "illegal arguments\n"); + goto cleanup; + } + + if (!file) + rc = fy_parser_set_input_fp(fyp, "stdin", stdin); + else + rc = set_parser_input(fyp, file, false); + + if (rc) { + fprintf(stderr, "failed to set parser input to %s for filter\n", + file ? file : "stdin"); + goto cleanup; + } + + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + + for (i = optind, j = 0; i < argc; i += step, j++) { + + fyn = fy_node_by_path(fy_document_root(fyd), argv[i], FY_NT, + follow ? FYNWF_FOLLOW : FYNWF_DONT_FOLLOW); + + /* ignore not found paths */ + if (!fyn) { + if (!(cfg.flags & FYPCF_QUIET)) + fprintf(stderr, "filter: could not find '%s'\n", + argv[i]); + continue; + } + + fyn_emit = fyn; + if (!fyn_emit) + continue; + + rc = fy_emit_document_start(emit, fyd, fyn_emit); + if (rc) + goto cleanup; + + rc = fy_emit_root_node(emit, fyn_emit); + if (rc) + goto cleanup; + + rc = fy_emit_document_end(emit); + if (rc) + goto cleanup; + + } + + fy_parse_document_destroy(fyp, fyd); + } + + if (fy_parser_get_stream_error(fyp)) + goto cleanup; + break; + + case OPT_JOIN: + if (optind >= argc) { + fprintf(stderr, "missing yaml file(s) to join\n"); + goto cleanup; + } + + fyd_join = NULL; + for (i = optind; ; i++) { + + if (optind < argc) { + if (i >= argc) + break; + input_arg = argv[i]; + } else { + if (i >= argc + 1) + break; + input_arg = "-"; + } + + rc = set_parser_input(fyp, input_arg, false); + if (rc) { + fprintf(stderr, "failed to set parser input to '%s' for join\n", input_arg); + goto cleanup; + } + + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + + if (!fyd_join) { + fyd_join = fyd; + continue; + } + + fyn_to = fy_node_by_path(fy_document_root(fyd_join), to, FY_NT, + follow ? FYNWF_FOLLOW : FYNWF_DONT_FOLLOW); + if (!fyn_to) { + fprintf(stderr, "unable to find to=%s\n", to); + goto cleanup; + } + + fyn_from = fy_node_by_path(fy_document_root(fyd), from, FY_NT, + follow ? FYNWF_FOLLOW : FYNWF_DONT_FOLLOW); + + if (!fyn_from) { + fprintf(stderr, "unable to find from=%s\n", from); + goto cleanup; + } + + rc = fy_node_insert(fyn_to, fyn_from); + if (rc) { + fprintf(stderr, "fy_node_insert() failed\n"); + goto cleanup; + } + + fy_document_destroy(fyd); + } + + if (fy_parser_get_stream_error(fyp)) + goto cleanup; + } + + /* perform the resolution at the end */ + if (join_resolve) { + rc = fy_document_resolve(fyd_join); + if (rc) + goto cleanup; + } + + fyn_emit = fy_node_by_path(fy_document_root(fyd_join), trim, FY_NT, + follow ? FYNWF_FOLLOW : FYNWF_DONT_FOLLOW); + + /* nothing to output ? */ + if (!fyn_emit) { + if (!(cfg.flags & FYPCF_QUIET)) + fprintf(stderr, "warning: empty document\n"); + } + + rc = fy_emit_document_start(emit, fyd_join, fyn_emit); + if (rc) + goto cleanup; + + rc = fy_emit_root_node(emit, fyn_emit); + if (rc) + goto cleanup; + + rc = fy_emit_document_end(emit); + if (rc) + goto cleanup; + + break; + + case OPT_YPATH: + step = 1; + if ((argc - optind) < 1) { + fprintf(stderr, "missing path expression\n"); + goto cleanup; + } + + memset(&pcfg, 0, sizeof(pcfg)); + pcfg.diag = diag; + + i = optind++; + expr = fy_path_expr_build_from_string(&pcfg, argv[i], FY_NT); + if (!expr) { + fprintf(stderr, "failed to parse path expression %s\n", argv[i]); + goto cleanup; + } + + if (dump_pathexpr) { + struct fy_document *fyd_pe; + + fy_path_expr_dump(expr, diag, FYET_ERROR, 0, "ypath expression:"); + + fyd_pe = fy_path_expr_to_document(expr); + if (!fyd_pe) { + fprintf(stderr, "failed to convert path expression to document\n"); + goto cleanup; + } + fy_emit_document(emit, fyd_pe); + + fy_document_destroy(fyd_pe); + } + + /* nothing more */ + if (noexec) { + exitcode = EXIT_SUCCESS; + goto cleanup; + } + + memset(&xcfg, 0, sizeof(xcfg)); + xcfg.diag = diag; + + fypx = fy_path_exec_create(&xcfg); + if (!fypx) { + fprintf(stderr, "failed to create a path executor\n"); + goto cleanup; + } + + /* if no more arguments use stdin */ + if (optind >= argc) { + rc = fy_parser_set_input_fp(fyp, "stdin", stdin); + if (rc) { + fprintf(stderr, "failed to set parser input to %s for ypath\n", + "stdin"); + goto cleanup; + } + stdin_input = true; + } else + stdin_input = false; + + for (;;) { + + if (!stdin_input) { + i = optind++; + rc = fy_parser_set_input_file(fyp, argv[i]); + if (rc) { + fprintf(stderr, "failed to set parser input to %s for ypath\n", + argv[i]); + goto cleanup; + } + } + + fy_path_exec_reset(fypx); + + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + + fyn_start = fy_node_by_path(fy_document_root(fyd), from, FY_NT, + follow ? FYNWF_FOLLOW : FYNWF_DONT_FOLLOW); + + /* if from is not found, then it's a null document */ + if (!fyn_start) { + if (!(cfg.flags & FYPCF_QUIET)) + fprintf(stderr, "filter: could not find starting point'%s'\n", + from); + continue; + } + + rc = fy_path_exec_execute(fypx, expr, fyn_start); + if (rc) { + fprintf(stderr, "failed to fy_path_exec_execute() - %d\n", rc); + goto cleanup; + } + + res_iter = NULL; + while ((fyn_emit = fy_path_exec_results_iterate(fypx, &res_iter)) != NULL) { + + rc = fy_emit_document_start(emit, fyd, fyn_emit); + if (rc) + goto cleanup; + + rc = fy_emit_root_node(emit, fyn_emit); + if (rc) + goto cleanup; + + rc = fy_emit_document_end(emit); + if (rc) + goto cleanup; + } + + fy_path_exec_reset(fypx); + + fy_parse_document_destroy(fyp, fyd); + } + + if (optind >= argc) + break; + + } + + if (fy_parser_get_stream_error(fyp)) + goto cleanup; + break; + + case OPT_SCAN_DUMP: + case OPT_PARSE_DUMP: + if (optind >= argc) { + fprintf(stderr, "missing yaml file to %s-dump\n", + tool_mode == OPT_SCAN_DUMP ? "scan" : "dump"); + goto cleanup; + } + + for (i = optind; i < argc; i++) { + rc = set_parser_input(fyp, argv[i], false); + if (rc) { + fprintf(stderr, "failed to set parser input to '%s' for dump\n", argv[i]); + goto cleanup; + } + + if (tool_mode == OPT_SCAN_DUMP) { + while ((fyt = fy_scan(fyp)) != NULL) { + dump_scan_token(fyp, fyt, dcfg.colorize); + fy_scan_token_free(fyp, fyt); + } + } else { + while ((fyev = fy_parser_parse(fyp)) != NULL) { + dump_parse_event(fyp, fyev, dcfg.colorize); + fy_parser_event_free(fyp, fyev); + } + } + + if (fy_parser_get_stream_error(fyp)) + goto cleanup; + } + break; + + case OPT_COMPOSE: + if (optind >= argc) { + fprintf(stderr, "missing yaml file to dump\n"); + goto cleanup; + } + + memset(&cd, 0, sizeof(cd)); + cd.fyp = fyp; + cd.emit = emit; + cd.null_output = null_output; + cd.single_document = false; + cd.verbose = dump_path; + + for (i = optind; i < argc; i++) { + rc = set_parser_input(fyp, argv[i], false); + if (rc) { + fprintf(stderr, "failed to set parser input to '%s' for dump\n", argv[i]); + goto cleanup; + } + } + + /* ignore errors for now */ + rc = fy_parse_compose(fyp, compose_process_event, &cd); + + /* NULL OK */ + fy_document_destroy(cd.fyd); + + if (rc || fy_parser_get_stream_error(fyp)) + goto cleanup; + + break; + + + } + exitcode = EXIT_SUCCESS; + +cleanup: + if (fypx) + fy_path_exec_destroy(fypx); + + if (expr) + fy_path_expr_free(expr); + + if (fyd_join) + fy_document_destroy(fyd_join); + + if (fyd_ins) { + for (j = 0; j < count_ins; j++) + fy_document_destroy(fyd_ins[j]); + } + + if (emit) + fy_emitter_destroy(emit); + + if (fyp) + fy_parser_destroy(fyp); + + if (diag) { + if (collect_errors) { + struct fy_diag_error *err; + void *iter; + + iter = NULL; + while ((err = fy_diag_errors_iterate(diag, &iter)) != NULL) { + fprintf(stderr, "%s:%d:%d %s\n", err->file, err->line, err->column, err->msg); + } + + } + fy_diag_destroy(diag); + } + + /* make valgrind happy */ + fy_shutdown(); + + return exitcode; + +err_out_usage: + exitcode = EXIT_FAILURE; + display_usage(stderr, progname, tool_mode); + goto cleanup; +} diff --git a/src/util/fy-align.h b/src/util/fy-align.h new file mode 100644 index 0000000..1e7e180 --- /dev/null +++ b/src/util/fy-align.h @@ -0,0 +1,95 @@ +/* + * fy-align.h - various align related stuff + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_ALIGN_H +#define FY_ALIGN_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +#define FY_ALIGNED_TO(x) __attribute__ ((aligned(x))) +#elif defined(_MSC_VER) +#define FY_ALIGNED_TO(x) __declspec(align(x)) +#else +#define FY_ALIGNED_TO(x) /* nothing */ +#endif + +#define FY_ALIGN(_align, _x) (((_x) + ((_align) - 1)) & ~((_align) - 1)) + +/* cachelines are universally 64 bytes */ +#define FY_CACHELINE_SIZE 64 +#define FY_CACHELINE_SIZE_ALIGN(_x) FY_ALIGN(FY_CACHELINE_SIZE, _x) +#define FY_CACHELINE_ALIGN FY_ALIGNED_TO(FY_CACHELINE_SIZE) + +/* provide posix_memalign for platforms that don't have it */ +#ifdef _WIN32 +static inline int posix_memalign(void **ptr, size_t align, size_t size) +{ + void *p; + + /* must be a power of two */ + if ((size & (size -1)) != 0) { + *ptr = NULL; + return EINVAL; + } + + p = _aligned_malloc(size, align); + if (!p) { + *ptr = NULL; + return ENOMEM; + } + return p; +} + +static inline void posix_memalign_free(void *p) +{ + _aligned_free(p); +} + +#else +/* normal implementations just use free */ +static inline void posix_memalign_free(void *p) +{ + free(p); +} +#endif + +static inline void *fy_align_alloc(size_t align, size_t size) +{ + void *p; + int rc; + + size = FY_ALIGN(align, size); + rc = posix_memalign(&p, align, size); + if (rc) + return NULL; + return p; +} + +static inline void fy_align_free(void *p) +{ + if (p) + posix_memalign_free(p); +} + +static inline void *fy_cacheline_alloc(size_t size) +{ + return fy_align_alloc(FY_CACHELINE_SIZE, size); +} + +static inline void fy_cacheline_free(void *p) +{ + fy_align_free(p); +} + +#endif diff --git a/src/util/fy-atomics.h b/src/util/fy-atomics.h new file mode 100644 index 0000000..bfc177f --- /dev/null +++ b/src/util/fy-atomics.h @@ -0,0 +1,202 @@ +/* + * fy-atomic.h - fy atomics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_ATOMIC_H +#define FY_ATOMIC_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "fy-utils.h" + +/* Detect standard C11 atomics */ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) +#include +#define FY_HAVE_STDATOMIC_H +#define FY_HAVE_C11_ATOMICS +#endif + +/* Detect compiler extensions for _Atomic */ +#if !defined(FY_HAVE_C11_ATOMICS) +#undef FY_HAVE_STDATOMIC_H + +#if defined(__GNUC__) && !defined(__STRICT_ANSI__) +#define FY_HAVE_C11_ATOMICS +#elif defined(__clang__) && defined(__has_extension) +#if __has_extension(c_atomic) +#define FY_HAVE_C11_ATOMICS +#endif +#endif + +#endif + +/* Decide macro */ +#ifdef FY_HAVE_C11_ATOMICS +#define FY_HAVE_ATOMICS +#else +#undef FY_HAVE_ATOMICS +#define _Atomic(_x) _x +#endif + +#if defined(FY_HAVE_STDATOMIC_H) +#define FY_HAVE_SAFE_ATOMIC_OPS +#else + +#undef FY_HAVE_SAFE_ATOMIC_OPS + +typedef bool atomic_flag; + +#define atomic_load(_ptr) \ + (*(_ptr)) + +#define atomic_store(_ptr, _val) \ + do { \ + *(_obj) = (_val); \ + } while(0) + +#define atomic_exchange(_ptr, _v) \ + ({ \ + __typeof__(_ptr) __ptr = (_ptr); \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr = (_v); \ + __old; \ + }) + +#define atomic_compare_exchange_strong(_ptr, _exp, _des) \ + ({ \ + __typeof__(_ptr) __ptr = (_ptr); \ + __typeof__(*(_ptr)) __old = *__ptr; \ + bool __res; \ + if (*__ptr == *(_exp)) { \ + *__ptr = (_des); \ + __res = true; \ + } else \ + *__res = false; \ + __res; \ + }) + +#define atomic_compare_exchange_weak(_ptr, _exp, _des) \ + atomic_compare_exchange_strong((_ptr), (_exp), (_des)) + +#define atomic_fetch_add(_ptr, _v) \ + ({ \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr += (_v); \ + __old; \ + }) + +#define atomic_fetch_sub(_ptr, _v) \ + ({ \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr -= (_v); \ + __old; \ + }) + +#define atomic_fetch_or(_ptr, _v) \ + ({ \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr |= (_v); \ + __old; \ + }) + +#define atomic_fetch_xor(_ptr, _v) \ + ({ \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr ^= (_v); \ + __old; \ + }) + +#define atomic_fetch_and(_ptr, _v) \ + ({ \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr &= (_v); \ + __old; \ + }) + +#define atomic_flag_clear(_ptr) \ + do { \ + *(_ptr) = false; \ + } while(0) + +#define atomic_flag_set(_ptr) \ + do { \ + *(_ptr) = true; \ + } while(0) + + +#define atomic_flag_test_and_set(_ptr) \ + ({ \ + volatile atomic_flag *__ptr = (_ptr); \ + if (!*__ptr) { \ + *__ptr = true; \ + __ret = true; \ + } else \ + __ret = false; \ + __ret; \ + } + +#endif + +/* OK, now define FY version */ +#define FY_ATOMIC(_x) _Atomic(_x) +#define fy_atomic_flag atomic_flag + +#define fy_atomic_load(_ptr) \ + atomic_load((_ptr)) +#define fy_atomic_store(_ptr, _v) \ + atomic_store((_ptr), (_v)) +#define fy_atomic_exchange(_ptr, _v) \ + atomic_exchange((_ptr), (_v)) +#define fy_atomic_compare_exchange_strong(_ptr, _e, _d) \ + atomic_compare_exchange_strong((_ptr), (_e), (_d)) +#define fy_atomic_compare_exchange_weak(_ptr, _e, _d) \ + atomic_compare_exchange_weak((_ptr), (_e), (_d)) +#define fy_atomic_fetch_add(_ptr, _v) \ + atomic_fetch_add((_ptr), (_v)) +#define fy_atomic_fetch_sub(_ptr, _v) \ + atomic_fetch_sub((_ptr), (_v)) +#define fy_atomic_fetch_or(_ptr, _v) \ + atomic_fetch_or((_ptr), (_v)) +#define fy_atomic_fetch_xor(_ptr, _v) \ + atomic_fetch_xor((_ptr), (_v)) +#define fy_atomic_fetch_and(_ptr, _v) \ + atomic_fetch_and((_ptr), (_v)) +#define fy_atomic_flag_clear(_ptr) \ + atomic_flag_clear((_ptr)) +#define fy_atomic_flag_set(_ptr) \ + atomic_flag_set((_ptr)) +#define fy_atomic_flag_test_and_set(_ptr) \ + atomic_flag_test_and_set((_ptr)) + +static inline void fy_cpu_relax(void) +{ +#if defined(__x86_64__) || defined(__i386__) + __builtin_ia32_pause(); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + _mm_pause(); +#elif defined(__aarch64__) || defined(__arm__) + __asm__ volatile ("yield"); +#elif defined(__powerpc__) + __asm__ volatile ("or 27,27,27"); +#else + __asm__ volatile ("" : : : "memory"); +#endif +} + +#define fy_atomic_get_and_clear_counter(_ptr) \ + ({ \ + __typeof__(_ptr) __ptr = (_ptr); \ + __typeof__(*__ptr + 0) __v = fy_atomic_load(__ptr); \ + fy_atomic_fetch_sub(__ptr, __v); \ + __v; \ + }) + +#endif diff --git a/src/util/fy-bit64.h b/src/util/fy-bit64.h new file mode 100644 index 0000000..48e2c4a --- /dev/null +++ b/src/util/fy-bit64.h @@ -0,0 +1,98 @@ +/* + * fy-bit64.h - various bit64 methods + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_BIT64_H +#define FY_BIT64_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#define FY_BIT64(x) ((uint64_t)1 << (x)) +#define FY_BIT64_COUNT(_x) (((_x) + 63) / 64) +#define FY_BIT64_SIZE(_x) ((size_t)((((_x) + 63) / 64) * sizeof(uint64_t))) + +#if (defined(__GNUC__) && __GNUC__ >= 4) || defined(__clang__) + +#define FY_BIT64_LOWEST(_x) ((unsigned int)__builtin_ctzll((uint64_t)(_x))) +#define FY_BIT64_HIGHEST(_x) ((unsigned int)__builtin_clzll((uint64_t)(_x))) +#define FY_BIT64_POPCNT(_x) ((unsigned int)__builtin_popcountl((uint64_t)(_x))) +#define FY_BIT64_FFS(_x) ((unsigned int)__builtin_ffsll((uint64_t)(_x))) + +static inline unsigned int fy_bit64_lowest(uint64_t x) +{ + return FY_BIT64_LOWEST(x); +} + +static inline unsigned int fy_bit64_highest(uint64_t x) +{ + return FY_BIT64_HIGHEST(x); +} + +static inline unsigned int fy_bit64_popcnt(uint64_t x) +{ + return FY_BIT64_POPCNT(x); +} + +static inline unsigned int fy_bit64_ffs(uint64_t x) +{ + return FY_BIT64_FFS(x); +} + +#else /* portable implementation */ + +static inline unsigned int fy_bit64_lowest(uint64_t x) +{ + unsigned int c = 0; + + if (!(x & 0x00000000ffffffffULL)) { x >>= 32; c += 32; } + if (!(x & 0x000000000000ffffULL)) { x >>= 16; c += 16; } + if (!(x & 0x00000000000000ffULL)) { x >>= 8; c += 8; } + if (!(x & 0x000000000000000fULL)) { x >>= 4; c += 4; } + if (!(x & 0x0000000000000003ULL)) { x >>= 2; c += 2; } + if (!(x & 0x0000000000000001ULL)) { c += 1; } + return c; +} + +static inline unsigned int fy_bit64_highest(uint64_t x) +{ + unsigned int c = 0; + + if (x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } + if (x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } + if (x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } + if (x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } + if (x & 0x000000000000000cULL) { x >>= 2; c += 2; } + if (x & 0x0000000000000002ULL) { c += 1; } + return c; +} + +static inline unsigned int fy_bit64_popcnt(uint64_t x) +{ + unsigned int count; + + for (count = 0; x; x &= x - 1) + count++; + return count; +} + +static inline unsigned int fy_bit64_ffs(uint64_t x) +{ + return x ? (fy_bit64_lowest(x) + 1) : 0; +} + +#define FY_BIT64_LOWEST(_x) fy_bit64_lowest(_x) +#define FY_BIT64_HIGEST(_x) fy_bit64_highest(_x) +#define FY_BIT64_POPCNT(_x) fy_bit64_popcnt(_x) +#define FY_BIT64_FFS(_x) fy_bit64_ffs(_x) + +#endif + +#endif diff --git a/src/util/fy-blob.c b/src/util/fy-blob.c new file mode 100644 index 0000000..9ed73a9 --- /dev/null +++ b/src/util/fy-blob.c @@ -0,0 +1,143 @@ +/* + * fy-blob.c - binary blob handling + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include + +#include "fy-blob.h" + +/* define optimized methods for probing */ +BR_DEFINE_WX(br_probe_w, 8, false, false); +BR_DEFINE_WX(br_probe_w, 16, false, false); +BR_DEFINE_WX(br_probe_w, 32, false, false); +BR_DEFINE_WX(br_probe_w, 64, false, false); + +/* define optimized methods for writing */ +BR_DEFINE_WX(br_native_w, 8, false, true); +BR_DEFINE_WX(br_native_w, 16, false, true); +BR_DEFINE_WX(br_native_w, 32, false, true); +BR_DEFINE_WX(br_native_w, 64, false, true); + +BR_DEFINE_WX(br_bswap_w, 8, true, true); +BR_DEFINE_WX(br_bswap_w, 16, true, true); +BR_DEFINE_WX(br_bswap_w, 32, true, true); +BR_DEFINE_WX(br_bswap_w, 64, true, true); + +void *fy_blob_read(const char *file, size_t *sizep) +{ + struct stat sb; + void *blob = NULL; + uint8_t *p; + size_t size, left; + ssize_t rdn; + int fd = -1, rc; + + if (!file || !sizep) + goto err_out; + + fd = open(file, O_RDONLY); + if (fd < 0) + goto err_out; + + rc = fstat(fd, &sb); + if (rc < 0) + goto err_out; + + size = sb.st_size; + blob = malloc(size); + if (!blob) + goto err_out; + + p = blob; + left = size; + while (left > 0) { + do { + rdn = read(fd, p, left); + } while (rdn == -1 && errno == EAGAIN); + if (rdn < 0) + goto err_out; + p += rdn; + left -= rdn; + } + close(fd); + + *sizep = size; + return blob; + +err_out: + if (blob) + free(blob); + if (fd >= 0) + close(fd); + return NULL; +} + +int fy_blob_write(const char *file, const void *blob, size_t size) +{ + int fd = -1; + const uint8_t *p; + size_t left; + ssize_t wrn; + + if (!file || !blob) + return -1; + + fd = open(file, O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IRGRP | S_IROTH); + if (fd < 0) + goto err_out; + + p = blob; + left = size; + while (left > 0) { + do { + wrn = write(fd, p, left); + } while (wrn == -1 && errno == EAGAIN); + if (wrn < 0) + goto err_out; + p += wrn; + left -= wrn; + } + close(fd); + + return 0; + +err_out: + if (fd >= 0) + close(fd); + return -1; +} + +size_t br_wstr(struct blob_region *br, bool dedup, const char *str) +{ + const char *p, *e, *pt; + size_t tlen, len; + + len = strlen(str); + + /* search in the string table for duplicates */ + if (dedup && br->wstart) { + p = (const char *)br->wstart; + e = p + br->curr; + while (p < e) { + tlen = strlen(p); + if (tlen >= len) { + pt = p + (tlen - len); + if (!memcmp(pt, str, len)) + return (size_t)(pt - (const char *)br->wstart); + } + p += tlen + 1; + } + } + return br_write(br, str, len + 1); +} diff --git a/src/util/fy-blob.h b/src/util/fy-blob.h new file mode 100644 index 0000000..28631d6 --- /dev/null +++ b/src/util/fy-blob.h @@ -0,0 +1,430 @@ +/* + * fy-blob.h - binary blob handling support + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_BLOB_H +#define FY_BLOB_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#ifdef HAVE_BYTESWAP_H +#include +#else +#ifdef HAVE___BUILTIN_BSWAP16 +#define bswap_16(value) __builtin_bswap16(value) +#else +#define bswap_16(value) ((((value)&0xff) << 8) | ((value) >> 8)) +#endif + +#ifdef HAVE___BUILTIN_BSWAP32 +#define bswap_32(value) __builtin_bswap32(value) +#else +#define bswap_32(value) \ + (((uint32_t)bswap_16((uint16_t)((value)&0xffff)) << 16) | \ + (uint32_t)bswap_16((uint16_t)((value) >> 16))) +#endif + +#ifdef HAVE___BUILTIN_BSWAP64 +#define bswap_64(value) __builtin_bswap64(value) +#else +#define bswap_64(value) \ + (((uint64_t)bswap_32((uint32_t)((value)&0xffffffff)) << 32) | \ + (uint64_t)bswap_32((uint32_t)((value) >> 32))) +#endif +#endif + +#include +#include + +#include "fy-utils.h" +#include "fy-endian.h" + +/* special unaligned types for pointer accesses */ + +#if defined(__GNUC__) && __GNUC__ >= 4 +typedef union { uint64_t v; } __attribute__((__packed__)) br_64; +typedef union { uint32_t v; } __attribute__((__packed__)) br_32; +typedef union { uint16_t v; } __attribute__((__packed__)) br_16; +typedef union { uint8_t v; } br_8; +#elif defined(_MSC_VER) +#pragma pack(push, 1) +typedef union { uint64_t v; } br_64; +typedef union { uint32_t v; } br_32; +typedef union { uint16_t v; } br_16; +typedef union { uint8_t v; } br_8; +#pragma pack(pop) +#else +#error Unsupported compiler +#endif + +enum blob_id_size { + BID_U8, + BID_U16, + BID_U32, + BID_U64, +}; + +static inline enum blob_id_size +blob_count_to_id_size(uintmax_t count) +{ + if (count <= (1LLU << 8)) + return BID_U8; + if (count <= (1LLU << 16)) + return BID_U16; + if (count <= (1LLU << 32)) + return BID_U32; + return BID_U64; +} + +static inline unsigned int +blob_id_size_to_byte_size(enum blob_id_size size) +{ + return 1U << size; +} + +static inline unsigned int +blob_id_size_to_bit_size(enum blob_id_size size) +{ + return blob_id_size_to_byte_size(size) * 8; +} + +enum blob_endian_type { + BET_NATIVE_ENDIAN, + BET_LITTLE_ENDIAN, + BET_BIG_ENDIAN +}; + +struct blob_region { + union { + uint8_t *wstart; + const uint8_t *rstart; + }; + size_t size; + enum blob_endian_type endian; + bool bswap; + size_t curr; +}; + +#define BR_DEFINE_WX(_pfx, _bits, _bswap, _write) \ +size_t _pfx ## _bits (struct blob_region *br, uint ## _bits ## _t v) \ +{ \ + size_t pos; \ +\ + if (_write) { \ + if (_bswap) \ + v = bswap_ ## _bits (v); \ + ((br_ ## _bits *)(br->wstart + br->curr))->v = v; \ + } \ + pos = br->curr; \ + br->curr += sizeof(v); \ + return pos; \ +} \ +struct __useless_struct_to_allow_semicolon + +#define BR_DEFINE_RX(_pfx, _bits, _bswap) \ +uint ## _bits ## _t _pfx ## _bits (struct blob_region *br) \ +{ \ + uint ## _bits ## _t v; \ +\ + v = ((const br_ ## _bits *)(br->rstart + br->curr))->v; \ + if (_bswap) \ + v = bswap_ ## _bits (v); \ + br->curr += sizeof(v); \ + return v; \ +} \ +struct __useless_struct_to_allow_semicolon + +static inline void br_setup_common(struct blob_region *br, size_t size, enum blob_endian_type endian) +{ + memset(br, 0, sizeof(*br)); + br->size = size; + br->endian = endian; + + switch (br->endian) { + case BET_NATIVE_ENDIAN: + /* native endian never needs swap */ + br->bswap = false; + break; + + case BET_LITTLE_ENDIAN: +#if __BYTE_ORDER == __LITTLE_ENDIAN + br->bswap = false; +#else + br->bswap = true; +#endif + break; + + case BET_BIG_ENDIAN: +#if __BYTE_ORDER == __LITTLE_ENDIAN + br->bswap = true; +#else + br->bswap = false; +#endif + break; + } + + br->curr = 0; +} + +static inline void br_wsetup(struct blob_region *br, void *data, size_t size, enum blob_endian_type endian) +{ + /* write with null data means probe */ + br_setup_common(br, size, endian); + br->wstart = data; +} + +static inline void br_rsetup(struct blob_region *br, const void *data, size_t size, enum blob_endian_type endian) +{ + /* read makes no sense without data */ + br_setup_common(br, size, endian); + br->rstart = data; +} + +static inline void br_reset(struct blob_region *br) +{ + br->curr = 0; +} + +static inline size_t br_curr(struct blob_region *br) +{ + return br->curr; +} + +static inline size_t br_write(struct blob_region *br, const void *data, size_t size) +{ + size_t pos; + + if (br->wstart) + memcpy(br->wstart + br->curr, data, size); + pos = br->curr; + br->curr += size; + return pos; +} + +static inline size_t br_w0(struct blob_region *br, size_t size) +{ + size_t pos; + + if (br->wstart) + memset(br->wstart + br->curr, 0, size); + pos = br->curr; + br->curr += size; + return pos; +} + +static inline size_t br_wskip(struct blob_region *br, size_t size) +{ + return br_w0(br, size); +} + +static inline size_t br_wskip_to(struct blob_region *br, size_t offset) +{ + return br_w0(br, offset - br->curr); +} + +static inline BR_DEFINE_WX(br_w, 8, br->bswap, br->wstart != NULL); +static inline BR_DEFINE_WX(br_w, 16, br->bswap, br->wstart != NULL); +static inline BR_DEFINE_WX(br_w, 32, br->bswap, br->wstart != NULL); +static inline BR_DEFINE_WX(br_w, 64, br->bswap, br->wstart != NULL); + +typedef size_t (*br_wid_func)(struct blob_region *br, int id); + +static inline size_t br_wid8(struct blob_region *br, int id) +{ + return br_w8(br, (uint8_t)id); +} + +static inline size_t br_wid16(struct blob_region *br, int id) +{ + return br_w16(br, (uint16_t)id); +} + +static inline size_t br_wid32(struct blob_region *br, int id) +{ + return br_w32(br, (uint32_t)id); +} + +static inline size_t br_wid64(struct blob_region *br, int id) +{ + return br_w64(br, (uint64_t)id); +} + +static inline br_wid_func br_wid_get_func(enum blob_id_size id_size) +{ + switch (id_size) { + case BID_U8: + return br_wid8; + + case BID_U16: + return br_wid16; + + case BID_U32: + return br_wid32; + + case BID_U64: + return br_wid64; + } + return NULL; +} + +/* write a string with optional dedup */ +size_t br_wstr(struct blob_region *br, bool dedup, const char *str); + +static inline size_t br_wX(struct blob_region *br, enum blob_id_size x_size, uint64_t x) +{ + switch (x_size) { + case BID_U8: + return br_w8(br, (uint8_t)x); + + case BID_U16: + return br_w16(br, (uint16_t)x); + + case BID_U32: + return br_w32(br, (uint32_t)x); + + case BID_U64: + return br_w64(br, x); + } + return (size_t)-1; +} + +static inline size_t br_wid(struct blob_region *br, enum blob_id_size id_size, int id) +{ + switch (id_size) { + case BID_U8: + return br_wid8(br, id); + + case BID_U16: + return br_wid16(br, id); + + case BID_U32: + return br_wid32(br, id); + + case BID_U64: + return br_wid64(br, id); + } + return br->curr; +} + +static inline size_t br_read(struct blob_region *br, void *data, size_t size) +{ + size_t pos; + + memcpy(data, br->rstart + br->curr, size); + pos = br->curr; + br->curr += size; + return pos; +} + +static inline void br_rskip(struct blob_region *br, size_t size) +{ + br->curr += size; +} + +static inline void br_rskip_to(struct blob_region *br, size_t offset) +{ + br->curr = offset; +} + +static inline BR_DEFINE_RX(br_r, 8, br->bswap); +static inline BR_DEFINE_RX(br_r, 16, br->bswap); +static inline BR_DEFINE_RX(br_r, 32, br->bswap); +static inline BR_DEFINE_RX(br_r, 64, br->bswap); + +typedef int (*br_rid_func)(struct blob_region *br); + +static inline int br_rid8(struct blob_region *br) +{ + return (int)br_r8(br); +} + +static inline int br_rid16(struct blob_region *br) +{ + return (int)br_r16(br); +} + +static inline int br_rid32(struct blob_region *br) +{ + return (int)br_r32(br); +} + +static inline int br_rid64(struct blob_region *br) +{ + return (int)br_r64(br); +} + +static inline br_rid_func br_rid_get_func(enum blob_id_size id_size) +{ + switch (id_size) { + case BID_U8: + return br_rid8; + + case BID_U16: + return br_rid16; + + case BID_U32: + return br_rid32; + + case BID_U64: + return br_rid64; + } + return NULL; +} + +static inline uint64_t br_rX(struct blob_region *br, enum blob_id_size x_size) +{ + switch (x_size) { + case BID_U8: + return br_r8(br); + + case BID_U16: + return br_r16(br); + + case BID_U32: + return br_r32(br); + + case BID_U64: + return br_r64(br); + } + FY_IMPOSSIBLE_ABORT(); +} + +static inline int br_rid(struct blob_region *br, enum blob_id_size id_size) +{ + int id; + + switch (id_size) { + case BID_U8: + id = br_rid8(br); + break; + + case BID_U16: + id = br_rid16(br); + break; + + case BID_U32: + id = br_rid32(br); + break; + + case BID_U64: + id = br_rid64(br); + break; + + default: + id = -1; + FY_IMPOSSIBLE_ABORT(); + } + return id; +} + +void *fy_blob_read(const char *file, size_t *sizep); +int fy_blob_write(const char *file, const void *blob, size_t size); + +#endif diff --git a/src/util/fy-ctype.c b/src/util/fy-ctype.c new file mode 100644 index 0000000..1c9463f --- /dev/null +++ b/src/util/fy-ctype.c @@ -0,0 +1,67 @@ +/* + * fy-ctype.c - ctype utilities + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + * + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include "fy-ctype.h" + +const char *fy_uri_esc(const char *s, size_t len, uint8_t *code, int *code_len) +{ + const char *e = s + len; + int j, k, width; + uint8_t octet; + char c; + + width = 0; + k = 0; + do { + /* check for enough space for %XX */ + if ((e - s) < 3) + return NULL; + + /* if more than one run, expect '%' */ + if (s[0] != '%') + return NULL; + + octet = 0; + for (j = 0; j < 2; j++) { + c = s[1 + j]; + octet <<= 4; + if (c >= '0' && c <= '9') + octet |= c - '0'; + else if (c >= 'a' && c <= 'f') + octet |= 10 + c - 'a'; + else + octet |= 10 + c - 'A'; + } + if (!width) { + width = fy_utf8_width_by_first_octet(octet); + if (!width) + return NULL; + k = 0; + } + if (k >= *code_len) + return NULL; + + code[k++] = octet; + + /* skip over the 3 character escape */ + s += 3; + + } while (--width > 0); + + *code_len = k; + + return s; +} + diff --git a/src/util/fy-ctype.h b/src/util/fy-ctype.h new file mode 100644 index 0000000..7c20791 --- /dev/null +++ b/src/util/fy-ctype.h @@ -0,0 +1,363 @@ +/* + * fy-ctype.h - ctype like macros header + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_CTYPE_H +#define FY_CTYPE_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "fy-utf8.h" + +enum fy_lb_mode { + fylb_cr_nl, /* only \r, \n (json + >= yaml1.2 */ + fylb_cr_nl_N_L_P, /* NEL/LS/PS (yaml1.1) */ +}; + +enum fy_flow_ws_mode { + fyfws_space_tab, /* space + TAB (yaml) */ + fyfws_space, /* only space (json) */ +}; + +static inline bool fy_is_first_alpha(const int c) +{ + return (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + c == '_'; +} + +static inline bool fy_is_alpha(const int c) +{ + return fy_is_first_alpha(c) || c == '-'; +} + +static inline bool fy_is_num(const int c) +{ + return c >= '0' && c <= '9'; +} + +static inline bool fy_is_digit(const int c) +{ + return c >= '0' && c <= '9'; +} + +static inline bool fy_is_xdigit(const int c) +{ + return fy_is_digit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + +static inline bool fy_is_first_alnum(const int c) +{ + return fy_is_first_alpha(c); +} + +static inline bool fy_is_alnum(const int c) +{ + return fy_is_alpha(c) || fy_is_num(c); +} + +static inline bool fy_is_space(const int c) +{ + return fy_utf8_is_space(c); +} + +static inline bool fy_is_tab(const int c) +{ + return fy_utf8_is_tab(c); +} + +static inline bool fy_is_ws(const int c) +{ + return fy_utf8_is_ws(c); +} + +static inline bool fy_is_hex(const int c) +{ + return fy_utf8_is_hex(c); +} + +static inline bool fy_is_flow_indicator(const int c) +{ + return fy_utf8_is_flow_indicator(c); +} + +static inline bool fy_is_lb_r_n(const int c) +{ + return fy_utf8_is_lb(c); +} + +static inline bool fy_is_blank(const int c) +{ + return fy_utf8_is_ws(c); +} + +static inline bool fy_is_blank_lb_r_n(const int c) +{ + return fy_utf8_is_ws_lb(c); +} + +static inline bool fy_is_uri(const int c) +{ + return fy_is_alnum(c) || fy_utf8_strchr(";/?:@&=+$,.!~*\'()[]%", c); +} + +static inline bool fy_is_lb_NEL(const int c) +{ + return c == 0x85; +} + +static inline bool fy_is_lb_LS_PS(const int c) +{ + return c == 0x2028 || c == 0x2029; +} + +static inline bool fy_is_unicode_lb(const int c) +{ + /* note that YAML1.1 supports NEL #x85, LS #x2028 and PS #x2029 as linebreaks */ + /* YAML1.2 and higher does not */ + return fy_is_lb_NEL(c) || fy_is_lb_LS_PS(c); +} + +static inline bool fy_is_any_lb(const int c) +{ + return fy_is_lb_r_n(c) || fy_is_unicode_lb(c); +} + +static inline bool fy_is_z(const int c) +{ + return c <= 0; +} + +#define FY_UTF8_BOM 0xfeff + +static inline bool fy_is_print(const int c) +{ + return c == '\n' || c == '\r' || + (c >= 0x0020 && c <= 0x007e) || + (c >= 0x00a0 && c <= 0xd7ff) || + (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM); +} + +static inline bool fy_is_printq(const int c) +{ + return c != '\t' && c != 0xa0 && !fy_is_any_lb(c) && fy_is_print(c); +} + +static inline bool fy_is_nb_char(const int c) +{ + return (c >= 0x0020 && c <= 0x007e) || + (c >= 0x00a0 && c <= 0xd7ff) || + (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM); +} + +static inline bool fy_is_ns_char(const int c) +{ + return fy_is_nb_char(c) && !fy_is_ws(c); +} + +static inline bool fy_is_start_indicator(const int c) +{ + return !!fy_utf8_strchr(",[]{}#&*!|>'\"%%@`", c); +} + +static inline bool fy_is_indicator_before_space(const int c) +{ + return !!fy_utf8_strchr("-:?`", c); +} + +static inline bool fy_is_path_flow_scalar_start(const int c) +{ + return c == '\'' || c == '"'; +} + +static inline bool fy_is_path_flow_key_start(const int c) +{ + return c == '"' || c == '\'' || c == '{' || c == '['; +} + +static inline bool fy_is_path_flow_key_end(const int c) +{ + return c == '"' || c == '\'' || c == '}' || c == ']'; +} + +static inline bool fy_is_unicode_control(const int c) +{ + return (c >= 0 && c <= 0x1f) || (c >= 0x80 && c <= 0x9f); +} + +static inline bool fy_is_unicode_space(const int c) +{ + return c == 0x20 || c == 0xa0 || + (c >= 0x2000 && c <= 0x200a) || + c == 0x202f || c == 0x205f || c == 0x3000; +} + +static inline bool fy_is_json_unescaped(const int c) +{ + return c >= 0x20 && c <= 0x110000 && c != '"' && c != '\\'; +} + +static inline bool fy_is_json_unescaped_range_only(const int c) +{ + return c >= 0x20 && c <= 0x110000; +} + +static inline bool fy_is_lb_m(const int c, const enum fy_lb_mode lb_mode) +{ + if (fy_is_lb_r_n(c)) + return true; + return lb_mode == fylb_cr_nl_N_L_P && fy_is_unicode_lb(c); +} + +static inline bool fy_is_generic_lb_m(const int c, const enum fy_lb_mode lb_mode) +{ + if (fy_is_lb_r_n(c)) + return true; + return lb_mode == fylb_cr_nl_N_L_P && fy_is_lb_NEL(c); +} + +static inline bool fy_is_blank_lb_m(const int c, const enum fy_lb_mode lb_mode) +{ + if (fy_is_blank_lb_r_n(c)) + return true; + return lb_mode == fylb_cr_nl_N_L_P && fy_is_unicode_lb(c); +} + +static inline bool fy_is_lbz_m(const int c, const enum fy_lb_mode lb_mode) +{ + return fy_is_lb_m(c, lb_mode) || fy_is_z(c); +} + +static inline bool fy_is_generic_lbz_m(const int c, const enum fy_lb_mode lb_mode) +{ + return fy_is_generic_lb_m(c, lb_mode) || fy_is_z(c); +} + +static inline bool fy_is_blankz_m(const int c, const enum fy_lb_mode lb_mode) +{ + return fy_is_ws(c) || fy_is_lbz_m(c, lb_mode); +} + +static inline bool fy_is_generic_blankz_m(const int c, const enum fy_lb_mode lb_mode) +{ + return fy_is_ws(c) || fy_is_generic_lbz_m(c, lb_mode); +} + +static inline bool fy_is_flow_ws_m(const int c, const enum fy_flow_ws_mode fws_mode) +{ + return fy_is_space(c) || (fws_mode == fyfws_space_tab && fy_is_tab(c)); +} + +#define FY_CTYPE_AT_BUILDER(_kind) \ +static inline const void * \ +fy_find_ ## _kind (const void *s, size_t len) \ +{ \ + const void *e = s + len; \ + int c, w; \ + for (; s < e && (c = fy_utf8_get(s, e - s, &w)) >= 0; s += w) { \ + assert(w); \ + if (fy_is_ ## _kind (c)) \ + return s; \ + } \ + return NULL; \ +} \ +static inline const void * \ +fy_find_non_ ## _kind (const void *s, size_t len) \ +{ \ + const void *e = s + len; \ + int c, w; \ + for (; s < e && (c = fy_utf8_get(s, e - s, &w)) >= 0; s += w) { \ + assert(w); \ + if (!(fy_is_ ## _kind (c))) \ + return s; \ + assert(w); \ + } \ + return NULL; \ +} \ +struct useless_struct_for_semicolon + +FY_CTYPE_AT_BUILDER(first_alpha); +FY_CTYPE_AT_BUILDER(alpha); +FY_CTYPE_AT_BUILDER(num); +FY_CTYPE_AT_BUILDER(digit); +FY_CTYPE_AT_BUILDER(xdigit); +FY_CTYPE_AT_BUILDER(first_alnum); +FY_CTYPE_AT_BUILDER(alnum); +FY_CTYPE_AT_BUILDER(space); +FY_CTYPE_AT_BUILDER(tab); +FY_CTYPE_AT_BUILDER(ws); +FY_CTYPE_AT_BUILDER(hex); +FY_CTYPE_AT_BUILDER(uri); +FY_CTYPE_AT_BUILDER(z); +FY_CTYPE_AT_BUILDER(any_lb); +FY_CTYPE_AT_BUILDER(blank); +FY_CTYPE_AT_BUILDER(print); +FY_CTYPE_AT_BUILDER(printq); +FY_CTYPE_AT_BUILDER(nb_char); +FY_CTYPE_AT_BUILDER(ns_char); +FY_CTYPE_AT_BUILDER(start_indicator); +FY_CTYPE_AT_BUILDER(indicator_before_space); +FY_CTYPE_AT_BUILDER(flow_indicator); +FY_CTYPE_AT_BUILDER(path_flow_key_start); +FY_CTYPE_AT_BUILDER(path_flow_key_end); +FY_CTYPE_AT_BUILDER(unicode_control); +FY_CTYPE_AT_BUILDER(unicode_space); +FY_CTYPE_AT_BUILDER(json_unescaped); + +/* + * Very special linebreak/ws methods + * Things get interesting due to \r\n and + * unicode linebreaks/spaces + */ + +/* skip for a _single_ linebreak */ +static inline const void *fy_skip_lb(const void *ptr, size_t left) +{ + int c, width; + + /* get the utf8 character at this point */ + c = fy_utf8_get(ptr, left, &width); + if (c < 0 || !fy_is_any_lb(c)) + return NULL; + + /* MS-DOS: check if next character is '\n' */ + if (c == '\r' && left > (size_t)width && *(char *)ptr == '\n') + width++; + + return ptr + width; +} + +/* given a pointer to a chunk of memory, return pointer to first + * ws character after the last non-ws character, or the end + * of the chunk + */ +static inline const void *fy_last_non_ws(const void *ptr, size_t left) +{ + const char *s, *e; + int c; + + s = ptr; + e = s + left; + while (e > s) { + c = e[-1]; + if (c != ' ' && c != '\t') + return e; + e--; + + } + return NULL; +} + +const char *fy_uri_esc(const char *s, size_t len, uint8_t *code, int *code_len); + +#endif diff --git a/src/util/fy-endian.h b/src/util/fy-endian.h new file mode 100644 index 0000000..5778037 --- /dev/null +++ b/src/util/fy-endian.h @@ -0,0 +1,50 @@ +/* + * fy-endian.h - simple system endian header wrapper + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ENDIAN_H +#define FY_ENDIAN_H + +#if defined(__linux__) || defined(__CYGWIN__) || defined(__OpenBSD__) || defined(__GNU__) +# include +#elif defined(__APPLE__) +# include +# include +#elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) +# include +#elif defined(_MSC_VER) +# include +# ifdef __GNUC__ +# include +# endif +#else +# error unsupported platform +#endif + +#if !defined(__BYTE_ORDER) && defined(BYTE_ORDER) +#define __BYTE_ORDER BYTE_ORDER +#endif + +#if !defined(__BIG_ENDIAN) && defined(BIG_ENDIAN) +#define __BIG_ENDIAN BIG_ENDIAN +#endif + +#if !defined(__LITTLE_ENDIAN) && defined(LITTLE_ENDIAN) +#define __LITTLE_ENDIAN LITTLE_ENDIAN +#endif + +#if !defined(__BYTE_ORDER) || !defined(__BIG_ENDIAN) || !defined(__LITTLE_ENDIAN) +# error Platform does not define endian macros +#endif + +/* no-one cares about PDP endian anymore */ + +/* make the macros work for 8 bit too */ +#ifndef bswap_8 +#define bswap_8(x) (x) +#endif + +#endif diff --git a/src/util/fy-id.h b/src/util/fy-id.h new file mode 100644 index 0000000..aae5b9f --- /dev/null +++ b/src/util/fy-id.h @@ -0,0 +1,242 @@ +/* + * fy-id.h - small ID allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ID_H +#define FY_ID_H + +#include +#include +#include +#include + +#include + +#include "fy-bit64.h" +#include "fy-atomics.h" + +typedef uint64_t fy_id_bits_non_atomic; +typedef FY_ATOMIC(fy_id_bits_non_atomic) fy_id_bits; + +#define FY_ID_BITS_SZ ((size_t)(sizeof(fy_id_bits))) +#define FY_ID_BITS_BITS (FY_ID_BITS_SZ * 8) /* yes, every character is 8 bits in 2023 */ +#define FY_ID_BITS_MASK ((uint64_t)(FY_ID_BITS_BITS - 1)) + +#define FY_ID_BITS_ARRAY_COUNT_BITS(_bits) \ + (((_bits) + FY_ID_BITS_MASK) & ~FY_ID_BITS_MASK) + +#define FY_ID_BITS_ARRAY_COUNT(_bits) \ + (FY_ID_BITS_ARRAY_COUNT_BITS(_bits) / FY_ID_BITS_BITS) + +#define FY_ID_OFFSET(_id) ((_id) / FY_ID_BITS_BITS) +#define FY_ID_BIT_MASK(_id) ((uint64_t)1 << ((_id) & FY_ID_BITS_MASK)) + +static inline int fy_id_ffs(const fy_id_bits_non_atomic id_bit) +{ + if (!id_bit) + return -1; + return FY_BIT64_FFS(id_bit) - 1; +} + +static inline int fy_id_popcount(const fy_id_bits_non_atomic id_bit) +{ + return FY_BIT64_POPCNT(id_bit); +} + +static inline void fy_id_reset(fy_id_bits *bits, const size_t count) +{ + size_t i; + + for (i = 0; i < count; i++, bits++) + fy_atomic_store(bits, 0); +} + +static inline int fy_id_alloc(fy_id_bits *bits, const size_t count) +{ + size_t i; + int pos; + fy_id_bits_non_atomic v, new_v; + + for (i = 0; i < count; i++, bits++) { + + for (;;) { + v = fy_atomic_load(bits); + pos = fy_id_ffs(~v); + if (pos < 0) + break; + new_v = v | FY_ID_BIT_MASK(pos); + if (fy_atomic_compare_exchange_strong(bits, &v, new_v)) + return (i * FY_ID_BITS_BITS) + pos; + } + } + return -1; +} + +static inline int fy_id_alloc_fixed(fy_id_bits *bits, const size_t count, const int id) +{ + size_t offset; + fy_id_bits_non_atomic m, v, new_v; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return -1; + bits += offset; + + m = FY_ID_BIT_MASK(id); + v = fy_atomic_load(bits); + if (v & m) + return -1; + new_v = v | m; + if (!fy_atomic_compare_exchange_strong(bits, &v, new_v)) + return -1; + + return id; +} + +static inline bool fy_id_is_used(fy_id_bits *bits, const size_t count, const int id) +{ + fy_id_bits_non_atomic v; + size_t offset; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return false; + bits += offset; + v = fy_atomic_load(bits); + return !!(v & FY_ID_BIT_MASK(id)); +} + +static inline bool fy_id_is_free(fy_id_bits *bits, const size_t count, const int id) +{ + fy_id_bits_non_atomic v; + size_t offset; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return false; + bits += offset; + v = fy_atomic_load(bits); + return !(v & FY_ID_BIT_MASK(id)); +} + +static inline void fy_id_free(fy_id_bits *bits, const size_t count, int id) +{ + size_t offset; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return; + bits += offset; + fy_atomic_fetch_and(bits, ~FY_ID_BIT_MASK(id)); /* no need to check, if bit is free, it is still free */ +} + +static inline void fy_id_set_used(fy_id_bits *bits, const size_t count, const int id) +{ + size_t offset; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return; + bits += offset; + fy_atomic_fetch_or(bits, FY_ID_BIT_MASK(id)); +} + +static inline void fy_id_set_free(fy_id_bits *bits, const size_t count, const int id) +{ + size_t offset; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return; + bits += offset; + fy_atomic_fetch_and(bits, ~FY_ID_BIT_MASK(id)); +} + +static inline size_t fy_id_count_used(fy_id_bits *bits, const size_t count) +{ + fy_id_bits v; + size_t i, popcount; + + popcount = 0; + for (i = 0; i < count; i++, bits++) { + v = fy_atomic_load(bits); + popcount += fy_id_popcount(v); + } + return popcount; +} + +static inline size_t fy_id_count_free(fy_id_bits *bits, const size_t count) +{ + return count - fy_id_count_used(bits, count); +} + +struct fy_id_iter { + const fy_id_bits *s; + fy_id_bits m; +}; + +static inline void fy_id_iter_begin(const fy_id_bits *bits, const size_t count, struct fy_id_iter *iterp) +{ + iterp->s = bits; + iterp->m = ~(fy_id_bits_non_atomic)0; +} + +static inline int fy_id_iter_next(const fy_id_bits *bits, const size_t count, struct fy_id_iter *iterp) +{ + const fy_id_bits *s, *e; + fy_id_bits_non_atomic v, m; + int pos; + + s = iterp->s; + if (!s) + return -1; + + m = iterp->m; + e = bits + count; + + if (s >= e) + return -1; + + /* scan until we find a set bit or hit the end */ + pos = -1; + while (s < e) { + v = fy_atomic_load(s); + pos = fy_id_ffs(v & m); + if (pos >= 0) + break; + s++; + m = ~(fy_id_bits_non_atomic)0; + } + + /* not found */ + if (pos < 0) { + iterp->s = NULL; + iterp->m = 0; + return -1; + } + + /* remove this bit from the mask */ + m &= ~((fy_id_bits_non_atomic)1 << pos); + + /* advance by the number of fy_id_bits we scanned */ + pos += (int)(s - bits) * FY_ID_BITS_BITS; + + /* if we run out of mask, advance */ + if (!m) { + s++; + m = ~(fy_id_bits_non_atomic)0; + } + iterp->s = s; + iterp->m = m; + return pos; +} + +static inline void fy_id_iter_end(const fy_id_bits *bits, const size_t count, const struct fy_id_iter *iterp) +{ + /* nothing */ +} + +#endif diff --git a/src/util/fy-list.h b/src/util/fy-list.h new file mode 100644 index 0000000..892d6cd --- /dev/null +++ b/src/util/fy-list.h @@ -0,0 +1,91 @@ +/* + * fy-list.h - Circular doubly-linked list implementation + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_LIST_H +#define FY_LIST_H + +#include "fy-utils.h" + +struct list_head { + struct list_head *next; + struct list_head *prev; +}; + +static inline void list_init(struct list_head *list) +{ + list->next = list->prev = list; +} + +static inline void list_add(struct list_head *new_item, struct list_head *head) +{ + struct list_head *prev = head; + struct list_head *next = head->next; + + next->prev = new_item; + new_item->next = next; + new_item->prev = prev; + prev->next = new_item; +} + +static inline void list_add_tail(struct list_head *new_item, struct list_head *head) +{ + struct list_head *prev = head->prev; + struct list_head *next = head; + + next->prev = new_item; + new_item->next = next; + new_item->prev = prev; + prev->next = new_item; +} + +static inline void list_del(struct list_head *entry) +{ + struct list_head *prev = entry->prev; + struct list_head *next = entry->next; + + next->prev = prev; + prev->next = next; +} + +static inline int list_empty(const struct list_head *head) +{ + return head->next == head; +} + +static inline int list_is_singular(const struct list_head *head) +{ + return !list_empty(head) && head->next == head->prev; +} + +static inline void list_splice(const struct list_head *list, + struct list_head *head) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + struct list_head *prev = head; + struct list_head *next = head->next; + + if (list_empty(list)) + return; + + first->prev = prev; + prev->next = first; + + last->next = next; + next->prev = last; +} + +#define list_entry(ptr, type, member) \ + container_of(ptr, type, member) + +#define list_first_entry(ptr, type, member) \ + list_entry((ptr)->next, type, member) + +#define list_last_entry(ptr, type, member) \ + list_entry((ptr)->prev, type, member) + +#endif diff --git a/src/util/fy-typelist.h b/src/util/fy-typelist.h new file mode 100644 index 0000000..a560e01 --- /dev/null +++ b/src/util/fy-typelist.h @@ -0,0 +1,159 @@ +/* + * fy-typelist.h - typed list method builders + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_TYPELIST_H +#define FY_TYPELIST_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#include "fy-list.h" + +/* declare type methods */ + +#define FY_TYPE_FWD_DECL_LIST(_type) \ +/* type safe list wrapper */ \ +struct fy_ ## _type ## _list { struct list_head _lh; }; \ +\ +struct __useless_struct_to_allow_semicolon + +#define FY_TYPE_DECL_LIST(_type) \ +static inline void fy_ ## _type ## _list_init(struct fy_ ## _type ## _list *_l) \ +{ \ + list_init(&_l->_lh); \ +} \ +static inline void fy_ ## _type ## _list_add(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (_l && _n) \ + list_add(&_n->node, &_l->_lh); \ +} \ +static inline void fy_ ## _type ## _list_add_tail(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (_l && _n) \ + list_add_tail(&_n->node, &_l->_lh); \ +} \ +static inline void fy_ ## _type ## _list_push(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (_l && _n) \ + fy_ ## _type ## _list_add(_l, _n); \ +} \ +static inline void fy_ ## _type ## _list_push_tail(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (_l && _n) \ + fy_ ## _type ## _list_add_tail(_l, _n); \ +} \ +static inline bool fy_ ## _type ## _list_empty(struct fy_ ## _type ## _list *_l) \ +{ \ + return _l ? list_empty(&_l->_lh) : true; \ +} \ +static inline bool fy_ ## _type ## _list_is_singular(struct fy_ ## _type ## _list *_l) \ +{ \ + return _l ? list_is_singular(&_l->_lh) : true; \ +} \ +static inline void fy_ ## _type ## _list_del(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (_l && _n) \ + list_del(&_n->node); \ +} \ +static inline void fy_ ## _type ## _list_insert_after(struct fy_ ## _type ## _list *_l, \ + struct fy_ ## _type *_p, struct fy_ ## _type *_n) \ +{ \ + if (_l && _p && _n) \ + list_add(&_n->node, &_p->node); \ +} \ +static inline void fy_ ## _type ## _list_insert_before(struct fy_ ## _type ## _list *_l, \ + struct fy_ ## _type *_p, struct fy_ ## _type *_n) \ +{ \ + if (_l && _p && _n) \ + list_add_tail(&_n->node, &_p->node); \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_head(struct fy_ ## _type ## _list *_l) \ +{ \ + return !fy_ ## _type ## _list_empty(_l) ? list_first_entry(&_l->_lh, struct fy_ ## _type, node) : NULL; \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_tail(struct fy_ ## _type ## _list *_l) \ +{ \ + return !fy_ ## _type ## _list_empty(_l) ? list_last_entry(&_l->_lh, struct fy_ ## _type, node) : NULL; \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_first(struct fy_ ## _type ## _list *_l) \ +{ \ + return fy_ ## _type ## _list_head(_l); \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_last(struct fy_ ## _type ## _list *_l) \ +{ \ + return fy_ ## _type ## _list_tail(_l); \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_pop(struct fy_ ## _type ## _list *_l) \ +{ \ + struct fy_ ## _type *_n; \ + \ + _n = fy_ ## _type ## _list_head(_l); \ + if (!_n) \ + return NULL; \ + fy_ ## _type ## _list_del(_l, _n); \ + return _n; \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _list_pop_tail(struct fy_ ## _type ## _list *_l) \ +{ \ + struct fy_ ## _type *_n; \ + \ + _n = fy_ ## _type ## _list_tail(_l); \ + if (!_n) \ + return NULL; \ + fy_ ## _type ## _list_del(_l, _n); \ + return _n; \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _next(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (!_n || !_l || _n->node.next == &_l->_lh) \ + return NULL; \ + return list_entry(_n->node.next, struct fy_ ## _type, node); \ +} \ +static inline struct fy_ ## _type *fy_ ## _type ## _prev(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ +{ \ + if (!_n || !_l || _n->node.prev == &_l->_lh) \ + return NULL; \ + return list_entry(_n->node.prev, struct fy_ ## _type, node); \ +} \ +static inline void fy_ ## _type ## _lists_splice( \ + struct fy_ ## _type ## _list *_l, \ + struct fy_ ## _type ## _list *_lfrom) \ +{ \ + /* check arguments for sanity and lists are not empty */ \ + if (!_l || !_lfrom || \ + fy_ ## _type ## _list_empty(_lfrom)) \ + return; \ + list_splice(&_lfrom->_lh, &_l->_lh); \ +} \ +static inline void fy_ ## _type ## _list_splice_after( \ + struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n, \ + struct fy_ ## _type ## _list *_lfrom) \ +{ \ + /* check arguments for sanity and lists are not empty */ \ + if (!_l || !_n || !_lfrom || \ + fy_ ## _type ## _list_empty(_lfrom)) \ + return; \ + list_splice(&_lfrom->_lh, &_n->node); \ +} \ +static inline void fy_ ## _type ## _list_splice_before( \ + struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n, \ + struct fy_ ## _type ## _list *_lfrom) \ +{ \ + /* check arguments for sanity and lists are not empty */ \ + if (!_l || !_n || !_lfrom || \ + fy_ ## _type ## _list_empty(_lfrom)) \ + return; \ + list_splice(&_lfrom->_lh, _n->node.prev); \ +} \ +struct __useless_struct_to_allow_semicolon + +#endif diff --git a/src/util/fy-utf8.c b/src/util/fy-utf8.c new file mode 100644 index 0000000..228879a --- /dev/null +++ b/src/util/fy-utf8.c @@ -0,0 +1,1058 @@ +/* + * fy-utf8.c - utf8 handling methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#include "fy-utf8.h" + +/* to avoid dragging in libfyaml.h */ +#ifndef FY_BIT +#define FY_BIT(x) (1U << (x)) +#endif + +const int8_t fy_utf8_width_table[32] = { + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, + 2, 2, 2, 2, 3, 3, 4, 0, +}; + +int fy_utf8_get_generic(const void *ptr, size_t left, int *widthp) +{ + const uint8_t *p = ptr; + int i, width, value; + + if (left < 1) + return FYUG_EOF; + + /* this is the slow path */ + width = fy_utf8_width_by_first_octet(p[0]); + if (!width) + return FYUG_INV; + if ((size_t)width > left) + return FYUG_PARTIAL; + + /* initial value */ + value = *p++ & (0xff >> width); + for (i = 1; i < width; i++) { + if ((*p & 0xc0) != 0x80) + return FYUG_INV; + value = (value << 6) | (*p++ & 0x3f); + } + + if (!fy_utf8_is_valid(value)) + return FYUG_INV; + + *widthp = width; + + return value; +} + +int fy_utf8_get_right_generic(const void *ptr, size_t left, int *widthp) +{ + const uint8_t *s, *e; + uint8_t v; + + s = ptr; + e = s + left; + + if (left < 1) + return FYUG_EOF; + + /* single byte sequence */ + v = e[-1]; + if ((v & 0x80) == 0) { + if (widthp) + *widthp = 1; + return (int)v & 0x7f; + } + + /* the last byte must be & 0xc0 == 0x80 */ + if ((v & 0xc0) != 0x80) + return FYUG_INV; + + /* at least two byte sequence */ + if (left < 2) + return FYUG_EOF; + + v = e[-2]; + /* the first byte of the sequence (must be a two byte sequence) */ + if ((v & 0xc0) != 0x80) { + /* two byte start is 110x_xxxx */ + if ((v & 0xe0) != 0xc0) + return FYUG_INV; + return fy_utf8_get(e - 2, 2, widthp); + } + + /* at least three byte sequence */ + if (left < 3) + return FYUG_EOF; + + v = e[-3]; + /* the first byte of the sequence (must be a three byte sequence) */ + if ((v & 0xc0) != 0x80) { + /* three byte start is 1110_xxxx */ + if ((v & 0xf0) != 0xe0) + return FYUG_INV; + return fy_utf8_get(e - 3, 3, widthp); + } + + /* at least four byte sequence */ + if (left < 4) + return FYUG_EOF; + + v = e[-4]; + + /* the first byte of the sequence (must be a four byte sequence) */ + /* four byte start is 1111_0xxx */ + if ((v & 0xf8) != 0xf0) { + return FYUG_INV; + } + return fy_utf8_get(e - 4, 4, widthp); +} + +struct fy_utf8_fmt_esc_map { + const int *ch; + const int *map; +}; + +static const struct fy_utf8_fmt_esc_map esc_all = { + .ch = (const int []){ '\\', '\0', '\b', '\r', '\t', '\f', '\n', '\v', '\a', '\e', 0x85, 0xa0, 0x2028, 0x2029, -1 }, + .map = (const int []){ '\\', '0', 'b', 'r', 't', 'f', 'n', 'v', 'a', 'e', 'N', '_', 'L', 'P', 0 } +}; + +static inline int esc_map(const struct fy_utf8_fmt_esc_map *esc_map, int c) +{ + const int *ch; + int cc; + + ch = esc_map->ch; + while ((cc = *ch++) >= 0) { + if (cc == c) + return esc_map->map[(ch - esc_map->ch) - 1]; + } + return -1; +} + +static inline int fy_utf8_esc_map(int c, enum fy_utf8_escape esc) +{ + if (esc == fyue_none) + return -1; + if (esc == fyue_singlequote && c == '\'') + return '\''; + if (fy_utf8_escape_is_any_doublequote(esc) && c == '"') + return '"'; + return esc_map(&esc_all, c); +} + +size_t fy_utf8_format_text_length(const char *buf, size_t len, + enum fy_utf8_escape esc) +{ + int c, w; + ssize_t l; + const char *s, *e; + + s = buf; + e = buf + len; + l = 0; + while (s < e) { + c = fy_utf8_get(s, e - s, &w); + if (!w || c < 0) + break; + s += w; + if (fy_utf8_esc_map(c, esc)) + w = 2; + l += w; + } + return l + 1; +} + +char *fy_utf8_format_text(const char *buf, size_t len, + char *out, size_t maxsz, + enum fy_utf8_escape esc) +{ + int c, w, cc; + const char *s, *e; + char *os, *oe; + + s = buf; + e = buf + len; + os = out; + oe = out + maxsz - 1; + while (s < e) { + c = fy_utf8_get(s, e - s, &w); + if (!w || c < 0) + break; + s += w; + + if ((cc = fy_utf8_esc_map(c, esc)) > 0) { + if (os + 2 > oe) + break; + *os++ = '\\'; + *os++ = cc; + continue; + } + + if (os + w > oe) + break; + + os = fy_utf8_put_unchecked(os, c); + } + *os++ = '\0'; + return out; +} + +char *fy_utf8_format(int c, char *buf, enum fy_utf8_escape esc) +{ + int cc; + char *s; + + if (!fy_utf8_is_valid(c)) { + *buf = '\0'; + return buf; + } + + s = buf; + if ((cc = fy_utf8_esc_map(c, esc)) > 0) { + *s++ = '\\'; + *s++ = cc; + } else + s = fy_utf8_put_unchecked(s, c); + *s = '\0'; + return buf; +} + +char *fy_utf8_format_text_alloc(const char *buf, size_t len, enum fy_utf8_escape esc) +{ + size_t outsz; + char *out; + + outsz = fy_utf8_format_text_length(buf, len, esc); + if ((ssize_t)outsz < 0) + return NULL; + out = malloc(outsz); + if (!out) + return NULL; + fy_utf8_format_text(buf, len, out, outsz, esc); + + return out; +} + +const void *fy_utf8_memchr_generic(const void *s, int c, size_t n) +{ + int cc, w; + const void *e; + + e = s + n; + while (s < e && (cc = fy_utf8_get(s, e - s, &w)) >= 0) { + if (c == cc) + return s; + s += w; + } + + return NULL; +} + +/* parse an escape and return utf8 value */ +int fy_utf8_parse_escape(const char **strp, size_t len, enum fy_utf8_escape esc) +{ + const char *s, *e; + char c; + int i, value, code_length, cc, w; + unsigned int hi_surrogate, lo_surrogate; + + /* why do you bother us? */ + if (esc == fyue_none) + return -1; + + if (!strp || !*strp || len < 2) + return -1; + + value = -1; + + s = *strp; + e = s + len; + + c = *s++; + + if (esc == fyue_singlequote) { + if (c != '\'') + goto out; + c = *s++; + if (c != '\'') + goto out; + + value = '\''; + goto out; + } + + /* get '\\' */ + if (c != '\\') + goto out; + + c = *s++; + + /* common YAML & JSON escapes */ + switch (c) { + case 'b': + value = '\b'; + break; + case 'f': + value = '\f'; + break; + case 'n': + value = '\n'; + break; + case 'r': + value = '\r'; + break; + case 't': + value = '\t'; + break; + case '"': + value = '"'; + break; + case '/': + value = '/'; + break; + case '\\': + value = '\\'; + break; + default: + break; + } + + if (value >= 0) + goto out; + + if (esc == fyue_doublequote || esc == fyue_doublequote_yaml_1_1) { + switch (c) { + case '0': + value = '\0'; + break; + case 'a': + value = '\a'; + break; + case '\t': + value = '\t'; + break; + case 'v': + value = '\v'; + break; + case 'e': + value = '\e'; + break; + case ' ': + value = ' '; + break; + case 'N': + value = 0x85; /* NEL */ + break; + case '_': + value = 0xa0; + break; + case 'L': + value = 0x2028; /* LS */ + break; + case 'P': /* PS 0x2029 */ + value = 0x2029; /* PS */ + break; + default: + /* weird unicode escapes */ + if ((uint8_t)c >= 0x80) { + /* in non yaml-1.1 mode we don't allow this craziness */ + if (esc == fyue_doublequote) + goto out; + + cc = fy_utf8_get(s - 1, e - (s - 1), &w); + switch (cc) { + case 0x2028: + case 0x2029: + case 0x85: + case 0xa0: + value = cc; + break; + default: + break; + } + } + break; + } + if (value >= 0) + goto out; + } + + /* finally try the unicode escapes */ + code_length = 0; + + if (esc == fyue_doublequote || esc == fyue_doublequote_yaml_1_1) { + switch (c) { + case 'x': + code_length = 2; + break; + case 'u': + code_length = 4; + break; + case 'U': + code_length = 8; + break; + default: + return -1; + } + } else if (esc == fyue_doublequote_json && c == 'u') + code_length = 4; + + if (!code_length || code_length > (e - s)) + goto out; + + value = 0; + for (i = 0; i < code_length; i++) { + c = *s++; + value <<= 4; + if (c >= '0' && c <= '9') + value |= c - '0'; + else if (c >= 'a' && c <= 'f') + value |= 10 + c - 'a'; + else if (c >= 'A' && c <= 'F') + value |= 10 + c - 'A'; + else + goto out; + } + + /* hi/lo surrogate pair */ + if (code_length == 4 && value >= 0xd800 && value <= 0xdbff && + (e - s) >= 6 && s[0] == '\\' && s[1] == 'u') { + hi_surrogate = value; + + s += 2; + + value = 0; + for (i = 0; i < code_length; i++) { + c = *s++; + value <<= 4; + if (c >= '0' && c <= '9') + value |= c - '0'; + else if (c >= 'a' && c <= 'f') + value |= 10 + c - 'a'; + else if (c >= 'A' && c <= 'F') + value |= 10 + c - 'A'; + else + return -1; + } + lo_surrogate = value; + value = 0x10000 + (hi_surrogate - 0xd800) * 0x400 + (lo_surrogate - 0xdc00); + } + +out: + *strp = s; + + return value; +} + +const uint8_t fy_utf8_low_ascii_flags[256] = { + [0x00] = 0, // NUL '\0' (null character) + [0x01] = 0, // SOH (start of heading) + [0x02] = 0, // STX (start of text) + [0x03] = 0, // ETX (end of text) + [0x04] = 0, // EOT (end of transmission) + [0x05] = 0, // ENQ (enquiry) + [0x06] = 0, // ACK (acknowledge) + [0x07] = 0, // BEL '\a' (bell) + [0x08] = 0, // BS '\b' (backspace) + [0x09] = F_WS, // HT '\t' (horizontal tab) + [0x0A] = F_LB, // LF '\n' (new line) + [0x0B] = 0, // VT '\v' (vertical tab) + [0x0C] = 0, // FF '\f' (form feed) + [0x0D] = F_LB, // CR '\r' (carriage ret) + [0x0E] = 0, // SO (shift out) + [0x0F] = 0, // SI (shift in) + [0x10] = 0, // DLE (data link escape) + [0x11] = 0, // DC1 (device control 1) + [0x12] = 0, // DC2 (device control 2) + [0x13] = 0, // DC3 (device control 3) + [0x14] = 0, // DC4 (device control 4) + [0x15] = 0, // NAK (negative ack.) + [0x16] = 0, // SYN (synchronous idle) + [0x17] = 0, // ETB (end of trans. blk) + [0x18] = 0, // CAN (cancel) + [0x19] = 0, // EM (end of medium) + [0x1A] = 0, // SUB (substitute) + [0x1B] = 0, // ESC (escape) + [0x1C] = 0, // FS (file separator) + [0x1D] = 0, // GS (group separator) + [0x1E] = 0, // RS (record separator) + [0x1F] = 0, // US (unit separator) + [' '] = F_DIRECT_PRINT | F_WS, + ['!'] = F_DIRECT_PRINT, + ['"'] = F_DIRECT_PRINT, + ['#'] = F_DIRECT_PRINT, + ['$'] = F_DIRECT_PRINT, + ['%'] = F_DIRECT_PRINT, + ['&'] = F_DIRECT_PRINT, + ['\''] = F_DIRECT_PRINT, + ['('] = F_DIRECT_PRINT, + [')'] = F_DIRECT_PRINT, + ['*'] = F_DIRECT_PRINT, + ['+'] = F_DIRECT_PRINT, + [','] = F_DIRECT_PRINT | F_FLOW_INDICATOR, + ['-'] = F_DIRECT_PRINT, + ['.'] = F_DIRECT_PRINT, + ['/'] = F_DIRECT_PRINT, + ['0'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['1'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['2'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['3'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['4'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['5'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['6'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['7'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['8'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['9'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + [':'] = F_DIRECT_PRINT, + [';'] = F_DIRECT_PRINT, + ['<'] = F_DIRECT_PRINT, + ['='] = F_DIRECT_PRINT, + ['>'] = F_DIRECT_PRINT, + ['?'] = F_DIRECT_PRINT, + ['@'] = F_DIRECT_PRINT, + ['A'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['B'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['C'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['D'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['E'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['F'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['G'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['H'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['I'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['J'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['K'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['L'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['M'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['N'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['O'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['P'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['Q'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['R'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['S'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['T'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['U'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['V'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['W'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['X'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['Y'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['Z'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['['] = F_DIRECT_PRINT | F_FLOW_INDICATOR, + ['\\'] = F_DIRECT_PRINT, // '\\' + [']'] = F_DIRECT_PRINT | F_FLOW_INDICATOR, + ['^'] = F_DIRECT_PRINT, + ['_'] = F_DIRECT_PRINT | F_SIMPLE_SCALAR, + ['`'] = F_DIRECT_PRINT, + ['a'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['b'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['c'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['d'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['e'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['f'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['g'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['h'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['i'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['j'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['k'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['l'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['m'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['n'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['o'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['p'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['q'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['r'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['s'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['t'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['u'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['v'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['w'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['x'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['y'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['z'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['{'] = F_DIRECT_PRINT | F_FLOW_INDICATOR, + ['|'] = F_DIRECT_PRINT, + ['}'] = F_DIRECT_PRINT | F_FLOW_INDICATOR, + ['~'] = F_DIRECT_PRINT, + [0x7F] = 0, // DEL + /* the rest are zero */ +}; + +void *fy_utf8_split_posix(const char *str, int *argcp, const char * const *argvp[]) +{ + enum split_state { + SS_WS, /* at whitespace */ + SS_WS_BS, /* backslash at whitespace */ + SS_UQ, /* at unquoted */ + SS_UQ_BS, /* backslash at unquoted */ + SS_SQ, /* at single quote */ + SS_SQ_BS, /* backslash at single quote */ + SS_DQ, /* at double quote */ + SS_DQ_BS, /* backslash at double quote */ + SS_DQ_BS_OCT1, /* in \nnn octal escape digit 1 */ + SS_DQ_BS_OCT2, /* in \nnn octal escape digit 2 */ + SS_DQ_BS_HEX0, /* in \xNN hex escape digit 0 */ + SS_DQ_BS_HEX1, /* in \xNN hex escape digit 1 */ + SS_DQ_BS_HEX2, /* in \xNN hex escape digit 2 */ + SS_DQ_BS_C, /* in \cN control character escape */ + SS_WS_BS_ERR_EOL, /* EOL while waiting for char after \ */ + SS_UQ_BS_ERR_EOL, /* EOL while waiting for char after \ */ + SS_SQ_BS_ERR_EOL, /* EOL while waiting for char after \ */ + SS_DQ_BS_ERR_EOL, /* EOL while waiting for char after \ */ + SS_DQ_BS_ERR_BAD, /* bad escape */ + SS_SQ_NC_ERR, /* no closing single quote */ + SS_DQ_NC_ERR, /* no closing double quote */ + SS_DQ_BS_ERR_OCT_BAD, /* bad octal escape */ + SS_DQ_BS_ERR_HEX_BAD, /* bad hex escape */ + SS_DQ_BS_ERR_C_BAD, /* bad control char */ + SS_DONE /* final state */ + }; + const char *s, *e; + enum split_state ss; + int c, w, val, tmp, outc, i; + size_t arg_alloc = strlen(str) + 1, tmplen; + char *arg = NULL, *args, *arge, *tmparg; + int argv_alloc = 64, argv_count = 0; + char **argv = NULL, **tmpargv; + void *mem; + + if (!str || !argcp || !argvp) + return NULL; + + /* the temporary is guaranteed to fit one split */ + arg = alloca(arg_alloc); + args = arg; + arge = arg + arg_alloc - 1; + + argv = alloca(sizeof(*argv) * argv_alloc); + + e = str + strlen(str); + +#undef OUTC +#define OUTC(_c) \ + do { \ + assert(args < arge); \ + args = fy_utf8_put(args, arge - args, (_c)); \ + assert(args); \ + } while(0) + +#undef NEW_ARGV +#define NEW_ARGV() \ + do { \ + if (args > arg) { \ + if (argv_count + 1 >= argv_alloc) { \ + argv_alloc *= 2; \ + tmpargv = alloca((sizeof(*tmpargv) * argv_alloc)); \ + memcpy(tmpargv, argv, argv_count * sizeof(*tmpargv)); \ + } \ + tmplen = args - arg; \ + tmparg = alloca(tmplen + 1); \ + memcpy(tmparg, arg, tmplen); \ + tmparg[tmplen] = '\0'; \ + argv[argv_count] = tmparg; \ + argv[++argv_count] = NULL; \ + } \ + args = arg; \ + } while(0) + +#define GOTO(_ss) \ + do { \ + ss = (_ss); \ + } while(0) + + + ss = SS_WS; + s = str; + val = 0; + while (!(ss >= SS_WS_BS_ERR_EOL && ss <= SS_DONE)) { + c = fy_utf8_get(s, e - s, &w); + switch (ss) { + case SS_WS: + if (c < 0) { + NEW_ARGV(); + GOTO(SS_DONE); + break; + } + switch (c) { + case ' ': + case '\t': + break; /* no change */ + case '\\': + GOTO(SS_WS_BS); + break; + case '"': + GOTO(SS_DQ); /* start double quoted */ + NEW_ARGV(); + break; + case '\'': + GOTO(SS_SQ); /* start single quoted */ + NEW_ARGV(); + break; + default: + GOTO(SS_UQ); /* start unquoted */ + NEW_ARGV(); + OUTC(c); + break; + } + break; + + case SS_WS_BS: + if (c < 0) { + GOTO(SS_WS_BS_ERR_EOL); + break; + } + if (c == '\n') { /* backslash newline, continuation */ + GOTO(SS_WS); + break; + } + GOTO(SS_UQ); + NEW_ARGV(); + OUTC(c); + break; + + case SS_UQ: + if (c <= 0) { + NEW_ARGV(); + GOTO(SS_DONE); + break; + } + + switch (c) { + case ' ': + case '\t': + GOTO(SS_WS); + NEW_ARGV(); + break; /* no change */ + case '\\': + GOTO(SS_UQ_BS); + break; + case '"': + GOTO(SS_DQ); /* double quoted */ + break; + case '\'': + GOTO(SS_SQ); /* single quoted */ + break; + default: + OUTC(c); + break; + } + break; + + case SS_UQ_BS: + if (c < 0) { + GOTO(SS_UQ_BS_ERR_EOL); + break; + } + if (c == '\n') { /* backslash new line only */ + GOTO(SS_UQ); + break; + } + GOTO(SS_UQ); + OUTC(c); + break; + + case SS_SQ: + if (c < 0) { + GOTO(SS_SQ_NC_ERR); /* no closing ' */ + break; + } + switch (c) { + case '\'': /* end of quote, back to unquoted */ + GOTO(SS_UQ); + break; + case '\\': + OUTC(c); + GOTO(SS_SQ_BS); /* backslash */ + break; + default: + OUTC(c); + break; + } + break; + + case SS_SQ_BS: + if (c < 0) { + GOTO(SS_SQ_BS_ERR_EOL); + break; + } + if (c == '\n') { /* backslash new line only */ + GOTO(SS_SQ); /* back to single quoted */ + break; + } + GOTO(SS_UQ); /* always back to */ + OUTC(c); + break; + + case SS_DQ: + if (c < 0) { + GOTO(SS_DQ_NC_ERR); /* no closing ' */ + break; + } + switch (c) { + case '"': /* end of quote, back to unquoted */ + GOTO(SS_UQ); + break; + case '\\': + GOTO(SS_DQ_BS); /* backslash */ + break; + default: + OUTC(c); + break; + } + break; + + case SS_DQ_BS: + if (c < 0) { + GOTO(SS_DQ_BS_ERR_EOL); + break; + } + outc = -1; + switch (c) { + case 'a': + outc = '\a'; + break; + case 'b': + outc = '\b'; + break; + case 'e': + outc = '\x1b'; /* escape */ + break; + case 'n': + outc = '\n'; + break; + case 'r': + outc = '\r'; + break; + case 't': + outc = '\t'; + break; + case 'v': + outc = '\v'; + break; + case '\\': + outc = '\\'; + break; + case '\'': + outc = '\''; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + val = c - '0'; + GOTO(SS_DQ_BS_OCT1); + break; + case 'x': + val = 0; + GOTO(SS_DQ_BS_HEX0); + break; + case 'c': + val = 0; + GOTO(SS_DQ_BS_C); + break; + default: + GOTO(SS_DQ_BS_ERR_BAD); /* unknown escape */ + break; + } + if (outc > 0) { + OUTC(outc); + GOTO(SS_DQ); + } + break; + + case SS_DQ_BS_OCT1: + case SS_DQ_BS_OCT2: + if (c < 0) { + OUTC(val); + NEW_ARGV(); + GOTO(SS_DONE); + break; + } + if (!(c >= '0' && c <= '7')) { + OUTC(val); + GOTO(SS_DQ); + w = 0; /* redo, same */ + break; + } + val *= 8; + val += c - '0'; + if (ss == SS_DQ_BS_OCT2) { + OUTC(val); + GOTO(SS_DQ); + } else + GOTO(SS_DQ_BS_OCT2); + break; + + case SS_DQ_BS_HEX0: + case SS_DQ_BS_HEX1: + case SS_DQ_BS_HEX2: + tmp = -1; + if (c >= '0' && c <= '9') + tmp = c - '0'; + else if (c >= 'a' && c <= 'f') + tmp = 10 + c - 'a'; + else if (c >= 'A' && c <= 'F') + tmp = 10 + c - 'A'; + + switch (ss) { + case SS_DQ_BS_HEX0: + if (tmp < 0) { + fprintf(stderr, "tmp=%d c=%c\n", tmp, c); + GOTO(SS_DQ_BS_ERR_HEX_BAD); + break; + } + val = tmp; + GOTO(SS_DQ_BS_HEX1); + break; + + case SS_DQ_BS_HEX1: + case SS_DQ_BS_HEX2: + if (tmp < 0) { + GOTO(SS_DQ); + OUTC(val); + w = 0; /* redo */ + break; + } + val *= 16; + val += tmp; + if (ss == SS_DQ_BS_HEX1) + GOTO(SS_DQ_BS_HEX2); + else { + OUTC(val); + GOTO(SS_DQ); + } + break; + + default: + FY_IMPOSSIBLE_ABORT(); + } + break; + + case SS_DQ_BS_C: + outc = -1; + if (c >= 'a' && c <= 'z') + outc = c - 'a' + 1; + else if (c >= 'A' && c <= 'Z') + outc = c - 'Z' + 1; + if (outc > 0x20) { + outc = -1; + GOTO(SS_DQ_BS_ERR_C_BAD); + } else + GOTO(SS_DQ); + if (outc > 0) + OUTC(outc); + break; + default: + FY_IMPOSSIBLE_ABORT(); + } + s += w; + } + + /* someething went wrong */ + if (ss != SS_DONE) + return NULL; + + tmplen = (argv_count + 1) * sizeof(*argv); + for (i = 0; i < argv_count; i++) + tmplen += strlen(argv[i]) + 1; + + mem = malloc(tmplen); + if (!mem) + return NULL; + + tmpargv = mem; + tmparg = mem + (argv_count + 1) * sizeof(*tmpargv); + for (i = 0; i < argv_count; i++) { + tmpargv[i] = tmparg; + strcpy(tmparg, argv[i]); + tmparg += strlen(argv[i]) + 1; + } + tmpargv[i] = NULL; + + *argcp = i; + *argvp = mem; + + return mem; +} + +int fy_utf8_get_generic_s(const void *ptr, const void *ptr_end, int *widthp) +{ + const uint8_t *p = ptr; + int i, width, value; + + if (ptr >= ptr_end) + return FYUG_EOF; + + /* this is the slow path */ + width = fy_utf8_width_by_first_octet(p[0]); + if (!width) + return FYUG_INV; + if (ptr + width > ptr_end) + return FYUG_PARTIAL; + + /* initial value */ + value = *p++ & (0xff >> width); + for (i = 1; i < width; i++) { + if ((*p & 0xc0) != 0x80) + return FYUG_INV; + value = (value << 6) | (*p++ & 0x3f); + } + + /* check for validity */ + if ((width == 4 && value < 0x10000) || + (width == 3 && value < 0x800) || + (width == 2 && value < 0x80) || + (value >= 0xd800 && value <= 0xdfff) || value >= 0x110000) + return FYUG_INV; + + *widthp = width; + + return value; +} + +int fy_utf8_get_generic_s_nocheck(const void *ptr, int *widthp) +{ + const uint8_t *p = ptr; + int i, width, value; + + /* this is the slow path */ + width = fy_utf8_width_by_first_octet(p[0]); + if (!width) + return FYUG_INV; + + /* initial value */ + value = *p++ & (0xff >> width); + for (i = 1; i < width; i++) { + if ((*p & 0xc0) != 0x80) + return FYUG_INV; + value = (value << 6) | (*p++ & 0x3f); + } + + /* check for validity */ + if ((width == 4 && value < 0x10000) || + (width == 3 && value < 0x800) || + (width == 2 && value < 0x80) || + (value >= 0xd800 && value <= 0xdfff) || value >= 0x110000) + return FYUG_INV; + + *widthp = width; + + return value; +} diff --git a/src/util/fy-utf8.h b/src/util/fy-utf8.h new file mode 100644 index 0000000..9a40af7 --- /dev/null +++ b/src/util/fy-utf8.h @@ -0,0 +1,647 @@ +/* + * fy-utf8.h - UTF-8 methods + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_UTF8_H +#define FY_UTF8_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + + +#include "fy-utils.h" + +#define FY_UTF8_MIN_WIDTH 1 +#define FY_UTF8_MAX_WIDTH 4 + +extern const int8_t fy_utf8_width_table[32]; + +static inline int +fy_utf8_width_by_first_octet_no_table(const uint8_t c) +{ + return (c & 0x80) == 0x00 ? 1 : + (c & 0xe0) == 0xc0 ? 2 : + (c & 0xf0) == 0xe0 ? 3 : + (c & 0xf8) == 0xf0 ? 4 : 0; +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_width_by_first_octet(uint8_t c) +{ + return fy_utf8_width_table[c >> 3]; +} + +/* assumes valid utf8 character */ +static inline size_t +fy_utf8_width(const int c) +{ + return 1 + (c >= 0x80) + (c >= 0x800) + (c >= 0x10000); +} + +static inline bool +fy_utf8_is_valid(const int c) +{ + return c >= 0 && !((c >= 0xd800 && c <= 0xdfff) || c >= 0x110000); +} + +/* generic utf8 decoder (not inlined) */ +int fy_utf8_get_generic(const void *ptr, size_t left, int *widthp); + +/* -1 for end of input, -2 for invalid character, -3 for partial */ +#define FYUG_EOF -1 +#define FYUG_INV -2 +#define FYUG_PARTIAL -3 + +#define FY_UTF8_64_C(_x) ((int)(int32_t)(_x)) +#define FY_UTF8_64_W(_x) (((int)((_x) >> 32))) + +#define FY_UTF8_64_MAKE(_w, _c) ((int64_t)(((int64_t)((_w)) << 32) | (int64_t)(_c))) + +static inline FY_ALWAYS_INLINE int64_t +fy_utf8_get_branch_64(const void *ptr, size_t left) +{ + const uint8_t *s = ptr; + unsigned int a, b, c, d; + uint32_t code; + + if (!left) + return FYUG_EOF; + + a = s[0]; + if (a < 0x80) // 1 byte: 0xxxxxxx + return FY_UTF8_64_MAKE(1, a); + + if (left < 2) + return FYUG_PARTIAL; + + b = s[1]; + if ((a & 0xe0) == 0xc0) { // 2 bytes: 110xxxxx 10xxxxxx + if ((b & 0xc0) != 0x80) + return FYUG_INV; + + code = (int)(((a & 0x1f) << 6) | (b & 0x3f)); + if (code < 0x80) + return FYUG_INV; // overlong + + return FY_UTF8_64_MAKE(2, code); + } + + if (left < 3) + return FYUG_PARTIAL; + + c = s[2]; + if ((a & 0xf0) == 0xe0) { // 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx + + if (((b | c) & 0xc0) != 0x80) + return FYUG_INV; + + code = (int)(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f)); + if (code < 0x800 || (code >= 0xd800 && code <= 0xdfff)) + return FYUG_INV; // overlong or surrogate + + return FY_UTF8_64_MAKE(3, code); + } + + if (left < 4) + return FYUG_PARTIAL; + + d = s[3]; + if ((a & 0xf8) == 0xf0) { // 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + + if (((b | c | d) & 0xc0) != 0x80) + return FYUG_INV; + + code = ((a & 0x07) << 18) | ((b & 0x3f) << 12) | + ((c & 0x3f) << 6) | (d & 0x3f); + + if (code < 0x10000 || code > 0x10ffff) + return FYUG_INV; // overlong or out of range + + return FY_UTF8_64_MAKE(4, code); + } + + return FYUG_INV; +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get_branch(const void *ptr, size_t left, int *widthp) +{ + const uint8_t *s = ptr; + unsigned int a, b, c, d; + int code; + + if (!left) + goto err_eof; + + a = s[0]; + if (a < 0x80) { // 1 byte: 0xxxxxxx + code = (int)a; + *widthp = 1; + return code; + } + + if (left < 2) + goto err_partial; + + b = s[1]; + if ((a & 0xe0) == 0xc0) { // 2 bytes: 110xxxxx 10xxxxxx + if ((b & 0xc0) != 0x80) + goto err_inv; + + code = (int)(((a & 0x1f) << 6) | (b & 0x3f)); + if (code < 0x80) + goto err_inv; // overlong + + *widthp = 2; + return code; + } + + if (left < 3) + goto err_partial; + + c = s[2]; + if ((a & 0xf0) == 0xe0) { // 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx + + if (((b | c) & 0xc0) != 0x80) + goto err_inv; + + code = (int)(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f)); + if (code < 0x800 || (code >= 0xd800 && code <= 0xdfff)) + goto err_inv; // overlong or surrogate + + *widthp = 3; + return code; + } + + if (left < 4) + goto err_partial; + + d = s[3]; + if ((a & 0xf8) == 0xf0) { // 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + + if (((b | c | d) & 0xc0) != 0x80) + goto err_inv; + + code = ((a & 0x07) << 18) | ((b & 0x3f) << 12) | + ((c & 0x3f) << 6) | (d & 0x3f); + + if (code < 0x10000 || code > 0x10ffff) + goto err_inv; // overlong or out of range + + *widthp = 4; + return code; + } + +err_inv: + *widthp = 0; + return FYUG_INV; + +err_partial: + *widthp = 0; + return FYUG_PARTIAL; + +err_eof: + *widthp = 0; + return FYUG_EOF; +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get_table(const void *ptr, size_t left, int *widthp) +{ + const uint8_t *s = ptr; + unsigned int a, b, c, d; + int code, w; + + if (!left) + goto err_eof; + + w = fy_utf8_width_by_first_octet(s[0]); + if ((size_t)w > left) + goto err_partial; + + *widthp = w; + + switch (w) { + case 1: + a = s[0]; + code = (int)a; + return code; + case 2: + a = s[0]; + b = s[1]; + if ((b & 0xc0) != 0x80) + goto err_inv; + + code = (int)(((a & 0x1f) << 6) | (b & 0x3f)); + if (code < 0x80) + goto err_inv; // overlong + + return code; + case 3: + a = s[0]; + b = s[1]; + c = s[2]; + + if (((b | c) & 0xc0) != 0x80) + goto err_inv; + + code = (int)(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f)); + if (code < 0x800 || (code >= 0xd800 && code <= 0xdfff)) + goto err_inv; // overlong or surrogate + + return code; + case 4: + a = s[0]; + b = s[1]; + c = s[2]; + d = s[3]; + + if (((b | c | d) & 0xc0) != 0x80) + goto err_inv; + + code = ((a & 0x07) << 18) | ((b & 0x3f) << 12) | + ((c & 0x3f) << 6) | (d & 0x3f); + + if (code < 0x10000 || code > 0x10ffff) + goto err_inv; // overlong or out of range + + return code; + + default: + break; + } + +err_inv: + *widthp = 0; + return FYUG_INV; + +err_partial: + *widthp = 0; + return FYUG_PARTIAL; + +err_eof: + *widthp = 0; + return FYUG_EOF; +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get(const void *ptr, size_t left, int *widthp) +{ + return fy_utf8_get_branch(ptr, left, widthp); +} + +static inline FY_ALWAYS_INLINE int64_t +fy_utf8_get_64(const void *ptr, size_t left) +{ + return fy_utf8_get_branch_64(ptr, left); +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get_no_width(const void *ptr, size_t left) +{ + int w; + + return fy_utf8_get(ptr, left, &w); +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get_end(const void *ptr, const void *ptr_end, int *widthp) +{ + return fy_utf8_get(ptr, (size_t)(ptr_end - ptr), widthp); +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get_end_no_width(const void *ptr, const void *ptr_end) +{ + int w; + + return fy_utf8_get(ptr, (size_t)(ptr_end - ptr), &w); +} + +int fy_utf8_get_right_generic(const void *ptr, size_t left, int *widthp); + +static inline int fy_utf8_get_right(const void *ptr, size_t left, int *widthp) +{ + const uint8_t *p = ptr + left; + + /* single byte (hot path) */ + if (left > 0 && !(p[-1] & 0x80)) { + if (widthp) + *widthp = 1; + return p[-1] & 0x7f; + } + return fy_utf8_get_right_generic(ptr, left, widthp); +} + + +/* for when you _know_ that there's enough room and c is valid */ +static inline void *fy_utf8_put_unchecked(void *ptr, int c) +{ + uint8_t *s = ptr; + + assert(c >= 0); + if (c < 0x80) + *s++ = c; + else if (c < 0x800) { + *s++ = (c >> 6) | 0xc0; + *s++ = (c & 0x3f) | 0x80; + } else if (c < 0x10000) { + *s++ = (c >> 12) | 0xe0; + *s++ = ((c >> 6) & 0x3f) | 0x80; + *s++ = (c & 0x3f) | 0x80; + } else { + *s++ = (c >> 18) | 0xf0; + *s++ = ((c >> 12) & 0x3f) | 0x80; + *s++ = ((c >> 6) & 0x3f) | 0x80; + *s++ = (c & 0x3f) | 0x80; + } + return s; +} + +static inline void *fy_utf8_put(void *ptr, size_t left, int c) +{ + if (!fy_utf8_is_valid(c) || fy_utf8_width(c) > left) + return NULL; + + return fy_utf8_put_unchecked(ptr, c); +} + +/* buffer must contain at least 5 characters */ +#define FY_UTF8_FORMAT_BUFMIN 5 +enum fy_utf8_escape { + fyue_none, + fyue_singlequote, + fyue_doublequote, + fyue_doublequote_json, + fyue_doublequote_yaml_1_1, +}; + +static inline bool fy_utf8_escape_is_any_doublequote(const enum fy_utf8_escape esc) +{ + return esc >= fyue_doublequote && esc <= fyue_doublequote_yaml_1_1; +} + +char *fy_utf8_format(int c, char *buf, const enum fy_utf8_escape esc); + +#define fy_utf8_format_a(_c, _esc) \ + ({ \ + char *_buf = alloca(FY_UTF8_FORMAT_BUFMIN); \ + fy_utf8_format((_c), _buf, _esc); \ + }) + +size_t fy_utf8_format_text_length(const char *buf, size_t len, + enum fy_utf8_escape esc); +char *fy_utf8_format_text(const char *buf, size_t len, + char *out, size_t maxsz, + enum fy_utf8_escape esc); + +#define fy_utf8_format_text_a(_buf, _len, _esc) \ + ({ \ + const char *__buf = (_buf); \ + size_t __len = (_len); \ + enum fy_utf8_escape __esc = (_esc); \ + size_t _outsz = fy_utf8_format_text_length(__buf, __len, __esc); \ + char *_out = alloca(_outsz + 1); \ + fy_utf8_format_text(__buf, __len, _out, _outsz, __esc); \ + }) + +char *fy_utf8_format_text_alloc(const char *buf, size_t len, const enum fy_utf8_escape esc); + +const void *fy_utf8_memchr_generic(const void *s, int c, size_t n); + +static inline const void *fy_utf8_memchr(const void *s, int c, size_t n) +{ + if (c < 0 || !n) + return NULL; + if (c < 0x80) + return memchr(s, c, n); + return fy_utf8_memchr_generic(s, c, n); +} + +static inline const void *fy_utf8_strchr(const void *s, int c) +{ + if (c < 0) + return NULL; + if (c < 0x80) + return strchr(s, c); + return fy_utf8_memchr_generic(s, c, strlen(s)); +} + +static inline int fy_utf8_count(const void *ptr, size_t len) +{ + const uint8_t *s = ptr, *e = ptr + len; + int w, count; + + count = 0; + while (s < e) { + w = fy_utf8_width_by_first_octet(*s); + + /* malformed? */ + if (!w || s + w > e) + break; + s += w; + + count++; + } + + return count; +} + +int fy_utf8_parse_escape(const char **strp, size_t len, const enum fy_utf8_escape esc); + +#define F_NONE 0 +#define F_SIMPLE_SCALAR FY_BIT(0) /* part of simple scalar 0-9a-zA-Z_ */ +#define F_DIRECT_PRINT FY_BIT(1) /* 0x20..0x7e */ +#define F_LB FY_BIT(2) /* is a linebreak */ +#define F_WS FY_BIT(3) /* is a whitespace */ +#define F_LETTER FY_BIT(4) /* is a letter a..z A..Z */ +#define F_DIGIT FY_BIT(5) /* is a digit 0..9 */ +#define F_XDIGIT FY_BIT(6) /* is a hex digit 0..9 a-f A-F */ +#define F_FLOW_INDICATOR FY_BIT(7) /* is ,[]{} */ + +extern const uint8_t fy_utf8_low_ascii_flags[256]; + +void *fy_utf8_split_posix(const char *str, int *argcp, const char * const *argvp[]); + +int fy_utf8_get_generic_s(const void *ptr, const void *ptr_end, int *widthp); +int fy_utf8_get_generic_s_nocheck(const void *ptr, int *widthp); + +static inline int fy_utf8_get_s(const void *ptr, const void *ptr_end, int *widthp) +{ + const uint8_t *p = ptr; + + /* single byte (hot path) */ + if (ptr >= ptr_end) { + *widthp = 0; + return FYUG_EOF; + } + + if (!(p[0] & 0x80)) { + *widthp = 1; + return p[0]; + } + return fy_utf8_get_generic_s(ptr, ptr_end, widthp); +} + +static inline int fy_utf8_get_s_nocheck(const void *ptr, int *widthp) +{ + const uint8_t *p = ptr; + + if (!(p[0] & 0x80)) { + *widthp = 1; + return p[0]; + } + return fy_utf8_get_generic_s_nocheck(ptr, widthp); +} + +/* for most 64 bit arches this will fit in a single register */ +struct fy_utf8_result { + int c; + int w; +}; + +/* probably the most performant version */ +static inline struct fy_utf8_result fy_utf8_get_s_res(const void *ptr, const void *ptr_end) +{ + const uint8_t *p = ptr; + int c, width; + + /* single byte (hot path) */ + if (ptr >= ptr_end) + return (struct fy_utf8_result){ FYUG_EOF, 0 }; + + if (!(p[0] & 0x80)) + return (struct fy_utf8_result){ p[0], 1 }; + + c = fy_utf8_get_generic_s(ptr, ptr_end, &width); + if (c < 0) + return (struct fy_utf8_result){ c, 0 }; + return (struct fy_utf8_result){ c, width }; +} + +static inline bool fy_utf8_is_space(const int c) +{ + return c == ' '; +} + +static inline bool fy_utf8_is_tab(const int c) +{ + return c == '\t'; +} + +static inline bool fy_utf8_is_simple_scalar_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_SIMPLE_SCALAR; +} + +static inline bool +fy_utf8_is_printable_ascii_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_DIRECT_PRINT; +} + +static inline bool fy_utf8_is_lb_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_LB; +} + +static inline bool fy_utf8_is_ws_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_WS; +} + +static inline bool fy_utf8_is_ws_lb_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & (F_WS | F_LB); +} + +static inline bool fy_utf8_is_letter_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_LETTER; +} + +static inline bool fy_utf8_is_digit_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_DIGIT; +} + +static inline bool fy_utf8_is_hex_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_XDIGIT; +} + +static inline bool fy_utf8_is_flow_indicator_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_FLOW_INDICATOR; +} + +static inline bool fy_utf8_is_low_ascii(const int c) +{ + return (unsigned int)c < 128; +} + +static inline bool fy_utf8_is_simple_scalar(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_simple_scalar_no_check(c); +} + +static inline bool +fy_utf8_is_printable_ascii_x(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_printable_ascii_no_check(c); +} + +static inline bool +fy_utf8_is_printable_ascii(const int c) +{ + return c >= 0x20 && c <= 0x7e; +} + +static inline bool fy_utf8_is_lb(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_lb_no_check(c); +} + +static inline bool fy_utf8_is_ws(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_ws_no_check(c); +} + +static inline bool fy_utf8_is_ws_lb(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_ws_lb_no_check(c); +} + +static inline bool fy_utf8_is_letter(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_letter_no_check(c); +} + +static inline bool fy_utf8_is_digit(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_digit_no_check(c); +} + +static inline bool fy_utf8_is_hex(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_hex_no_check(c); +} + +static inline bool fy_utf8_is_flow_indicator(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_flow_indicator_no_check(c); +} + +#endif diff --git a/src/util/fy-utils.c b/src/util/fy-utils.c new file mode 100644 index 0000000..40ce77b --- /dev/null +++ b/src/util/fy-utils.c @@ -0,0 +1,900 @@ +/* + * fy-utils.c - Generic utilities for functionality that's missing + * from platforms. + * + * For now only used to implement memstream for Apple platforms. + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fy-utf8.h" +#include "fy-ctype.h" +#include "fy-utils.h" + +#include "xxhash.h" + +#if defined(__APPLE__) && (_POSIX_C_SOURCE < 200809L) + +/* + * adapted from http://piumarta.com/software/memstream/ + * + * Under the MIT license. + */ + +/* + * ---------------------------------------------------------------------------- + * + * OPEN_MEMSTREAM(3) BSD and Linux Library Functions OPEN_MEMSTREAM(3) + * + * SYNOPSIS + * #include "memstream.h" + * + * FILE *open_memstream(char **bufp, size_t *sizep); + * + * DESCRIPTION + * The open_memstream() function opens a stream for writing to a buffer. + * The buffer is dynamically allocated (as with malloc(3)), and + * automatically grows as required. After closing the stream, the caller + * should free(3) this buffer. + * + * When the stream is closed (fclose(3)) or flushed (fflush(3)), the + * locations pointed to by bufp and sizep are updated to contain, + * respectively, a pointer to the buffer and the current size of the + * buffer. These values remain valid only as long as the caller performs + * no further output on the stream. If further output is performed, then + * the stream must again be flushed before trying to access these + * variables. + * + * A null byte is maintained at the end of the buffer. This byte is not + * included in the size value stored at sizep. + * + * The stream's file position can be changed with fseek(3) or fseeko(3). + * Moving the file position past the end of the data already written fills + * the intervening space with zeros. + * + * RETURN VALUE + * Upon successful completion open_memstream() returns a FILE pointer. + * Otherwise, NULL is returned and errno is set to indicate the error. + * + * CONFORMING TO + * POSIX.1-2008 + * + * ---------------------------------------------------------------------------- + */ + +#ifndef min +#define min(X, Y) (((X) < (Y)) ? (X) : (Y)) +#endif + +struct memstream { + size_t position; + size_t size; + size_t capacity; + char *contents; + char **ptr; + size_t *sizeloc; +}; + +static int memstream_grow(struct memstream *ms, size_t minsize) +{ + size_t newcap; + char *newcontents; + + newcap = ms->capacity * 2; + while (newcap <= minsize + 1) + newcap *= 2; + newcontents = realloc(ms->contents, newcap); + if (!newcontents) + return -1; + ms->contents = newcontents; + memset(ms->contents + ms->capacity, 0, newcap - ms->capacity); + ms->capacity = newcap; + *ms->ptr = ms->contents; + return 0; +} + +static int memstream_read(void *cookie, char *buf, int count) +{ + struct memstream *ms = cookie; + size_t n; + + n = min(ms->size - ms->position, (size_t)count); + if (n < 1) + return 0; + memcpy(buf, ms->contents, n); + ms->position += n; + return n; +} + +static int memstream_write(void *cookie, const char *buf, int count) +{ + struct memstream *ms = cookie; + + if (ms->capacity <= ms->position + (size_t)count && + memstream_grow(ms, ms->position + (size_t)count) < 0) + return -1; + memcpy(ms->contents + ms->position, buf, count); + ms->position += count; + ms->contents[ms->position] = '\0'; + if (ms->size < ms->position) + *ms->sizeloc = ms->size = ms->position; + + return count; +} + +static fpos_t memstream_seek(void *cookie, fpos_t offset, int whence) +{ + struct memstream *ms = cookie; + fpos_t pos= 0; + + switch (whence) { + case SEEK_SET: + pos = offset; + break; + case SEEK_CUR: + pos = ms->position + offset; + break; + case SEEK_END: + pos = ms->size + offset; + break; + default: + errno= EINVAL; + return -1; + } + if (pos >= (fpos_t)ms->capacity && memstream_grow(ms, pos) < 0) + return -1; + ms->position = pos; + if (ms->size < ms->position) + *ms->sizeloc = ms->size = ms->position; + return pos; +} + +static int memstream_close(void *cookie) +{ + struct memstream *ms = cookie; + + ms->size = min(ms->size, ms->position); + *ms->ptr = ms->contents; + *ms->sizeloc = ms->size; + ms->contents[ms->size]= 0; + /* ms->contents is what's returned */ + free(ms); + return 0; +} + +FILE *open_memstream(char **ptr, size_t *sizeloc) +{ + struct memstream *ms; + FILE *fp; + + if (!ptr || !sizeloc) { + errno= EINVAL; + goto err_out; + } + + ms = calloc(1, sizeof(struct memstream)); + if (!ms) + goto err_out; + + ms->position = ms->size= 0; + ms->capacity = 4096; + ms->contents = calloc(ms->capacity, 1); + if (!ms->contents) + goto err_free_ms; + ms->ptr = ptr; + ms->sizeloc = sizeloc; + fp= funopen(ms, memstream_read, memstream_write, + memstream_seek, memstream_close); + if (!fp) + goto err_free_all; + *ptr = ms->contents; + *sizeloc = ms->size; + return fp; + +err_free_all: + free(ms->contents); +err_free_ms: + free(ms); +err_out: + return NULL; +} + +#endif /* __APPLE__ && _POSIX_C_SOURCE < 200809L */ + +bool fy_tag_uri_is_valid(const char *data, size_t len) +{ + const char *s, *e; + int w, j, k, width, c; + uint8_t octet, esc_octets[4]; + + s = data; + e = s + len; + + while ((c = fy_utf8_get(s, e - s, &w)) >= 0) { + if (c != '%') { + s += w; + continue; + } + + width = 0; + k = 0; + do { + /* short URI escape */ + if ((e - s) < 3) + return false; + + if (width > 0) { + c = fy_utf8_get(s, e - s, &w); + if (c != '%') + return false; + } + + s += w; + + octet = 0; + + for (j = 0; j < 2; j++) { + c = fy_utf8_get(s, e - s, &w); + if (!fy_is_hex(c)) + return false; + s += w; + + octet <<= 4; + if (c >= '0' && c <= '9') + octet |= c - '0'; + else if (c >= 'a' && c <= 'f') + octet |= 10 + c - 'a'; + else + octet |= 10 + c - 'A'; + } + if (!width) { + width = fy_utf8_width_by_first_octet(octet); + + if (width < 1 || width > 4) + return false; + k = 0; + } + esc_octets[k++] = octet; + + } while (--width > 0); + + /* now convert to utf8 */ + c = fy_utf8_get(esc_octets, k, &w); + + if (c < 0) + return false; + } + + return true; +} + +int fy_tag_handle_length(const char *data, size_t len) +{ + const char *s, *e; + int c, w; + + s = data; + e = s + len; + + c = fy_utf8_get(s, e - s, &w); + if (c != '!') + return -1; + s += w; + + c = fy_utf8_get(s, e - s, &w); + if (c == -1) + return len; + + if (fy_is_ws(c)) + return s - data; + /* if first character is !, empty handle */ + if (c == '!') { + s += w; + return s - data; + } + if (!fy_is_first_alpha(c)) + return -1; + s += w; + while (fy_is_alnum(c = fy_utf8_get(s, e - s, &w))) + s += w; + if (c == '!') + s += w; + + return s - data; +} + +int fy_tag_uri_length(const char *data, size_t len) +{ + const char *s, *e; + int c, w, cn, wn, uri_length; + + s = data; + e = s + len; + + while (fy_is_uri(c = fy_utf8_get(s, e - s, &w))) { + cn = fy_utf8_get(s + w, e - (s + w), &wn); + if ((fy_is_z(cn) || fy_is_blank(cn) || fy_is_any_lb(cn)) && fy_utf8_strchr(",}]", c)) + break; + s += w; + } + uri_length = s - data; + + if (!fy_tag_uri_is_valid(data, uri_length)) + return -1; + + return uri_length; +} + +int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info) +{ + const char *s, *e; + int total_length, handle_length, uri_length, prefix_length, suffix_length; + int c, cn, w, wn; + + s = data; + e = s + len; + + prefix_length = 0; + + /* it must start with '!' */ + c = fy_utf8_get(s, e - s, &w); + if (c != '!') + return -1; + cn = fy_utf8_get(s + w, e - (s + w), &wn); + if (cn == '<') { + prefix_length = 2; + suffix_length = 1; + } else + prefix_length = suffix_length = 0; + + if (prefix_length) { + handle_length = 0; /* set the handle to '' */ + s += prefix_length; + } else { + /* either !suffix or !handle!suffix */ + /* we scan back to back, and split handle/suffix */ + handle_length = fy_tag_handle_length(s, e - s); + if (handle_length < 0) + return -1; + s += handle_length; + } + + uri_length = fy_tag_uri_length(s, e - s); + if (uri_length < 0) + return -1; + + /* a handle? */ + if (!prefix_length && (handle_length == 0 || data[handle_length - 1] != '!')) { + /* special case, '!', handle set to '' and suffix to '!' */ + if (handle_length == 1 && uri_length == 0) { + handle_length = 0; + uri_length = 1; + } else { + uri_length = handle_length - 1 + uri_length; + handle_length = 1; + } + } + total_length = prefix_length + handle_length + uri_length + suffix_length; + + if (total_length != (int)len) + return -1; + + info->total_length = total_length; + info->handle_length = handle_length; + info->uri_length = uri_length; + info->prefix_length = prefix_length; + info->suffix_length = suffix_length; + + return 0; +} + +/* simple terminal methods; mainly for getting size of terminal */ +int fy_term_set_raw(int fd, struct termios *oldt) +{ + struct termios newt, t; + int ret; + + /* must be a terminal */ + if (!isatty(fd)) + return -1; + + ret = tcgetattr(fd, &t); + if (ret != 0) + return ret; + + newt = t; + + cfmakeraw(&newt); + + ret = tcsetattr(fd, TCSANOW, &newt); + if (ret != 0) + return ret; + + if (oldt) + *oldt = t; + + return 0; +} + +int fy_term_restore(int fd, const struct termios *oldt) +{ + /* must be a terminal */ + if (!isatty(fd)) + return -1; + + return tcsetattr(fd, TCSANOW, oldt); +} + +ssize_t fy_term_write(int fd, const void *data, size_t count) +{ + ssize_t wrn, r; + + if (!isatty(fd)) + return -1; + r = 0; + wrn = 0; + while (count > 0) { + do { + r = write(fd, data, count); + } while (r == -1 && errno == EAGAIN); + if (r < 0) + break; + wrn += r; + data += r; + count -= r; + } + + /* return the amount written, or the last error code */ + return wrn > 0 ? wrn : r; +} + +int fy_term_safe_write(int fd, const void *data, size_t count) +{ + if (!isatty(fd)) + return -1; + + return fy_term_write(fd, data, count) == (ssize_t)count ? 0 : -1; +} + +ssize_t fy_term_read(int fd, void *data, size_t count, int timeout_us) +{ + ssize_t rdn, r; + struct timeval tv, tvto, *tvp; + fd_set rdfds; + + if (!isatty(fd)) + return -1; + + FD_ZERO(&rdfds); + + memset(&tvto, 0, sizeof(tvto)); + memset(&tv, 0, sizeof(tv)); + + if (timeout_us >= 0) { + tvto.tv_sec = timeout_us / 1000000; + tvto.tv_usec = timeout_us % 1000000; + tvp = &tv; + } else { + tvp = NULL; + } + + r = 0; + rdn = 0; + while (count > 0) { + do { + FD_SET(fd, &rdfds); + if (tvp) + *tvp = tvto; + r = select(fd + 1, &rdfds, NULL, NULL, tvp); + } while (r == -1 && errno == EAGAIN); + + /* select ends, or something weird */ + if (r <= 0 || !FD_ISSET(fd, &rdfds)) + break; + + /* now read */ + do { + r = read(fd, data, count); + } while (r == -1 && errno == EAGAIN); + if (r < 0) + break; + + rdn += r; + data += r; + count -= r; + } + + /* return the amount written, or the last error code */ + return rdn > 0 ? rdn : r; +} + +ssize_t fy_term_read_escape(int fd, void *buf, size_t count) +{ + char *p; + int r, rdn; + char c; + + /* at least 3 characters */ + if (count < 3) + return -1; + + p = buf; + rdn = 0; + + /* ESC */ + r = fy_term_read(fd, &c, 1, 100 * 1000); + if (r != 1 || c != '\x1b') + return -1; + *p++ = c; + count--; + rdn++; + + /* [ */ + r = fy_term_read(fd, &c, 1, 100 * 1000); + if (r != 1 || c != '[') + return rdn; + *p++ = c; + count--; + rdn++; + + /* read until error, out of buffer, or < 0x40 || > 0x7e */ + r = -1; + while (count > 0) { + r = fy_term_read(fd, &c, 1, 100 * 1000); + if (r != 1) + r = -1; + if (r != 1) + break; + *p++ = c; + count--; + rdn++; + + /* end of escape */ + if (c >= 0x40 && c <= 0x7e) + break; + } + + return rdn; +} + +int fy_term_query_size_raw(int fd, int *rows, int *cols) +{ + char buf[32]; + char *s, *e; + ssize_t r; + + /* must be a terminal */ + if (!isatty(fd)) + return -1; + + *rows = *cols = 0; + + /* query text area */ + r = fy_term_safe_write(fd, "\x1b[18t", 5); + if (r != 0) + return r; + + /* read a character */ + r = fy_term_read_escape(fd, buf, sizeof(buf)); + + /* return must be ESC[8;;;t */ + + if (r < 8 || r >= (int)sizeof(buf) - 2) /* minimum ESC[8;1;1t */ + return -1; + + s = buf; + e = s + r; + + /* correct response? starts with ESC[8; */ + if (s[0] != '\x1b' || s[1] != '[' || s[2] != '8' || s[3] != ';') + return -1; + s += 4; + + /* must end with t */ + if (e[-1] != 't') + return -1; + *--e = '\0'; /* remove trailing t, and zero terminate */ + + /* scan two ints separated by ; */ + r = sscanf(s, "%d;%d", rows, cols); + if (r != 2) + return -1; + + return 0; +} + +int fy_term_query_size(int fd, int *rows, int *cols) +{ + struct termios old_term; + int ret, r; + + if (!isatty(fd)) + return -1; + + r = fy_term_set_raw(fd, &old_term); + if (r != 0) + return -1; + + ret = fy_term_query_size_raw(fd, rows, cols); + + r = fy_term_restore(fd, &old_term); + if (r != 0) + return -1; + + return ret; +} + +int fy_comment_iter_begin(const char *comment, size_t size, struct fy_comment_iter *iter) +{ + if (!comment || !iter) + return -1; + + memset(iter, 0, sizeof(*iter)); + iter->start = comment; + iter->size = size == (size_t)-1 ? strlen(comment) : size; + iter->end = iter->start + iter->size; + + if (!iter->size) + return -1; + + iter->next = iter->start; + iter->line = 0; + + return 0; +} + +const char *fy_comment_iter_next_line(struct fy_comment_iter *iter, size_t *lenp) +{ + const char *s, *e, *le, *t; + + if (!iter || !lenp || !iter->start) + return NULL; + + /* no more */ + if (iter->next >= iter->end) + return NULL; +again: + *lenp = 0; + + s = iter->next; + e = iter->end; + + /* skip whitespace */ + while (s < e && isblank((unsigned char)*s)) + s++; + + if (s >= e) + return NULL; + + /* find end of line */ + le = memchr(s, '\n', e - s); + if (!le) { + le = e; + iter->next = e; + } else { + iter->next = le + 1; + } + + /* final? check if ends in *\/ */ + if (le >= e && (le - s) > 2 && le[-2] == '*' && le[-1] == '/') + le -= 2; + + /* backtrack while there's space at the end of line */ + while (le > s && isblank((unsigned char)le[-1])) + le--; + + /* check if the whole line is punctuation so it's formatting */ + t = s; + while (t < le && ispunct((unsigned char)*t)) + t++; + + /* everything is punctuation? */ + if (t > s && t >= le) { + if (((le - s) == 2 && s[0] == '/' && s[1] == '/') || /* '//' -> empty line */ + ((le - s) == 1 && s[0] == '*')) { /* '*' -> empty line */ + iter->line++; + return ""; + } + /* for anything else, just try again */ + goto again; + } + + /* something is there, skip over '// ' or '* ' */ + if ((le - s) > 3 && s[0] == '/' && s[1] == '/' && isblank((unsigned char)s[2])) + s += 3; + else if ((le - s) > 2 && s[0] == '*' && isblank((unsigned char)s[1])) + s += 2; + else if (iter->line == 0 && (le - s) > 3 && s[0] == '/' && s[1] == '*' && isblank((unsigned char)s[2])) + s += 3; + + iter->line++; + *lenp = le - s; + return s; +} + +void fy_comment_iter_end(struct fy_comment_iter *iter) +{ + /* nothing */ +} + +char *fy_get_cooked_comment(const char *raw_comment, size_t size) +{ + struct fy_comment_iter iter; + FILE *fp; + char *buf; + const char *line; + size_t len, line_len; + int ret; + + if (!raw_comment) + return NULL; + + fp = open_memstream(&buf, &len); + if (!fp) + return NULL; + + ret = 0; + fy_comment_iter_begin(raw_comment, size, &iter); + while ((line = fy_comment_iter_next_line(&iter, &line_len)) != NULL) { + ret = fprintf(fp, "%.*s\n", (int)line_len, line); + if (ret < 0) + break; + } + fy_comment_iter_end(&iter); + + fclose(fp); + + if (ret < 0) { + if (buf) + free(buf); + return NULL; + } + + /* must be freed */ + return buf; +} + +int fy_keyword_iter_begin(const char *text, size_t size, const char *keyword, struct fy_keyword_iter *iter) +{ + if (!text || !size || !keyword || !iter) + return -1; + + memset(iter, 0, sizeof(*iter)); + iter->keyword = keyword; + iter->keyword_len = strlen(keyword); + iter->start = text; + iter->size = size == (size_t)-1 ? strlen(text) : size; + iter->end = iter->start + iter->size; + iter->pc = '\n'; + + if (!iter->size) + return -1; + + iter->next = iter->start; + + return 0; +} + +const char *fy_keyword_iter_next(struct fy_keyword_iter *iter) +{ + const char *keyword; + size_t keyword_len; + const char *s, *e; + int c, w, pc, mode; + + if (!iter || !iter->start) + return NULL; + + /* no more */ + if (iter->next >= iter->end) + return NULL; + + s = iter->next; + e = iter->end; + + keyword = iter->keyword; + keyword_len = iter->keyword_len; + + mode = 0; + pc = iter->pc; + while ((c = fy_utf8_get_s(s, e, &w)) >= 0) { + + /* simple state machine to handle quoting */ + switch (mode) { + case 0: /* unquoted */ + if (c == keyword[0] && (fy_is_any_lb(pc) || fy_is_ws(pc)) && + (size_t)(e - s) > (keyword_len + 1) && !memcmp(s, keyword, keyword_len) && + (fy_is_ws(s[keyword_len]) || fy_is_any_lb(s[keyword_len]))) { + + iter->next = s; + iter->pc = pc; + return s; + } + + if (c == '\'') + mode = 1; + else if (c == '"') + mode = 3; + break; + case 1: /* single quote */ + if (c == '\\') + mode = 2; /* escaped single quote? */ + else if (c == '\'') + mode = 0; /* back to unquoted */ + break; + case 2: /* single quote backslash */ + mode = 1; /* back to single quote mode always */ + break; + case 3: /* double quote */ + if (c == '\\') + mode = 4; /* escaped quote? */ + else if (c == '"') + mode = 0; /* back to unquoted */ + break; + case 4: + mode = 3; /* back to double quote mode always */ + break; + } + s += w; + pc = c; + } + + return NULL; +} + +void fy_keyword_iter_advance(struct fy_keyword_iter *iter, size_t advance) +{ + int w; + const char *prev; + + if (!iter || !iter->next) + return; + + prev = iter->next; + + iter->next += advance; + if (iter->next >= iter->end) + iter->next = iter->end; + + iter->pc = fy_utf8_get_right(prev, (size_t)(iter->next - prev), &w); + if (iter->pc < 0) + iter->pc = '\n'; +} + +void fy_keyword_iter_end(struct fy_keyword_iter *iter) +{ + /* nothing */ +} + +#define FY_XXHASH64_SEED ((uint64_t)0x1973198120142019U) + +uint64_t fy_iovec_xxhash64(const struct iovec *iov, int iovcnt) +{ + XXH64_state_t xxstate; + uint64_t hash; + int i; + + XXH64_reset(&xxstate, FY_XXHASH64_SEED); + for (i = 0; i < iovcnt; i++) + XXH64_update(&xxstate, iov[i].iov_base, iov[i].iov_len); + hash = XXH64_digest(&xxstate); + /* XXX we never return a hash value of zero */ + if (!hash) + hash++; + return hash; +} diff --git a/src/util/fy-utils.h b/src/util/fy-utils.h new file mode 100644 index 0000000..8e5fa4e --- /dev/null +++ b/src/util/fy-utils.h @@ -0,0 +1,619 @@ +/* + * fy-utils.h - internal utilities header file + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_UTILS_H +#define FY_UTILS_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Define FY_CPP_SHORT_NAMES to use shortened macro names in the implementation. + * This significantly reduces expansion buffer usage during recursive macro expansion, + * which can help avoid internal compiler errors and speed up compilation. + * + * The public API (FY_CPP_*) remains the same regardless of this setting. + */ + +#define FY_CPP_SHORT_NAMES + +/* to avoid dragging in libfyaml.h */ +#ifndef FY_BIT +#define FY_BIT(x) (1U << (x)) +#endif + +#if defined(__linux__) +#include +#endif + +#if defined(__APPLE__) && (_POSIX_C_SOURCE < 200809L) +FILE *open_memstream(char **ptr, size_t *sizeloc); +#endif + +int fy_tag_handle_length(const char *data, size_t len); +bool fy_tag_uri_is_valid(const char *data, size_t len); +int fy_tag_uri_length(const char *data, size_t len); + +struct fy_tag_scan_info { + int total_length; + int handle_length; + int uri_length; + int prefix_length; + int suffix_length; +}; + +int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info); + +#ifndef container_of +#define container_of(ptr, type, member) \ + ({ \ + const __typeof__(((type *)0)->member) *__mptr = (ptr); \ + (type *)((void *)__mptr - offsetof(type,member)); \ + }) +#endif + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) ((sizeof(x)/sizeof((x)[0]))) +#endif + +#if defined(NDEBUG) && (defined(__GNUC__) && __GNUC__ >= 4) +#define FY_ALWAYS_INLINE __attribute__((always_inline)) +#else +#define FY_ALWAYS_INLINE /* nothing */ +#endif + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define FY_UNUSED __attribute__((unused)) +#else +#define FY_UNUSED /* nothing */ +#endif + +#if defined(NDEBUG) && defined(__GNUC__) && __GNUC__ >= 4 +#define FY_DEBUG_UNUSED __attribute__((unused)) +#else +#define FY_DEBUG_UNUSED /* nothing */ +#endif + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define FY_CONSTRUCTOR __attribute__((constructor)) +#define FY_DESTRUCTOR __attribute__((destructor)) +#define FY_HAS_CONSTRUCTOR +#define FY_HAS_DESTRUCTOR +#else +#define FY_CONSTRUCTOR /* nothing */ +#define FY_DESTRUCTOR /* nothing */ +#endif + +#if defined(FY_DESTRUCTOR) && defined(NDEBUG) +#define FY_DESTRUCTOR_SHOW_LEFTOVERS +#endif + +/* something impossible is happening, assert/abort */ +#define FY_IMPOSSIBLE_ABORT() \ + do { \ + assert(0); \ + abort(); \ + } while(0) + +static inline void *fy_ptr_align(void *p, size_t align) +{ + return (void *)(((uintptr_t)p + (align - 1)) & ~(align - 1)); +} + +static inline size_t fy_size_t_align(size_t size, size_t align) +{ + return (size + (align - 1)) & ~(align - 1); +} + +int fy_term_set_raw(int fd, struct termios *oldt); +int fy_term_restore(int fd, const struct termios *oldt); +ssize_t fy_term_write(int fd, const void *data, size_t count); +int fy_term_safe_write(int fd, const void *data, size_t count); +ssize_t fy_term_read(int fd, void *data, size_t count, int timeout_us); +ssize_t fy_term_read_escape(int fd, void *buf, size_t count); + +/* the raw methods require the terminal to be in raw mode */ +int fy_term_query_size_raw(int fd, int *rows, int *cols); + +/* the non raw methods will set the terminal to raw and then restore */ +int fy_term_query_size(int fd, int *rows, int *cols); + +struct fy_comment_iter { + const char *start; + size_t size; + const char *end; + const char *next; + int line; +}; + +int fy_comment_iter_begin(const char *comment, size_t size, struct fy_comment_iter *iter); +const char *fy_comment_iter_next_line(struct fy_comment_iter *iter, size_t *lenp); +void fy_comment_iter_end(struct fy_comment_iter *iter); + +char *fy_get_cooked_comment(const char *raw_comment, size_t size); + +struct fy_keyword_iter { + const char *keyword; + size_t keyword_len; + const char *start; + size_t size; + const char *end; + const char *next; + int pc; +}; + +int fy_keyword_iter_begin(const char *text, size_t size, const char *keyword, struct fy_keyword_iter *iter); +const char *fy_keyword_iter_next(struct fy_keyword_iter *iter); +void fy_keyword_iter_advance(struct fy_keyword_iter *iter, size_t advance); +void fy_keyword_iter_end(struct fy_keyword_iter *iter); + +#if !defined(S_ISREG) && defined(S_IFMT) && defined(S_IFREG) +#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) +#endif + +#if !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR) +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +#endif + +/* if expression is zero, then the build will break */ +#define FY_COMPILE_ERROR_ON_ZERO(_e) ((void)(sizeof(char[1 - 2*!!(_e)]))) + +/* make things fail easy on non GCC/clang */ +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +/* true, if types are the same, false otherwise (depends on builtin_types_compatible_p) */ +#if __has_builtin(__builtin_types_compatible_p) +#define FY_SAME_TYPE(_a, _b) __builtin_types_compatible_p(__typeof__(_a), __typeof__(_b)) +#else +#define FY_SAME_TYPE(_a, _b) true +#endif + +/* compile error if types are not the same */ +#define FY_CHECK_SAME_TYPE(_a, _b) \ + FY_COMPILE_ERROR_ON_ZERO(!FY_SAME_TYPE(_a, _b)) + +/* type safe add overflow */ +#if __has_builtin(__builtin_add_overflow) +#define FY_ADD_OVERFLOW __builtin_add_overflow +#else +#define FY_ADD_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + FY_CHECK_SAME_TYPE(__a, __b); \ + \ + __res = __a + __b; \ + /* overflow when signs of a, b same, but results different */ \ + __overflow = ((__a ^ __result) & (__b & __result)) < 0; \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +/* type safe sub overflow */ +#if __has_builtin(__builtin_sub_overflow) +#define FY_SUB_OVERFLOW __builtin_sub_overflow +#else +#define FY_SUB_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + FY_CHECK_SAME_TYPE(__a, __b); \ + \ + __res = __a - __b; \ + /* overflow when signs of a, b differ, but results different from minuend */ \ + __overflow = ((__a ^ __b) & (__a & __result)) < 0; \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +/* type safe multiply overflow */ +#if __has_builtin(__builtin_mul_overflow) +#define FY_MUL_OVERFLOW __builtin_mul_overflow +#else +#define FY_MUL_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + FY_CHECK_SAME_TYPE(__a, __b); \ + \ + if (!__a || !__b) { \ + __overflow = false; \ + __res = 0; \ + } else { \ + __res = __a * __b; \ + /* overflow when division of the result differs */ \ + __overflow = (__res / __a) != __b; \ + } \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +static inline void fy_strip_trailing_nl(char *str) +{ + char *s; + + if (str) { + s = str + strlen(str); + while (s > str && s[-1] == '\n') + *--s = '\0'; + } +} + +/* alloca formatted print methods */ +#define fy_vsprintfa(_fmt, _ap) \ + ({ \ + const char *__fmt = (_fmt); \ + va_list _ap_orig; \ + int _size; \ + char *_buf; \ + \ + va_copy(_ap_orig, (_ap)); \ + _size = vsnprintf(NULL, 0, __fmt, _ap_orig); \ + va_end(_ap_orig); \ + _buf = alloca(_size + 1); \ + (void)vsnprintf(_buf, _size + 1, __fmt, (_ap)); \ + _buf; \ + }) + +#define fy_sprintfa(_fmt, ...) \ + ({ \ + const char *__fmt = (_fmt); \ + int _size; \ + char *_buf; \ + \ + _size = snprintf(NULL, 0, __fmt, ## __VA_ARGS__); \ + _buf = alloca(_size + 1); \ + (void)snprintf(_buf, _size + 1, __fmt, __VA_ARGS__); \ + _buf; \ + }) + +#if !defined(NDEBUG) && defined(HAVE_DEVMODE) && HAVE_DEVMODE +#define FY_DEVMODE +#else +#undef FY_DEVMODE +#endif + +#ifdef FY_DEVMODE +#define __FY_DEBUG_UNUSED__ /* nothing */ +#else +#if defined(__GNUC__) && __GNUC__ >= 4 +#define __FY_DEBUG_UNUSED__ __attribute__((__unused__)) +#else +#define __FY_DEBUG_UNUSED__ /* nothing */ +#endif +#endif + +// C preprocessor magic follows (pilfered and adapted from h4x0r.org) + +/* applies an expansion over a varargs list */ + +#ifdef FY_CPP_SHORT_NAMES + +/* Short-name implementation - reduces expansion buffer usage by ~60% */ + +#define _E1(...) __VA_ARGS__ +#define _E2(...) _E1(_E1(__VA_ARGS__)) +#define _E4(...) _E2(_E2(__VA_ARGS__)) +#define _E8(...) _E4(_E4(__VA_ARGS__)) +#define _E16(...) _E8(_E8(__VA_ARGS__)) +#define _E32(...) _E16(_E16(__VA_ARGS__)) +#define _E(...) _E32(_E32(__VA_ARGS__)) + +#define _FMT() +#define _FP1(m) m _FMT() +#define _FP2(a, m) m _FMT() + +#define _F1(_1, ...) _1 +#define _F1F(...) _F1(__VA_OPT__(__VA_ARGS__ ,) 0) + +#define _F2(_1, _2, ...) _2 +#define _F2F(...) _F2(__VA_OPT__(__VA_ARGS__ ,) 0, 0) + +#define _F3(_1, _2, _3, ...) _3 +#define _F3F(...) _F3(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0) + +#define _F4(_1, _2, _3, _4, ...) _4 +#define _F4F(...) _F4(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0) + +#define _F5(_1, _2, _3, _4, _5, ...) _5 +#define _F5F(...) \ + _F5(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0, 0) + +#define _F6(_1, _2, _3, _4, _5, _6, ...) _6 +#define _F6F(...) \ + _F6(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0, 0, 0) + +#define _FR(x, ...) __VA_ARGS__ +#define _FRF(...) __VA_OPT__(_FR(__VA_ARGS__)) + +#define _FM(m, ...) \ + __VA_OPT__(_E(_FM1(m, __VA_ARGS__))) +#define _FM1(m, x, ...) m(x) \ + __VA_OPT__(_FP1(_FMI)()(m, __VA_ARGS__)) +#define _FMI() _FM1 + +#define _FCB(x) +1 +#define _FVC(...) (__VA_OPT__((_FM(_FCB, __VA_ARGS__) + 0)) + 0) + +#define _FI1(a) a +#define _FIL(a) , _FI1(a) +#define _FILS(...) _FM(_FIL, __VA_ARGS__) + +#define _FVIS(...) \ + _FI1(_F1F(__VA_ARGS__)) \ + _FILS(_FRF(__VA_ARGS__)) + +#define _FVISF(_t, ...) \ + ((_t [_FVC(__VA_ARGS__)]) { _FVIS(__VA_ARGS__) }) + +#define _FM2(a, m, ...) \ + __VA_OPT__(_E(_FM12(a, m, __VA_ARGS__))) + +#define _FM12(a, m, x, ...) m(a, x) \ + __VA_OPT__(_FP2(a, _FMI2)()(a, m, __VA_ARGS__)) +#define _FMI2() _FM12 + +/* Public API - maps to short names */ +#define FY_CPP_EVAL1 _E1 +#define FY_CPP_EVAL2 _E2 +#define FY_CPP_EVAL4 _E4 +#define FY_CPP_EVAL8 _E8 +#if !defined(__clang__) +#define FY_CPP_EVAL16 _E16 +#endif +#define FY_CPP_EVAL _E +#define FY_CPP_EMPTY _FMT +#define FY_CPP_POSTPONE1 _FP1 +#define FY_CPP_POSTPONE2 _FP2 + +#define FY_CPP_FIRST _F1F +#define FY_CPP_SECOND _F2F +#define FY_CPP_THIRD _F3F +#define FY_CPP_FOURTH _F4F +#define FY_CPP_FIFTH _F5F +#define FY_CPP_SIXTH _F6F +#define FY_CPP_REST _FRF + +#define FY_CPP_MAP _FM +#define _FY_CPP_MAP_ONE _FM1 +#define _FY_CPP_MAP_INDIRECT _FMI +#define _FY_CPP_COUNT_BODY _FCB +#define FY_CPP_VA_COUNT _FVC +#define _FY_CPP_ITEM_ONE _FI1 +#define _FY_CPP_ITEM_LATER_ARG _FIL +#define _FY_CPP_ITEM_LIST _FILS +#define _FY_CPP_VA_ITEMS _FVIS +#define FY_CPP_VA_ITEMS _FVISF +#define FY_CPP_MAP2 _FM2 +#define _FY_CPP_MAP_ONE2 _FM12 +#define _FY_CPP_MAP_INDIRECT2 _FMI2 + +#else /* !FY_CPP_SHORT_NAMES */ + +/* Original long-name implementation */ + +#define FY_CPP_EVAL1(...) __VA_ARGS__ +#define FY_CPP_EVAL2(...) FY_CPP_EVAL1(FY_CPP_EVAL1(__VA_ARGS__)) +#define FY_CPP_EVAL4(...) FY_CPP_EVAL2(FY_CPP_EVAL2(__VA_ARGS__)) +#define FY_CPP_EVAL8(...) FY_CPP_EVAL4(FY_CPP_EVAL4(__VA_ARGS__)) +#if !defined(__clang__) +// gcc is better, goes to 16 +#define FY_CPP_EVAL16(...) FY_CPP_EVAL8(FY_CPP_EVAL8(__VA_ARGS__)) +#define FY_CPP_EVAL(...) FY_CPP_EVAL16(FY_CPP_EVAL16(__VA_ARGS__)) +#else +// clang craps out at 8 +#define FY_CPP_EVAL(...) FY_CPP_EVAL8(FY_CPP_EVAL8(__VA_ARGS__)) +#endif + +#define FY_CPP_EMPTY() +#define FY_CPP_POSTPONE1(macro) macro FY_CPP_EMPTY() + +#define FY_CPP_MAP(macro, ...) \ + __VA_OPT__(FY_CPP_EVAL(_FY_CPP_MAP_ONE(macro, __VA_ARGS__))) +#define _FY_CPP_MAP_ONE(macro, x, ...) macro(x) \ + __VA_OPT__(FY_CPP_POSTPONE1(_FY_CPP_MAP_INDIRECT)()(macro, __VA_ARGS__)) +#define _FY_CPP_MAP_INDIRECT() _FY_CPP_MAP_ONE + +#define FY_CPP_POSTPONE2(a, macro) macro FY_CPP_EMPTY() + +#define _FY_CPP_FIRST(_1, ...) _1 +#define FY_CPP_FIRST(...) _FY_CPP_FIRST(__VA_OPT__(__VA_ARGS__ ,) 0) + +#define _FY_CPP_SECOND(_1, _2, ...) _2 +#define FY_CPP_SECOND(...) _FY_CPP_SECOND(__VA_OPT__(__VA_ARGS__ ,) 0, 0) + +#define _FY_CPP_THIRD(_1, _2, _3, ...) _3 +#define FY_CPP_THIRD(...) _FY_CPP_THIRD(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0) + +#define _FY_CPP_FOURTH(_1, _2, _3, _4, ...) _4 +#define FY_CPP_FOURTH(...) _FY_CPP_FOURTH(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0) + +#define _FY_CPP_FIFTH(_1, _2, _3, _4, _5, ...) _5 +#define FY_CPP_FIFTH(...) \ + _FY_CPP_FIFTH(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0, 0) + +#define _FY_CPP_SIXTH(_1, _2, _3, _4, _5, _6, ...) _6 +#define FY_CPP_SIXTH(...) \ + _FY_CPP_SIXTH(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0, 0, 0) + +#define _FY_CPP_REST(x, ...) __VA_ARGS__ +#define FY_CPP_REST(...) __VA_OPT__(_FY_CPP_REST(__VA_ARGS__)) + +#define _FY_CPP_COUNT_BODY(x) +1 +#define FY_CPP_VA_COUNT(...) (__VA_OPT__((FY_CPP_MAP(_FY_CPP_COUNT_BODY, __VA_ARGS__) + 0)) + 0) + +#define _FY_CPP_ITEM_ONE(arg) arg +#define _FY_CPP_ITEM_LATER_ARG(arg) , _FY_CPP_ITEM_ONE(arg) +#define _FY_CPP_ITEM_LIST(...) FY_CPP_MAP(_FY_CPP_ITEM_LATER_ARG, __VA_ARGS__) + +#define _FY_CPP_VA_ITEMS(...) \ + _FY_CPP_ITEM_ONE(FY_CPP_FIRST(__VA_ARGS__)) \ + _FY_CPP_ITEM_LIST(FY_CPP_REST(__VA_ARGS__)) + +#define FY_CPP_VA_ITEMS(_type, ...) \ + ((_type [FY_CPP_VA_COUNT(__VA_ARGS__)]) { _FY_CPP_VA_ITEMS(__VA_ARGS__) }) + +#define FY_CPP_MAP2(a, macro, ...) \ + __VA_OPT__(FY_CPP_EVAL(_FY_CPP_MAP_ONE2(a, macro, __VA_ARGS__))) + +#define _FY_CPP_MAP_ONE2(a, macro, x, ...) macro(a, x) \ + __VA_OPT__(FY_CPP_POSTPONE2(a, _FY_CPP_MAP_INDIRECT2)()(a, macro, __VA_ARGS__)) +#define _FY_CPP_MAP_INDIRECT2() _FY_CPP_MAP_ONE2 + +#endif /* FY_CPP_SHORT_NAMES */ + +/* + * example usage: + * + * int do_sum(int count, int *items) + * { + * int i; + * int sum; + * + * sum = 0; + * for (i = 0; i < count; i++) + * sum += items[i]; + * + * return sum; + * } + * + * #define do_sum_macro(...) \ + * do_sum( \ + * FY_CPP_VA_COUNT(__VA_ARGS__), \ + * FY_CPP_VA_ITEMS(int, __VA_ARGS__)) + * + * sum = do_sum_macro(1, 2, 5, 100); + * + * will expand to: + * + * sum = do_sum(4, ((int [4]){ 1, 2, 5, 100 })); + * + * printf("sum=%d\n", sum); + */ + +#define fy_alloca_align(_sz, _align) \ + ({ \ + const size_t __sz = (_sz); \ + const size_t __align = (_align); \ + void *__p; \ + __p = __align <= sizeof(max_align_t) ? alloca(__sz) : fy_ptr_align(alloca(__sz + __align - 1), __align); \ + __p; \ + }) + +static inline size_t +fy_iovec_size(const struct iovec *iov, int iovcnt) +{ + size_t size; + int i; + + size = 0; + for (i = 0; i < iovcnt; i++) { + if (FY_ADD_OVERFLOW(size, iov[i].iov_len, &size)) + return SIZE_MAX; + } + return size; +} + +static inline void * +fy_iovec_copy_from(const struct iovec *iov, int iovcnt, void *dst) +{ + size_t size; + int i; + + for (i = 0; i < iovcnt; i++, dst += size) { + size = iov[i].iov_len; + memcpy(dst, iov[i].iov_base, size); + } + return dst; +} + +static inline const void * +fy_iovec_copy_to(const struct iovec *iov, int iovcnt, const void *src) +{ + size_t size; + int i; + + for (i = 0; i < iovcnt; i++, src += size) { + size = iov[i].iov_len; + memcpy(iov[i].iov_base, src, size); + } + return src; +} + +static inline int +fy_iovec_cmp(const struct iovec *iov, int iovcnt, const void *data) +{ + const void *s = data; + size_t size; + int i, ret; + + for (i = 0; i < iovcnt; i++) { + size = iov[i].iov_len; + ret = memcmp(iov[i].iov_base, s, size); + if (ret) + return ret; + s += size; + } + return 0; +} + +/* safe bet for 2025 */ +#define FY_CACHE_LINE_SZ 64 +#define FY_CACHE_ALIGNED __attribute__((aligned(FY_CACHE_LINE_SZ))) + +uint64_t fy_iovec_xxhash64(const struct iovec *iov, int iovcnt); + +#define FY_CONCAT(_a, _b) FY_CONCAT_INNER(_a, _b) +#define FY_CONCAT_INNER(_a, _b) _a ## _b +#define FY_UNIQUE(_base) FY_CONCAT(_base, __COUNTER__) +#define FY_LUNIQUE(_base) FY_CONCAT(_base, __LINE__) + +#if __has_builtin(__builtin_stack_save) && __has_builtin(__builtin_stack_restore) +#define FY_STACK_SAVE() __builtin_stack_save() +#define FY_STACK_RESTORE(_x) __builtin_stack_restore((_x)) +#else +/* no builtin? just ignore */ +#define FY_STACK_SAVE() ((void *)NULL) +#define FY_STACK_RESTORE(_x) do { (void)(_x); } while(0) +#endif + +/* possible have lambdas, either gcc or clang with block support */ +#ifdef __clang__ +#ifdef __BLOCKS__ +#define FY_HAVE_LAMBDAS +#define FY_HAVE_BLOCK_LAMBDAS +#endif +#elif defined(__GNUC__) +#define FY_HAVE_LAMBDAS +#define FY_HAVE_NESTED_FUNC_LAMBDAS +#endif + +#endif diff --git a/src/util/fy-vlsize.h b/src/util/fy-vlsize.h new file mode 100644 index 0000000..568407d --- /dev/null +++ b/src/util/fy-vlsize.h @@ -0,0 +1,564 @@ +/* + * fy-vlsize.h - variable length size encoding + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_VLSIZE_H +#define FY_VLSIZE_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +// size encoding for 64 bit +// +// high bit is set, more follow with 0 ending the run +// The final 9th byte always terminates the run +// +// 0 1 2 3 4 5 6 7 8 +// -- -- -- -- -- -- -- -- -- +// 0k 7 bit length k +// 1k 0l 14 bit length kl +// 1k 1l 0m 21 bit length klm +// 1k 1l 1m 0n 28 bit length klmn +// 1k 1l 1m 1n 0o 35 bit length klmno +// 1k 1l 1m 1n 1o 0p 42 bit length klmnop +// 1k 1l 1m 1n 1o 1p 0q 49 bit length klmnopq +// 1k 1l 1m 1n 1o 1p 1q 0r 56 bit length klmnopqr +// 1k 1l 1m 1n 1o 1p 1q 1r t 64 bit length klmnopqrt +// +// size encoding for 32 bit +// +// high bit is set, more follow with 0 ending the run +// The final 9th byte always terminates the run +// +// 0 1 2 3 4 5 6 7 8 +// -- -- -- -- -- -- -- -- -- +// 0k 7 bit length k +// 1k 0l 14 bit length kl +// 1k 1l 0m 21 bit length klm +// 1k 1l 1m 0n 28 bit length klmn +// 1xk 1l 1m 1n o 32 bit length klmno 4 high bits ignored (at byte 0) +// + +#define FYVL_SIZE_ENCODING_MAX_64 9 // 7 * 8 + 8 = 64 bits +#define FYVL_SIZE_ENCODING_MAX_32 5 // 7 * 4 + 4 = 32 bits + +/* 32 bit specific */ +static inline unsigned int +fy_encode_size32_bytes(uint32_t size) +{ + if (size < ((uint32_t)1 << 7)) + return 1; + if (size < ((uint32_t)1 << 14)) + return 2; + if (size < ((uint32_t)1 << 21)) + return 3; + if (size < ((uint32_t)1 << 28)) + return 4; + return 5; +} + +static inline uint8_t * +fy_encode_size32(uint8_t *p, uint32_t bufsz, uint32_t size) +{ + uint8_t *end = p + bufsz; + + if (size < ((uint32_t)1 << 7)) { + if (p + 1 > end) + return NULL; + p[0] = (uint8_t)size; + return p + 1; + } + if (size < ((uint32_t)1 << 14)) { + if (p + 2 > end) + return NULL; + p[0] = (uint8_t)(size >> 7) | 0x80; + p[1] = (uint8_t)size & 0x7f; + return p + 2; + } + if (size < ((uint32_t)1 << 21)) { + if (p + 3 > end) + return NULL; + p[0] = (uint8_t)(size >> 14) | 0x80; + p[1] = (uint8_t)(size >> 7) | 0x80; + p[2] = (uint8_t)size & 0x7f; + return p + 3; + } + if (size < ((uint32_t)1 << 28)) { + if (p + 4 > end) + return NULL; + p[0] = (uint8_t)(size >> 21) | 0x80; + p[1] = (uint8_t)(size >> 14) | 0x80; + p[2] = (uint8_t)(size >> 7) | 0x80; + p[3] = (uint8_t)size & 0x7f; + return p + 4; + } + if (p + 5 > end) + return NULL; + p[0] = (uint8_t)(size >> 29) | 0x80; + p[1] = (uint8_t)(size >> 22) | 0x80; + p[2] = (uint8_t)(size >> 15) | 0x80; + p[3] = (uint8_t)(size >> 8) | 0x80; + p[4] = (uint8_t)size; + return p + 5; +} + +static inline const uint8_t * +fy_decode_size32(const uint8_t *start, size_t bufsz, uint32_t *sizep) +{ + const uint8_t *p, *end, *end_scan, *end_max_scan; + uint32_t size; + + end = start + bufsz; + + end_max_scan = start + FYVL_SIZE_ENCODING_MAX_32; + + if (end_max_scan < end) + end_scan = end_max_scan; + else + end_scan = end; + + end_max_scan--; + + p = start; + size = 0; + while (p < end_scan) { + if (p < end_max_scan) { + size <<= 7; + size |= (*p & 0x7f); + if (!(*p & 0x80)) + goto done; + } else { + /* last one is always the full 8 bit */ + size <<= 8; + size |= *p; + goto done; + } + p++; + } + + *sizep = (uint32_t)-1; + return NULL; +done: + if (++p >= end) + p = end; + *sizep = size; + return p; +} + +static inline const uint8_t * +fy_decode_size32_nocheck(const uint8_t *start, uint64_t *sizep) +{ + const uint8_t *p; + size_t size; + unsigned int i; + + p = start; + size = 0; + for (i = 0; i < FYVL_SIZE_ENCODING_MAX_32-1; i++) { + size <<= 7; + size |= (p[i] & 0x7f); + if ((int8_t)p[i] >= 0) + goto out; + } + size <<= 8; + size |= p[i]; +out: + *sizep = size; + return p + i + 1; +} + +static inline const uint8_t * +fy_skip_size32(const uint8_t *start, size_t bufsz) +{ + const uint8_t *p, *end, *end_scan, *end_max_scan; + + end = start + bufsz; + + end_max_scan = start + FYVL_SIZE_ENCODING_MAX_32; + + if (end_max_scan < end) + end_scan = end_max_scan; + else + end_scan = end; + + end_max_scan--; + + p = start; + while (p < end_scan) { + if (p < end_max_scan) { + if (!(*p & 0x80)) + goto done; + } else + goto done; + p++; + } + + return NULL; +done: + if (++p >= end) + p = end; + return p; +} + +static inline const uint8_t * +fy_skip_size32_nocheck(const uint8_t *p) +{ + unsigned int i; + + for (i = 0; i < FYVL_SIZE_ENCODING_MAX_32; ) { + if ((int8_t)p[i++] >= 0) + break; + } + return p + i; +} + +static inline unsigned int +fy_encode_size64_bytes(uint64_t size) +{ + if (size < ((uint64_t)1 << 7)) + return 1; + if (size < ((uint64_t)1 << 14)) + return 2; + if (size < ((uint64_t)1 << 21)) + return 3; + if (size < ((uint64_t)1 << 28)) + return 4; + if (size < ((uint64_t)1 << 35)) + return 5; + if (size < ((uint64_t)1 << 42)) + return 6; + if (size < ((uint64_t)1 << 49)) + return 7; + if (size < ((uint64_t)1 << 56)) + return 8; + return 9; +} + +static inline uint8_t * +fy_encode_size64(uint8_t *p, size_t bufsz, uint64_t size) +{ + uint8_t *end = p + bufsz; + + if (size < ((uint64_t)1 << 7)) { + if (p + 1 > end) + return NULL; + p[0] = (uint8_t)size; + return p + 1; + } + if (size < ((uint64_t)1 << 14)) { + if (p + 2 > end) + return NULL; + p[0] = (uint8_t)(size >> 7) | 0x80; + p[1] = (uint8_t)size & 0x7f; + return p + 2; + } + if (size < ((uint64_t)1 << 21)) { + if (p + 3 > end) + return NULL; + p[0] = (uint8_t)(size >> 14) | 0x80; + p[1] = (uint8_t)(size >> 7) | 0x80; + p[2] = (uint8_t)size & 0x7f; + return p + 3; + } + if (size < ((uint64_t)1 << 28)) { + if (p + 4 > end) + return NULL; + p[0] = (uint8_t)(size >> 21) | 0x80; + p[1] = (uint8_t)(size >> 14) | 0x80; + p[2] = (uint8_t)(size >> 7) | 0x80; + p[3] = (uint8_t)size & 0x7f; + return p + 4; + } + if (size < ((uint64_t)1 << 35)) { + if (p + 5 > end) + return NULL; + p[0] = (uint8_t)(size >> 28) | 0x80; + p[1] = (uint8_t)(size >> 21) | 0x80; + p[2] = (uint8_t)(size >> 14) | 0x80; + p[3] = (uint8_t)(size >> 7) | 0x80; + p[4] = (uint8_t)size & 0x7f; + return p + 5; + } + if (size < ((uint64_t)1 << 42)) { + if (p + 6 > end) + return NULL; + p[0] = (uint8_t)(size >> 35) | 0x80; + p[1] = (uint8_t)(size >> 28) | 0x80; + p[2] = (uint8_t)(size >> 21) | 0x80; + p[3] = (uint8_t)(size >> 14) | 0x80; + p[4] = (uint8_t)(size >> 7) | 0x80; + p[5] = (uint8_t)size & 0x7f; + return p + 6; + } + if (size < ((uint64_t)1 << 49)) { + if (p + 7 > end) + return NULL; + p[0] = (uint8_t)(size >> 42) | 0x80; + p[1] = (uint8_t)(size >> 35) | 0x80; + p[2] = (uint8_t)(size >> 28) | 0x80; + p[3] = (uint8_t)(size >> 21) | 0x80; + p[4] = (uint8_t)(size >> 14) | 0x80; + p[5] = (uint8_t)(size >> 7) | 0x80; + p[6] = (uint8_t)size & 0x7f; + return p + 7; + } + if (size < ((uint64_t)1 << 56)) { + if (p + 8 > end) + return NULL; + p[0] = (uint8_t)(size >> 49) | 0x80; + p[1] = (uint8_t)(size >> 42) | 0x80; + p[2] = (uint8_t)(size >> 35) | 0x80; + p[3] = (uint8_t)(size >> 28) | 0x80; + p[4] = (uint8_t)(size >> 21) | 0x80; + p[5] = (uint8_t)(size >> 14) | 0x80; + p[6] = (uint8_t)(size >> 7) | 0x80; + p[7] = (uint8_t)size & 0x7f; + return p + 8; + } + if (p + 9 > end) + return NULL; + p[0] = (uint8_t)(size >> 57) | 0x80; + p[1] = (uint8_t)(size >> 50) | 0x80; + p[2] = (uint8_t)(size >> 43) | 0x80; + p[3] = (uint8_t)(size >> 36) | 0x80; + p[4] = (uint8_t)(size >> 29) | 0x80; + p[5] = (uint8_t)(size >> 22) | 0x80; + p[6] = (uint8_t)(size >> 15) | 0x80; + p[7] = (uint8_t)(size >> 8) | 0x80; + p[8] = (uint8_t)size; + return p + 9; +} + +static inline const uint8_t * +fy_decode_size64(const uint8_t *start, size_t bufsz, uint64_t *sizep) +{ + const uint8_t *p, *end, *end_scan, *end_max_scan; + size_t size; + + end = start + bufsz; + + end_max_scan = start + FYVL_SIZE_ENCODING_MAX_64; + + if (end_max_scan < end) + end_scan = end_max_scan; + else + end_scan = end; + + end_max_scan--; + + p = start; + size = 0; + while (p < end_scan) { + if (p < end_max_scan) { + size <<= 7; + size |= (*p & 0x7f); + if (!(*p & 0x80)) + goto done; + } else { + /* last one is always 8 bit */ + size <<= 8; + size |= *p; + goto done; + } + p++; + } + + *sizep = (size_t)-1; + return NULL; +done: + if (++p >= end) + p = end; + *sizep = size; + return p; +} + +static inline const uint8_t * +fy_decode_size64_nocheck(const uint8_t *start, uint64_t *sizep) +{ + const uint8_t *p; + size_t size; + unsigned int i; + + p = start; + size = 0; + for (i = 0; i < FYVL_SIZE_ENCODING_MAX_64-1; i++) { + size <<= 7; + size |= (p[i] & 0x7f); + if ((int8_t)p[i] >= 0) + goto out; + } + size <<= 8; + size |= p[i]; +out: + *sizep = size; + return p + i + 1; +} + +static inline const uint8_t * +fy_skip_size64(const uint8_t *start, size_t bufsz) +{ + const uint8_t *p, *end, *end_scan, *end_max_scan; + + end = start + bufsz; + + end_max_scan = start + FYVL_SIZE_ENCODING_MAX_64; + + if (end_max_scan < end) + end_scan = end_max_scan; + else + end_scan = end; + + end_max_scan--; + + p = start; + while (p < end_scan) { + if (p < end_max_scan) { + if (!(*p & 0x80)) + goto done; + } else + goto done; + p++; + } + + return NULL; +done: + if (++p >= end) + p = end; + return p; +} + +static inline const uint8_t * +fy_skip_size64_nocheck(const uint8_t *p) +{ + unsigned int i; + + for (i = 0; i < FYVL_SIZE_ENCODING_MAX_64; ) { + if ((int8_t)p[i++] >= 0) + break; + } + return p + i; +} + +/* is pointless to pretend size_t is anything other than 64 or 32 bits */ +#if SIZE_MAX == UINT64_MAX + +static inline unsigned int +fy_encode_size_bytes(size_t size) +{ + return fy_encode_size64_bytes(size); +} + +static inline uint8_t * +fy_encode_size(uint8_t *p, size_t bufsz, size_t size) +{ + return fy_encode_size64(p, bufsz, size); +} + +static inline const uint8_t * +fy_decode_size(const uint8_t *start, size_t bufsz, size_t *sizep) +{ + uint64_t sz; + const uint8_t *ret; + + ret = fy_decode_size64(start, bufsz, &sz); + if (!ret) { + *sizep = 0; + return NULL; + } + *sizep = sz; + return ret; +} + +static inline const uint8_t * +fy_decode_size_nocheck(const uint8_t *start, size_t *sizep) +{ + uint64_t size64; + const uint8_t *s; + + switch (sizeof(size_t)) { + case sizeof(uint64_t): + s = fy_decode_size64_nocheck(start, &size64); + break; + case sizeof(uint32_t): + s = fy_decode_size32_nocheck(start, &size64); + break; + default: + size64 = 0; + s = NULL; + break; + } + *sizep = (size_t)size64; + return s; +} + +static inline const uint8_t * +fy_skip_size(const uint8_t *start, size_t bufsz) +{ + return fy_skip_size64(start, bufsz); +} + +static inline const uint8_t * +fy_skip_size_nocheck(const uint8_t *start) +{ + return fy_skip_size64_nocheck(start); +} + +#define FYVL_SIZE_ENCODING_MAX FYVL_SIZE_ENCODING_MAX_64 + +#else + +static inline unsigned int +fy_encode_size_bytes(size_t size) +{ + return fy_encode_size32_bytes(size); +} + +static inline uint8_t * +fy_encode_size(uint8_t *p, size_t bufsz, size_t size) +{ + return fy_encode_size32(p, bufsz, size); +} + +static inline const uint8_t * +fy_decode_size(const uint8_t *start, size_t bufsz, size_t *sizep) +{ + uint32_t sz; + const uint8_t *ret; + + ret = fy_decode_size32(start, bufsz, &sz); + if (!ret) + return NULL; + *sizep = sz; + return ret; +} + +static inline const uint8_t * +fy_decode_size_nocheck(const uint8_t *start, size_t *sizep) +{ + return fy_decode_size32_nocheck(start, sizep); +} + +static inline const uint8_t * +fy_skip_size(const uint8_t *start, size_t bufsz) +{ + return fy_skip_size32(start, bufsz, &sz); +} + +static inline const uint8_t * +fy_skip_size_nocheck(const uint8_t *start) +{ + return fy_skip_size32_nocheck(start); +} + +#define FYVL_SIZE_ENCODING_MAX FYVL_SIZE_ENCODING_MAX_32 + +#endif + +#endif diff --git a/src/valgrind/fy-valgrind.h b/src/valgrind/fy-valgrind.h new file mode 100644 index 0000000..36a64bd --- /dev/null +++ b/src/valgrind/fy-valgrind.h @@ -0,0 +1,166 @@ +/* + * fy-valgrind.h - valgrind auto option handling + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_VALGRIND_H +#define FY_VALGRIND_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + +#ifdef __linux__ +#include +#endif + +enum fy_valgrind_tool { + fyvt_none, + fyvt_valgrind, + fyvt_callgrind, + fyvt_massif, +}; + +static inline void fy_valgrind_check(int *argcp, char ***argvp) +{ + int argc = *argcp, va_argc; + char **argv = *argvp, **va_argv; + int i; +#ifdef __linux__ + char exe[PATH_MAX]; + ssize_t ret; +#endif + char *argv0 = NULL; + char *valgrind; + enum fy_valgrind_tool tool = fyvt_none; + + /* check for environment variables at first */ + if (getenv("USE_VALGRIND")) { + tool = fyvt_valgrind; + goto do_valgrind_no_opt; + } + if (getenv("USE_CALLGRIND") || getenv("USE_CACHEGRIND")) { + tool = fyvt_callgrind; + goto do_valgrind_no_opt; + } + if (getenv("USE_MASSIF")) { + tool = fyvt_callgrind; + goto do_valgrind_no_opt; + } + for (i = 1; i < argc; i++) { + /* -- is end of options */ + if (!strcmp(argv[i], "--")) + break; + + if (!strcmp(argv[i], "--valgrind")) { + tool = fyvt_valgrind; + break; + } + if (!strcmp(argv[i], "--callgrind") || !strcmp(argv[i], "--cachegrind")) { + tool = fyvt_callgrind; + break; + } + if (!strcmp(argv[i], "--massif")) { + tool = fyvt_massif; + break; + } + } + + if (tool == fyvt_none) + return; + + /* remove --valgrind/--callgrind/--massif from the option list */ + memmove(argv + i, argv + i + 1, (argc - i) * sizeof(*argv)); + (*argcp)--; + argc--; + +do_valgrind_no_opt: + /* clear those environment variables in any case */ + unsetenv("USE_VALGRIND"); + unsetenv("USE_CALLGRIND"); + unsetenv("USE_MASSIF"); + +#ifdef __linux__ + /* it's a Linuxism! but it should fail gracefully */ + ret = readlink("/proc/self/exe", exe, sizeof(exe)); + if (ret == 0) + argv0 = exe; +#endif + if (!argv0) + argv0 = argv[0]; + + valgrind = getenv("VALGRIND"); + if (valgrind) + unsetenv("VALGRIND"); + else + valgrind = "valgrind"; + + switch (tool) { + + case fyvt_valgrind: + va_argc = 1 + 5 + argc - 1; + va_argv = alloca(sizeof(*va_argv) * (va_argc + 1)); + va_argv[0] = valgrind; + va_argv[1] = "--leak-check=full"; + va_argv[2] = "--show-leak-kinds=all"; + va_argv[3] = "--track-origins=yes"; + va_argv[4] = "--error-exitcode=5"; + va_argv[5] = argv0; + memcpy(va_argv + 1 + 5, argv + 1, argc * sizeof(*va_argv)); + break; + + case fyvt_callgrind: + va_argc = 1 + 5 + argc - 1; + va_argv = alloca(sizeof(*va_argv) * (va_argc + 1)); + va_argv[0] = valgrind; + va_argv[1] = "--tool=callgrind"; + va_argv[2] = "--dump-instr=yes"; + va_argv[3] = "--simulate-cache=yes"; + va_argv[4] = "--collect-jumps=yes"; + va_argv[5] = argv0; + memcpy(va_argv + 1 + 5, argv + 1, argc * sizeof(*va_argv)); + break; + + case fyvt_massif: + va_argc = 1 + 2 + argc - 1; + va_argv = alloca(sizeof(*va_argv) * (va_argc + 1)); + va_argv[0] = valgrind; + va_argv[1] = "--tool=massif"; + va_argv[2] = argv0; + memcpy(va_argv + 1 + 2, argv + 1, argc * sizeof(*va_argv)); + break; + + default: + abort(); /* should never get here */ + break; + } + + assert(va_argv[va_argc] == NULL); + + for (i = 0; i < va_argc; i++) + fprintf(stderr, "[%d]: %s\n", i, va_argv[i]); + fprintf(stderr, "[%d]: %s\n", i, va_argv[i]); + + setenv("FY_VALGRIND", "1", 1); + + execvp("valgrind", va_argv); + + fprintf(stderr, "warning: failed to start valgrind, continue without"); +} + +#endif diff --git a/src/xxhash/xxhash.c b/src/xxhash/xxhash.c new file mode 100644 index 0000000..892c601 --- /dev/null +++ b/src/xxhash/xxhash.c @@ -0,0 +1,1881 @@ +/* +xxHash - Fast Hash algorithm +Copyright (C) 2012-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash source repository : http://code.google.com/p/xxhash/ +- public discussion board : https://groups.google.com/forum/#!forum/lz4c +*/ + + +//************************************** +// Tuning parameters +//************************************** +// Unaligned memory access is automatically enabled for "common" CPU, such as x86. +// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. +// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. +// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). +#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +// panto: -fsanitize fails with this so disable if compiling with it enabled +# if !defined(__SANITIZE_ADDRESS__) +# define XXH_USE_UNALIGNED_ACCESS 1 +# endif +#endif + +// XXH_ACCEPT_NULL_INPUT_POINTER : +// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. +// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. +// This option has a very small performance cost (only measurable on small inputs). +// By default, this option is disabled. To enable it, uncomment below define : +// #define XXH_ACCEPT_NULL_INPUT_POINTER 1 + +// XXH_FORCE_NATIVE_FORMAT : +// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. +// Results are therefore identical for little-endian and big-endian CPU. +// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. +// Should endian-independance be of no importance for your application, you may set the #define below to 1. +// It will improve speed for Big-endian CPU. +// This option has no impact on Little_Endian CPU. +#define XXH_FORCE_NATIVE_FORMAT 0 + +//************************************** +// Compiler Specific Options +//************************************** +// Disable some Visual warning messages +#ifdef _MSC_VER // Visual Studio +# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant +#endif + +#ifdef _MSC_VER // Visual Studio +# define FORCE_INLINE static __forceinline +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + +//************************************** +// Includes & Memory related functions +//************************************** +#include "xxhash.h" +// Modify the local functions below should you wish to use some other memory routines +// for malloc(), free() +#include +FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); } +FORCE_INLINE void XXH_free (void* p) { free(p); } +// for memcpy() +#include +FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +//************************************** +// Basic Types +//************************************** +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 +# include +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +#else +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; +#endif + + +#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# ifdef __IBMC__ +# pragma pack(1) +# else +# pragma pack(push, 1) +# endif +#endif + +typedef struct _U32_S +{ + U32 v; +} _PACKED U32_S; +typedef struct _U64_S +{ + U64 v; +} _PACKED U64_S; + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# pragma pack(pop) +#endif + +#define A32(x) (((U32_S *)(x))->v) +#define A64(x) (((U64_S *)(x))->v) + + +//*************************************** +// Compiler-specific Functions and Macros +//*************************************** +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +// Note : although _rotl exists for minGW (GCC under windows), performance seems poor +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + + +#if defined(_MSC_VER) // Visual Studio +# define XXH_swap32 _byteswap_ulong +# define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +# define XXH_swap64 __builtin_bswap64 +#else +FORCE_INLINE U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +FORCE_INLINE U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + +//************************************** +// Constants +//************************************** +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + +#define PRIME64_1 11400714785074694791ULL +#define PRIME64_2 14029467366897019727ULL +#define PRIME64_3 1609587929392839161ULL +#define PRIME64_4 9650029242287828579ULL +#define PRIME64_5 2870177450012600261ULL + +//************************************** +// Architecture Macros +//************************************** +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch +static const int one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) +#endif + + +//************************************** +// Macros +//************************************** +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations + + +//**************************** +// Memory reads +//**************************** +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); + else + return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +FORCE_INLINE U64 XXH_readLE64_align(const U64* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr)); + else + return endian==XXH_littleEndian ? *ptr : XXH_swap64(*ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const U64* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + +//**************************** +// Simple Hash Functions +//**************************** +FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align((const U32*)p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do + { + v1 += XXH_get32bits(p) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_get32bits(p) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_get32bits(p) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_get32bits(p) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } + else + { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p+4<=bEnd) + { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +unsigned int XXH32 (const void* input, size_t len, unsigned seed) +{ +#if 0 + // Simple version, good for code maintenance, but unfortunately slow for small inputs + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, input, len); + return XXH32_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align((const U64*)p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do + { + v1 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + v2 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + v3 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + v4 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + } + while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64 * PRIME64_1 + PRIME64_4; + } + else + { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p+8<=bEnd) + { + U64 k1 = XXH_get64bits(p); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) + { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; + +#undef XXH_get64bits +} + + +unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + // Simple version, good for code maintenance, but unfortunately slow for small inputs + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, input, len); + return XXH64_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +FORCE_INLINE void XXH128_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align, void* out) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U64 h1, h2; +#define XXH_get64bits(p) XXH_readLE64_align((const U64*)p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do + { + v1 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + v2 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + v3 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + v4 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + } + while (p<=limit); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h1 = v1; + h2 = ( XXH_rotl64(h1, 27) + h1 ) * PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 33); + v2 *= PRIME64_1; + h2 ^= v2; + h1 ^= ( XXH_rotl64(h2, 27) + h2 ) * PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 29); + v3 *= PRIME64_1; + h1 ^= v3; + h2 ^= ( XXH_rotl64(h1, 27) + h1 ) * PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 27); + v4 *= PRIME64_1; + h2 ^= v4; + h1 ^= ( XXH_rotl64(h2, 27) + h2 ) * PRIME64_1 + PRIME64_4; + } + else + { + h1 = seed + PRIME64_5; + h2 = seed + PRIME64_1; + } + + switch(len & 31) + { + case 31: h2 ^= ((U64)p[30]) << 48; + /* fall-through */ + case 30: h2 ^= ((U64)p[29]) << 40; + /* fall-through */ + case 29: h2 ^= ((U64)p[28]) << 32; + /* fall-through */ + case 28: h2 ^= ((U64)p[27]) << 24; + /* fall-through */ + case 27: h2 ^= ((U64)p[26]) << 16; + /* fall-through */ + case 26: h2 ^= ((U64)p[25]) << 8; + /* fall-through */ + case 25: h2 ^= ((U64)p[24]) << 0; + h1 ^= XXH_rotl64(h2 * PRIME64_2, 11) * PRIME64_1; + /* fall-through */ + case 24: h1 ^= ((U64)p[23]) << 56; + /* fall-through */ + case 23: h1 ^= ((U64)p[22]) << 48; + /* fall-through */ + case 22: h1 ^= ((U64)p[21]) << 40; + /* fall-through */ + case 21: h1 ^= ((U64)p[20]) << 32; + /* fall-through */ + case 20: h1 ^= ((U64)p[19]) << 24; + /* fall-through */ + case 19: h1 ^= ((U64)p[18]) << 16; + /* fall-through */ + case 18: h1 ^= ((U64)p[17]) << 8; + /* fall-through */ + case 17: h1 ^= ((U64)p[16]) << 0; + h2 ^= XXH_rotl64(h1 * PRIME64_2, 11) * PRIME64_1; + /* fall-through */ + case 16: h2 ^= ((U64)p[15]) << 56; + /* fall-through */ + case 15: h2 ^= ((U64)p[14]) << 48; + /* fall-through */ + case 14: h2 ^= ((U64)p[13]) << 40; + /* fall-through */ + case 13: h2 ^= ((U64)p[12]) << 32; + /* fall-through */ + case 12: h2 ^= ((U64)p[11]) << 24; + /* fall-through */ + case 11: h2 ^= ((U64)p[10]) << 16; + /* fall-through */ + case 10: h2 ^= ((U64)p[9]) << 8; + /* fall-through */ + case 9: h2 ^= ((U64)p[8]) << 0; + h1 ^= XXH_rotl64(h2 * PRIME64_2, 11) * PRIME64_1; + /* fall-through */ + case 8: h1 ^= ((U64)p[7]) << 56; + /* fall-through */ + case 7: h1 ^= ((U64)p[6]) << 48; + /* fall-through */ + case 6: h1 ^= ((U64)p[5]) << 40; + /* fall-through */ + case 5: h1 ^= ((U64)p[4]) << 32; + /* fall-through */ + case 4: h1 ^= ((U64)p[3]) << 24; + /* fall-through */ + case 3: h1 ^= ((U64)p[2]) << 16; + /* fall-through */ + case 2: h1 ^= ((U64)p[1]) << 8; + /* fall-through */ + case 1: h1 ^= ((U64)p[0]) << 0; + h2 ^= XXH_rotl64(h1 * PRIME64_5, 11) * PRIME64_1; + /* fall-through */ + } + + h1 = XXH_rotl64(h2, 27) * PRIME64_1 + PRIME64_4; + + h1 += (U64) len; + h2 += (U64) len; + + h2 ^= h1 >> 33; + h2 *= PRIME64_2; + h1 ^= h2 >> 29; + h1 *= PRIME64_3; + h2 ^= h1 >> 32; + + ((U64*)out)[0] = h1; + ((U64*)out)[1] = h2; + +#undef XXH_get64bits +} + +void XXH128 (const void* input, size_t len, unsigned long long seed, void* out) +{ +#if 0 + XXH128_state_t state; + XXH128_reset(&state, seed); + XXH128_update(&state, input, len); + XXH128_digest(&state, out); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + XXH128_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned, out); + else + XXH128_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned, out); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + XXH128_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned, out); + else + XXH128_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned, out); +#endif +} + + +FORCE_INLINE void XXH256_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align, void* out) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U64 h1, h2, h3, h4; + +#define XXH_get64bits(p) XXH_readLE64_align((const U64*)p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do + { + v1 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + v2 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + v3 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + v4 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + } + while (p<=limit); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h1 = v1; + h2 = ( XXH_rotl64(h1, 27) + h1 ) * PRIME64_1 + PRIME64_2; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 33); + v2 *= PRIME64_1; + h2 ^= v2; + h3 = ( XXH_rotl64(h2, 29) + h2 ) * PRIME64_2 + PRIME64_3; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 29); + v3 *= PRIME64_1; + h3 ^= v3; + h4 = ( XXH_rotl64(h3, 31) + h3 ) * PRIME64_3 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 27); + v4 *= PRIME64_1; + h4 ^= v4; + h1 ^= ( XXH_rotl64(h4, 33) + h4 ) * PRIME64_4 + PRIME64_5; + } + else + { + h1 = seed + PRIME64_5; + h2 = seed + PRIME64_1; + h3 = seed + PRIME64_4; + h4 = seed + PRIME64_2; + } + + switch(len & 31) + { + case 31: h4 ^= ((U64)p[30]) << 48; + /* fall-through */ + case 30: h4 ^= ((U64)p[29]) << 40; + /* fall-through */ + case 29: h4 ^= ((U64)p[28]) << 32; + /* fall-through */ + case 28: h4 ^= ((U64)p[27]) << 24; + /* fall-through */ + case 27: h4 ^= ((U64)p[26]) << 16; + /* fall-through */ + case 26: h4 ^= ((U64)p[25]) << 8; + /* fall-through */ + case 25: h4 ^= ((U64)p[24]) << 0; + h3 ^= XXH_rotl64(h4 * PRIME64_5, 17) * PRIME64_1; + /* fall-through */ + case 24: h3 ^= ((U64)p[23]) << 56; + /* fall-through */ + case 23: h3 ^= ((U64)p[22]) << 48; + /* fall-through */ + case 22: h3 ^= ((U64)p[21]) << 40; + /* fall-through */ + case 21: h3 ^= ((U64)p[20]) << 32; + /* fall-through */ + case 20: h3 ^= ((U64)p[19]) << 24; + /* fall-through */ + case 19: h3 ^= ((U64)p[18]) << 16; + /* fall-through */ + case 18: h3 ^= ((U64)p[17]) << 8; + /* fall-through */ + case 17: h3 ^= ((U64)p[16]) << 0; + h2 ^= XXH_rotl64(h3 * PRIME64_5, 13) * PRIME64_1; + /* fall-through */ + case 16: h2 ^= ((U64)p[15]) << 56; + /* fall-through */ + case 15: h2 ^= ((U64)p[14]) << 48; + /* fall-through */ + case 14: h2 ^= ((U64)p[13]) << 40; + /* fall-through */ + case 13: h2 ^= ((U64)p[12]) << 32; + /* fall-through */ + case 12: h2 ^= ((U64)p[11]) << 24; + /* fall-through */ + case 11: h2 ^= ((U64)p[10]) << 16; + /* fall-through */ + case 10: h2 ^= ((U64)p[9]) << 8; + /* fall-through */ + case 9: h2 ^= ((U64)p[8]) << 0; + h1 ^= XXH_rotl64(h2 * PRIME64_5, 11) * PRIME64_1; + /* fall-through */ + case 8: h1 ^= ((U64)p[7]) << 56; + /* fall-through */ + case 7: h1 ^= ((U64)p[6]) << 48; + /* fall-through */ + case 6: h1 ^= ((U64)p[5]) << 40; + /* fall-through */ + case 5: h1 ^= ((U64)p[4]) << 32; + /* fall-through */ + case 4: h1 ^= ((U64)p[3]) << 24; + /* fall-through */ + case 3: h1 ^= ((U64)p[2]) << 16; + /* fall-through */ + case 2: h1 ^= ((U64)p[1]) << 8; + /* fall-through */ + case 1: h1 ^= ((U64)p[0]) << 0; + h4 ^= XXH_rotl64(h1 * PRIME64_5, 7) * PRIME64_1; + /* fall-through */ + } + + h2 ^= ( XXH_rotl64(h1, 27) + h1 ) * PRIME64_1 + PRIME64_4; + h3 ^= ( XXH_rotl64(h2, 29) + h2 ) * PRIME64_2 + PRIME64_3; + h4 ^= ( XXH_rotl64(h3, 31) + h3 ) * PRIME64_3 + PRIME64_2; + h1 ^= ( XXH_rotl64(h4, 33) + h4 ) * PRIME64_4 + PRIME64_1; + + h1 += (U64) len; + h2 += (U64) len; + h3 += (U64) len; + h4 += (U64) len; + + h4 ^= h1 >> 33; + h4 *= PRIME64_2; + h1 ^= h4 >> 29; + h1 *= PRIME64_3; + h4 ^= h1 >> 32; + + h3 ^= h2 >> 33; + h3 *= PRIME64_2; + h2 ^= h3 >> 29; + h2 *= PRIME64_3; + h3 ^= h2 >> 32; + + ((unsigned long long*)out)[0] = h1; + ((unsigned long long*)out)[1] = h2; + ((unsigned long long*)out)[2] = h3; + ((unsigned long long*)out)[3] = h4; + +#undef XXH_get64bits +} + +void XXH256 (const void* input, size_t len, unsigned long long seed, void* out) +{ +#if 0 + XXH256_state_t state; + XXH256_reset(&state, seed); + XXH256_update(&state, input, len); + XXH256_digest(&state, out); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + XXH256_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned, out); + else + XXH256_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned, out); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + XXH256_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned, out); + else + XXH256_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned, out); +#endif +} + + +/**************************************************** + * Advanced Hash Functions +****************************************************/ + +/*** Allocation ***/ +typedef struct +{ + U64 total_len; + U32 seed; + U32 v1; + U32 v2; + U32 v3; + U32 v4; + U32 memsize; + char memory[16]; +} XXH_istate32_t; + +typedef struct +{ + U64 total_len; + U64 seed; + U64 v1; + U64 v2; + U64 v3; + U64 v4; + U32 memsize; + char memory[32]; +} XXH_istate64_t; + + +typedef struct +{ + U64 total_len; + U64 seed; + U64 v1; + U64 v2; + U64 v3; + U64 v4; + char memory[64]; + U32 memsize; +} XXH_istate128_t; + +typedef struct +{ + U64 total_len; + U64 seed; + U64 v1; + U64 v2; + U64 v3; + U64 v4; + char memory[64]; + U32 memsize; +} XXH_istate256_t; + + +XXH32_state_t* XXH32_createState(void) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); // A compilation error here means XXH32_state_t is not large enough + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +}; + +XXH64_state_t* XXH64_createState(void) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); // A compilation error here means XXH64_state_t is not large enough + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +}; + +XXH128_state_t* XXH128_createState(void) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_state_t) >= sizeof(XXH_istate128_t)); // A compilation error here means XXH128_state_t is not large enough + return (XXH128_state_t*)XXH_malloc(sizeof(XXH128_state_t)); +} +XXH_errorcode XXH128_freeState(XXH128_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + + +/*** Hash feed ***/ + +XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed) +{ + XXH_istate32_t* state = (XXH_istate32_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME32_1 + PRIME32_2; + state->v2 = seed + PRIME32_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME32_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + +XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed) +{ + XXH_istate64_t* state = (XXH_istate64_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME64_1 + PRIME64_2; + state->v2 = seed + PRIME64_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME64_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + +XXH_errorcode XXH128_reset(XXH128_state_t* state_in, unsigned long long seed) +{ + XXH_istate128_t* state = (XXH_istate128_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME64_1 + PRIME64_2; + state->v2 = seed + PRIME64_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME64_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + +XXH_errorcode XXH256_reset(XXH256_state_t* state_in, unsigned long long seed) +{ + XXH_istate256_t* state = (XXH_istate256_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME64_1 + PRIME64_2; + state->v2 = seed + PRIME64_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME64_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate32_t* state = (XXH_istate32_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 16) // fill in tmp buffer + { + XXH_memcpy(state->memory + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); + { + const U32* p32 = (const U32*)state->memory; + state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v1 = XXH_rotl32(state->v1, 13); + state->v1 *= PRIME32_1; + p32++; + state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v2 = XXH_rotl32(state->v2, 13); + state->v2 *= PRIME32_1; + p32++; + state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v3 = XXH_rotl32(state->v3, 13); + state->v3 *= PRIME32_1; + p32++; + state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v4 = XXH_rotl32(state->v4, 13); + state->v4 *= PRIME32_1; + p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do + { + v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->memory, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) +{ + XXH_istate32_t* state = (XXH_istate32_t*) state_in; + const BYTE * p = (const BYTE*)state->memory; + BYTE* bEnd = (BYTE*)state->memory + state->memsize; + U32 h32; + + if (state->total_len >= 16) + { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } + else + { + h32 = state->seed + PRIME32_5; + } + + h32 += (U32) state->total_len; + + while (p+4<=bEnd) + { + h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + +FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) // fill in tmp buffer + { + XXH_memcpy(state->memory + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy(state->memory + state->memsize, input, 32-state->memsize); + { + const U64* p64 = (const U64*)state->memory; + state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v1 = XXH_rotl64(state->v1, 31); + state->v1 *= PRIME64_1; + p64++; + state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v2 = XXH_rotl64(state->v2, 31); + state->v2 *= PRIME64_1; + p64++; + state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v3 = XXH_rotl64(state->v3, 31); + state->v3 *= PRIME64_1; + p64++; + state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v4 = XXH_rotl64(state->v4, 31); + state->v4 *= PRIME64_1; + p64++; + } + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do + { + v1 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + p+=8; + v2 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + p+=8; + v3 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + p+=8; + v4 += XXH_readLE64((const U64*)p, endian) * PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + p+=8; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->memory, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) +{ + XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const BYTE * p = (const BYTE*)state->memory; + BYTE* bEnd = (BYTE*)state->memory + state->memsize; + U64 h64; + + if (state->total_len >= 32) + { + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64*PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64*PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64*PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64*PRIME64_1 + PRIME64_4; + } + else + { + h64 = state->seed + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p+8<=bEnd) + { + U64 k1 = XXH_readLE64((const U64*)p, endian); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) + { + h64 ^= (U64)(XXH_readLE32((const U32*)p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + +FORCE_INLINE XXH_errorcode XXH128_update_endian (XXH128_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate128_t * state = (XXH_istate128_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; +#define XXH_get64bits(p) XXH_readLE64((const U64*)p, endian) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) // fill in tmp buffer + { + XXH_memcpy(state->memory + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy(state->memory + state->memsize, input, 32-state->memsize); + { + const BYTE* ps = (const BYTE*)state->memory; + state->v1 += XXH_get64bits(ps) * PRIME64_2; + state->v1 = XXH_rotl64(state->v1, 31); + state->v1 *= PRIME64_1; + ps+=8; + state->v2 += XXH_get64bits(ps) * PRIME64_2; + state->v2 = XXH_rotl64(state->v2, 31); + state->v2 *= PRIME64_1; + ps+=8; + state->v3 += XXH_get64bits(ps) * PRIME64_2; + state->v3 = XXH_rotl64(state->v3, 31); + state->v3 *= PRIME64_1; + ps+=8; + state->v4 += XXH_get64bits(ps) * PRIME64_2; + state->v4 = XXH_rotl64(state->v4, 31); + state->v4 *= PRIME64_1; + ps+=8; + } + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do + { + v1 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + v2 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + v3 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + v4 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->memory, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; + +#undef XXH_get64bits +} + +XXH_errorcode XXH128_update (XXH128_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH128_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH128_update_endian(state_in, input, len, XXH_bigEndian); +} + + +FORCE_INLINE void XXH128_digest_endian (const XXH128_state_t* state_in, XXH_endianess endian, void* out) +{ + (void)endian; + XXH_istate128_t * state = (XXH_istate128_t *) state_in; + const BYTE * p = (const BYTE*)state->memory; + U64 h1, h2; + + if (state->total_len >= 32) + { + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h1 = v1; + h2 = ( XXH_rotl64(h1, 27) + h1 ) * PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 33); + v2 *= PRIME64_1; + h2 ^= v2; + h1 ^= ( XXH_rotl64(h2, 27) + h2 ) * PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 29); + v3 *= PRIME64_1; + h1 ^= v3; + h2 ^= ( XXH_rotl64(h1, 27) + h1 ) * PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 27); + v4 *= PRIME64_1; + h2 ^= v4; + h1 ^= ( XXH_rotl64(h2, 27) + h2 ) * PRIME64_1 + PRIME64_4; + } + else + { + h1 = state->seed + PRIME64_5; + h2 = state->seed + PRIME64_1; + } + + switch(state->total_len & 31) + { + case 31: h2 ^= ((U64)p[30]) << 48; + /* fall-through */ + case 30: h2 ^= ((U64)p[29]) << 40; + /* fall-through */ + case 29: h2 ^= ((U64)p[28]) << 32; + /* fall-through */ + case 28: h2 ^= ((U64)p[27]) << 24; + /* fall-through */ + case 27: h2 ^= ((U64)p[26]) << 16; + /* fall-through */ + case 26: h2 ^= ((U64)p[25]) << 8; + /* fall-through */ + case 25: h2 ^= ((U64)p[24]) << 0; + h1 ^= XXH_rotl64(h2 * PRIME64_2, 11) * PRIME64_1; + /* fall-through */ + case 24: h1 ^= ((U64)p[23]) << 56; + /* fall-through */ + case 23: h1 ^= ((U64)p[22]) << 48; + /* fall-through */ + case 22: h1 ^= ((U64)p[21]) << 40; + /* fall-through */ + case 21: h1 ^= ((U64)p[20]) << 32; + /* fall-through */ + case 20: h1 ^= ((U64)p[19]) << 24; + /* fall-through */ + case 19: h1 ^= ((U64)p[18]) << 16; + /* fall-through */ + case 18: h1 ^= ((U64)p[17]) << 8; + /* fall-through */ + case 17: h1 ^= ((U64)p[16]) << 0; + h2 ^= XXH_rotl64(h1 * PRIME64_2, 11) * PRIME64_1; + /* fall-through */ + case 16: h2 ^= ((U64)p[15]) << 56; + /* fall-through */ + case 15: h2 ^= ((U64)p[14]) << 48; + /* fall-through */ + case 14: h2 ^= ((U64)p[13]) << 40; + /* fall-through */ + case 13: h2 ^= ((U64)p[12]) << 32; + /* fall-through */ + case 12: h2 ^= ((U64)p[11]) << 24; + /* fall-through */ + case 11: h2 ^= ((U64)p[10]) << 16; + /* fall-through */ + case 10: h2 ^= ((U64)p[9]) << 8; + /* fall-through */ + case 9: h2 ^= ((U64)p[8]) << 0; + h1 ^= XXH_rotl64(h2 * PRIME64_2, 11) * PRIME64_1; + /* fall-through */ + case 8: h1 ^= ((U64)p[7]) << 56; + /* fall-through */ + case 7: h1 ^= ((U64)p[6]) << 48; + /* fall-through */ + case 6: h1 ^= ((U64)p[5]) << 40; + /* fall-through */ + case 5: h1 ^= ((U64)p[4]) << 32; + /* fall-through */ + case 4: h1 ^= ((U64)p[3]) << 24; + /* fall-through */ + case 3: h1 ^= ((U64)p[2]) << 16; + /* fall-through */ + case 2: h1 ^= ((U64)p[1]) << 8; + /* fall-through */ + case 1: h1 ^= ((U64)p[0]) << 0; + h2 ^= XXH_rotl64(h1 * PRIME64_5, 11) * PRIME64_1; + /* fall-through */ + } + + h1 = XXH_rotl64(h2, 27) * PRIME64_1 + PRIME64_4; + + h1 += (U64) state->total_len; + h2 += (U64) state->total_len; + + h2 ^= h1 >> 33; + h2 *= PRIME64_2; + h1 ^= h2 >> 29; + h1 *= PRIME64_3; + h2 ^= h1 >> 32; + + ((U64*)out)[0] = h1; + ((U64*)out)[1] = h2; +} + +void XXH128_digest (const XXH128_state_t* state_in, void* out) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH128_digest_endian(state_in, XXH_littleEndian, (unsigned long long*)out); + else + return XXH128_digest_endian(state_in, XXH_bigEndian, (unsigned long long*)out); +} + + +FORCE_INLINE XXH_errorcode XXH256_update_endian (XXH256_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate256_t * state = (XXH_istate256_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) // fill in tmp buffer + { + XXH_memcpy(state->memory + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) // some data left from previous update + { + XXH_memcpy(state->memory + state->memsize, input, 32-state->memsize); + { + const U64* p64 = (const U64*)state->memory; + state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v1 = XXH_rotl64(state->v1, 31); + state->v1 *= PRIME64_1; + p64++; + state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v2 = XXH_rotl64(state->v2, 31); + state->v2 *= PRIME64_1; + p64++; + state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v3 = XXH_rotl64(state->v3, 31); + state->v3 *= PRIME64_1; + p64++; + state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v4 = XXH_rotl64(state->v4, 31); + state->v4 *= PRIME64_1; + p64++; + } + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do + { + v1 += XXH_readLE64((const U64*)p+0, endian) * PRIME64_2; + v1 = XXH_rotl64(v1, 31) * PRIME64_1; + + v2 += XXH_readLE64((const U64*)p+1, endian) * PRIME64_2; + v2 = XXH_rotl64(v2, 31) * PRIME64_1; + + v3 += XXH_readLE64((const U64*)p+2, endian) * PRIME64_2; + v3 = XXH_rotl64(v3, 31) * PRIME64_1; + + v4 += XXH_readLE64((const U64*)p+3, endian) * PRIME64_2; + v4 = XXH_rotl64(v4, 31) * PRIME64_1; + + p+=32; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->memory, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH256_update (XXH256_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH256_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH256_update_endian(state_in, input, len, XXH_bigEndian); +} + + +FORCE_INLINE void XXH256_digest_endian (const XXH256_state_t* state_in, XXH_endianess endian, void* out) +{ + (void)endian; + XXH_istate256_t * state = (XXH_istate256_t *) state_in; + const BYTE * p = (const BYTE*)state->memory; + U64 h1, h2, h3, h4; + + if (state->total_len >= 32) + { + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h1 = v1; + h2 = ( XXH_rotl64(h1, 27) + h1 ) * PRIME64_1 + PRIME64_2; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 33); + v2 *= PRIME64_1; + h2 ^= v2; + h3 = ( XXH_rotl64(h2, 29) + h2 ) * PRIME64_2 + PRIME64_3; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 29); + v3 *= PRIME64_1; + h3 ^= v3; + h4 = ( XXH_rotl64(h3, 31) + h3 ) * PRIME64_3 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 27); + v4 *= PRIME64_1; + h4 ^= v4; + h1 ^= ( XXH_rotl64(h4, 33) + h4 ) * PRIME64_4 + PRIME64_5; + } + else + { + h1 = state->seed + PRIME64_5; + h2 = state->seed + PRIME64_1; + h3 = state->seed + PRIME64_4; + h4 = state->seed + PRIME64_2; + } + + switch(state->total_len & 31) + { + case 31: h4 ^= ((U64)p[30]) << 48; + /* fall-through */ + case 30: h4 ^= ((U64)p[29]) << 40; + /* fall-through */ + case 29: h4 ^= ((U64)p[28]) << 32; + /* fall-through */ + case 28: h4 ^= ((U64)p[27]) << 24; + /* fall-through */ + case 27: h4 ^= ((U64)p[26]) << 16; + /* fall-through */ + case 26: h4 ^= ((U64)p[25]) << 8; + /* fall-through */ + case 25: h4 ^= ((U64)p[24]) << 0; + h3 ^= XXH_rotl64(h4 * PRIME64_5, 17) * PRIME64_1; + /* fall-through */ + case 24: h3 ^= ((U64)p[23]) << 56; + /* fall-through */ + case 23: h3 ^= ((U64)p[22]) << 48; + /* fall-through */ + case 22: h3 ^= ((U64)p[21]) << 40; + /* fall-through */ + case 21: h3 ^= ((U64)p[20]) << 32; + /* fall-through */ + case 20: h3 ^= ((U64)p[19]) << 24; + /* fall-through */ + case 19: h3 ^= ((U64)p[18]) << 16; + /* fall-through */ + case 18: h3 ^= ((U64)p[17]) << 8; + /* fall-through */ + case 17: h3 ^= ((U64)p[16]) << 0; + h2 ^= XXH_rotl64(h3 * PRIME64_5, 13) * PRIME64_1; + /* fall-through */ + case 16: h2 ^= ((U64)p[15]) << 56; + /* fall-through */ + case 15: h2 ^= ((U64)p[14]) << 48; + /* fall-through */ + case 14: h2 ^= ((U64)p[13]) << 40; + /* fall-through */ + case 13: h2 ^= ((U64)p[12]) << 32; + /* fall-through */ + case 12: h2 ^= ((U64)p[11]) << 24; + /* fall-through */ + case 11: h2 ^= ((U64)p[10]) << 16; + /* fall-through */ + case 10: h2 ^= ((U64)p[9]) << 8; + /* fall-through */ + case 9: h2 ^= ((U64)p[8]) << 0; + h1 ^= XXH_rotl64(h2 * PRIME64_5, 11) * PRIME64_1; + /* fall-through */ + case 8: h1 ^= ((U64)p[7]) << 56; + /* fall-through */ + case 7: h1 ^= ((U64)p[6]) << 48; + /* fall-through */ + case 6: h1 ^= ((U64)p[5]) << 40; + /* fall-through */ + case 5: h1 ^= ((U64)p[4]) << 32; + /* fall-through */ + case 4: h1 ^= ((U64)p[3]) << 24; + /* fall-through */ + case 3: h1 ^= ((U64)p[2]) << 16; + /* fall-through */ + case 2: h1 ^= ((U64)p[1]) << 8; + /* fall-through */ + case 1: h1 ^= ((U64)p[0]) << 0; + h4 ^= XXH_rotl64(h1 * PRIME64_5, 7) * PRIME64_1; + /* fall-through */ + } + + h2 ^= ( XXH_rotl64(h1, 27) + h1 ) * PRIME64_1 + PRIME64_4; + h3 ^= ( XXH_rotl64(h2, 29) + h2 ) * PRIME64_2 + PRIME64_3; + h4 ^= ( XXH_rotl64(h3, 31) + h3 ) * PRIME64_3 + PRIME64_2; + h1 ^= ( XXH_rotl64(h4, 33) + h4 ) * PRIME64_4 + PRIME64_1; + + h1 += (U64) state->total_len; + h2 += (U64) state->total_len; + h3 += (U64) state->total_len; + h4 += (U64) state->total_len; + + h4 ^= h1 >> 33; + h4 *= PRIME64_2; + h1 ^= h4 >> 29; + h1 *= PRIME64_3; + h4 ^= h1 >> 32; + + h3 ^= h2 >> 33; + h3 *= PRIME64_2; + h2 ^= h3 >> 29; + h2 *= PRIME64_3; + h3 ^= h2 >> 32; + + ((unsigned long long*)out)[0] = h1; + ((unsigned long long*)out)[1] = h2; + ((unsigned long long*)out)[2] = h3; + ((unsigned long long*)out)[3] = h4; +} + +void XXH256_digest (const XXH256_state_t* state_in, void* out) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH256_digest_endian(state_in, XXH_littleEndian, (unsigned long long*)out); + else + return XXH256_digest_endian(state_in, XXH_bigEndian, (unsigned long long*)out); +} diff --git a/src/xxhash/xxhash.h b/src/xxhash/xxhash.h new file mode 100644 index 0000000..3dc3967 --- /dev/null +++ b/src/xxhash/xxhash.h @@ -0,0 +1,193 @@ +#ifndef XXHASH_H +#define XXHASH_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2014, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : http://code.google.com/p/xxhash/ + + Modified for internal use for libfyaml by: + Pantelis Antoniou + + Minor cosmetic and warning fixes +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. +*/ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + + +/***************************** + Includes +*****************************/ +#include /* size_t */ + + +/***************************** + Type +*****************************/ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + + +/***************************** + Simple Hash Functions +*****************************/ + +unsigned int XXH32 (const void* input, size_t length, unsigned seed); +unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); +void XXH128 (const void* input, size_t length, unsigned long long seed, void* out); +void XXH256 (const void* input, size_t length, unsigned long long seed, void* out); + +/* +XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + This function successfully passes all SMHasher tests. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". +XXH128(): + Calculate the 128-bits hash of sequence of length "len" stored at memory address "input". + Output is stored in the 16 byte array "out" +XXH256(): + Calculate the 256-bits hash of sequence of length "len" stored at memory address "input". + Output is stored in the 32 byte array "out" +*/ + + + +/***************************** + Advanced Hash Functions +*****************************/ +typedef struct { long long ll[ 6]; } XXH32_state_t; +typedef struct { long long ll[11]; } XXH64_state_t; +typedef struct { long long ll[28]; } XXH128_state_t; +typedef struct { long long ll[28]; } XXH256_state_t; + +/* +These structures allow static allocation of XXH states. +States must then be initialized using XXHnn_reset() before first use. + +If you prefer dynamic allocation, please refer to functions below. +*/ + +XXH32_state_t* XXH32_createState(void); +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH64_state_t* XXH64_createState(void); +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +XXH128_state_t* XXH128_createState(void); +XXH_errorcode XXH128_freeState(XXH128_state_t* statePtr); + +XXH256_state_t* XXH256_createState(void); +XXH_errorcode XXH256_freeState(XXH256_state_t* statePtr); + +/* +These functions create and release memory for XXH state. +States must then be initialized using XXHnn_reset() before first use. +*/ + + +XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed); +XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +unsigned int XXH32_digest (const XXH32_state_t* statePtr); + +XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +unsigned long long XXH64_digest (const XXH64_state_t* statePtr); + +XXH_errorcode XXH128_reset (XXH128_state_t* statePtr, unsigned long long seed); +XXH_errorcode XXH128_update (XXH128_state_t* statePtr, const void* input, size_t length); +void XXH128_digest (const XXH128_state_t* statePtr, void* out); + +XXH_errorcode XXH256_reset (XXH256_state_t* statePtr, unsigned long long seed); +XXH_errorcode XXH256_update (XXH256_state_t* statePtr, const void* input, size_t length); +void XXH256_digest (const XXH256_state_t* statePtr, void* out); + +/* +These functions calculate the xxHash of an input provided in multiple smaller packets, +as opposed to an input provided as a single block. + +XXH state space must first be allocated, using either static or dynamic method provided above. + +Start a new hash by initializing state with a seed, using XXHnn_reset(). + +Then, feed the hash state by calling XXHnn_update() as many times as necessary. +Obviously, input must be valid, meaning allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, you can produce a hash anytime, by using XXHnn_digest(). +This function returns the final nn-bits hash. +You can nonetheless continue feeding the hash state with more input, +and therefore get some new hashes, by calling again XXHnn_digest(). + +When you are done, don't forget to free XXH state space, using typically XXHnn_freeState(). +*/ + +#if defined (__cplusplus) +} +#endif + +#endif diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..610a062 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +*.timestamp diff --git a/test/Makefile.am b/test/Makefile.am new file mode 100644 index 0000000..6da175b --- /dev/null +++ b/test/Makefile.am @@ -0,0 +1,329 @@ +# test-suite (run with 'make check') + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include +AM_CFLAGS = + +AM_TESTS_ENVIRONMENT= \ + TOP_SRCDIR="${top_srcdir}" \ + TOP_BUILDDIR="${top_builddir}" \ + SRCDIR="${srcdir}" \ + BUILDDIR="${builddir}" \ + JQ="@JQ@" + +TESTS_ENVIRONMENT= \ + TOP_SRCDIR="${top_srcdir}" \ + TOP_BUILDDIR="${top_builddir}" \ + SRCDIR="${srcdir}" \ + BUILDDIR="${builddir}" \ + JQ="@JQ@" + +# TEST_EXTENSIONS = .test + +TEST_LOG_COMPILE = $(SHELL) +TEST_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) $(top_srcdir)/build-aux/tap-driver.sh + +TESTS = + +# When cross-compiling, we can build the tests but cannot execute them +# since they are compiled for the target architecture +if CROSS_COMPILING +check_PROGRAMS = +else + +if HAVE_COMPATIBLE_CHECK + +check_PROGRAMS = libfyaml-test +libfyaml_test_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/valgrind/ \ + -I$(top_srcdir)/src/lib/ \ + -I$(top_srcdir)/src/util \ + -I$(top_srcdir)/src/allocator +libfyaml_test_LDADD = $(AM_LDADD) $(CHECK_LIBS) $(top_builddir)/src/libfyaml.la +libfyaml_test_CFLAGS = $(AM_CFLAGS) $(CHECK_CFLAGS) +libfyaml_test_LDFLAGS = $(AM_LDFLAGS) $(CHECK_LDFLAGS) + +libfyaml_test_SOURCES = \ + libfyaml-test.c \ + libfyaml-test-core.c \ + libfyaml-test-meta.c \ + libfyaml-test-emit.c \ + libfyaml-test-allocator.c + +# the private tests require static compilation +# if we're compiling shared only, disable the private parts +if HAVE_STATIC +libfyaml_test_SOURCES += \ + libfyaml-test-private.c \ + libfyaml-test-private-id.c \ + libfyaml-test-parser.c \ + libfyaml-test-thread.c +libfyaml_test_LDFLAGS += -static +endif + +TESTS += libfyaml.test + +endif + +if HAVE_NETWORK +if HAVE_GIT + +# normal YAML testsuite +TESTS += testsuite.test + +# YAML testsuite JSON generation +if HAVE_JQ +TESTS += testsuite-json.test +endif + +# normal JSON testsuite +TESTS += jsontestsuite.test + +# normal YAML testsuite using document event stream +TESTS += testsuite-evstream.test + +# resolution equivalence +TESTS += testsuite-resolution.test + +testsuite.test: test-suite-data.checkout.timestamp +if HAVE_JQ +testsuite-json.test: test-suite-data.checkout.timestamp +endif +testsuite-evstream.test: test-suite-data.checkout.timestamp +testsuite-resolution.test: test-suite-data.checkout.timestamp + +test-suite-data.checkout.timestamp: + @GIT@ clone -q "@TESTSUITEURL@" -- test-suite-data && \ + @GIT@ -C test-suite-data checkout -q --detach @TESTSUITECHECKOUT@ && \ + touch test-suite-data.checkout.timestamp + +jsontestsuite.test: json-test-suite-data.checkout.timestamp + +json-test-suite-data.checkout.timestamp: + @GIT@ clone -q "@JSONTESTSUITEURL@" -- json-test-suite-data && \ + @GIT@ -C json-test-suite-data checkout -q --detach @JSONTESTSUITECHECKOUT@ && \ + touch json-test-suite-data.checkout.timestamp + +distclean-local: + @rm -rf test-suite-data test-suite-data.checkout.timestamp + @rm -rf json-test-suite-data json-test-suite-data.checkout.timestamp + +endif +endif + +TESTS += testerrors.test + +TESTS += testemitter.test +TESTS += testemitter-streaming.test +TESTS += testemitter-restreaming.test + +endif # !CROSS_COMPILING + +EXTRA_DIST = test-env $(TESTS) + +# Add the error test files +EXTRA_DIST += \ + test-errors/0002/=== \ + test-errors/0002/in.yaml \ + test-errors/0002/test.error \ + test-errors/0003/=== \ + test-errors/0003/in.yaml \ + test-errors/0003/test.error \ + test-errors/0004/=== \ + test-errors/0004/in.yaml \ + test-errors/0004/test.error \ + test-errors/0005/=== \ + test-errors/0005/in.yaml \ + test-errors/0005/test.error \ + test-errors/0006/=== \ + test-errors/0006/in.yaml \ + test-errors/0006/test.error \ + test-errors/0007/=== \ + test-errors/0007/in.yaml \ + test-errors/0007/test.error \ + test-errors/0008/=== \ + test-errors/0008/in.yaml \ + test-errors/0008/test.error \ + test-errors/0009/=== \ + test-errors/0009/in.yaml \ + test-errors/0009/test.error \ + test-errors/0010/=== \ + test-errors/0010/in.yaml \ + test-errors/0010/test.error \ + test-errors/0011/=== \ + test-errors/0011/in.yaml \ + test-errors/0011/test.error + +# Add the emitter test files +EXTRA_DIST += \ + emitter-examples/anchors-1.yaml \ + emitter-examples/anchors-2.yaml \ + emitter-examples/anchors-3.yaml \ + emitter-examples/anchors-4.1.yaml \ + emitter-examples/anchors-4.yaml \ + emitter-examples/anchors-on-empty-scalars1.yaml \ + emitter-examples/anchors-on-empty-scalars2.yaml \ + emitter-examples/anchors-on-empty-scalars3.yaml \ + emitter-examples/anchors-on-empty-scalars4.yaml \ + emitter-examples/anchors-on-empty-scalars.yaml \ + emitter-examples/anchors.yaml \ + emitter-examples/array.yaml \ + emitter-examples/block2.yaml \ + emitter-examples/block3.yaml \ + emitter-examples/block4.yaml \ + emitter-examples/block6.yaml \ + emitter-examples/block7.yaml \ + emitter-examples/blocked.yaml \ + emitter-examples/blockind.yaml \ + emitter-examples/block.yaml \ + emitter-examples/c10.yaml \ + emitter-examples/c11.yaml \ + emitter-examples/c12.yaml \ + emitter-examples/c13.yaml \ + emitter-examples/c1.yaml \ + emitter-examples/c2.yaml \ + emitter-examples/c3.yaml \ + emitter-examples/c4.yaml \ + emitter-examples/c5.yaml \ + emitter-examples/c6.yaml \ + emitter-examples/c7.yaml \ + emitter-examples/c8.yaml \ + emitter-examples/c9.yaml \ + emitter-examples/compact1.yaml \ + emitter-examples/compactblockmap.yaml \ + emitter-examples/complexkey2.yaml \ + emitter-examples/complexkey3.yaml \ + emitter-examples/complexkey4.yaml \ + emitter-examples/complexkey5.yaml \ + emitter-examples/complexkey6.yaml \ + emitter-examples/complexkey7.yaml \ + emitter-examples/complexkey8.yaml \ + emitter-examples/complexkey9.yaml \ + emitter-examples/complexkey.yaml \ + emitter-examples/docstartend.yaml \ + emitter-examples/dqscalar.yaml \ + emitter-examples/dqzero.yaml \ + emitter-examples/emoji.yaml \ + emitter-examples/emptydoc.yaml \ + emitter-examples/emptykey.yaml \ + emitter-examples/emptystream.yaml \ + emitter-examples/flow1.yaml \ + emitter-examples/flow2.yaml \ + emitter-examples/flow.yaml \ + emitter-examples/fold2.yaml \ + emitter-examples/fold3.yaml \ + emitter-examples/fold4.yaml \ + emitter-examples/fold5.yaml \ + emitter-examples/folded2.yaml \ + emitter-examples/folded.yaml \ + emitter-examples/folding.yaml \ + emitter-examples/fold.yaml \ + emitter-examples/global-tag.yaml \ + emitter-examples/invoice.yaml \ + emitter-examples/json.yaml \ + emitter-examples/keyflow.yaml \ + emitter-examples/keykey2.yaml \ + emitter-examples/keykey.yaml \ + emitter-examples/line.yaml \ + emitter-examples/literal1.yaml \ + emitter-examples/literal2.yaml \ + emitter-examples/literal3.yaml \ + emitter-examples/literal4.yaml \ + emitter-examples/literal.yaml \ + emitter-examples/mapping.yaml \ + emitter-examples/mergekeyspec.yaml \ + emitter-examples/multi-document.yaml \ + emitter-examples/multiline-quoted-key.yaml \ + emitter-examples/multiline-simple-key.yaml \ + emitter-examples/nodeprop2.yaml \ + emitter-examples/nodeprop.yaml \ + emitter-examples/numbers-flow.yaml \ + emitter-examples/numbers.yaml \ + emitter-examples/plainlines.yaml \ + emitter-examples/plain-scalars-with-commas.yaml \ + emitter-examples/plainscalar.yaml \ + emitter-examples/quotedbackslash.yaml \ + emitter-examples/quoted.yaml \ + emitter-examples/scalar-multiline.yaml \ + emitter-examples/scalars2.yaml \ + emitter-examples/scalar-singlequoted.yaml \ + emitter-examples/scalar-space1.yaml \ + emitter-examples/scalar-space.yaml \ + emitter-examples/scalars.yaml \ + emitter-examples/scanner-c-10.yaml \ + emitter-examples/scanner-c-11.yaml \ + emitter-examples/scanner-c-12.yaml \ + emitter-examples/scanner-c-13.yaml \ + emitter-examples/scanner-c-1.yaml \ + emitter-examples/scanner-c-2.yaml \ + emitter-examples/scanner-c-3.yaml \ + emitter-examples/scanner-c-4.yaml \ + emitter-examples/scanner-c-5.yaml \ + emitter-examples/scanner-c-6.yaml \ + emitter-examples/scanner-c-7.yaml \ + emitter-examples/scanner-c-8-2.yaml \ + emitter-examples/scanner-c-8.yaml \ + emitter-examples/scanner-c-9.yaml \ + emitter-examples/seq1.yaml \ + emitter-examples/seq2.yaml \ + emitter-examples/seq3.yaml \ + emitter-examples/seq4.yaml \ + emitter-examples/seq5.yaml \ + emitter-examples/seq6.yaml \ + emitter-examples/seq.yaml \ + emitter-examples/sets.yaml \ + emitter-examples/simple1.yaml \ + emitter-examples/simple2.yaml \ + emitter-examples/simpleanchor1.yaml \ + emitter-examples/simpleanchor2.yaml \ + emitter-examples/simpleanchor3.yaml \ + emitter-examples/simpleanchor4.yaml \ + emitter-examples/simpleanchor.yaml \ + emitter-examples/simplefolded.yaml \ + emitter-examples/simplekey1.yaml \ + emitter-examples/simplekey2.yaml \ + emitter-examples/simplekey3.yaml \ + emitter-examples/simplekey4.yaml \ + emitter-examples/simplekey5.yaml \ + emitter-examples/simplekey.yaml \ + emitter-examples/simpleliteral.yaml \ + emitter-examples/simpleseq1.yaml \ + emitter-examples/simpleseq.yaml \ + emitter-examples/simple.yaml \ + emitter-examples/singlepairimp2.yaml \ + emitter-examples/singlepairimp.yaml \ + emitter-examples/sqscalarspace.yaml \ + emitter-examples/sqscalar.yaml \ + emitter-examples/strings.yaml \ + emitter-examples/t1.yaml \ + emitter-examples/t2.yaml \ + emitter-examples/t3.yaml \ + emitter-examples/t4.yaml \ + emitter-examples/t5.yaml \ + emitter-examples/tabsmix.yaml \ + emitter-examples/tagdirective.yaml \ + emitter-examples/tagesc.yaml \ + emitter-examples/tags-1.yaml \ + emitter-examples/tags.yaml \ + emitter-examples/test1.yaml \ + emitter-examples/test2.yaml \ + emitter-examples/test.yaml \ + emitter-examples/t.yaml \ + emitter-examples/u1.yaml \ + emitter-examples/u2.yaml \ + emitter-examples/u3.yaml \ + emitter-examples/utf8-simple.yaml \ + emitter-examples/utf8.yaml \ + emitter-examples/u.yaml \ + emitter-examples/v1.yaml \ + emitter-examples/v2.yaml \ + emitter-examples/version.yaml \ + emitter-examples/v.yaml \ + emitter-examples/weirdplain.yaml \ + emitter-examples/ws0.yaml \ + emitter-examples/ws1.yaml \ + emitter-examples/ws2.yaml \ + emitter-examples/ws3.yaml \ + emitter-examples/yaml-version.yaml \ + emitter-examples/y.yaml \ + emitter-examples/yy.yaml \ + emitter-examples/zeroexplicit.yaml diff --git a/test/emitter-examples/anchors-1.yaml b/test/emitter-examples/anchors-1.yaml new file mode 100644 index 0000000..beaf8ec --- /dev/null +++ b/test/emitter-examples/anchors-1.yaml @@ -0,0 +1,2 @@ +base: &base + name: this-is-a-name diff --git a/test/emitter-examples/anchors-2.yaml b/test/emitter-examples/anchors-2.yaml new file mode 100644 index 0000000..3574f01 --- /dev/null +++ b/test/emitter-examples/anchors-2.yaml @@ -0,0 +1,2 @@ +base: &base { name: this-is-a-name } + diff --git a/test/emitter-examples/anchors-3.yaml b/test/emitter-examples/anchors-3.yaml new file mode 100644 index 0000000..0d0dc92 --- /dev/null +++ b/test/emitter-examples/anchors-3.yaml @@ -0,0 +1,2 @@ +&base { name: this-is-a-name } + diff --git a/test/emitter-examples/anchors-4.1.yaml b/test/emitter-examples/anchors-4.1.yaml new file mode 100644 index 0000000..6881ba7 --- /dev/null +++ b/test/emitter-examples/anchors-4.1.yaml @@ -0,0 +1,3 @@ +base: &base + name: Everyone has same name + age: 10 diff --git a/test/emitter-examples/anchors-4.yaml b/test/emitter-examples/anchors-4.yaml new file mode 100644 index 0000000..05b387e --- /dev/null +++ b/test/emitter-examples/anchors-4.yaml @@ -0,0 +1,3 @@ +base: &base + name: Everyone has same name + age: 10 diff --git a/test/emitter-examples/anchors-on-empty-scalars.yaml b/test/emitter-examples/anchors-on-empty-scalars.yaml new file mode 100644 index 0000000..c027125 --- /dev/null +++ b/test/emitter-examples/anchors-on-empty-scalars.yaml @@ -0,0 +1,12 @@ +- &a +- a +- + &a1 : a + b: &b +- + &c : &a2 +- + ? &d +- + ? &e + : &a3 diff --git a/test/emitter-examples/anchors-on-empty-scalars1.yaml b/test/emitter-examples/anchors-on-empty-scalars1.yaml new file mode 100644 index 0000000..2be6683 --- /dev/null +++ b/test/emitter-examples/anchors-on-empty-scalars1.yaml @@ -0,0 +1,3 @@ +- + ? &e + : &a diff --git a/test/emitter-examples/anchors-on-empty-scalars2.yaml b/test/emitter-examples/anchors-on-empty-scalars2.yaml new file mode 100644 index 0000000..7a0145a --- /dev/null +++ b/test/emitter-examples/anchors-on-empty-scalars2.yaml @@ -0,0 +1,3 @@ +- + ? + : diff --git a/test/emitter-examples/anchors-on-empty-scalars3.yaml b/test/emitter-examples/anchors-on-empty-scalars3.yaml new file mode 100644 index 0000000..47c32ca --- /dev/null +++ b/test/emitter-examples/anchors-on-empty-scalars3.yaml @@ -0,0 +1,3 @@ +- + ? a + : diff --git a/test/emitter-examples/anchors-on-empty-scalars4.yaml b/test/emitter-examples/anchors-on-empty-scalars4.yaml new file mode 100644 index 0000000..f5814d5 --- /dev/null +++ b/test/emitter-examples/anchors-on-empty-scalars4.yaml @@ -0,0 +1,3 @@ +- + ? &a + : diff --git a/test/emitter-examples/anchors.yaml b/test/emitter-examples/anchors.yaml new file mode 100644 index 0000000..8755853 --- /dev/null +++ b/test/emitter-examples/anchors.yaml @@ -0,0 +1,10 @@ +base: &base + name: Everyone has same name + +foo: &foo + <<: *base + age: 10 + +bar: &bar + <<: *base + age: 20 diff --git a/test/emitter-examples/array.yaml b/test/emitter-examples/array.yaml new file mode 100644 index 0000000..18efd12 --- /dev/null +++ b/test/emitter-examples/array.yaml @@ -0,0 +1,2 @@ +- member +- member2 diff --git a/test/emitter-examples/block.yaml b/test/emitter-examples/block.yaml new file mode 100644 index 0000000..fb6d80a --- /dev/null +++ b/test/emitter-examples/block.yaml @@ -0,0 +1,7 @@ +- item 1 +# STREAM_START FYTT_STREAM_START +# BLOCK_SEQUENCE_START FYTT_BLOCK_SEQUENCE_START +# BLOCK_ENTRY FYTT_BLOCK_ENTRY +# SCALAR value='item 1' style=PLAIN FYTT_SCALAR value='item 1' style=PLAIN +# BLOCK_END FYTT_BLOCK_END +# STREAM_END FYTT_STREAM_END diff --git a/test/emitter-examples/block2.yaml b/test/emitter-examples/block2.yaml new file mode 100644 index 0000000..cb745e0 --- /dev/null +++ b/test/emitter-examples/block2.yaml @@ -0,0 +1,10 @@ +- item 1 +- item 2 +# STREAM_START +# BLOCK_SEQUENCE_START +# BLOCK_ENTRY +# SCALAR value='item 1' style=PLAIN +# BLOCK_ENTRY +# SCALAR value='item 2' style=PLAIN +# BLOCK_END +# STREAM_END diff --git a/test/emitter-examples/block3.yaml b/test/emitter-examples/block3.yaml new file mode 100644 index 0000000..1c0593c --- /dev/null +++ b/test/emitter-examples/block3.yaml @@ -0,0 +1,3 @@ +one: one-value +two: two-value +three: three-value diff --git a/test/emitter-examples/block4.yaml b/test/emitter-examples/block4.yaml new file mode 100644 index 0000000..bbea92a --- /dev/null +++ b/test/emitter-examples/block4.yaml @@ -0,0 +1,3 @@ +test: | + + foo diff --git a/test/emitter-examples/block6.yaml b/test/emitter-examples/block6.yaml new file mode 100644 index 0000000..1ff9bfa --- /dev/null +++ b/test/emitter-examples/block6.yaml @@ -0,0 +1,5 @@ +--- |+ + ab + + +... diff --git a/test/emitter-examples/block7.yaml b/test/emitter-examples/block7.yaml new file mode 100644 index 0000000..418a586 --- /dev/null +++ b/test/emitter-examples/block7.yaml @@ -0,0 +1,6 @@ +block-scalar: | + Test + 12 + testing + FOO +... diff --git a/test/emitter-examples/blocked.yaml b/test/emitter-examples/blocked.yaml new file mode 100644 index 0000000..19824b4 --- /dev/null +++ b/test/emitter-examples/blocked.yaml @@ -0,0 +1,4 @@ +|- + ab + +... diff --git a/test/emitter-examples/blockind.yaml b/test/emitter-examples/blockind.yaml new file mode 100644 index 0000000..608ea19 --- /dev/null +++ b/test/emitter-examples/blockind.yaml @@ -0,0 +1,7 @@ +sequence: +- one +- two +mapping: + ? sky + : blue + sea : green diff --git a/test/emitter-examples/c1.yaml b/test/emitter-examples/c1.yaml new file mode 100644 index 0000000..473afea --- /dev/null +++ b/test/emitter-examples/c1.yaml @@ -0,0 +1,2 @@ +--- +foo # comment diff --git a/test/emitter-examples/c10.yaml b/test/emitter-examples/c10.yaml new file mode 100644 index 0000000..48c6082 --- /dev/null +++ b/test/emitter-examples/c10.yaml @@ -0,0 +1,9 @@ +# root comment + + # top comment of A +- A # right comment of A + # bottom comment of A + + # top comment of B +- B # right comment of B + # bottom comment of B diff --git a/test/emitter-examples/c11.yaml b/test/emitter-examples/c11.yaml new file mode 100644 index 0000000..525d6e7 --- /dev/null +++ b/test/emitter-examples/c11.yaml @@ -0,0 +1,10 @@ +[ + # top comment of A + A, # right comment of A + # bottom comment of A + + # top comment of A + B, # right comment of A + # bottom comment of A +] + diff --git a/test/emitter-examples/c12.yaml b/test/emitter-examples/c12.yaml new file mode 100644 index 0000000..9002396 --- /dev/null +++ b/test/emitter-examples/c12.yaml @@ -0,0 +1,4 @@ +key: # key-comment + value # value-comment +key2: + key3: value diff --git a/test/emitter-examples/c13.yaml b/test/emitter-examples/c13.yaml new file mode 100644 index 0000000..46e22fa --- /dev/null +++ b/test/emitter-examples/c13.yaml @@ -0,0 +1,3 @@ +foo: bar # comment +{ foo: bar }: # comment2 + baz diff --git a/test/emitter-examples/c2.yaml b/test/emitter-examples/c2.yaml new file mode 100644 index 0000000..c0d56ad --- /dev/null +++ b/test/emitter-examples/c2.yaml @@ -0,0 +1,4 @@ +--- +# top comment +foo # right comment +# bottom comment diff --git a/test/emitter-examples/c3.yaml b/test/emitter-examples/c3.yaml new file mode 100644 index 0000000..7040aa9 --- /dev/null +++ b/test/emitter-examples/c3.yaml @@ -0,0 +1,2 @@ +--- +foo: bar # comment diff --git a/test/emitter-examples/c4.yaml b/test/emitter-examples/c4.yaml new file mode 100644 index 0000000..50d4be9 --- /dev/null +++ b/test/emitter-examples/c4.yaml @@ -0,0 +1,5 @@ +--- +{ + foo: bar, # comment + baz: frooz +} diff --git a/test/emitter-examples/c5.yaml b/test/emitter-examples/c5.yaml new file mode 100644 index 0000000..7e39af8 --- /dev/null +++ b/test/emitter-examples/c5.yaml @@ -0,0 +1,6 @@ +--- +{ + foo: bar, # comment + # top comment + baz: frooz # last comment +} diff --git a/test/emitter-examples/c6.yaml b/test/emitter-examples/c6.yaml new file mode 100644 index 0000000..c56a8c1 --- /dev/null +++ b/test/emitter-examples/c6.yaml @@ -0,0 +1,5 @@ +--- +[ + foo, # comment + bar +] diff --git a/test/emitter-examples/c7.yaml b/test/emitter-examples/c7.yaml new file mode 100644 index 0000000..627eef2 --- /dev/null +++ b/test/emitter-examples/c7.yaml @@ -0,0 +1,5 @@ +--- +[ + [ foo, bar ], # comment + baz +] diff --git a/test/emitter-examples/c8.yaml b/test/emitter-examples/c8.yaml new file mode 100644 index 0000000..e1adf02 --- /dev/null +++ b/test/emitter-examples/c8.yaml @@ -0,0 +1,6 @@ +--- +{ + foo: bar, # foo:bar comment + baz: frooz, # baz:frooz comment + whee: yikes # whee: yikes comment +} diff --git a/test/emitter-examples/c9.yaml b/test/emitter-examples/c9.yaml new file mode 100644 index 0000000..9058f07 --- /dev/null +++ b/test/emitter-examples/c9.yaml @@ -0,0 +1,17 @@ +--- +# 1 +# top foo:bar comment +# 2 +foo: bar # foo:bar comment + # continue foo:bar comment + +# 3 +# top baz:frooz comment +# 4 + # break +baz: frooz # baz:frooz comment + +# top whee comment +whee: # whee: comment + level: # level: comment + 1: level-1 # level-1 comment diff --git a/test/emitter-examples/compact1.yaml b/test/emitter-examples/compact1.yaml new file mode 100644 index 0000000..5925e07 --- /dev/null +++ b/test/emitter-examples/compact1.yaml @@ -0,0 +1,6 @@ +- # Empty +- | + block node +- - one # Compact +- - two # sequence +- - one: two # Compact mapping diff --git a/test/emitter-examples/compactblockmap.yaml b/test/emitter-examples/compactblockmap.yaml new file mode 100644 index 0000000..476b362 --- /dev/null +++ b/test/emitter-examples/compactblockmap.yaml @@ -0,0 +1,2 @@ +? a: b +: c: d diff --git a/test/emitter-examples/complexkey.yaml b/test/emitter-examples/complexkey.yaml new file mode 100644 index 0000000..d5f6d26 --- /dev/null +++ b/test/emitter-examples/complexkey.yaml @@ -0,0 +1 @@ +? complex : value diff --git a/test/emitter-examples/complexkey2.yaml b/test/emitter-examples/complexkey2.yaml new file mode 100644 index 0000000..4fbbddb --- /dev/null +++ b/test/emitter-examples/complexkey2.yaml @@ -0,0 +1 @@ +{ ? complex: value } diff --git a/test/emitter-examples/complexkey3.yaml b/test/emitter-examples/complexkey3.yaml new file mode 100644 index 0000000..4586d1c --- /dev/null +++ b/test/emitter-examples/complexkey3.yaml @@ -0,0 +1,12 @@ +{ + ? complex key: value +} +# STREAM_START FYTT_STREAM_START +# FLOW_MAPPING_START FYTT_FLOW_MAPPING_START +# KEY FYTT_KEY +# SCALAR value='complex key' style=PLAIN FYTT_KEY +# VALUE FYTT_SCALAR value='complex key' style=PLAIN +# SCALAR value='value' style=PLAIN FYTT_VALUE +# FLOW_MAPPING_END FYTT_SCALAR value='value' style=PLAIN +# STREAM_END FYTT_FLOW_MAPPING_END +# FYTT_STREAM_END diff --git a/test/emitter-examples/complexkey4.yaml b/test/emitter-examples/complexkey4.yaml new file mode 100644 index 0000000..c8cd88c --- /dev/null +++ b/test/emitter-examples/complexkey4.yaml @@ -0,0 +1,10 @@ +[ ? complex: value ] +# STREAM_START FYTT_STREAM_START +# FLOW_SEQUENCE_START FYTT_FLOW_SEQUENCE_START +# KEY FYTT_KEY +# SCALAR value='complex key' style=PLAIN FYTT_KEY +# VALUE FYTT_SCALAR value='complex key' style=PLAIN +# SCALAR value='value' style=PLAIN FYTT_VALUE +# FLOW_SEQUENCE_END FYTT_SCALAR value='value' style=PLAIN +# STREAM_END FYTT_FLOW_SEQUENCE_END +# FYTT_STREAM_END diff --git a/test/emitter-examples/complexkey5.yaml b/test/emitter-examples/complexkey5.yaml new file mode 100644 index 0000000..181aa2a --- /dev/null +++ b/test/emitter-examples/complexkey5.yaml @@ -0,0 +1 @@ +{ ? { complex: 2 }: value } diff --git a/test/emitter-examples/complexkey6.yaml b/test/emitter-examples/complexkey6.yaml new file mode 100644 index 0000000..7baf45d --- /dev/null +++ b/test/emitter-examples/complexkey6.yaml @@ -0,0 +1,2 @@ +? complex +: value diff --git a/test/emitter-examples/complexkey7.yaml b/test/emitter-examples/complexkey7.yaml new file mode 100644 index 0000000..4dbae17 --- /dev/null +++ b/test/emitter-examples/complexkey7.yaml @@ -0,0 +1 @@ +- ? complex : value diff --git a/test/emitter-examples/complexkey8.yaml b/test/emitter-examples/complexkey8.yaml new file mode 100644 index 0000000..1495237 --- /dev/null +++ b/test/emitter-examples/complexkey8.yaml @@ -0,0 +1 @@ +[ ]: foo diff --git a/test/emitter-examples/complexkey9.yaml b/test/emitter-examples/complexkey9.yaml new file mode 100644 index 0000000..cbaba7f --- /dev/null +++ b/test/emitter-examples/complexkey9.yaml @@ -0,0 +1,3 @@ +? a: b + c: d +: e diff --git a/test/emitter-examples/docstartend.yaml b/test/emitter-examples/docstartend.yaml new file mode 100644 index 0000000..f0d5983 --- /dev/null +++ b/test/emitter-examples/docstartend.yaml @@ -0,0 +1,3 @@ +--- +foo +... diff --git a/test/emitter-examples/dqscalar.yaml b/test/emitter-examples/dqscalar.yaml new file mode 100644 index 0000000..d6bbd62 --- /dev/null +++ b/test/emitter-examples/dqscalar.yaml @@ -0,0 +1,14 @@ +- "quoted" +- "quoted\nescaped new line" +- "quoted + explicit new line" +- "" +- " + " +- " pre-spaces" +- "post-spaces " +- "key": value +- "key\nnewline": value +- "\r\"\\foo" +- "\x20" +- "testing \0null" diff --git a/test/emitter-examples/dqzero.yaml b/test/emitter-examples/dqzero.yaml new file mode 100644 index 0000000..b5db05f --- /dev/null +++ b/test/emitter-examples/dqzero.yaml @@ -0,0 +1 @@ +"zero\0zero" diff --git a/test/emitter-examples/emoji.yaml b/test/emitter-examples/emoji.yaml new file mode 100644 index 0000000..3487828 --- /dev/null +++ b/test/emitter-examples/emoji.yaml @@ -0,0 +1,3846 @@ +- chars: ! "\U0001F600" + codes: 1F600 + name: GRINNING FACE +- chars: ! "\U0001F601" + codes: 1F601 + name: GRINNING FACE WITH SMILING EYES +- chars: ! "\U0001F602" + codes: 1F602 + name: FACE WITH TEARS OF JOY +- chars: ! "\U0001F603" + codes: 1F603 + name: SMILING FACE WITH OPEN MOUTH +- chars: ! "\U0001F604" + codes: 1F604 + name: SMILING FACE WITH OPEN MOUTH AND SMILING EYES +- chars: ! "\U0001F605" + codes: 1F605 + name: SMILING FACE WITH OPEN MOUTH AND COLD SWEAT +- chars: ! "\U0001F606" + codes: 1F606 + name: SMILING FACE WITH OPEN MOUTH AND TIGHTLY-CLOSED EYES +- chars: ! "\U0001F609" + codes: 1F609 + name: WINKING FACE +- chars: ! "\U0001F60A" + codes: 1F60A + name: SMILING FACE WITH SMILING EYES +- chars: ! "\U0001F60B" + codes: 1F60B + name: FACE SAVOURING DELICIOUS FOOD +- chars: ! "\U0001F60E" + codes: 1F60E + name: SMILING FACE WITH SUNGLASSES +- chars: ! "\U0001F60D" + codes: 1F60D + name: SMILING FACE WITH HEART-SHAPED EYES +- chars: ! "\U0001F618" + codes: 1F618 + name: FACE THROWING A KISS +- chars: ! "\U0001F617" + codes: 1F617 + name: KISSING FACE +- chars: ! "\U0001F619" + codes: 1F619 + name: KISSING FACE WITH SMILING EYES +- chars: ! "\U0001F61A" + codes: 1F61A + name: KISSING FACE WITH CLOSED EYES +- chars: ☺️ + codes: 263A + name: WHITE SMILING FACE +- chars: ! "\U0001F642" + codes: 1F642 + name: SLIGHTLY SMILING FACE +- chars: ! "\U0001F917" + codes: 1F917 + name: HUGGING FACE +- chars: ! "\U0001F607" + codes: 1F607 + name: SMILING FACE WITH HALO +- chars: ! "\U0001F914" + codes: 1F914 + name: THINKING FACE +- chars: ! "\U0001F610" + codes: 1F610 + name: NEUTRAL FACE +- chars: ! "\U0001F611" + codes: 1F611 + name: EXPRESSIONLESS FACE +- chars: ! "\U0001F636" + codes: 1F636 + name: FACE WITHOUT MOUTH +- chars: ! "\U0001F644" + codes: 1F644 + name: FACE WITH ROLLING EYES +- chars: ! "\U0001F60F" + codes: 1F60F + name: SMIRKING FACE +- chars: ! "\U0001F623" + codes: 1F623 + name: PERSEVERING FACE +- chars: ! "\U0001F625" + codes: 1F625 + name: DISAPPOINTED BUT RELIEVED FACE +- chars: ! "\U0001F62E" + codes: 1F62E + name: FACE WITH OPEN MOUTH +- chars: ! "\U0001F910" + codes: 1F910 + name: ZIPPER-MOUTH FACE +- chars: ! "\U0001F62F" + codes: 1F62F + name: HUSHED FACE +- chars: ! "\U0001F62A" + codes: 1F62A + name: SLEEPY FACE +- chars: ! "\U0001F62B" + codes: 1F62B + name: TIRED FACE +- chars: ! "\U0001F634" + codes: 1F634 + name: SLEEPING FACE +- chars: ! "\U0001F60C" + codes: 1F60C + name: RELIEVED FACE +- chars: ! "\U0001F913" + codes: 1F913 + name: NERD FACE +- chars: ! "\U0001F61B" + codes: 1F61B + name: FACE WITH STUCK-OUT TONGUE +- chars: ! "\U0001F61C" + codes: 1F61C + name: FACE WITH STUCK-OUT TONGUE AND WINKING EYE +- chars: ! "\U0001F61D" + codes: 1F61D + name: FACE WITH STUCK-OUT TONGUE AND TIGHTLY-CLOSED EYES +- chars: ☹ + codes: '2639' + name: WHITE FROWNING FACE +- chars: ! "\U0001F641" + codes: 1F641 + name: SLIGHTLY FROWNING FACE +- chars: ! "\U0001F612" + codes: 1F612 + name: UNAMUSED FACE +- chars: ! "\U0001F613" + codes: 1F613 + name: FACE WITH COLD SWEAT +- chars: ! "\U0001F614" + codes: 1F614 + name: PENSIVE FACE +- chars: ! "\U0001F615" + codes: 1F615 + name: CONFUSED FACE +- chars: ! "\U0001F616" + codes: 1F616 + name: CONFOUNDED FACE +- chars: ! "\U0001F643" + codes: 1F643 + name: UPSIDE-DOWN FACE +- chars: ! "\U0001F637" + codes: 1F637 + name: FACE WITH MEDICAL MASK +- chars: ! "\U0001F912" + codes: 1F912 + name: FACE WITH THERMOMETER +- chars: ! "\U0001F915" + codes: 1F915 + name: FACE WITH HEAD-BANDAGE +- chars: ! "\U0001F911" + codes: 1F911 + name: MONEY-MOUTH FACE +- chars: ! "\U0001F632" + codes: 1F632 + name: ASTONISHED FACE +- chars: ! "\U0001F61E" + codes: 1F61E + name: DISAPPOINTED FACE +- chars: ! "\U0001F61F" + codes: 1F61F + name: WORRIED FACE +- chars: ! "\U0001F624" + codes: 1F624 + name: FACE WITH LOOK OF TRIUMPH +- chars: ! "\U0001F622" + codes: 1F622 + name: CRYING FACE +- chars: ! "\U0001F62D" + codes: 1F62D + name: LOUDLY CRYING FACE +- chars: ! "\U0001F626" + codes: 1F626 + name: FROWNING FACE WITH OPEN MOUTH +- chars: ! "\U0001F627" + codes: 1F627 + name: ANGUISHED FACE +- chars: ! "\U0001F628" + codes: 1F628 + name: FEARFUL FACE +- chars: ! "\U0001F629" + codes: 1F629 + name: WEARY FACE +- chars: ! "\U0001F62C" + codes: 1F62C + name: GRIMACING FACE +- chars: ! "\U0001F630" + codes: 1F630 + name: FACE WITH OPEN MOUTH AND COLD SWEAT +- chars: ! "\U0001F631" + codes: 1F631 + name: FACE SCREAMING IN FEAR +- chars: ! "\U0001F633" + codes: 1F633 + name: FLUSHED FACE +- chars: ! "\U0001F635" + codes: 1F635 + name: DIZZY FACE +- chars: ! "\U0001F621" + codes: 1F621 + name: POUTING FACE +- chars: ! "\U0001F620" + codes: 1F620 + name: ANGRY FACE +- chars: ! "\U0001F608" + codes: 1F608 + name: SMILING FACE WITH HORNS +- chars: ! "\U0001F47F" + codes: 1F47F + name: IMP +- chars: ! "\U0001F479" + codes: 1F479 + name: JAPANESE OGRE +- chars: ! "\U0001F47A" + codes: 1F47A + name: JAPANESE GOBLIN +- chars: ! "\U0001F480" + codes: 1F480 + name: SKULL +- chars: ☠ + codes: '2620' + name: SKULL AND CROSSBONES +- chars: ! "\U0001F47B" + codes: 1F47B + name: GHOST +- chars: ! "\U0001F47D" + codes: 1F47D + name: EXTRATERRESTRIAL ALIEN +- chars: ! "\U0001F47E" + codes: 1F47E + name: ALIEN MONSTER +- chars: ! "\U0001F916" + codes: 1F916 + name: ROBOT FACE +- chars: ! "\U0001F4A9" + codes: 1F4A9 + name: PILE OF POO +- chars: ! "\U0001F63A" + codes: 1F63A + name: SMILING CAT FACE WITH OPEN MOUTH +- chars: ! "\U0001F638" + codes: 1F638 + name: GRINNING CAT FACE WITH SMILING EYES +- chars: ! "\U0001F639" + codes: 1F639 + name: CAT FACE WITH TEARS OF JOY +- chars: ! "\U0001F63B" + codes: 1F63B + name: SMILING CAT FACE WITH HEART-SHAPED EYES +- chars: ! "\U0001F63C" + codes: 1F63C + name: CAT FACE WITH WRY SMILE +- chars: ! "\U0001F63D" + codes: 1F63D + name: KISSING CAT FACE WITH CLOSED EYES +- chars: ! "\U0001F640" + codes: 1F640 + name: WEARY CAT FACE +- chars: ! "\U0001F63F" + codes: 1F63F + name: CRYING CAT FACE +- chars: ! "\U0001F63E" + codes: 1F63E + name: POUTING CAT FACE +- chars: ! "\U0001F648" + codes: 1F648 + name: SEE-NO-EVIL MONKEY +- chars: ! "\U0001F649" + codes: 1F649 + name: HEAR-NO-EVIL MONKEY +- chars: ! "\U0001F64A" + codes: 1F64A + name: SPEAK-NO-EVIL MONKEY +- chars: ! "\U0001F466" + codes: 1F466 + name: BOY +- chars: ! "\U0001F467" + codes: 1F467 + name: GIRL +- chars: ! "\U0001F468" + codes: 1F468 + name: MAN +- chars: ! "\U0001F469" + codes: 1F469 + name: WOMAN +- chars: ! "\U0001F474" + codes: 1F474 + name: OLDER MAN +- chars: ! "\U0001F475" + codes: 1F475 + name: OLDER WOMAN +- chars: ! "\U0001F476" + codes: 1F476 + name: BABY +- chars: ! "\U0001F471" + codes: 1F471 + name: PERSON WITH BLOND HAIR +- chars: ! "\U0001F46E" + codes: 1F46E + name: POLICE OFFICER +- chars: ! "\U0001F472" + codes: 1F472 + name: MAN WITH GUA PI MAO +- chars: ! "\U0001F473" + codes: 1F473 + name: MAN WITH TURBAN +- chars: ! "\U0001F477" + codes: 1F477 + name: CONSTRUCTION WORKER +- chars: ⛑ + codes: 26D1 + name: HELMET WITH WHITE CROSS +- chars: ! "\U0001F478" + codes: 1F478 + name: PRINCESS +- chars: ! "\U0001F482" + codes: 1F482 + name: GUARDSMAN +- chars: ! "\U0001F575" + codes: 1F575 + name: SLEUTH OR SPY +- chars: ! "\U0001F385" + codes: 1F385 + name: FATHER CHRISTMAS +- chars: ! "\U0001F47C" + codes: 1F47C + name: BABY ANGEL +- chars: ! "\U0001F46F" + codes: 1F46F + name: WOMAN WITH BUNNY EARS +- chars: ! "\U0001F486" + codes: 1F486 + name: FACE MASSAGE +- chars: ! "\U0001F487" + codes: 1F487 + name: HAIRCUT +- chars: ! "\U0001F470" + codes: 1F470 + name: BRIDE WITH VEIL +- chars: ! "\U0001F64D" + codes: 1F64D + name: PERSON FROWNING +- chars: ! "\U0001F64E" + codes: 1F64E + name: PERSON WITH POUTING FACE +- chars: ! "\U0001F645" + codes: 1F645 + name: FACE WITH NO GOOD GESTURE +- chars: ! "\U0001F646" + codes: 1F646 + name: FACE WITH OK GESTURE +- chars: ! "\U0001F481" + codes: 1F481 + name: INFORMATION DESK PERSON +- chars: ! "\U0001F64B" + codes: 1F64B + name: HAPPY PERSON RAISING ONE HAND +- chars: ! "\U0001F647" + codes: 1F647 + name: PERSON BOWING DEEPLY +- chars: ! "\U0001F64C" + codes: 1F64C + name: PERSON RAISING BOTH HANDS IN CELEBRATION +- chars: ! "\U0001F64F" + codes: 1F64F + name: PERSON WITH FOLDED HANDS +- chars: ! "\U0001F5E3" + codes: 1F5E3 + name: SPEAKING HEAD IN SILHOUETTE +- chars: ! "\U0001F464" + codes: 1F464 + name: BUST IN SILHOUETTE +- chars: ! "\U0001F465" + codes: 1F465 + name: BUSTS IN SILHOUETTE +- chars: ! "\U0001F6B6" + codes: 1F6B6 + name: PEDESTRIAN +- chars: ! "\U0001F3C3" + codes: 1F3C3 + name: RUNNER +- chars: ! "\U0001F483" + codes: 1F483 + name: DANCER +- chars: ! "\U0001F574" + codes: 1F574 + name: MAN IN BUSINESS SUIT LEVITATING +- chars: ! "\U0001F48F" + codes: 1F48F + name: KISS +- chars: ! "\U0001F491" + codes: 1F491 + name: COUPLE WITH HEART +- chars: ! "\U0001F46A" + codes: 1F46A + name: FAMILY +- chars: ! "\U0001F46B" + codes: 1F46B + name: MAN AND WOMAN HOLDING HANDS +- chars: ! "\U0001F46C" + codes: 1F46C + name: TWO MEN HOLDING HANDS +- chars: ! "\U0001F46D" + codes: 1F46D + name: TWO WOMEN HOLDING HANDS +- chars: ! "\U0001F3FB" + codes: 1F3FB + name: EMOJI MODIFIER FITZPATRICK TYPE-1-2 +- chars: ! "\U0001F3FC" + codes: 1F3FC + name: EMOJI MODIFIER FITZPATRICK TYPE-3 +- chars: ! "\U0001F3FD" + codes: 1F3FD + name: EMOJI MODIFIER FITZPATRICK TYPE-4 +- chars: ! "\U0001F3FE" + codes: 1F3FE + name: EMOJI MODIFIER FITZPATRICK TYPE-5 +- chars: ! "\U0001F3FF" + codes: 1F3FF + name: EMOJI MODIFIER FITZPATRICK TYPE-6 +- chars: ! "\U0001F4AA" + codes: 1F4AA + name: FLEXED BICEPS +- chars: ! "\U0001F448" + codes: 1F448 + name: WHITE LEFT POINTING BACKHAND INDEX +- chars: ! "\U0001F449" + codes: 1F449 + name: WHITE RIGHT POINTING BACKHAND INDEX +- chars: ☝️ + codes: 261D + name: WHITE UP POINTING INDEX +- chars: ! "\U0001F446" + codes: 1F446 + name: WHITE UP POINTING BACKHAND INDEX +- chars: ! "\U0001F595" + codes: 1F595 + name: REVERSED HAND WITH MIDDLE FINGER EXTENDED +- chars: ! "\U0001F447" + codes: 1F447 + name: WHITE DOWN POINTING BACKHAND INDEX +- chars: ✌️ + codes: 270C + name: VICTORY HAND +- chars: ! "\U0001F596" + codes: 1F596 + name: RAISED HAND WITH PART BETWEEN MIDDLE AND RING FINGERS +- chars: ! "\U0001F918" + codes: 1F918 + name: SIGN OF THE HORNS +- chars: ! "\U0001F590" + codes: 1F590 + name: RAISED HAND WITH FINGERS SPLAYED +- chars: ✊ + codes: 270A + name: RAISED FIST +- chars: ✋ + codes: 270B + name: RAISED HAND +- chars: ! "\U0001F44A" + codes: 1F44A + name: FISTED HAND SIGN +- chars: ! "\U0001F44C" + codes: 1F44C + name: OK HAND SIGN +- chars: ! "\U0001F44D" + codes: 1F44D + name: THUMBS UP SIGN +- chars: ! "\U0001F44E" + codes: 1F44E + name: THUMBS DOWN SIGN +- chars: ! "\U0001F44B" + codes: 1F44B + name: WAVING HAND SIGN +- chars: ! "\U0001F44F" + codes: 1F44F + name: CLAPPING HANDS SIGN +- chars: ! "\U0001F450" + codes: 1F450 + name: OPEN HANDS SIGN +- chars: ✍ + codes: 270D + name: WRITING HAND +- chars: ! "\U0001F485" + codes: 1F485 + name: NAIL POLISH +- chars: ! "\U0001F442" + codes: 1F442 + name: EAR +- chars: ! "\U0001F443" + codes: 1F443 + name: NOSE +- chars: ! "\U0001F463" + codes: 1F463 + name: FOOTPRINTS +- chars: ! "\U0001F440" + codes: 1F440 + name: EYES +- chars: ! "\U0001F441" + codes: 1F441 + name: EYE +- chars: ! "\U0001F445" + codes: 1F445 + name: TONGUE +- chars: ! "\U0001F444" + codes: 1F444 + name: MOUTH +- chars: ! "\U0001F48B" + codes: 1F48B + name: KISS MARK +- chars: ! "\U0001F498" + codes: 1F498 + name: HEART WITH ARROW +- chars: ❤️ + codes: '2764' + name: HEAVY BLACK HEART +- chars: ! "\U0001F493" + codes: 1F493 + name: BEATING HEART +- chars: ! "\U0001F494" + codes: 1F494 + name: BROKEN HEART +- chars: ! "\U0001F495" + codes: 1F495 + name: TWO HEARTS +- chars: ! "\U0001F496" + codes: 1F496 + name: SPARKLING HEART +- chars: ! "\U0001F497" + codes: 1F497 + name: GROWING HEART +- chars: ! "\U0001F499" + codes: 1F499 + name: BLUE HEART +- chars: ! "\U0001F49A" + codes: 1F49A + name: GREEN HEART +- chars: ! "\U0001F49B" + codes: 1F49B + name: YELLOW HEART +- chars: ! "\U0001F49C" + codes: 1F49C + name: PURPLE HEART +- chars: ! "\U0001F49D" + codes: 1F49D + name: HEART WITH RIBBON +- chars: ! "\U0001F49E" + codes: 1F49E + name: REVOLVING HEARTS +- chars: ! "\U0001F49F" + codes: 1F49F + name: HEART DECORATION +- chars: ❣ + codes: '2763' + name: HEAVY HEART EXCLAMATION MARK ORNAMENT +- chars: ! "\U0001F48C" + codes: 1F48C + name: LOVE LETTER +- chars: ! "\U0001F4A4" + codes: 1F4A4 + name: SLEEPING SYMBOL +- chars: ! "\U0001F4A2" + codes: 1F4A2 + name: ANGER SYMBOL +- chars: ! "\U0001F4A3" + codes: 1F4A3 + name: BOMB +- chars: ! "\U0001F4A5" + codes: 1F4A5 + name: COLLISION SYMBOL +- chars: ! "\U0001F4A6" + codes: 1F4A6 + name: SPLASHING SWEAT SYMBOL +- chars: ! "\U0001F4A8" + codes: 1F4A8 + name: DASH SYMBOL +- chars: ! "\U0001F4AB" + codes: 1F4AB + name: DIZZY SYMBOL +- chars: ! "\U0001F4AC" + codes: 1F4AC + name: SPEECH BALLOON +- chars: ! "\U0001F5E8" + codes: 1F5E8 + name: LEFT SPEECH BUBBLE +- chars: ! "\U0001F5EF" + codes: 1F5EF + name: RIGHT ANGER BUBBLE +- chars: ! "\U0001F4AD" + codes: 1F4AD + name: THOUGHT BALLOON +- chars: ! "\U0001F573" + codes: 1F573 + name: HOLE +- chars: ! "\U0001F453" + codes: 1F453 + name: EYEGLASSES +- chars: ! "\U0001F576" + codes: 1F576 + name: DARK SUNGLASSES +- chars: ! "\U0001F454" + codes: 1F454 + name: NECKTIE +- chars: ! "\U0001F455" + codes: 1F455 + name: T-SHIRT +- chars: ! "\U0001F456" + codes: 1F456 + name: JEANS +- chars: ! "\U0001F457" + codes: 1F457 + name: DRESS +- chars: ! "\U0001F458" + codes: 1F458 + name: KIMONO +- chars: ! "\U0001F459" + codes: 1F459 + name: BIKINI +- chars: ! "\U0001F45A" + codes: 1F45A + name: WOMANS CLOTHES +- chars: ! "\U0001F45B" + codes: 1F45B + name: PURSE +- chars: ! "\U0001F45C" + codes: 1F45C + name: HANDBAG +- chars: ! "\U0001F45D" + codes: 1F45D + name: POUCH +- chars: ! "\U0001F6CD" + codes: 1F6CD + name: SHOPPING BAGS +- chars: ! "\U0001F392" + codes: 1F392 + name: SCHOOL SATCHEL +- chars: ! "\U0001F45E" + codes: 1F45E + name: MANS SHOE +- chars: ! "\U0001F45F" + codes: 1F45F + name: ATHLETIC SHOE +- chars: ! "\U0001F460" + codes: 1F460 + name: HIGH-HEELED SHOE +- chars: ! "\U0001F461" + codes: 1F461 + name: WOMANS SANDAL +- chars: ! "\U0001F462" + codes: 1F462 + name: WOMANS BOOTS +- chars: ! "\U0001F451" + codes: 1F451 + name: CROWN +- chars: ! "\U0001F452" + codes: 1F452 + name: WOMANS HAT +- chars: ! "\U0001F3A9" + codes: 1F3A9 + name: TOP HAT +- chars: ! "\U0001F393" + codes: 1F393 + name: GRADUATION CAP +- chars: ! "\U0001F4FF" + codes: 1F4FF + name: PRAYER BEADS +- chars: ! "\U0001F484" + codes: 1F484 + name: LIPSTICK +- chars: ! "\U0001F48D" + codes: 1F48D + name: RING +- chars: ! "\U0001F48E" + codes: 1F48E + name: GEM STONE +- chars: ! "\U0001F435" + codes: 1F435 + name: MONKEY FACE +- chars: ! "\U0001F412" + codes: 1F412 + name: MONKEY +- chars: ! "\U0001F436" + codes: 1F436 + name: DOG FACE +- chars: ! "\U0001F415" + codes: 1F415 + name: DOG +- chars: ! "\U0001F429" + codes: 1F429 + name: POODLE +- chars: ! "\U0001F43A" + codes: 1F43A + name: WOLF FACE +- chars: ! "\U0001F431" + codes: 1F431 + name: CAT FACE +- chars: ! "\U0001F408" + codes: 1F408 + name: CAT +- chars: ! "\U0001F981" + codes: 1F981 + name: LION FACE +- chars: ! "\U0001F42F" + codes: 1F42F + name: TIGER FACE +- chars: ! "\U0001F405" + codes: 1F405 + name: TIGER +- chars: ! "\U0001F406" + codes: 1F406 + name: LEOPARD +- chars: ! "\U0001F434" + codes: 1F434 + name: HORSE FACE +- chars: ! "\U0001F40E" + codes: 1F40E + name: HORSE +- chars: ! "\U0001F984" + codes: 1F984 + name: UNICORN FACE +- chars: ! "\U0001F42E" + codes: 1F42E + name: COW FACE +- chars: ! "\U0001F402" + codes: 1F402 + name: OX +- chars: ! "\U0001F403" + codes: 1F403 + name: WATER BUFFALO +- chars: ! "\U0001F404" + codes: 1F404 + name: COW +- chars: ! "\U0001F437" + codes: 1F437 + name: PIG FACE +- chars: ! "\U0001F416" + codes: 1F416 + name: PIG +- chars: ! "\U0001F417" + codes: 1F417 + name: BOAR +- chars: ! "\U0001F43D" + codes: 1F43D + name: PIG NOSE +- chars: ! "\U0001F40F" + codes: 1F40F + name: RAM +- chars: ! "\U0001F411" + codes: 1F411 + name: SHEEP +- chars: ! "\U0001F410" + codes: 1F410 + name: GOAT +- chars: ! "\U0001F42A" + codes: 1F42A + name: DROMEDARY CAMEL +- chars: ! "\U0001F42B" + codes: 1F42B + name: BACTRIAN CAMEL +- chars: ! "\U0001F418" + codes: 1F418 + name: ELEPHANT +- chars: ! "\U0001F42D" + codes: 1F42D + name: MOUSE FACE +- chars: ! "\U0001F401" + codes: 1F401 + name: MOUSE +- chars: ! "\U0001F400" + codes: 1F400 + name: RAT +- chars: ! "\U0001F439" + codes: 1F439 + name: HAMSTER FACE +- chars: ! "\U0001F430" + codes: 1F430 + name: RABBIT FACE +- chars: ! "\U0001F407" + codes: 1F407 + name: RABBIT +- chars: ! "\U0001F43F" + codes: 1F43F + name: CHIPMUNK +- chars: ! "\U0001F43B" + codes: 1F43B + name: BEAR FACE +- chars: ! "\U0001F428" + codes: 1F428 + name: KOALA +- chars: ! "\U0001F43C" + codes: 1F43C + name: PANDA FACE +- chars: ! "\U0001F43E" + codes: 1F43E + name: PAW PRINTS +- chars: ! "\U0001F983" + codes: 1F983 + name: TURKEY +- chars: ! "\U0001F414" + codes: 1F414 + name: CHICKEN +- chars: ! "\U0001F413" + codes: 1F413 + name: ROOSTER +- chars: ! "\U0001F423" + codes: 1F423 + name: HATCHING CHICK +- chars: ! "\U0001F424" + codes: 1F424 + name: BABY CHICK +- chars: ! "\U0001F425" + codes: 1F425 + name: FRONT-FACING BABY CHICK +- chars: ! "\U0001F426" + codes: 1F426 + name: BIRD +- chars: ! "\U0001F427" + codes: 1F427 + name: PENGUIN +- chars: ! "\U0001F54A" + codes: 1F54A + name: DOVE OF PEACE +- chars: ! "\U0001F438" + codes: 1F438 + name: FROG FACE +- chars: ! "\U0001F40A" + codes: 1F40A + name: CROCODILE +- chars: ! "\U0001F422" + codes: 1F422 + name: TURTLE +- chars: ! "\U0001F40D" + codes: 1F40D + name: SNAKE +- chars: ! "\U0001F432" + codes: 1F432 + name: DRAGON FACE +- chars: ! "\U0001F409" + codes: 1F409 + name: DRAGON +- chars: ! "\U0001F433" + codes: 1F433 + name: SPOUTING WHALE +- chars: ! "\U0001F40B" + codes: 1F40B + name: WHALE +- chars: ! "\U0001F42C" + codes: 1F42C + name: DOLPHIN +- chars: ! "\U0001F41F" + codes: 1F41F + name: FISH +- chars: ! "\U0001F420" + codes: 1F420 + name: TROPICAL FISH +- chars: ! "\U0001F421" + codes: 1F421 + name: BLOWFISH +- chars: ! "\U0001F419" + codes: 1F419 + name: OCTOPUS +- chars: ! "\U0001F41A" + codes: 1F41A + name: SPIRAL SHELL +- chars: ! "\U0001F980" + codes: 1F980 + name: CRAB +- chars: ! "\U0001F40C" + codes: 1F40C + name: SNAIL +- chars: ! "\U0001F41B" + codes: 1F41B + name: BUG +- chars: ! "\U0001F41C" + codes: 1F41C + name: ANT +- chars: ! "\U0001F41D" + codes: 1F41D + name: HONEYBEE +- chars: ! "\U0001F41E" + codes: 1F41E + name: LADY BEETLE +- chars: ! "\U0001F577" + codes: 1F577 + name: SPIDER +- chars: ! "\U0001F578" + codes: 1F578 + name: SPIDER WEB +- chars: ! "\U0001F982" + codes: 1F982 + name: SCORPION +- chars: ! "\U0001F490" + codes: 1F490 + name: BOUQUET +- chars: ! "\U0001F338" + codes: 1F338 + name: CHERRY BLOSSOM +- chars: ! "\U0001F4AE" + codes: 1F4AE + name: WHITE FLOWER +- chars: ! "\U0001F3F5" + codes: 1F3F5 + name: ROSETTE +- chars: ! "\U0001F339" + codes: 1F339 + name: ROSE +- chars: ! "\U0001F33A" + codes: 1F33A + name: HIBISCUS +- chars: ! "\U0001F33B" + codes: 1F33B + name: SUNFLOWER +- chars: ! "\U0001F33C" + codes: 1F33C + name: BLOSSOM +- chars: ! "\U0001F337" + codes: 1F337 + name: TULIP +- chars: ☘ + codes: '2618' + name: SHAMROCK +- chars: ! "\U0001F331" + codes: 1F331 + name: SEEDLING +- chars: ! "\U0001F332" + codes: 1F332 + name: EVERGREEN TREE +- chars: ! "\U0001F333" + codes: 1F333 + name: DECIDUOUS TREE +- chars: ! "\U0001F334" + codes: 1F334 + name: PALM TREE +- chars: ! "\U0001F335" + codes: 1F335 + name: CACTUS +- chars: ! "\U0001F33E" + codes: 1F33E + name: EAR OF RICE +- chars: ! "\U0001F33F" + codes: 1F33F + name: HERB +- chars: ! "\U0001F340" + codes: 1F340 + name: FOUR LEAF CLOVER +- chars: ! "\U0001F341" + codes: 1F341 + name: MAPLE LEAF +- chars: ! "\U0001F342" + codes: 1F342 + name: FALLEN LEAF +- chars: ! "\U0001F343" + codes: 1F343 + name: LEAF FLUTTERING IN WIND +- chars: ! "\U0001F347" + codes: 1F347 + name: GRAPES +- chars: ! "\U0001F348" + codes: 1F348 + name: MELON +- chars: ! "\U0001F349" + codes: 1F349 + name: WATERMELON +- chars: ! "\U0001F34A" + codes: 1F34A + name: TANGERINE +- chars: ! "\U0001F34B" + codes: 1F34B + name: LEMON +- chars: ! "\U0001F34C" + codes: 1F34C + name: BANANA +- chars: ! "\U0001F34D" + codes: 1F34D + name: PINEAPPLE +- chars: ! "\U0001F34E" + codes: 1F34E + name: RED APPLE +- chars: ! "\U0001F34F" + codes: 1F34F + name: GREEN APPLE +- chars: ! "\U0001F350" + codes: 1F350 + name: PEAR +- chars: ! "\U0001F351" + codes: 1F351 + name: PEACH +- chars: ! "\U0001F352" + codes: 1F352 + name: CHERRIES +- chars: ! "\U0001F353" + codes: 1F353 + name: STRAWBERRY +- chars: ! "\U0001F345" + codes: 1F345 + name: TOMATO +- chars: ! "\U0001F346" + codes: 1F346 + name: AUBERGINE +- chars: ! "\U0001F33D" + codes: 1F33D + name: EAR OF MAIZE +- chars: ! "\U0001F336" + codes: 1F336 + name: HOT PEPPER +- chars: ! "\U0001F344" + codes: 1F344 + name: MUSHROOM +- chars: ! "\U0001F330" + codes: 1F330 + name: CHESTNUT +- chars: ! "\U0001F35E" + codes: 1F35E + name: BREAD +- chars: ! "\U0001F9C0" + codes: 1F9C0 + name: CHEESE WEDGE +- chars: ! "\U0001F356" + codes: 1F356 + name: MEAT ON BONE +- chars: ! "\U0001F357" + codes: 1F357 + name: POULTRY LEG +- chars: ! "\U0001F354" + codes: 1F354 + name: HAMBURGER +- chars: ! "\U0001F35F" + codes: 1F35F + name: FRENCH FRIES +- chars: ! "\U0001F355" + codes: 1F355 + name: SLICE OF PIZZA +- chars: ! "\U0001F32D" + codes: 1F32D + name: HOT DOG +- chars: ! "\U0001F32E" + codes: 1F32E + name: TACO +- chars: ! "\U0001F32F" + codes: 1F32F + name: BURRITO +- chars: ! "\U0001F37F" + codes: 1F37F + name: POPCORN +- chars: ! "\U0001F372" + codes: 1F372 + name: POT OF FOOD +- chars: ! "\U0001F371" + codes: 1F371 + name: BENTO BOX +- chars: ! "\U0001F358" + codes: 1F358 + name: RICE CRACKER +- chars: ! "\U0001F359" + codes: 1F359 + name: RICE BALL +- chars: ! "\U0001F35A" + codes: 1F35A + name: COOKED RICE +- chars: ! "\U0001F35B" + codes: 1F35B + name: CURRY AND RICE +- chars: ! "\U0001F35C" + codes: 1F35C + name: STEAMING BOWL +- chars: ! "\U0001F35D" + codes: 1F35D + name: SPAGHETTI +- chars: ! "\U0001F360" + codes: 1F360 + name: ROASTED SWEET POTATO +- chars: ! "\U0001F362" + codes: 1F362 + name: ODEN +- chars: ! "\U0001F363" + codes: 1F363 + name: SUSHI +- chars: ! "\U0001F364" + codes: 1F364 + name: FRIED SHRIMP +- chars: ! "\U0001F365" + codes: 1F365 + name: FISH CAKE WITH SWIRL DESIGN +- chars: ! "\U0001F361" + codes: 1F361 + name: DANGO +- chars: ! "\U0001F366" + codes: 1F366 + name: SOFT ICE CREAM +- chars: ! "\U0001F367" + codes: 1F367 + name: SHAVED ICE +- chars: ! "\U0001F368" + codes: 1F368 + name: ICE CREAM +- chars: ! "\U0001F369" + codes: 1F369 + name: DOUGHNUT +- chars: ! "\U0001F36A" + codes: 1F36A + name: COOKIE +- chars: ! "\U0001F382" + codes: 1F382 + name: BIRTHDAY CAKE +- chars: ! "\U0001F370" + codes: 1F370 + name: SHORTCAKE +- chars: ! "\U0001F36B" + codes: 1F36B + name: CHOCOLATE BAR +- chars: ! "\U0001F36C" + codes: 1F36C + name: CANDY +- chars: ! "\U0001F36D" + codes: 1F36D + name: LOLLIPOP +- chars: ! "\U0001F36E" + codes: 1F36E + name: CUSTARD +- chars: ! "\U0001F36F" + codes: 1F36F + name: HONEY POT +- chars: ! "\U0001F37C" + codes: 1F37C + name: BABY BOTTLE +- chars: ☕️ + codes: '2615' + name: HOT BEVERAGE +- chars: ! "\U0001F375" + codes: 1F375 + name: TEACUP WITHOUT HANDLE +- chars: ! "\U0001F376" + codes: 1F376 + name: SAKE BOTTLE AND CUP +- chars: ! "\U0001F37E" + codes: 1F37E + name: BOTTLE WITH POPPING CORK +- chars: ! "\U0001F377" + codes: 1F377 + name: WINE GLASS +- chars: ! "\U0001F378" + codes: 1F378 + name: COCKTAIL GLASS +- chars: ! "\U0001F379" + codes: 1F379 + name: TROPICAL DRINK +- chars: ! "\U0001F37A" + codes: 1F37A + name: BEER MUG +- chars: ! "\U0001F37B" + codes: 1F37B + name: CLINKING BEER MUGS +- chars: ! "\U0001F37D" + codes: 1F37D + name: FORK AND KNIFE WITH PLATE +- chars: ! "\U0001F374" + codes: 1F374 + name: FORK AND KNIFE +- chars: ! "\U0001F373" + codes: 1F373 + name: COOKING +- chars: ! "\U0001F3FA" + codes: 1F3FA + name: AMPHORA +- chars: ! "\U0001F30D" + codes: 1F30D + name: EARTH GLOBE EUROPE-AFRICA +- chars: ! "\U0001F30E" + codes: 1F30E + name: EARTH GLOBE AMERICAS +- chars: ! "\U0001F30F" + codes: 1F30F + name: EARTH GLOBE ASIA-AUSTRALIA +- chars: ! "\U0001F310" + codes: 1F310 + name: GLOBE WITH MERIDIANS +- chars: ! "\U0001F5FA" + codes: 1F5FA + name: WORLD MAP +- chars: ! "\U0001F3D4" + codes: 1F3D4 + name: SNOW CAPPED MOUNTAIN +- chars: ⛰ + codes: 26F0 + name: MOUNTAIN +- chars: ! "\U0001F30B" + codes: 1F30B + name: VOLCANO +- chars: ! "\U0001F5FB" + codes: 1F5FB + name: MOUNT FUJI +- chars: ! "\U0001F3D5" + codes: 1F3D5 + name: CAMPING +- chars: ! "\U0001F3D6" + codes: 1F3D6 + name: BEACH WITH UMBRELLA +- chars: ! "\U0001F3DC" + codes: 1F3DC + name: DESERT +- chars: ! "\U0001F3DD" + codes: 1F3DD + name: DESERT ISLAND +- chars: ! "\U0001F3DE" + codes: 1F3DE + name: NATIONAL PARK +- chars: ! "\U0001F3DF" + codes: 1F3DF + name: STADIUM +- chars: ! "\U0001F3DB" + codes: 1F3DB + name: CLASSICAL BUILDING +- chars: ! "\U0001F3D7" + codes: 1F3D7 + name: BUILDING CONSTRUCTION +- chars: ! "\U0001F3D8" + codes: 1F3D8 + name: HOUSE BUILDINGS +- chars: ! "\U0001F3D9" + codes: 1F3D9 + name: CITYSCAPE +- chars: ! "\U0001F3DA" + codes: 1F3DA + name: DERELICT HOUSE BUILDING +- chars: ! "\U0001F3E0" + codes: 1F3E0 + name: HOUSE BUILDING +- chars: ! "\U0001F3E1" + codes: 1F3E1 + name: HOUSE WITH GARDEN +- chars: ⛪️ + codes: 26EA + name: CHURCH +- chars: ! "\U0001F54B" + codes: 1F54B + name: KAABA +- chars: ! "\U0001F54C" + codes: 1F54C + name: MOSQUE +- chars: ! "\U0001F54D" + codes: 1F54D + name: SYNAGOGUE +- chars: ⛩ + codes: '26E9' + name: SHINTO SHRINE +- chars: ! "\U0001F3E2" + codes: 1F3E2 + name: OFFICE BUILDING +- chars: ! "\U0001F3E3" + codes: 1F3E3 + name: JAPANESE POST OFFICE +- chars: ! "\U0001F3E4" + codes: 1F3E4 + name: EUROPEAN POST OFFICE +- chars: ! "\U0001F3E5" + codes: 1F3E5 + name: HOSPITAL +- chars: ! "\U0001F3E6" + codes: 1F3E6 + name: BANK +- chars: ! "\U0001F3E8" + codes: 1F3E8 + name: HOTEL +- chars: ! "\U0001F3E9" + codes: 1F3E9 + name: LOVE HOTEL +- chars: ! "\U0001F3EA" + codes: 1F3EA + name: CONVENIENCE STORE +- chars: ! "\U0001F3EB" + codes: 1F3EB + name: SCHOOL +- chars: ! "\U0001F3EC" + codes: 1F3EC + name: DEPARTMENT STORE +- chars: ! "\U0001F3ED" + codes: 1F3ED + name: FACTORY +- chars: ! "\U0001F3EF" + codes: 1F3EF + name: JAPANESE CASTLE +- chars: ! "\U0001F3F0" + codes: 1F3F0 + name: EUROPEAN CASTLE +- chars: ! "\U0001F492" + codes: 1F492 + name: WEDDING +- chars: ! "\U0001F5FC" + codes: 1F5FC + name: TOKYO TOWER +- chars: ! "\U0001F5FD" + codes: 1F5FD + name: STATUE OF LIBERTY +- chars: ! "\U0001F5FE" + codes: 1F5FE + name: SILHOUETTE OF JAPAN +- chars: ⛲️ + codes: 26F2 + name: FOUNTAIN +- chars: ⛺️ + codes: 26FA + name: TENT +- chars: ! "\U0001F301" + codes: 1F301 + name: FOGGY +- chars: ! "\U0001F303" + codes: 1F303 + name: NIGHT WITH STARS +- chars: ! "\U0001F304" + codes: 1F304 + name: SUNRISE OVER MOUNTAINS +- chars: ! "\U0001F305" + codes: 1F305 + name: SUNRISE +- chars: ! "\U0001F306" + codes: 1F306 + name: CITYSCAPE AT DUSK +- chars: ! "\U0001F307" + codes: 1F307 + name: SUNSET OVER BUILDINGS +- chars: ! "\U0001F309" + codes: 1F309 + name: BRIDGE AT NIGHT +- chars: ♨️ + codes: '2668' + name: HOT SPRINGS +- chars: ! "\U0001F30C" + codes: 1F30C + name: MILKY WAY +- chars: ! "\U0001F3A0" + codes: 1F3A0 + name: CAROUSEL HORSE +- chars: ! "\U0001F3A1" + codes: 1F3A1 + name: FERRIS WHEEL +- chars: ! "\U0001F3A2" + codes: 1F3A2 + name: ROLLER COASTER +- chars: ! "\U0001F488" + codes: 1F488 + name: BARBER POLE +- chars: ! "\U0001F3AA" + codes: 1F3AA + name: CIRCUS TENT +- chars: ! "\U0001F3AD" + codes: 1F3AD + name: PERFORMING ARTS +- chars: ! "\U0001F5BC" + codes: 1F5BC + name: FRAME WITH PICTURE +- chars: ! "\U0001F3A8" + codes: 1F3A8 + name: ARTIST PALETTE +- chars: ! "\U0001F3B0" + codes: 1F3B0 + name: SLOT MACHINE +- chars: ! "\U0001F682" + codes: 1F682 + name: STEAM LOCOMOTIVE +- chars: ! "\U0001F683" + codes: 1F683 + name: RAILWAY CAR +- chars: ! "\U0001F684" + codes: 1F684 + name: HIGH-SPEED TRAIN +- chars: ! "\U0001F685" + codes: 1F685 + name: HIGH-SPEED TRAIN WITH BULLET NOSE +- chars: ! "\U0001F686" + codes: 1F686 + name: TRAIN +- chars: ! "\U0001F687" + codes: 1F687 + name: METRO +- chars: ! "\U0001F688" + codes: 1F688 + name: LIGHT RAIL +- chars: ! "\U0001F689" + codes: 1F689 + name: STATION +- chars: ! "\U0001F68A" + codes: 1F68A + name: TRAM +- chars: ! "\U0001F69D" + codes: 1F69D + name: MONORAIL +- chars: ! "\U0001F69E" + codes: 1F69E + name: MOUNTAIN RAILWAY +- chars: ! "\U0001F68B" + codes: 1F68B + name: TRAM CAR +- chars: ! "\U0001F68C" + codes: 1F68C + name: BUS +- chars: ! "\U0001F68D" + codes: 1F68D + name: ONCOMING BUS +- chars: ! "\U0001F68E" + codes: 1F68E + name: TROLLEYBUS +- chars: ! "\U0001F68F" + codes: 1F68F + name: BUS STOP +- chars: ! "\U0001F690" + codes: 1F690 + name: MINIBUS +- chars: ! "\U0001F691" + codes: 1F691 + name: AMBULANCE +- chars: ! "\U0001F692" + codes: 1F692 + name: FIRE ENGINE +- chars: ! "\U0001F693" + codes: 1F693 + name: POLICE CAR +- chars: ! "\U0001F694" + codes: 1F694 + name: ONCOMING POLICE CAR +- chars: ! "\U0001F695" + codes: 1F695 + name: TAXI +- chars: ! "\U0001F696" + codes: 1F696 + name: ONCOMING TAXI +- chars: ! "\U0001F697" + codes: 1F697 + name: AUTOMOBILE +- chars: ! "\U0001F698" + codes: 1F698 + name: ONCOMING AUTOMOBILE +- chars: ! "\U0001F699" + codes: 1F699 + name: RECREATIONAL VEHICLE +- chars: ! "\U0001F69A" + codes: 1F69A + name: DELIVERY TRUCK +- chars: ! "\U0001F69B" + codes: 1F69B + name: ARTICULATED LORRY +- chars: ! "\U0001F69C" + codes: 1F69C + name: TRACTOR +- chars: ! "\U0001F6B2" + codes: 1F6B2 + name: BICYCLE +- chars: ⛽️ + codes: 26FD + name: FUEL PUMP +- chars: ! "\U0001F6E3" + codes: 1F6E3 + name: MOTORWAY +- chars: ! "\U0001F6E4" + codes: 1F6E4 + name: RAILWAY TRACK +- chars: ! "\U0001F6A8" + codes: 1F6A8 + name: POLICE CARS REVOLVING LIGHT +- chars: ! "\U0001F6A5" + codes: 1F6A5 + name: HORIZONTAL TRAFFIC LIGHT +- chars: ! "\U0001F6A6" + codes: 1F6A6 + name: VERTICAL TRAFFIC LIGHT +- chars: ! "\U0001F6A7" + codes: 1F6A7 + name: CONSTRUCTION SIGN +- chars: ⚓️ + codes: '2693' + name: ANCHOR +- chars: ⛵️ + codes: 26F5 + name: SAILBOAT +- chars: ! "\U0001F6A3" + codes: 1F6A3 + name: ROWBOAT +- chars: ! "\U0001F6A4" + codes: 1F6A4 + name: SPEEDBOAT +- chars: ! "\U0001F6F3" + codes: 1F6F3 + name: PASSENGER SHIP +- chars: ⛴ + codes: 26F4 + name: FERRY +- chars: ! "\U0001F6E5" + codes: 1F6E5 + name: MOTOR BOAT +- chars: ! "\U0001F6A2" + codes: 1F6A2 + name: SHIP +- chars: ✈️ + codes: '2708' + name: AIRPLANE +- chars: ! "\U0001F6E9" + codes: 1F6E9 + name: SMALL AIRPLANE +- chars: ! "\U0001F6EB" + codes: 1F6EB + name: AIRPLANE DEPARTURE +- chars: ! "\U0001F6EC" + codes: 1F6EC + name: AIRPLANE ARRIVING +- chars: ! "\U0001F4BA" + codes: 1F4BA + name: SEAT +- chars: ! "\U0001F681" + codes: 1F681 + name: HELICOPTER +- chars: ! "\U0001F69F" + codes: 1F69F + name: SUSPENSION RAILWAY +- chars: ! "\U0001F6A0" + codes: 1F6A0 + name: MOUNTAIN CABLEWAY +- chars: ! "\U0001F6A1" + codes: 1F6A1 + name: AERIAL TRAMWAY +- chars: ! "\U0001F680" + codes: 1F680 + name: ROCKET +- chars: ! "\U0001F6F0" + codes: 1F6F0 + name: SATELLITE +- chars: ! "\U0001F6CE" + codes: 1F6CE + name: BELLHOP BELL +- chars: ! "\U0001F6AA" + codes: 1F6AA + name: DOOR +- chars: ! "\U0001F6CC" + codes: 1F6CC + name: SLEEPING ACCOMMODATION +- chars: ! "\U0001F6CF" + codes: 1F6CF + name: BED +- chars: ! "\U0001F6CB" + codes: 1F6CB + name: COUCH AND LAMP +- chars: ! "\U0001F6BD" + codes: 1F6BD + name: TOILET +- chars: ! "\U0001F6BF" + codes: 1F6BF + name: SHOWER +- chars: ! "\U0001F6C0" + codes: 1F6C0 + name: BATH +- chars: ! "\U0001F6C1" + codes: 1F6C1 + name: BATHTUB +- chars: ⌛️ + codes: 231B + name: HOURGLASS +- chars: ⏳ + codes: 23F3 + name: HOURGLASS WITH FLOWING SAND +- chars: ⌚️ + codes: 231A + name: WATCH +- chars: ⏰ + codes: 23F0 + name: ALARM CLOCK +- chars: ⏱ + codes: 23F1 + name: STOPWATCH +- chars: ⏲ + codes: 23F2 + name: TIMER CLOCK +- chars: ! "\U0001F570" + codes: 1F570 + name: MANTELPIECE CLOCK +- chars: ! "\U0001F55B" + codes: 1F55B + name: CLOCK FACE TWELVE OCLOCK +- chars: ! "\U0001F567" + codes: 1F567 + name: CLOCK FACE TWELVE-THIRTY +- chars: ! "\U0001F550" + codes: 1F550 + name: CLOCK FACE ONE OCLOCK +- chars: ! "\U0001F55C" + codes: 1F55C + name: CLOCK FACE ONE-THIRTY +- chars: ! "\U0001F551" + codes: 1F551 + name: CLOCK FACE TWO OCLOCK +- chars: ! "\U0001F55D" + codes: 1F55D + name: CLOCK FACE TWO-THIRTY +- chars: ! "\U0001F552" + codes: 1F552 + name: CLOCK FACE THREE OCLOCK +- chars: ! "\U0001F55E" + codes: 1F55E + name: CLOCK FACE THREE-THIRTY +- chars: ! "\U0001F553" + codes: 1F553 + name: CLOCK FACE FOUR OCLOCK +- chars: ! "\U0001F55F" + codes: 1F55F + name: CLOCK FACE FOUR-THIRTY +- chars: ! "\U0001F554" + codes: 1F554 + name: CLOCK FACE FIVE OCLOCK +- chars: ! "\U0001F560" + codes: 1F560 + name: CLOCK FACE FIVE-THIRTY +- chars: ! "\U0001F555" + codes: 1F555 + name: CLOCK FACE SIX OCLOCK +- chars: ! "\U0001F561" + codes: 1F561 + name: CLOCK FACE SIX-THIRTY +- chars: ! "\U0001F556" + codes: 1F556 + name: CLOCK FACE SEVEN OCLOCK +- chars: ! "\U0001F562" + codes: 1F562 + name: CLOCK FACE SEVEN-THIRTY +- chars: ! "\U0001F557" + codes: 1F557 + name: CLOCK FACE EIGHT OCLOCK +- chars: ! "\U0001F563" + codes: 1F563 + name: CLOCK FACE EIGHT-THIRTY +- chars: ! "\U0001F558" + codes: 1F558 + name: CLOCK FACE NINE OCLOCK +- chars: ! "\U0001F564" + codes: 1F564 + name: CLOCK FACE NINE-THIRTY +- chars: ! "\U0001F559" + codes: 1F559 + name: CLOCK FACE TEN OCLOCK +- chars: ! "\U0001F565" + codes: 1F565 + name: CLOCK FACE TEN-THIRTY +- chars: ! "\U0001F55A" + codes: 1F55A + name: CLOCK FACE ELEVEN OCLOCK +- chars: ! "\U0001F566" + codes: 1F566 + name: CLOCK FACE ELEVEN-THIRTY +- chars: ! "\U0001F311" + codes: 1F311 + name: NEW MOON SYMBOL +- chars: ! "\U0001F312" + codes: 1F312 + name: WAXING CRESCENT MOON SYMBOL +- chars: ! "\U0001F313" + codes: 1F313 + name: FIRST QUARTER MOON SYMBOL +- chars: ! "\U0001F314" + codes: 1F314 + name: WAXING GIBBOUS MOON SYMBOL +- chars: ! "\U0001F315" + codes: 1F315 + name: FULL MOON SYMBOL +- chars: ! "\U0001F316" + codes: 1F316 + name: WANING GIBBOUS MOON SYMBOL +- chars: ! "\U0001F317" + codes: 1F317 + name: LAST QUARTER MOON SYMBOL +- chars: ! "\U0001F318" + codes: 1F318 + name: WANING CRESCENT MOON SYMBOL +- chars: ! "\U0001F319" + codes: 1F319 + name: CRESCENT MOON +- chars: ! "\U0001F31A" + codes: 1F31A + name: NEW MOON WITH FACE +- chars: ! "\U0001F31B" + codes: 1F31B + name: FIRST QUARTER MOON WITH FACE +- chars: ! "\U0001F31C" + codes: 1F31C + name: LAST QUARTER MOON WITH FACE +- chars: ! "\U0001F321" + codes: 1F321 + name: THERMOMETER +- chars: ☀️ + codes: '2600' + name: BLACK SUN WITH RAYS +- chars: ! "\U0001F31D" + codes: 1F31D + name: FULL MOON WITH FACE +- chars: ! "\U0001F31E" + codes: 1F31E + name: SUN WITH FACE +- chars: ⭐️ + codes: 2B50 + name: WHITE MEDIUM STAR +- chars: ! "\U0001F31F" + codes: 1F31F + name: GLOWING STAR +- chars: ! "\U0001F320" + codes: 1F320 + name: SHOOTING STAR +- chars: ☁️ + codes: '2601' + name: CLOUD +- chars: ⛅️ + codes: 26C5 + name: SUN BEHIND CLOUD +- chars: ⛈ + codes: 26C8 + name: THUNDER CLOUD AND RAIN +- chars: ! "\U0001F324" + codes: 1F324 + name: WHITE SUN WITH SMALL CLOUD +- chars: ! "\U0001F325" + codes: 1F325 + name: WHITE SUN BEHIND CLOUD +- chars: ! "\U0001F326" + codes: 1F326 + name: WHITE SUN BEHIND CLOUD WITH RAIN +- chars: ! "\U0001F327" + codes: 1F327 + name: CLOUD WITH RAIN +- chars: ! "\U0001F328" + codes: 1F328 + name: CLOUD WITH SNOW +- chars: ! "\U0001F329" + codes: 1F329 + name: CLOUD WITH LIGHTNING +- chars: ! "\U0001F32A" + codes: 1F32A + name: CLOUD WITH TORNADO +- chars: ! "\U0001F32B" + codes: 1F32B + name: FOG +- chars: ! "\U0001F32C" + codes: 1F32C + name: WIND BLOWING FACE +- chars: ! "\U0001F300" + codes: 1F300 + name: CYCLONE +- chars: ! "\U0001F308" + codes: 1F308 + name: RAINBOW +- chars: ! "\U0001F302" + codes: 1F302 + name: CLOSED UMBRELLA +- chars: ☂ + codes: '2602' + name: UMBRELLA +- chars: ☔️ + codes: '2614' + name: UMBRELLA WITH RAIN DROPS +- chars: ⛱ + codes: 26F1 + name: UMBRELLA ON GROUND +- chars: ⚡️ + codes: 26A1 + name: HIGH VOLTAGE SIGN +- chars: ❄️ + codes: '2744' + name: SNOWFLAKE +- chars: ☃ + codes: '2603' + name: SNOWMAN +- chars: ⛄️ + codes: 26C4 + name: SNOWMAN WITHOUT SNOW +- chars: ☄ + codes: '2604' + name: COMET +- chars: ! "\U0001F525" + codes: 1F525 + name: FIRE +- chars: ! "\U0001F4A7" + codes: 1F4A7 + name: DROPLET +- chars: ! "\U0001F30A" + codes: 1F30A + name: WATER WAVE +- chars: ! "\U0001F383" + codes: 1F383 + name: JACK-O-LANTERN +- chars: ! "\U0001F384" + codes: 1F384 + name: CHRISTMAS TREE +- chars: ! "\U0001F386" + codes: 1F386 + name: FIREWORKS +- chars: ! "\U0001F387" + codes: 1F387 + name: FIREWORK SPARKLER +- chars: ✨ + codes: '2728' + name: SPARKLES +- chars: ! "\U0001F388" + codes: 1F388 + name: BALLOON +- chars: ! "\U0001F389" + codes: 1F389 + name: PARTY POPPER +- chars: ! "\U0001F38A" + codes: 1F38A + name: CONFETTI BALL +- chars: ! "\U0001F38B" + codes: 1F38B + name: TANABATA TREE +- chars: ! "\U0001F38C" + codes: 1F38C + name: CROSSED FLAGS +- chars: ! "\U0001F38D" + codes: 1F38D + name: PINE DECORATION +- chars: ! "\U0001F38E" + codes: 1F38E + name: JAPANESE DOLLS +- chars: ! "\U0001F38F" + codes: 1F38F + name: CARP STREAMER +- chars: ! "\U0001F390" + codes: 1F390 + name: WIND CHIME +- chars: ! "\U0001F391" + codes: 1F391 + name: MOON VIEWING CEREMONY +- chars: ! "\U0001F380" + codes: 1F380 + name: RIBBON +- chars: ! "\U0001F381" + codes: 1F381 + name: WRAPPED PRESENT +- chars: ! "\U0001F396" + codes: 1F396 + name: MILITARY MEDAL +- chars: ! "\U0001F397" + codes: 1F397 + name: REMINDER RIBBON +- chars: ! "\U0001F39E" + codes: 1F39E + name: FILM FRAMES +- chars: ! "\U0001F39F" + codes: 1F39F + name: ADMISSION TICKETS +- chars: ! "\U0001F3AB" + codes: 1F3AB + name: TICKET +- chars: ! "\U0001F3F7" + codes: 1F3F7 + name: LABEL +- chars: ⚽️ + codes: 26BD + name: SOCCER BALL +- chars: ⚾️ + codes: 26BE + name: BASEBALL +- chars: ! "\U0001F3C0" + codes: 1F3C0 + name: BASKETBALL AND HOOP +- chars: ! "\U0001F3C8" + codes: 1F3C8 + name: AMERICAN FOOTBALL +- chars: ! "\U0001F3C9" + codes: 1F3C9 + name: RUGBY FOOTBALL +- chars: ! "\U0001F3BE" + codes: 1F3BE + name: TENNIS RACQUET AND BALL +- chars: ! "\U0001F3B1" + codes: 1F3B1 + name: BILLIARDS +- chars: ! "\U0001F3B3" + codes: 1F3B3 + name: BOWLING +- chars: ⛳️ + codes: 26F3 + name: FLAG IN HOLE +- chars: ! "\U0001F3CC" + codes: 1F3CC + name: GOLFER +- chars: ⛸ + codes: 26F8 + name: ICE SKATE +- chars: ! "\U0001F3A3" + codes: 1F3A3 + name: FISHING POLE AND FISH +- chars: ! "\U0001F3BD" + codes: 1F3BD + name: RUNNING SHIRT WITH SASH +- chars: ! "\U0001F3BF" + codes: 1F3BF + name: SKI AND SKI BOOT +- chars: ⛷ + codes: 26F7 + name: SKIER +- chars: ! "\U0001F3C2" + codes: 1F3C2 + name: SNOWBOARDER +- chars: ! "\U0001F3C4" + codes: 1F3C4 + name: SURFER +- chars: ! "\U0001F3C7" + codes: 1F3C7 + name: HORSE RACING +- chars: ! "\U0001F3CA" + codes: 1F3CA + name: SWIMMER +- chars: ⛹ + codes: 26F9 + name: PERSON WITH BALL +- chars: ! "\U0001F3CB" + codes: 1F3CB + name: WEIGHT LIFTER +- chars: ! "\U0001F6B4" + codes: 1F6B4 + name: BICYCLIST +- chars: ! "\U0001F6B5" + codes: 1F6B5 + name: MOUNTAIN BICYCLIST +- chars: ! "\U0001F3CE" + codes: 1F3CE + name: RACING CAR +- chars: ! "\U0001F3CD" + codes: 1F3CD + name: RACING MOTORCYCLE +- chars: ! "\U0001F3C5" + codes: 1F3C5 + name: SPORTS MEDAL +- chars: ! "\U0001F3C6" + codes: 1F3C6 + name: TROPHY +- chars: ! "\U0001F3CF" + codes: 1F3CF + name: CRICKET BAT AND BALL +- chars: ! "\U0001F3D0" + codes: 1F3D0 + name: VOLLEYBALL +- chars: ! "\U0001F3D1" + codes: 1F3D1 + name: FIELD HOCKEY STICK AND BALL +- chars: ! "\U0001F3D2" + codes: 1F3D2 + name: ICE HOCKEY STICK AND PUCK +- chars: ! "\U0001F3D3" + codes: 1F3D3 + name: TABLE TENNIS PADDLE AND BALL +- chars: ! "\U0001F3F8" + codes: 1F3F8 + name: BADMINTON RACQUET AND SHUTTLECOCK +- chars: ! "\U0001F3AF" + codes: 1F3AF + name: DIRECT HIT +- chars: ! "\U0001F3AE" + codes: 1F3AE + name: VIDEO GAME +- chars: ! "\U0001F579" + codes: 1F579 + name: JOYSTICK +- chars: ! "\U0001F3B2" + codes: 1F3B2 + name: GAME DIE +- chars: ♠️ + codes: '2660' + name: BLACK SPADE SUIT +- chars: ♥️ + codes: '2665' + name: BLACK HEART SUIT +- chars: ♦️ + codes: '2666' + name: BLACK DIAMOND SUIT +- chars: ♣️ + codes: '2663' + name: BLACK CLUB SUIT +- chars: ! "\U0001F0CF" + codes: 1F0CF + name: PLAYING CARD BLACK JOKER +- chars: ! "\U0001F004️" + codes: 1F004 + name: MAHJONG TILE RED DRAGON +- chars: ! "\U0001F3B4" + codes: 1F3B4 + name: FLOWER PLAYING CARDS +- chars: ! "\U0001F507" + codes: 1F507 + name: SPEAKER WITH CANCELLATION STROKE +- chars: ! "\U0001F508" + codes: 1F508 + name: SPEAKER +- chars: ! "\U0001F509" + codes: 1F509 + name: SPEAKER WITH ONE SOUND WAVE +- chars: ! "\U0001F50A" + codes: 1F50A + name: SPEAKER WITH THREE SOUND WAVES +- chars: ! "\U0001F4E2" + codes: 1F4E2 + name: PUBLIC ADDRESS LOUDSPEAKER +- chars: ! "\U0001F4E3" + codes: 1F4E3 + name: CHEERING MEGAPHONE +- chars: ! "\U0001F4EF" + codes: 1F4EF + name: POSTAL HORN +- chars: ! "\U0001F514" + codes: 1F514 + name: BELL +- chars: ! "\U0001F515" + codes: 1F515 + name: BELL WITH CANCELLATION STROKE +- chars: ! "\U0001F3BC" + codes: 1F3BC + name: MUSICAL SCORE +- chars: ! "\U0001F3B5" + codes: 1F3B5 + name: MUSICAL NOTE +- chars: ! "\U0001F3B6" + codes: 1F3B6 + name: MULTIPLE MUSICAL NOTES +- chars: ! "\U0001F399" + codes: 1F399 + name: STUDIO MICROPHONE +- chars: ! "\U0001F39A" + codes: 1F39A + name: LEVEL SLIDER +- chars: ! "\U0001F39B" + codes: 1F39B + name: CONTROL KNOBS +- chars: ! "\U0001F3A4" + codes: 1F3A4 + name: MICROPHONE +- chars: ! "\U0001F3A7" + codes: 1F3A7 + name: HEADPHONE +- chars: ! "\U0001F3B7" + codes: 1F3B7 + name: SAXOPHONE +- chars: ! "\U0001F3B8" + codes: 1F3B8 + name: GUITAR +- chars: ! "\U0001F3B9" + codes: 1F3B9 + name: MUSICAL KEYBOARD +- chars: ! "\U0001F3BA" + codes: 1F3BA + name: TRUMPET +- chars: ! "\U0001F3BB" + codes: 1F3BB + name: VIOLIN +- chars: ! "\U0001F4FB" + codes: 1F4FB + name: RADIO +- chars: ! "\U0001F4F1" + codes: 1F4F1 + name: MOBILE PHONE +- chars: ! "\U0001F4F2" + codes: 1F4F2 + name: MOBILE PHONE WITH RIGHTWARDS ARROW AT LEFT +- chars: ☎️ + codes: '260E' + name: BLACK TELEPHONE +- chars: ! "\U0001F4DE" + codes: 1F4DE + name: TELEPHONE RECEIVER +- chars: ! "\U0001F4DF" + codes: 1F4DF + name: PAGER +- chars: ! "\U0001F4E0" + codes: 1F4E0 + name: FAX MACHINE +- chars: ! "\U0001F50B" + codes: 1F50B + name: BATTERY +- chars: ! "\U0001F50C" + codes: 1F50C + name: ELECTRIC PLUG +- chars: ! "\U0001F4BB" + codes: 1F4BB + name: PERSONAL COMPUTER +- chars: ! "\U0001F5A5" + codes: 1F5A5 + name: DESKTOP COMPUTER +- chars: ! "\U0001F5A8" + codes: 1F5A8 + name: PRINTER +- chars: ⌨ + codes: '2328' + name: KEYBOARD +- chars: ! "\U0001F5B1" + codes: 1F5B1 + name: THREE BUTTON MOUSE +- chars: ! "\U0001F5B2" + codes: 1F5B2 + name: TRACKBALL +- chars: ! "\U0001F4BD" + codes: 1F4BD + name: MINIDISC +- chars: ! "\U0001F4BE" + codes: 1F4BE + name: FLOPPY DISK +- chars: ! "\U0001F4BF" + codes: 1F4BF + name: OPTICAL DISC +- chars: ! "\U0001F4C0" + codes: 1F4C0 + name: DVD +- chars: ! "\U0001F3A5" + codes: 1F3A5 + name: MOVIE CAMERA +- chars: ! "\U0001F3AC" + codes: 1F3AC + name: CLAPPER BOARD +- chars: ! "\U0001F4FD" + codes: 1F4FD + name: FILM PROJECTOR +- chars: ! "\U0001F4FA" + codes: 1F4FA + name: TELEVISION +- chars: ! "\U0001F4F7" + codes: 1F4F7 + name: CAMERA +- chars: ! "\U0001F4F8" + codes: 1F4F8 + name: CAMERA WITH FLASH +- chars: ! "\U0001F4F9" + codes: 1F4F9 + name: VIDEO CAMERA +- chars: ! "\U0001F4FC" + codes: 1F4FC + name: VIDEOCASSETTE +- chars: ! "\U0001F50D" + codes: 1F50D + name: LEFT-POINTING MAGNIFYING GLASS +- chars: ! "\U0001F50E" + codes: 1F50E + name: RIGHT-POINTING MAGNIFYING GLASS +- chars: ! "\U0001F52C" + codes: 1F52C + name: MICROSCOPE +- chars: ! "\U0001F52D" + codes: 1F52D + name: TELESCOPE +- chars: ! "\U0001F4E1" + codes: 1F4E1 + name: SATELLITE ANTENNA +- chars: ! "\U0001F56F" + codes: 1F56F + name: CANDLE +- chars: ! "\U0001F4A1" + codes: 1F4A1 + name: ELECTRIC LIGHT BULB +- chars: ! "\U0001F526" + codes: 1F526 + name: ELECTRIC TORCH +- chars: ! "\U0001F3EE" + codes: 1F3EE + name: IZAKAYA LANTERN +- chars: ! "\U0001F4D4" + codes: 1F4D4 + name: NOTEBOOK WITH DECORATIVE COVER +- chars: ! "\U0001F4D5" + codes: 1F4D5 + name: CLOSED BOOK +- chars: ! "\U0001F4D6" + codes: 1F4D6 + name: OPEN BOOK +- chars: ! "\U0001F4D7" + codes: 1F4D7 + name: GREEN BOOK +- chars: ! "\U0001F4D8" + codes: 1F4D8 + name: BLUE BOOK +- chars: ! "\U0001F4D9" + codes: 1F4D9 + name: ORANGE BOOK +- chars: ! "\U0001F4DA" + codes: 1F4DA + name: BOOKS +- chars: ! "\U0001F4D3" + codes: 1F4D3 + name: NOTEBOOK +- chars: ! "\U0001F4D2" + codes: 1F4D2 + name: LEDGER +- chars: ! "\U0001F4C3" + codes: 1F4C3 + name: PAGE WITH CURL +- chars: ! "\U0001F4DC" + codes: 1F4DC + name: SCROLL +- chars: ! "\U0001F4C4" + codes: 1F4C4 + name: PAGE FACING UP +- chars: ! "\U0001F4F0" + codes: 1F4F0 + name: NEWSPAPER +- chars: ! "\U0001F5DE" + codes: 1F5DE + name: ROLLED-UP NEWSPAPER +- chars: ! "\U0001F4D1" + codes: 1F4D1 + name: BOOKMARK TABS +- chars: ! "\U0001F516" + codes: 1F516 + name: BOOKMARK +- chars: ! "\U0001F4B0" + codes: 1F4B0 + name: MONEY BAG +- chars: ! "\U0001F4B4" + codes: 1F4B4 + name: BANKNOTE WITH YEN SIGN +- chars: ! "\U0001F4B5" + codes: 1F4B5 + name: BANKNOTE WITH DOLLAR SIGN +- chars: ! "\U0001F4B6" + codes: 1F4B6 + name: BANKNOTE WITH EURO SIGN +- chars: ! "\U0001F4B7" + codes: 1F4B7 + name: BANKNOTE WITH POUND SIGN +- chars: ! "\U0001F4B8" + codes: 1F4B8 + name: MONEY WITH WINGS +- chars: ! "\U0001F4B3" + codes: 1F4B3 + name: CREDIT CARD +- chars: ! "\U0001F4B9" + codes: 1F4B9 + name: CHART WITH UPWARDS TREND AND YEN SIGN +- chars: ✉️ + codes: '2709' + name: ENVELOPE +- chars: ! "\U0001F4E7" + codes: 1F4E7 + name: E-MAIL SYMBOL +- chars: ! "\U0001F4E8" + codes: 1F4E8 + name: INCOMING ENVELOPE +- chars: ! "\U0001F4E9" + codes: 1F4E9 + name: ENVELOPE WITH DOWNWARDS ARROW ABOVE +- chars: ! "\U0001F4E4" + codes: 1F4E4 + name: OUTBOX TRAY +- chars: ! "\U0001F4E5" + codes: 1F4E5 + name: INBOX TRAY +- chars: ! "\U0001F4E6" + codes: 1F4E6 + name: PACKAGE +- chars: ! "\U0001F4EB" + codes: 1F4EB + name: CLOSED MAILBOX WITH RAISED FLAG +- chars: ! "\U0001F4EA" + codes: 1F4EA + name: CLOSED MAILBOX WITH LOWERED FLAG +- chars: ! "\U0001F4EC" + codes: 1F4EC + name: OPEN MAILBOX WITH RAISED FLAG +- chars: ! "\U0001F4ED" + codes: 1F4ED + name: OPEN MAILBOX WITH LOWERED FLAG +- chars: ! "\U0001F4EE" + codes: 1F4EE + name: POSTBOX +- chars: ! "\U0001F5F3" + codes: 1F5F3 + name: BALLOT BOX WITH BALLOT +- chars: ✏️ + codes: 270F + name: PENCIL +- chars: ✒️ + codes: '2712' + name: BLACK NIB +- chars: ! "\U0001F58B" + codes: 1F58B + name: LOWER LEFT FOUNTAIN PEN +- chars: ! "\U0001F58A" + codes: 1F58A + name: LOWER LEFT BALLPOINT PEN +- chars: ! "\U0001F58C" + codes: 1F58C + name: LOWER LEFT PAINTBRUSH +- chars: ! "\U0001F58D" + codes: 1F58D + name: LOWER LEFT CRAYON +- chars: ! "\U0001F4DD" + codes: 1F4DD + name: MEMO +- chars: ! "\U0001F4BC" + codes: 1F4BC + name: BRIEFCASE +- chars: ! "\U0001F4C1" + codes: 1F4C1 + name: FILE FOLDER +- chars: ! "\U0001F4C2" + codes: 1F4C2 + name: OPEN FILE FOLDER +- chars: ! "\U0001F5C2" + codes: 1F5C2 + name: CARD INDEX DIVIDERS +- chars: ! "\U0001F4C5" + codes: 1F4C5 + name: CALENDAR +- chars: ! "\U0001F4C6" + codes: 1F4C6 + name: TEAR-OFF CALENDAR +- chars: ! "\U0001F5D2" + codes: 1F5D2 + name: SPIRAL NOTE PAD +- chars: ! "\U0001F5D3" + codes: 1F5D3 + name: SPIRAL CALENDAR PAD +- chars: ! "\U0001F4C7" + codes: 1F4C7 + name: CARD INDEX +- chars: ! "\U0001F4C8" + codes: 1F4C8 + name: CHART WITH UPWARDS TREND +- chars: ! "\U0001F4C9" + codes: 1F4C9 + name: CHART WITH DOWNWARDS TREND +- chars: ! "\U0001F4CA" + codes: 1F4CA + name: BAR CHART +- chars: ! "\U0001F4CB" + codes: 1F4CB + name: CLIPBOARD +- chars: ! "\U0001F4CC" + codes: 1F4CC + name: PUSHPIN +- chars: ! "\U0001F4CD" + codes: 1F4CD + name: ROUND PUSHPIN +- chars: ! "\U0001F4CE" + codes: 1F4CE + name: PAPERCLIP +- chars: ! "\U0001F587" + codes: 1F587 + name: LINKED PAPERCLIPS +- chars: ! "\U0001F4CF" + codes: 1F4CF + name: STRAIGHT RULER +- chars: ! "\U0001F4D0" + codes: 1F4D0 + name: TRIANGULAR RULER +- chars: ✂️ + codes: '2702' + name: BLACK SCISSORS +- chars: ! "\U0001F5C3" + codes: 1F5C3 + name: CARD FILE BOX +- chars: ! "\U0001F5C4" + codes: 1F5C4 + name: FILE CABINET +- chars: ! "\U0001F5D1" + codes: 1F5D1 + name: WASTEBASKET +- chars: ! "\U0001F512" + codes: 1F512 + name: LOCK +- chars: ! "\U0001F513" + codes: 1F513 + name: OPEN LOCK +- chars: ! "\U0001F50F" + codes: 1F50F + name: LOCK WITH INK PEN +- chars: ! "\U0001F510" + codes: 1F510 + name: CLOSED LOCK WITH KEY +- chars: ! "\U0001F511" + codes: 1F511 + name: KEY +- chars: ! "\U0001F5DD" + codes: 1F5DD + name: OLD KEY +- chars: ! "\U0001F528" + codes: 1F528 + name: HAMMER +- chars: ⛏ + codes: 26CF + name: PICK +- chars: ⚒ + codes: '2692' + name: HAMMER AND PICK +- chars: ! "\U0001F6E0" + codes: 1F6E0 + name: HAMMER AND WRENCH +- chars: ! "\U0001F527" + codes: 1F527 + name: WRENCH +- chars: ! "\U0001F529" + codes: 1F529 + name: NUT AND BOLT +- chars: ⚙ + codes: '2699' + name: GEAR +- chars: ! "\U0001F5DC" + codes: 1F5DC + name: COMPRESSION +- chars: ⚗ + codes: '2697' + name: ALEMBIC +- chars: ⚖ + codes: '2696' + name: SCALES +- chars: ! "\U0001F517" + codes: 1F517 + name: LINK SYMBOL +- chars: ⛓ + codes: 26D3 + name: CHAINS +- chars: ! "\U0001F489" + codes: 1F489 + name: SYRINGE +- chars: ! "\U0001F48A" + codes: 1F48A + name: PILL +- chars: ! "\U0001F5E1" + codes: 1F5E1 + name: DAGGER KNIFE +- chars: ! "\U0001F52A" + codes: 1F52A + name: HOCHO +- chars: ⚔ + codes: '2694' + name: CROSSED SWORDS +- chars: ! "\U0001F52B" + codes: 1F52B + name: PISTOL +- chars: ! "\U0001F6E1" + codes: 1F6E1 + name: SHIELD +- chars: ! "\U0001F3F9" + codes: 1F3F9 + name: BOW AND ARROW +- chars: ! "\U0001F3C1" + codes: 1F3C1 + name: CHEQUERED FLAG +- chars: ! "\U0001F3F3" + codes: 1F3F3 + name: WAVING WHITE FLAG +- chars: ! "\U0001F3F4" + codes: 1F3F4 + name: WAVING BLACK FLAG +- chars: ! "\U0001F6A9" + codes: 1F6A9 + name: TRIANGULAR FLAG ON POST +- chars: ! "\U0001F6AC" + codes: 1F6AC + name: SMOKING SYMBOL +- chars: ⚰ + codes: 26B0 + name: COFFIN +- chars: ⚱ + codes: 26B1 + name: FUNERAL URN +- chars: ! "\U0001F5FF" + codes: 1F5FF + name: MOYAI +- chars: ! "\U0001F6E2" + codes: 1F6E2 + name: OIL DRUM +- chars: ! "\U0001F52E" + codes: 1F52E + name: CRYSTAL BALL +- chars: ! "\U0001F3E7" + codes: 1F3E7 + name: AUTOMATED TELLER MACHINE +- chars: ! "\U0001F6AE" + codes: 1F6AE + name: PUT LITTER IN ITS PLACE SYMBOL +- chars: ! "\U0001F6B0" + codes: 1F6B0 + name: POTABLE WATER SYMBOL +- chars: ♿️ + codes: 267F + name: WHEELCHAIR SYMBOL +- chars: ! "\U0001F6B9" + codes: 1F6B9 + name: MENS SYMBOL +- chars: ! "\U0001F6BA" + codes: 1F6BA + name: WOMENS SYMBOL +- chars: ! "\U0001F6BB" + codes: 1F6BB + name: RESTROOM +- chars: ! "\U0001F6BC" + codes: 1F6BC + name: BABY SYMBOL +- chars: ! "\U0001F6BE" + codes: 1F6BE + name: WATER CLOSET +- chars: ! "\U0001F6C2" + codes: 1F6C2 + name: PASSPORT CONTROL +- chars: ! "\U0001F6C3" + codes: 1F6C3 + name: CUSTOMS +- chars: ! "\U0001F6C4" + codes: 1F6C4 + name: BAGGAGE CLAIM +- chars: ! "\U0001F6C5" + codes: 1F6C5 + name: LEFT LUGGAGE +- chars: ⚠️ + codes: 26A0 + name: WARNING SIGN +- chars: ! "\U0001F6B8" + codes: 1F6B8 + name: CHILDREN CROSSING +- chars: ⛔️ + codes: 26D4 + name: NO ENTRY +- chars: ! "\U0001F6AB" + codes: 1F6AB + name: NO ENTRY SIGN +- chars: ! "\U0001F6B3" + codes: 1F6B3 + name: NO BICYCLES +- chars: ! "\U0001F6AD" + codes: 1F6AD + name: NO SMOKING SYMBOL +- chars: ! "\U0001F6AF" + codes: 1F6AF + name: DO NOT LITTER SYMBOL +- chars: ! "\U0001F6B1" + codes: 1F6B1 + name: NON-POTABLE WATER SYMBOL +- chars: ! "\U0001F6B7" + codes: 1F6B7 + name: NO PEDESTRIANS +- chars: ☢ + codes: '2622' + name: RADIOACTIVE SIGN +- chars: ☣ + codes: '2623' + name: BIOHAZARD SIGN +- chars: ⬆️ + codes: 2B06 + name: UPWARDS BLACK ARROW +- chars: ↗️ + codes: '2197' + name: NORTH EAST ARROW +- chars: ➡️ + codes: 27A1 + name: BLACK RIGHTWARDS ARROW +- chars: ↘️ + codes: '2198' + name: SOUTH EAST ARROW +- chars: ⬇️ + codes: 2B07 + name: DOWNWARDS BLACK ARROW +- chars: ↙️ + codes: '2199' + name: SOUTH WEST ARROW +- chars: ⬅️ + codes: 2B05 + name: LEFTWARDS BLACK ARROW +- chars: ↖️ + codes: '2196' + name: NORTH WEST ARROW +- chars: ↕️ + codes: '2195' + name: UP DOWN ARROW +- chars: ↔️ + codes: '2194' + name: LEFT RIGHT ARROW +- chars: ↩️ + codes: 21A9 + name: LEFTWARDS ARROW WITH HOOK +- chars: ↪️ + codes: 21AA + name: RIGHTWARDS ARROW WITH HOOK +- chars: ⤴️ + codes: '2934' + name: ARROW POINTING RIGHTWARDS THEN CURVING UPWARDS +- chars: ⤵️ + codes: '2935' + name: ARROW POINTING RIGHTWARDS THEN CURVING DOWNWARDS +- chars: ! "\U0001F503" + codes: 1F503 + name: CLOCKWISE DOWNWARDS AND UPWARDS OPEN CIRCLE ARROWS +- chars: ! "\U0001F504" + codes: 1F504 + name: ANTICLOCKWISE DOWNWARDS AND UPWARDS OPEN CIRCLE ARROWS +- chars: ! "\U0001F519" + codes: 1F519 + name: BACK WITH LEFTWARDS ARROW ABOVE +- chars: ! "\U0001F51A" + codes: 1F51A + name: END WITH LEFTWARDS ARROW ABOVE +- chars: ! "\U0001F51B" + codes: 1F51B + name: ON WITH EXCLAMATION MARK WITH LEFT RIGHT ARROW ABOVE +- chars: ! "\U0001F51C" + codes: 1F51C + name: SOON WITH RIGHTWARDS ARROW ABOVE +- chars: ! "\U0001F51D" + codes: 1F51D + name: TOP WITH UPWARDS ARROW ABOVE +- chars: ! "\U0001F6D0" + codes: 1F6D0 + name: PLACE OF WORSHIP +- chars: ⚛ + codes: 269B + name: ATOM SYMBOL +- chars: ! "\U0001F549" + codes: 1F549 + name: OM SYMBOL +- chars: ✡ + codes: '2721' + name: STAR OF DAVID +- chars: ☸ + codes: '2638' + name: WHEEL OF DHARMA +- chars: ☯ + codes: 262F + name: YIN YANG +- chars: ✝ + codes: 271D + name: LATIN CROSS +- chars: ☦ + codes: '2626' + name: ORTHODOX CROSS +- chars: ☪ + codes: 262A + name: STAR AND CRESCENT +- chars: ☮ + codes: '262E' + name: PEACE SYMBOL +- chars: ! "\U0001F54E" + codes: 1F54E + name: MENORAH WITH NINE BRANCHES +- chars: ! "\U0001F52F" + codes: 1F52F + name: SIX POINTED STAR WITH MIDDLE DOT +- chars: ♻️ + codes: 267B + name: BLACK UNIVERSAL RECYCLING SYMBOL +- chars: ! "\U0001F4DB" + codes: 1F4DB + name: NAME BADGE +- chars: ⚜ + codes: 269C + name: FLEUR-DE-LIS +- chars: ! "\U0001F530" + codes: 1F530 + name: JAPANESE SYMBOL FOR BEGINNER +- chars: ! "\U0001F531" + codes: 1F531 + name: TRIDENT EMBLEM +- chars: ⭕️ + codes: 2B55 + name: HEAVY LARGE CIRCLE +- chars: ✅ + codes: '2705' + name: WHITE HEAVY CHECK MARK +- chars: ☑️ + codes: '2611' + name: BALLOT BOX WITH CHECK +- chars: ✔️ + codes: '2714' + name: HEAVY CHECK MARK +- chars: ✖️ + codes: '2716' + name: HEAVY MULTIPLICATION X +- chars: ❌ + codes: 274C + name: CROSS MARK +- chars: ❎ + codes: '274E' + name: NEGATIVE SQUARED CROSS MARK +- chars: ➕ + codes: '2795' + name: HEAVY PLUS SIGN +- chars: ➖ + codes: '2796' + name: HEAVY MINUS SIGN +- chars: ➗ + codes: '2797' + name: HEAVY DIVISION SIGN +- chars: ➰ + codes: 27B0 + name: CURLY LOOP +- chars: ➿ + codes: 27BF + name: DOUBLE CURLY LOOP +- chars: 〽️ + codes: 303D + name: PART ALTERNATION MARK +- chars: ✳️ + codes: '2733' + name: EIGHT SPOKED ASTERISK +- chars: ✴️ + codes: '2734' + name: EIGHT POINTED BLACK STAR +- chars: ❇️ + codes: '2747' + name: SPARKLE +- chars: ! "\U0001F4B1" + codes: 1F4B1 + name: CURRENCY EXCHANGE +- chars: ! "\U0001F4B2" + codes: 1F4B2 + name: HEAVY DOLLAR SIGN +- chars: ‼️ + codes: 203C + name: DOUBLE EXCLAMATION MARK +- chars: ⁉️ + codes: '2049' + name: EXCLAMATION QUESTION MARK +- chars: ❓ + codes: '2753' + name: BLACK QUESTION MARK ORNAMENT +- chars: ❔ + codes: '2754' + name: WHITE QUESTION MARK ORNAMENT +- chars: ❕ + codes: '2755' + name: WHITE EXCLAMATION MARK ORNAMENT +- chars: ❗️ + codes: '2757' + name: HEAVY EXCLAMATION MARK SYMBOL +- chars: 〰️ + codes: '3030' + name: WAVY DASH +- chars: ©️ + codes: 00A9 + name: COPYRIGHT SIGN +- chars: ®️ + codes: 00AE + name: REGISTERED SIGN +- chars: ™️ + codes: '2122' + name: TRADE MARK SIGN +- chars: ♈️ + codes: '2648' + name: ARIES +- chars: ♉️ + codes: '2649' + name: TAURUS +- chars: ♊️ + codes: 264A + name: GEMINI +- chars: ♋️ + codes: 264B + name: CANCER +- chars: ♌️ + codes: 264C + name: LEO +- chars: ♍️ + codes: 264D + name: VIRGO +- chars: ♎️ + codes: '264E' + name: LIBRA +- chars: ♏️ + codes: 264F + name: SCORPIUS +- chars: ♐️ + codes: '2650' + name: SAGITTARIUS +- chars: ♑️ + codes: '2651' + name: CAPRICORN +- chars: ♒️ + codes: '2652' + name: AQUARIUS +- chars: ♓️ + codes: '2653' + name: PISCES +- chars: ⛎ + codes: 26CE + name: OPHIUCHUS +- chars: ! "\U0001F500" + codes: 1F500 + name: TWISTED RIGHTWARDS ARROWS +- chars: ! "\U0001F501" + codes: 1F501 + name: CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE ARROWS +- chars: ! "\U0001F502" + codes: 1F502 + name: CLOCKWISE RIGHTWARDS AND LEFTWARDS OPEN CIRCLE ARROWS WITH CIRCLED ONE OVERLAY +- chars: ▶️ + codes: 25B6 + name: BLACK RIGHT-POINTING TRIANGLE +- chars: ⏩ + codes: '23E9' + name: BLACK RIGHT-POINTING DOUBLE TRIANGLE +- chars: ⏭ + codes: 23ED + name: BLACK RIGHT-POINTING DOUBLE TRIANGLE WITH VERTICAL BAR +- chars: ⏯ + codes: 23EF + name: BLACK RIGHT-POINTING TRIANGLE WITH DOUBLE VERTICAL BAR +- chars: ◀️ + codes: 25C0 + name: BLACK LEFT-POINTING TRIANGLE +- chars: ⏪ + codes: 23EA + name: BLACK LEFT-POINTING DOUBLE TRIANGLE +- chars: ⏮ + codes: '23EE' + name: BLACK LEFT-POINTING DOUBLE TRIANGLE WITH VERTICAL BAR +- chars: ! "\U0001F53C" + codes: 1F53C + name: UP-POINTING SMALL RED TRIANGLE +- chars: ⏫ + codes: 23EB + name: BLACK UP-POINTING DOUBLE TRIANGLE +- chars: ! "\U0001F53D" + codes: 1F53D + name: DOWN-POINTING SMALL RED TRIANGLE +- chars: ⏬ + codes: 23EC + name: BLACK DOWN-POINTING DOUBLE TRIANGLE +- chars: ⏸ + codes: 23F8 + name: DOUBLE VERTICAL BAR +- chars: ⏹ + codes: 23F9 + name: BLACK SQUARE FOR STOP +- chars: ⏺ + codes: 23FA + name: BLACK CIRCLE FOR RECORD +- chars: ⏏ + codes: 23CF + name: EJECT SYMBOL +- chars: ! "\U0001F3A6" + codes: 1F3A6 + name: CINEMA +- chars: ! "\U0001F505" + codes: 1F505 + name: LOW BRIGHTNESS SYMBOL +- chars: ! "\U0001F506" + codes: 1F506 + name: HIGH BRIGHTNESS SYMBOL +- chars: ! "\U0001F4F6" + codes: 1F4F6 + name: ANTENNA WITH BARS +- chars: ! "\U0001F4F5" + codes: 1F4F5 + name: NO MOBILE PHONES +- chars: ! "\U0001F4F3" + codes: 1F4F3 + name: VIBRATION MODE +- chars: ! "\U0001F4F4" + codes: 1F4F4 + name: MOBILE PHONE OFF +- chars: ! '#' + codes: 0023 20E3 + name: Keycap NUMBER SIGN +- chars: ! '*⃣' + codes: 002A 20E3 + name: Keycap ASTERISK +- chars: '0' + codes: 0030 20E3 + name: Keycap DIGIT ZERO +- chars: '1' + codes: 0031 20E3 + name: Keycap DIGIT ONE +- chars: '2' + codes: 0032 20E3 + name: Keycap DIGIT TWO +- chars: '3' + codes: 0033 20E3 + name: Keycap DIGIT THREE +- chars: '4' + codes: 0034 20E3 + name: Keycap DIGIT FOUR +- chars: '5' + codes: 0035 20E3 + name: Keycap DIGIT FIVE +- chars: '6' + codes: 0036 20E3 + name: Keycap DIGIT SIX +- chars: '7' + codes: 0037 20E3 + name: Keycap DIGIT SEVEN +- chars: '8' + codes: 0038 20E3 + name: Keycap DIGIT EIGHT +- chars: '9' + codes: 0039 20E3 + name: Keycap DIGIT NINE +- chars: ! "\U0001F51F" + codes: 1F51F + name: KEYCAP TEN +- chars: ! "\U0001F4AF" + codes: 1F4AF + name: HUNDRED POINTS SYMBOL +- chars: ! "\U0001F51E" + codes: 1F51E + name: NO ONE UNDER EIGHTEEN SYMBOL +- chars: ! "\U0001F520" + codes: 1F520 + name: INPUT SYMBOL FOR LATIN CAPITAL LETTERS +- chars: ! "\U0001F521" + codes: 1F521 + name: INPUT SYMBOL FOR LATIN SMALL LETTERS +- chars: ! "\U0001F522" + codes: 1F522 + name: INPUT SYMBOL FOR NUMBERS +- chars: ! "\U0001F523" + codes: 1F523 + name: INPUT SYMBOL FOR SYMBOLS +- chars: ! "\U0001F524" + codes: 1F524 + name: INPUT SYMBOL FOR LATIN LETTERS +- chars: ! "\U0001F170️" + codes: 1F170 + name: NEGATIVE SQUARED LATIN CAPITAL LETTER A +- chars: ! "\U0001F18E" + codes: 1F18E + name: NEGATIVE SQUARED AB +- chars: ! "\U0001F171️" + codes: 1F171 + name: NEGATIVE SQUARED LATIN CAPITAL LETTER B +- chars: ! "\U0001F191" + codes: 1F191 + name: SQUARED CL +- chars: ! "\U0001F192" + codes: 1F192 + name: SQUARED COOL +- chars: ! "\U0001F193" + codes: 1F193 + name: SQUARED FREE +- chars: ℹ️ + codes: '2139' + name: INFORMATION SOURCE +- chars: ! "\U0001F194" + codes: 1F194 + name: SQUARED ID +- chars: Ⓜ️ + codes: 24C2 + name: CIRCLED LATIN CAPITAL LETTER M +- chars: ! "\U0001F195" + codes: 1F195 + name: SQUARED NEW +- chars: ! "\U0001F196" + codes: 1F196 + name: SQUARED NG +- chars: ! "\U0001F17E️" + codes: 1F17E + name: NEGATIVE SQUARED LATIN CAPITAL LETTER O +- chars: ! "\U0001F197" + codes: 1F197 + name: SQUARED OK +- chars: ! "\U0001F17F️" + codes: 1F17F + name: NEGATIVE SQUARED LATIN CAPITAL LETTER P +- chars: ! "\U0001F198" + codes: 1F198 + name: SQUARED SOS +- chars: ! "\U0001F199" + codes: 1F199 + name: SQUARED UP WITH EXCLAMATION MARK +- chars: ! "\U0001F19A" + codes: 1F19A + name: SQUARED VS +- chars: ! "\U0001F201" + codes: 1F201 + name: SQUARED KATAKANA KOKO +- chars: ! "\U0001F202️" + codes: 1F202 + name: SQUARED KATAKANA SA +- chars: ! "\U0001F237️" + codes: 1F237 + name: SQUARED CJK UNIFIED IDEOGRAPH-6708 +- chars: ! "\U0001F236" + codes: 1F236 + name: SQUARED CJK UNIFIED IDEOGRAPH-6709 +- chars: ! "\U0001F22F️" + codes: 1F22F + name: SQUARED CJK UNIFIED IDEOGRAPH-6307 +- chars: ! "\U0001F250" + codes: 1F250 + name: CIRCLED IDEOGRAPH ADVANTAGE +- chars: ! "\U0001F239" + codes: 1F239 + name: SQUARED CJK UNIFIED IDEOGRAPH-5272 +- chars: ! "\U0001F21A️" + codes: 1F21A + name: SQUARED CJK UNIFIED IDEOGRAPH-7121 +- chars: ! "\U0001F232" + codes: 1F232 + name: SQUARED CJK UNIFIED IDEOGRAPH-7981 +- chars: ! "\U0001F251" + codes: 1F251 + name: CIRCLED IDEOGRAPH ACCEPT +- chars: ! "\U0001F238" + codes: 1F238 + name: SQUARED CJK UNIFIED IDEOGRAPH-7533 +- chars: ! "\U0001F234" + codes: 1F234 + name: SQUARED CJK UNIFIED IDEOGRAPH-5408 +- chars: ! "\U0001F233" + codes: 1F233 + name: SQUARED CJK UNIFIED IDEOGRAPH-7A7A +- chars: ㊗️ + codes: '3297' + name: CIRCLED IDEOGRAPH CONGRATULATION +- chars: ㊙️ + codes: '3299' + name: CIRCLED IDEOGRAPH SECRET +- chars: ! "\U0001F23A" + codes: 1F23A + name: SQUARED CJK UNIFIED IDEOGRAPH-55B6 +- chars: ! "\U0001F235" + codes: 1F235 + name: SQUARED CJK UNIFIED IDEOGRAPH-6E80 +- chars: ▪️ + codes: 25AA + name: BLACK SMALL SQUARE +- chars: ▫️ + codes: 25AB + name: WHITE SMALL SQUARE +- chars: ◻️ + codes: 25FB + name: WHITE MEDIUM SQUARE +- chars: ◼️ + codes: 25FC + name: BLACK MEDIUM SQUARE +- chars: ◽️ + codes: 25FD + name: WHITE MEDIUM SMALL SQUARE +- chars: ◾️ + codes: 25FE + name: BLACK MEDIUM SMALL SQUARE +- chars: ⬛️ + codes: 2B1B + name: BLACK LARGE SQUARE +- chars: ⬜️ + codes: 2B1C + name: WHITE LARGE SQUARE +- chars: ! "\U0001F536" + codes: 1F536 + name: LARGE ORANGE DIAMOND +- chars: ! "\U0001F537" + codes: 1F537 + name: LARGE BLUE DIAMOND +- chars: ! "\U0001F538" + codes: 1F538 + name: SMALL ORANGE DIAMOND +- chars: ! "\U0001F539" + codes: 1F539 + name: SMALL BLUE DIAMOND +- chars: ! "\U0001F53A" + codes: 1F53A + name: UP-POINTING RED TRIANGLE +- chars: ! "\U0001F53B" + codes: 1F53B + name: DOWN-POINTING RED TRIANGLE +- chars: ! "\U0001F4A0" + codes: 1F4A0 + name: DIAMOND SHAPE WITH A DOT INSIDE +- chars: ! "\U0001F518" + codes: 1F518 + name: RADIO BUTTON +- chars: ! "\U0001F532" + codes: 1F532 + name: BLACK SQUARE BUTTON +- chars: ! "\U0001F533" + codes: 1F533 + name: WHITE SQUARE BUTTON +- chars: ⚪️ + codes: 26AA + name: MEDIUM WHITE CIRCLE +- chars: ⚫️ + codes: 26AB + name: MEDIUM BLACK CIRCLE +- chars: ! "\U0001F534" + codes: 1F534 + name: LARGE RED CIRCLE +- chars: ! "\U0001F535" + codes: 1F535 + name: LARGE BLUE CIRCLE +- chars: ! "\U0001F1E6\U0001F1E8" + codes: 1F1E6 1F1E8 + name: Flag for Ascension Island +- chars: ! "\U0001F1E6\U0001F1E9" + codes: 1F1E6 1F1E9 + name: Flag for Andorra +- chars: ! "\U0001F1E6\U0001F1EA" + codes: 1F1E6 1F1EA + name: Flag for United Arab Emirates +- chars: ! "\U0001F1E6\U0001F1EB" + codes: 1F1E6 1F1EB + name: Flag for Afghanistan +- chars: ! "\U0001F1E6\U0001F1EC" + codes: 1F1E6 1F1EC + name: Flag for Antigua & Barbuda +- chars: ! "\U0001F1E6\U0001F1EE" + codes: 1F1E6 1F1EE + name: Flag for Anguilla +- chars: ! "\U0001F1E6\U0001F1F1" + codes: 1F1E6 1F1F1 + name: Flag for Albania +- chars: ! "\U0001F1E6\U0001F1F2" + codes: 1F1E6 1F1F2 + name: Flag for Armenia +- chars: ! "\U0001F1E6\U0001F1F4" + codes: 1F1E6 1F1F4 + name: Flag for Angola +- chars: ! "\U0001F1E6\U0001F1F6" + codes: 1F1E6 1F1F6 + name: Flag for Antarctica +- chars: ! "\U0001F1E6\U0001F1F7" + codes: 1F1E6 1F1F7 + name: Flag for Argentina +- chars: ! "\U0001F1E6\U0001F1F8" + codes: 1F1E6 1F1F8 + name: Flag for American Samoa +- chars: ! "\U0001F1E6\U0001F1F9" + codes: 1F1E6 1F1F9 + name: Flag for Austria +- chars: ! "\U0001F1E6\U0001F1FA" + codes: 1F1E6 1F1FA + name: Flag for Australia +- chars: ! "\U0001F1E6\U0001F1FC" + codes: 1F1E6 1F1FC + name: Flag for Aruba +- chars: ! "\U0001F1E6\U0001F1FD" + codes: 1F1E6 1F1FD + name: Flag for Åland Islands +- chars: ! "\U0001F1E6\U0001F1FF" + codes: 1F1E6 1F1FF + name: Flag for Azerbaijan +- chars: ! "\U0001F1E7\U0001F1E6" + codes: 1F1E7 1F1E6 + name: Flag for Bosnia & Herzegovina +- chars: ! "\U0001F1E7\U0001F1E7" + codes: 1F1E7 1F1E7 + name: Flag for Barbados +- chars: ! "\U0001F1E7\U0001F1E9" + codes: 1F1E7 1F1E9 + name: Flag for Bangladesh +- chars: ! "\U0001F1E7\U0001F1EA" + codes: 1F1E7 1F1EA + name: Flag for Belgium +- chars: ! "\U0001F1E7\U0001F1EB" + codes: 1F1E7 1F1EB + name: Flag for Burkina Faso +- chars: ! "\U0001F1E7\U0001F1EC" + codes: 1F1E7 1F1EC + name: Flag for Bulgaria +- chars: ! "\U0001F1E7\U0001F1ED" + codes: 1F1E7 1F1ED + name: Flag for Bahrain +- chars: ! "\U0001F1E7\U0001F1EE" + codes: 1F1E7 1F1EE + name: Flag for Burundi +- chars: ! "\U0001F1E7\U0001F1EF" + codes: 1F1E7 1F1EF + name: Flag for Benin +- chars: ! "\U0001F1E7\U0001F1F1" + codes: 1F1E7 1F1F1 + name: Flag for St. Barthélemy +- chars: ! "\U0001F1E7\U0001F1F2" + codes: 1F1E7 1F1F2 + name: Flag for Bermuda +- chars: ! "\U0001F1E7\U0001F1F3" + codes: 1F1E7 1F1F3 + name: Flag for Brunei +- chars: ! "\U0001F1E7\U0001F1F4" + codes: 1F1E7 1F1F4 + name: Flag for Bolivia +- chars: ! "\U0001F1E7\U0001F1F6" + codes: 1F1E7 1F1F6 + name: Flag for Caribbean Netherlands +- chars: ! "\U0001F1E7\U0001F1F7" + codes: 1F1E7 1F1F7 + name: Flag for Brazil +- chars: ! "\U0001F1E7\U0001F1F8" + codes: 1F1E7 1F1F8 + name: Flag for Bahamas +- chars: ! "\U0001F1E7\U0001F1F9" + codes: 1F1E7 1F1F9 + name: Flag for Bhutan +- chars: ! "\U0001F1E7\U0001F1FB" + codes: 1F1E7 1F1FB + name: Flag for Bouvet Island +- chars: ! "\U0001F1E7\U0001F1FC" + codes: 1F1E7 1F1FC + name: Flag for Botswana +- chars: ! "\U0001F1E7\U0001F1FE" + codes: 1F1E7 1F1FE + name: Flag for Belarus +- chars: ! "\U0001F1E7\U0001F1FF" + codes: 1F1E7 1F1FF + name: Flag for Belize +- chars: ! "\U0001F1E8\U0001F1E6" + codes: 1F1E8 1F1E6 + name: Flag for Canada +- chars: ! "\U0001F1E8\U0001F1E8" + codes: 1F1E8 1F1E8 + name: Flag for Cocos Islands +- chars: ! "\U0001F1E8\U0001F1E9" + codes: 1F1E8 1F1E9 + name: Flag for Congo - Kinshasa +- chars: ! "\U0001F1E8\U0001F1EB" + codes: 1F1E8 1F1EB + name: Flag for Central African Republic +- chars: ! "\U0001F1E8\U0001F1EC" + codes: 1F1E8 1F1EC + name: Flag for Congo - Brazzaville +- chars: ! "\U0001F1E8\U0001F1ED" + codes: 1F1E8 1F1ED + name: Flag for Switzerland +- chars: ! "\U0001F1E8\U0001F1EE" + codes: 1F1E8 1F1EE + name: Flag for Côte d’Ivoire +- chars: ! "\U0001F1E8\U0001F1F0" + codes: 1F1E8 1F1F0 + name: Flag for Cook Islands +- chars: ! "\U0001F1E8\U0001F1F1" + codes: 1F1E8 1F1F1 + name: Flag for Chile +- chars: ! "\U0001F1E8\U0001F1F2" + codes: 1F1E8 1F1F2 + name: Flag for Cameroon +- chars: ! "\U0001F1E8\U0001F1F3" + codes: 1F1E8 1F1F3 + name: Flag for China +- chars: ! "\U0001F1E8\U0001F1F4" + codes: 1F1E8 1F1F4 + name: Flag for Colombia +- chars: ! "\U0001F1E8\U0001F1F5" + codes: 1F1E8 1F1F5 + name: Flag for Clipperton Island +- chars: ! "\U0001F1E8\U0001F1F7" + codes: 1F1E8 1F1F7 + name: Flag for Costa Rica +- chars: ! "\U0001F1E8\U0001F1FA" + codes: 1F1E8 1F1FA + name: Flag for Cuba +- chars: ! "\U0001F1E8\U0001F1FB" + codes: 1F1E8 1F1FB + name: Flag for Cape Verde +- chars: ! "\U0001F1E8\U0001F1FC" + codes: 1F1E8 1F1FC + name: Flag for Curaçao +- chars: ! "\U0001F1E8\U0001F1FD" + codes: 1F1E8 1F1FD + name: Flag for Christmas Island +- chars: ! "\U0001F1E8\U0001F1FE" + codes: 1F1E8 1F1FE + name: Flag for Cyprus +- chars: ! "\U0001F1E8\U0001F1FF" + codes: 1F1E8 1F1FF + name: Flag for Czech Republic +- chars: ! "\U0001F1E9\U0001F1EA" + codes: 1F1E9 1F1EA + name: Flag for Germany +- chars: ! "\U0001F1E9\U0001F1EC" + codes: 1F1E9 1F1EC + name: Flag for Diego Garcia +- chars: ! "\U0001F1E9\U0001F1EF" + codes: 1F1E9 1F1EF + name: Flag for Djibouti +- chars: ! "\U0001F1E9\U0001F1F0" + codes: 1F1E9 1F1F0 + name: Flag for Denmark +- chars: ! "\U0001F1E9\U0001F1F2" + codes: 1F1E9 1F1F2 + name: Flag for Dominica +- chars: ! "\U0001F1E9\U0001F1F4" + codes: 1F1E9 1F1F4 + name: Flag for Dominican Republic +- chars: ! "\U0001F1E9\U0001F1FF" + codes: 1F1E9 1F1FF + name: Flag for Algeria +- chars: ! "\U0001F1EA\U0001F1E6" + codes: 1F1EA 1F1E6 + name: Flag for Ceuta & Melilla +- chars: ! "\U0001F1EA\U0001F1E8" + codes: 1F1EA 1F1E8 + name: Flag for Ecuador +- chars: ! "\U0001F1EA\U0001F1EA" + codes: 1F1EA 1F1EA + name: Flag for Estonia +- chars: ! "\U0001F1EA\U0001F1EC" + codes: 1F1EA 1F1EC + name: Flag for Egypt +- chars: ! "\U0001F1EA\U0001F1ED" + codes: 1F1EA 1F1ED + name: Flag for Western Sahara +- chars: ! "\U0001F1EA\U0001F1F7" + codes: 1F1EA 1F1F7 + name: Flag for Eritrea +- chars: ! "\U0001F1EA\U0001F1F8" + codes: 1F1EA 1F1F8 + name: Flag for Spain +- chars: ! "\U0001F1EA\U0001F1F9" + codes: 1F1EA 1F1F9 + name: Flag for Ethiopia +- chars: ! "\U0001F1EA\U0001F1FA" + codes: 1F1EA 1F1FA + name: Flag for European Union +- chars: ! "\U0001F1EB\U0001F1EE" + codes: 1F1EB 1F1EE + name: Flag for Finland +- chars: ! "\U0001F1EB\U0001F1EF" + codes: 1F1EB 1F1EF + name: Flag for Fiji +- chars: ! "\U0001F1EB\U0001F1F0" + codes: 1F1EB 1F1F0 + name: Flag for Falkland Islands +- chars: ! "\U0001F1EB\U0001F1F2" + codes: 1F1EB 1F1F2 + name: Flag for Micronesia +- chars: ! "\U0001F1EB\U0001F1F4" + codes: 1F1EB 1F1F4 + name: Flag for Faroe Islands +- chars: ! "\U0001F1EB\U0001F1F7" + codes: 1F1EB 1F1F7 + name: Flag for France +- chars: ! "\U0001F1EC\U0001F1E6" + codes: 1F1EC 1F1E6 + name: Flag for Gabon +- chars: ! "\U0001F1EC\U0001F1E7" + codes: 1F1EC 1F1E7 + name: Flag for United Kingdom +- chars: ! "\U0001F1EC\U0001F1E9" + codes: 1F1EC 1F1E9 + name: Flag for Grenada +- chars: ! "\U0001F1EC\U0001F1EA" + codes: 1F1EC 1F1EA + name: Flag for Georgia +- chars: ! "\U0001F1EC\U0001F1EB" + codes: 1F1EC 1F1EB + name: Flag for French Guiana +- chars: ! "\U0001F1EC\U0001F1EC" + codes: 1F1EC 1F1EC + name: Flag for Guernsey +- chars: ! "\U0001F1EC\U0001F1ED" + codes: 1F1EC 1F1ED + name: Flag for Ghana +- chars: ! "\U0001F1EC\U0001F1EE" + codes: 1F1EC 1F1EE + name: Flag for Gibraltar +- chars: ! "\U0001F1EC\U0001F1F1" + codes: 1F1EC 1F1F1 + name: Flag for Greenland +- chars: ! "\U0001F1EC\U0001F1F2" + codes: 1F1EC 1F1F2 + name: Flag for Gambia +- chars: ! "\U0001F1EC\U0001F1F3" + codes: 1F1EC 1F1F3 + name: Flag for Guinea +- chars: ! "\U0001F1EC\U0001F1F5" + codes: 1F1EC 1F1F5 + name: Flag for Guadeloupe +- chars: ! "\U0001F1EC\U0001F1F6" + codes: 1F1EC 1F1F6 + name: Flag for Equatorial Guinea +- chars: ! "\U0001F1EC\U0001F1F7" + codes: 1F1EC 1F1F7 + name: Flag for Greece +- chars: ! "\U0001F1EC\U0001F1F8" + codes: 1F1EC 1F1F8 + name: Flag for South Georgia & South Sandwich Islands +- chars: ! "\U0001F1EC\U0001F1F9" + codes: 1F1EC 1F1F9 + name: Flag for Guatemala +- chars: ! "\U0001F1EC\U0001F1FA" + codes: 1F1EC 1F1FA + name: Flag for Guam +- chars: ! "\U0001F1EC\U0001F1FC" + codes: 1F1EC 1F1FC + name: Flag for Guinea-Bissau +- chars: ! "\U0001F1EC\U0001F1FE" + codes: 1F1EC 1F1FE + name: Flag for Guyana +- chars: ! "\U0001F1ED\U0001F1F0" + codes: 1F1ED 1F1F0 + name: Flag for Hong Kong +- chars: ! "\U0001F1ED\U0001F1F2" + codes: 1F1ED 1F1F2 + name: Flag for Heard & McDonald Islands +- chars: ! "\U0001F1ED\U0001F1F3" + codes: 1F1ED 1F1F3 + name: Flag for Honduras +- chars: ! "\U0001F1ED\U0001F1F7" + codes: 1F1ED 1F1F7 + name: Flag for Croatia +- chars: ! "\U0001F1ED\U0001F1F9" + codes: 1F1ED 1F1F9 + name: Flag for Haiti +- chars: ! "\U0001F1ED\U0001F1FA" + codes: 1F1ED 1F1FA + name: Flag for Hungary +- chars: ! "\U0001F1EE\U0001F1E8" + codes: 1F1EE 1F1E8 + name: Flag for Canary Islands +- chars: ! "\U0001F1EE\U0001F1E9" + codes: 1F1EE 1F1E9 + name: Flag for Indonesia +- chars: ! "\U0001F1EE\U0001F1EA" + codes: 1F1EE 1F1EA + name: Flag for Ireland +- chars: ! "\U0001F1EE\U0001F1F1" + codes: 1F1EE 1F1F1 + name: Flag for Israel +- chars: ! "\U0001F1EE\U0001F1F2" + codes: 1F1EE 1F1F2 + name: Flag for Isle of Man +- chars: ! "\U0001F1EE\U0001F1F3" + codes: 1F1EE 1F1F3 + name: Flag for India +- chars: ! "\U0001F1EE\U0001F1F4" + codes: 1F1EE 1F1F4 + name: Flag for British Indian Ocean Territory +- chars: ! "\U0001F1EE\U0001F1F6" + codes: 1F1EE 1F1F6 + name: Flag for Iraq +- chars: ! "\U0001F1EE\U0001F1F7" + codes: 1F1EE 1F1F7 + name: Flag for Iran +- chars: ! "\U0001F1EE\U0001F1F8" + codes: 1F1EE 1F1F8 + name: Flag for Iceland +- chars: ! "\U0001F1EE\U0001F1F9" + codes: 1F1EE 1F1F9 + name: Flag for Italy +- chars: ! "\U0001F1EF\U0001F1EA" + codes: 1F1EF 1F1EA + name: Flag for Jersey +- chars: ! "\U0001F1EF\U0001F1F2" + codes: 1F1EF 1F1F2 + name: Flag for Jamaica +- chars: ! "\U0001F1EF\U0001F1F4" + codes: 1F1EF 1F1F4 + name: Flag for Jordan +- chars: ! "\U0001F1EF\U0001F1F5" + codes: 1F1EF 1F1F5 + name: Flag for Japan +- chars: ! "\U0001F1F0\U0001F1EA" + codes: 1F1F0 1F1EA + name: Flag for Kenya +- chars: ! "\U0001F1F0\U0001F1EC" + codes: 1F1F0 1F1EC + name: Flag for Kyrgyzstan +- chars: ! "\U0001F1F0\U0001F1ED" + codes: 1F1F0 1F1ED + name: Flag for Cambodia +- chars: ! "\U0001F1F0\U0001F1EE" + codes: 1F1F0 1F1EE + name: Flag for Kiribati +- chars: ! "\U0001F1F0\U0001F1F2" + codes: 1F1F0 1F1F2 + name: Flag for Comoros +- chars: ! "\U0001F1F0\U0001F1F3" + codes: 1F1F0 1F1F3 + name: Flag for St. Kitts & Nevis +- chars: ! "\U0001F1F0\U0001F1F5" + codes: 1F1F0 1F1F5 + name: Flag for North Korea +- chars: ! "\U0001F1F0\U0001F1F7" + codes: 1F1F0 1F1F7 + name: Flag for South Korea +- chars: ! "\U0001F1F0\U0001F1FC" + codes: 1F1F0 1F1FC + name: Flag for Kuwait +- chars: ! "\U0001F1F0\U0001F1FE" + codes: 1F1F0 1F1FE + name: Flag for Cayman Islands +- chars: ! "\U0001F1F0\U0001F1FF" + codes: 1F1F0 1F1FF + name: Flag for Kazakhstan +- chars: ! "\U0001F1F1\U0001F1E6" + codes: 1F1F1 1F1E6 + name: Flag for Laos +- chars: ! "\U0001F1F1\U0001F1E7" + codes: 1F1F1 1F1E7 + name: Flag for Lebanon +- chars: ! "\U0001F1F1\U0001F1E8" + codes: 1F1F1 1F1E8 + name: Flag for St. Lucia +- chars: ! "\U0001F1F1\U0001F1EE" + codes: 1F1F1 1F1EE + name: Flag for Liechtenstein +- chars: ! "\U0001F1F1\U0001F1F0" + codes: 1F1F1 1F1F0 + name: Flag for Sri Lanka +- chars: ! "\U0001F1F1\U0001F1F7" + codes: 1F1F1 1F1F7 + name: Flag for Liberia +- chars: ! "\U0001F1F1\U0001F1F8" + codes: 1F1F1 1F1F8 + name: Flag for Lesotho +- chars: ! "\U0001F1F1\U0001F1F9" + codes: 1F1F1 1F1F9 + name: Flag for Lithuania +- chars: ! "\U0001F1F1\U0001F1FA" + codes: 1F1F1 1F1FA + name: Flag for Luxembourg +- chars: ! "\U0001F1F1\U0001F1FB" + codes: 1F1F1 1F1FB + name: Flag for Latvia +- chars: ! "\U0001F1F1\U0001F1FE" + codes: 1F1F1 1F1FE + name: Flag for Libya +- chars: ! "\U0001F1F2\U0001F1E6" + codes: 1F1F2 1F1E6 + name: Flag for Morocco +- chars: ! "\U0001F1F2\U0001F1E8" + codes: 1F1F2 1F1E8 + name: Flag for Monaco +- chars: ! "\U0001F1F2\U0001F1E9" + codes: 1F1F2 1F1E9 + name: Flag for Moldova +- chars: ! "\U0001F1F2\U0001F1EA" + codes: 1F1F2 1F1EA + name: Flag for Montenegro +- chars: ! "\U0001F1F2\U0001F1EB" + codes: 1F1F2 1F1EB + name: Flag for St. Martin +- chars: ! "\U0001F1F2\U0001F1EC" + codes: 1F1F2 1F1EC + name: Flag for Madagascar +- chars: ! "\U0001F1F2\U0001F1ED" + codes: 1F1F2 1F1ED + name: Flag for Marshall Islands +- chars: ! "\U0001F1F2\U0001F1F0" + codes: 1F1F2 1F1F0 + name: Flag for Macedonia +- chars: ! "\U0001F1F2\U0001F1F1" + codes: 1F1F2 1F1F1 + name: Flag for Mali +- chars: ! "\U0001F1F2\U0001F1F2" + codes: 1F1F2 1F1F2 + name: Flag for Myanmar +- chars: ! "\U0001F1F2\U0001F1F3" + codes: 1F1F2 1F1F3 + name: Flag for Mongolia +- chars: ! "\U0001F1F2\U0001F1F4" + codes: 1F1F2 1F1F4 + name: Flag for Macau +- chars: ! "\U0001F1F2\U0001F1F5" + codes: 1F1F2 1F1F5 + name: Flag for Northern Mariana Islands +- chars: ! "\U0001F1F2\U0001F1F6" + codes: 1F1F2 1F1F6 + name: Flag for Martinique +- chars: ! "\U0001F1F2\U0001F1F7" + codes: 1F1F2 1F1F7 + name: Flag for Mauritania +- chars: ! "\U0001F1F2\U0001F1F8" + codes: 1F1F2 1F1F8 + name: Flag for Montserrat +- chars: ! "\U0001F1F2\U0001F1F9" + codes: 1F1F2 1F1F9 + name: Flag for Malta +- chars: ! "\U0001F1F2\U0001F1FA" + codes: 1F1F2 1F1FA + name: Flag for Mauritius +- chars: ! "\U0001F1F2\U0001F1FB" + codes: 1F1F2 1F1FB + name: Flag for Maldives +- chars: ! "\U0001F1F2\U0001F1FC" + codes: 1F1F2 1F1FC + name: Flag for Malawi +- chars: ! "\U0001F1F2\U0001F1FD" + codes: 1F1F2 1F1FD + name: Flag for Mexico +- chars: ! "\U0001F1F2\U0001F1FE" + codes: 1F1F2 1F1FE + name: Flag for Malaysia +- chars: ! "\U0001F1F2\U0001F1FF" + codes: 1F1F2 1F1FF + name: Flag for Mozambique +- chars: ! "\U0001F1F3\U0001F1E6" + codes: 1F1F3 1F1E6 + name: Flag for Namibia +- chars: ! "\U0001F1F3\U0001F1E8" + codes: 1F1F3 1F1E8 + name: Flag for New Caledonia +- chars: ! "\U0001F1F3\U0001F1EA" + codes: 1F1F3 1F1EA + name: Flag for Niger +- chars: ! "\U0001F1F3\U0001F1EB" + codes: 1F1F3 1F1EB + name: Flag for Norfolk Island +- chars: ! "\U0001F1F3\U0001F1EC" + codes: 1F1F3 1F1EC + name: Flag for Nigeria +- chars: ! "\U0001F1F3\U0001F1EE" + codes: 1F1F3 1F1EE + name: Flag for Nicaragua +- chars: ! "\U0001F1F3\U0001F1F1" + codes: 1F1F3 1F1F1 + name: Flag for Netherlands +- chars: ! "\U0001F1F3\U0001F1F4" + codes: 1F1F3 1F1F4 + name: Flag for Norway +- chars: ! "\U0001F1F3\U0001F1F5" + codes: 1F1F3 1F1F5 + name: Flag for Nepal +- chars: ! "\U0001F1F3\U0001F1F7" + codes: 1F1F3 1F1F7 + name: Flag for Nauru +- chars: ! "\U0001F1F3\U0001F1FA" + codes: 1F1F3 1F1FA + name: Flag for Niue +- chars: ! "\U0001F1F3\U0001F1FF" + codes: 1F1F3 1F1FF + name: Flag for New Zealand +- chars: ! "\U0001F1F4\U0001F1F2" + codes: 1F1F4 1F1F2 + name: Flag for Oman +- chars: ! "\U0001F1F5\U0001F1E6" + codes: 1F1F5 1F1E6 + name: Flag for Panama +- chars: ! "\U0001F1F5\U0001F1EA" + codes: 1F1F5 1F1EA + name: Flag for Peru +- chars: ! "\U0001F1F5\U0001F1EB" + codes: 1F1F5 1F1EB + name: Flag for French Polynesia +- chars: ! "\U0001F1F5\U0001F1EC" + codes: 1F1F5 1F1EC + name: Flag for Papua New Guinea +- chars: ! "\U0001F1F5\U0001F1ED" + codes: 1F1F5 1F1ED + name: Flag for Philippines +- chars: ! "\U0001F1F5\U0001F1F0" + codes: 1F1F5 1F1F0 + name: Flag for Pakistan +- chars: ! "\U0001F1F5\U0001F1F1" + codes: 1F1F5 1F1F1 + name: Flag for Poland +- chars: ! "\U0001F1F5\U0001F1F2" + codes: 1F1F5 1F1F2 + name: Flag for St. Pierre & Miquelon +- chars: ! "\U0001F1F5\U0001F1F3" + codes: 1F1F5 1F1F3 + name: Flag for Pitcairn Islands +- chars: ! "\U0001F1F5\U0001F1F7" + codes: 1F1F5 1F1F7 + name: Flag for Puerto Rico +- chars: ! "\U0001F1F5\U0001F1F8" + codes: 1F1F5 1F1F8 + name: Flag for Palestinian Territories +- chars: ! "\U0001F1F5\U0001F1F9" + codes: 1F1F5 1F1F9 + name: Flag for Portugal +- chars: ! "\U0001F1F5\U0001F1FC" + codes: 1F1F5 1F1FC + name: Flag for Palau +- chars: ! "\U0001F1F5\U0001F1FE" + codes: 1F1F5 1F1FE + name: Flag for Paraguay +- chars: ! "\U0001F1F6\U0001F1E6" + codes: 1F1F6 1F1E6 + name: Flag for Qatar +- chars: ! "\U0001F1F7\U0001F1EA" + codes: 1F1F7 1F1EA + name: Flag for Réunion +- chars: ! "\U0001F1F7\U0001F1F4" + codes: 1F1F7 1F1F4 + name: Flag for Romania +- chars: ! "\U0001F1F7\U0001F1F8" + codes: 1F1F7 1F1F8 + name: Flag for Serbia +- chars: ! "\U0001F1F7\U0001F1FA" + codes: 1F1F7 1F1FA + name: Flag for Russia +- chars: ! "\U0001F1F7\U0001F1FC" + codes: 1F1F7 1F1FC + name: Flag for Rwanda +- chars: ! "\U0001F1F8\U0001F1E6" + codes: 1F1F8 1F1E6 + name: Flag for Saudi Arabia +- chars: ! "\U0001F1F8\U0001F1E7" + codes: 1F1F8 1F1E7 + name: Flag for Solomon Islands +- chars: ! "\U0001F1F8\U0001F1E8" + codes: 1F1F8 1F1E8 + name: Flag for Seychelles +- chars: ! "\U0001F1F8\U0001F1E9" + codes: 1F1F8 1F1E9 + name: Flag for Sudan +- chars: ! "\U0001F1F8\U0001F1EA" + codes: 1F1F8 1F1EA + name: Flag for Sweden +- chars: ! "\U0001F1F8\U0001F1EC" + codes: 1F1F8 1F1EC + name: Flag for Singapore +- chars: ! "\U0001F1F8\U0001F1ED" + codes: 1F1F8 1F1ED + name: Flag for St. Helena +- chars: ! "\U0001F1F8\U0001F1EE" + codes: 1F1F8 1F1EE + name: Flag for Slovenia +- chars: ! "\U0001F1F8\U0001F1EF" + codes: 1F1F8 1F1EF + name: Flag for Svalbard & Jan Mayen +- chars: ! "\U0001F1F8\U0001F1F0" + codes: 1F1F8 1F1F0 + name: Flag for Slovakia +- chars: ! "\U0001F1F8\U0001F1F1" + codes: 1F1F8 1F1F1 + name: Flag for Sierra Leone +- chars: ! "\U0001F1F8\U0001F1F2" + codes: 1F1F8 1F1F2 + name: Flag for San Marino +- chars: ! "\U0001F1F8\U0001F1F3" + codes: 1F1F8 1F1F3 + name: Flag for Senegal +- chars: ! "\U0001F1F8\U0001F1F4" + codes: 1F1F8 1F1F4 + name: Flag for Somalia +- chars: ! "\U0001F1F8\U0001F1F7" + codes: 1F1F8 1F1F7 + name: Flag for Suriname +- chars: ! "\U0001F1F8\U0001F1F8" + codes: 1F1F8 1F1F8 + name: Flag for South Sudan +- chars: ! "\U0001F1F8\U0001F1F9" + codes: 1F1F8 1F1F9 + name: Flag for São Tomé & Príncipe +- chars: ! "\U0001F1F8\U0001F1FB" + codes: 1F1F8 1F1FB + name: Flag for El Salvador +- chars: ! "\U0001F1F8\U0001F1FD" + codes: 1F1F8 1F1FD + name: Flag for Sint Maarten +- chars: ! "\U0001F1F8\U0001F1FE" + codes: 1F1F8 1F1FE + name: Flag for Syria +- chars: ! "\U0001F1F8\U0001F1FF" + codes: 1F1F8 1F1FF + name: Flag for Swaziland +- chars: ! "\U0001F1F9\U0001F1E6" + codes: 1F1F9 1F1E6 + name: Flag for Tristan da Cunha +- chars: ! "\U0001F1F9\U0001F1E8" + codes: 1F1F9 1F1E8 + name: Flag for Turks & Caicos Islands +- chars: ! "\U0001F1F9\U0001F1E9" + codes: 1F1F9 1F1E9 + name: Flag for Chad +- chars: ! "\U0001F1F9\U0001F1EB" + codes: 1F1F9 1F1EB + name: Flag for French Southern Territories +- chars: ! "\U0001F1F9\U0001F1EC" + codes: 1F1F9 1F1EC + name: Flag for Togo +- chars: ! "\U0001F1F9\U0001F1ED" + codes: 1F1F9 1F1ED + name: Flag for Thailand +- chars: ! "\U0001F1F9\U0001F1EF" + codes: 1F1F9 1F1EF + name: Flag for Tajikistan +- chars: ! "\U0001F1F9\U0001F1F0" + codes: 1F1F9 1F1F0 + name: Flag for Tokelau +- chars: ! "\U0001F1F9\U0001F1F1" + codes: 1F1F9 1F1F1 + name: Flag for Timor-Leste +- chars: ! "\U0001F1F9\U0001F1F2" + codes: 1F1F9 1F1F2 + name: Flag for Turkmenistan +- chars: ! "\U0001F1F9\U0001F1F3" + codes: 1F1F9 1F1F3 + name: Flag for Tunisia +- chars: ! "\U0001F1F9\U0001F1F4" + codes: 1F1F9 1F1F4 + name: Flag for Tonga +- chars: ! "\U0001F1F9\U0001F1F7" + codes: 1F1F9 1F1F7 + name: Flag for Turkey +- chars: ! "\U0001F1F9\U0001F1F9" + codes: 1F1F9 1F1F9 + name: Flag for Trinidad & Tobago +- chars: ! "\U0001F1F9\U0001F1FB" + codes: 1F1F9 1F1FB + name: Flag for Tuvalu +- chars: ! "\U0001F1F9\U0001F1FC" + codes: 1F1F9 1F1FC + name: Flag for Taiwan +- chars: ! "\U0001F1F9\U0001F1FF" + codes: 1F1F9 1F1FF + name: Flag for Tanzania +- chars: ! "\U0001F1FA\U0001F1E6" + codes: 1F1FA 1F1E6 + name: Flag for Ukraine +- chars: ! "\U0001F1FA\U0001F1EC" + codes: 1F1FA 1F1EC + name: Flag for Uganda +- chars: ! "\U0001F1FA\U0001F1F2" + codes: 1F1FA 1F1F2 + name: Flag for U.S. Outlying Islands +- chars: ! "\U0001F1FA\U0001F1F8" + codes: 1F1FA 1F1F8 + name: Flag for United States +- chars: ! "\U0001F1FA\U0001F1FE" + codes: 1F1FA 1F1FE + name: Flag for Uruguay +- chars: ! "\U0001F1FA\U0001F1FF" + codes: 1F1FA 1F1FF + name: Flag for Uzbekistan +- chars: ! "\U0001F1FB\U0001F1E6" + codes: 1F1FB 1F1E6 + name: Flag for Vatican City +- chars: ! "\U0001F1FB\U0001F1E8" + codes: 1F1FB 1F1E8 + name: Flag for St. Vincent & Grenadines +- chars: ! "\U0001F1FB\U0001F1EA" + codes: 1F1FB 1F1EA + name: Flag for Venezuela +- chars: ! "\U0001F1FB\U0001F1EC" + codes: 1F1FB 1F1EC + name: Flag for British Virgin Islands +- chars: ! "\U0001F1FB\U0001F1EE" + codes: 1F1FB 1F1EE + name: Flag for U.S. Virgin Islands +- chars: ! "\U0001F1FB\U0001F1F3" + codes: 1F1FB 1F1F3 + name: Flag for Vietnam +- chars: ! "\U0001F1FB\U0001F1FA" + codes: 1F1FB 1F1FA + name: Flag for Vanuatu +- chars: ! "\U0001F1FC\U0001F1EB" + codes: 1F1FC 1F1EB + name: Flag for Wallis & Futuna +- chars: ! "\U0001F1FC\U0001F1F8" + codes: 1F1FC 1F1F8 + name: Flag for Samoa +- chars: ! "\U0001F1FD\U0001F1F0" + codes: 1F1FD 1F1F0 + name: Flag for Kosovo +- chars: ! "\U0001F1FE\U0001F1EA" + codes: 1F1FE 1F1EA + name: Flag for Yemen +- chars: ! "\U0001F1FE\U0001F1F9" + codes: 1F1FE 1F1F9 + name: Flag for Mayotte +- chars: ! "\U0001F1FF\U0001F1E6" + codes: 1F1FF 1F1E6 + name: Flag for South Africa +- chars: ! "\U0001F1FF\U0001F1F2" + codes: 1F1FF 1F1F2 + name: Flag for Zambia +- chars: ! "\U0001F1FF\U0001F1FC" + codes: 1F1FF 1F1FC + name: Flag for Zimbabwe diff --git a/test/emitter-examples/emptydoc.yaml b/test/emitter-examples/emptydoc.yaml new file mode 100644 index 0000000..a845151 --- /dev/null +++ b/test/emitter-examples/emptydoc.yaml @@ -0,0 +1,2 @@ +--- +--- diff --git a/test/emitter-examples/emptykey.yaml b/test/emitter-examples/emptykey.yaml new file mode 100644 index 0000000..7c784ca --- /dev/null +++ b/test/emitter-examples/emptykey.yaml @@ -0,0 +1 @@ +- [ YAML : separate ] diff --git a/test/emitter-examples/emptystream.yaml b/test/emitter-examples/emptystream.yaml new file mode 100644 index 0000000..e69de29 diff --git a/test/emitter-examples/flow.yaml b/test/emitter-examples/flow.yaml new file mode 100644 index 0000000..95f2303 --- /dev/null +++ b/test/emitter-examples/flow.yaml @@ -0,0 +1,5 @@ +a: + b: + - c + - d + - e diff --git a/test/emitter-examples/flow1.yaml b/test/emitter-examples/flow1.yaml new file mode 100644 index 0000000..0f591db --- /dev/null +++ b/test/emitter-examples/flow1.yaml @@ -0,0 +1,5 @@ +a: + b: [ c, + d, + e ] + diff --git a/test/emitter-examples/flow2.yaml b/test/emitter-examples/flow2.yaml new file mode 100644 index 0000000..61f52d2 --- /dev/null +++ b/test/emitter-examples/flow2.yaml @@ -0,0 +1,8 @@ +a: + b: [ c, + d, + e, [ f, g, h, i, j + k, l, m, n, o + ] + ] + diff --git a/test/emitter-examples/fold-tricky.yaml b/test/emitter-examples/fold-tricky.yaml new file mode 100644 index 0000000..3c8c714 --- /dev/null +++ b/test/emitter-examples/fold-tricky.yaml @@ -0,0 +1,72 @@ +--- > + 1 +--- > + 2 + 2 +--- > + 3 + + 3 +--- > + 4 + + 4 +--- > + 5 + + + 5 +--- > + 6 + + + 6 +--- > + 7 + + + 7 +--- > + 8 + + 8 +--- > + 9 + + 9 + 9 +--- > + a + + a + + a +--- > + b + b + b +--- > + c + + c + c +--- > + d + + d + + d +--- > + e + e + e +--- > + f f + f + f + +--- > + g g + g + g + diff --git a/test/emitter-examples/fold.yaml b/test/emitter-examples/fold.yaml new file mode 100644 index 0000000..9252180 --- /dev/null +++ b/test/emitter-examples/fold.yaml @@ -0,0 +1,25 @@ +fold: > + 1 2 + 3 + 4 +fold2: > + 1 2 + 3 # comment + 4 +fold3: > + 1 2 + 3 + 4 +fold4: > + 1 2 + + 3 + 4 +fold5: > + 1 + + 2 +fold6: > + 1 2 + 3 + 4 diff --git a/test/emitter-examples/fold2.yaml b/test/emitter-examples/fold2.yaml new file mode 100644 index 0000000..eb186d6 --- /dev/null +++ b/test/emitter-examples/fold2.yaml @@ -0,0 +1,5 @@ +fold: > + 1 2 + 3 4 + 5 + 6 diff --git a/test/emitter-examples/fold3.yaml b/test/emitter-examples/fold3.yaml new file mode 100644 index 0000000..0fc4ca9 --- /dev/null +++ b/test/emitter-examples/fold3.yaml @@ -0,0 +1,5 @@ +fold4: > + 1 2 + + 3 + 4 diff --git a/test/emitter-examples/fold4.yaml b/test/emitter-examples/fold4.yaml new file mode 100644 index 0000000..7140e98 --- /dev/null +++ b/test/emitter-examples/fold4.yaml @@ -0,0 +1,4 @@ +fold: > + 1 + + 2 diff --git a/test/emitter-examples/fold5.yaml b/test/emitter-examples/fold5.yaml new file mode 100644 index 0000000..9ef791b --- /dev/null +++ b/test/emitter-examples/fold5.yaml @@ -0,0 +1,4 @@ +fold3: > + 1 2 + 3 + 4 diff --git a/test/emitter-examples/folded-endbreaks.yaml b/test/emitter-examples/folded-endbreaks.yaml new file mode 100644 index 0000000..313401f --- /dev/null +++ b/test/emitter-examples/folded-endbreaks.yaml @@ -0,0 +1,66 @@ +--- > + 0 +--- >+ + 1 +--- >- + 2 +--- > + 3 + 3 +--- >+ + 4 + 4 +--- >- + 5 + 5 +--- > + 6 + + 6 +--- >+ + 7 + + 7 +--- >- + 8 + + 8 +--- > + 9 + + 9 +--- >+ + a + + a +--- >- + b + + b +--- > + c + c + c +--- >+ + d + d + d +--- >- + e + e + e +--- > + f + + + f +--- >+ + g + + + g +--- >- + h + + + h diff --git a/test/emitter-examples/folded.yaml b/test/emitter-examples/folded.yaml new file mode 100644 index 0000000..ae24857 --- /dev/null +++ b/test/emitter-examples/folded.yaml @@ -0,0 +1,33 @@ +clip: > + This is a single clipped line +keep: >+ + This is a single kept line +strip: >- + This is a single stripped line +clip2: > + These are two + clipped lines +keep2: >+ + These are two + kept lines +strip2: >- + These are two + striped lines +clip3: > + These are two + clipped lines with trailing line breaks + + + +keep3: >+ + These are two + kept lines with trailing line breaks + + + +strip3: >- + These are two + striped lines with trailing line breaks + + + diff --git a/test/emitter-examples/folded2.yaml b/test/emitter-examples/folded2.yaml new file mode 100644 index 0000000..eda4b49 --- /dev/null +++ b/test/emitter-examples/folded2.yaml @@ -0,0 +1,5 @@ +"folded +to a space, + +to a line feed, or \ + \ non-content" diff --git a/test/emitter-examples/folding.yaml b/test/emitter-examples/folding.yaml new file mode 100644 index 0000000..0896cc6 --- /dev/null +++ b/test/emitter-examples/folding.yaml @@ -0,0 +1,6 @@ +> + foo + + bar + + baz diff --git a/test/emitter-examples/global-tag.yaml b/test/emitter-examples/global-tag.yaml new file mode 100644 index 0000000..1180757 --- /dev/null +++ b/test/emitter-examples/global-tag.yaml @@ -0,0 +1,14 @@ +%TAG ! tag:clarkevans.com,2002: +--- !shape + # Use the ! handle for presenting + # tag:clarkevans.com,2002:circle +- !circle + center: &ORIGIN {x: 73, y: 129} + radius: 7 +- !line + start: *ORIGIN + finish: { x: 89, y: 102 } +- !label + start: *ORIGIN + color: 0xFFEEBB + text: Pretty vector drawing. diff --git a/test/emitter-examples/invoice.yaml b/test/emitter-examples/invoice.yaml new file mode 100644 index 0000000..434f080 --- /dev/null +++ b/test/emitter-examples/invoice.yaml @@ -0,0 +1,9 @@ +invoice: 34843 +date : !!str 2001-01-23 +bill-to: &id001 + given : Chris + family : Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 diff --git a/test/emitter-examples/json.yaml b/test/emitter-examples/json.yaml new file mode 100644 index 0000000..7822ddc --- /dev/null +++ b/test/emitter-examples/json.yaml @@ -0,0 +1 @@ +{"key": ["value", 3]} diff --git a/test/emitter-examples/keyflow.yaml b/test/emitter-examples/keyflow.yaml new file mode 100644 index 0000000..074e0fc --- /dev/null +++ b/test/emitter-examples/keyflow.yaml @@ -0,0 +1,5 @@ +{ + a: b, + c: { d: e }, + { f: g }: h, +} diff --git a/test/emitter-examples/keykey.yaml b/test/emitter-examples/keykey.yaml new file mode 100644 index 0000000..00a179c --- /dev/null +++ b/test/emitter-examples/keykey.yaml @@ -0,0 +1,3 @@ +top1: + key1 : value1 +"top2": value2 diff --git a/test/emitter-examples/keykey2.yaml b/test/emitter-examples/keykey2.yaml new file mode 100644 index 0000000..e5a5af9 --- /dev/null +++ b/test/emitter-examples/keykey2.yaml @@ -0,0 +1 @@ +{ }: value diff --git a/test/emitter-examples/line.yaml b/test/emitter-examples/line.yaml new file mode 100644 index 0000000..a1230a8 --- /dev/null +++ b/test/emitter-examples/line.yaml @@ -0,0 +1,8 @@ +text: | + a + + b + + c + + d diff --git a/test/emitter-examples/literal-endbreaks.yaml b/test/emitter-examples/literal-endbreaks.yaml new file mode 100644 index 0000000..a455245 --- /dev/null +++ b/test/emitter-examples/literal-endbreaks.yaml @@ -0,0 +1,93 @@ +--- | + 0 +--- |+ + 1 +--- |- + 2 +--- | + 3 + +--- |+ + 4 + +--- |- + 5 + +--- | + 6 + + +--- |+ + 7 + + +--- |- + 8 + + +--- | + 9 + +--- |+ + a + +--- |- + b + +--- | + c + + +--- |+ + d + + +--- |- + e + + +--- | + f + + f +--- |+ + g + + g +--- |- + h + + h +--- | + i + + i +--- |+ + j + + j +--- |- + k + + k +--- | + l + + + + + +--- |+ + m + + + + + +--- |- + n + + + + + diff --git a/test/emitter-examples/literal.yaml b/test/emitter-examples/literal.yaml new file mode 100644 index 0000000..018bcda --- /dev/null +++ b/test/emitter-examples/literal.yaml @@ -0,0 +1,33 @@ +clip: | + This is a single clipped line +keep: |+ + This is a single kept line +strip: |- + This is a single stripped line +clip2: | + These are two + clipped lines +keep2: |+ + These are two + kept lines +strip2: |- + These are two + striped lines +clip3: | + These are two + clipped lines with trailing line breaks + + + +keep3: |+ + These are two + kept lines with trailing line breaks + + + +strip3: |- + These are two + striped lines with trailing line breaks + + + diff --git a/test/emitter-examples/literal1.yaml b/test/emitter-examples/literal1.yaml new file mode 100644 index 0000000..f2d2e90 --- /dev/null +++ b/test/emitter-examples/literal1.yaml @@ -0,0 +1,3 @@ +a: | + literal +... diff --git a/test/emitter-examples/literal2.yaml b/test/emitter-examples/literal2.yaml new file mode 100644 index 0000000..9d537cb --- /dev/null +++ b/test/emitter-examples/literal2.yaml @@ -0,0 +1,9 @@ +| + + + literal + + + text + + # Comment diff --git a/test/emitter-examples/literal3.yaml b/test/emitter-examples/literal3.yaml new file mode 100644 index 0000000..5a3259b --- /dev/null +++ b/test/emitter-examples/literal3.yaml @@ -0,0 +1,5 @@ +--- | +foo +bar +... +foo diff --git a/test/emitter-examples/literal4.yaml b/test/emitter-examples/literal4.yaml new file mode 100644 index 0000000..370ecc3 --- /dev/null +++ b/test/emitter-examples/literal4.yaml @@ -0,0 +1,6 @@ +keep3: |+ + These are two + kept lines with trailing line breaks + + + diff --git a/test/emitter-examples/mapping.yaml b/test/emitter-examples/mapping.yaml new file mode 100644 index 0000000..53d2567 --- /dev/null +++ b/test/emitter-examples/mapping.yaml @@ -0,0 +1,2 @@ +key: value +other-key: other-value diff --git a/test/emitter-examples/mergekeyspec.yaml b/test/emitter-examples/mergekeyspec.yaml new file mode 100644 index 0000000..ee4a48f --- /dev/null +++ b/test/emitter-examples/mergekeyspec.yaml @@ -0,0 +1,27 @@ +--- +- &CENTER { x: 1, y: 2 } +- &LEFT { x: 0, y: 2 } +- &BIG { r: 10 } +- &SMALL { r: 1 } + +# All the following maps are equal: + +- # Explicit keys + x: 1 + y: 2 + r: 10 + label: center/big + +- # Merge one map + << : *CENTER + r: 10 + label: center/big + +- # Merge multiple maps + << : [ *CENTER, *BIG ] + label: center/big + +- # Override + << : [ *BIG, *LEFT, *SMALL ] + x: 1 + label: center/big diff --git a/test/emitter-examples/multi-document.yaml b/test/emitter-examples/multi-document.yaml new file mode 100644 index 0000000..2fea5d0 --- /dev/null +++ b/test/emitter-examples/multi-document.yaml @@ -0,0 +1,8 @@ +%YAML 1.1 +--- +First +... +%YAML 1.2 +--- +Second +... diff --git a/test/emitter-examples/multiline-quoted-key.yaml b/test/emitter-examples/multiline-quoted-key.yaml new file mode 100644 index 0000000..577642f --- /dev/null +++ b/test/emitter-examples/multiline-quoted-key.yaml @@ -0,0 +1,2 @@ +{ "foo + bar" : 20 } diff --git a/test/emitter-examples/multiline-simple-key.yaml b/test/emitter-examples/multiline-simple-key.yaml new file mode 100644 index 0000000..c608157 --- /dev/null +++ b/test/emitter-examples/multiline-simple-key.yaml @@ -0,0 +1,2 @@ +{ foo + bar : 20 } diff --git a/test/emitter-examples/nodeprop.yaml b/test/emitter-examples/nodeprop.yaml new file mode 100644 index 0000000..66d75f3 --- /dev/null +++ b/test/emitter-examples/nodeprop.yaml @@ -0,0 +1,3 @@ +!!str &a1 "foo": + !!str bar +&a2 baz : *a1 diff --git a/test/emitter-examples/nodeprop2.yaml b/test/emitter-examples/nodeprop2.yaml new file mode 100644 index 0000000..32dd542 --- /dev/null +++ b/test/emitter-examples/nodeprop2.yaml @@ -0,0 +1,3 @@ +"foo": + bar +&a2 baz : fuzz diff --git a/test/emitter-examples/numbers-flow.yaml b/test/emitter-examples/numbers-flow.yaml new file mode 100644 index 0000000..dc06e9e --- /dev/null +++ b/test/emitter-examples/numbers-flow.yaml @@ -0,0 +1,4 @@ +- 100 +- 12.5 +- -130 +- 1.3e9 diff --git a/test/emitter-examples/numbers.yaml b/test/emitter-examples/numbers.yaml new file mode 100644 index 0000000..45d2bf0 --- /dev/null +++ b/test/emitter-examples/numbers.yaml @@ -0,0 +1 @@ +[100, 12.5, -130, 1.3e+9] diff --git a/test/emitter-examples/plain-scalars-with-commas.yaml b/test/emitter-examples/plain-scalars-with-commas.yaml new file mode 100644 index 0000000..ed82142 --- /dev/null +++ b/test/emitter-examples/plain-scalars-with-commas.yaml @@ -0,0 +1,6 @@ +- Aa, + Bb, C, + D +- Eee, + F, Gg, + E diff --git a/test/emitter-examples/plainlines.yaml b/test/emitter-examples/plainlines.yaml new file mode 100644 index 0000000..0499250 --- /dev/null +++ b/test/emitter-examples/plainlines.yaml @@ -0,0 +1,4 @@ +1st non-empty + + 2nd non-empty + 3rd non-empty diff --git a/test/emitter-examples/plainscalar.yaml b/test/emitter-examples/plainscalar.yaml new file mode 100644 index 0000000..a7b828f --- /dev/null +++ b/test/emitter-examples/plainscalar.yaml @@ -0,0 +1,11 @@ +- plain +- plain with spaces +- plain with + breaks and stuff +- plain which should break at some point since it is somewhat longer than usual. nice weather eh... +- plain with breaks + + and + + gaps +- various two space stuff diff --git a/test/emitter-examples/quoted.yaml b/test/emitter-examples/quoted.yaml new file mode 100644 index 0000000..aed6e4b --- /dev/null +++ b/test/emitter-examples/quoted.yaml @@ -0,0 +1,8 @@ +--- +" + foo + + bar + + baz +" diff --git a/test/emitter-examples/quotedbackslash.yaml b/test/emitter-examples/quotedbackslash.yaml new file mode 100644 index 0000000..9ea0d05 --- /dev/null +++ b/test/emitter-examples/quotedbackslash.yaml @@ -0,0 +1,4 @@ +"\ + 123\ + 456\ + 789" diff --git a/test/emitter-examples/scalar-multiline.yaml b/test/emitter-examples/scalar-multiline.yaml new file mode 100644 index 0000000..c40c86c --- /dev/null +++ b/test/emitter-examples/scalar-multiline.yaml @@ -0,0 +1,36 @@ +comment: + Deliver here, but if not + possible deliver there. + + + Tatta! + + + +comment-2: + " Deliver here, but if not + possible deliver there. + + + Tatta! " + +comment-3: + ' Deliver here, but if not + possible deliver there. + + + Tatta! ' + +comment-4: + Deliver here, but if not + possible deliver there. + + + Tatta! + + +comment-5: + "" + +comment-6: " + " diff --git a/test/emitter-examples/scalar-singlequoted.yaml b/test/emitter-examples/scalar-singlequoted.yaml new file mode 100644 index 0000000..67ddb0c --- /dev/null +++ b/test/emitter-examples/scalar-singlequoted.yaml @@ -0,0 +1,14 @@ +simple: 'simple' +multiline: 'multi line + singled quoted scalar' +multiline2: 'multi line + singled quoted scalar + + with a newline ' +comment-3: + ' Deliver here, but if not + possible deliver there. + + + Tatta! ' + diff --git a/test/emitter-examples/scalar-space.yaml b/test/emitter-examples/scalar-space.yaml new file mode 100644 index 0000000..868f393 --- /dev/null +++ b/test/emitter-examples/scalar-space.yaml @@ -0,0 +1,17 @@ +z: " + " +aa: foo + bar +a: " + + + " +b: "" +c: + + b +d: " + " +e: " + + " diff --git a/test/emitter-examples/scalar-space1.yaml b/test/emitter-examples/scalar-space1.yaml new file mode 100644 index 0000000..5c4a085 --- /dev/null +++ b/test/emitter-examples/scalar-space1.yaml @@ -0,0 +1,6 @@ +0: " + " +a: " + + + " diff --git a/test/emitter-examples/scalars.yaml b/test/emitter-examples/scalars.yaml new file mode 100644 index 0000000..2f17998 --- /dev/null +++ b/test/emitter-examples/scalars.yaml @@ -0,0 +1,24 @@ +--- +plain-simple : plain-simple +plain: This is a plain scalar # alright +key with spaces-1 : scalar with spaces too +key with spaces-2: scalar with spaces too 2 +key with spaces-3 : scalar with spaces too 3# comment +key with spaces-4 : scalar with spaces too 4 # comment + +single-quoted: 'This is a single quoted scalar' +single-quoted-escaped: 'This is a single quoted scalar with ''escapes''' +double-quoted: "This is a double quoted scalar" +double-quoted-escaped: "This is a double quoted\tscalar\nwith escapes" +double-quoted-escaped2: "This is a double quoted\tscalar\nwith \x20 escapes" +literal: | + This is a literal scalar # comments included +literal2: | + This is a literal scalar 2 + More than one line + Indentation + Indeed +folded: > + This is a folded + scalar +... diff --git a/test/emitter-examples/scalars2.yaml b/test/emitter-examples/scalars2.yaml new file mode 100644 index 0000000..8f7fd0c --- /dev/null +++ b/test/emitter-examples/scalars2.yaml @@ -0,0 +1,24 @@ +key spaces: | + Normal indent + One more indent + Back to normal +folded : > + Folding is easy to do + Indenting too + Roses are blue +foo: bar +another-fold: >- + fold + 1 + 2 + 3 +yet-another-fold: >+ + another-fold + 11 + 12 + 13 +more-fold: > + more-fold + 21 + 22 + 23 diff --git a/test/emitter-examples/scanner-c-1.yaml b/test/emitter-examples/scanner-c-1.yaml new file mode 100644 index 0000000..0f1c2df --- /dev/null +++ b/test/emitter-examples/scanner-c-1.yaml @@ -0,0 +1,4 @@ +%YAML 1.1 +%TAG ! !foo +%TAG !yaml! tag:yaml.org,2002: +--- diff --git a/test/emitter-examples/scanner-c-10.yaml b/test/emitter-examples/scanner-c-10.yaml new file mode 100644 index 0000000..c4b3415 --- /dev/null +++ b/test/emitter-examples/scanner-c-10.yaml @@ -0,0 +1,18 @@ +{ + a simple key: a value, # Note that the KEY token is produced. + ? a complex key: another value, +} +# STREAM_START FYTT_STREAM_START +# FLOW_MAPPING_START FYTT_FLOW_MAPPING_START +# KEY FYTT_KEY +# SCALAR value='a simple key' style=PLAIN FYTT_SCALAR value='a simple key' style=PLAIN +# VALUE FYTT_VALUE +# SCALAR value='a value' style=PLAIN FYTT_SCALAR value='a value' style=PLAIN +# FLOW_ENTRY FYTT_FLOW_ENTRY +# KEY FYTT_KEY +# SCALAR value='a complex key' style=PLAIN FYTT_SCALAR value='a complex key' style=PLAIN +# VALUE FYTT_VALUE +# SCALAR value='another value' style=PLAIN FYTT_SCALAR value='another value' style=PLAIN +# FLOW_ENTRY FYTT_FLOW_ENTRY +# FLOW_MAPPING_END FYTT_FLOW_MAPPING_END +# STREAM_END FYTT_STREAM_END diff --git a/test/emitter-examples/scanner-c-11.yaml b/test/emitter-examples/scanner-c-11.yaml new file mode 100644 index 0000000..1e70b94 --- /dev/null +++ b/test/emitter-examples/scanner-c-11.yaml @@ -0,0 +1,34 @@ +- item 1 +- item 2 +- + - item 3.1 + - item 3.2 +- + key 1: value 1 + key 2: value 2 +# STREAM_START +# BLOCK_SEQUENCE_START +# BLOCK_ENTRY +# SCALAR value='item 1' style=PLAIN +# BLOCK_ENTRY +# SCALAR value='item 2' style=PLAIN +# BLOCK_ENTRY +# BLOCK_SEQUENCE_START +# BLOCK_ENTRY +# SCALAR value='item 3.1' style=PLAIN +# BLOCK_ENTRY +# SCALAR value='item 3.2' style=PLAIN +# BLOCK_END +# BLOCK_ENTRY +# BLOCK_MAPPING_START +# KEY +# SCALAR value='key 1' style=PLAIN +# VALUE +# SCALAR value='value 1' style=PLAIN +# KEY +# SCALAR value='key 2' style=PLAIN +# VALUE +# SCALAR value='value 2' style=PLAIN +# BLOCK_END +# BLOCK_END +# STREAM_END diff --git a/test/emitter-examples/scanner-c-12.yaml b/test/emitter-examples/scanner-c-12.yaml new file mode 100644 index 0000000..c00f23f --- /dev/null +++ b/test/emitter-examples/scanner-c-12.yaml @@ -0,0 +1,42 @@ +a simple key: a value # The KEY token is produced here. +? a complex key +: another value +a mapping: + key 1: value 1 + key 2: value 2 +a sequence: + - item 1 + - item 2 +# STREAM-START(utf-8) +# BLOCK-MAPPING-START +# KEY +# SCALAR("a simple key",plain) +# VALUE +# SCALAR("a value",plain) +# KEY +# SCALAR("a complex key",plain) +# VALUE +# SCALAR("another value",plain) +# KEY +# SCALAR("a mapping",plain) +# BLOCK-MAPPING-START +# KEY +# SCALAR("key 1",plain) +# VALUE +# SCALAR("value 1",plain) +# KEY +# SCALAR("key 2",plain) +# VALUE +# SCALAR("value 2",plain) +# BLOCK-END +# KEY +# SCALAR("a sequence",plain) +# VALUE +# BLOCK-SEQUENCE-START +# BLOCK-ENTRY +# SCALAR("item 1",plain) +# BLOCK-ENTRY +# SCALAR("item 2",plain) +# BLOCK-END +# BLOCK-END +# STREAM-END diff --git a/test/emitter-examples/scanner-c-13.yaml b/test/emitter-examples/scanner-c-13.yaml new file mode 100644 index 0000000..4107762 --- /dev/null +++ b/test/emitter-examples/scanner-c-13.yaml @@ -0,0 +1,13 @@ +key: +- item 1 # BLOCK-SEQUENCE-START is NOT produced here. +- item 2 +# STREAM-START(utf-8) +# BLOCK-MAPPING-START +# KEY +# SCALAR("key",plain) +# VALUE +# BLOCK-ENTRY +# SCALAR("item 1",plain) +# BLOCK-ENTRY +# SCALAR("item 2",plain) +# BLOCK-END diff --git a/test/emitter-examples/scanner-c-2.yaml b/test/emitter-examples/scanner-c-2.yaml new file mode 100644 index 0000000..91da2a7 --- /dev/null +++ b/test/emitter-examples/scanner-c-2.yaml @@ -0,0 +1,2 @@ +--- +... diff --git a/test/emitter-examples/scanner-c-3.yaml b/test/emitter-examples/scanner-c-3.yaml new file mode 100644 index 0000000..ee53cc5 --- /dev/null +++ b/test/emitter-examples/scanner-c-3.yaml @@ -0,0 +1 @@ +'a scalar' diff --git a/test/emitter-examples/scanner-c-4.yaml b/test/emitter-examples/scanner-c-4.yaml new file mode 100644 index 0000000..dbfa5a7 --- /dev/null +++ b/test/emitter-examples/scanner-c-4.yaml @@ -0,0 +1,3 @@ +--- +'a scalar' +... diff --git a/test/emitter-examples/scanner-c-5.yaml b/test/emitter-examples/scanner-c-5.yaml new file mode 100644 index 0000000..76df81e --- /dev/null +++ b/test/emitter-examples/scanner-c-5.yaml @@ -0,0 +1,5 @@ +'a scalar' +--- +'another scalar' +--- +'yet another scalar' diff --git a/test/emitter-examples/scanner-c-6.yaml b/test/emitter-examples/scanner-c-6.yaml new file mode 100644 index 0000000..1d4cdd8 --- /dev/null +++ b/test/emitter-examples/scanner-c-6.yaml @@ -0,0 +1 @@ +&A [ *A ] diff --git a/test/emitter-examples/scanner-c-7.yaml b/test/emitter-examples/scanner-c-7.yaml new file mode 100644 index 0000000..2d75792 --- /dev/null +++ b/test/emitter-examples/scanner-c-7.yaml @@ -0,0 +1 @@ +!!float "3.14" # A good approximation. diff --git a/test/emitter-examples/scanner-c-8-2.yaml b/test/emitter-examples/scanner-c-8-2.yaml new file mode 100644 index 0000000..64a248c --- /dev/null +++ b/test/emitter-examples/scanner-c-8-2.yaml @@ -0,0 +1,28 @@ +--- # Implicit empty plain scalars do not produce tokens. +--- a plain scalar +--- 'a single-quoted scalar' +--- "a double-quoted scalar" +--- |- + a literal scalar (|-) + + test + + +--- >- + a folded (>-) + scalar + + test + + + +--- | + a literal scalar (|) +--- > + a folded (>) + scalar +--- |+ + a literal scalar (|+) +--- >+ + a folded (>+) + scalar diff --git a/test/emitter-examples/scanner-c-8.yaml b/test/emitter-examples/scanner-c-8.yaml new file mode 100644 index 0000000..4176f15 --- /dev/null +++ b/test/emitter-examples/scanner-c-8.yaml @@ -0,0 +1,9 @@ +--- # Implicit empty plain scalars do not produce tokens. +--- a plain scalar +--- 'a single-quoted scalar' +--- "a double-quoted scalar" +--- |- + a literal scalar +--- >- + a folded + scalar diff --git a/test/emitter-examples/scanner-c-9.yaml b/test/emitter-examples/scanner-c-9.yaml new file mode 100644 index 0000000..cb6d8ef --- /dev/null +++ b/test/emitter-examples/scanner-c-9.yaml @@ -0,0 +1 @@ +[item 1, item 2, item 3] diff --git a/test/emitter-examples/seq.yaml b/test/emitter-examples/seq.yaml new file mode 100644 index 0000000..db250df --- /dev/null +++ b/test/emitter-examples/seq.yaml @@ -0,0 +1,8 @@ +# Outside flow collection: +- ::vector +- ": - ()" +- Up, up, and away! +- -123 +- http://example.com/foo#bar +# Inside flow collection: +- [ ::vector ] diff --git a/test/emitter-examples/seq1.yaml b/test/emitter-examples/seq1.yaml new file mode 100644 index 0000000..7cf7dca --- /dev/null +++ b/test/emitter-examples/seq1.yaml @@ -0,0 +1,6 @@ +- 1 +- 2 +- 3 +- key: + - value1 + - value 2 diff --git a/test/emitter-examples/seq2.yaml b/test/emitter-examples/seq2.yaml new file mode 100644 index 0000000..fdff3b5 --- /dev/null +++ b/test/emitter-examples/seq2.yaml @@ -0,0 +1,3 @@ +- [ YAML : separate ] +- [ : empty key entry ] +- [ {JSON: like}:adjacent ] diff --git a/test/emitter-examples/seq3.yaml b/test/emitter-examples/seq3.yaml new file mode 100644 index 0000000..c82dff5 --- /dev/null +++ b/test/emitter-examples/seq3.yaml @@ -0,0 +1,3 @@ +# - [ YAML : separate ] +- [ "" : empty key entry ] +# - [ {JSON: like}:adjacent ] diff --git a/test/emitter-examples/seq4.yaml b/test/emitter-examples/seq4.yaml new file mode 100644 index 0000000..1ff4f2b --- /dev/null +++ b/test/emitter-examples/seq4.yaml @@ -0,0 +1 @@ +[ : empty ] diff --git a/test/emitter-examples/seq5.yaml b/test/emitter-examples/seq5.yaml new file mode 100644 index 0000000..31bca7d --- /dev/null +++ b/test/emitter-examples/seq5.yaml @@ -0,0 +1 @@ +- : empty diff --git a/test/emitter-examples/seq6.yaml b/test/emitter-examples/seq6.yaml new file mode 100644 index 0000000..328d363 --- /dev/null +++ b/test/emitter-examples/seq6.yaml @@ -0,0 +1,3 @@ +key: +- value1 +- value 2 diff --git a/test/emitter-examples/sets.yaml b/test/emitter-examples/sets.yaml new file mode 100644 index 0000000..30e2ee8 --- /dev/null +++ b/test/emitter-examples/sets.yaml @@ -0,0 +1,8 @@ +# Sets are represented as a +# Mapping where each key is +# associated with a null value +--- +? a +? b +? c +? d diff --git a/test/emitter-examples/simple.yaml b/test/emitter-examples/simple.yaml new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/test/emitter-examples/simple.yaml @@ -0,0 +1 @@ +5 diff --git a/test/emitter-examples/simple1.yaml b/test/emitter-examples/simple1.yaml new file mode 100644 index 0000000..8025a0e --- /dev/null +++ b/test/emitter-examples/simple1.yaml @@ -0,0 +1,2 @@ +--- !!str &anchor +foo diff --git a/test/emitter-examples/simple2.yaml b/test/emitter-examples/simple2.yaml new file mode 100644 index 0000000..14e24d4 --- /dev/null +++ b/test/emitter-examples/simple2.yaml @@ -0,0 +1 @@ +"foobar" diff --git a/test/emitter-examples/simpleanchor.yaml b/test/emitter-examples/simpleanchor.yaml new file mode 100644 index 0000000..b617b80 --- /dev/null +++ b/test/emitter-examples/simpleanchor.yaml @@ -0,0 +1 @@ +&anchor key: value diff --git a/test/emitter-examples/simpleanchor1.yaml b/test/emitter-examples/simpleanchor1.yaml new file mode 100644 index 0000000..6cfc23e --- /dev/null +++ b/test/emitter-examples/simpleanchor1.yaml @@ -0,0 +1,3 @@ +a: &anchor b +c: *anchor +d: e diff --git a/test/emitter-examples/simpleanchor2.yaml b/test/emitter-examples/simpleanchor2.yaml new file mode 100644 index 0000000..9b14943 --- /dev/null +++ b/test/emitter-examples/simpleanchor2.yaml @@ -0,0 +1,3 @@ +a: &anchor [ b, bb, b3, b4 ] +c: *anchor +d: e diff --git a/test/emitter-examples/simpleanchor3.yaml b/test/emitter-examples/simpleanchor3.yaml new file mode 100644 index 0000000..950acfa --- /dev/null +++ b/test/emitter-examples/simpleanchor3.yaml @@ -0,0 +1,8 @@ +a: &anchor + - b + - &another bb + - b3 + - b4 +c: *anchor +d: e +f: *another diff --git a/test/emitter-examples/simpleanchor4.yaml b/test/emitter-examples/simpleanchor4.yaml new file mode 100644 index 0000000..2d2807f --- /dev/null +++ b/test/emitter-examples/simpleanchor4.yaml @@ -0,0 +1,9 @@ +a: &anchor + - b + - &another bb + - b3 + - b4 +c: + d: + e: *anchor +f: *another diff --git a/test/emitter-examples/simplefolded.yaml b/test/emitter-examples/simplefolded.yaml new file mode 100644 index 0000000..40a4a31 --- /dev/null +++ b/test/emitter-examples/simplefolded.yaml @@ -0,0 +1,3 @@ +> + this is a folded + scalar diff --git a/test/emitter-examples/simplekey.yaml b/test/emitter-examples/simplekey.yaml new file mode 100644 index 0000000..3c494f3 --- /dev/null +++ b/test/emitter-examples/simplekey.yaml @@ -0,0 +1 @@ +key : value diff --git a/test/emitter-examples/simplekey1.yaml b/test/emitter-examples/simplekey1.yaml new file mode 100644 index 0000000..bbac3ce --- /dev/null +++ b/test/emitter-examples/simplekey1.yaml @@ -0,0 +1 @@ +{ key : value } diff --git a/test/emitter-examples/simplekey2.yaml b/test/emitter-examples/simplekey2.yaml new file mode 100644 index 0000000..167e6bf --- /dev/null +++ b/test/emitter-examples/simplekey2.yaml @@ -0,0 +1,5 @@ +root: # level 0 + key: # level 1 + value # level 2 + key2: # level 1 + value2 # level 2 diff --git a/test/emitter-examples/simplekey3.yaml b/test/emitter-examples/simplekey3.yaml new file mode 100644 index 0000000..56df99a --- /dev/null +++ b/test/emitter-examples/simplekey3.yaml @@ -0,0 +1,5 @@ +{ + root: { + key: value + } +} diff --git a/test/emitter-examples/simplekey4.yaml b/test/emitter-examples/simplekey4.yaml new file mode 100644 index 0000000..5851568 --- /dev/null +++ b/test/emitter-examples/simplekey4.yaml @@ -0,0 +1,2 @@ +a: b +c: d diff --git a/test/emitter-examples/simplekey5.yaml b/test/emitter-examples/simplekey5.yaml new file mode 100644 index 0000000..2ac4cd4 --- /dev/null +++ b/test/emitter-examples/simplekey5.yaml @@ -0,0 +1,2 @@ +key: + value diff --git a/test/emitter-examples/simpleliteral.yaml b/test/emitter-examples/simpleliteral.yaml new file mode 100644 index 0000000..59da07a --- /dev/null +++ b/test/emitter-examples/simpleliteral.yaml @@ -0,0 +1,3 @@ +| + this is a literal + scalar diff --git a/test/emitter-examples/simpleseq.yaml b/test/emitter-examples/simpleseq.yaml new file mode 100644 index 0000000..a0cd6f3 --- /dev/null +++ b/test/emitter-examples/simpleseq.yaml @@ -0,0 +1,3 @@ +- 1 +- 2 +- 3 diff --git a/test/emitter-examples/simpleseq1.yaml b/test/emitter-examples/simpleseq1.yaml new file mode 100644 index 0000000..a9cd5ae --- /dev/null +++ b/test/emitter-examples/simpleseq1.yaml @@ -0,0 +1,5 @@ +- 1 +- 2 +- 3 +- 4: + foo diff --git a/test/emitter-examples/singlepairimp.yaml b/test/emitter-examples/singlepairimp.yaml new file mode 100644 index 0000000..0f0b325 --- /dev/null +++ b/test/emitter-examples/singlepairimp.yaml @@ -0,0 +1,3 @@ +- [ YAML : separate ] +- [ ffo : empty key entry ] +- [ {JSON: like}:adjacent ] diff --git a/test/emitter-examples/singlepairimp2.yaml b/test/emitter-examples/singlepairimp2.yaml new file mode 100644 index 0000000..38c3b31 --- /dev/null +++ b/test/emitter-examples/singlepairimp2.yaml @@ -0,0 +1,2 @@ +[ [ foo ]: bar ] +# [ foo ]: bar diff --git a/test/emitter-examples/sqscalar.yaml b/test/emitter-examples/sqscalar.yaml new file mode 100644 index 0000000..e74b73e --- /dev/null +++ b/test/emitter-examples/sqscalar.yaml @@ -0,0 +1,13 @@ +- 'quoted' +- 'quoted\nescaped new line' +- 'quoted + explicit new line' +- '' +- ' ' +- ' + ' +- ' pre-spaces' +- 'post-spaces ' +- 'key': value +- 'key/newline': value +- 'escaped '' quote' diff --git a/test/emitter-examples/sqscalarspace.yaml b/test/emitter-examples/sqscalarspace.yaml new file mode 100644 index 0000000..266109e --- /dev/null +++ b/test/emitter-examples/sqscalarspace.yaml @@ -0,0 +1,2 @@ +' + ' diff --git a/test/emitter-examples/strings.yaml b/test/emitter-examples/strings.yaml new file mode 100644 index 0000000..3a196df --- /dev/null +++ b/test/emitter-examples/strings.yaml @@ -0,0 +1,7 @@ +unquoted: string +literal-block: | + This entire block of text will be the value of the 'literal-block' key, + with line breaks being preserved. +folded: > + This entire block of text will be the value of 'folded', but this + time, all newlines will be replaced with a single space. diff --git a/test/emitter-examples/t.yaml b/test/emitter-examples/t.yaml new file mode 100644 index 0000000..2caf1b2 --- /dev/null +++ b/test/emitter-examples/t.yaml @@ -0,0 +1 @@ +: a diff --git a/test/emitter-examples/t1.yaml b/test/emitter-examples/t1.yaml new file mode 100644 index 0000000..aa03859 --- /dev/null +++ b/test/emitter-examples/t1.yaml @@ -0,0 +1 @@ +x: a diff --git a/test/emitter-examples/t2.yaml b/test/emitter-examples/t2.yaml new file mode 100644 index 0000000..a8ca83d --- /dev/null +++ b/test/emitter-examples/t2.yaml @@ -0,0 +1 @@ +? : a diff --git a/test/emitter-examples/t3.yaml b/test/emitter-examples/t3.yaml new file mode 100644 index 0000000..9ab723f --- /dev/null +++ b/test/emitter-examples/t3.yaml @@ -0,0 +1 @@ +? x : a diff --git a/test/emitter-examples/t4.yaml b/test/emitter-examples/t4.yaml new file mode 100644 index 0000000..7eb2a3a --- /dev/null +++ b/test/emitter-examples/t4.yaml @@ -0,0 +1,2 @@ +? x # foo +: a diff --git a/test/emitter-examples/t5.yaml b/test/emitter-examples/t5.yaml new file mode 100644 index 0000000..f35eed8 --- /dev/null +++ b/test/emitter-examples/t5.yaml @@ -0,0 +1 @@ +{ ? : a } diff --git a/test/emitter-examples/tabsmix.yaml b/test/emitter-examples/tabsmix.yaml new file mode 100644 index 0000000..3b895bb --- /dev/null +++ b/test/emitter-examples/tabsmix.yaml @@ -0,0 +1,3 @@ +- > + + detected diff --git a/test/emitter-examples/tagdirective.yaml b/test/emitter-examples/tagdirective.yaml new file mode 100644 index 0000000..82e04a9 --- /dev/null +++ b/test/emitter-examples/tagdirective.yaml @@ -0,0 +1,2 @@ +%TAG !yaml! tag:yaml.org,2002: +--- diff --git a/test/emitter-examples/tagesc.yaml b/test/emitter-examples/tagesc.yaml new file mode 100644 index 0000000..70365f4 --- /dev/null +++ b/test/emitter-examples/tagesc.yaml @@ -0,0 +1,5 @@ +%TAG !e! tag:example.com,2000:app/ +--- +- !local foo +- !!str bar +- !e!tag%21 baz diff --git a/test/emitter-examples/tags-1.yaml b/test/emitter-examples/tags-1.yaml new file mode 100644 index 0000000..bb179ee --- /dev/null +++ b/test/emitter-examples/tags-1.yaml @@ -0,0 +1 @@ +explicit_string: !!str 0.5 diff --git a/test/emitter-examples/tags.yaml b/test/emitter-examples/tags.yaml new file mode 100644 index 0000000..3d5a010 --- /dev/null +++ b/test/emitter-examples/tags.yaml @@ -0,0 +1,7 @@ +gif_file: !!binary | + R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5 + OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+ + +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC + AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs= +explicit_string: !!str 0.5 +python_tag: !!python/complex '1.0+2.0j' diff --git a/test/emitter-examples/test.yaml b/test/emitter-examples/test.yaml new file mode 100644 index 0000000..9a24056 --- /dev/null +++ b/test/emitter-examples/test.yaml @@ -0,0 +1,6 @@ +- + a: b + c: d +- + e: f + g: h diff --git a/test/emitter-examples/test1.yaml b/test/emitter-examples/test1.yaml new file mode 100644 index 0000000..a1199a7 --- /dev/null +++ b/test/emitter-examples/test1.yaml @@ -0,0 +1,3 @@ +aa: + bb: cc + dd: ee diff --git a/test/emitter-examples/test2.yaml b/test/emitter-examples/test2.yaml new file mode 100644 index 0000000..5851568 --- /dev/null +++ b/test/emitter-examples/test2.yaml @@ -0,0 +1,2 @@ +a: b +c: d diff --git a/test/emitter-examples/u.yaml b/test/emitter-examples/u.yaml new file mode 100644 index 0000000..aa86103 --- /dev/null +++ b/test/emitter-examples/u.yaml @@ -0,0 +1,4 @@ +{ + foo : !!str, + !!str : bar, +} diff --git a/test/emitter-examples/u1.yaml b/test/emitter-examples/u1.yaml new file mode 100644 index 0000000..329f064 --- /dev/null +++ b/test/emitter-examples/u1.yaml @@ -0,0 +1 @@ +- [ : empty ] diff --git a/test/emitter-examples/u2.yaml b/test/emitter-examples/u2.yaml new file mode 100644 index 0000000..f1805e4 --- /dev/null +++ b/test/emitter-examples/u2.yaml @@ -0,0 +1 @@ +- { : empty } diff --git a/test/emitter-examples/u3.yaml b/test/emitter-examples/u3.yaml new file mode 100644 index 0000000..182599f --- /dev/null +++ b/test/emitter-examples/u3.yaml @@ -0,0 +1 @@ +: empty diff --git a/test/emitter-examples/utf8-simple.yaml b/test/emitter-examples/utf8-simple.yaml new file mode 100644 index 0000000..dd9a18a --- /dev/null +++ b/test/emitter-examples/utf8-simple.yaml @@ -0,0 +1 @@ +Τιμή ελληνική diff --git a/test/emitter-examples/utf8.yaml b/test/emitter-examples/utf8.yaml new file mode 100644 index 0000000..296532c --- /dev/null +++ b/test/emitter-examples/utf8.yaml @@ -0,0 +1,10 @@ +unqouted: string +literal-block: | + This entire block of text will be the value of the 'literal-block' key, + with line breaks being preserved. +folded: > + This entire block of text will be the value of 'folded', but this + time, all newlines will be replaced with a single space. +key-en: "Τιμή ελληνική" +κλειδί-ελ: "English value" + diff --git a/test/emitter-examples/v.yaml b/test/emitter-examples/v.yaml new file mode 100644 index 0000000..5a1554e --- /dev/null +++ b/test/emitter-examples/v.yaml @@ -0,0 +1,8 @@ +--- +a: 0 +--- +? b : 1 +--- +{ c: 2 } +--- +{ ? d : e } diff --git a/test/emitter-examples/v1.yaml b/test/emitter-examples/v1.yaml new file mode 100644 index 0000000..3ae0210 --- /dev/null +++ b/test/emitter-examples/v1.yaml @@ -0,0 +1 @@ +? key : value diff --git a/test/emitter-examples/v2.yaml b/test/emitter-examples/v2.yaml new file mode 100644 index 0000000..57eb12e --- /dev/null +++ b/test/emitter-examples/v2.yaml @@ -0,0 +1 @@ +{ ? d : e } diff --git a/test/emitter-examples/version.yaml b/test/emitter-examples/version.yaml new file mode 100644 index 0000000..96cd702 --- /dev/null +++ b/test/emitter-examples/version.yaml @@ -0,0 +1,4 @@ +# test +%YAML 1.1 +--- +# more comments diff --git a/test/emitter-examples/weirdplain.yaml b/test/emitter-examples/weirdplain.yaml new file mode 100644 index 0000000..020458e --- /dev/null +++ b/test/emitter-examples/weirdplain.yaml @@ -0,0 +1 @@ +this is a:w weird plain diff --git a/test/emitter-examples/ws0.yaml b/test/emitter-examples/ws0.yaml new file mode 100644 index 0000000..6440382 --- /dev/null +++ b/test/emitter-examples/ws0.yaml @@ -0,0 +1 @@ +--- scalar diff --git a/test/emitter-examples/ws1.yaml b/test/emitter-examples/ws1.yaml new file mode 100644 index 0000000..8171dcc --- /dev/null +++ b/test/emitter-examples/ws1.yaml @@ -0,0 +1 @@ +--- scalar diff --git a/test/emitter-examples/ws2.yaml b/test/emitter-examples/ws2.yaml new file mode 100644 index 0000000..5df48e7 --- /dev/null +++ b/test/emitter-examples/ws2.yaml @@ -0,0 +1,12 @@ +"top1" : + "key1" : &alias1 scalar1 +'top2' : + 'key2' : &alias2 scalar2 +top3: &node3 + *alias1 : scalar3 +top4: + *alias2 : scalar4 +top5 : + scalar5 +top6: + &anchor6 'key6' : scalar6 diff --git a/test/emitter-examples/ws3.yaml b/test/emitter-examples/ws3.yaml new file mode 100644 index 0000000..e1e1113 --- /dev/null +++ b/test/emitter-examples/ws3.yaml @@ -0,0 +1,6 @@ +{ first: Sammy, last: Sosa }: +# Statistics: + hr: # Home runs + 65 + avg: # Average + 0.278 diff --git a/test/emitter-examples/y.yaml b/test/emitter-examples/y.yaml new file mode 100644 index 0000000..e14d59c --- /dev/null +++ b/test/emitter-examples/y.yaml @@ -0,0 +1,3 @@ +a: + ? b + : c diff --git a/test/emitter-examples/yaml-version.yaml b/test/emitter-examples/yaml-version.yaml new file mode 100644 index 0000000..122b508 --- /dev/null +++ b/test/emitter-examples/yaml-version.yaml @@ -0,0 +1,3 @@ +%YAML 1.1 +--- +[1, 2, 3] diff --git a/test/emitter-examples/yy.yaml b/test/emitter-examples/yy.yaml new file mode 100644 index 0000000..a44dcdf --- /dev/null +++ b/test/emitter-examples/yy.yaml @@ -0,0 +1,2 @@ +a: + ? b : c diff --git a/test/emitter-examples/zeroexplicit.yaml b/test/emitter-examples/zeroexplicit.yaml new file mode 100644 index 0000000..5637a1a --- /dev/null +++ b/test/emitter-examples/zeroexplicit.yaml @@ -0,0 +1,7 @@ +--- +? +- a +- b +: +- c +- d diff --git a/test/jsontestsuite.test b/test/jsontestsuite.test new file mode 100755 index 0000000..4b0cbc1 --- /dev/null +++ b/test/jsontestsuite.test @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +count=0 + +# count expected to pass +for f in json-test-suite-data/test_parsing/y_*.json; do + count=`expr $count + 1` +done + +# add expected to fail +for f in json-test-suite-data/test_parsing/n_*.json; do + count=`expr $count + 1` +done + +# add implementation defined +for f in json-test-suite-data/test_parsing/i_*.json; do + count=`expr $count + 1` +done + +# output plan +echo 1..$count + +i=0 + +# expected to pass +for f in json-test-suite-data/test_parsing/y_*.json; do + + i=`expr $i + 1` + tf=`basename $f` + + ${TOP_BUILDDIR}/src/fy-tool --testsuite --streaming "$f" >/dev/null + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + + echo "$res $i - $tf" +done + +# expected to fail +for f in json-test-suite-data/test_parsing/n_*.json; do + + i=`expr $i + 1` + tf=`basename $f` + + ${TOP_BUILDDIR}/src/fy-tool --testsuite --streaming "$f" >/dev/null + if [ $? -eq 0 ]; then + res="not ok" + else + res="ok" + fi + + echo "$res $i - $tf" +done + +# implementation defined +for f in json-test-suite-data/test_parsing/i_*.json; do + + i=`expr $i + 1` + tf=`basename $f` + + ${TOP_BUILDDIR}/src/fy-tool --testsuite --streaming "$f" >/dev/null + if [ $? -eq 0 ]; then + ires="i-pass" + else + ires="i-fail" + fi + res="ok" + + echo "$res $i - $ires $tf" +done diff --git a/test/libfyaml-test-allocator.c b/test/libfyaml-test-allocator.c new file mode 100644 index 0000000..b5435cb --- /dev/null +++ b/test/libfyaml-test-allocator.c @@ -0,0 +1,356 @@ +/* + * libfyaml-test-allocator.c - libfyaml test public allocator interface + * + * Copyright (c) 2024 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) ((sizeof(x)/sizeof((x)[0]))) +#endif + +static const char *builtin_allocators[] = { + "linear", + "malloc", + "mremap", + "dedup", + "auto", + NULL, +}; + +START_TEST(allocator_builtins) +{ + const char **pp, *p; + + /* verify that all the builtins are available */ + pp = builtin_allocators; + while ((p = *pp++) != NULL) { + fprintf(stderr, "checking builtin allocator: %s\n", p); + ck_assert(fy_allocator_is_available(p)); + } +} +END_TEST + +static void test_allocator_alignment(struct fy_allocator *a, int tag) +{ + size_t sz, align; + void *p; + + /* 1, 2, 4, 8, 16 bytes allocations */ + for (sz = 1; sz <= 16; sz <<= 1) { + + align = sz; + + /* allocate and check alignment */ + p = fy_allocator_alloc(a, tag, sz, align); + ck_assert_ptr_ne(p, NULL); + ck_assert(((uintptr_t)p & (align - 1)) == 0); + } +} + +START_TEST(allocator_linear_buf) +{ + /* align to 64 bits */ + union { + char buf[1024]; + uint64_t dummy; + } u; + struct fy_linear_allocator_cfg lcfg; + struct fy_allocator *a = NULL; + int tag = -1; + void *p; + + memset(&lcfg, 0, sizeof(lcfg)); + lcfg.buf = u.buf; + lcfg.size = sizeof(u.buf); + + /* create */ + a = fy_allocator_create("linear", &lcfg); + ck_assert_ptr_ne(a, NULL); + + /* get the tag */ + tag = fy_allocator_get_tag(a); + ck_assert_int_ne(tag, -1); + + test_allocator_alignment(a, tag); + + /* allocate something too large to fit */ + p = fy_allocator_alloc(a, tag, sizeof(u.buf) + 1, 16); + ck_assert_ptr_eq(p, NULL); + + /* destroy */ + fy_allocator_destroy(a); +} +END_TEST + +START_TEST(allocator_linear_alloc) +{ + struct fy_linear_allocator_cfg lcfg; + struct fy_allocator *a = NULL; + int tag = -1; + void *p; + + memset(&lcfg, 0, sizeof(lcfg)); + lcfg.size = 1024; + + /* create */ + a = fy_allocator_create("linear", &lcfg); + ck_assert_ptr_ne(a, NULL); + + /* get the tag */ + tag = fy_allocator_get_tag(a); + ck_assert_int_ne(tag, -1); + + test_allocator_alignment(a, tag); + + /* allocate something too large to fit */ + p = fy_allocator_alloc(a, tag, 1024 + 1, 16); + ck_assert_ptr_eq(p, NULL); + + /* destroy */ + fy_allocator_destroy(a); +} +END_TEST + +START_TEST(allocator_malloc) +{ + struct fy_allocator *a = NULL; + int tag0 = -1, tag1 = -1; + + /* create */ + a = fy_allocator_create("malloc", NULL); + ck_assert_ptr_ne(a, NULL); + + /* get the first tag */ + tag0 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag0, -1); + + /* get the second tag */ + tag1 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag1, -1); + + /* tags must be different */ + ck_assert_int_ne(tag0, tag1); + + test_allocator_alignment(a, tag0); + + test_allocator_alignment(a, tag1); + + /* destroy */ + fy_allocator_destroy(a); +} +END_TEST + +START_TEST(allocator_mremap) +{ + struct fy_allocator *a = NULL; + int tag0 = -1, tag1 = -1; + + /* create (default everything) */ + a = fy_allocator_create("mremap", NULL); + ck_assert_ptr_ne(a, NULL); + + /* get the first tag */ + tag0 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag0, -1); + + /* get the second tag */ + tag1 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag1, -1); + + /* tags must be different */ + ck_assert_int_ne(tag0, tag1); + + test_allocator_alignment(a, tag0); + + test_allocator_alignment(a, tag1); + + /* destroy */ + fy_allocator_destroy(a); +} +END_TEST + +static inline bool +scenario_is_single_tagged(int scenario) +{ + return scenario >= FYAST_SINGLE_LINEAR_RANGE && + scenario <= FYAST_SINGLE_LINEAR_RANGE_DEDUP; +} + +static inline bool +scenario_is_dedup(int scenario) +{ + return scenario == FYAST_PER_TAG_FREE_DEDUP || + scenario == FYAST_PER_OBJ_FREE_DEDUP || + scenario == FYAST_SINGLE_LINEAR_RANGE_DEDUP; +} + +static inline bool +scenario_is_fixed_size(int scenario) +{ + return scenario >= FYAST_SINGLE_LINEAR_RANGE && + scenario <= FYAST_SINGLE_LINEAR_RANGE_DEDUP; +} + +START_TEST(allocator_auto) +{ + static const struct { + int scenario; + const char *name; + } auto_scenarios[] = { + { + .scenario = FYAST_PER_TAG_FREE, + .name = "per-tag-free", + }, { + .scenario = FYAST_PER_TAG_FREE_DEDUP, + .name = "per-tag-free-dedup", + }, { + .scenario = FYAST_PER_OBJ_FREE, + .name = "per-obj-free", + }, { + .scenario = FYAST_PER_OBJ_FREE_DEDUP, + .name = "per-obj-free-dedup", + }, { + .scenario = FYAST_SINGLE_LINEAR_RANGE, + .name = "single-linear-range", + }, { + .scenario = FYAST_SINGLE_LINEAR_RANGE_DEDUP, + .name = "single-linear-range-dedup", + } + }; + struct fy_auto_allocator_cfg acfg; + struct fy_allocator *a = NULL; + enum fy_auto_allocator_scenario_type sc; + int tag0 = -1, tag1 = -1; + unsigned int i; + + /* create (default everything) */ + for (i = 0; i < ARRAY_SIZE(auto_scenarios); i++) { + + printf("scenario #%d %s\n", i, auto_scenarios[i].name); + sc = auto_scenarios[i].scenario; + + memset(&acfg, 0, sizeof(acfg)); + acfg.scenario = sc; + + /* for fixed sizes, make it 1MB */ + if (scenario_is_fixed_size(sc)) + acfg.estimated_max_size = 1 << 20; + + a = fy_allocator_create("auto", &acfg); + ck_assert_ptr_ne(a, NULL); + + /* get the first tag */ + tag0 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag0, -1); + + if (!scenario_is_single_tagged(sc)) { + /* get the second tag */ + tag1 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag1, -1); + + /* tags must be different */ + ck_assert_int_ne(tag0, tag1); + } + + test_allocator_alignment(a, tag0); + + if (!scenario_is_single_tagged(sc)) + test_allocator_alignment(a, tag1); + + /* destroy */ + fy_allocator_destroy(a); + a = NULL; + } +} +END_TEST + +START_TEST(allocator_linear_inplace) +{ + struct fy_allocator *a; + char buf[FY_LINEAR_ALLOCATOR_IN_PLACE_MIN_SIZE]; + int tag; + const void *p; + + a = fy_linear_allocator_create_in_place(buf, sizeof(buf)); + ck_assert_ptr_ne(a, NULL); + + tag = fy_allocator_get_tag(a); + ck_assert_int_ne(tag, -1); + + p = fy_allocator_store(a, tag, "Hello", 6, 1); + ck_assert_ptr_ne(p, NULL); + + ck_assert(!strcmp(p, "Hello")); + + /* no release */ +} +END_TEST + +START_TEST(allocator_dedup_inplace) +{ + struct fy_allocator *a; + char buf[FY_DEDUP_ALLOCATOR_IN_PLACE_MIN_SIZE]; + int tag; + const void *p1, *p2; + + a = fy_dedup_allocator_create_in_place(buf, sizeof(buf)); + ck_assert_ptr_ne(a, NULL); + + tag = fy_allocator_get_tag(a); + ck_assert_int_ne(tag, -1); + + p1 = fy_allocator_store(a, tag, "Hello", 6, 1); + ck_assert_ptr_ne(p1, NULL); + + ck_assert(!strcmp(p1, "Hello")); + + p2 = fy_allocator_store(a, tag, "Hello", 6, 1); + ck_assert_ptr_ne(p2, NULL); + + ck_assert(!strcmp(p2, "Hello")); + + /* dedup must return the same pointer */ + ck_assert(p1 == p2); + + /* no release */ +} +END_TEST + +TCase *libfyaml_case_allocator(void) +{ + TCase *tc; + + tc = tcase_create("allocator"); + + tcase_add_test(tc, allocator_builtins); + + tcase_add_test(tc, allocator_linear_buf); + tcase_add_test(tc, allocator_linear_alloc); + tcase_add_test(tc, allocator_malloc); + tcase_add_test(tc, allocator_mremap); + tcase_add_test(tc, allocator_auto); + + tcase_add_test(tc, allocator_linear_inplace); + tcase_add_test(tc, allocator_dedup_inplace); + + return tc; +} diff --git a/test/libfyaml-test-core.c b/test/libfyaml-test-core.c new file mode 100644 index 0000000..47d8fac --- /dev/null +++ b/test/libfyaml-test-core.c @@ -0,0 +1,2117 @@ +/* + * libfyaml-test-core.c - libfyaml core testing harness + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +START_TEST(doc_build_simple) +{ + struct fy_document *fyd; + + /* setup */ + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* cleanup */ + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_build_parse_check) +{ + struct fy_document *fyd; + char *buf; + + /* build document (with comments, newlines etc) */ + fyd = fy_document_build_from_string(NULL, "#comment\n[ 42, \n 12 ] # comment\n", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* convert to string */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + + /* compare with expected result */ + ck_assert_str_eq(buf, "[42, 12]\n"); + + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_build_scalar) +{ + struct fy_document *fyd; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "plain scalar # comment", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* compare with expected result */ + ck_assert_str_eq(fy_node_get_scalar0(fy_document_root(fyd)), "plain scalar"); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_build_sequence) +{ + struct fy_document *fyd; + struct fy_node *fyn; + int count; + void *iter; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "[ 10, 11, foo ] # comment", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* check for correct count value */ + count = fy_node_sequence_item_count(fy_document_root(fyd)); + ck_assert_int_eq(count, 3); + + /* try forward iterator first */ + iter = NULL; + + /* 0 */ + fyn = fy_node_sequence_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + /* 1 */ + fyn = fy_node_sequence_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "11"); + + /* 2 */ + fyn = fy_node_sequence_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "foo"); + + /* final, iterator must return NULL */ + fyn = fy_node_sequence_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_eq(fyn, NULL); + + /* reverse iterator */ + iter = NULL; + + /* 2 */ + fyn = fy_node_sequence_reverse_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "foo"); + + /* 1 */ + fyn = fy_node_sequence_reverse_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "11"); + + /* 0 */ + fyn = fy_node_sequence_reverse_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + /* final, iterator must return NULL */ + fyn = fy_node_sequence_reverse_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_eq(fyn, NULL); + + /* do forward index based accesses */ + + /* 0 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), 0); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + /* 1 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), 1); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "11"); + + /* 2 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), 2); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "foo"); + + /* 3, it must not exist */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), 3); + ck_assert_ptr_eq(fyn, NULL); + + /* do backward index based accesses */ + + /* 2 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -1); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "foo"); + + /* 1 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -2); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "11"); + + /* 0 */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -3); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + /* -1, it must not exist */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -4); + ck_assert_ptr_eq(fyn, NULL); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_build_mapping) +{ + struct fy_document *fyd; + struct fy_node_pair *fynp; + int count; + void *iter; + + fyd = fy_document_build_from_string(NULL, "{ foo: 10, bar : 20, baz: [100, 101], [frob, 1]: boo }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* check for correct count value */ + count = fy_node_mapping_item_count(fy_document_root(fyd)); + ck_assert_int_eq(count, 4); + + /* forward iterator first */ + iter = NULL; + + /* 0 */ + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "foo"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "10"); + + /* 1 */ + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "bar"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "20"); + + /* 2 */ + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "baz"); + ck_assert_int_eq(fy_node_sequence_item_count(fy_node_pair_value(fynp)), 2); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_value(fynp), 0)), "100"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_value(fynp), 1)), "101"); + + /* 3 */ + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_int_eq(fy_node_sequence_item_count(fy_node_pair_key(fynp)), 2); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_key(fynp), 0)), "frob"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_key(fynp), 1)), "1"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "boo"); + + /* 4, it must not exist */ + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_eq(fynp, NULL); + + /* reverse iterator */ + iter = NULL; + + /* 3 */ + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_int_eq(fy_node_sequence_item_count(fy_node_pair_key(fynp)), 2); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_key(fynp), 0)), "frob"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_key(fynp), 1)), "1"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "boo"); + + /* 2 */ + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "baz"); + ck_assert_int_eq(fy_node_sequence_item_count(fy_node_pair_value(fynp)), 2); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_value(fynp), 0)), "100"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_sequence_get_by_index(fy_node_pair_value(fynp), 1)), "101"); + + /* 1 */ + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "bar"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "20"); + + /* 0 */ + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "foo"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "10"); + + /* -1, it must not exist */ + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_eq(fynp, NULL); + + /* key lookups (note how only the contents are compared) */ + ck_assert(fy_node_compare_string(fy_node_mapping_lookup_by_string(fy_document_root(fyd), "foo", FY_NT), "10", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_mapping_lookup_by_string(fy_document_root(fyd), "bar", FY_NT), "20", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_mapping_lookup_by_string(fy_document_root(fyd), "baz", FY_NT), "- 100\n- 101", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_mapping_lookup_by_string(fy_document_root(fyd), "- 'frob'\n- \"\x31\"", FY_NT), "boo", FY_NT) == true); + + fy_document_destroy(fyd); + fyd = NULL; +} +END_TEST + +START_TEST(doc_path_access) +{ + struct fy_document *fyd; + struct fy_node *fyn; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "{ " + "foo: 10, bar : 20, baz:{ frob: boo }, " + "frooz: [ seq1, { key: value} ], \"zero\\0zero\" : 0, " + "{ key2: value2 }: { key3: value3 } " + "}", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* check that getting root node works */ + fyn = fy_node_by_path(fy_document_root(fyd), "/", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fy_document_root(fyd)); + + /* check access to scalars */ + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW), "10", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "bar", FY_NT, FYNWF_DONT_FOLLOW), "20", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "baz/frob", FY_NT, FYNWF_DONT_FOLLOW), "boo", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/frooz/0", FY_NT, FYNWF_DONT_FOLLOW), "seq1", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/frooz/1/key", FY_NT, FYNWF_DONT_FOLLOW), "value", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "\"zero\\0zero\"", FY_NT, FYNWF_DONT_FOLLOW), "0", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/{ key2: value2 }/key3", FY_NT, FYNWF_DONT_FOLLOW), "value3", FY_NT) == true); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_path_node) +{ + struct fy_document *fyd; + char *path; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "{ " + "foo: 10, bar : 20, baz:{ frob: boo }, " + "frooz: [ seq1, { key: value} ], \"zero\\0zero\" : 0, " + "{ key2: value2 }: { key3: value3 } " + "}", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fyd), "/", FY_NT, FYNWF_DONT_FOLLOW)); + ck_assert_str_eq(path, "/"); + free(path); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fyd), "/frooz", FY_NT, FYNWF_DONT_FOLLOW)); + ck_assert_str_eq(path, "/frooz"); + free(path); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fyd), "/frooz/0", FY_NT, FYNWF_DONT_FOLLOW)); + ck_assert_str_eq(path, "/frooz/0"); + free(path); + + path = fy_node_get_path(fy_node_by_path(fy_document_root(fyd), "/{ key2: value2 }/key3", FY_NT, FYNWF_DONT_FOLLOW)); + ck_assert_str_eq(path, "/{key2: value2}/key3"); + free(path); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_path_parent) +{ + struct fy_document *fyd; + struct fy_node *fyn_root, *fyn_foo, *fyn_bar, *fyn_baz, *fyn_frob, *fyn_ten; + struct fy_node *fyn_deep, *fyn_deeper; + char *path; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "{ " + "foo: 10, bar : [ ten, 20 ], baz:{ frob: boo, deep: { deeper: yet } }, " + "}", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_root = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + + fyn_foo = fy_node_by_path(fyn_root, "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_foo, NULL); + + fyn_bar = fy_node_by_path(fyn_root, "/bar", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_bar, NULL); + + fyn_baz = fy_node_by_path(fyn_root, "/baz", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_baz, NULL); + + fyn_frob = fy_node_by_path(fyn_root, "/baz/frob", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_frob, NULL); + + fyn_ten = fy_node_by_path(fyn_root, "/bar/0", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_ten, NULL); + + fyn_deep = fy_node_by_path(fyn_root, "/baz/deep", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_deep, NULL); + + fyn_deeper = fy_node_by_path(fyn_root, "/baz/deep/deeper", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_deeper, NULL); + + /* check parent paths of foo, frob, ten */ + path = fy_node_get_parent_address(fyn_foo); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "foo"); + free(path); + + path = fy_node_get_parent_address(fyn_frob); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "frob"); + free(path); + + path = fy_node_get_parent_address(fyn_ten); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "0"); + free(path); + + /* check relative paths to root */ + path = fy_node_get_path_relative_to(NULL, fyn_foo); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "foo"); + free(path); + + path = fy_node_get_path_relative_to(fyn_root, fyn_foo); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "foo"); + free(path); + + path = fy_node_get_path_relative_to(fyn_root, fyn_frob); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "baz/frob"); + free(path); + + path = fy_node_get_path_relative_to(fyn_root, fyn_ten); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "bar/0"); + free(path); + + /* check relative paths to other parents */ + path = fy_node_get_path_relative_to(fyn_baz, fyn_frob); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "frob"); + free(path); + + path = fy_node_get_path_relative_to(fyn_bar, fyn_ten); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "0"); + free(path); + + path = fy_node_get_path_relative_to(fyn_baz, fyn_deeper); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "deep/deeper"); + free(path); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_short_path) +{ + struct fy_document *fyd; + struct fy_node *fyn_root, *fyn_foo, *fyn_notfoo, *fyn_bar, *fyn_baz; + const char *str; + + /* build document */ + fyd = fy_document_build_from_string(NULL, + "--- &r\n" + " foo: &f\n" + " bar: [ 0, two, baz: what ]\n" + " frob: true\n" + " notfoo: false\n" + , FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_root = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + + fyn_foo = fy_node_by_path(fyn_root, "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_foo, NULL); + + fyn_notfoo = fy_node_by_path(fyn_root, "/notfoo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_notfoo, NULL); + + fyn_bar = fy_node_by_path(fyn_root, "/foo/bar", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_bar, NULL); + + fyn_baz = fy_node_by_path(fyn_root, "/foo/bar/2/baz", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_baz, NULL); + + str = fy_node_get_short_path_alloca(fyn_root); + ck_assert_ptr_ne(str, NULL); + ck_assert_str_eq(str, "*r"); + ck_assert_ptr_eq(fy_node_by_path(fy_document_root(fyd), str, FY_NT, FYNWF_FOLLOW), fyn_root); + + str = fy_node_get_short_path_alloca(fyn_foo); + ck_assert_ptr_ne(str, NULL); + ck_assert_str_eq(str, "*f"); + ck_assert_ptr_eq(fy_node_by_path(fy_document_root(fyd), str, FY_NT, FYNWF_FOLLOW), fyn_foo); + + str = fy_node_get_short_path_alloca(fyn_notfoo); + ck_assert_ptr_ne(str, NULL); + ck_assert_str_eq(str, "*r/notfoo"); + ck_assert_ptr_eq(fy_node_by_path(fy_document_root(fyd), str, FY_NT, FYNWF_FOLLOW), fyn_notfoo); + + str = fy_node_get_short_path_alloca(fyn_bar); + ck_assert_ptr_ne(str, NULL); + ck_assert_str_eq(str, "*f/bar"); + ck_assert_ptr_eq(fy_node_by_path(fy_document_root(fyd), str, FY_NT, FYNWF_FOLLOW), fyn_bar); + + str = fy_node_get_short_path_alloca(fyn_baz); + ck_assert_ptr_ne(str, NULL); + ck_assert_str_eq(str, "*f/bar/2/baz"); + ck_assert_ptr_eq(fy_node_by_path(fy_document_root(fyd), str, FY_NT, FYNWF_FOLLOW), fyn_baz); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_scalar_path) +{ + struct fy_document *fyd; + struct fy_node *fyn_root, *fyn_foo; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "--- foo\n", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_root = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + + /* get the scalar root and verify */ + fyn_foo = fy_node_by_path(fyn_root, "/", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_foo, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn_foo), "foo"); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_scalar_path_array) +{ + struct fy_document *fyd; + struct fy_node *fyn_root, *fynt; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "--- [ foo, bar, baz ]\n", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_root = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + + /* get the scalars in the array and verify */ + fynt = fy_node_by_path(fyn_root, "/0", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fynt, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fynt), "foo"); + + fynt = fy_node_by_path(fyn_root, "/1", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fynt, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fynt), "bar"); + + fynt = fy_node_by_path(fyn_root, "/2", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fynt, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fynt), "baz"); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_nearest_anchor) +{ + struct fy_document *fyd; + struct fy_node *fyn, *fyn_root, *fyn_foo, *fyn_notfoo, *fyn_bar, *fyn_baz; + + /* build document */ + fyd = fy_document_build_from_string(NULL, + "--- &r\n" + " foo: &f\n" + " bar: [ 0, two, baz: what ]\n" + " frob: true\n" + " notfoo: false\n" + , FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_root = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + + fyn_foo = fy_node_by_path(fyn_root, "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_foo, NULL); + + fyn_notfoo = fy_node_by_path(fyn_root, "/notfoo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_notfoo, NULL); + + fyn_bar = fy_node_by_path(fyn_root, "/foo/bar", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_bar, NULL); + + fyn_baz = fy_node_by_path(fyn_root, "/foo/bar/2/baz", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_baz, NULL); + + /* get nearest anchor of root (is root) */ + fyn = fy_anchor_node(fy_node_get_nearest_anchor(fyn_root)); + ck_assert_ptr_eq(fyn, fyn_root); + + /* get nearest anchor of notfoo (is root) */ + fyn = fy_anchor_node(fy_node_get_nearest_anchor(fyn_notfoo)); + ck_assert_ptr_eq(fyn, fyn_root); + + /* get nearest anchor of baz (is foo) */ + fyn = fy_anchor_node(fy_node_get_nearest_anchor(fyn_baz)); + ck_assert_ptr_eq(fyn, fyn_foo); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_references) +{ + struct fy_document *fyd; + struct fy_node *fyn, *fyn_ref, *fyn_root, *fyn_foo, *fyn_notfoo, *fyn_bar, *fyn_baz; + char *path; + + /* build document */ + fyd = fy_document_build_from_string(NULL, + "---\n" + " foo: &f\n" + " bar: [ 0, two, baz: what ]\n" + " frob: true\n" + " notfoo: false\n" + , FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_root = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + + fyn_foo = fy_node_by_path(fyn_root, "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_foo, NULL); + + fyn_notfoo = fy_node_by_path(fyn_root, "/notfoo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_notfoo, NULL); + + fyn_bar = fy_node_by_path(fyn_root, "/foo/bar", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_bar, NULL); + + fyn_baz = fy_node_by_path(fyn_root, "/foo/bar/2/baz", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_baz, NULL); + + /* get reference to root */ + path = fy_node_get_reference(fyn_root); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "*/"); + free(path); + + /* get reference to /foo */ + path = fy_node_get_reference(fyn_foo); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "*f"); + free(path); + + /* get reference to /notfoo */ + path = fy_node_get_reference(fyn_notfoo); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "*/notfoo"); + free(path); + + /* get reference to /foo/bar/2/baz */ + path = fy_node_get_reference(fyn_baz); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "*/foo/bar/2/baz"); + free(path); + + /* create reference to root and verify it points there */ + fyn_ref = fy_node_create_reference(fyn_root); + ck_assert_ptr_ne(fyn_ref, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_root); + fy_node_free(fyn_ref); + + /* get reference to /foo */ + fyn_ref = fy_node_create_reference(fyn_foo); + ck_assert_ptr_ne(path, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_foo); + fy_node_free(fyn_ref); + + /* get reference to /notfoo */ + fyn_ref = fy_node_create_reference(fyn_notfoo); + ck_assert_ptr_ne(path, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_notfoo); + fy_node_free(fyn_ref); + + /* get reference to /bar */ + fyn_ref = fy_node_create_reference(fyn_bar); + ck_assert_ptr_ne(path, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_bar); + fy_node_free(fyn_ref); + + /* get reference to /baz */ + fyn_ref = fy_node_create_reference(fyn_baz); + ck_assert_ptr_ne(path, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_baz); + fy_node_free(fyn_ref); + + /* get relative reference to /foo starting at / */ + path = fy_node_get_relative_reference(fyn_root, fyn_foo); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "*/foo"); + free(path); + + /* get relative reference to /foo/bar/2/baz starting at / */ + path = fy_node_get_relative_reference(fyn_root, fyn_baz); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "*/foo/bar/2/baz"); + free(path); + + /* get relative reference to /foo/bar/2/baz starting at /foo */ + path = fy_node_get_relative_reference(fyn_foo, fyn_baz); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "*f/bar/2/baz"); + free(path); + + /* get relative reference to /notfoo at /foo (will return absolute) */ + path = fy_node_get_relative_reference(fyn_foo, fyn_notfoo); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "*/notfoo"); + free(path); + + /* create relative reference to /foo starting at / */ + fyn_ref = fy_node_create_relative_reference(fyn_root, fyn_foo); + ck_assert_ptr_ne(fyn_ref, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_foo); + fy_node_free(fyn_ref); + + /* create relative reference to /foo/bar/2/baz starting at / */ + fyn_ref = fy_node_create_relative_reference(fyn_root, fyn_baz); + ck_assert_ptr_ne(fyn_ref, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_baz); + fy_node_free(fyn_ref); + + /* create relative reference to /foo/bar/2/baz starting at /foo */ + fyn_ref = fy_node_create_relative_reference(fyn_foo, fyn_baz); + ck_assert_ptr_ne(fyn_ref, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_baz); + fy_node_free(fyn_ref); + + /* create relative reference to /notfoo starting at /foo (will use absolute) */ + fyn_ref = fy_node_create_relative_reference(fyn_foo, fyn_notfoo); + ck_assert_ptr_ne(fyn_ref, NULL); + fyn = fy_node_resolve_alias(fyn_ref); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_notfoo); + fy_node_free(fyn_ref); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_nearest_child_of) +{ + struct fy_document *fyd; + struct fy_node *fyn, *fyn_root, *fyn_foo, *fyn_bar, *fyn_baz; + + /* build document */ + fyd = fy_document_build_from_string(NULL, + "foo:\n" + " bar:\n" + " barz: [ zero, baz: true ]\n" + " frooz: notfoo\n" + , FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_root = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + + fyn_foo = fy_node_by_path(fyn_root, "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_foo, NULL); + + fyn_bar = fy_node_by_path(fyn_root, "/foo/bar", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_bar, NULL); + + fyn_baz = fy_node_by_path(fyn_root, "/foo/bar/barz/1/baz", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_baz, NULL); + + /* nearest child to the root of /foo is /foo */ + fyn = fy_node_get_nearest_child_of(fyn_root, fyn_foo); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_foo); + + /* nearest child to the root of /foo/bar/barz/1/baz is /foo */ + fyn = fy_node_get_nearest_child_of(fyn_root, fyn_baz); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_foo); + + /* nearest child to foo of /foo/bar/barz/1/baz is /foo/bar */ + fyn = fy_node_get_nearest_child_of(fyn_foo, fyn_baz); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_ptr_eq(fyn, fyn_bar); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_create_empty_seq1) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_build_from_string(fyd, "[ ]", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* convert to string */ + buf = fy_emit_node_to_string(fy_document_root(fyd), FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + + /* compare with expected result */ + ck_assert_str_eq(buf, "[]"); + + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_create_empty_seq2) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* convert to string */ + buf = fy_emit_node_to_string(fy_document_root(fyd), FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + + /* compare with expected result */ + ck_assert_str_eq(buf, "[]"); + + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_create_empty_map1) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_build_from_string(fyd, "{ }", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* convert to string */ + buf = fy_emit_node_to_string(fy_document_root(fyd), FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + + /* compare with expected result */ + ck_assert_str_eq(buf, "{}"); + + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_create_empty_map2) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_mapping(fyd); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* convert to string */ + buf = fy_emit_node_to_string(fy_document_root(fyd), FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + + /* compare with expected result */ + ck_assert_str_eq(buf, "{}"); + + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_create_test_seq1) +{ + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn, NULL); + + ret = fy_node_sequence_append(fyn, fy_node_create_scalar(fyd, "foo", FY_NT)); + ck_assert_int_eq(ret, 0); + + ret = fy_node_sequence_append(fyn, fy_node_create_scalar(fyd, "bar", FY_NT)); + ck_assert_int_eq(ret, 0); + + ret = fy_node_sequence_append(fyn, fy_node_build_from_string(fyd, "{ baz: frooz }", FY_NT)); + ck_assert_int_eq(ret, 0); + + fy_document_set_root(fyd, fyn); + + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/0", FY_NT, FYNWF_DONT_FOLLOW), "foo", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/1", FY_NT, FYNWF_DONT_FOLLOW), "bar", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/2/baz", FY_NT, FYNWF_DONT_FOLLOW), "frooz", FY_NT) == true); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_create_test_map1) +{ + struct fy_document *fyd; + struct fy_node *fyn, *fyn1, *fyn2, *fyn3; + int ret; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_mapping(fyd); + ck_assert_ptr_ne(fyn, NULL); + + ret = fy_node_mapping_append(fyn, + fy_node_build_from_string(fyd, "seq", FY_NT), + fy_node_build_from_string(fyd, "[ zero, one ]", FY_NT)); + ck_assert_int_eq(ret, 0); + + ret = fy_node_mapping_append(fyn, NULL, + fy_node_build_from_string(fyd, "value-of-null-key", FY_NT)); + ck_assert_int_eq(ret, 0); + + ret = fy_node_mapping_append(fyn, + fy_node_build_from_string(fyd, "key-of-null-value", FY_NT), NULL); + ck_assert_int_eq(ret, 0); + + fy_document_set_root(fyd, fyn); + + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/seq/0", FY_NT, FYNWF_DONT_FOLLOW), "zero", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/seq/1", FY_NT, FYNWF_DONT_FOLLOW), "one", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/''", FY_NT, FYNWF_DONT_FOLLOW), "value-of-null-key", FY_NT) == true); + + fyn1 = fy_node_by_path(fy_document_root(fyd), "/key-of-null-value", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_eq(fyn1, NULL); + + /* try to append duplicate key (it should fail) */ + fyn2 = fy_node_build_from_string(fyd, "seq", FY_NT); + ck_assert_ptr_ne(fyn2, NULL); + fyn3 = fy_node_create_scalar(fyd, "dupl", FY_NT); + ck_assert_ptr_ne(fyn3, NULL); + ret = fy_node_mapping_append(fyn, fyn2, fyn3); + ck_assert_int_ne(ret, 0); + + fy_node_free(fyn3); + fy_node_free(fyn2); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_insert_remove_seq) +{ + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fy_document_set_root(fyd, fy_node_build_from_string(fyd, "[ one, two, four ]", FY_NT)); + + /* check that the order is correct */ + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/0", FY_NT, FYNWF_DONT_FOLLOW), "one", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/1", FY_NT, FYNWF_DONT_FOLLOW), "two", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/2", FY_NT, FYNWF_DONT_FOLLOW), "four", FY_NT) == true); + + ret = fy_node_sequence_append(fy_document_root(fyd), fy_node_build_from_string(fyd, "five", FY_NT)); + ck_assert_int_eq(ret, 0); + + ret = fy_node_sequence_prepend(fy_document_root(fyd), fy_node_build_from_string(fyd, "zero", FY_NT)); + ck_assert_int_eq(ret, 0); + + ret = fy_node_sequence_insert_after(fy_document_root(fyd), + fy_node_by_path(fy_document_root(fyd), "/2", FY_NT, FYNWF_DONT_FOLLOW), + fy_node_build_from_string(fyd, "three", FY_NT)); + ck_assert_int_eq(ret, 0); + + ret = fy_node_sequence_insert_before(fy_document_root(fyd), + fy_node_by_path(fy_document_root(fyd), "/3", FY_NT, FYNWF_DONT_FOLLOW), + fy_node_build_from_string(fyd, "two-and-a-half", FY_NT)); + ck_assert_int_eq(ret, 0); + + fyn = fy_node_sequence_remove(fy_document_root(fyd), + fy_node_by_path(fy_document_root(fyd), "/3", FY_NT, FYNWF_DONT_FOLLOW)); + ck_assert_ptr_ne(fyn, NULL); + + fy_node_free(fyn); + + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/0", FY_NT, FYNWF_DONT_FOLLOW), "zero", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/1", FY_NT, FYNWF_DONT_FOLLOW), "one", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/2", FY_NT, FYNWF_DONT_FOLLOW), "two", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/3", FY_NT, FYNWF_DONT_FOLLOW), "three", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/4", FY_NT, FYNWF_DONT_FOLLOW), "four", FY_NT) == true); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_insert_remove_map) +{ + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_build_from_string(NULL, "{ one: 1, two: 2, four: 4 }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* check that the order is correct */ + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/one", FY_NT, FYNWF_DONT_FOLLOW), "1", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/two", FY_NT, FYNWF_DONT_FOLLOW), "2", FY_NT) == true); + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/four", FY_NT, FYNWF_DONT_FOLLOW), "4", FY_NT) == true); + + ret = fy_node_mapping_append(fy_document_root(fyd), + fy_node_build_from_string(fyd, "three", FY_NT), + fy_node_build_from_string(fyd, "3", FY_NT)); + ck_assert_int_eq(ret, 0); + + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/three", FY_NT, FYNWF_DONT_FOLLOW), "3", FY_NT) == true); + + ret = fy_node_mapping_prepend(fy_document_root(fyd), + fy_node_build_from_string(fyd, "zero", FY_NT), + fy_node_build_from_string(fyd, "0", FY_NT)); + ck_assert_int_eq(ret, 0); + + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/zero", FY_NT, FYNWF_DONT_FOLLOW), "0", FY_NT) == true); + + ret = fy_node_mapping_append(fy_document_root(fyd), + fy_node_build_from_string(fyd, "two-and-a-half", FY_NT), + fy_node_build_from_string(fyd, "2.5", FY_NT)); + ck_assert_int_eq(ret, 0); + + ck_assert(fy_node_compare_string(fy_node_by_path(fy_document_root(fyd), "/two-and-a-half", FY_NT, FYNWF_DONT_FOLLOW), "2.5", FY_NT) == true); + + fyn = fy_node_mapping_remove_by_key(fy_document_root(fyd), + fy_node_build_from_string(fyd, "two-and-a-half", FY_NT)); + ck_assert_ptr_ne(fyn, NULL); + + fy_node_free(fyn); + + /* it must be removed */ + fyn = fy_node_by_path(fy_document_root(fyd), "/two-and-a-half", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_eq(fyn, NULL); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_sort) +{ + struct fy_document *fyd; + void *fynp; + void *iter; + int ret, count; + + fyd = fy_document_build_from_string(NULL, "{ a: 5, { z: bar }: 1, z: 7, " + "[ a, b, c] : 3, { a: whee } : 2 , " + "b: 6, [ z ]: 4 }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + ret = fy_node_sort(fy_document_root(fyd), NULL, NULL); + ck_assert_int_eq(ret, 0); + + /* check for correct count value */ + count = fy_node_mapping_item_count(fy_document_root(fyd)); + ck_assert_int_eq(count, 7); + + /* forward iterator first */ + iter = NULL; + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "1"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "2"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "3"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "4"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "5"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "6"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "7"); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_insert_at) +{ + struct fy_document *fyd; + struct fy_node *fyn; + int rc; + + /* build document */ + fyd = fy_document_build_from_string(NULL, + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " annotations:\n" + " x: 2024-06-12t14:50:43z\n", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* verify that x exists */ + fyn = fy_node_by_path(fy_document_root(fyd), "/metadata/annotations/x", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + + /* delete the x node */ + rc = fy_document_insert_at(fyd, "/metadata/annotations/x", FY_NT, NULL); + ck_assert_int_eq(rc, 0); + + /* verify that x now gone */ + fyn = fy_node_by_path(fy_document_root(fyd), "/metadata/annotations/x", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_eq(fyn, NULL); + + /* verify that y does not exist */ + fyn = fy_node_by_path(fy_document_root(fyd), "/metadata/annotations/y", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_eq(fyn, NULL); + + /* insert a y node */ + rc = fy_document_insert_at(fyd, "/metadata/annotations", FY_NT, + fy_node_buildf(fyd, "y: 2024-06-12t14:50:43z")); + ck_assert_int_eq(rc, 0); + + /* verify that y exists now */ + fyn = fy_node_by_path(fy_document_root(fyd), "/metadata/annotations/y", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_destroy(fyd); +} +END_TEST + +static char *join_docs(const char *tgt_text, const char *tgt_path, + const char *src_text, const char *src_path, + const char *emit_path) +{ + struct fy_document *fyd_tgt, *fyd_src; + struct fy_node *fyn_tgt, *fyn_src, *fyn_emit; + char *output; + int ret; + + /* insert which overwrites root ( <- ) */ + fyd_tgt = fy_document_build_from_string(NULL, tgt_text, FY_NT); + ck_assert_ptr_ne(fyd_tgt, NULL); + + fyd_src = fy_document_build_from_string(NULL, src_text, FY_NT); + ck_assert_ptr_ne(fyd_src, NULL); + + fyn_tgt = fy_node_by_path(fy_document_root(fyd_tgt), tgt_path, FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_tgt, NULL); + + fyn_src = fy_node_by_path(fy_document_root(fyd_src), src_path, FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_src, NULL); + + ret = fy_node_insert(fyn_tgt, fyn_src); + ck_assert_int_eq(ret, 0); + + ret = fy_document_set_parent(fyd_tgt, fyd_src); + ck_assert_int_eq(ret, 0); + + fyn_emit = fy_node_by_path(fy_document_root(fyd_tgt), emit_path, FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_emit, NULL); + + output = fy_emit_node_to_string(fyn_emit, FYECF_MODE_FLOW_ONELINE | FYECF_WIDTH_INF); + ck_assert_ptr_ne(output, NULL); + + fy_document_destroy(fyd_tgt); + + return output; +} + +START_TEST(doc_join_scalar_to_scalar) +{ + char *output; + + output = join_docs( + "foo", "/", /* target */ + "bar", "/", /* source */ + "/"); /* emit path */ + + ck_assert_str_eq(output, "bar"); + free(output); +} +END_TEST + +START_TEST(doc_join_scalar_to_map) +{ + char *output; + + output = join_docs( + "{ foo: baz }", "/", /* target */ + "bar", "/", /* source */ + "/"); /* emit path */ + + ck_assert_str_eq(output, "bar"); + free(output); +} +END_TEST + +START_TEST(doc_join_scalar_to_seq) +{ + char *output; + + output = join_docs( + "[ foo, baz ]", "/", /* target */ + "bar", "/", /* source */ + "/"); /* emit path */ + + ck_assert_str_eq(output, "bar"); + free(output); +} +END_TEST + +START_TEST(doc_join_map_to_scalar) +{ + char *output; + + output = join_docs( + "foo", "/", /* target */ + "{bar: baz}", "/", /* source */ + "/"); /* emit path */ + + ck_assert_str_eq(output, "{bar: baz}"); + free(output); +} +END_TEST + +START_TEST(doc_join_map_to_seq) +{ + char *output; + + output = join_docs( + "[foo, frooz]", "/", /* target */ + "{bar: baz}", "/", /* source */ + "/"); /* emit path */ + + ck_assert_str_eq(output, "{bar: baz}"); + free(output); +} +END_TEST + +START_TEST(doc_join_map_to_map) +{ + char *output; + + output = join_docs( + "{foo: frooz}", "/", /* target */ + "{bar: baz}", "/", /* source */ + "/"); /* emit path */ + + ck_assert_str_eq(output, "{foo: frooz, bar: baz}"); + free(output); +} +END_TEST + +START_TEST(doc_join_seq_to_scalar) +{ + char *output; + + output = join_docs( + "foo", "/", /* target */ + "[bar, baz]", "/", /* source */ + "/"); /* emit path */ + + ck_assert_str_eq(output, "[bar, baz]"); + free(output); +} +END_TEST + +START_TEST(doc_join_seq_to_seq) +{ + char *output; + + output = join_docs( + "[foo, frooz]", "/", /* target */ + "[bar, baz]", "/", /* source */ + "/"); /* emit path */ + + ck_assert_str_eq(output, "[foo, frooz, bar, baz]"); + free(output); +} +END_TEST + +START_TEST(doc_join_seq_to_map) +{ + char *output; + + output = join_docs( + "{foo: frooz}", "/", /* target */ + "[bar, baz]", "/", /* source */ + "/"); /* emit path */ + + ck_assert_str_eq(output, "[bar, baz]"); + free(output); +} +END_TEST + +START_TEST(doc_join_tags) +{ + char *output; + + output = join_docs( + "%TAG !a! tag:a.com,2019:\n" + "---\n" + "- !a!foo\n" + " foo: bar\n", "/", + "%TAG !b! tag:b.com,2019:\n" + "---\n" + "- !b!bar\n" + " something: other\n", "/", + "/"); + + ck_assert_str_eq(output, "[!a!foo {foo: bar}, !b!bar {something: other}]"); + free(output); +} +END_TEST + +START_TEST(doc_build_with_tags) +{ + struct fy_document *fyd; + struct fy_node *fyn; + struct fy_token *fyt; + char *buf; + int rc; + + /* build document */ + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* create a sequence and set it as root */ + fyn = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + fyn = NULL; + + /* create a node, containing a new tag */ + fyn = fy_node_build_from_string(fyd, "%TAG !e! tag:example.com,2000:app/\n---\n- foo\n- !e!foo bar\n", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + + /* append it to the root of the document */ + rc = fy_node_sequence_append(fy_document_root(fyd), fyn); + ck_assert_int_eq(rc, 0); + fyn = NULL; + + /* there must be a new tag */ + fyt = fy_document_tag_directive_lookup(fyd, "!e!"); + ck_assert_ptr_ne(fyt, NULL); + + /* try to build another, but with a different !e! prefix, it must fail */ + fyn = fy_node_build_from_string(fyd, "%TAG !e! tag:example.com,2019:app/\n---\n- foo\n- !e!foo bar\n", FY_NT); + ck_assert_ptr_eq(fyn, NULL); + + /* manually add a tag */ + rc = fy_document_tag_directive_add(fyd, "!f!", "tag:example.com,2019:f/"); + ck_assert_int_eq(rc, 0); + + /* build a node with a tag that's already in the document */ + fyn = fy_node_build_from_string(fyd, "!f!whiz frooz\n", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + + /* append it to the root of the document */ + rc = fy_node_sequence_append(fy_document_root(fyd), fyn); + ck_assert_int_eq(rc, 0); + fyn = NULL; + + /* convert to string */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(doc_attach_check) +{ + struct fy_document *fyd; + struct fy_node *fyn, *fyn_seq, *fyn_map; + struct fy_node *fyn_foo, *fyn_foo2, *fyn_bar, *fyn_baz; + struct fy_node_pair *fynp; + int rc; + + /* build document */ + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* create a sequence */ + fyn_seq = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn_seq, NULL); + + /* create a mapping */ + fyn_map = fy_node_create_mapping(fyd); + ck_assert_ptr_ne(fyn_map, NULL); + + /* create a simple scalar node foo */ + fyn_foo = fy_node_build_from_string(fyd, "foo", FY_NT); + ck_assert_ptr_ne(fyn_foo, NULL); + + /* create another simple scalar node bar */ + fyn_bar = fy_node_build_from_string(fyd, "bar", FY_NT); + ck_assert_ptr_ne(fyn_bar, NULL); + + /* create another simple scalar node baz */ + fyn_baz = fy_node_build_from_string(fyd, "baz", FY_NT); + ck_assert_ptr_ne(fyn_baz, NULL); + + /* create a scalar node with the same content as foo */ + fyn_foo2 = fy_node_build_from_string(fyd, "foo", FY_NT); + ck_assert_ptr_ne(fyn_foo2, NULL); + + /* set the root as the sequence */ + rc = fy_document_set_root(fyd, fyn_seq); + ck_assert_int_eq(rc, 0); + + /* should fail since fyn_seq is now attached */ + rc = fy_document_set_root(fyd, fyn_seq); + ck_assert_int_ne(rc, 0); + + /* freeing should fail, since it's attached too */ + rc = fy_node_free(fyn_seq); + ck_assert_int_ne(rc, 0); + + /* append it to the sequence */ + rc = fy_node_sequence_append(fyn_seq, fyn_foo); + ck_assert_int_eq(rc, 0); + + /* freeing should fail, since it's attached to the seq */ + rc = fy_node_free(fyn_foo); + ck_assert_int_ne(rc, 0); + + /* trying to append it to the sequence again should fail */ + rc = fy_node_sequence_append(fyn_seq, fyn_foo); + ck_assert_int_ne(rc, 0); + + /* append the mapping to the sequence */ + rc = fy_node_sequence_append(fyn_seq, fyn_map); + ck_assert_int_eq(rc, 0); + + /* this should fail, since foo is attached to the sequence */ + rc = fy_node_mapping_append(fyn_map, fyn_foo, fyn_bar); + ck_assert_int_ne(rc, 0); + + /* this should be OK, since foo2 is not attached */ + rc = fy_node_mapping_append(fyn_map, fyn_foo2, fyn_bar); + ck_assert_int_eq(rc, 0); + + /* remove foo from the sequence */ + fyn = fy_node_sequence_remove(fyn_seq, fyn_foo); + ck_assert_ptr_eq(fyn, fyn_foo); + + /* trying to append the same key should fail */ + rc = fy_node_mapping_append(fyn_map, fyn_foo, NULL); + ck_assert_int_ne(rc, 0); + + /* append the baz: NULL mapping */ + rc = fy_node_mapping_append(fyn_map, fyn_baz, NULL); + ck_assert_int_eq(rc, 0); + + /* get the baz: null node pair */ + fynp = fy_node_mapping_lookup_pair(fyn_map, fyn_baz); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_ptr_eq(fy_node_pair_key(fynp), fyn_baz); + ck_assert_ptr_eq(fy_node_pair_value(fynp), NULL); + + /* trying to set the same key in the mapping should fail */ + rc = fy_node_pair_set_key(fynp, fyn_foo); + ck_assert_int_ne(rc, 0); + + /* get the foo: bar node pair */ + fynp = fy_node_mapping_lookup_pair(fyn_map, fyn_foo); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_ptr_eq(fy_node_pair_key(fynp), fyn_foo2); + ck_assert_ptr_eq(fy_node_pair_value(fynp), fyn_bar); + + /* we're setting the same key to the mapping, but that's OK + * since the key is replaced */ + rc = fy_node_pair_set_key(fynp, fyn_foo); + ck_assert_int_eq(rc, 0); + + /* fyn_foo has been freed */ + fyn_foo = NULL; + + /* convert to string */ + rc = fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stderr); + ck_assert_int_eq(rc, 0); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(manual_scalar_esc) +{ +#undef MANUAL_SCALAR_ESC +#undef MANUAL_SCALAR_ESC_TXT +#define MANUAL_SCALAR_ESC "\\\"\0\a\b\t\v\f\r\e\xc2\x85\xc2\xa0\xe2\x80\xa8\xe2\x80\xa9" +#define MANUAL_SCALAR_ESC_TXT "\"\\\\\\\"\\0\\a\\b\\t\\v\\f\\r\\e\\N\\_\\L\\P\"" + const char *what = MANUAL_SCALAR_ESC; + size_t what_sz = sizeof(MANUAL_SCALAR_ESC) - 1; + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + const char *buf2; + size_t sz2; + int rc; + + /* build document */ + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* create a manual scalar with all the escapes */ + fyn = fy_node_create_scalar(fyd, what, what_sz); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + fyn = NULL; + + /* emit to a buffer */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + + /* destroy the old document */ + fy_document_destroy(fyd); + fyd = NULL; + + /* verify that the resulting document is in the escaped form */ + ck_assert_str_eq(buf, MANUAL_SCALAR_ESC_TXT "\n"); + + /* now load the result back and verify that it contains exactly the same */ + fyd = fy_document_build_from_string(NULL, buf, FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* get the scalar content */ + buf2 = fy_node_get_scalar(fy_document_root(fyd), &sz2); + ck_assert_ptr_ne(buf2, NULL); + + /* sizes must match */ + ck_assert_int_eq(what_sz, sz2); + + /* and the strings too */ + rc = memcmp(what, buf2, what_sz); + ck_assert_int_eq(rc, 0); + + /* free the document */ + fy_document_destroy(fyd); + fyd = NULL; + + free(buf); +} +END_TEST + +START_TEST(manual_scalar_quoted) +{ +#undef MANUAL_SCALAR_QUOTED +#undef MANUAL_SCALAR_QUOTED_TXT +#define MANUAL_SCALAR_QUOTED "&foo" +#define MANUAL_SCALAR_QUOTED_TXT "\"&foo\"" + const char *what = MANUAL_SCALAR_QUOTED; + size_t what_sz = sizeof(MANUAL_SCALAR_QUOTED) - 1; + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + const char *buf2; + size_t sz2; + int rc; + + /* build document */ + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* create a manual scalar with all the escapes */ + fyn = fy_node_create_scalar(fyd, what, what_sz); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + fyn = NULL; + + /* emit to a buffer */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + + /* destroy the old document */ + fy_document_destroy(fyd); + fyd = NULL; + + /* verify that the resulting document is in the escaped form */ + ck_assert_str_eq(buf, MANUAL_SCALAR_QUOTED_TXT "\n"); + + /* now load the result back and verify that it contains exactly the same */ + fyd = fy_document_build_from_string(NULL, buf, FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* get the scalar content */ + buf2 = fy_node_get_scalar(fy_document_root(fyd), &sz2); + ck_assert_ptr_ne(buf2, NULL); + + /* sizes must match */ + ck_assert_int_eq(what_sz, sz2); + + /* and the strings too */ + rc = memcmp(what, buf2, what_sz); + ck_assert_int_eq(rc, 0); + + /* free the document */ + fy_document_destroy(fyd); + fyd = NULL; + + free(buf); +} +END_TEST + +START_TEST(manual_scalar_copy) +{ +#undef MANUAL_SCALAR_COPY +#define MANUAL_SCALAR_COPY "foo" + const char *what = MANUAL_SCALAR_COPY; + size_t what_sz = sizeof(MANUAL_SCALAR_COPY) - 1; + char *what_copy; + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + /* build document */ + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + what_copy = malloc(what_sz); + ck_assert_ptr_ne(what_copy, NULL); + memcpy(what_copy, what, what_sz); + + /* create a manual scalar with all the escapes */ + fyn = fy_node_create_scalar_copy(fyd, what_copy, what_sz); + ck_assert_ptr_ne(fyn, NULL); + + /* free the data */ + free(what_copy); + + fy_document_set_root(fyd, fyn); + fyn = NULL; + + /* emit to a buffer */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + + /* verify that the resulting document is the one we used + '\n' */ + ck_assert_str_eq(buf, MANUAL_SCALAR_COPY "\n"); + + /* destroy the old document */ + fy_document_destroy(fyd); + fyd = NULL; + + free(buf); + +} +END_TEST + +START_TEST(manual_scalarf) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + /* build document */ + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* create a manual scalar using the printf interface */ + fyn = fy_node_create_scalarf(fyd, "foo%d", 13); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + fyn = NULL; + + /* emit to a buffer */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + + /* verify that the resulting document is the one we used + '\n' */ + ck_assert_str_eq(buf, "foo13" "\n"); + + /* destroy the old document */ + fy_document_destroy(fyd); + fyd = NULL; + + free(buf); +} +END_TEST + +START_TEST(manual_valid_anchor) +{ + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + fyn = fy_node_create_scalar(fyd, "foo", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + + ret = fy_node_sequence_append(fy_document_root(fyd), fyn); + ck_assert_int_eq(ret, 0); + + /* create a valid anchor */ + ret = fy_node_set_anchor(fyn, "foo", FY_NT); + ck_assert_int_eq(ret, 0); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(manual_invalid_anchor) +{ + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + fyn = fy_node_create_scalar(fyd, "foo", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + + ret = fy_node_sequence_append(fy_document_root(fyd), fyn); + ck_assert_int_eq(ret, 0); + + /* create an invalid anchor */ + ret = fy_node_set_anchor(fyn, "*foo", FY_NT); + ck_assert_int_ne(ret, 0); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(manual_anchor_removal) +{ + struct fy_document *fyd; + struct fy_node *fyn; + int ret; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + fyn = fy_node_create_scalar(fyd, "foo", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + + ret = fy_node_sequence_append(fy_document_root(fyd), fyn); + ck_assert_int_eq(ret, 0); + + /* create a valid anchor */ + ret = fy_node_set_anchor(fyn, "foo", FY_NT); + ck_assert_int_eq(ret, 0); + + fprintf(stderr, "---\n# with anchor\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stderr); + + /* should fail (an anchor already exists) */ + ret = fy_node_set_anchor(fyn, "bar", FY_NT); + ck_assert_int_ne(ret, 0); + + /* should succeed */ + ret = fy_node_remove_anchor(fyn); + ck_assert_int_eq(ret, 0); + + fprintf(stderr, "---\n# without anchor\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stderr); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(manual_block_flow_mix) +{ + struct fy_document *fyd; + struct fy_node *fyn_mapping, *fyn_key, *fyn_value; + char *buf; + int ret; + + fyd = fy_document_build_from_string(NULL, "--- &root\n{\n}\n", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_mapping = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_mapping, NULL); + + ck_assert(fy_node_is_mapping(fyn_mapping) == true); + + fyn_key = fy_node_create_scalar(fyd, "key", FY_NT); + ck_assert_ptr_ne(fyn_key, NULL); + + fyn_value = fy_node_build_from_string(fyd, "|\n literal\n", FY_NT); + ck_assert_ptr_ne(fyn_value, NULL); + + ret = fy_node_mapping_append(fyn_mapping, fyn_key, fyn_value); + ck_assert_int_eq(ret, 0); + + /* emit document to buffer */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + + /* destroy the first document */ + fy_document_destroy(fyd); + fyd = NULL; + + /* read the emitted document back */ + fyd = fy_document_build_from_string(NULL, buf, FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* compare with expected result */ + ck_assert_str_eq(fy_node_get_scalar0(fy_node_by_path(fy_document_root(fyd), "/key", FY_NT, FYNWF_DONT_FOLLOW)), "literal\n"); + + /* destroy the second document */ + fy_document_destroy(fyd); + fyd = NULL; + + free(buf); + +} +END_TEST + +START_TEST(alloca_check) +{ + struct fy_document *fyd; + char *buf; + const char *abuf; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "{ " + "foo: 10, bar : [ ten, 20 ], baz:{ frob: boo, deep: { deeper: yet } }, " + "}", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* fy_emit_document_to_string*() */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + abuf = fy_emit_document_to_string_alloca(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(abuf, NULL); + ck_assert_str_eq(buf, abuf); + free(buf); + + /* fy_emit_node_to_string*() */ + buf = fy_emit_node_to_string(fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW), FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + abuf = fy_emit_node_to_string_alloca(fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW), FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(abuf, NULL); + ck_assert_str_eq(buf, abuf); + free(buf); + + /* path check eq */ + buf = fy_node_get_path(fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW)); + ck_assert_ptr_ne(buf, NULL); + ck_assert_str_eq(buf, "/foo"); + abuf = fy_node_get_path_alloca(fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW)); + ck_assert_ptr_ne(abuf, NULL); + ck_assert_str_eq(abuf, "/foo"); + ck_assert_str_eq(buf, abuf); + free(buf); + + /* check that a bad path is "" */ + abuf = fy_node_get_path_alloca(NULL); + ck_assert_ptr_ne(abuf, NULL); + ck_assert_str_eq(abuf, ""); + + fy_document_destroy(fyd); + +} +END_TEST + +START_TEST(scanf_check) +{ + struct fy_document *fyd; + struct fy_node *fyn_root; + int ret, ival; + char sval[256]; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "{ " + "foo: 10, bar : 20, baz:{ frob: boo }, " + "frooz: [ 1, { key: value }, three ]" + "}", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_root = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + + /* check scanf accesses to scalars */ + ret = fy_node_scanf(fyn_root, "/foo %d", &ival); + ck_assert_int_eq(ret, 1); + ck_assert_int_eq(ival, 10); + + ret = fy_node_scanf(fyn_root, "/bar %d", &ival); + ck_assert_int_eq(ret, 1); + ck_assert_int_eq(ival, 20); + + ret = fy_node_scanf(fyn_root, "/baz/frob %s", sval); + ck_assert_int_eq(ret, 1); + ck_assert_str_eq(sval, "boo"); + + ret = fy_node_scanf(fyn_root, "/frooz/0 %d", &ival); + ck_assert_int_eq(ret, 1); + ck_assert_int_eq(ival, 1); + + ret = fy_node_scanf(fyn_root, "/frooz/1/key %s", sval); + ck_assert_int_eq(ret, 1); + ck_assert_str_eq(sval, "value"); + + ret = fy_node_scanf(fyn_root, "/frooz/2 %s", sval); + ck_assert_int_eq(ret, 1); + ck_assert_str_eq(sval, "three"); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(token_test) +{ + struct fy_document *fyd; + struct fy_node *fyn_sequence, *fyn_mapping, *fyn_scalar; + struct fy_token *fyn_sequence_start, *fyn_sequence_end, + *fyn_mapping_start, *fyn_mapping_end, *fyn_scalar_token; + struct fy_parse_cfg cfg = { + .flags = FYPCF_PARSE_COMMENTS + }; + + /* build document */ + + fyd = fy_document_build_from_string(&cfg, "- name: key\n" + " value: value\n", FY_NT); + + + /* Root (sequence) */ + fyn_sequence = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_sequence, NULL); + + fyn_sequence_start = fy_node_get_start_token(fyn_sequence); + ck_assert_ptr_ne(fyn_sequence_start, NULL); + + ck_assert_ptr_ne(fy_token_start_mark(fyn_sequence_start), NULL); + + ck_assert_int_eq(fy_token_start_mark(fyn_sequence_start)->line, 0); + ck_assert_int_eq(fy_token_start_mark(fyn_sequence_start)->column, 0); + + fyn_sequence_end = fy_node_get_end_token(fyn_sequence); + ck_assert_ptr_ne(fyn_sequence_end, NULL); + + ck_assert_int_eq(fy_token_end_mark(fyn_sequence_end)->line, 2); + ck_assert_int_eq(fy_token_end_mark(fyn_sequence_end)->column, 0); + + /* Mapping (sequence item) */ + fyn_mapping = fy_node_sequence_get_by_index(fyn_sequence, 0); + ck_assert_ptr_ne(fyn_mapping, NULL); + + fyn_mapping_start = fy_node_get_start_token(fyn_mapping); + ck_assert_ptr_ne(fyn_mapping_start, NULL); + + ck_assert_ptr_ne(fy_token_start_mark(fyn_mapping_start), NULL); + + ck_assert_int_eq(fy_token_start_mark(fyn_mapping_start)->line, 0); + ck_assert_int_eq(fy_token_start_mark(fyn_mapping_start)->column, 2); + + fyn_mapping_end = fy_node_get_end_token(fyn_mapping); + ck_assert_ptr_ne(fyn_mapping_end, NULL); + + ck_assert_ptr_ne(fy_token_start_mark(fyn_mapping_end), NULL); + + ck_assert_int_eq(fy_token_start_mark(fyn_mapping_end)->line, 2); + ck_assert_int_eq(fy_token_start_mark(fyn_mapping_end)->column, 0); + + /* Scalar (key) */ + fyn_scalar = fy_node_pair_key(fy_node_mapping_get_by_index(fyn_mapping, 0)); + ck_assert_ptr_ne(fyn_scalar, NULL); + + fyn_scalar_token = fy_node_get_start_token(fyn_scalar); + ck_assert_ptr_ne(fyn_scalar_token, NULL); + ck_assert_ptr_eq(fyn_scalar_token, fy_node_get_end_token(fyn_scalar)); + + ck_assert_ptr_ne(fy_token_start_mark(fyn_scalar_token), NULL); + ck_assert_int_eq(fy_token_start_mark(fyn_scalar_token)->line, 0); + ck_assert_int_eq(fy_token_start_mark(fyn_scalar_token)->column, 2); + + ck_assert_ptr_ne(fy_token_end_mark(fyn_scalar_token), NULL); + ck_assert_int_eq(fy_token_end_mark(fyn_scalar_token)->line, 0); + ck_assert_int_eq(fy_token_end_mark(fyn_scalar_token)->column, 6); + + /* cleanup */ + fy_document_destroy(fyd); +} +END_TEST + +TCase *libfyaml_case_core(void) +{ + TCase *tc; + + tc = tcase_create("core"); + + tcase_add_test(tc, doc_build_simple); + tcase_add_test(tc, doc_build_parse_check); + tcase_add_test(tc, doc_build_scalar); + tcase_add_test(tc, doc_build_sequence); + tcase_add_test(tc, doc_build_mapping); + + tcase_add_test(tc, doc_path_access); + tcase_add_test(tc, doc_path_node); + tcase_add_test(tc, doc_path_parent); + tcase_add_test(tc, doc_short_path); + tcase_add_test(tc, doc_scalar_path); + tcase_add_test(tc, doc_scalar_path_array); + + tcase_add_test(tc, doc_nearest_anchor); + tcase_add_test(tc, doc_references); + tcase_add_test(tc, doc_nearest_child_of); + + tcase_add_test(tc, doc_create_empty_seq1); + tcase_add_test(tc, doc_create_empty_seq2); + tcase_add_test(tc, doc_create_empty_map1); + tcase_add_test(tc, doc_create_empty_map2); + + tcase_add_test(tc, doc_create_test_seq1); + tcase_add_test(tc, doc_create_test_map1); + + tcase_add_test(tc, doc_insert_remove_seq); + tcase_add_test(tc, doc_insert_remove_map); + + tcase_add_test(tc, doc_sort); + tcase_add_test(tc, doc_insert_at); + + tcase_add_test(tc, doc_join_scalar_to_scalar); + tcase_add_test(tc, doc_join_scalar_to_map); + tcase_add_test(tc, doc_join_scalar_to_seq); + + tcase_add_test(tc, doc_join_map_to_scalar); + tcase_add_test(tc, doc_join_map_to_seq); + tcase_add_test(tc, doc_join_map_to_map); + + tcase_add_test(tc, doc_join_seq_to_scalar); + tcase_add_test(tc, doc_join_seq_to_seq); + tcase_add_test(tc, doc_join_seq_to_map); + + tcase_add_test(tc, doc_join_tags); + + tcase_add_test(tc, doc_build_with_tags); + + tcase_add_test(tc, doc_attach_check); + + tcase_add_test(tc, manual_scalar_esc); + tcase_add_test(tc, manual_scalar_quoted); + tcase_add_test(tc, manual_scalar_copy); + tcase_add_test(tc, manual_scalarf); + + tcase_add_test(tc, manual_valid_anchor); + tcase_add_test(tc, manual_invalid_anchor); + tcase_add_test(tc, manual_anchor_removal); + + tcase_add_test(tc, manual_block_flow_mix); + + tcase_add_test(tc, alloca_check); + + tcase_add_test(tc, scanf_check); + + tcase_add_test(tc, token_test); + + return tc; +} diff --git a/test/libfyaml-test-emit.c b/test/libfyaml-test-emit.c new file mode 100644 index 0000000..6b3e8f4 --- /dev/null +++ b/test/libfyaml-test-emit.c @@ -0,0 +1,124 @@ +/* + * libfyaml-test-emit.c - libfyaml test public emitter interface + * + * Copyright (c) 2021 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +struct test_emitter_data { + struct fy_emitter *emit; + struct fy_emitter_cfg cfg; + size_t alloc; + size_t count; + char *buf; +}; + +static int collect_output(struct fy_emitter *emit, enum fy_emitter_write_type type, + const char *str, int len, void *userdata) +{ + struct test_emitter_data *data = userdata; + char *newbuf; + size_t alloc, need; + + need = data->count + len + 1; + alloc = data->alloc; + if (!alloc) + alloc = 512; /* start at 512 bytes and double */ + while (need > alloc) + alloc <<= 1; + + if (alloc > data->alloc) { + newbuf = realloc(data->buf, alloc); + if (!newbuf) + return -1; + data->buf = newbuf; + data->alloc = alloc; + } + assert(data->alloc >= need); + memcpy(data->buf + data->count, str, len); + data->count += len; + *(char *)(data->buf + data->count) = '\0'; + data->count++; + + return len; +} + +struct fy_emitter *setup_test_emitter(struct test_emitter_data *data) +{ + memset(data, 0, sizeof(*data)); + data->cfg.output = collect_output; + data->cfg.userdata = data; + data->cfg.flags = FYECF_DEFAULT; + data->emit = fy_emitter_create(&data->cfg); + return data->emit; + +} + +static void cleanup_test_emitter(struct test_emitter_data *data) +{ + if (data->emit) + fy_emitter_destroy(data->emit); + if (data->buf) + free(data->buf); +} + +START_TEST(emit_simple) +{ + struct test_emitter_data data; + struct fy_emitter *emit; + int rc; + + emit = setup_test_emitter(&data); + ck_assert_ptr_ne(emit, NULL); + + rc = fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_START)); + ck_assert_int_eq(rc, 0); + + rc = fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_START, true, NULL, NULL)); + ck_assert_int_eq(rc, 0); + + rc = fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "simple", FY_NT, NULL, NULL)); + ck_assert_int_eq(rc, 0); + + rc = fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_END, true, NULL, NULL)); + ck_assert_int_eq(rc, 0); + + rc = fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_END)); + ck_assert_int_eq(rc, 0); + + ck_assert_ptr_ne(data.buf, NULL); + + /* the contents must be 'simple' (without a newline) */ + ck_assert_str_eq(data.buf, "simple"); + + cleanup_test_emitter(&data); +} +END_TEST + +TCase *libfyaml_case_emit(void) +{ + TCase *tc; + + tc = tcase_create("emit"); + + tcase_add_test(tc, emit_simple); + + return tc; +} diff --git a/test/libfyaml-test-meta.c b/test/libfyaml-test-meta.c new file mode 100644 index 0000000..7bf096b --- /dev/null +++ b/test/libfyaml-test-meta.c @@ -0,0 +1,199 @@ +/* + * libfyaml-test-meta.c - libfyaml meta testing harness + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +START_TEST(meta_basic) +{ + struct fy_document *fyd; + struct fy_node *fyn_root; + void *meta; + int meta_value; + int ret; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "100", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* check that root exist */ + fyn_root = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + + /* compare with expected result */ + ck_assert_str_eq(fy_node_get_scalar0(fyn_root), "100"); + + /* check that no meta at start */ + meta = fy_node_get_meta(fyn_root); + ck_assert_ptr_eq(meta, NULL); + + /* set a simple value meta */ + ret = fy_node_set_meta(fyn_root, (void *)(uintptr_t)100); + ck_assert_int_eq(ret, 0); + + /* retrieve it and verify */ + meta = fy_node_get_meta(fyn_root); + meta_value = (int)(uintptr_t)meta; + ck_assert_int_eq(meta_value, 100); + + /* clear the meta */ + fy_node_clear_meta(fyn_root); + + /* check that no meta exists now */ + meta = fy_node_get_meta(fyn_root); + ck_assert_ptr_eq(meta, NULL); + + /* setting new meta now should work */ + ret = fy_node_set_meta(fyn_root, (void *)(uintptr_t)200); + ck_assert_int_eq(ret, 0); + + /* retrieve it and verify */ + meta = fy_node_get_meta(fyn_root); + meta_value = (int)(uintptr_t)meta; + ck_assert_int_eq(meta_value, 200); + + /* setting new meta override */ + ret = fy_node_set_meta(fyn_root, (void *)(uintptr_t)201); + ck_assert_int_eq(ret, 0); + + /* retrieve it and verify */ + meta = fy_node_get_meta(fyn_root); + meta_value = (int)(uintptr_t)meta; + ck_assert_int_eq(meta_value, 201); + + fy_document_destroy(fyd); +} +END_TEST + +static void test_meta_clear_cb(struct fy_node *fyn, void *meta, void *user) +{ + int meta_value = (int)(uintptr_t)meta; + int *clear_counter_p = user; + + *clear_counter_p += meta_value; +} + +START_TEST(meta_clear_cb) +{ + struct fy_document *fyd; + struct fy_node *fyn_root, *fyn_0, *fyn_1; + int ret; + int clear_counter; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "[ 100, 101 ]", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* check that root exist */ + fyn_root = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + + /* register the meta clear callback */ + clear_counter = 0; + ret = fy_document_register_meta(fyd, test_meta_clear_cb, &clear_counter); + + /* setting new meta */ + ret = fy_node_set_meta(fyn_root, (void *)(uintptr_t)1000); + ck_assert_int_eq(ret, 0); + + /* the clear counter must be unchanged */ + ck_assert_int_eq(clear_counter, 0); + + /* get the two items of the sequence and assign meta */ + fyn_0 = fy_node_by_path(fyn_root, "/0", (size_t)-1, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_0, NULL); + ret = fy_node_set_meta(fyn_root, (void *)(uintptr_t)100); + ck_assert_int_eq(ret, 0); + + fyn_1 = fy_node_by_path(fyn_root, "/1", (size_t)-1, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_1, NULL); + ret = fy_node_set_meta(fyn_root, (void *)(uintptr_t)200); + ck_assert_int_eq(ret, 0); + + /* destroy the document */ + fy_document_destroy(fyd); + + /* the end result of the counter must be the sum of all */ + ck_assert_int_eq(clear_counter, 1000 + 100 + 200); +} +END_TEST + +START_TEST(meta_unregister) +{ + struct fy_document *fyd; + struct fy_node *fyn_root, *fyn_0, *fyn_1; + int ret; + int clear_counter; + + /* build document */ + fyd = fy_document_build_from_string(NULL, "[ 100, 101 ]", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* check that root exist */ + fyn_root = fy_document_root(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + + /* register the meta clear callback */ + clear_counter = 0; + ret = fy_document_register_meta(fyd, test_meta_clear_cb, &clear_counter); + + /* setting new meta */ + ret = fy_node_set_meta(fyn_root, (void *)(uintptr_t)1000); + ck_assert_int_eq(ret, 0); + + /* the clear counter must be unchanged */ + ck_assert_int_eq(clear_counter, 0); + + /* get the two items of the sequence and assign meta */ + fyn_0 = fy_node_by_path(fyn_root, "/0", (size_t)-1, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_0, NULL); + ret = fy_node_set_meta(fyn_root, (void *)(uintptr_t)100); + ck_assert_int_eq(ret, 0); + + fyn_1 = fy_node_by_path(fyn_root, "/1", (size_t)-1, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_1, NULL); + ret = fy_node_set_meta(fyn_root, (void *)(uintptr_t)200); + ck_assert_int_eq(ret, 0); + + /* unregister */ + fy_document_unregister_meta(fyd); + + /* the counter must be the sum of all after unregistering */ + ck_assert_int_eq(clear_counter, 1000 + 100 + 200); + + /* destroy the document */ + fy_document_destroy(fyd); + +} +END_TEST + +TCase *libfyaml_case_meta(void) +{ + TCase *tc; + + tc = tcase_create("meta"); + + tcase_add_test(tc, meta_basic); + tcase_add_test(tc, meta_clear_cb); + tcase_add_test(tc, meta_unregister); + + return tc; +} diff --git a/test/libfyaml-test-parser.c b/test/libfyaml-test-parser.c new file mode 100644 index 0000000..13927bd --- /dev/null +++ b/test/libfyaml-test-parser.c @@ -0,0 +1,2165 @@ +/* + * libfyaml-test-parser.c - libfyaml parser testing (extracted from libfyaml-parser.c) + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include + +#include +#include "fy-parse.h" +#include "fy-utf8.h" + +/* Test: Mapping iterator (forward and reverse) */ +START_TEST(parser_mapping_iterator) +{ + struct fy_document *fyd; + struct fy_node_pair *fynp; + int count; + void *iter; + + /* Build a mapping with multiple entries including complex keys */ + fyd = fy_document_build_from_string(NULL, + "{ foo: 10, bar: 20, baz: [100, 101], [frob, 1]: boo }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Verify count */ + count = fy_node_mapping_item_count(fy_document_root(fyd)); + ck_assert_int_eq(count, 4); + + /* Test forward iterator */ + iter = NULL; + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "foo"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "10"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "bar"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "20"); + + /* Test reverse iterator */ + iter = NULL; + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + /* Last item should be the complex key */ + + /* Test index-based access */ + fynp = fy_node_mapping_get_by_index(fy_document_root(fyd), 0); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "foo"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "10"); + + fynp = fy_node_mapping_get_by_index(fy_document_root(fyd), 1); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "bar"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Mapping key lookup */ +START_TEST(parser_mapping_key_lookup) +{ + struct fy_document *fyd; + struct fy_node *fyn; + + fyd = fy_document_build_from_string(NULL, + "{ foo: 10, bar: 20, baz: [100, 101], [frob, 1]: boo }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Lookup simple keys */ + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fyd), "foo", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fyd), "bar", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "20"); + + /* Lookup key with sequence value */ + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fyd), "baz", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_sequence(fyn)); + + /* Lookup complex key */ + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fyd), "[ frob, 1 ]", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "boo"); + + /* Lookup non-existent key */ + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fyd), "nonexistent", FY_NT); + ck_assert_ptr_eq(fyn, NULL); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Path-based node queries */ +START_TEST(parser_path_queries) +{ + struct fy_document *fyd; + struct fy_node *fyn; + + fyd = fy_document_build_from_string(NULL, + "{ foo: 10, bar: 20, baz:{ frob: boo }, " + "frooz: [ seq1, { key: value} ], \"zero\\0zero\": 0, " + "{ key2: value2 }: { key3: value3 } }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Query root */ + fyn = fy_node_by_path(fy_document_root(fyd), "/", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + /* Query simple keys */ + fyn = fy_node_by_path(fy_document_root(fyd), "foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + fyn = fy_node_by_path(fy_document_root(fyd), "bar", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "20"); + + /* Query nested path */ + fyn = fy_node_by_path(fy_document_root(fyd), "baz/frob", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "boo"); + + /* Query sequence elements by index */ + fyn = fy_node_by_path(fy_document_root(fyd), "/frooz/[0]", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "seq1"); + + /* Query nested in sequence */ + fyn = fy_node_by_path(fy_document_root(fyd), "/frooz/[1]/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "value"); + + /* Query with quoted key */ + fyn = fy_node_by_path(fy_document_root(fyd), "\"foo\"", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + /* Query with null byte in key */ + fyn = fy_node_by_path(fy_document_root(fyd), "\"zero\\0zero\"", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "0"); + + /* Query complex key mapping */ + fyn = fy_node_by_path(fy_document_root(fyd), "/{ key2: value2 }", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + /* Query nested in complex key */ + fyn = fy_node_by_path(fy_document_root(fyd), "/{ key2: value2 }/key3", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "value3"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node path generation */ +START_TEST(parser_node_path_generation) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *path; + + fyd = fy_document_build_from_string(NULL, + "{ foo: 10, frooz: [ seq1, { key: value} ], " + "{ key2: value2 }: { key3: value3 } }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Get path of root */ + fyn = fy_node_by_path(fy_document_root(fyd), "/", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + path = fy_node_get_path(fyn); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "/"); + free(path); + + /* Get path of simple key */ + fyn = fy_node_by_path(fy_document_root(fyd), "/frooz", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + path = fy_node_get_path(fyn); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "/frooz"); + free(path); + + /* Get path of sequence element */ + fyn = fy_node_by_path(fy_document_root(fyd), "/frooz/[0]", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + path = fy_node_get_path(fyn); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "/frooz/0"); + free(path); + + /* Get path of nested element in complex key */ + fyn = fy_node_by_path(fy_document_root(fyd), "/{ key2: value2 }/key3", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + path = fy_node_get_path(fyn); + ck_assert_ptr_ne(path, NULL); + /* Path should be valid */ + ck_assert_ptr_ne(path, NULL); + free(path); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation from scratch */ +START_TEST(parser_node_creation_scalar) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + /* Create document and scalar node */ + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_scalar(fyd, "foo", 3); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* Emit and verify */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + ck_assert_str_eq(buf, "foo\n"); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation - multiline scalar */ +START_TEST(parser_node_creation_multiline_scalar) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_scalar(fyd, "foo\nfoo", 7); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* Emit and verify - multiline scalars should be emitted with literal or folded style */ + buf = fy_emit_document_to_string(fyd, 0); + ck_assert_ptr_ne(buf, NULL); + /* Just verify it emits successfully and contains the content */ + ck_assert(strstr(buf, "foo") != NULL); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation - empty sequence */ +START_TEST(parser_node_creation_empty_sequence) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* Emit and verify */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + ck_assert_str_eq(buf, "[]\n"); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation - empty mapping */ +START_TEST(parser_node_creation_empty_mapping) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_mapping(fyd); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* Emit and verify */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + ck_assert_str_eq(buf, "{}\n"); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation - populated sequence */ +START_TEST(parser_node_creation_populated_sequence) +{ + struct fy_document *fyd; + struct fy_node *fyn, *fyn_seq; + int ret; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn_seq = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn_seq, NULL); + + /* Append elements */ + fyn = fy_node_create_scalar(fyd, "foo", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ret = fy_node_sequence_append(fyn_seq, fyn); + ck_assert_int_eq(ret, 0); + + fyn = fy_node_create_scalar(fyd, "bar", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ret = fy_node_sequence_append(fyn_seq, fyn); + ck_assert_int_eq(ret, 0); + + fyn = fy_node_build_from_string(fyd, "{ baz: frooz }", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ret = fy_node_sequence_append(fyn_seq, fyn); + ck_assert_int_eq(ret, 0); + + fy_document_set_root(fyd, fyn_seq); + + /* Verify count */ + ck_assert_int_eq(fy_node_sequence_item_count(fyn_seq), 3); + + /* Verify content */ + fyn = fy_node_sequence_get_by_index(fyn_seq, 0); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "foo"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 1); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "bar"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 2); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation - populated mapping */ +START_TEST(parser_node_creation_populated_mapping) +{ + struct fy_document *fyd; + struct fy_node *fyn, *fyn_key, *fyn_val, *fyn_map; + int ret; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn_map = fy_node_create_mapping(fyd); + ck_assert_ptr_ne(fyn_map, NULL); + + /* Append key-value pairs */ + fyn_key = fy_node_create_scalar(fyd, "foo", FY_NT); + ck_assert_ptr_ne(fyn_key, NULL); + fyn_val = fy_node_create_scalar(fyd, "10", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + ret = fy_node_mapping_append(fyn_map, fyn_key, fyn_val); + ck_assert_int_eq(ret, 0); + + fyn_key = fy_node_create_scalar(fyd, "bar", FY_NT); + ck_assert_ptr_ne(fyn_key, NULL); + fyn_val = fy_node_build_from_string(fyd, "[ 1, 2, 3 ]", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + ret = fy_node_mapping_append(fyn_map, fyn_key, fyn_val); + ck_assert_int_eq(ret, 0); + + fy_document_set_root(fyd, fyn_map); + + /* Verify count */ + ck_assert_int_eq(fy_node_mapping_item_count(fyn_map), 2); + + /* Verify lookup */ + fyn = fy_node_mapping_lookup_by_string(fyn_map, "foo", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + fyn = fy_node_mapping_lookup_by_string(fyn_map, "bar", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_sequence(fyn)); + ck_assert_int_eq(fy_node_sequence_item_count(fyn), 3); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Build node from string within document */ +START_TEST(parser_build_node_from_string) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* Build a complex node from string */ + fyn = fy_node_build_from_string(fyd, "{ }", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + fy_document_set_root(fyd, fyn); + + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + ck_assert_str_eq(buf, "{}\n"); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Sequence operations - reverse index access */ +START_TEST(parser_sequence_negative_index) +{ + struct fy_document *fyd; + struct fy_node *fyn; + + fyd = fy_document_build_from_string(NULL, "[ first, second, third ]", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Access from end using negative indices */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -1); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "third"); + + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -2); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "second"); + + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -3); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "first"); + + /* Out of bounds negative index */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -4); + ck_assert_ptr_eq(fyn, NULL); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Complex nested structure */ +START_TEST(parser_complex_nested_structure) +{ + struct fy_document *fyd; + struct fy_node *fyn; + int count; + + /* Build a complex nested structure */ + fyd = fy_document_build_from_string(NULL, + "root:\n" + " level1:\n" + " level2:\n" + " - item1\n" + " - item2\n" + " - key: value\n" + " nested: data\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Navigate to nested sequence */ + fyn = fy_node_by_path(fy_document_root(fyd), "root/level1/level2", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_sequence(fyn)); + + count = fy_node_sequence_item_count(fyn); + ck_assert_int_eq(count, 3); + + /* Check first scalar item */ + fyn = fy_node_by_path(fy_document_root(fyd), "root/level1/level2/[0]", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "item1"); + + /* Check nested mapping in sequence */ + fyn = fy_node_by_path(fy_document_root(fyd), "root/level1/level2/[2]/nested", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "data"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Anchor and alias resolution */ +START_TEST(parser_anchor_alias_resolution) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + int rc; + + /* Build document with anchor and alias */ + fyd = fy_document_build_from_string(NULL, + "base: &base\n" + " name: this-is-a-name\n" + " value: 42\n" + "copy: *base\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Before resolution, alias should exist */ + fyn = fy_node_by_path(fy_document_root(fyd), "copy", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_alias(fyn)); + + /* Resolve the document */ + rc = fy_document_resolve(fyd); + ck_assert_int_eq(rc, 0); + + /* After resolution, we should be able to access through the alias */ + fyn = fy_node_by_path(fy_document_root(fyd), "copy/name", FY_NT, FYNWF_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "this-is-a-name"); + + /* Emit resolved document */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document insertion at path */ +START_TEST(parser_document_insert_at) +{ + struct fy_document *fyd; + struct fy_node *fyn, *fyn_inserted; + int rc; + + /* Create base document */ + fyd = fy_document_build_from_string(NULL, + "base:\n" + " name: original\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Build a mapping node to insert (key: value) */ + fyn = fy_node_buildf(fyd, "new-key: inserted-value"); + ck_assert_ptr_ne(fyn, NULL); + + /* Insert the mapping at /base (merges into existing mapping) */ + rc = fy_document_insert_at(fyd, "/base", FY_NT, fyn); + ck_assert_int_eq(rc, 0); + + /* Verify insertion */ + fyn_inserted = fy_node_by_path(fy_document_root(fyd), "/base/new-key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_inserted, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn_inserted), "inserted-value"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document emit with different flags */ +START_TEST(parser_document_emit_flags) +{ + struct fy_document *fyd; + char *buf; + + /* Build test document */ + fyd = fy_document_build_from_string(NULL, + "{ z: 1, a: 2, m: 3 }", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Emit with default flags */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + ck_assert(strstr(buf, "z") != NULL); + ck_assert(strstr(buf, "a") != NULL); + ck_assert(strstr(buf, "m") != NULL); + free(buf); + + /* Emit with sorted keys */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE | FYECF_SORT_KEYS); + ck_assert_ptr_ne(buf, NULL); + /* In sorted output, 'a' should come before 'z' */ + ck_assert(strstr(buf, "a") < strstr(buf, "z")); + ck_assert(strstr(buf, "m") < strstr(buf, "z")); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Multi-document stream parsing */ +START_TEST(parser_multi_document_stream) +{ + struct fy_parser fyp_data, *fyp = &fyp_data; + struct fy_parse_cfg cfg = { + .flags = FYPCF_DEFAULT_DOC, + }; + struct fy_document *fyd; + int count; + int rc; + + /* Setup parser */ + rc = fy_parse_setup(fyp, &cfg); + ck_assert_int_eq(rc, 0); + + /* Create multi-document input */ + const char *yaml_multi = + "---\n" + "doc: 1\n" + "---\n" + "doc: 2\n" + "---\n" + "doc: 3\n"; + + /* Add input */ + struct fy_input_cfg fyic = { + .type = fyit_memory, + .memory.data = yaml_multi, + .memory.size = strlen(yaml_multi), + }; + rc = fy_parse_input_append(fyp, &fyic); + ck_assert_int_eq(rc, 0); + + /* Parse all documents */ + count = 0; + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + count++; + + /* Verify document content */ + struct fy_node *fyn = fy_node_by_path(fy_document_root(fyd), "doc", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + + int doc_num = atoi(fy_node_get_scalar0(fyn)); + ck_assert_int_eq(doc_num, count); + + fy_parse_document_destroy(fyp, fyd); + } + + ck_assert_int_eq(count, 3); + + fy_parse_cleanup(fyp); +} +END_TEST + +/* Test: Empty document handling */ +START_TEST(parser_empty_document) +{ + struct fy_document *fyd; + char *buf; + + /* Null document (YAML null/empty) */ + fyd = fy_document_build_from_string(NULL, "null", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Should have scalar null root */ + ck_assert_ptr_ne(fy_document_root(fyd), NULL); + + /* Should emit */ + buf = fy_emit_document_to_string(fyd, 0); + ck_assert_ptr_ne(buf, NULL); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document with comments (requires FYPCF_PARSE_COMMENTS) */ +START_TEST(parser_document_with_comments) +{ + struct fy_parse_cfg cfg = { + .flags = FYPCF_PARSE_COMMENTS, + }; + struct fy_document *fyd; + + /* Build document with comments */ + fyd = fy_document_build_from_string(&cfg, + "# Top comment\n" + "key: value # Right comment\n" + "# Bottom comment\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Verify content (comments should be preserved in structure) */ + struct fy_node *fyn = fy_node_by_path(fy_document_root(fyd), "key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "value"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Sequence append and prepend */ +START_TEST(parser_sequence_append_prepend) +{ + struct fy_document *fyd; + struct fy_node *fyn_seq, *fyn; + int rc; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* Create sequence */ + fyn_seq = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn_seq, NULL); + fy_document_set_root(fyd, fyn_seq); + + /* Append items */ + fyn = fy_node_create_scalar(fyd, "second", FY_NT); + rc = fy_node_sequence_append(fyn_seq, fyn); + ck_assert_int_eq(rc, 0); + + /* Prepend item */ + fyn = fy_node_create_scalar(fyd, "first", FY_NT); + rc = fy_node_sequence_prepend(fyn_seq, fyn); + ck_assert_int_eq(rc, 0); + + /* Append another */ + fyn = fy_node_create_scalar(fyd, "third", FY_NT); + rc = fy_node_sequence_append(fyn_seq, fyn); + ck_assert_int_eq(rc, 0); + + /* Verify order */ + fyn = fy_node_sequence_get_by_index(fyn_seq, 0); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "first"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 1); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "second"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 2); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "third"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Mapping prepend */ +START_TEST(parser_mapping_prepend) +{ + struct fy_document *fyd; + struct fy_node *fyn_map, *fyn_key, *fyn_val; + struct fy_node_pair *fynp; + int rc; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn_map = fy_node_create_mapping(fyd); + ck_assert_ptr_ne(fyn_map, NULL); + fy_document_set_root(fyd, fyn_map); + + /* Append a pair */ + fyn_key = fy_node_create_scalar(fyd, "second", FY_NT); + fyn_val = fy_node_create_scalar(fyd, "2", FY_NT); + rc = fy_node_mapping_append(fyn_map, fyn_key, fyn_val); + ck_assert_int_eq(rc, 0); + + /* Prepend a pair */ + fyn_key = fy_node_create_scalar(fyd, "first", FY_NT); + fyn_val = fy_node_create_scalar(fyd, "1", FY_NT); + rc = fy_node_mapping_prepend(fyn_map, fyn_key, fyn_val); + ck_assert_int_eq(rc, 0); + + /* Verify order */ + fynp = fy_node_mapping_get_by_index(fyn_map, 0); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "first"); + + fynp = fy_node_mapping_get_by_index(fyn_map, 1); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "second"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node removal from sequence */ +START_TEST(parser_sequence_remove) +{ + struct fy_document *fyd; + struct fy_node *fyn_seq, *fyn; + int count; + + /* Build a sequence */ + fyd = fy_document_build_from_string(NULL, "[ a, b, c, d ]", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_seq = fy_document_root(fyd); + ck_assert_int_eq(fy_node_sequence_item_count(fyn_seq), 4); + + /* Remove middle element */ + fyn = fy_node_sequence_get_by_index(fyn_seq, 1); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "b"); + + fyn = fy_node_sequence_remove(fyn_seq, fyn); + ck_assert_ptr_ne(fyn, NULL); + fy_node_free(fyn); + + /* Verify count and order */ + count = fy_node_sequence_item_count(fyn_seq); + ck_assert_int_eq(count, 3); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 0); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "a"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 1); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "c"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 2); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "d"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node removal from mapping */ +START_TEST(parser_mapping_remove) +{ + struct fy_document *fyd; + struct fy_node *fyn_map, *fyn_key, *fyn_val; + int count; + + /* Build a mapping */ + fyd = fy_document_build_from_string(NULL, "{ a: 1, b: 2, c: 3 }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_map = fy_document_root(fyd); + ck_assert_int_eq(fy_node_mapping_item_count(fyn_map), 3); + + /* Remove by key */ + fyn_key = fy_node_build_from_string(fyd, "b", FY_NT); + ck_assert_ptr_ne(fyn_key, NULL); + + fyn_val = fy_node_mapping_remove_by_key(fyn_map, fyn_key); + ck_assert_ptr_ne(fyn_val, NULL); + fy_node_free(fyn_val); + + /* Verify count */ + count = fy_node_mapping_item_count(fyn_map); + ck_assert_int_eq(count, 2); + + /* Verify 'b' is gone */ + fyn_val = fy_node_mapping_lookup_by_string(fyn_map, "b", FY_NT); + ck_assert_ptr_eq(fyn_val, NULL); + + /* Verify others remain */ + fyn_val = fy_node_mapping_lookup_by_string(fyn_map, "a", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + + fyn_val = fy_node_mapping_lookup_by_string(fyn_map, "c", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document iterator functionality */ +START_TEST(parser_document_iterator) +{ + struct fy_document *fyd; + struct fy_document_iterator *fydi; + struct fy_node *fyn; + int count; + + /* Build test document with nested structure */ + fyd = fy_document_build_from_string(NULL, + "root:\n" + " scalar: value\n" + " seq:\n" + " - item1\n" + " - item2\n" + " map:\n" + " key: val\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Create iterator */ + fydi = fy_document_iterator_create(); + ck_assert_ptr_ne(fydi, NULL); + + /* Start iteration from root */ + fy_document_iterator_node_start(fydi, fy_document_root(fyd)); + + /* Count all nodes */ + count = 0; + while ((fyn = fy_document_iterator_node_next(fydi)) != NULL) { + count++; + + /* Verify node type detection works */ + if (fy_node_is_scalar(fyn)) { + const char *text; + size_t len; + text = fy_node_get_scalar(fyn, &len); + ck_assert_ptr_ne(text, NULL); + } else if (fy_node_is_sequence(fyn)) { + /* Verify it's a sequence */ + ck_assert(fy_node_is_sequence(fyn)); + } else if (fy_node_is_mapping(fyn)) { + /* Verify it's a mapping */ + ck_assert(fy_node_is_mapping(fyn)); + } + } + + /* We should have iterated through multiple nodes */ + ck_assert_int_gt(count, 0); + + /* Cleanup */ + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document iterator with key detection */ +START_TEST(parser_document_iterator_key_detection) +{ + struct fy_document *fyd; + struct fy_document_iterator *fydi; + struct fy_node *fyn; + + /* Build mapping document */ + fyd = fy_document_build_from_string(NULL, + "key1: value1\n" + "key2: value2\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fydi = fy_document_iterator_create(); + ck_assert_ptr_ne(fydi, NULL); + + fy_document_iterator_node_start(fydi, fy_document_root(fyd)); + + /* Iterate and check for key nodes */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_ne(fyn, NULL); + /* must be a mapping */ + ck_assert(fy_node_is_mapping(fyn)); + /* must be the root */ + ck_assert_ptr_eq(fyn, fy_document_root(fyd)); + + /* get the first key */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_ne(fyn, NULL); + /* must be a scalar */ + ck_assert(fy_node_is_scalar(fyn)); + ck_assert(!strcmp(fy_node_get_scalar0(fyn), "key1")); + + /* get the first value */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_ne(fyn, NULL); + /* must be a scalar */ + ck_assert(fy_node_is_scalar(fyn)); + ck_assert(!strcmp(fy_node_get_scalar0(fyn), "value1")); + + /* get the second key */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_ne(fyn, NULL); + /* must be a scalar */ + ck_assert(fy_node_is_scalar(fyn)); + ck_assert(!strcmp(fy_node_get_scalar0(fyn), "key2")); + + /* get the second value */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_ne(fyn, NULL); + /* must be a scalar */ + ck_assert(fy_node_is_scalar(fyn)); + ck_assert(!strcmp(fy_node_get_scalar0(fyn), "value2")); + + /* final, must be out of nodes */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_eq(fyn, NULL); + + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Comment retrieval from tokens */ +START_TEST(parser_comment_retrieval) +{ + struct fy_parse_cfg cfg = { + .flags = FYPCF_PARSE_COMMENTS, + }; + struct fy_document *fyd; + struct fy_document_iterator *fydi; + struct fy_node *fyn; + struct fy_token *fyt; + char buf[256]; + bool found_comment = false; + + /* Build document with comments */ + fyd = fy_document_build_from_string(&cfg, + "# Top comment\n" + "scalar: value # Right comment\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fydi = fy_document_iterator_create(); + ck_assert_ptr_ne(fydi, NULL); + + fy_document_iterator_node_start(fydi, fy_document_root(fyd)); + + /* Iterate and check for comments */ + while ((fyn = fy_document_iterator_node_next(fydi)) != NULL) { + if (!fy_node_is_scalar(fyn)) + continue; + + fyt = fy_node_get_scalar_token(fyn); + if (!fyt || !fy_token_has_any_comment(fyt)) + continue; + + /* Try to get comments at different placements */ + for (int placement = fycp_top; placement < fycp_max; placement++) { + if (fy_token_get_comment(fyt, buf, sizeof(buf), placement)) { + ck_assert_ptr_ne(buf, NULL); + ck_assert_int_gt(strlen(buf), 0); + found_comment = true; + } + } + } + + ck_assert(found_comment); + + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Alias node detection in iterator */ +START_TEST(parser_iterator_alias_detection) +{ + struct fy_document *fyd; + struct fy_document_iterator *fydi; + struct fy_node *fyn; + bool found_alias = false; + + /* Build document with anchor and alias */ + fyd = fy_document_build_from_string(NULL, + "anchor: &ref value\n" + "alias: *ref\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fydi = fy_document_iterator_create(); + ck_assert_ptr_ne(fydi, NULL); + + fy_document_iterator_node_start(fydi, fy_document_root(fyd)); + + /* Iterate and check for alias nodes */ + while ((fyn = fy_document_iterator_node_next(fydi)) != NULL) { + if (fy_node_is_scalar(fyn) && fy_node_is_alias(fyn)) { + found_alias = true; + } + } + + ck_assert(found_alias); + + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Event-based parsing */ +START_TEST(parser_event_generation) +{ + struct fy_parser fyp_data, *fyp = &fyp_data; + struct fy_parse_cfg cfg = { + .flags = FYPCF_DEFAULT_DOC, + }; + struct fy_event *event; + int rc; + bool got_stream_start = false; + bool got_doc_start = false; + bool got_scalar = false; + bool got_doc_end = false; + bool got_stream_end = false; + + /* Setup parser */ + rc = fy_parse_setup(fyp, &cfg); + ck_assert_int_eq(rc, 0); + + /* Add simple YAML input */ + const char *yaml = "key: value\n"; + struct fy_input_cfg fyic = { + .type = fyit_memory, + .memory.data = yaml, + .memory.size = strlen(yaml), + }; + rc = fy_parse_input_append(fyp, &fyic); + ck_assert_int_eq(rc, 0); + + /* Parse events */ + while ((event = fy_parser_parse(fyp)) != NULL) { + switch (event->type) { + case FYET_STREAM_START: + got_stream_start = true; + break; + case FYET_DOCUMENT_START: + got_doc_start = true; + break; + case FYET_SCALAR: + got_scalar = true; + break; + case FYET_DOCUMENT_END: + got_doc_end = true; + break; + case FYET_STREAM_END: + got_stream_end = true; + break; + default: + break; + } + fy_parser_event_free(fyp, event); + } + + /* Verify we got expected events */ + ck_assert(got_stream_start); + ck_assert(got_doc_start); + ck_assert(got_scalar); + ck_assert(got_doc_end); + ck_assert(got_stream_end); + + fy_parse_cleanup(fyp); +} +END_TEST + +/* Test: Scalar style detection */ +START_TEST(parser_scalar_styles) +{ + struct fy_document *fyd; + struct fy_node *fyn; + struct fy_token *fyt; + enum fy_scalar_style style; + + /* Build document with different scalar styles */ + fyd = fy_document_build_from_string(NULL, + "plain: plain value\n" + "single: 'single quoted'\n" + "double: \"double quoted\"\n" + "literal: |\n" + " literal block\n" + "folded: >\n" + " folded block\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Check plain style */ + fyn = fy_node_by_path(fy_document_root(fyd), "plain", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + fyt = fy_node_get_scalar_token(fyn); + ck_assert_ptr_ne(fyt, NULL); + style = fy_token_scalar_style(fyt); + ck_assert_int_eq(style, FYSS_PLAIN); + + /* Check single quoted style */ + fyn = fy_node_by_path(fy_document_root(fyd), "single", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + fyt = fy_node_get_scalar_token(fyn); + ck_assert_ptr_ne(fyt, NULL); + style = fy_token_scalar_style(fyt); + ck_assert_int_eq(style, FYSS_SINGLE_QUOTED); + + /* Check double quoted style */ + fyn = fy_node_by_path(fy_document_root(fyd), "double", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + fyt = fy_node_get_scalar_token(fyn); + ck_assert_ptr_ne(fyt, NULL); + style = fy_token_scalar_style(fyt); + ck_assert_int_eq(style, FYSS_DOUBLE_QUOTED); + + /* Check literal style */ + fyn = fy_node_by_path(fy_document_root(fyd), "literal", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + fyt = fy_node_get_scalar_token(fyn); + ck_assert_ptr_ne(fyt, NULL); + style = fy_token_scalar_style(fyt); + ck_assert_int_eq(style, FYSS_LITERAL); + + /* Check folded style */ + fyn = fy_node_by_path(fy_document_root(fyd), "folded", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + fyt = fy_node_get_scalar_token(fyn); + ck_assert_ptr_ne(fyt, NULL); + style = fy_token_scalar_style(fyt); + ck_assert_int_eq(style, FYSS_FOLDED); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Tag handling */ +START_TEST(parser_tag_handling) +{ + struct fy_document *fyd; + struct fy_node *fyn; + const char *tag; + + /* Build document with tags */ + fyd = fy_document_build_from_string(NULL, + "string: !!str tagged string\n" + "integer: !!int 42\n" + "custom: !custom custom tag\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Check string tag */ + fyn = fy_node_by_path(fy_document_root(fyd), "string", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + tag = fy_node_get_tag0(fyn); + ck_assert_ptr_ne(tag, NULL); + ck_assert(strstr(tag, "str") != NULL); + + /* Check integer tag */ + fyn = fy_node_by_path(fy_document_root(fyd), "integer", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + tag = fy_node_get_tag0(fyn); + ck_assert_ptr_ne(tag, NULL); + ck_assert(strstr(tag, "int") != NULL); + + /* Check custom tag */ + fyn = fy_node_by_path(fy_document_root(fyd), "custom", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + tag = fy_node_get_tag0(fyn); + ck_assert_ptr_ne(tag, NULL); + ck_assert(strstr(tag, "custom") != NULL); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: YAML version directives */ +START_TEST(parser_yaml_version) +{ + struct fy_parse_cfg cfg_11 = { + .flags = FYPCF_DEFAULT_VERSION_1_1, + }; + struct fy_parse_cfg cfg_12 = { + .flags = FYPCF_DEFAULT_VERSION_1_2, + }; + struct fy_document *fyd; + + /* Parse with YAML 1.1 */ + fyd = fy_document_build_from_string(&cfg_11, "key: value", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + fy_document_destroy(fyd); + + /* Parse with YAML 1.2 */ + fyd = fy_document_build_from_string(&cfg_12, "key: value", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + fy_document_destroy(fyd); + + /* Parse with explicit version directive */ + fyd = fy_document_build_from_string(NULL, "%YAML 1.2\n---\nkey: value", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Flow and block styles */ +START_TEST(parser_flow_block_styles) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + /* Build document with mixed flow and block styles */ + fyd = fy_document_build_from_string(NULL, + "block_map:\n" + " key: value\n" + "flow_map: {key: value}\n" + "block_seq:\n" + " - item\n" + "flow_seq: [item]\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Verify block mapping */ + fyn = fy_node_by_path(fy_document_root(fyd), "block_map", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + /* Verify flow mapping */ + fyn = fy_node_by_path(fy_document_root(fyd), "flow_map", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + /* Verify block sequence */ + fyn = fy_node_by_path(fy_document_root(fyd), "block_seq", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_sequence(fyn)); + + /* Verify flow sequence */ + fyn = fy_node_by_path(fy_document_root(fyd), "flow_seq", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_sequence(fyn)); + + /* Emit and verify output */ + buf = fy_emit_document_to_string(fyd, 0); + ck_assert_ptr_ne(buf, NULL); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document builder API */ +START_TEST(parser_document_builder) +{ + struct fy_document *fyd; + struct fy_node *fyn_root, *fyn_key, *fyn_val; + int rc; + + /* Create document using builder pattern */ + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* Build root mapping */ + fyn_root = fy_node_create_mapping(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + fy_document_set_root(fyd, fyn_root); + + /* Add key-value pairs using builder */ + fyn_key = fy_node_build_from_string(fyd, "key1", FY_NT); + ck_assert_ptr_ne(fyn_key, NULL); + fyn_val = fy_node_build_from_string(fyd, "value1", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + rc = fy_node_mapping_append(fyn_root, fyn_key, fyn_val); + ck_assert_int_eq(rc, 0); + + /* Add another pair with complex value */ + fyn_key = fy_node_build_from_string(fyd, "key2", FY_NT); + ck_assert_ptr_ne(fyn_key, NULL); + fyn_val = fy_node_build_from_string(fyd, "[1, 2, 3]", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + rc = fy_node_mapping_append(fyn_root, fyn_key, fyn_val); + ck_assert_int_eq(rc, 0); + + /* Verify the built document */ + ck_assert_int_eq(fy_node_mapping_item_count(fyn_root), 2); + + fyn_val = fy_node_mapping_lookup_by_string(fyn_root, "key1", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn_val), "value1"); + + fyn_val = fy_node_mapping_lookup_by_string(fyn_root, "key2", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + ck_assert(fy_node_is_sequence(fyn_val)); + ck_assert_int_eq(fy_node_sequence_item_count(fyn_val), 3); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Shell-like string splitting (from do_shell_split) */ +START_TEST(parser_shell_split) +{ + const char * const *argv; + int argc; + void *mem; + + /* Test simple splitting */ + mem = fy_utf8_split_posix("arg1 arg2 arg3", &argc, &argv); + ck_assert_ptr_ne(mem, NULL); + ck_assert_int_eq(argc, 3); + ck_assert_str_eq(argv[0], "arg1"); + ck_assert_str_eq(argv[1], "arg2"); + ck_assert_str_eq(argv[2], "arg3"); + ck_assert_ptr_eq(argv[argc], NULL); + free(mem); + + /* Test quoted strings */ + mem = fy_utf8_split_posix("'single quoted' \"double quoted\"", &argc, &argv); + ck_assert_ptr_ne(mem, NULL); + ck_assert_int_eq(argc, 2); + ck_assert_str_eq(argv[0], "single quoted"); + ck_assert_str_eq(argv[1], "double quoted"); + ck_assert_ptr_eq(argv[argc], NULL); + free(mem); + + /* Test escape sequences */ + mem = fy_utf8_split_posix("arg1\\ with\\ spaces arg2", &argc, &argv); + ck_assert_ptr_ne(mem, NULL); + ck_assert_int_eq(argc, 2); + ck_assert_str_eq(argv[0], "arg1 with spaces"); + ck_assert_str_eq(argv[1], "arg2"); + ck_assert_ptr_eq(argv[argc], NULL); + free(mem); + + /* Test empty string */ + mem = fy_utf8_split_posix("", &argc, &argv); + ck_assert_ptr_ne(mem, NULL); + ck_assert_int_eq(argc, 0); + ck_assert_ptr_eq(argv[argc], NULL); + free(mem); + + /* Test multiple spaces */ + mem = fy_utf8_split_posix(" arg1 arg2 ", &argc, &argv); + ck_assert_ptr_ne(mem, NULL); + ck_assert_int_eq(argc, 2); + ck_assert_str_eq(argv[0], "arg1"); + ck_assert_str_eq(argv[1], "arg2"); + ck_assert_ptr_eq(argv[argc], NULL); + free(mem); +} +END_TEST + +/* Test: UTF-8 validation (from do_bad_utf8) */ +START_TEST(parser_utf8_validation) +{ + const char *valid_ascii = "hello world"; + const char *s, *e; + int c, w, pos; + + /* Test valid ASCII forward */ + s = valid_ascii; + e = s + strlen(valid_ascii); + pos = 0; + while (s < e) { + c = fy_utf8_get(s, e - s, &w); + ck_assert_int_ge(c, 0); + ck_assert_int_ge(w, 1); + ck_assert_int_le(w, 4); + s += w; + pos++; + } + ck_assert_int_eq(pos, 11); + + /* Test valid UTF-8 forward (Greek characters) */ + const char *greek = "\xCE\xA4\xCE\xB9\xCE\xBC\xCE\xAE"; /* Τιμή */ + s = greek; + e = s + strlen(greek); + pos = 0; + while (s < e) { + c = fy_utf8_get(s, e - s, &w); + ck_assert_int_ge(c, 0); + ck_assert_int_eq(w, 2); /* Greek chars are 2 bytes */ + s += w; + pos++; + } + ck_assert_int_eq(pos, 4); + + /* Test valid ASCII backward */ + s = valid_ascii; + e = s + strlen(valid_ascii); + pos = 0; + while (s < e) { + c = fy_utf8_get_right(s, e - s, &w); + ck_assert_int_ge(c, 0); + ck_assert_int_ge(w, 1); + ck_assert_int_le(w, 4); + e -= w; + pos++; + } + ck_assert_int_eq(pos, 11); + + /* Test invalid UTF-8 sequence */ + const char invalid[] = { 0x67, 0xe7, 0x67, 0x00 }; /* 'g' followed by incomplete UTF-8 */ + s = invalid; + e = s + 3; /* Don't include null terminator */ + c = fy_utf8_get(s, 1, &w); /* First char should be valid */ + ck_assert_int_eq(c, 0x67); + ck_assert_int_eq(w, 1); + s += w; + c = fy_utf8_get(s, e - s, &w); /* Second char should be invalid or partial */ + ck_assert_int_lt(c, 0); /* Should return FYUG_INV or FYUG_PARTIAL */ + + /* Test EOF condition */ + c = fy_utf8_get(valid_ascii, 0, &w); + ck_assert_int_eq(c, FYUG_EOF); + ck_assert_int_eq(w, 0); +} +END_TEST + +/* Test: Token scanning - basic scalars */ +START_TEST(parser_token_scan_scalars) +{ + struct fy_parser *fyp; + struct fy_parse_cfg cfg; + struct fy_token *fyt; + const char *yaml; + int token_count; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + /* Test simple scalar */ + yaml = "hello"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + ck_assert_ptr_ne(fyt, NULL); + token_count++; + fy_token_unref(fyt); + } + ck_assert_int_gt(token_count, 0); + fy_parser_destroy(fyp); + + /* Test multiple scalars */ + yaml = "foo bar baz"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + token_count++; + fy_token_unref(fyt); + } + ck_assert_int_gt(token_count, 0); + fy_parser_destroy(fyp); +} +END_TEST + +/* Test: Token scanning - mappings */ +START_TEST(parser_token_scan_mapping) +{ + struct fy_parser *fyp; + struct fy_parse_cfg cfg; + struct fy_token *fyt; + const char *yaml; + int token_count; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + yaml = "key: value"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + ck_assert_ptr_ne(fyt, NULL); + token_count++; + fy_token_unref(fyt); + } + /* Should have multiple tokens: STREAM-START, BLOCK-MAPPING-START, KEY, SCALAR, VALUE, SCALAR, etc. */ + ck_assert_int_gt(token_count, 5); + fy_parser_destroy(fyp); + + /* Test nested mapping */ + yaml = "outer:\n inner: value"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + token_count++; + fy_token_unref(fyt); + } + ck_assert_int_gt(token_count, 5); + fy_parser_destroy(fyp); +} +END_TEST + +/* Test: Token scanning - sequences */ +START_TEST(parser_token_scan_sequence) +{ + struct fy_parser *fyp; + struct fy_parse_cfg cfg; + struct fy_token *fyt; + const char *yaml; + int token_count; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + /* Test flow sequence */ + yaml = "[1, 2, 3]"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + token_count++; + fy_token_unref(fyt); + } + ck_assert_int_gt(token_count, 5); + fy_parser_destroy(fyp); + + /* Test block sequence */ + yaml = "- item1\n- item2\n- item3"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + token_count++; + fy_token_unref(fyt); + } + ck_assert_int_gt(token_count, 5); + fy_parser_destroy(fyp); +} +END_TEST + +/* Test: Token scanning - scalar styles */ +START_TEST(parser_token_scan_scalar_styles) +{ + struct fy_parser *fyp; + struct fy_parse_cfg cfg; + struct fy_token *fyt; + const char *yaml; + int scalar_count; + enum fy_scalar_style style; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + /* Test different scalar styles */ + yaml = "plain: value\n" + "single: 'quoted value'\n" + "double: \"quoted value\"\n"; + + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + scalar_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + if (fyt->type == FYTT_SCALAR) { + style = fy_token_scalar_style(fyt); + /* Verify style is valid */ + ck_assert(style == FYSS_PLAIN || + style == FYSS_SINGLE_QUOTED || + style == FYSS_DOUBLE_QUOTED || + style == FYSS_LITERAL || + style == FYSS_FOLDED); + scalar_count++; + } + fy_token_unref(fyt); + } + ck_assert_int_ge(scalar_count, 6); /* 6 scalars: 3 keys + 3 values */ + fy_parser_destroy(fyp); +} +END_TEST + +/* Test: Manual emitter mode - emit events directly */ +START_TEST(parser_manual_emitter) +{ + struct fy_emitter_cfg ecfg; + struct fy_emitter *emit; + + memset(&ecfg, 0, sizeof(ecfg)); + ecfg.flags = FYECF_MODE_MANUAL; + + /* Test block style manual emission */ + emit = fy_emitter_create(&ecfg); + ck_assert_ptr_ne(emit, NULL); + + /* Emit: key: [{ a: 1 }] in block style */ + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_START)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_START, false, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "key", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "a", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "1", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_END, true, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_END)); + + /* If we got here without crashes, the manual emitter block style works */ + fy_emitter_destroy(emit); + + /* Test flow style manual emission */ + emit = fy_emitter_create(&ecfg); + ck_assert_ptr_ne(emit, NULL); + + /* Emit: {key: [{a: 1}]} in flow style */ + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_START)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_START, false, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_FLOW, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "key", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_START, FYNS_FLOW, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_FLOW, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "a", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "1", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_END, true, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_END)); + + /* If we got here without crashes, the manual emitter flow style works */ + fy_emitter_destroy(emit); +} +END_TEST + +START_TEST(parser_parse_load_document) +{ + struct fy_parse_cfg cfg; + struct fy_parser *fyp; + struct fy_document *fyd; + const char *yaml = "---\nfoo: bar\n---\nbaz: qux\n"; + int count; + + /* Test loading multiple documents from a parser stream */ + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + + fy_parser_set_string(fyp, yaml, FY_NT); + + /* Load first document */ + count = 0; + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + const char *value; + struct fy_node *fyn; + + count++; + + /* Verify document content */ + if (count == 1) { + fyn = fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + value = fy_node_get_scalar0(fyn); + ck_assert_str_eq(value, "bar"); + } else if (count == 2) { + fyn = fy_node_by_path(fy_document_root(fyd), "/baz", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + value = fy_node_get_scalar0(fyn); + ck_assert_str_eq(value, "qux"); + } + + fy_parse_document_destroy(fyp, fyd); + } + + ck_assert_int_eq(count, 2); + + fy_parser_destroy(fyp); +} +END_TEST + +START_TEST(parser_document_resolve) +{ + struct fy_document *fyd; + struct fy_node *fyn; + const char *yaml = "anchor: &data\n key: value\nref: *data\n"; + const char *val; + int rc; + + /* Test document resolution (resolving aliases) */ + fyd = fy_document_build_from_string(NULL, yaml, FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Before resolution, alias nodes exist */ + fyn = fy_node_by_path(fy_document_root(fyd), "/ref", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_alias(fyn)); + + /* Resolve all aliases in the document */ + rc = fy_document_resolve(fyd); + ck_assert_int_eq(rc, 0); + + /* After resolution, verify both paths are accessible and have same values */ + fyn = fy_node_by_path(fy_document_root(fyd), "/anchor/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + val = fy_node_get_scalar0(fyn); + ck_assert_str_eq(val, "value"); + + fyn = fy_node_by_path(fy_document_root(fyd), "/ref/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + val = fy_node_get_scalar0(fyn); + ck_assert_str_eq(val, "value"); + + /* After resolution, the ref node should no longer be an alias */ + fyn = fy_node_by_path(fy_document_root(fyd), "/ref", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(!fy_node_is_alias(fyn)); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(parser_document_clone) +{ + struct fy_document *fyd, *fyd_clone; + struct fy_node *fyn, *fyn_clone; + const char *yaml = "foo: bar\nbaz: [1, 2, 3]\n"; + const char *val1, *val2; + + /* Test document cloning */ + fyd = fy_document_build_from_string(NULL, yaml, FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Clone the document */ + fyd_clone = fy_document_clone(fyd); + ck_assert_ptr_ne(fyd_clone, NULL); + + /* Verify original document content */ + fyn = fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + val1 = fy_node_get_scalar0(fyn); + ck_assert_str_eq(val1, "bar"); + + /* Verify cloned document has same content */ + fyn_clone = fy_node_by_path(fy_document_root(fyd_clone), "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_clone, NULL); + val2 = fy_node_get_scalar0(fyn_clone); + ck_assert_str_eq(val2, "bar"); + + /* But they are different node objects */ + ck_assert_ptr_ne(fyn, fyn_clone); + + fy_document_destroy(fyd); + fy_document_destroy(fyd_clone); +} +END_TEST + +START_TEST(parser_node_copy) +{ + struct fy_document *fyd; + struct fy_node *fyn_src, *fyn_copy, *fyn_root; + const char *yaml = "original: {key: value}\n"; + const char *val; + + /* Test node copying */ + fyd = fy_document_build_from_string(NULL, yaml, FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Get the source node */ + fyn_src = fy_node_by_path(fy_document_root(fyd), "/original", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_src, NULL); + + /* Copy the node */ + fyn_copy = fy_node_copy(fyd, fyn_src); + ck_assert_ptr_ne(fyn_copy, NULL); + + /* Verify the copy has same content */ + fyn_root = fy_node_by_path(fyn_copy, "/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_root, NULL); + val = fy_node_get_scalar0(fyn_root); + ck_assert_str_eq(val, "value"); + + /* Add the copy to the document with a new key */ + fy_node_mapping_append(fy_document_root(fyd), + fy_node_build_from_string(fyd, "copied", FY_NT), + fyn_copy); + + /* Verify it exists at new location */ + fyn_root = fy_node_by_path(fy_document_root(fyd), "/copied/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_root, NULL); + val = fy_node_get_scalar0(fyn_root); + ck_assert_str_eq(val, "value"); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(parser_document_builder_load) +{ + struct fy_parse_cfg parse_cfg; + struct fy_document_builder_cfg builder_cfg; + struct fy_parser *fyp; + struct fy_document_builder *fydb; + struct fy_document *fyd; + const char *yaml = "---\nkey: value\n---\nfoo: bar\n"; + int count; + + /* Test document builder API */ + memset(&parse_cfg, 0, sizeof(parse_cfg)); + parse_cfg.flags = FYPCF_DEFAULT_PARSE; + + memset(&builder_cfg, 0, sizeof(builder_cfg)); + builder_cfg.parse_cfg = parse_cfg; + + fyp = fy_parser_create(&parse_cfg); + ck_assert_ptr_ne(fyp, NULL); + + fy_parser_set_string(fyp, yaml, FY_NT); + + /* Create document builder with configuration */ + fydb = fy_document_builder_create(&builder_cfg); + ck_assert_ptr_ne(fydb, NULL); + + /* Load documents using builder */ + count = 0; + while ((fyd = fy_document_builder_load_document(fydb, fyp)) != NULL) { + const char *value; + struct fy_node *fyn; + + count++; + + /* Verify document content */ + if (count == 1) { + fyn = fy_node_by_path(fy_document_root(fyd), "/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + value = fy_node_get_scalar0(fyn); + ck_assert_str_eq(value, "value"); + } else if (count == 2) { + fyn = fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + value = fy_node_get_scalar0(fyn); + ck_assert_str_eq(value, "bar"); + } + + fy_document_destroy(fyd); + } + + ck_assert_int_eq(count, 2); + + fy_document_builder_destroy(fydb); + fy_parser_destroy(fyp); +} +END_TEST + +/* Composer callback record entry */ +struct compose_record { + enum fy_event_type type; + char path[128]; + int depth; + char scalar_value[64]; +}; + +/* Composer callback data for parser_compose_callback test */ +struct compose_test_data { + struct compose_record records[100]; + int record_count; +}; + +/* Composer callback function for testing */ +static enum fy_composer_return +compose_test_callback(struct fy_parser *fyp, struct fy_event *fye, + struct fy_path *path, void *userdata) +{ + struct compose_test_data *data = userdata; + struct compose_record *rec; + char *path_text; + const char *scalar; + size_t len; + + if (data->record_count >= 100) + return FYCR_OK_CONTINUE; + + rec = &data->records[data->record_count++]; + rec->type = fye->type; + rec->depth = fy_path_depth(path); + + /* Get path as text */ + path_text = fy_path_get_text(path); + if (path_text) { + snprintf(rec->path, sizeof(rec->path), "%s", path_text); + free(path_text); + } else { + rec->path[0] = '\0'; + } + + /* For scalar events, capture the value */ + rec->scalar_value[0] = '\0'; + if (fye->type == FYET_SCALAR && fye->scalar.value) { + scalar = fy_token_get_text(fye->scalar.value, &len); + if (scalar && len < sizeof(rec->scalar_value) - 1) { + memcpy(rec->scalar_value, scalar, len); + rec->scalar_value[len] = '\0'; + } + } + + return FYCR_OK_CONTINUE; +} + +START_TEST(parser_compose_callback) +{ + struct fy_parse_cfg cfg; + struct fy_parser *fyp; + struct compose_test_data data; + const char *yaml = + "invoice: 34843\n" + "date : !!str 2001-01-23\n" + "bill-to: &id001\n" + " given : Chris\n" + " family : Dumars\n" + " address:\n" + " lines: |\n" + " 458 Walkman Dr.\n" + " Suite #292\n"; + int rc; + int i; + bool found_invoice_key = false; + bool found_invoice_value = false; + bool found_bill_to_path = false; + + /* Test compose callback mechanism */ + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + + fy_parser_set_string(fyp, yaml, FY_NT); + + /* Initialize test data */ + memset(&data, 0, sizeof(data)); + + /* Compose with callback */ + rc = fy_parse_compose(fyp, compose_test_callback, &data); + ck_assert_int_eq(rc, 0); + + /* Verify we got events */ + ck_assert(data.record_count > 0); + + /* Verify we have the expected events and paths */ + for (i = 0; i < data.record_count; i++) { + struct compose_record *rec = &data.records[i]; + + /* Look for specific events we expect */ + if (rec->type == FYET_SCALAR && strcmp(rec->scalar_value, "invoice") == 0) { + found_invoice_key = true; + /* Verify path for invoice key */ + ck_assert(strstr(rec->path, "invoice") != NULL || strcmp(rec->path, "/") == 0); + } + + if (rec->type == FYET_SCALAR && strcmp(rec->scalar_value, "34843") == 0) { + found_invoice_value = true; + } + + /* Check for bill-to path */ + if (rec->type == FYET_MAPPING_START && strstr(rec->path, "bill-to") != NULL) { + found_bill_to_path = true; + /* Verify depth is reasonable (should be at least 1) */ + ck_assert(rec->depth > 0); + } + } + + /* Verify we found the expected elements */ + ck_assert(found_invoice_key); + ck_assert(found_invoice_value); + ck_assert(found_bill_to_path); + + /* Verify first event is stream start */ + ck_assert_int_eq(data.records[0].type, FYET_STREAM_START); + + /* Verify last event is stream end */ + ck_assert_int_eq(data.records[data.record_count - 1].type, FYET_STREAM_END); + + fy_parser_destroy(fyp); +} +END_TEST + +TCase *libfyaml_case_parser(void) +{ + TCase *tc; + + tc = tcase_create("parser"); + + /* Mapping tests */ + tcase_add_test(tc, parser_mapping_iterator); + tcase_add_test(tc, parser_mapping_key_lookup); + tcase_add_test(tc, parser_mapping_prepend); + tcase_add_test(tc, parser_mapping_remove); + + /* Path query tests */ + tcase_add_test(tc, parser_path_queries); + tcase_add_test(tc, parser_node_path_generation); + + /* Node creation tests */ + tcase_add_test(tc, parser_node_creation_scalar); + tcase_add_test(tc, parser_node_creation_multiline_scalar); + tcase_add_test(tc, parser_node_creation_empty_sequence); + tcase_add_test(tc, parser_node_creation_empty_mapping); + tcase_add_test(tc, parser_node_creation_populated_sequence); + tcase_add_test(tc, parser_node_creation_populated_mapping); + tcase_add_test(tc, parser_build_node_from_string); + + /* Sequence tests */ + tcase_add_test(tc, parser_sequence_negative_index); + tcase_add_test(tc, parser_sequence_append_prepend); + tcase_add_test(tc, parser_sequence_remove); + + /* Complex structure tests */ + tcase_add_test(tc, parser_complex_nested_structure); + + /* Anchor/alias tests */ + tcase_add_test(tc, parser_anchor_alias_resolution); + + /* Document operations */ + tcase_add_test(tc, parser_document_insert_at); + tcase_add_test(tc, parser_document_emit_flags); + tcase_add_test(tc, parser_multi_document_stream); + tcase_add_test(tc, parser_empty_document); + tcase_add_test(tc, parser_document_with_comments); + + /* Iterator tests */ + tcase_add_test(tc, parser_document_iterator); + tcase_add_test(tc, parser_document_iterator_key_detection); + tcase_add_test(tc, parser_iterator_alias_detection); + + /* Comment tests */ + tcase_add_test(tc, parser_comment_retrieval); + + /* Event and parsing tests */ + tcase_add_test(tc, parser_event_generation); + tcase_add_test(tc, parser_scalar_styles); + tcase_add_test(tc, parser_tag_handling); + tcase_add_test(tc, parser_yaml_version); + tcase_add_test(tc, parser_flow_block_styles); + tcase_add_test(tc, parser_document_builder); + + /* Utility function tests */ + tcase_add_test(tc, parser_shell_split); + tcase_add_test(tc, parser_utf8_validation); + + /* Token scanning tests */ + tcase_add_test(tc, parser_token_scan_scalars); + tcase_add_test(tc, parser_token_scan_mapping); + tcase_add_test(tc, parser_token_scan_sequence); + tcase_add_test(tc, parser_token_scan_scalar_styles); + + /* Manual emitter tests */ + tcase_add_test(tc, parser_manual_emitter); + + /* Parser and document builder tests */ + tcase_add_test(tc, parser_parse_load_document); + tcase_add_test(tc, parser_document_resolve); + tcase_add_test(tc, parser_document_clone); + tcase_add_test(tc, parser_node_copy); + tcase_add_test(tc, parser_document_builder_load); + + /* Compose callback tests */ + tcase_add_test(tc, parser_compose_callback); + + return tc; +} diff --git a/test/libfyaml-test-private-id.c b/test/libfyaml-test-private-id.c new file mode 100644 index 0000000..f943868 --- /dev/null +++ b/test/libfyaml-test-private-id.c @@ -0,0 +1,366 @@ +/* + * libfyaml-test-private-id.c - libfyaml id allocation and handling + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_ALLOCA_H +#include +#endif + + +#include + +#include + +#include "fy-utils.h" +#include "fy-id.h" + +/* check ffs works */ +START_TEST(id_ffs) +{ + static const struct { + fy_id_bits v; + int r; + } ffs_check[] = { + { .v = 0, .r = -1 }, + { .v = ~(fy_id_bits_non_atomic)0, .r = 0 }, + { .v = ((fy_id_bits_non_atomic)1 << 0), .r = 0 }, + { .v = ((fy_id_bits_non_atomic)1 << (FY_ID_BITS_BITS - 1)), .r = (FY_ID_BITS_BITS - 1) }, + { .v = ((fy_id_bits_non_atomic)1 << 0) | ((fy_id_bits_non_atomic)1 << (FY_ID_BITS_BITS - 1)), .r = 0 }, + { .v = ((fy_id_bits_non_atomic)1 << (FY_ID_BITS_BITS / 2)), .r = (FY_ID_BITS_BITS / 2), }, + }; + unsigned int i; + fy_id_bits v; + int r; + + for (i = 0; i < ARRAY_SIZE(ffs_check); i++) { + fy_atomic_store(&v, ffs_check[i].v); + r = fy_id_ffs(v); + ck_assert_int_eq(r, ffs_check[i].r); + } +} +END_TEST + +/* a random bit number for the following tests, not a power of 2 */ +#define BA_BITS 67 + +#define BA_DECL \ + size_t ba_count = FY_ID_BITS_ARRAY_COUNT(BA_BITS); \ + fy_id_bits *ba = alloca(ba_count * sizeof(*ba)) + +/* verify that reset clears everything */ +START_TEST(id_reset) +{ + BA_DECL; + unsigned int i; + + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_count; i++) + ck_assert_int_eq(ba[i], 0); +} +END_TEST + +/* verify that allocation to the full works, and then fails */ +START_TEST(id_alloc_full) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i; + int id, expected_id; + + fy_id_reset(ba, ba_count); + + /* allocate all, verify that we get numbers in sequence */ + expected_id = 0; + for (i = 0; i < ba_bits_actual; i++) { + id = fy_id_alloc(ba, ba_count); + ck_assert_int_ne(id, -1); + ck_assert_int_eq(id, expected_id); + expected_id++; + } + + /* full, it must fail now */ + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, -1); +} +END_TEST + +/* verify that allocation when almost full works, and then fails */ +START_TEST(id_alloc_almost_full) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i; + int id, expected_id; + + /* fill the array */ + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_bits_actual; i++) + fy_id_set_used(ba, ba_count, i); + + /* allocate all, verify that we get numbers in sequence */ + expected_id = 0; + for (i = 0; i < ba_bits_actual; i++) { + /* free one, and allocate, it must succeed at the exact spot */ + fy_id_free(ba, ba_count, i); + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, expected_id); + expected_id++; + + /* now it must fail */ + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, -1); + } +} +END_TEST + +/* verify that allocation of even bits works */ +START_TEST(id_alloc_even) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i, j; + int id, expected_id; + + /* fill the array */ + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_bits_actual; i++) + fy_id_set_used(ba, ba_count, i); + + /* free the even ids */ + j = ba_bits_actual / 2; + for (i = 0; i < j; i++) + fy_id_free(ba, ba_count, i * 2); + + /* allocate all, verify that we get numbers in sequence */ + expected_id = 0; + for (i = 0; i < j; i++) { + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, expected_id); + expected_id += 2; + } +} +END_TEST + +/* verify that allocation of odd bits works */ +START_TEST(id_alloc_odd) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i, j; + int id, expected_id; + + /* fill the array */ + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_bits_actual; i++) + fy_id_set_used(ba, ba_count, i); + + /* free the even ids */ + j = ba_bits_actual / 2; + for (i = 0; i < j; i++) + fy_id_free(ba, ba_count, i * 2 + 1); + + /* allocate all, verify that we get numbers in sequence */ + expected_id = 1; + for (i = 0; i < j; i++) { + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, expected_id); + expected_id += 2; + } +} +END_TEST + +/* verify that a specific allocation sequence works */ +START_TEST(id_alloc_seq) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i; + int id, expected_id; + int check_ids[] = { + 0, + ba_bits_actual / 2 - 1, + ba_bits_actual / 2, + ba_bits_actual / 2 + 1, + ba_bits_actual - 1 + }; + + /* fill the array */ + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_bits_actual; i++) + fy_id_set_used(ba, ba_count, i); + + /* free those specific bits */ + for (i = 0; i < ARRAY_SIZE(check_ids); i++) + fy_id_free(ba, ba_count, check_ids[i]); + + /* now allocate in sequence */ + expected_id = 0; + for (i = 0; i < ARRAY_SIZE(check_ids); i++) { + expected_id = check_ids[i]; + id = fy_id_alloc(ba, ba_count); + ck_assert_int_eq(id, expected_id); + } +} +END_TEST + +/* verify that a iterator works for a single bit in array */ +START_TEST(id_iter_single) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i; + int id, expected_id, found_id; + struct fy_id_iter iter; + + for (i = 0; i < ba_bits_actual; i++) { + fy_id_reset(ba, ba_count); + fy_id_set_used(ba, ba_count, i); + + expected_id = i; + found_id = -1; + fy_id_iter_begin(ba, ba_count, &iter); + while ((id = fy_id_iter_next(ba, ba_count, &iter)) >= 0) { + /* must find a single one */ + ck_assert_int_lt(found_id, 0); + found_id = id; + ck_assert_int_eq(id, expected_id); + } + fy_id_iter_end(ba, ba_count, &iter); + } + +} +END_TEST + +/* verify that a iterator works for a full array */ +START_TEST(id_iter_full) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + unsigned int i; + int id, expected_id; + struct fy_id_iter iter; + + fy_id_reset(ba, ba_count); + for (i = 0; i < ba_bits_actual; i++) + fy_id_set_used(ba, ba_count, i); + + expected_id = 0; + fy_id_iter_begin(ba, ba_count, &iter); + while ((id = fy_id_iter_next(ba, ba_count, &iter)) >= 0) { + ck_assert_int_eq(id, expected_id); + expected_id++; + } + + /* we must have run through the whole array */ + ck_assert_int_eq(expected_id, ba_bits_actual); + + fy_id_iter_end(ba, ba_count, &iter); +} +END_TEST + +/* verify that a iterator works for sequences up to 3 bits */ +START_TEST(id_iter_seq) +{ + BA_DECL; + unsigned int ba_bits_actual = FY_ID_BITS_ARRAY_COUNT_BITS(BA_BITS); + int idtab[][3] = { + { 0, ba_bits_actual - 1, -1 }, + { 0, 1, -1 }, + { ba_bits_actual - 2, ba_bits_actual - 1, -1 }, + { 0, FY_ID_BITS_BITS - 1, -1 }, + { 0, FY_ID_BITS_BITS, -1 }, + { FY_ID_BITS_BITS - 1, FY_ID_BITS_BITS, -1 }, + { FY_ID_BITS_BITS, FY_ID_BITS_BITS + 1, -1 }, + { 0, 1, 2 }, + { 0, ba_bits_actual - 2, ba_bits_actual - 1 }, + { ba_bits_actual - 3, ba_bits_actual - 2, ba_bits_actual - 1 }, + }; + unsigned int i; + int id, expected_id; + int p0, p1, p2; + struct fy_id_iter iter; + + for (i = 0; i < ARRAY_SIZE(idtab); i++) { + + p0 = idtab[i][0]; + p1 = idtab[i][1]; + p2 = idtab[i][2]; + + fy_id_reset(ba, ba_count); + if (p0 >= 0) + fy_id_set_used(ba, ba_count, p0); + if (p1 >= 0) + fy_id_set_used(ba, ba_count, p1); + if (p2 >= 0) + fy_id_set_used(ba, ba_count, p2); + + fy_id_iter_begin(ba, ba_count, &iter); + + if (p0 >= 0) { + id = fy_id_iter_next(ba, ba_count, &iter); + expected_id = p0; + ck_assert_int_eq(id, expected_id); + fy_id_set_free(ba, ba_count, expected_id); + } + + if (p1 >= 0) { + id = fy_id_iter_next(ba, ba_count, &iter); + expected_id = p1; + ck_assert_int_eq(id, expected_id); + fy_id_set_free(ba, ba_count, expected_id); + } + + if (p2 >= 0) { + id = fy_id_iter_next(ba, ba_count, &iter); + expected_id = p2; + ck_assert_int_eq(id, expected_id); + fy_id_set_free(ba, ba_count, expected_id); + } + fy_id_iter_end(ba, ba_count, &iter); + + /* and now it should be empty */ + fy_id_iter_begin(ba, ba_count, &iter); + id = fy_id_iter_next(ba, ba_count, &iter); + ck_assert_int_eq(id, -1); + fy_id_iter_end(ba, ba_count, &iter); + } +} +END_TEST + +TCase *libfyaml_case_private_id(void) +{ + TCase *tc; + + tc = tcase_create("private-id"); + + tcase_add_test(tc, id_ffs); + + tcase_add_test(tc, id_reset); + tcase_add_test(tc, id_alloc_full); + tcase_add_test(tc, id_alloc_almost_full); + tcase_add_test(tc, id_alloc_even); + tcase_add_test(tc, id_alloc_odd); + tcase_add_test(tc, id_alloc_seq); + + tcase_add_test(tc, id_iter_single); + tcase_add_test(tc, id_iter_full); + tcase_add_test(tc, id_iter_seq); + + return tc; +} diff --git a/test/libfyaml-test-private.c b/test/libfyaml-test-private.c new file mode 100644 index 0000000..c428682 --- /dev/null +++ b/test/libfyaml-test-private.c @@ -0,0 +1,166 @@ +/* + * libfyaml-test-private.c - libfyaml private API test harness + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include "fy-parse.h" + +static const struct fy_parse_cfg default_parse_cfg = { + .search_path = "", + .flags = FYPCF_QUIET, +}; + +START_TEST(parser_setup) +{ + struct fy_parser ctx, *fyp = &ctx; + const struct fy_parse_cfg *cfg = &default_parse_cfg; + int rc; + + /* setup */ + rc = fy_parse_setup(fyp, cfg); + ck_assert_int_eq(rc, 0); + + /* cleanup */ + fy_parse_cleanup(fyp); +} +END_TEST + +START_TEST(scan_simple) +{ + struct fy_parser ctx, *fyp = &ctx; + const struct fy_parse_cfg *cfg = &default_parse_cfg; + static const struct fy_input_cfg fyic = { + .type = fyit_memory, + .memory.data = "42", + .memory.size = 2, + }; + struct fy_token *fyt; + int rc; + + /* setup */ + rc = fy_parse_setup(fyp, cfg); + ck_assert_int_eq(rc, 0); + + /* add the input */ + rc = fy_parse_input_append(fyp, &fyic); + ck_assert_int_eq(rc, 0); + + /* STREAM_START */ + fyt = fy_scan(fyp); + ck_assert_ptr_ne(fyt, NULL); + ck_assert(fyt->type == FYTT_STREAM_START); + fy_token_unref(fyt); + + /* SCALAR */ + fyt = fy_scan(fyp); + ck_assert_ptr_ne(fyt, NULL); + ck_assert(fyt->type == FYTT_SCALAR); + ck_assert(fyt->scalar.style == FYSS_PLAIN); + ck_assert_str_eq(fy_token_get_text0(fyt), "42"); + fy_token_unref(fyt); + + /* STREAM_END */ + fyt = fy_scan(fyp); + ck_assert_ptr_ne(fyt, NULL); + ck_assert(fyt->type == FYTT_STREAM_END); + fy_token_unref(fyt); + + /* EOF */ + fyt = fy_scan(fyp); + ck_assert_ptr_eq(fyt, NULL); + + /* cleanup */ + fy_parse_cleanup(fyp); +} +END_TEST + +START_TEST(parse_simple) +{ + struct fy_parser ctx, *fyp = &ctx; + const struct fy_parse_cfg *cfg = &default_parse_cfg; + static const struct fy_input_cfg fyic = { + .type = fyit_memory, + .memory.data = "42", + .memory.size = 2, + }; + struct fy_eventp *fyep; + int rc; + + /* setup */ + rc = fy_parse_setup(fyp, cfg); + ck_assert_int_eq(rc, 0); + + /* add the input */ + rc = fy_parse_input_append(fyp, &fyic); + ck_assert_int_eq(rc, 0); + + /* STREAM_START */ + fyep = fy_parse_private(fyp); + ck_assert_ptr_ne(fyep, NULL); + ck_assert(fyep->e.type == FYET_STREAM_START); + fy_parse_eventp_recycle(fyp, fyep); + + /* DOCUMENT_START */ + fyep = fy_parse_private(fyp); + ck_assert_ptr_ne(fyep, NULL); + ck_assert(fyep->e.type == FYET_DOCUMENT_START); + fy_parse_eventp_recycle(fyp, fyep); + + /* SCALAR */ + fyep = fy_parse_private(fyp); + ck_assert_ptr_ne(fyep, NULL); + ck_assert(fyep->e.type == FYET_SCALAR); + ck_assert_str_eq(fy_token_get_text0(fyep->e.scalar.value), "42"); + fy_parse_eventp_recycle(fyp, fyep); + + /* DOCUMENT_END */ + fyep = fy_parse_private(fyp); + ck_assert_ptr_ne(fyep, NULL); + ck_assert(fyep->e.type == FYET_DOCUMENT_END); + fy_parse_eventp_recycle(fyp, fyep); + + /* STREAM_END */ + fyep = fy_parse_private(fyp); + ck_assert_ptr_ne(fyep, NULL); + ck_assert(fyep->e.type == FYET_STREAM_END); + fy_parse_eventp_recycle(fyp, fyep); + + /* EOF */ + fyep = fy_parse_private(fyp); + ck_assert_ptr_eq(fyep, NULL); + + /* cleanup */ + fy_parse_cleanup(fyp); +} +END_TEST + +TCase *libfyaml_case_private(void) +{ + TCase *tc; + + tc = tcase_create("private"); + + tcase_add_test(tc, parser_setup); + tcase_add_test(tc, scan_simple); + tcase_add_test(tc, parse_simple); + + return tc; +} diff --git a/test/libfyaml-test-thread.c b/test/libfyaml-test-thread.c new file mode 100644 index 0000000..581b4df --- /dev/null +++ b/test/libfyaml-test-thread.c @@ -0,0 +1,267 @@ +/* + * libfyaml-test-thread.c - libfyaml threading tests + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include + +/* Test: Basic thread pool creation and destruction */ +START_TEST(thread_pool_create_destroy) +{ + struct fy_thread_pool_cfg cfg; + struct fy_thread_pool *tp; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = 0; + cfg.num_threads = 2; + cfg.userdata = NULL; + + tp = fy_thread_pool_create(&cfg); + ck_assert_ptr_ne(tp, NULL); + + /* Verify we can get the configuration */ + const struct fy_thread_pool_cfg *got_cfg = fy_thread_pool_get_cfg(tp); + ck_assert_ptr_ne(got_cfg, NULL); + ck_assert_int_eq(got_cfg->num_threads, 2); + + /* Verify we can get the number of threads */ + int num_threads = fy_thread_pool_get_num_threads(tp); + ck_assert_int_eq(num_threads, 2); + + fy_thread_pool_destroy(tp); +} +END_TEST + +/* Worker function that atomically increments a counter */ +static void atomic_increment_worker(void *arg) +{ + _Atomic(int) *p = arg; + int v, exp_v; + + /* Atomically increase the counter */ + v = atomic_load(p); + for (;;) { + exp_v = v; + if (atomic_compare_exchange_strong(p, &exp_v, v + 1)) + return; + v = exp_v; + } +} + +/* Test: Thread reserve, submit work, wait, unreserve */ +START_TEST(thread_reserve_submit_wait) +{ + struct fy_thread_pool_cfg cfg; + struct fy_thread_pool *tp; + struct fy_thread *threads[4]; + struct fy_thread_work works[4]; + _Atomic(int) counter = 0; + unsigned int i; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = 0; + cfg.num_threads = 4; + cfg.userdata = NULL; + + tp = fy_thread_pool_create(&cfg); + ck_assert_ptr_ne(tp, NULL); + + /* Reserve all threads */ + for (i = 0; i < 4; i++) { + threads[i] = fy_thread_reserve(tp); + ck_assert_ptr_ne(threads[i], NULL); + } + + /* Submit work to all threads */ + for (i = 0; i < 4; i++) { + works[i].fn = atomic_increment_worker; + works[i].arg = &counter; + fy_thread_submit_work(threads[i], &works[i]); + } + + /* Wait for all threads to complete */ + for (i = 0; i < 4; i++) { + fy_thread_wait_work(threads[i]); + } + + /* Verify counter was incremented 4 times */ + ck_assert_int_eq(atomic_load(&counter), 4); + + /* Unreserve all threads */ + for (i = 0; i < 4; i++) { + fy_thread_unreserve(threads[i]); + } + + fy_thread_pool_destroy(tp); +} +END_TEST + +/* Test: Thread arg join */ +START_TEST(thread_arg_join) +{ + struct fy_thread_pool_cfg cfg; + struct fy_thread_pool *tp; + _Atomic(int) counter = 0; + unsigned int num_tasks = 8; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = 0; + cfg.num_threads = 4; + cfg.userdata = NULL; + + tp = fy_thread_pool_create(&cfg); + ck_assert_ptr_ne(tp, NULL); + + /* Use arg_join to execute the same function multiple times */ + fy_thread_arg_join(tp, atomic_increment_worker, NULL, &counter, num_tasks); + + /* Verify counter was incremented num_tasks times */ + ck_assert_int_eq(atomic_load(&counter), (int)num_tasks); + + fy_thread_pool_destroy(tp); +} +END_TEST + +/* Worker function for sum test */ +struct sum_arg { + const int *values; + int count; + int result; +}; + +static void sum_worker(void *arg) +{ + struct sum_arg *s = arg; + int i; + int sum = 0; + + for (i = 0; i < s->count; i++) { + sum += s->values[i]; + } + s->result = sum; +} + +/* Test: Thread array join with different arguments */ +START_TEST(thread_arg_array_join) +{ + struct fy_thread_pool_cfg cfg; + struct fy_thread_pool *tp; + int values[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + struct sum_arg args[2]; + int total_sum; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = 0; + cfg.num_threads = 2; + cfg.userdata = NULL; + + tp = fy_thread_pool_create(&cfg); + ck_assert_ptr_ne(tp, NULL); + + /* Split the work into two tasks */ + args[0].values = values; + args[0].count = 5; + args[0].result = 0; + + args[1].values = values + 5; + args[1].count = 5; + args[1].result = 0; + + /* Execute both tasks in parallel */ + fy_thread_arg_array_join(tp, sum_worker, NULL, args, sizeof(struct sum_arg), 2); + + /* Verify results */ + total_sum = args[0].result + args[1].result; + ck_assert_int_eq(total_sum, 55); /* 1+2+3+...+10 = 55 */ + ck_assert_int_eq(args[0].result, 15); /* 1+2+3+4+5 = 15 */ + ck_assert_int_eq(args[1].result, 40); /* 6+7+8+9+10 = 40 */ + + fy_thread_pool_destroy(tp); +} +END_TEST + +/* Worker function for steal mode test */ +static void steal_mode_worker(void *arg) +{ + _Atomic(int) *p = arg; + int v, exp_v; + int i; + + /* Increment counter 100 times */ + for (i = 0; i < 100; i++) { + v = atomic_load(p); + for (;;) { + exp_v = v; + if (atomic_compare_exchange_strong(p, &exp_v, v + 1)) + break; + v = exp_v; + } + } +} + +/* Test: Work stealing mode */ +START_TEST(thread_steal_mode) +{ + struct fy_thread_pool_cfg cfg; + struct fy_thread_pool *tp; + _Atomic(int) counter = 0; + unsigned int num_tasks = 16; /* More tasks than threads */ + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYTPCF_STEAL_MODE; + cfg.num_threads = 4; + cfg.userdata = NULL; + + tp = fy_thread_pool_create(&cfg); + ck_assert_ptr_ne(tp, NULL); + + /* Execute many tasks with work stealing enabled */ + fy_thread_arg_join(tp, steal_mode_worker, NULL, &counter, num_tasks); + + /* Verify all tasks completed: 16 tasks * 100 increments each */ + ck_assert_int_eq(atomic_load(&counter), (int)(num_tasks * 100)); + + fy_thread_pool_destroy(tp); +} +END_TEST + +TCase *libfyaml_case_thread(void) +{ + TCase *tc; + + tc = tcase_create("thread"); + + /* Basic thread pool tests */ + tcase_add_test(tc, thread_pool_create_destroy); + + /* Thread work submission tests */ + tcase_add_test(tc, thread_reserve_submit_wait); + + /* Join API tests */ + tcase_add_test(tc, thread_arg_join); + tcase_add_test(tc, thread_arg_array_join); + + /* Work stealing tests */ + tcase_add_test(tc, thread_steal_mode); + + return tc; +} diff --git a/test/libfyaml-test.c b/test/libfyaml-test.c new file mode 100644 index 0000000..0fbf83e --- /dev/null +++ b/test/libfyaml-test.c @@ -0,0 +1,105 @@ +/* + * libfyaml-test.c - C API testing harness for libyaml + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#include + +#include "fy-valgrind.h" + +#define QUIET_DEFAULT false + +static struct option lopts[] = { + {"quiet", no_argument, 0, 'q' }, + {"help", no_argument, 0, 'h' }, + {0, 0, 0, 0 }, +}; + +static void display_usage(FILE *fp, char *progname) +{ + fprintf(fp, "Usage: %s [options] [files]\n", progname); + fprintf(fp, "\nOptions:\n\n"); + fprintf(fp, "\t--quiet, -q : Quiet operation, do not " + "output messages (default %s)\n", + QUIET_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--help, -h : Display help message\n"); + fprintf(fp, "\ne.g. %s\n", progname); +} + +#if defined(HAVE_STATIC) && HAVE_STATIC +extern TCase *libfyaml_case_private(void); +extern TCase *libfyaml_case_private_id(void); +#endif +extern TCase *libfyaml_case_core(void); +extern TCase *libfyaml_case_meta(void); +extern TCase *libfyaml_case_emit(void); +extern TCase *libfyaml_case_allocator(void); +extern TCase *libfyaml_case_parser(void); +extern TCase *libfyaml_case_thread(void); + +Suite *libfyaml_suite(void) +{ + Suite *s; + + s = suite_create("libfyaml"); + +#if defined(HAVE_STATIC) && HAVE_STATIC + suite_add_tcase(s, libfyaml_case_private()); + suite_add_tcase(s, libfyaml_case_private_id()); +#endif + suite_add_tcase(s, libfyaml_case_core()); + suite_add_tcase(s, libfyaml_case_meta()); + suite_add_tcase(s, libfyaml_case_emit()); + suite_add_tcase(s, libfyaml_case_allocator()); + suite_add_tcase(s, libfyaml_case_parser()); + suite_add_tcase(s, libfyaml_case_thread()); + + return s; +} + +int main(int argc, char *argv[]) +{ + int exitcode = EXIT_FAILURE, opt, lidx; + bool quiet = QUIET_DEFAULT; + int number_failed; + Suite *s; + SRunner *sr; + + fy_valgrind_check(&argc, &argv); + + while ((opt = getopt_long(argc, argv, "qh", lopts, &lidx)) != -1) { + switch (opt) { + case 'q': + quiet = true; + break; + case 'h' : + default: + if (opt != 'h') + fprintf(stderr, "Unknown option\n"); + display_usage(opt == 'h' ? stdout : stderr, argv[0]); + return EXIT_SUCCESS; + } + } + + s = libfyaml_suite(); + sr = srunner_create(s); + srunner_set_tap(sr, "-"); + srunner_run_all(sr, quiet ? CK_SILENT : CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + + exitcode = !number_failed ? EXIT_SUCCESS : EXIT_FAILURE; + + return exitcode; +} diff --git a/test/libfyaml.test b/test/libfyaml.test new file mode 100755 index 0000000..d77cc2f --- /dev/null +++ b/test/libfyaml.test @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +if [ "x$builddir" = "x" ]; then + T=`realpath ./libfyaml-test` +else + T=`realpath ${builddir}/libfyaml-test` +fi +${T} +exit $? diff --git a/test/test-env b/test/test-env new file mode 100644 index 0000000..abb4867 --- /dev/null +++ b/test/test-env @@ -0,0 +1 @@ +# test environment for libfyaml diff --git a/test/test-errors/0002/=== b/test/test-errors/0002/=== new file mode 100644 index 0000000..e7f4f89 --- /dev/null +++ b/test/test-errors/0002/=== @@ -0,0 +1 @@ +Duplicate key (plain scalar) diff --git a/test/test-errors/0002/in.yaml b/test/test-errors/0002/in.yaml new file mode 100644 index 0000000..c8b14cf --- /dev/null +++ b/test/test-errors/0002/in.yaml @@ -0,0 +1,2 @@ +foo: bar +foo: baz diff --git a/test/test-errors/0002/test.error b/test/test-errors/0002/test.error new file mode 100644 index 0000000..55e8955 --- /dev/null +++ b/test/test-errors/0002/test.error @@ -0,0 +1 @@ +:2:1: error: duplicate key diff --git a/test/test-errors/0003/=== b/test/test-errors/0003/=== new file mode 100644 index 0000000..5441272 --- /dev/null +++ b/test/test-errors/0003/=== @@ -0,0 +1 @@ +Duplicate key (plain scalar, quoted scalar) diff --git a/test/test-errors/0003/in.yaml b/test/test-errors/0003/in.yaml new file mode 100644 index 0000000..41a9f78 --- /dev/null +++ b/test/test-errors/0003/in.yaml @@ -0,0 +1,2 @@ +foo: bar +'foo': baz diff --git a/test/test-errors/0003/test.error b/test/test-errors/0003/test.error new file mode 100644 index 0000000..2ac16d5 --- /dev/null +++ b/test/test-errors/0003/test.error @@ -0,0 +1 @@ +:2:2: error: duplicate key diff --git a/test/test-errors/0004/=== b/test/test-errors/0004/=== new file mode 100644 index 0000000..3053517 --- /dev/null +++ b/test/test-errors/0004/=== @@ -0,0 +1 @@ +Duplicate key (plain scalar, literal scalar) diff --git a/test/test-errors/0004/in.yaml b/test/test-errors/0004/in.yaml new file mode 100644 index 0000000..43e2e52 --- /dev/null +++ b/test/test-errors/0004/in.yaml @@ -0,0 +1,4 @@ +foo: bar +? |- + foo +: baz diff --git a/test/test-errors/0004/test.error b/test/test-errors/0004/test.error new file mode 100644 index 0000000..5ad00f9 --- /dev/null +++ b/test/test-errors/0004/test.error @@ -0,0 +1 @@ +:3:1: error: duplicate key diff --git a/test/test-errors/0005/=== b/test/test-errors/0005/=== new file mode 100644 index 0000000..fe15eed --- /dev/null +++ b/test/test-errors/0005/=== @@ -0,0 +1 @@ +Duplicate key (sequence) diff --git a/test/test-errors/0005/in.yaml b/test/test-errors/0005/in.yaml new file mode 100644 index 0000000..dff9b3b --- /dev/null +++ b/test/test-errors/0005/in.yaml @@ -0,0 +1,2 @@ +[ foo ]: bar +[ foo ]: baz diff --git a/test/test-errors/0005/test.error b/test/test-errors/0005/test.error new file mode 100644 index 0000000..55e8955 --- /dev/null +++ b/test/test-errors/0005/test.error @@ -0,0 +1 @@ +:2:1: error: duplicate key diff --git a/test/test-errors/0006/=== b/test/test-errors/0006/=== new file mode 100644 index 0000000..b018d4d --- /dev/null +++ b/test/test-errors/0006/=== @@ -0,0 +1 @@ +Duplicate key (simple mapping) diff --git a/test/test-errors/0006/in.yaml b/test/test-errors/0006/in.yaml new file mode 100644 index 0000000..df2fabd --- /dev/null +++ b/test/test-errors/0006/in.yaml @@ -0,0 +1,2 @@ +{ foo: bar }: baz +{ foo: bar }: frooz diff --git a/test/test-errors/0006/test.error b/test/test-errors/0006/test.error new file mode 100644 index 0000000..55e8955 --- /dev/null +++ b/test/test-errors/0006/test.error @@ -0,0 +1 @@ +:2:1: error: duplicate key diff --git a/test/test-errors/0007/=== b/test/test-errors/0007/=== new file mode 100644 index 0000000..b8fdf31 --- /dev/null +++ b/test/test-errors/0007/=== @@ -0,0 +1 @@ +Duplicate key (complex sorted mapping) diff --git a/test/test-errors/0007/in.yaml b/test/test-errors/0007/in.yaml new file mode 100644 index 0000000..a2e231a --- /dev/null +++ b/test/test-errors/0007/in.yaml @@ -0,0 +1,2 @@ +{ a: b, { e: f, g: h}: d}: foo +{ { g: h, e: f}: d, a: b}: bar diff --git a/test/test-errors/0007/test.error b/test/test-errors/0007/test.error new file mode 100644 index 0000000..55e8955 --- /dev/null +++ b/test/test-errors/0007/test.error @@ -0,0 +1 @@ +:2:1: error: duplicate key diff --git a/test/test-errors/0008/=== b/test/test-errors/0008/=== new file mode 100644 index 0000000..4556c58 --- /dev/null +++ b/test/test-errors/0008/=== @@ -0,0 +1 @@ +Unknown alias diff --git a/test/test-errors/0008/in.yaml b/test/test-errors/0008/in.yaml new file mode 100644 index 0000000..c9afc53 --- /dev/null +++ b/test/test-errors/0008/in.yaml @@ -0,0 +1 @@ +a: *b diff --git a/test/test-errors/0008/test.error b/test/test-errors/0008/test.error new file mode 100644 index 0000000..35154e4 --- /dev/null +++ b/test/test-errors/0008/test.error @@ -0,0 +1 @@ +:1:5: error: invalid alias diff --git a/test/test-errors/0009/=== b/test/test-errors/0009/=== new file mode 100644 index 0000000..ffc2f0c --- /dev/null +++ b/test/test-errors/0009/=== @@ -0,0 +1 @@ +Invalid merge key (referencing not a mapping) diff --git a/test/test-errors/0009/in.yaml b/test/test-errors/0009/in.yaml new file mode 100644 index 0000000..48e4aaf --- /dev/null +++ b/test/test-errors/0009/in.yaml @@ -0,0 +1,2 @@ +- &FOO foo +- << : *FOO diff --git a/test/test-errors/0009/test.error b/test/test-errors/0009/test.error new file mode 100644 index 0000000..891d8e0 --- /dev/null +++ b/test/test-errors/0009/test.error @@ -0,0 +1 @@ +:2:9: error: invalid merge key value diff --git a/test/test-errors/0010/=== b/test/test-errors/0010/=== new file mode 100644 index 0000000..59e4371 --- /dev/null +++ b/test/test-errors/0010/=== @@ -0,0 +1 @@ +Invalid merge key (not an alias, scalar) diff --git a/test/test-errors/0010/in.yaml b/test/test-errors/0010/in.yaml new file mode 100644 index 0000000..b177cc3 --- /dev/null +++ b/test/test-errors/0010/in.yaml @@ -0,0 +1 @@ +- << : FOO diff --git a/test/test-errors/0010/test.error b/test/test-errors/0010/test.error new file mode 100644 index 0000000..5794178 --- /dev/null +++ b/test/test-errors/0010/test.error @@ -0,0 +1 @@ +:1:8: error: invalid merge key value diff --git a/test/test-errors/0011/=== b/test/test-errors/0011/=== new file mode 100644 index 0000000..77f5aea --- /dev/null +++ b/test/test-errors/0011/=== @@ -0,0 +1 @@ +Invalid merge key (not an alias sequence item) diff --git a/test/test-errors/0011/in.yaml b/test/test-errors/0011/in.yaml new file mode 100644 index 0000000..e3c2ec2 --- /dev/null +++ b/test/test-errors/0011/in.yaml @@ -0,0 +1,2 @@ +- &FOO foo +- << : [ *FOO, bar ] diff --git a/test/test-errors/0011/test.error b/test/test-errors/0011/test.error new file mode 100644 index 0000000..ca287e1 --- /dev/null +++ b/test/test-errors/0011/test.error @@ -0,0 +1 @@ +:2:8: error: invalid merge key value diff --git a/test/testemitter-restreaming.test b/test/testemitter-restreaming.test new file mode 100755 index 0000000..4a48178 --- /dev/null +++ b/test/testemitter-restreaming.test @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +${SRCDIR}/testemitter.test --restreaming diff --git a/test/testemitter-streaming.test b/test/testemitter-streaming.test new file mode 100755 index 0000000..84aa774 --- /dev/null +++ b/test/testemitter-streaming.test @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +${SRCDIR}/testemitter.test --streaming diff --git a/test/testemitter.test b/test/testemitter.test new file mode 100755 index 0000000..cef826e --- /dev/null +++ b/test/testemitter.test @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +EXTRA_DUMP_ARGS="" + +if [ "x$1" = "x--streaming" ]; then + EXTRA_DUMP_ARGS="$EXTRA_DUMP_ARGS --streaming" +elif [ "x$1" = "x--restreaming" ]; then + EXTRA_DUMP_ARGS="$EXTRA_DUMP_ARGS --streaming --recreating" +fi + +count=0 +for f in "${SRCDIR}"/emitter-examples/*.yaml; do + count=`expr $count + 1` +done + +# output plan +echo 1..$count + +i=0 +for f in "${SRCDIR}"/emitter-examples/*.yaml; do + + i=`expr $i + 1` + tf=`basename $f` + + t1=`mktemp` + t2=`mktemp` + + res="not ok" + + pass_parse=0 + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers "$f" >"$t1" + if [ $? -eq 0 ]; then + ${TOP_BUILDDIR}/src/fy-tool --dump ${EXTRA_DUMP_ARGS} "$f" | \ + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t2" + if [ $? -eq 0 ]; then + pass_parse=1 + fi + fi + + # all errors test are expected to fail + if [ "$pass_parse" = "1" ]; then + diff -u "$t1" "$t2" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + else + res="not ok" + fi + + rm -f "$t1" "$t2" + + echo "$res $i $tf" +done diff --git a/test/testerrors.test b/test/testerrors.test new file mode 100755 index 0000000..bbf79c0 --- /dev/null +++ b/test/testerrors.test @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +count=0 +for dir in "${SRCDIR}"/test-errors/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + count=`expr $count + 1` +done + +# output plan +echo 1..$count + +i=0 +for dir in "${SRCDIR}"/test-errors/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + + i=`expr $i + 1` + desctxt=`cat 2>/dev/null "$dir/==="` + tdir=`basename $dir` + + t=`mktemp` + + res="not ok" + + pass_yaml=0 + ${TOP_BUILDDIR}/src/fy-tool --dump --no-streaming -r "$dir/in.yaml" >"$t" 2>&1 + if [ $? -eq 0 ]; then + pass_yaml=1 + fi + + errmsg=`cat "$t" | head -n1 | sed -e 's/^[^:]*//'` + echo "errmsg: $errmsg" + + # replace with error message + echo "$errmsg" >"$t" + + # all errors test are expected to fail + if [ "$pass_yaml" = "0" ]; then + res="ok" + + # diff is pointless under valgrind + if [ "x$USE_VALGRIND" = "x" ]; then + diff_err=0 + diff -u "$dir/test.error" "$t" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + fi + else + res="not ok" + fi + + rm -f "$t" + + echo "$res $i $tdir - $desctxt" +done diff --git a/test/testsuite-evstream.test b/test/testsuite-evstream.test new file mode 100755 index 0000000..b7b5d25 --- /dev/null +++ b/test/testsuite-evstream.test @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +count=0 +for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do + # there must be a test there + if [ ! -e "$tst/===" ]; then + continue + fi + # skip tests that are expected to fail + if [ -e "$tst/error" ]; then + continue + fi + count=`expr $count + 1` + done +done + +# output plan +echo 1..$count + +declare -A skips=( + [2JQS]="duplicate keys in testcase; cannot load as document" +) +declare -A xfails=() + +i=0 +for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + + for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do + + # there must be a test there + if [ ! -e "$tst/===" ]; then + continue + fi + + # skip tests that are expected to fail + if [ -e "$tst/error" ]; then + continue + fi + + i=`expr $i + 1` + + # strip trailing / + t=${tst%/} + # remove test-suite-data/ + test_subtest_id=`echo $t | cut -d/ -f2-` + test_id=`echo $test_subtest_id | cut -d/ -f1` + subtest_id=`echo $test_subtest_id | cut -s -d/ -f2` + + desctxt=`cat 2>/dev/null "$tst/==="` + + directive="" + skip_reason="${skips[$test_subtest_id]}" + if [ "x$skip_reason" != "x" ]; then + directive=" # SKIP: ${skip_reason}" + fi + + res="ok" + if [ "x$skip_reason" = "x" ]; then + res="not ok" + + t=`mktemp` + + # run the test using document-event-stream + ${TOP_BUILDDIR}/src/fy-tool --testsuite --document-event-stream "$tst/in.yaml" >"$t" + if [ $? -eq 0 ]; then + diff -u "$tst/test.event" "$t" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + else + res="not ok" + fi + + rm -f "$t" + + xfail_reason="${xfails[$test_subtest_id]}" + if [ "x$xfail_reason" != "x" ]; then + directive=" # TODO: ${xfail_reason}" + fi + fi + + echo "$res $i $test_subtest_id - $desctxt$directive" + done +done diff --git a/test/testsuite-json.test b/test/testsuite-json.test new file mode 100755 index 0000000..f607119 --- /dev/null +++ b/test/testsuite-json.test @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +count=0 +for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do + # there must be a test there + if [ ! -e "$tst/===" ]; then + continue + fi + # ignore tests which are expected to fail + if [ -e "$tst/error" ]; then + continue + fi + # a json file must be there + if [ ! -e "$tst/in.json" ]; then + continue + fi + count=`expr $count + 1` + done +done + +# output plan +echo 1..$count + +declare -A skips=( + [UGM3]="Later jq versions rewrite numbers like 12.00 -> 12 which breaks diff" +) +declare -A xfails=( + [C4HZ]="requires schema support which libfyaml does not support yet." +) + +i=0 +for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + + for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do + + # ignore tests which are expected to fail + if [ -e "$tst/error" ]; then + continue + fi + # a json file must be there + if [ ! -e "$tst/in.json" ]; then + continue + fi + + i=`expr $i + 1` + + # strip trailing / + t=${tst%/} + # remove test-suite-data/ + test_subtest_id=`echo $t | cut -d/ -f2-` + test_id=`echo $test_subtest_id | cut -d/ -f1` + subtest_id=`echo $test_subtest_id | cut -s -d/ -f2` + + desctxt=`cat 2>/dev/null "$tst/==="` + + directive="" + skip_reason="${skips[$test_subtest_id]}" + if [ "x$skip_reason" != "x" ]; then + directive=" # SKIP: ${skip_reason}" + fi + + res="ok" + if [ "x$skip_reason" = "x" ]; then + t1=`mktemp` + t2=`mktemp` + + # output yaml in json format + ${TOP_BUILDDIR}/src/fy-tool --dump --strip-labels --strip-tags --strip-doc -r -mjson "$tst/in.yaml" | "${JQ}" --sort-keys . >"$t1" + + # do the same with the json input (canonicalize) + cat "$tst/in.json" | "${JQ}" --sort-keys . > "$t2" + + diff -u "$t1" "$t2" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + + rm -f "$t1" "$t2" + + xfail_reason="${xfails[$test_subtest_id]}" + if [ "x$xfail_reason" != "x" ]; then + directive=" # TODO: ${xfail_reason}" + fi + fi + + echo "$res $i $test_subtest_id - $desctxt (JSON)$directive" + done +done diff --git a/test/testsuite-resolution.test b/test/testsuite-resolution.test new file mode 100755 index 0000000..7856bad --- /dev/null +++ b/test/testsuite-resolution.test @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +count=0 +for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do + # there must be a test there + if [ ! -e "$tst/===" ]; then + continue + fi + # skip tests that are expected to fail + if [ -e "$tst/error" ]; then + continue + fi + # there must be an alias in the test + grep -q '\*[A-Za-z]' "$tst/in.yaml" + if [ $? -ne 0 ]; then + continue + fi + count=`expr $count + 1` + done +done + +# output plan +echo 1..$count + +declare -A skips=( + [2JQS]="duplicate keys in testcase; cannot load as document" + [X38W]="duplicate keys after resolution" +) +declare -A xfails=( +) + +i=0 +for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + + for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do + + # there must be a test there + if [ ! -e "$tst/===" ]; then + continue + fi + + # skip tests that are expected to fail + if [ -e "$tst/error" ]; then + continue + fi + + # there must be an alias in the test + grep -q '\*[A-Za-z]' "$tst/in.yaml" + if [ $? -ne 0 ]; then + continue + fi + + i=`expr $i + 1` + + # strip trailing / + t=${tst%/} + # remove test-suite-data/ + test_subtest_id=`echo $t | cut -d/ -f2-` + test_id=`echo $test_subtest_id | cut -d/ -f1` + subtest_id=`echo $test_subtest_id | cut -s -d/ -f2` + + desctxt=`cat 2>/dev/null "$tst/==="` + + t1=`mktemp` + t2=`mktemp` + + directive="" + skip_reason="${skips[$test_subtest_id]}" + if [ "x$skip_reason" != "x" ]; then + directive=" # SKIP: ${skip_reason}" + fi + + res="ok" + if [ "x$skip_reason" = "x" ]; then + res="not ok" + + ${TOP_BUILDDIR}/src/fy-tool --dump --resolve "$tst/in.yaml" | ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t1" + ${TOP_BUILDDIR}/src/fy-tool --dump --resolve --streaming "$tst/in.yaml" | ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t2" + + diff -u "$t1" "$t2" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + + rm -f "$t1" "$t2" + + xfail_reason="${xfails[$test_subtest_id]}" + if [ "x$xfail_reason" != "x" ]; then + directive=" # TODO: ${xfail_reason}" + fi + fi + + echo "$res $i $test_subtest_id - $desctxt$directive" + done +done diff --git a/test/testsuite.test b/test/testsuite.test new file mode 100755 index 0000000..38cd521 --- /dev/null +++ b/test/testsuite.test @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +count=0 +for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do + # there must be a test there + if [ ! -e "$tst/===" ]; then + continue + fi + count=`expr $count + 1` + done +done + +# output plan +echo 1..$count + +skiplist="" +xfaillist="" + +i=0 +for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do + + for tst in ${basetst} ${basetst}[0-9][0-9]/ ${basetst}[0-9][0-9][0-9]/; do + + # there must be a test there + if [ ! -e "$tst/===" ]; then + continue + fi + + i=`expr $i + 1` + + # strip trailing / + t=${tst%/} + # remove test-suite-data/ + test_subtest_id=`echo $t | cut -d/ -f2-` + test_id=`echo $test_subtest_id | cut -d/ -f1` + subtest_id=`echo $test_subtest_id | cut -s -d/ -f2` + + desctxt=`cat 2>/dev/null "$tst/==="` + + t=`mktemp` + + directive="" + for skip in $skiplist; do + if [ "$test_subtest_id" = "$skip" ]; then + directive=" # SKIP" + break + fi + done + + res="ok" + if [ "x$directive" = "x" ]; then + res="not ok" + + pass_yaml=0 + ${TOP_BUILDDIR}/src/fy-tool --testsuite "$tst/in.yaml" >"$t" + if [ $? -eq 0 ]; then + pass_yaml=1 + fi + + if [ -e "$tst/error" ]; then + # test is expected to fail + if [ $pass_yaml = "0" ]; then + res="ok" + else + res="not ok" + fi + else + # test is expected to pass + if [ $pass_yaml = "1" ]; then + diff_yaml=0 + # diff -u "$tst/test.event" "$t" | sed -e 's/^/# /' + diff -u "$tst/test.event" "$t" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + else + res="not ok" + fi + fi + + rm -f "$t" + + for xfail in $xfaillist; do + if [ "$test_subtest_id" = "$xfail" ]; then + directive=" # TODO: known failure." + break + fi + done + fi + + echo "$res $i $test_subtest_id - $desctxt$directive" + done +done