From 4e2d1540ba0da4860e82b451e1b85ae8caa4197d Mon Sep 17 00:00:00 2001 From: =?utf8?q?Timo=20R=C3=B6hling?= Date: Fri, 2 Jan 2026 23:42:16 +0100 Subject: [PATCH] New upstream version 0.9.2 --- .gitignore | 1 + .libtool-version | 2 +- .tarball-version | 2 +- CHANGELOG.md | 133 ++ CMakeLists.txt | 1621 +++++++++++++++++-- Dockerfile | 4 +- Dockerfile-build-deb | 2 +- Makefile.am | 4 +- README.md | 884 +++++++---- bootstrap.sh | 2 +- cmake/CheckClangBlocks.cmake | 53 + cmake/CheckLibClang.cmake | 77 + cmake/add-tap-subtests.cmake | 690 ++++++++ cmake/config.h.in | 84 + cmake/generate-jsontestsuite-list.sh | 135 ++ cmake/generate-testsuite-list.sh | 164 ++ cmake/jsontestsuite-tests.cmake | 342 ++++ cmake/libfyaml-config.cmake.in | 54 + cmake/libfyaml.pc.in | 10 + cmake/run-single-tap-test.sh | 314 ++++ cmake/setup-yaml-test-suite.cmake | 42 + cmake/tags.cmake | 21 + cmake/testsuite-tests.cmake | 771 +++++++++ configure.ac | 198 ++- doc/conf.py | 2 +- doc/include | 1 + examples/CMakeLists.txt | 69 + examples/README.md | 281 ++++ examples/basic-parsing.c | 50 + examples/build-from-scratch.c | 111 ++ examples/config.yaml | 40 + examples/document-manipulation.c | 111 ++ examples/event-streaming.c | 117 ++ examples/invoice.yaml | 42 + examples/path-queries.c | 80 + examples/quick-start.c | 70 + include/libfyaml.h | 1981 +++++++++++++++++++++-- libfyaml.pc.in | 2 +- scripts/get-gpg-key-id-by-email.sh | 2 +- scripts/run-check-errors.sh | 2 +- scripts/run-compare-dump.sh | 2 +- scripts/run-compare-examples.sh | 2 +- scripts/run-compare-parse.sh | 2 +- scripts/run-compare-scan.sh | 2 +- scripts/run-compare-testsuite.sh | 2 +- scripts/run-emit-check.sh | 2 +- scripts/run-list-testsuite.sh | 2 +- scripts/run-test.sh | 2 +- scripts/show-desc.sh | 2 +- src/Makefile.am | 102 +- src/allocator/fy-allocator-auto.c | 399 +++++ src/allocator/fy-allocator-auto.h | 26 + src/allocator/fy-allocator-dedup.c | 1222 +++++++++++++++ src/allocator/fy-allocator-dedup.h | 89 ++ src/allocator/fy-allocator-linear.c | 391 +++++ src/allocator/fy-allocator-linear.h | 34 + src/allocator/fy-allocator-malloc.c | 717 +++++++++ src/allocator/fy-allocator-malloc.h | 49 + src/allocator/fy-allocator-mremap.c | 1180 ++++++++++++++ src/allocator/fy-allocator-mremap.h | 77 + src/allocator/fy-allocator.c | 756 +++++++++ src/allocator/fy-allocator.h | 284 ++++ src/internal/fy-allocators.c | 301 ++++ src/internal/fy-thread.c | 13 +- src/internal/libfyaml-parser.c | 12 +- src/lib/fy-accel.c | 12 +- src/lib/fy-atom.c | 14 +- src/lib/fy-atom.h | 61 +- src/lib/fy-composer-diag.c | 91 ++ src/lib/fy-composer.c | 66 +- src/lib/fy-composer.h | 116 +- src/lib/fy-diag.c | 510 ++---- src/lib/fy-diag.h | 650 +------- src/lib/fy-doc-diag.c | 91 ++ src/lib/fy-doc.c | 231 ++- src/lib/fy-doc.h | 142 +- src/lib/fy-docbuilder-diag.c | 92 ++ src/lib/fy-docbuilder.c | 53 +- src/lib/fy-docbuilder.h | 154 +- src/lib/fy-docstate.h | 1 + src/lib/fy-dump.c | 3 + src/lib/fy-emit.c | 819 +++++++--- src/lib/fy-emit.h | 13 +- src/lib/fy-event.c | 3 + src/lib/fy-input-diag.c | 92 ++ src/lib/fy-input.c | 286 +++- src/lib/fy-input.h | 490 ++++-- src/lib/fy-parse-diag.c | 162 ++ src/lib/fy-parse.c | 1624 ++++++++++++++++--- src/lib/fy-parse.h | 256 ++- src/lib/fy-path.c | 4 +- src/lib/fy-token.c | 108 +- src/lib/fy-token.h | 62 +- src/lib/fy-types.h | 2 +- src/lib/fy-walk.c | 437 +++++- src/lib/fy-walk.h | 11 +- src/thread/fy-thread.c | 112 +- src/thread/fy-thread.h | 17 +- src/tool/fy-tool-dump.c | 767 +++++++++ src/tool/fy-tool-util.c | 251 +++ src/tool/fy-tool-util.h | 255 +++ src/tool/fy-tool.c | 1138 +++----------- src/util/fy-atomics.h | 202 +++ src/util/fy-bit64.h | 26 +- src/util/fy-blob.c | 26 +- src/util/fy-blob.h | 43 +- src/util/fy-ctype.h | 122 +- src/util/fy-id.h | 141 +- src/util/fy-list.h | 686 +------- src/util/fy-typelist.h | 6 +- src/util/fy-utf8.c | 270 ++-- src/util/fy-utf8.h | 420 ++++- src/util/fy-utils.c | 25 +- src/util/fy-utils.h | 512 +++++- src/util/fy-vlsize.h | 564 +++++++ src/valgrind/fy-valgrind.h | 15 +- test/Makefile.am | 19 +- test/jsontestsuite.test | 2 +- test/libfyaml-test-allocator.c | 356 +++++ test/libfyaml-test-core.c | 48 +- test/libfyaml-test-parser.c | 2165 ++++++++++++++++++++++++++ test/libfyaml-test-private-id.c | 15 +- test/libfyaml-test-thread.c | 267 ++++ test/libfyaml-test.c | 6 + test/libfyaml.test | 2 +- test/testemitter-restreaming.test | 2 + test/testemitter-streaming.test | 2 +- test/testemitter.test | 4 +- test/testerrors.test | 4 +- test/testsuite-evstream.test | 35 +- test/testsuite-json.test | 33 +- test/testsuite-resolution.test | 33 +- test/testsuite.test | 2 +- 133 files changed, 24735 insertions(+), 4755 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 cmake/CheckClangBlocks.cmake create mode 100644 cmake/CheckLibClang.cmake create mode 100644 cmake/add-tap-subtests.cmake create mode 100644 cmake/config.h.in create mode 100755 cmake/generate-jsontestsuite-list.sh create mode 100755 cmake/generate-testsuite-list.sh create mode 100644 cmake/jsontestsuite-tests.cmake create mode 100644 cmake/libfyaml-config.cmake.in create mode 100644 cmake/libfyaml.pc.in create mode 100755 cmake/run-single-tap-test.sh create mode 100644 cmake/setup-yaml-test-suite.cmake create mode 100644 cmake/tags.cmake create mode 100644 cmake/testsuite-tests.cmake create mode 120000 doc/include create mode 100644 examples/CMakeLists.txt create mode 100644 examples/README.md create mode 100644 examples/basic-parsing.c create mode 100644 examples/build-from-scratch.c create mode 100644 examples/config.yaml create mode 100644 examples/document-manipulation.c create mode 100644 examples/event-streaming.c create mode 100644 examples/invoice.yaml create mode 100644 examples/path-queries.c create mode 100644 examples/quick-start.c create mode 100644 src/allocator/fy-allocator-auto.c create mode 100644 src/allocator/fy-allocator-auto.h create mode 100644 src/allocator/fy-allocator-dedup.c create mode 100644 src/allocator/fy-allocator-dedup.h create mode 100644 src/allocator/fy-allocator-linear.c create mode 100644 src/allocator/fy-allocator-linear.h create mode 100644 src/allocator/fy-allocator-malloc.c create mode 100644 src/allocator/fy-allocator-malloc.h create mode 100644 src/allocator/fy-allocator-mremap.c create mode 100644 src/allocator/fy-allocator-mremap.h create mode 100644 src/allocator/fy-allocator.c create mode 100644 src/allocator/fy-allocator.h create mode 100644 src/internal/fy-allocators.c create mode 100644 src/lib/fy-composer-diag.c create mode 100644 src/lib/fy-doc-diag.c create mode 100644 src/lib/fy-docbuilder-diag.c create mode 100644 src/lib/fy-input-diag.c create mode 100644 src/lib/fy-parse-diag.c create mode 100644 src/tool/fy-tool-dump.c create mode 100644 src/tool/fy-tool-util.c create mode 100644 src/tool/fy-tool-util.h create mode 100644 src/util/fy-atomics.h create mode 100644 src/util/fy-vlsize.h create mode 100644 test/libfyaml-test-allocator.c create mode 100644 test/libfyaml-test-parser.c create mode 100644 test/libfyaml-test-thread.c create mode 100755 test/testemitter-restreaming.test diff --git a/.gitignore b/.gitignore index 2378768..2ec5596 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ src/fy-tool src/libfyaml-parser src/fy-thread src/fy-b3sum +src/fy-allocators src/*.a stamp-h1 tags diff --git a/.libtool-version b/.libtool-version index 2531082..7e093fb 100644 --- a/.libtool-version +++ b/.libtool-version @@ -1 +1 @@ -1:4:2 +2:0:0 diff --git a/.tarball-version b/.tarball-version index b63ba69..2003b63 100644 --- a/.tarball-version +++ b/.tarball-version @@ -1 +1 @@ -0.9 +0.9.2 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1b0a930 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,133 @@ +# Changelog + +All notable changes to libfyaml will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.9.2] - 2025-12-29 + +### Fixed +- **#157**: dist tarball now includes the missing CMake files + +## [0.9.1] - 2025-12-28 + +### CRITICAL: Licensing Change + +**GPL Code Removed - Library Now Fully MIT Licensed** + +Replaced GPL-licensed list implementation with clean-room MIT-licensed minimal +implementation. The entire library is now consistently MIT licensed without any +GPL components. This is a major change for users requiring permissive licensing +for commercial or proprietary projects. + +### Added + +- Thread-safe lockless allocator infrastructure (linear, malloc, mremap, dedup, auto) +- Parser checkpointing API: `fy_parser_parse_peek()`, `fy_parser_skip()`, `fy_parser_count_sequence_items()`, `fy_parser_count_mapping_items()` +- Public composer and document builder interfaces +- YPath set operators: `@` for inclusion, `!` for exclusion +- Compact emitter formats (flow and JSON) +- `fy_emit_body_node()` for emitting single nodes without stream/doc events +- `fy_event_report()` methods for error diagnostics in composer interface +- `fy_node_get_tag0()` and `fy_node_get_tag_length()` tag utility methods +- `fy_node_delete()` method for explicit node removal +- `fy_allocator_contains()` for pointer containment checks +- `fy_event_get_type()` convenience wrapper +- CMake documentation build targets (doc-html, doc-latexpdf, doc-man) +- Cross-compilation support for both CMake and autotools build systems +- Optional libclang dependency detection for type-aware features +- `--enable-static-tools` configuration option +- `examples/` directory with library usage examples +- Comprehensive allocator and parser tests + +### Changed + +- **fy-tool now uses streaming mode by default** for dump operations (up to 24x faster on large files: 105MB in 0.776s vs 18.3s; document mode available via `--no-streaming`) +- Emission performance improved up to 2x through token analysis optimization +- UTF-8 handling significantly faster +- Plain scalar parsing and fetch reworked for better performance +- fy-tool no longer wraps output when stdout is not a TTY +- fy-tool now properly calls `fy_shutdown()` at exit for clean valgrind runs +- CMake build system now at feature parity with autotools +- Stream YAML version handling now per-stream (not per-document) +- Split diagnostics into separate modules by type +- Reorganized fy-tool directory structure +- Renamed `alloca_vsprintf` to `fy_vsprintfa` +- Improved block scalar end linebreak and whitespace handling + +### Fixed + +- **#133**: Segmentation fault in `fy_reader_*` when all input processed +- **#135**: Use-after-free when document acceleration fails +- **#134**: Double-free in `fy_node_setup_path_expr_data` +- **#132**: Use-after-free after document resolution +- **#123**: Crash when assertions compiled out in release mode +- **#122**: Crash when addressing expression as node incorrectly +- **#120**: Crash on error during walk evaluation +- **#119**: Segmentation fault on garbage input in `evaluate_new()` +- **#118**: Segmentation fault when block scalar indicators last in input +- **#115**: Assertion preventing node deletion via `fy_document_insert_at()` +- **#107**: Crash when comparing mappings with NULL values +- **#102**: Portability crash on NEC Aurora (enums in bitfields) +- Infinite loop in scan directive for malformed UTF-8 input +- Use-after-free in auto allocator (sub-allocator freeing order) +- Use-after-free in anchor setup before resolution +- Composer halt teardown event pumping +- Document state lifecycle in presence of errors +- Parser reset queued inputs initialization +- Blank/whitespace-only scalar creation via emit event API +- Zero-length scalars as plain mapping keys +- Prefix-only tags passing scan validation +- Plain scalars with linebreaks creation +- Pretty mode document end marker forcing +- MacOS `LIST_HEAD` warning and linker options +- Numerous compiler warnings at high optimization levels +- Testsuite failures with latest jq versions + +### Performance + +- fy-tool: Up to 24x faster on large files (streaming mode by default) +- Emission: ~2x faster (optimized plain scalar token analysis) +- Parsing: Faster plain scalar fetch with inline specialization +- UTF-8: Significantly improved validation and processing speed + +### Internal + +- New atomic helpers (fy-atomics) based on C atomics with fallbacks +- fy-shutdown mechanism for library cleanup +- FY_IMPOSSIBLE_ABORT() macro for impossible conditions +- Variable size encoding header (fy-vlsize) +- CPP varargs macros for recursive expansion +- iovec copy/from/xxhash methods +- FY_LAMBDA define for lambda support detection +- Enhanced blob infrastructure +- Improved utility functions (overflow checks, alloca sprintf, constructors) + +### API Compatibility + +- **Minor breakage**: Added `minimum_bucket_occupancy` to dedup allocator config + (no known external users affected) +- **Behavioral change**: fy-tool now defaults to streaming mode instead of document + mode (24x faster, but loses forward anchor references and strict duplicate key + checking; use `--no-streaming` for old behavior) +- All other changes are backwards compatible additions + +### Contributors + +Pantelis Antoniou, Alessandro Astone, Kevin Wooten, Henry Qin, Roland Wirth, +Yurii Rashkovskii, Alexandre Detiste, Benjamin Rodenberg, +Jose Luis Blanco-Claraco, Andrey Somov, Orange_233, Martin Diehl + +### Statistics + +- 171 commits since v0.9 +- 12 bug fix issues closed +- 12 contributors + +## [0.9] - 2023-09-28 + +Initial public release with comprehensive YAML 1.2 support. + +[0.9.1]: https://github.com/pantoniou/libfyaml/compare/v0.9...v0.9.1 +[0.9]: https://github.com/pantoniou/libfyaml/releases/tag/v0.9 diff --git a/CMakeLists.txt b/CMakeLists.txt index a77beaa..d4f0999 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,161 +1,1574 @@ +cmake_minimum_required(VERSION 3.10) + if (POLICY CMP0048) cmake_policy(SET CMP0048 NEW) # Allow project(xxx VERSION a.b.c) endif() -# version number below should be the currently under development version, -# so when doing a tag/release it is captured with the proper numbering. -project(fyaml LANGUAGES C VERSION 0.7.2) +if (POLICY CMP0074) + cmake_policy(SET CMP0074 NEW) # find_package uses _ROOT variables +endif() + +if (POLICY CMP0110) + cmake_policy(SET CMP0110 NEW) # add_test() supports arbitrary characters in test names +endif() + +# Get version from git-version-gen +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.tarball-version") + file(READ "${CMAKE_CURRENT_SOURCE_DIR}/.tarball-version" PROJECT_VERSION) + string(STRIP "${PROJECT_VERSION}" PROJECT_VERSION) +else() + execute_process( + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/build-aux/git-version-gen" "${CMAKE_CURRENT_SOURCE_DIR}/.tarball-version" + OUTPUT_VARIABLE PROJECT_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +endif() -cmake_minimum_required(VERSION 3.0) +# Parse version components +string(REGEX MATCH "^([0-9]+)(\\.([0-9]+)(\\.([0-9]+))?)?(-(.*))?$" _ "${PROJECT_VERSION}") +set(VERSION_MAJOR "${CMAKE_MATCH_1}") +if(CMAKE_MATCH_3) + set(VERSION_MINOR "${CMAKE_MATCH_3}") +else() + set(VERSION_MINOR "0") +endif() +if(CMAKE_MATCH_5) + set(VERSION_PATCH "${CMAKE_MATCH_5}") +else() + set(VERSION_PATCH "0") +endif() +if(CMAKE_MATCH_7) + set(VERSION_EXTRA "${CMAKE_MATCH_7}") +else() + set(VERSION_EXTRA "") +endif() -# Must use GNUInstallDirs to install libraries into correct -# locations on all platforms. +project(libfyaml LANGUAGES C ASM VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}) + +# Must use GNUInstallDirs to install libraries into correct locations on all platforms include(GNUInstallDirs) +include(CheckIncludeFile) +include(CheckFunctionExists) +include(CheckCSourceCompiles) +include(CheckCCompilerFlag) +include(CheckLibraryExists) +include(CheckSymbolExists) +include(CMakeDependentOption) +include(CTest) +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/CheckLibClang.cmake) + +# does this CMake version understand generator expressions? +if(${CMAKE_VERSION} VERSION_LESS "3.27") + set(HAVE_CMAKE_GEXPR 0) +else() + set(HAVE_CMAKE_GEXPR 1) +endif() + +# Macro to add libclang support to a target +macro(add_libclang_to_target target_name link_scope) + if(HAVE_LIBCLANG) + # Split LIBCLANG_CFLAGS into list and add as compile options + string(REPLACE " " ";" LIBCLANG_CFLAGS_LIST "${LIBCLANG_CFLAGS}") + target_compile_options(${target_name} PRIVATE ${LIBCLANG_CFLAGS_LIST}) + + # Split LIBCLANG_LDFLAGS into list and add as link directories/options + # LIBCLANG_LDFLAGS typically contains -L/path/to/lib and -Wl,flags + # Extract only the -L paths + string(REGEX MATCHALL "-L([^ ]+)" LIBCLANG_L_FLAGS "${LIBCLANG_LDFLAGS}") + set(LIBCLANG_LINK_DIRS_LOCAL "") + foreach(flag ${LIBCLANG_L_FLAGS}) + string(REGEX REPLACE "^-L" "" dir "${flag}") + list(APPEND LIBCLANG_LINK_DIRS_LOCAL "${dir}") + endforeach() + if(LIBCLANG_LINK_DIRS_LOCAL) + target_link_directories(${target_name} ${link_scope} ${LIBCLANG_LINK_DIRS_LOCAL}) + endif() + + # Add libclang libraries + target_link_libraries(${target_name} ${link_scope} ${LIBCLANG_LIBS}) + endif() +endmacro() + +# Options +option(BUILD_SHARED_LIBS "Build shared libraries" ON) +option(ENABLE_PORTABLE_TARGET "Enable portable mode (disable per-target optimizations)" OFF) +option(ENABLE_STATIC_TOOLS "Tools will be compiled as static executables" OFF) +option(ENABLE_ASAN "Enable ASAN support" OFF) +option(ENABLE_NETWORK "Enable tests requiring network access" ON) +option(ENABLE_DEVMODE "Enable development mode only debugging" OFF) +option(BUILD_TESTING "Build tests" ON) +option(ENABLE_LIBCLANG "Enable libclang support for reflection" ON) + +# Path to LLVM installation (optional, overrides automatic detection) +set(LLVM_ROOT "" CACHE PATH "Path to LLVM installation directory (e.g., /usr/lib/llvm-15)") + +# Read libtool version +file(READ "${CMAKE_CURRENT_SOURCE_DIR}/.libtool-version" LIBTOOL_VERSION) +string(STRIP "${LIBTOOL_VERSION}" LIBTOOL_VERSION) + +# Parse libtool version components (format is current:revision:age) +string(REGEX REPLACE "^([0-9]+):.*" "\\1" LIBTOOL_CURRENT "${LIBTOOL_VERSION}") +string(REGEX REPLACE "^[0-9]+:([0-9]+):.*" "\\1" LIBTOOL_REVISION "${LIBTOOL_VERSION}") +string(REGEX REPLACE "^[0-9]+:[0-9]+:([0-9]+)" "\\1" LIBTOOL_AGE "${LIBTOOL_VERSION}") + +# Calculate SO version (current - age) +math(EXPR SO_VERSION "${LIBTOOL_CURRENT} - ${LIBTOOL_AGE}") +set(SO_VERSION_FULL "${SO_VERSION}.${LIBTOOL_AGE}.${LIBTOOL_REVISION}") + +# Platform detection +if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + set(TARGET_CPU_X86_64 TRUE) + set(TARGET_CPU_ANY_X86 TRUE) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "i.86|x86") + set(TARGET_CPU_X86 TRUE) + set(TARGET_CPU_ANY_X86 TRUE) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(TARGET_CPU_ARM64 TRUE) + set(TARGET_CPU_ANY_ARM TRUE) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm") + set(TARGET_CPU_ARM TRUE) + set(TARGET_CPU_ANY_ARM TRUE) +endif() + +if(CMAKE_CROSSCOMPILING) + set(CROSS_COMPILING 1) + message(STATUS "Cross-compiling detected:") + message(STATUS " Build system: ${CMAKE_HOST_SYSTEM_NAME} ${CMAKE_HOST_SYSTEM_PROCESSOR}") + message(STATUS " Target system: ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}") +else() + set(CROSS_COMPILING 0) +endif() + +# Check for required dependencies +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_package(Threads REQUIRED) +if(NOT CMAKE_USE_PTHREADS_INIT) + message(FATAL_ERROR "Missing required pthread support") +endif() + +# Check for headers and functions +check_include_file("byteswap.h" HAVE_BYTESWAP_H) +check_function_exists(__builtin_bswap16 HAVE___BUILTIN_BSWAP16) +check_function_exists(__builtin_bswap32 HAVE___BUILTIN_BSWAP32) +check_function_exists(__builtin_bswap64 HAVE___BUILTIN_BSWAP64) +check_function_exists(qsort_r HAVE_QSORT_R) +check_function_exists(mremap HAVE_MREMAP) + +# Check for environment variables +check_c_source_compiles(" + extern char **environ; + int main() { return 0; } +" HAVE_DECL_ENVIRON) + +# Check for optional dependencies + +# libyaml detection - try CMake config first, then pkg-config +set(HAVE_LIBYAML 0) +find_package(yaml QUIET CONFIG) +if(yaml_FOUND) + set(HAVE_LIBYAML 1) + # CMake config found, use imported target + set(LIBYAML_LIBRARIES yaml) + message(STATUS "Found libyaml via CMake config") +else() + find_package(PkgConfig) + if(PKG_CONFIG_FOUND) + pkg_check_modules(LIBYAML yaml-0.1) + if(LIBYAML_FOUND) + set(HAVE_LIBYAML 1) + message(STATUS "Found libyaml via pkg-config") + else() + message(WARNING "failed to find libyaml; compatibility disabled") + endif() + endif() +endif() + +# check framework detection - try CMake config first, then pkg-config +set(HAVE_CHECK 0) +set(HAVE_COMPATIBLE_CHECK 0) +if(BUILD_TESTING) + find_package(check QUIET CONFIG) + if(check_FOUND) + set(HAVE_CHECK 1) + set(CHECK_LIBRARIES check) + get_target_property(CHECK_INCLUDE_DIRS check INTERFACE_INCLUDE_DIRECTORIES) + # Get library location to extract directory + get_target_property(CHECK_LIB_LOCATION check IMPORTED_LOCATION) + if(NOT CHECK_LIB_LOCATION) + get_target_property(CHECK_LIB_LOCATION check LOCATION) + endif() + if(CHECK_LIB_LOCATION) + get_filename_component(CHECK_LIBRARY_DIRS "${CHECK_LIB_LOCATION}" DIRECTORY) + endif() + message(STATUS "Found check framework via CMake config") + + # Check if libcheck has srunner_set_tap (jessie has outdated libcheck) + set(_TEMP_CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + set(_TEMP_CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES}") + # For CMake config targets, use the target directly + set(CMAKE_REQUIRED_LIBRARIES check) + set(CMAKE_REQUIRED_INCLUDES ${CHECK_INCLUDE_DIRS}) + + check_symbol_exists(srunner_set_tap "check.h" HAVE_SRUNNER_SET_TAP) + + set(CMAKE_REQUIRED_LIBRARIES "${_TEMP_CMAKE_REQUIRED_LIBRARIES}") + set(CMAKE_REQUIRED_INCLUDES "${_TEMP_CMAKE_REQUIRED_INCLUDES}") + unset(_TEMP_CMAKE_REQUIRED_LIBRARIES) + unset(_TEMP_CMAKE_REQUIRED_INCLUDES) + + if(HAVE_SRUNNER_SET_TAP) + set(HAVE_COMPATIBLE_CHECK 1) + endif() + else() + # Fall back to pkg-config + find_package(PkgConfig) + if(PKG_CONFIG_FOUND) + pkg_check_modules(CHECK check) + if(CHECK_FOUND) + set(HAVE_CHECK 1) + message(STATUS "Found check framework via pkg-config") + + # Check if libcheck has srunner_set_tap (jessie has outdated libcheck) + set(_TEMP_CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + set(_TEMP_CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES}") + set(_TEMP_CMAKE_REQUIRED_LINK_OPTIONS "${CMAKE_REQUIRED_LINK_OPTIONS}") + set(CMAKE_REQUIRED_LIBRARIES ${CHECK_LIBRARIES}) + set(CMAKE_REQUIRED_INCLUDES ${CHECK_INCLUDE_DIRS}) + # Add library directories to linker flags + foreach(libdir ${CHECK_LIBRARY_DIRS}) + list(APPEND CMAKE_REQUIRED_LINK_OPTIONS "-L${libdir}") + endforeach() + + check_symbol_exists(srunner_set_tap "check.h" HAVE_SRUNNER_SET_TAP) + + set(CMAKE_REQUIRED_LIBRARIES "${_TEMP_CMAKE_REQUIRED_LIBRARIES}") + set(CMAKE_REQUIRED_INCLUDES "${_TEMP_CMAKE_REQUIRED_INCLUDES}") + set(CMAKE_REQUIRED_LINK_OPTIONS "${_TEMP_CMAKE_REQUIRED_LINK_OPTIONS}") + unset(_TEMP_CMAKE_REQUIRED_LIBRARIES) + unset(_TEMP_CMAKE_REQUIRED_INCLUDES) + unset(_TEMP_CMAKE_REQUIRED_LINK_OPTIONS) + + if(HAVE_SRUNNER_SET_TAP) + set(HAVE_COMPATIBLE_CHECK 1) + endif() + endif() + endif() + endif() +endif() + +# Check for libclang (for reflection support) +set(HAVE_LIBCLANG 0) +set(LIBCLANG_CFLAGS "") +set(LIBCLANG_LDFLAGS "") +set(LIBCLANG_LIBS "") +set(LIBCLANG_FOUND FALSE) +set(LIBCLANG_DETECTION_METHOD "") + +if(NOT HAVE_CMAKE_GEXPR) + message(STATUS "libclang support disabled (CMake version is too old (<=3.27)") + set(ENABLE_LIBCLANG OFF CACHE BOOL "Disable my feature" FORCE) +endif() + +if(NOT ENABLE_LIBCLANG) + message(STATUS "libclang support disabled (ENABLE_LIBCLANG=OFF)") +else() + + # If user specified LLVM_ROOT, add it to CMAKE_PREFIX_PATH for this search + if(LLVM_ROOT) + message(STATUS "Using user-specified LLVM_ROOT: ${LLVM_ROOT}") + list(PREPEND CMAKE_PREFIX_PATH "${LLVM_ROOT}") + endif() + + # Try CMake config mode first + find_package(LLVM QUIET CONFIG) + + # If LLVM was found, help CMake find Clang in the same installation + if(LLVM_FOUND) + # Clang's config is typically in ${LLVM_INSTALL_PREFIX}/lib/cmake/clang + set(Clang_DIR "${LLVM_INSTALL_PREFIX}/lib/cmake/clang") + endif() + + find_package(Clang QUIET CONFIG) + + if(LLVM_FOUND AND Clang_FOUND) + + message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION} via CMake config") + message(STATUS " LLVM_INSTALL_PREFIX: ${LLVM_INSTALL_PREFIX}") + message(STATUS " LLVM_INCLUDE_DIRS: ${LLVM_INCLUDE_DIRS}") + + # Use LLVM's CMake variables directly + set(LIBCLANG_CFLAGS_LIST ${LLVM_DEFINITIONS}) + list(APPEND LIBCLANG_CFLAGS_LIST "-I${LLVM_INCLUDE_DIRS}") + list(APPEND LIBCLANG_CFLAGS_LIST "-I${CLANG_INCLUDE_DIRS}") + + # Set up library directories + set(LIBCLANG_LINK_DIRS ${LLVM_LIBRARY_DIRS}) + + # Link against libclang (C API) and LLVM core libs + # Need to link against LLVM core for LLVMShutdown() and other LLVM APIs + if(TARGET libclang) + set(LIBCLANG_LIBS_LIST libclang LLVM) + else() + set(LIBCLANG_LIBS_LIST clang LLVM) + endif() + + set(LIBCLANG_INCLUDES "${LLVM_INCLUDE_DIRS};${CLANG_INCLUDE_DIRS}") + set(LIBCLANG_FOUND TRUE) + set(LIBCLANG_DETECTION_METHOD "CMake config") + elseif(NOT CROSS_COMPILING) + # Fallback to llvm-config method (only when not cross-compiling) + message(STATUS "LLVM CMake config not found, trying llvm-config") + + # Try to find llvm-config (versions 20 down to 10) + set(LLVM_CONFIG_NAMES llvm-config) + foreach(ver RANGE 20 10 -1) + list(APPEND LLVM_CONFIG_NAMES llvm-config-${ver}) + endforeach() + + find_program(LLVM_CONFIG NAMES ${LLVM_CONFIG_NAMES}) + + if(LLVM_CONFIG) + execute_process(COMMAND ${LLVM_CONFIG} --cflags OUTPUT_VARIABLE LIBCLANG_CFLAGS_TEMP OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${LLVM_CONFIG} --ldflags OUTPUT_VARIABLE LIBCLANG_LDFLAGS_TEMP OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${LLVM_CONFIG} --libs OUTPUT_VARIABLE LLVM_LIBS_TEMP OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${LLVM_CONFIG} --system-libs OUTPUT_VARIABLE LLVM_SYSLIBS_TEMP OUTPUT_STRIP_TRAILING_WHITESPACE) + + set(LIBCLANG_CFLAGS_LIST "${LIBCLANG_CFLAGS_TEMP} ${LIBCLANG_LDFLAGS_TEMP}") + set(LIBCLANG_LIBS_LIST "-lclang;${LLVM_LIBS_TEMP};${LLVM_SYSLIBS_TEMP}") + set(LIBCLANG_CFLAGS "${LIBCLANG_CFLAGS_TEMP}") + set(LIBCLANG_LDFLAGS "${LIBCLANG_LDFLAGS_TEMP}") + set(LIBCLANG_FOUND TRUE) + set(LIBCLANG_DETECTION_METHOD "llvm-config: ${LLVM_CONFIG}") + endif() + else() + message(STATUS "Cross-compiling: LLVM CMake config not found") + message(STATUS " Set LLVM_ROOT to LLVM install directory (e.g., /usr/lib/llvm-18)") + message(STATUS " Or set CMAKE_PREFIX_PATH or LLVM_DIR to the directory containing LLVMConfig.cmake") + endif() + + # Test if libclang actually works (single test for all detection methods) + if(LIBCLANG_FOUND) + if(NOT CROSS_COMPILING) + check_libclang_works(LIBCLANG_WORKS + CFLAGS "${LIBCLANG_CFLAGS_LIST}" + INCLUDES "${LIBCLANG_INCLUDES}" + LINK_DIRS "${LIBCLANG_LINK_DIRS}" + LIBRARIES "${LIBCLANG_LIBS_LIST}" + ) + else() + # When cross-compiling, disable libclang by default to avoid linking host libraries + # User can set LLVM_ROOT, CMAKE_PREFIX_PATH, or LLVM_DIR to point to target-specific LLVM installation + message(STATUS "Cross-compiling: disabling libclang") + message(STATUS " To enable, set LLVM_ROOT or CMAKE_PREFIX_PATH to target LLVM installation") + set(LIBCLANG_WORKS FALSE) + endif() + + if(LIBCLANG_WORKS) + set(HAVE_LIBCLANG 1) + # Convert list to string for LIBCLANG_CFLAGS if needed (CMake config path) + if(LIBCLANG_DETECTION_METHOD STREQUAL "CMake config") + string(REPLACE ";" " " LIBCLANG_CFLAGS "${LIBCLANG_CFLAGS_LIST}") + endif() + set(LIBCLANG_LIBS "${LIBCLANG_LIBS_LIST}") + message(STATUS "Found libclang via ${LIBCLANG_DETECTION_METHOD}") + else() + message(STATUS "libclang not found or not working") + message(STATUS " Reflection support will be disabled") + message(STATUS " To specify libclang location: cmake -DLLVM_ROOT=/path/to/llvm") + message(STATUS " To disable this check: cmake -DENABLE_LIBCLANG=OFF") + endif() + else() + if(NOT LIBCLANG_FOUND) + message(STATUS "libclang not found") + message(STATUS " Reflection support will be disabled") + message(STATUS " To specify libclang location: cmake -DLLVM_ROOT=/path/to/llvm") + message(STATUS " To disable this check: cmake -DENABLE_LIBCLANG=OFF") + endif() + endif() + +endif() # ENABLE_LIBCLANG + +# Check for tools +find_program(GIT_EXECUTABLE git) +if(GIT_EXECUTABLE) + set(HAVE_GIT 1) +else() + set(HAVE_GIT 0) +endif() + +find_program(JQ_EXECUTABLE jq) +if(JQ_EXECUTABLE) + set(HAVE_JQ 1) +else() + set(HAVE_JQ 0) +endif() +find_program(DOCKER_EXECUTABLE docker) +if(DOCKER_EXECUTABLE) + set(HAVE_DOCKER 1) +else() + set(HAVE_DOCKER 0) +endif() + +# Test suite URLs +if(NOT TESTSUITEURL) + set(TESTSUITEURL "https://github.com/yaml/yaml-test-suite") +endif() +if(NOT TESTSUITECHECKOUT) + set(TESTSUITECHECKOUT "6e6c296ae9c9d2d5c4134b4b64d01b29ac19ff6f") +endif() +if(NOT JSONTESTSUITEURL) + set(JSONTESTSUITEURL "https://github.com/nst/JSONTestSuite") +endif() +if(NOT JSONTESTSUITECHECKOUT) + set(JSONTESTSUITECHECKOUT "d64aefb55228d9584d3e5b2433f720ea8fd00c82") +endif() + +# SIMD capability detection +set(TARGET_HAS_SSE2 FALSE) +set(TARGET_HAS_SSE41 FALSE) +set(TARGET_HAS_AVX2 FALSE) +set(TARGET_HAS_AVX512 FALSE) +set(TARGET_HAS_NEON FALSE) + +if(TARGET_CPU_ANY_X86 AND NOT ENABLE_PORTABLE_TARGET) + check_c_source_compiles(" + #include + int main() { __m128i a = _mm_setzero_si128(); return 0; } + " COMPILER_SUPPORTS_SSE2) + if(COMPILER_SUPPORTS_SSE2) + set(TARGET_HAS_SSE2 TRUE) + endif() + + check_c_source_compiles(" + #include + int main() { __m128i a = _mm_setzero_si128(); return 0; } + " COMPILER_SUPPORTS_SSE41) + if(COMPILER_SUPPORTS_SSE41) + set(TARGET_HAS_SSE41 TRUE) + endif() + + set(CMAKE_REQUIRED_FLAGS "-mavx2") + check_c_source_compiles(" + #include + int main() { __m256i a = _mm256_setzero_si256(); return 0; } + " COMPILER_SUPPORTS_AVX2) + set(CMAKE_REQUIRED_FLAGS "") + if(COMPILER_SUPPORTS_AVX2) + set(TARGET_HAS_AVX2 TRUE) + endif() + + set(CMAKE_REQUIRED_FLAGS "-mavx512f -mavx512vl") + check_c_source_compiles(" + #include + int main() { __m512i a = _mm512_setzero_si512(); return 0; } + " COMPILER_SUPPORTS_AVX512) + set(CMAKE_REQUIRED_FLAGS "") + if(COMPILER_SUPPORTS_AVX512) + set(TARGET_HAS_AVX512 TRUE) + endif() +endif() + +if(TARGET_CPU_ANY_ARM AND NOT ENABLE_PORTABLE_TARGET) + if(TARGET_CPU_ARM64) + set(TARGET_HAS_NEON TRUE) + else() + check_c_source_compiles(" + #include + int main() { uint8x16_t a = vdupq_n_u8(0); return 0; } + " COMPILER_SUPPORTS_NEON) + if(COMPILER_SUPPORTS_NEON) + set(TARGET_HAS_NEON TRUE) + endif() + endif() +endif() + +# ASAN support +set(HAVE_ASAN 0) +if(ENABLE_ASAN) + check_c_source_compiles(" + int main() { return 0; } + " ASAN_WORKS) + if(ASAN_WORKS) + set(HAVE_ASAN 1) + endif() +endif() + +# Build common compiler flags for all targets +set(COMMON_C_FLAGS) +set(COMMON_C_DEFINITIONS _GNU_SOURCE) + +if(NOT MSVC) + list(APPEND COMMON_C_FLAGS -Wall -Wsign-compare) + + check_c_compiler_flag(-Wno-unused-function COMPILER_SUPPORTS_WNO_UNUSED_FUNCTION) + if(COMPILER_SUPPORTS_WNO_UNUSED_FUNCTION) + list(APPEND COMMON_C_FLAGS -Wno-unused-function) + endif() + + check_c_compiler_flag(-Wno-stringop-overflow COMPILER_SUPPORTS_WNO_STRINGOP_OVERFLOW) + if(COMPILER_SUPPORTS_WNO_STRINGOP_OVERFLOW) + list(APPEND COMMON_C_FLAGS -Wno-stringop-overflow) + endif() + + check_c_compiler_flag(-Wno-tautological-constant-out-of-range-compare + COMPILER_SUPPORTS_WNO_TAUTOLOGICAL_CONSTANT_OUT_OF_RANGE_COMPARE) + if(COMPILER_SUPPORTS_WNO_TAUTOLOGICAL_CONSTANT_OUT_OF_RANGE_COMPARE) + list(APPEND COMMON_C_FLAGS -Wno-tautological-constant-out-of-range-compare) + endif() + + set(USE_STD 0) + + if(NOT USE_STD) + check_c_compiler_flag(-std=c2x COMPILER_SUPPORTS_GNU2X) + if(COMPILER_SUPPORTS_GNU2X) + list(APPEND COMMON_C_FLAGS -std=gnu2x) + set(USE_STD 1) + endif() + endif() + + if(NOT USE_STD) + check_c_compiler_flag(-std=c2x COMPILER_SUPPORTS_C2X) + if(COMPILER_SUPPORTS_C2X) + list(APPEND COMMON_C_FLAGS -std=c2x) + set(USE_STD 1) + endif() + endif() + + # GCC heap trampolines support (for nested functions) + # Heap trampolines are more secure than executable stack trampolines + # Available in GCC 14+ with -ftrampoline-impl=heap + # Supported targets: x86_64/i386/aarch64 Linux, x86_64/i386 Darwin + set(HAVE_HEAP_TRAMPOLINES 0) + if(CMAKE_C_COMPILER_ID STREQUAL "GNU") + check_c_compiler_flag(-ftrampoline-impl=heap COMPILER_SUPPORTS_HEAP_TRAMPOLINES) + if(COMPILER_SUPPORTS_HEAP_TRAMPOLINES) + set(HAVE_HEAP_TRAMPOLINES 1) + list(APPEND COMMON_C_FLAGS -ftrampoline-impl=heap) + message(STATUS "GCC heap trampolines enabled (-ftrampoline-impl=heap)") + else() + message(STATUS "GCC heap trampolines not available (GCC 14+ required), using stack trampolines") + endif() + endif() + +endif() + +# Network tests +if(ENABLE_NETWORK) + set(HAVE_NETWORK 1) +else() + set(HAVE_NETWORK 0) +endif() + +# Dev mode +if(ENABLE_DEVMODE) + set(HAVE_DEVMODE 1) +else() + set(HAVE_DEVMODE 0) +endif() + +# Static support +if(NOT BUILD_SHARED_LIBS) + set(HAVE_STATIC 1) +else() + set(HAVE_STATIC 0) +endif() + +# Library sources set(LIBHDRS - src/lib/fy-accel.h - src/lib/fy-atom.h - src/lib/fy-composer.h - src/lib/fy-diag.h - src/lib/fy-doc.h - src/lib/fy-docbuilder.h - src/lib/fy-docstate.h - src/lib/fy-dump.h - src/lib/fy-emit.h src/lib/fy-emit-accum.h - src/lib/fy-input.h - src/lib/fy-parse.h - src/lib/fy-path.h - src/lib/fy-token.h - src/lib/fy-types.h - src/lib/fy-walk.h - src/util/fy-blob.h - src/util/fy-ctype.h - src/util/fy-endian.h - src/util/fy-id.h - src/util/fy-list.h - src/util/fy-typelist.h - src/util/fy-utf8.h - src/util/fy-utils.h - src/xxhash/xxhash.h + src/lib/fy-accel.h + src/lib/fy-atom.h + src/lib/fy-composer.h + src/lib/fy-diag.h + src/lib/fy-doc.h + src/lib/fy-docbuilder.h + src/lib/fy-docstate.h + src/lib/fy-dump.h + src/lib/fy-emit.h + src/lib/fy-emit-accum.h + src/lib/fy-event.h + src/lib/fy-input.h + src/lib/fy-parse.h + src/lib/fy-path.h + src/lib/fy-token.h + src/lib/fy-types.h + src/lib/fy-walk.h +) + +set(UTILHDRS + src/util/fy-blob.h + src/util/fy-ctype.h + src/util/fy-endian.h + src/util/fy-id.h + src/util/fy-list.h + src/util/fy-typelist.h + src/util/fy-utf8.h + src/util/fy-utils.h + src/util/fy-align.h + src/util/fy-bit64.h + src/util/fy-vlsize.h ) + set(LIBSRCS - src/lib/fy-accel.c - src/lib/fy-atom.c - src/lib/fy-composer.c - src/lib/fy-diag.c - src/lib/fy-doc.c - src/lib/fy-docbuilder.c - src/lib/fy-docstate.c - src/lib/fy-dump.c - src/lib/fy-emit.c - src/lib/fy-event.c - src/lib/fy-input.c - src/lib/fy-parse.c - src/lib/fy-path.c - src/lib/fy-token.c - src/lib/fy-types.c - src/lib/fy-walk.c - src/util/fy-blob.c - src/util/fy-ctype.c - src/util/fy-utf8.c - src/util/fy-utils.c - src/xxhash/xxhash.c + src/lib/fy-accel.c + src/lib/fy-atom.c + src/lib/fy-composer.c + src/lib/fy-diag.c + src/lib/fy-doc.c + src/lib/fy-docbuilder.c + src/lib/fy-docstate.c + src/lib/fy-dump.c + src/lib/fy-emit.c + src/lib/fy-event.c + src/lib/fy-input.c + src/lib/fy-parse.c + src/lib/fy-path.c + src/lib/fy-token.c + src/lib/fy-types.c + src/lib/fy-walk.c + src/util/fy-blob.c + src/util/fy-ctype.c + src/util/fy-utf8.c + src/util/fy-utils.c + src/xxhash/xxhash.c + src/thread/fy-thread.c + src/allocator/fy-allocator.c + src/allocator/fy-allocator-linear.c + src/allocator/fy-allocator-malloc.c + src/allocator/fy-allocator-mremap.c + src/allocator/fy-allocator-dedup.c + src/allocator/fy-allocator-auto.c + src/blake3/blake3_host_state.c + src/blake3/blake3_backend.c + src/blake3/blake3_be_cpusimd.c + src/blake3/fy-blake3.c + src/lib/fy-composer-diag.c + src/lib/fy-doc-diag.c + src/lib/fy-docbuilder-diag.c + src/lib/fy-input-diag.c + src/lib/fy-parse-diag.c ) + set(LIBHDRSPUB include/libfyaml.h ) -add_library(${PROJECT_NAME} +# BLAKE3 portable library +add_library(b3portable OBJECT + src/blake3/blake3_portable.c + src/blake3/blake3.c +) +target_include_directories(b3portable PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 +) +target_compile_definitions(b3portable PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=portable + SIMD_DEGREE=1 +) +target_compile_options(b3portable PRIVATE ${COMMON_C_FLAGS}) + +# BLAKE3 SIMD libraries +set(BLAKE3_TARGETS b3portable) + +if(TARGET_HAS_SSE2) + add_library(b3sse2 OBJECT + src/blake3/blake3_sse2.c + src/blake3/blake3_sse2_x86-64_unix.S + src/blake3/blake3.c + ) + target_include_directories(b3sse2 PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ) + target_compile_definitions(b3sse2 PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=sse2 + SIMD_DEGREE=4 + ) + target_compile_options(b3sse2 PRIVATE ${COMMON_C_FLAGS} -msse2) + list(APPEND BLAKE3_TARGETS b3sse2) +endif() + +if(TARGET_HAS_SSE41) + add_library(b3sse41 OBJECT + src/blake3/blake3_sse41.c + src/blake3/blake3_sse41_x86-64_unix.S + src/blake3/blake3.c + ) + target_include_directories(b3sse41 PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ) + target_compile_definitions(b3sse41 PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=sse41 + SIMD_DEGREE=4 + ) + target_compile_options(b3sse41 PRIVATE ${COMMON_C_FLAGS} -msse4.1) + list(APPEND BLAKE3_TARGETS b3sse41) +endif() + +if(TARGET_HAS_AVX2) + add_library(b3avx2 OBJECT + src/blake3/blake3_avx2.c + src/blake3/blake3_avx2_x86-64_unix.S + src/blake3/blake3.c + ) + target_include_directories(b3avx2 PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ) + target_compile_definitions(b3avx2 PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=avx2 + SIMD_DEGREE=8 + ) + target_compile_options(b3avx2 PRIVATE ${COMMON_C_FLAGS} -mavx2) + list(APPEND BLAKE3_TARGETS b3avx2) +endif() + +if(TARGET_HAS_AVX512) + add_library(b3avx512 OBJECT + src/blake3/blake3_avx512.c + src/blake3/blake3_avx512_x86-64_unix.S + src/blake3/blake3.c + ) + target_include_directories(b3avx512 PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ) + target_compile_definitions(b3avx512 PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=avx512 + SIMD_DEGREE=16 + ) + target_compile_options(b3avx512 PRIVATE ${COMMON_C_FLAGS} -mavx512f -mavx512vl) + list(APPEND BLAKE3_TARGETS b3avx512) +endif() + +if(TARGET_HAS_NEON) + add_library(b3neon OBJECT + src/blake3/blake3_neon.c + src/blake3/blake3.c + ) + target_include_directories(b3neon PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ) + target_compile_definitions(b3neon PRIVATE + ${COMMON_C_DEFINITIONS} + HASHER_SUFFIX=neon + SIMD_DEGREE=4 + ) + target_compile_options(b3neon PRIVATE ${COMMON_C_FLAGS}) + if(TARGET_CPU_ARM) + target_compile_options(b3neon PRIVATE -mfpu=neon) + endif() + list(APPEND BLAKE3_TARGETS b3neon) +endif() + +# Main library +add_library(fyaml ${LIBHDRS} + ${UTILHDRS} ${LIBSRCS} ${LIBHDRSPUB} ) -include(CheckIncludeFile) -check_include_file("byteswap.h" HAVE_BYTESWAP_H) -include(CheckFunctionExists) -check_function_exists(__builtin_bswap16 HAVE___BUILTIN_BSWAP16) -check_function_exists(__builtin_bswap32 HAVE___BUILTIN_BSWAP32) -check_function_exists(__builtin_bswap64 HAVE___BUILTIN_BSWAP64) +# Link BLAKE3 object libraries +foreach(blake3_target ${BLAKE3_TARGETS}) + target_sources(fyaml PRIVATE $) +endforeach() + +add_library(libfyaml::libfyaml ALIAS fyaml) + +# Always build a static library for internal tools (they need access to internal APIs) +# This is separate from BUILD_SHARED_LIBS and matches autotools behavior +add_library(fyaml_static STATIC + ${LIBHDRS} + ${UTILHDRS} + ${LIBSRCS} + ${LIBHDRSPUB} +) -add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) +# Link BLAKE3 object libraries to static version +foreach(blake3_target ${BLAKE3_TARGETS}) + target_sources(fyaml_static PRIVATE $) +endforeach() -set_target_properties(${PROJECT_NAME} PROPERTIES FOLDER "Libraries") +set_target_properties(fyaml_static PROPERTIES + OUTPUT_NAME fyaml_static + POSITION_INDEPENDENT_CODE OFF + FOLDER "Libraries" +) -target_include_directories(${PROJECT_NAME} +# Configure static library with same settings as main library +target_include_directories(fyaml_static PUBLIC - $ - $ + $ + $ PRIVATE - ${${PROJECT_NAME}_SOURCE_DIR}/src/lib - ${${PROJECT_NAME}_SOURCE_DIR}/src/util - ${${PROJECT_NAME}_SOURCE_DIR}/src/xxhash + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/xxhash + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ${CMAKE_CURRENT_BINARY_DIR} ) -#target_link_libraries(${PROJECT_NAME} PUBLIC xx) -#target_compile_definitions(${PROJECT_NAME} PUBLIC xx) +target_link_libraries(fyaml_static + PUBLIC + Threads::Threads +) -target_compile_definitions(${PROJECT_NAME} PRIVATE - VERSION="0.5.7.34" +# Add libclang support if available +add_libclang_to_target(fyaml_static INTERFACE) + +# check if blocks are available, and if so link them +include(${CMAKE_SOURCE_DIR}/cmake/CheckClangBlocks.cmake) + +# Apply flags if available +if (HAVE_CLANG_BLOCKS) + add_compile_options(-fblocks) + list(APPEND COMMON_C_FLAGS -fblocks) + + if (CLANG_BLOCKS_LIB) + link_libraries(${CLANG_BLOCKS_LIB}) + endif() +endif() + +target_compile_definitions(fyaml_static PRIVATE + HAVE_CONFIG_H + VERSION="${PROJECT_VERSION}" + $<$>:NDEBUG> ) -target_compile_definitions(${PROJECT_NAME} PRIVATE - $<$>:NDEBUG> +target_compile_definitions(fyaml_static PRIVATE ${COMMON_C_DEFINITIONS}) +target_compile_options(fyaml_static PRIVATE ${COMMON_C_FLAGS}) + +if(NOT MSVC) + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_options(fyaml_static PRIVATE -O2) + endif() + + if(HAVE_ASAN) + target_compile_options(fyaml_static PRIVATE -fsanitize=address -fno-omit-frame-pointer) + # Don't add link options here - executables will link dynamically against libasan + endif() +endif() + +set_target_properties(fyaml PROPERTIES + OUTPUT_NAME fyaml + VERSION ${PROJECT_VERSION} + SOVERSION ${VERSION_MAJOR} + FOLDER "Libraries" + PUBLIC_HEADER "${LIBHDRSPUB}" +) + +target_include_directories(fyaml + PUBLIC + $ + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/xxhash + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ${CMAKE_CURRENT_SOURCE_DIR}/src/generic + ${CMAKE_CURRENT_SOURCE_DIR}/src/reflection +) + +target_link_libraries(fyaml + PUBLIC + Threads::Threads +) + +# Add libclang support if available +add_libclang_to_target(fyaml PRIVATE) + +# Generate config.h +configure_file( + ${CMAKE_SOURCE_DIR}/cmake/config.h.in + ${CMAKE_BINARY_DIR}/config.h ) -# Shared options between GCC and CLANG: -if (NOT MSVC) - target_compile_options(${PROJECT_NAME} PRIVATE - -Wall - -fPIC - -D_GNU_SOURCE +# Generate config.h +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/config.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/config.h" +) +target_include_directories(fyaml PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + +target_compile_definitions(fyaml PRIVATE + HAVE_CONFIG_H + VERSION="${PROJECT_VERSION}" + $<$>:NDEBUG> +) + +target_compile_definitions(fyaml PRIVATE ${COMMON_C_DEFINITIONS}) +target_compile_options(fyaml PRIVATE ${COMMON_C_FLAGS}) + +# Shared options between GCC and CLANG +if(NOT MSVC) + target_compile_options(fyaml PRIVATE -fvisibility=hidden) + + if(BUILD_SHARED_LIBS) + target_compile_options(fyaml PRIVATE -fPIC) + endif() + + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_options(fyaml PRIVATE -O2) + endif() + + if(HAVE_ASAN) + target_compile_options(fyaml PRIVATE -fsanitize=address -fno-omit-frame-pointer) + # Use INTERFACE to propagate link flags to executables linking against this library + target_link_options(fyaml INTERFACE -fsanitize=address) + endif() +endif() + +# Build fy-tool +add_executable(fy-tool + src/tool/fy-tool.c + src/tool/fy-tool-util.c + src/tool/fy-tool-dump.c + src/valgrind/fy-valgrind.h +) + +target_include_directories(fy-tool PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind +) + +target_link_libraries(fy-tool PRIVATE fyaml) + +if(ENABLE_STATIC_TOOLS AND NOT MSVC) + target_link_options(fy-tool PRIVATE -static) +endif() + +# Build internal tools - these are never installed, built for internal testing +# These tools use private/internal APIs and embed libfyaml code via fyaml_static +# System libraries are dynamically linked (works with or without ASAN) +# Skip when cross-compiling since these tools cannot be executed on build host + +if(NOT CROSS_COMPILING) + +# libfyaml-parser requires libyaml +if(HAVE_LIBYAML) + add_executable(libfyaml-parser + src/internal/libfyaml-parser.c + src/valgrind/fy-valgrind.h ) + target_compile_definitions(libfyaml-parser PRIVATE ${COMMON_C_DEFINITIONS}) + target_compile_options(libfyaml-parser PRIVATE ${COMMON_C_FLAGS}) - if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - target_compile_options(${PROJECT_NAME} PRIVATE - -O2 + target_include_directories(libfyaml-parser PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/xxhash + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_BINARY_DIR} + ${LIBYAML_INCLUDE_DIRS} + ) + target_link_directories(libfyaml-parser PRIVATE ${LIBYAML_LIBRARY_DIRS}) + if(NOT MSVC) + # Embed libfyaml code statically, allow dynamic linking for system libraries + if(APPLE) + target_link_libraries(libfyaml-parser PRIVATE + -Wl,-force_load,$ + ${LIBYAML_LIBRARIES} + $<$:${LIBCLANG_LIBS}> + ) + else() + target_link_libraries(libfyaml-parser PRIVATE + -Wl,--whole-archive fyaml_static -Wl,--no-whole-archive + ${LIBYAML_LIBRARIES} + $<$:${LIBCLANG_LIBS}> + ) + endif() + if(HAVE_ASAN) + target_link_options(libfyaml-parser PRIVATE -fsanitize=address) + endif() + else() + target_link_libraries(libfyaml-parser PRIVATE fyaml_static ${LIBYAML_LIBRARIES}) + endif() + if(HAVE_LIBCLANG) + target_link_directories(libfyaml-parser PRIVATE ${LIBCLANG_LINK_DIRS}) + endif() + add_dependencies(libfyaml-parser fyaml_static) +endif() + +add_executable(fy-thread + src/internal/fy-thread.c + src/valgrind/fy-valgrind.h +) +target_include_directories(fy-thread PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_BINARY_DIR} +) +if(NOT MSVC) + if(APPLE) + target_link_libraries(fy-thread PRIVATE + -Wl,-force_load,$ + $<$:${LIBCLANG_LIBS}> + ) + else() + target_link_libraries(fy-thread PRIVATE + -Wl,--whole-archive fyaml_static -Wl,--no-whole-archive + $<$:${LIBCLANG_LIBS}> ) - endif() + endif() + if(HAVE_ASAN) + target_link_options(fy-thread PRIVATE -fsanitize=address) + endif() +else() + target_link_libraries(fy-thread PRIVATE fyaml_static) endif() +if(HAVE_LIBCLANG) + target_link_directories(fy-thread PRIVATE ${LIBCLANG_LINK_DIRS}) +endif() +add_dependencies(fy-thread fyaml_static) -# ==== Install & export target ======== +add_executable(fy-b3sum + src/internal/fy-b3sum.c + src/valgrind/fy-valgrind.h +) +target_include_directories(fy-b3sum PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/blake3 + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_BINARY_DIR} +) +if(NOT MSVC) + if(APPLE) + target_link_libraries(fy-b3sum PRIVATE + -Wl,-force_load,$ + $<$:${LIBCLANG_LIBS}> + ) + else() + target_link_libraries(fy-b3sum PRIVATE + -Wl,--whole-archive fyaml_static -Wl,--no-whole-archive + $<$:${LIBCLANG_LIBS}> + ) + endif() + if(HAVE_ASAN) + target_link_options(fy-b3sum PRIVATE -fsanitize=address) + endif() +else() + target_link_libraries(fy-b3sum PRIVATE fyaml_static) +endif() +if(HAVE_LIBCLANG) + target_link_directories(fy-b3sum PRIVATE ${LIBCLANG_LINK_DIRS}) +endif() +add_dependencies(fy-b3sum fyaml_static) -install(TARGETS ${PROJECT_NAME} - EXPORT ${PROJECT_NAME}-targets - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} +add_executable(fy-allocators + src/internal/fy-allocators.c + src/valgrind/fy-valgrind.h ) -install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +target_include_directories(fy-allocators PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/xxhash + ${CMAKE_CURRENT_SOURCE_DIR}/src/thread + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_BINARY_DIR} +) +if(NOT MSVC) + if(APPLE) + target_link_libraries(fy-allocators PRIVATE + -Wl,-force_load,$ + $<$:${LIBCLANG_LIBS}> + ) + else() + target_link_libraries(fy-allocators PRIVATE + -Wl,--whole-archive fyaml_static -Wl,--no-whole-archive + $<$:${LIBCLANG_LIBS}> + ) + endif() + if(HAVE_ASAN) + target_link_options(fy-allocators PRIVATE -fsanitize=address) + endif() +else() + target_link_libraries(fy-allocators PRIVATE fyaml_static) +endif() +if(HAVE_LIBCLANG) + target_link_directories(fy-allocators PRIVATE ${LIBCLANG_LINK_DIRS}) +endif() +add_dependencies(fy-allocators fyaml_static) + +endif() # NOT CROSS_COMPILING (internal tools) + +# Testing +if(BUILD_TESTING AND NOT CROSS_COMPILING) + # Include TAP subtest registration functions + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/add-tap-subtests.cmake) -install(EXPORT ${PROJECT_NAME}-targets - FILE ${PROJECT_NAME}-targets.cmake - NAMESPACE ${PROJECT_NAME}:: - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} + # Create test directory + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test) + + # Create src directory for compatibility with test scripts + # Tests expect tools at build/src/ + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/src) + add_custom_command(TARGET fy-tool POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_BINARY_DIR}/fy-tool + ${CMAKE_CURRENT_BINARY_DIR}/src/fy-tool + COMMENT "Creating symlink for test compatibility" + ) + + # Create symlinks for internal tools + add_custom_command(TARGET fy-thread POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_BINARY_DIR}/fy-thread + ${CMAKE_CURRENT_BINARY_DIR}/src/fy-thread + ) + add_custom_command(TARGET fy-b3sum POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_BINARY_DIR}/fy-b3sum + ${CMAKE_CURRENT_BINARY_DIR}/src/fy-b3sum + ) + add_custom_command(TARGET fy-allocators POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_BINARY_DIR}/fy-allocators + ${CMAKE_CURRENT_BINARY_DIR}/src/fy-allocators + ) + if(HAVE_LIBYAML) + add_custom_command(TARGET libfyaml-parser POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_BINARY_DIR}/libfyaml-parser + ${CMAKE_CURRENT_BINARY_DIR}/src/libfyaml-parser + ) + endif() + + # Build libfyaml-test if check is available + if(HAVE_COMPATIBLE_CHECK) + + add_executable(libfyaml-test + test/libfyaml-test.c + test/libfyaml-test-core.c + test/libfyaml-test-meta.c + test/libfyaml-test-emit.c + test/libfyaml-test-allocator.c + test/libfyaml-test-private.c + test/libfyaml-test-private-id.c + test/libfyaml-test-parser.c + test/libfyaml-test-thread.c + ) + + target_compile_options(libfyaml-test PRIVATE ${COMMON_C_FLAGS}) + target_compile_definitions(libfyaml-test PRIVATE ${COMMON_C_DEFINITIONS}) + + target_include_directories(libfyaml-test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/src/valgrind + ${CMAKE_CURRENT_SOURCE_DIR}/src/lib + ${CMAKE_CURRENT_SOURCE_DIR}/src/util + ${CMAKE_CURRENT_SOURCE_DIR}/src/allocator + ${CMAKE_CURRENT_BINARY_DIR} + ${CHECK_INCLUDE_DIRS} + ) + + # Link against static library to access private symbols + # Use same pattern as internal tools + if(NOT MSVC) + if(APPLE) + target_link_libraries(libfyaml-test PRIVATE + -Wl,-force_load,$ + ${CHECK_LIBRARIES} + $<$:${LIBCLANG_LIBS}> + ) + else() + target_link_libraries(libfyaml-test PRIVATE + -Wl,--whole-archive fyaml_static -Wl,--no-whole-archive + ${CHECK_LIBRARIES} + $<$:${LIBCLANG_LIBS}> + ) + endif() + if(HAVE_ASAN) + target_link_options(libfyaml-test PRIVATE -fsanitize=address) + endif() + else() + target_link_libraries(libfyaml-test PRIVATE fyaml_static ${CHECK_LIBRARIES}) + endif() + if(CHECK_LIBRARY_DIRS) + target_link_directories(libfyaml-test PRIVATE ${CHECK_LIBRARY_DIRS}) + endif() + if(HAVE_LIBCLANG) + target_link_directories(libfyaml-test PRIVATE ${LIBCLANG_LINK_DIRS}) + endif() + + if (HAVE_CLANG_BLOCKS) + target_compile_options(libfyaml-test PRIVATE -fblocks) + if (CLANG_BLOCKS_LIB) + target_link_libraries(libfyaml-test PRIVATE ${CLANG_BLOCKS_LIB}) + endif() + endif() + + add_dependencies(libfyaml-test fyaml_static) + + # Set output directory to match autoconf behavior + set_target_properties(libfyaml-test PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Add libfyaml.test + add_test(NAME libfyaml + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test/libfyaml.test + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + set_tests_properties(libfyaml PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test" + ) + endif() + + # Error tests - register individual subtests + add_testerrors_tests() + + # Emitter tests - register individual subtests + add_testemitter_tests(testemitter "") + add_testemitter_tests(testemitter-streaming "--streaming") + add_testemitter_tests(testemitter-restreaming "--streaming --recreating") + + # Network and git-dependent tests + if(HAVE_NETWORK AND HAVE_GIT) + # YAML test suite - setup fixture + add_test(NAME yaml-test-suite-setup + COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR}/test + ${CMAKE_COMMAND} + -DGIT_EXECUTABLE=${GIT_EXECUTABLE} + -DTESTSUITEURL=${TESTSUITEURL} + -DTESTSUITECHECKOUT=${TESTSUITECHECKOUT} + -DTEST_DIR=${CMAKE_CURRENT_BINARY_DIR}/test + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/setup-yaml-test-suite.cmake + ) + set_tests_properties(yaml-test-suite-setup PROPERTIES + FIXTURES_SETUP yaml-test-suite-data + ) + + # YAML test suite tests - register individual subtests + # Three modes (handled automatically by add_testsuite_tests): + # 1. If test-suite-data exists: scan filesystem and register tests + # 2. If cmake/testsuite-tests.cmake exists: use pre-generated list + # 3. Otherwise: use monolithic test (re-run cmake after downloading data) + # To generate cmake/testsuite-tests.cmake, see cmake/README-testsuite-generation.md + add_testsuite_tests(testsuite testsuite.test) + add_testsuite_tests(testsuite-evstream testsuite-evstream.test) + add_testsuite_tests(testsuite-resolution testsuite-resolution.test) + if(HAVE_JQ) + add_testsuite_tests(testsuite-json testsuite-json.test) + endif() + + # JSON test suite - setup fixture + add_test(NAME json-test-suite-setup + COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR}/test + ${CMAKE_COMMAND} + -DGIT_EXECUTABLE=${GIT_EXECUTABLE} + -DTESTSUITEURL=${JSONTESTSUITEURL} + -DTESTSUITECHECKOUT=${JSONTESTSUITECHECKOUT} + -DTEST_DIR=${CMAKE_CURRENT_BINARY_DIR}/test + -DREPO_NAME=json-test-suite-data + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/setup-yaml-test-suite.cmake + ) + set_tests_properties(json-test-suite-setup PROPERTIES + FIXTURES_SETUP json-test-suite-data + ) + + # JSON test suite - register individual subtests + # Three modes (handled automatically by add_jsontestsuite_tests): + # 1. If json-test-suite-data exists: scan filesystem and register tests + # 2. If cmake/jsontestsuite-tests.cmake exists: use pre-generated list + # 3. Otherwise: use monolithic test (re-run cmake after downloading data) + # To generate cmake/jsontestsuite-tests.cmake, see cmake/README-testsuite-generation.md + add_jsontestsuite_tests() + endif() + + set(TEST_DRIVER ${CMAKE_SOURCE_DIR}/build-aux/tap-driver.sh) + + # Build all test dependencies as part of default 'all' target + # Note: CMake's 'make test' target doesn't trigger builds by design. + # Use 'make all test' or 'make check-am' or 'make build_and_test' for complete workflow. + set(TEST_DEPENDENCIES fy-tool fy-thread fy-b3sum fy-allocators) + if(HAVE_COMPATIBLE_CHECK) + list(APPEND TEST_DEPENDENCIES libfyaml-test) + endif() + if(HAVE_LIBYAML) + list(APPEND TEST_DEPENDENCIES libfyaml-parser) + endif() + + add_custom_target(build_tests ALL + DEPENDS ${TEST_DEPENDENCIES} + COMMENT "Building test dependencies" + ) + + # Convenience target that builds and runs tests in one command + add_custom_target(build_and_test + COMMAND ${CMAKE_CTEST_COMMAND} + DEPENDS ${TEST_DEPENDENCIES} + COMMENT "Building and running tests" + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + + # Autotools-compatible test runner using test-driver + # NOTE: Named 'check-am' instead of 'check' to avoid conflicts with check library target + add_custom_target(check-am + COMMENT "Running tests via Automake test-driver" + ) + + if(HAVE_COMPATIBLE_CHECK) + set(TEST_EXECUTABLES + libfyaml-test + ) + + foreach(exe IN LISTS TEST_EXECUTABLES) + add_custom_target(check_${exe} + COMMAND ${CMAKE_COMMAND} -E env TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR} TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR} SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test + ${TEST_DRIVER} + --test-name ${exe} + --log-file ${exe}.log + --trs-file ${exe}.trs + --comments + --color-tests yes + --expect-failure 0 + -- ${CMAKE_CURRENT_BINARY_DIR}/test/${exe} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + COMMENT "Running ${exe}" + ) + add_dependencies(check_${exe} ${exe}) + add_dependencies(check-am check_${exe}) + endforeach() + endif() + + set(TEST_SUITES + testsuite.test + jsontestsuite.test + testsuite-evstream.test + testsuite-resolution.test + testerrors.test + testemitter.test + testemitter-streaming.test + testemitter-restreaming.test + ) + + if(HAVE_JQ) + list(APPEND TEST_SUITES testsuite-json.test) + endif() + + if(HAVE_LIBCLANG) + list(APPEND TEST_SUITES testreflection.test) + endif() + + foreach(suite IN LISTS TEST_SUITES) + add_custom_target(check_${suite} + COMMAND ${CMAKE_COMMAND} -E env TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR} TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR} SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test + ${TEST_DRIVER} + --test-name ${suite} + --log-file ${suite}.log + --trs-file ${suite}.trs + --comments + --color-tests yes + --expect-failure 0 + -- ${CMAKE_SOURCE_DIR}/test/${suite} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + COMMENT "Running ${suite}" + ) + add_dependencies(check_${suite} fy-tool) + add_dependencies(check-am check_${suite}) + endforeach() + +endif() + +# Documentation +find_program(SPHINX_EXECUTABLE NAMES sphinx-build) + +if(SPHINX_EXECUTABLE) + set(HAVE_SPHINX TRUE) + set(SPHINX_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/doc") + set(SPHINX_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/doc/_build") + + # Add custom targets for documentation + add_custom_target(doc-html + COMMAND ${SPHINX_EXECUTABLE} -M html "${SPHINX_SOURCE_DIR}" "${SPHINX_BUILD_DIR}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Building HTML documentation with Sphinx" + VERBATIM + ) + + add_custom_target(doc-man + COMMAND ${SPHINX_EXECUTABLE} -M man "${SPHINX_SOURCE_DIR}" "${SPHINX_BUILD_DIR}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Building man pages with Sphinx" + VERBATIM + ) + + add_custom_target(doc-latexpdf + COMMAND ${SPHINX_EXECUTABLE} -M latexpdf "${SPHINX_SOURCE_DIR}" "${SPHINX_BUILD_DIR}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Building PDF documentation with Sphinx" + VERBATIM + ) + + add_custom_target(doc-clean + COMMAND ${CMAKE_COMMAND} -E remove_directory "${SPHINX_BUILD_DIR}" + COMMENT "Cleaning documentation build directory" + ) + + message(STATUS "Sphinx found: ${SPHINX_EXECUTABLE}") + message(STATUS " make doc-html - Build HTML documentation") + message(STATUS " make doc-man - Build man pages") + message(STATUS " make doc-latexpdf - Build PDF documentation") +else() + set(HAVE_SPHINX FALSE) + message(STATUS "Sphinx not found - documentation targets disabled") + message(STATUS " Install: pip3 install sphinx sphinx_rtd_theme sphinx-markdown-builder linuxdoc") +endif() + +# Install targets +install(TARGETS fyaml fy-tool + EXPORT libfyaml-targets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) -export( - TARGETS ${PROJECT_NAME} - FILE "${CMAKE_CURRENT_BINARY_DIR}/cmake/${PROJECT_NAME}-targets.cmake" - NAMESPACE ${PROJECT_NAME}:: + +# Create symlinks for fy-tool +install(CODE " + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-dump WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-filter WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-testsuite WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-join WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-ypath WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) + execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink fy-tool fy-compose WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) +") + +install(EXPORT libfyaml-targets + FILE libfyaml-targets.cmake + NAMESPACE libfyaml:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libfyaml ) +export( + TARGETS fyaml + FILE "${CMAKE_CURRENT_BINARY_DIR}/cmake/libfyaml-targets.cmake" + NAMESPACE libfyaml:: +) +# Package config include(CMakePackageConfigHelpers) configure_package_config_file( - ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${PROJECT_NAME}-config.cmake.in - ${CMAKE_CURRENT_BINARY_DIR}/cmake/${PROJECT_NAME}-config.cmake - INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/libfyaml-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/cmake/libfyaml-config.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libfyaml ) write_basic_package_version_file( - ${CMAKE_CURRENT_BINARY_DIR}/cmake/${PROJECT_NAME}-config-version.cmake + ${CMAKE_CURRENT_BINARY_DIR}/cmake/libfyaml-config-version.cmake VERSION ${PROJECT_VERSION} COMPATIBILITY AnyNewerVersion ) install( FILES - ${CMAKE_CURRENT_BINARY_DIR}/cmake/${PROJECT_NAME}-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/cmake/${PROJECT_NAME}-config-version.cmake - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} + ${CMAKE_CURRENT_BINARY_DIR}/cmake/libfyaml-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/cmake/libfyaml-config-version.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/libfyaml +) + +# pkg-config file +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/libfyaml.pc.in + ${CMAKE_CURRENT_BINARY_DIR}/libfyaml.pc + @ONLY ) + +install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/libfyaml.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig +) + +# tags and cscope support +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/tags.cmake) + +# Print configuration summary +message(STATUS "") +message(STATUS "---{ ${PROJECT_NAME} ${PROJECT_VERSION} }---") +message(STATUS "") +message(STATUS "VERSION: ${PROJECT_VERSION}") +message(STATUS "MAJOR.MINOR: ${VERSION_MAJOR}.${VERSION_MINOR}") +message(STATUS "PATCH: ${VERSION_PATCH}") +message(STATUS "EXTRA: ${VERSION_EXTRA}") +message(STATUS "LIBTOOL_VERSION: ${LIBTOOL_VERSION}") +message(STATUS "prefix: ${CMAKE_INSTALL_PREFIX}") +message(STATUS "Build system: ${CMAKE_HOST_SYSTEM_NAME} ${CMAKE_HOST_SYSTEM_PROCESSOR}") +message(STATUS "Target system: ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}") +message(STATUS "Cross compiling: ${CROSS_COMPILING}") +message(STATUS "C compiler: ${CMAKE_C_COMPILER}") +message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") +message(STATUS "HAVE_CHECK: ${HAVE_CHECK}") +message(STATUS "HAVE_COMPATIBLE_CHECK: ${HAVE_COMPATIBLE_CHECK}") +message(STATUS "HAVE_LIBYAML: ${HAVE_LIBYAML}") +message(STATUS "HAVE_CMAKE_GEXPR: ${HAVE_CMAKE_GEXPR}") +message(STATUS "HAVE_LIBCLANG: ${HAVE_LIBCLANG}") +message(STATUS "HAVE_CLANG_BLOCKS: ${HAVE_CLANG_BLOCKS}") +message(STATUS "HAVE_NETWORK: ${HAVE_NETWORK}") +message(STATUS "HAVE_DEVMODE: ${HAVE_DEVMODE}") +message(STATUS "HAVE_GIT: ${HAVE_GIT}") +message(STATUS "HAVE_JQ: ${HAVE_JQ}") +message(STATUS "HAVE_DOCKER: ${HAVE_DOCKER}") +message(STATUS "HAVE_SPHINX: ${HAVE_SPHINX}") +message(STATUS "HAVE_ASAN: ${HAVE_ASAN}") +message(STATUS "TARGET_HAS_SSE2: ${TARGET_HAS_SSE2}") +message(STATUS "TARGET_HAS_SSE41: ${TARGET_HAS_SSE41}") +message(STATUS "TARGET_HAS_AVX2: ${TARGET_HAS_AVX2}") +message(STATUS "TARGET_HAS_AVX512: ${TARGET_HAS_AVX512}") +message(STATUS "TARGET_HAS_NEON: ${TARGET_HAS_NEON}") +message(STATUS "HAVE_HEAP_TRAMPOLINES: ${HAVE_HEAP_TRAMPOLINES}") +message(STATUS "TESTSUITEURL: ${TESTSUITEURL}") +message(STATUS "TESTSUITECHECKOUT: ${TESTSUITECHECKOUT}") +message(STATUS "JSONTESTSUITEURL: ${JSONTESTSUITEURL}") +message(STATUS "JSONTESTSUITECHECKOUT: ${JSONTESTSUITECHECKOUT}") +message(STATUS "") +if(BUILD_TESTING AND NOT CROSS_COMPILING) + message(STATUS "Test targets:") + message(STATUS " make all test - Build everything then run CTest (2464 individual tests)") + message(STATUS " make build_and_test - Convenience target: build and test in one command") + message(STATUS " make check-am - Autotools-compatible TAP test runner (~10 test scripts)") + message(STATUS "") + message(STATUS "NOTE: 'make test' alone will fail if binaries aren't built. Use 'make all test'.") + message(STATUS "") +elseif(CROSS_COMPILING) + message(STATUS "Tests disabled when cross-compiling (target binaries cannot execute on build host)") + message(STATUS "") +endif() diff --git a/Dockerfile b/Dockerfile index b7347ad..6af6f28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,8 @@ RUN apt-get update -qq RUN apt-get install --no-install-recommends -y \ gcc autoconf automake libtool git make libyaml-dev libltdl-dev \ pkg-config check python3 python3-pip python3-setuptools -# install sphinx doc dependencies -RUN pip3 install wheel sphinx git+http://github.com/return42/linuxdoc.git sphinx_rtd_theme sphinx-markdown-builder +# install sphinx doc dependencies globally (without venv) +RUN pip3 install --break-system-packages wheel sphinx git+http://github.com/return42/linuxdoc.git sphinx_rtd_theme sphinx-markdown-builder # configure argument ARG CONFIG_ARGS ENV CONFIG_ARGS=${CONFIG_ARGS:-"--enable-debug --prefix=/usr"} diff --git a/Dockerfile-build-deb b/Dockerfile-build-deb index 079ffbc..268fb3d 100644 --- a/Dockerfile-build-deb +++ b/Dockerfile-build-deb @@ -5,7 +5,7 @@ RUN apt-get update -qq RUN apt-get install --no-install-recommends --fix-missing -y \ gcc autoconf automake libtool git make libyaml-dev libltdl-dev \ pkg-config check python3 python3-pip python3-setuptools \ - devscripts build-essential lintian debhelper dh-buildinfo dh-autoreconf fakeroot \ + devscripts build-essential lintian debhelper dh-autoreconf fakeroot \ gnupg # install sphinx doc dependencies RUN pip3 install wheel sphinx git+http://github.com/return42/linuxdoc.git sphinx_rtd_theme sphinx-markdown-builder diff --git a/Makefile.am b/Makefile.am index d097218..e593fea 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,7 +12,9 @@ EXTRA_DIST=bootstrap.sh \ build-aux/git-version-gen \ README.md LICENSE \ Dockerfile Dockerfile-build-deb \ - Dockerfile.alpine Dockerfile.centos + Dockerfile.alpine Dockerfile.centos \ + cmake CMakeLists.txt \ + examples MAINTAINERCLEANFILES = \ Makefile.in src/Makefile.in config.h.in configure \ diff --git a/README.md b/README.md index 72e9c9a..bdd7af2 100644 --- a/README.md +++ b/README.md @@ -1,466 +1,660 @@ -# libfyaml · ![](https://github.com/pantoniou/libfyaml/workflows/Standard%20Automake%20CI/badge.svg) +# libfyaml -A fancy 1.2 YAML and JSON parser/writer. +[![CI Status](https://github.com/pantoniou/libfyaml/workflows/Standard%20Automake%20CI/badge.svg)](https://github.com/pantoniou/libfyaml/actions) +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) +[![Language: C](https://img.shields.io/badge/language-C-brightgreen.svg)](https://en.wikipedia.org/wiki/C_(programming_language)) -Fully feature complete YAML parser and emitter, supporting the latest YAML spec and -passing the full YAML testsuite. +**A fully-featured YAML 1.2 and JSON parser/writer with zero-copy operation and no artificial limits.** -It is designed to be very efficient, avoiding copies of data, and -has no artificial limits like the 1024 character limit for implicit keys. +libfyaml is designed for high performance with zero content duplication, supporting arbitrarily large documents without the 1024-character limit on implicit keys found in other parsers. It passes the complete YAML test suite and provides both event-based (streaming) and document tree APIs for maximum flexibility. -libfyaml is using https://github.com/yaml/yaml-test-suite as a core part -of it's testsuite. +**Fully MIT Licensed** - v0.9.1 removes all GPL code, making the entire library consistently MIT licensed for use in commercial and proprietary projects. -## Features -* Fully supports YAML version 1.2. -* Attempts to adhere to features coming with YAML version 1.3 so that it - will be ready. -* Zero content copy operation, which means that content is never copied to - internal structures. On input types that support it (mmap files and - constant strings) that means that memory usage is kept low, and arbitrary - large content can be manipulated without problem. -* Parser may be used in event mode (like libyaml) or in document generating mode. -* Extensive programmable API capable of manipulating parsed YAML documents or - creating them from scratch. -* YAML emitter with programmable options, supporting colored output. -* Extensive testsuite for the API, the full YAML test-suite and correct - emitter operation. -* Easy printf/scanf based YAML creation and data extraction API. -* Accurate and descriptive error messages, in standard compiler format that - can be parsed by editors and developer GUIs. -* Testsuite supports running under valgrind and checking for memory leaks. No - leaks should be possible under normal operation, so it is usable for long- - running applications. - -## Contents -- [Features](#features) -- [Prerequisites](#prerequisites) -- [Building](#building) -- [Usage and examples](#usage-and-examples) -- [API documentation](#api-documentation) -- [fy-tool reference](#fy-tool-reference) -- [Missing Features](#missing-features) - -## Prerequisites +**Key Highlights:** +- **Zero-copy architecture** - minimal memory usage, handles arbitrarily large documents +- **YAML 1.2 & JSON** - full spec compliance, passes complete YAML test suite +- **Dual API** - event-based streaming (like libyaml) or document tree manipulation +- **No artificial limits** - unlike libyaml's 1024-char implicit key restriction +- **High performance** - up to 24x faster on large files with streaming mode, thread-safe lockless allocators +- **Rich manipulation** - path expressions, scanf/printf-style APIs, programmatic document building +- **Production-ready** - memory-safe, valgrind-clean, extensive test coverage -libfyaml is primarily developed on Linux based debian distros but Apple MacOS X builds -(using homebrew) are supported as well. +--- -On a based debian distro (i.e. ubuntu 19.04 disco) you should install the following -dependencies: +## Quick Start -* `sudo apt-get install gcc autoconf automake libtool git make libltdl-dev pkg-config` +Parse, query, and modify YAML documents with ease: -To enable the libyaml comparison checker: - -* `sudo apt-get install libyaml-dev` +```c +#include -For the API testsuite libcheck is required: +// Parse YAML from string or file +struct fy_document *fyd = fy_document_build_from_file(NULL, "config.yaml"); -* `sudo apt-get install check` +// Extract values using path-based scanf +unsigned int port; +char hostname[256]; +fy_document_scanf(fyd, "/server/port %u /server/host %255s", &port, hostname); -And finally in order to build the sphinx based documentation: +// Modify document using path-based insertion +fy_document_insert_at(fyd, "/server", FY_NT, fy_node_buildf(fyd, "timeout: 30")); -* `sudo apt-get install python3 python3-pip python3-setuptools` -* `pip3 install wheel sphinx git+http://github.com/return42/linuxdoc.git sphinx\_rtd\_theme sphinx-markdown-builder` +// Emit as YAML or JSON +fy_emit_document_to_file(fyd, FYECF_SORT_KEYS, "config.yaml"); +fy_document_destroy(fyd); +``` -Note that some older distros (like xenial) do not have a sufficiently recent -sphinx in their repos. In that case you can create a virtual environment -using scripts/create-virtual-env +--- -## Building +## What's New in v0.9.1 -``libfyaml`` uses a standard autotools based build scheme so: +**Major Updates:** -* `./bootstrap.sh` -* `./configure` -* `make` +- **GPL Code Removed**: All GPL-licensed code has been replaced with MIT-licensed implementations. The entire library is now consistently MIT licensed. +- **Performance**: Up to 24x faster on large files with new streaming mode default in fy-tool +- **Thread Safety**: New thread-safe lockless allocators for multi-threaded applications +- **Stability**: Fixed 12 critical bugs including segmentation faults and memory corruption +- **New APIs**: Parser checkpointing, composer interface, document builder, YPath set operators +- **Build System**: CMake now at feature parity with autotools, cross-compilation support -Will build the library and `fy-tool`. +See [CHANGELOG.md](CHANGELOG.md) for complete release notes. -* `make check` +--- -Will run the test-suite. +## Why libfyaml? -Binaries, libraries, header files and pkgconfig files maybe installed with +### Zero-Copy Efficiency +Content is never duplicated internally. libfyaml uses "atoms" that reference source data directly, keeping memory usage low and enabling handling of gigabyte-sized documents without performance degradation. -* `make install` +### No Artificial Limits +Unlike libyaml's 1024-character limit on implicit keys, libfyaml has no hardcoded restrictions. Parse real-world YAML documents of any complexity without worrying about hitting parser limits. -By default, the installation prefix will be `/usr/local`, which you can change -with the `--prefix ` option during configure. +### Dual API Design +Choose the right API for your use case: +- **Event-based** (streaming): For memory-efficient processing of large streams +- **Document tree**: For convenient manipulation, queries, and modifications -To build the documentation API in HTML format use: +Both APIs coexist and can be mixed as needed. -* `make doc-html` +### Powerful Manipulation +- **Path expressions** (YPATH): XPath-like queries (`/foo/bar`, `/items/[0]`) +- **scanf/printf style**: Extract and build data with format strings +- **Programmatic building**: Construct documents from scratch with simple APIs +- **Rich emitter**: Output as YAML (block/flow), JSON, with colors and formatting options -The documentation for the public API will be found in doc/\_build/html +### Production Quality +- **Memory safe**: Valgrind-clean, no leaks under normal operation +- **Accurate errors**: Compiler-format diagnostics that integrate with IDEs +- **Fully tested**: Passes complete YAML 1.2 test suite -* `make doc-latexpdf` +--- -Will generate a single pdf containing everything. +## Features -## Usage and examples +### YAML & JSON Support +- Full YAML 1.2 specification compliance +- YAML 1.3 compatibility (preparing for upcoming spec) +- JSON parsing and emission +- Comment preservation (experimental) +- Complete test suite coverage + +### Parsing Modes +- Event-based streaming parser (like libyaml) +- Document tree builder with full manipulation API +- Configurable resolution of anchors and merge keys +- Multiple input sources: files, strings, file pointers, streams + +### Document Manipulation +- Path-based queries and modifications (YPATH) +- scanf/printf-style data extraction and insertion +- Programmatic document and node creation +- Clone, copy, and merge operations +- Anchor and alias management +- Mapping key sorting + +### Output & Emission +- Multiple output modes: original, block, flow, JSON +- ANSI color output support +- Visible whitespace mode for debugging +- Configurable indentation and line width +- Streaming or buffered emission + +### Performance & Scalability +- Zero-copy operation on mmap'd files and constant strings +- No artificial limits on key lengths, nesting depth, or document size +- Thread-safe lockless allocators (linear, mremap, dedup, auto) for multi-threaded applications +- Streaming mode with up to 24x performance improvement on large files +- Configurable memory allocators for different usage patterns +- Efficient handling of arbitrarily large documents + +### Developer Experience +- Single header inclusion: `` +- Opaque pointers - stable ABI across versions +- Descriptive error messages in compiler format +- Extensive API documentation +- Multiple detailed examples + +--- + +## Installation + +### Using CMake (Recommended) + +Add libfyaml to your CMake project: + +```cmake +find_package(libfyaml 0.9.1 REQUIRED) +target_link_libraries(your_app PRIVATE libfyaml::libfyaml) +``` -Usage of libfyaml is somewhat similar to libyaml, but with a few notable differences. +Check for optional features: +```cmake +if(libfyaml_HAS_LIBCLANG) + message(STATUS "Reflection support available") +endif() +``` -1. The objects of the library are opaque, they are pointers that may be used but - may not be derefenced via library users. This makes the public API not be dependent of - internal changes in the library structures. +### Using pkg-config -2. The object pointers used are guaranteed to not 'move' like libyaml object pointers - so you may embed them freely in your own structures. +```bash +# Compile flags +pkg-config --cflags libfyaml -3. The convenience methods of libyaml allow you to avoid tedious iteration and code - duplication. While fully manual YAML document tree manipulation is available, if your - application is not performance sensitive when manipulating YAML, you are advised to - use the helpers. +# Linker flags +pkg-config --libs libfyaml +``` -### Using libfyaml in your projects +In your Makefile: +```make +CFLAGS += $(shell pkg-config --cflags libfyaml) +LDFLAGS += $(shell pkg-config --libs libfyaml) +``` -Typically you only have to include the single header file `libfyaml.h` and -link against the correct fyaml-\-\ library. +### Building from Source -It is recommended to use pkg-config, i.e. +**Using CMake:** +```bash +mkdir build && cd build +cmake .. +cmake --build . +ctest # Run tests +cmake --install . # Install +``` -```make -CFLAGS+= `pkg-config --cflags libfyaml` -LDFLAGS+= `pkg-config --libs libfyaml` +CMake configuration options: +```bash +cmake -DBUILD_SHARED_LIBS=ON \ # Build shared libraries (default: ON) + -DENABLE_ASAN=OFF \ # AddressSanitizer for debugging (default: OFF) + -DENABLE_STATIC_TOOLS=OFF \ # Static tool executables (default: OFF) + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + .. ``` -For use in an automake based project you may use the following fragment +**Using Autotools:** ```bash -PKG_CHECK_MODULES(LIBFYAML, [ libfyaml ], HAVE_LIBFYAML=1, HAVE_LIBFYAML=0) +./bootstrap.sh +./configure +make +make check # Run tests +sudo make install +``` -if test "x$HAVE_LIBFYAML" != "x1" ; then - AC_MSG_ERROR([failed to find libfyaml]) -fi +### Prerequisites -AC_SUBST(HAVE_LIBFYAML) -AC_SUBST(LIBFYAML_CFLAGS) -AC_SUBST(LIBFYAML_LIBS) -AC_DEFINE_UNQUOTED([HAVE_LIBFYAML], [$HAVE_LIBFYAML], [Define to 1 if you have libfyaml available]) -AM_CONDITIONAL([HAVE_LIBFYAML], [ test x$HAVE_LIBFYAML = x1 ]) +**Debian/Ubuntu:** +```bash +sudo apt install gcc autoconf automake libtool git make libltdl-dev pkg-config check ``` -The examples that follow will make things clear. +**Optional dependencies:** +- `libyaml-dev` - for comparison testing +- `llvm-dev libclang-dev` - for reflection support (experimental) -### Display libfyaml version example +--- -This is the minimal example that checks that you've compiled against the correct libfyaml. +## Usage Examples -```c -/* - * fy-version.c - libfyaml version example - * - * Copyright (c) 2019 Pantelis Antoniou - * - * SPDX-License-Identifier: MIT - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif +### Level 1: Basic Parsing +Parse and extract data from a YAML file: + +```c #include #include - #include -int main(int argc, char *argv[]) -{ - printf("%s\n", fy_library_version()); - return EXIT_SUCCESS; +int main(void) { + struct fy_document *fyd = fy_document_build_from_file(NULL, "config.yaml"); + if (!fyd) { + fprintf(stderr, "Failed to parse YAML\n"); + return EXIT_FAILURE; + } + + // Get root node and emit + fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stdout); + + fy_document_destroy(fyd); + return EXIT_SUCCESS; } ``` -### libfyaml example using simplified inprogram YAML generation +### Level 2: Path-Based Queries -This example simply parses an in-program YAML string and displays -a string. - -The standard header plus variables definition. +Extract specific values using YPATH expressions: ```c -/* - * inprogram.c - libfyaml inprogram YAML example - * - * Copyright (c) 2019 Pantelis Antoniou - * - * SPDX-License-Identifier: MIT - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include - #include -int main(int argc, char *argv[]) -{ - static const char *yaml = - "invoice: 34843\n" - "date : !!str 2001-01-23\n" - "bill-to: &id001\n" - " given : Chris\n" - " family : Dumars\n" - " address:\n" - " lines: |\n" - " 458 Walkman Dr.\n" - " Suite #292\n"; - struct fy_document *fyd = NULL; - int rc, count, ret = EXIT_FAILURE; - unsigned int invoice_nr; - char given[256 + 1]; +int main(void) { + const char *yaml = + "server:\n" + " host: localhost\n" + " port: 8080\n" + " ssl: true\n"; + + struct fy_document *fyd = fy_document_build_from_string(NULL, yaml, FY_NT); + + // Extract multiple values at once + char host[256]; + unsigned int port; + fy_document_scanf(fyd, + "/server/host %255s " + "/server/port %u", + host, &port); + + printf("Server: %s:%u\n", host, port); + + // Query single node by path + struct fy_node *ssl_node = fy_node_by_path( + fy_document_root(fyd), "/server/ssl", FY_NT, FYNWF_DONT_FOLLOW); + if (ssl_node && fy_node_compare_string(ssl_node, "true", FY_NT) == 0) { + printf("SSL is enabled\n"); + } + + fy_document_destroy(fyd); + return 0; +} ``` -Parsing and creating a YAML document from either the built-in -YAML, or an invoice file given on the command line: +### Level 3: Document Manipulation + +Modify existing documents programmatically: ```c - if (argc == 1) - fyd = fy_document_build_from_string(NULL, yaml, FY_NT); - else - fyd = fy_document_build_from_file(NULL, argv[1]); - if (!fyd) { - fprintf(stderr, "failed to build document"); - goto fail; - } +#include + +int main(void) { + struct fy_document *fyd = fy_document_build_from_file(NULL, "invoice.yaml"); + + // Read current invoice number + unsigned int invoice_nr; + char given_name[256]; + int count = fy_document_scanf(fyd, + "/invoice %u " + "/bill-to/given %255s", + &invoice_nr, given_name); + + if (count == 2) { + printf("Processing invoice #%u for %s\n", invoice_nr, given_name); + + // Update invoice number + fy_document_insert_at(fyd, "/invoice", FY_NT, + fy_node_buildf(fyd, "%u", invoice_nr + 1)); + + // Add new fields + fy_document_insert_at(fyd, "/bill-to", FY_NT, + fy_node_buildf(fyd, "spouse: %s", "Jane")); + + fy_document_insert_at(fyd, "/bill-to", FY_NT, + fy_node_buildf(fyd, + "delivery-address:\n" + " street: 123 Main St\n" + " city: Springfield\n")); + + // Save with sorted keys + fy_emit_document_to_file(fyd, + FYECF_DEFAULT | FYECF_SORT_KEYS, + "invoice_updated.yaml"); + } + + fy_document_destroy(fyd); + return 0; +} ``` -Get the invoice number and the given name using a single call. +### Level 4: Event-Based Streaming + +Process large YAML files with minimal memory: ```c - /* get the invoice number and the given name */ - count = fy_document_scanf(fyd, - "/invoice %u " - "/bill-to/given %256s", - &invoice_nr, given); - if (count != 2) { - fprintf(stderr, "Failed to retreive the two items\n"); - goto fail; - } - - /* print them as comments in the emitted YAML */ - printf("# invoice number was %u\n", invoice_nr); - printf("# given name is %s\n", given); +#include + +int main(void) { + struct fy_parser *fyp = fy_parser_create(NULL); + if (!fyp) + return EXIT_FAILURE; + + // Set input + fy_parser_set_input_file(fyp, "large_file.yaml"); + + // Process events + struct fy_event *fye; + while ((fye = fy_parser_parse(fyp)) != NULL) { + enum fy_event_type type = fye->type; + + switch (type) { + case FYET_SCALAR: { + const char *value = fy_token_get_text0(fy_event_get_token(fye)); + printf("Scalar: %s\n", value); + break; + } + case FYET_MAPPING_START: + printf("Mapping start\n"); + break; + case FYET_SEQUENCE_START: + printf("Sequence start\n"); + break; + default: + break; + } + + fy_parser_event_free(fyp, fye); + } + + fy_parser_destroy(fyp); + return 0; +} ``` -In sequence, increase the invoice number, add a spouse and a secondary -address. +### Level 5: Building Documents Programmatically + +Create YAML documents from scratch: ```c - rc = - /* set increased invoice number (modify existing node) */ - fy_document_insert_at(fyd, "/invoice", FY_NT, - fy_node_buildf(fyd, "%u", invoice_nr + 1)) || - /* add spouse (create new mapping pair) */ - fy_document_insert_at(fyd, "/bill-to", FY_NT, - fy_node_buildf(fyd, "spouse: %s", "Doris")) || - /* add a second address */ - fy_document_insert_at(fyd, "/bill-to", FY_NT, - fy_node_buildf(fyd, "delivery-address:\n" - " lines: |\n" - " 1226 Windward Ave.\n")); - if (rc) { - fprintf(stderr, "failed to insert to document\n"); - goto fail; - } +#include + +int main(void) { + // Create empty document + struct fy_document *fyd = fy_document_create(NULL); + + // Build root mapping using printf-style formatting + struct fy_node *root = fy_node_buildf(fyd, + "application: MyApp\n" + "version: %d.%d.%d\n" + "settings:\n" + " debug: %s\n" + " max_connections: %d\n" + " allowed_hosts:\n" + " - localhost\n" + " - 127.0.0.1\n", + 1, 2, 3, + "true", + 100); + + fy_document_set_root(fyd, root); + + // Add more fields programmatically + fy_document_insert_at(fyd, "/settings", FY_NT, + fy_node_buildf(fyd, "log_level: info")); + + // Output as JSON + printf("As JSON:\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_JSON, stdout); + + printf("\nAs YAML:\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_BLOCK | FYECF_SORT_KEYS, stdout); + + fy_document_destroy(fyd); + return 0; +} ``` -Emit the document to standard output (while sorting the keys) +--- -```c - /* emit the document to stdout (but sorted) */ - rc = fy_emit_document_to_fp(fyd, FYECF_DEFAULT | FYECF_SORT_KEYS, stdout); - if (rc) { - fprintf(stderr, "failed to emit document to stdout"); - goto fail; - } -``` +## API Overview -Finally exit and report condition. +libfyaml provides a comprehensive API organized into functional categories: -```c - ret = EXIT_SUCCESS; -fail: - fy_document_destroy(fyd); /* NULL is OK */ +| Category | Purpose | Key Functions | +|----------|---------|---------------| +| **Parser** | Event-based streaming YAML parsing | `fy_parser_create()`, `fy_parser_parse()`, `fy_parser_set_input_*()` | +| **Parser Checkpointing** | Look-ahead and peek operations | `fy_parser_parse_peek()`, `fy_parser_skip()`, `fy_parser_count_*()` | +| **Composer** | Event-based document composition | `fy_composer_create()`, `fy_composer_process_event()` | +| **Document** | High-level document tree manipulation | `fy_document_build_from_*()`, `fy_document_scanf()`, `fy_document_insert_at()` | +| **Document Builder** | Programmatic document construction | `fy_document_builder_create()`, `fy_document_builder_*()` | +| **Node** | Individual YAML node operations | `fy_node_create_*()`, `fy_node_by_path()`, `fy_node_buildf()`, `fy_node_delete()` | +| **Sequence** | Array/list operations | `fy_node_sequence_iterate()`, `fy_node_sequence_append()` | +| **Mapping** | Key-value dictionary operations | `fy_node_mapping_lookup_*()`, `fy_node_mapping_append()` | +| **Emitter** | YAML/JSON output generation | `fy_emitter_create()`, `fy_emit_document_to_*()`, `fy_emit_body_node()` | +| **Path** | XPath-like YAML queries (YPATH) | `fy_node_by_path()`, `fy_path_exec_execute()` | +| **Anchor** | Anchor and alias management | `fy_document_lookup_anchor()`, `fy_node_create_alias()` | +| **Allocator** | Memory allocation control | `fy_allocator_create()`, `fy_allocator_contains()` | - return ret; -} -``` +**See the [complete API documentation](https://pantoniou.github.io/libfyaml/) for detailed reference.** + +--- -## API documentation +## Tools -For complete documentation of libfyaml API, visit https://pantoniou.github.io/libfyaml/ +libfyaml includes `fy-tool`, a multi-function YAML manipulation utility: -## fy-tool reference +### fy-dump - Parse and Pretty-Print -A YAML manipulation tool is included in libfyaml, aptly name `fy-tool`. -It's a multi tool application, acting differently according to the name it has -when it's invoked. There are four tool modes, namely: +**Note**: As of v0.9.1, fy-dump uses streaming mode by default for up to 24x better performance on large files. Use `--no-streaming` if you need forward anchor references or strict duplicate key checking. -* fy-testsuite: Used for outputing a test-suite specific event stream which is - used for comparison with the expected output of the suite. -* fy-dump: General purpose YAML parser and dumper, with syntax coloring support, - visible whitespace options, and a number of output modes. -* fy-filter: YAML filtering tool allows to extract information out of a YAML - document. -* fy-join: YAML flexible join tool. +```bash +# Convert YAML to JSON (streaming mode) +fy-dump -m json config.yaml -### fy-testsuite usage +# Use document mode for forward anchors +fy-dump --no-streaming -m json config.yaml -A number of options are common in every fy-tool invocation: +# Format with visible whitespace +fy-dump -V -m block messy.yaml +# Pretty-print with sorted keys +fy-dump -s -m block config.yaml > formatted.yaml ``` -Usage : fy-tool [options] [args] - -Options: - - --include, -I : Add directory to include path (default path "") - --debug-level, -d : Set debug level to (default level 3) - --indent, -i : Set dump indent to (default indent 2) - --width, -w : Set dump width to (default width 80) - --resolve, -r : Perform anchor and merge key resolution (default false) - --color, -C : Color output can be one of on, off, auto (default auto) - --visible, -V : Make all whitespace and linebreaks visible (default false) - --follow, -l : Follow aliases when using paths (default false) - --strip-labels : Strip labels when emitting (default false) - --strip-tags : Strip tags when emitting (default false) - --strip-doc : Strip document headers and indicators when emitting (default false) - --quiet, -q : Quiet operation, do not output messages (default false) - --version, -v : Display libfyaml version - --help, -h : Display help message +### fy-filter - Extract Document Parts + +```bash +# Extract specific paths +fy-filter --file config.yaml /database/host /database/port + +# Filter with complex keys +echo "{ foo: bar }: baz" | fy-filter "/{foo: bar}/" +# Output: baz ``` +### fy-join - Merge YAML Documents + +```bash +# Merge two configuration files +fy-join base-config.yaml override-config.yaml + +# Merge inline YAML +fy-join ">foo: bar" ">baz: qux" +# Output: +# foo: bar +# baz: qux ``` -Usage: fy-testsuite [options] [args] - - [common options] - - Parse and dump test-suite event format - $ fy-testsuite input.yaml - ... - - Parse and dump of event example - $ echo "foo: bar" | fy-testsuite - - +STR - +DOC - +MAP - =VAL :foo - =VAL :bar - -MAP - -DOC - -STR + +### fy-testsuite - Test Suite Event Format + +```bash +# Generate test-suite event stream +echo "foo: bar" | fy-testsuite - +# Output: +# +STR +# +DOC +# +MAP +# =VAL :foo +# =VAL :bar +# -MAP +# -DOC +# -STR ``` -### fy-dump usage +**All tools support:** +- Syntax coloring (`--color`) +- Whitespace visualization (`--visible`) +- Anchor resolution (`--resolve`) +- Multiple output modes (`--mode`) +- Flexible input from files, stdin, or inline strings (`">yaml here"`) + +--- + +## Performance & Architecture + +### Zero-Copy Design + +libfyaml's core design principle is **zero content duplication**: + +- **Atoms**: Internal references to source data without copying +- **Memory efficiency**: Only metadata is allocated, content stays in source +- **Scalability**: Handles multi-gigabyte documents with minimal RAM +- **Speed**: No time wasted on redundant memory operations + +### Memory Usage + +For a typical YAML document: +- **libyaml**: Copies all content into internal structures +- **libfyaml**: References original data, allocates only ~1-5% overhead for structure + +### No Artificial Limits +Common parser limitations that libfyaml **does not have**: +- No 1024-char limit on implicit keys (unlike libyaml) +- No hardcoded nesting depth limits +- No document size restrictions +- No key count limits per mapping + +Configurable limits exist for safety but can be adjusted or removed entirely. + +--- + +## Documentation + +- **[API Reference](https://pantoniou.github.io/libfyaml/)** - Complete API documentation +- **[Examples](examples/)** - Sample code and use cases +- **[CMake Integration](cmake/README-package.md)** - Using libfyaml in CMake projects +- **[Changelog](CHANGELOG.md)** - Version history and release notes + +### Building Documentation Locally + +```bash +# HTML documentation +make doc-html +# Output: doc/_build/html/index.html + +# PDF documentation +make doc-latexpdf +# Output: doc/_build/latex/libfyaml.pdf ``` -Usage: fy-dump [options] [args] -Options: +--- + +## Platform Support - [common options] +**Primary Platform**: Linux (Debian/Ubuntu, Fedora, Arch, etc.) +**Also Supported**: macOS (via Homebrew) +**Not Yet Supported**: Windows (contributions welcome!) - --sort, -s : Perform mapping key sort (valid for dump) (default false) - --comment, -c : Output comments (experimental) (default false) - --mode, -m : Output mode can be one of original, block, flow, flow-oneline, json, json-tp, json-oneline (default original) - --streaming : Use streaming output mode (default false) +**Architectures**: x86, x86_64, ARM, ARM64, and others +**Compilers**: GCC, Clang, ICC - [common options] +--- - Parse and dump generated YAML document tree in the original YAML form - $ fy-dump input.yaml - ... +## Contributing - Parse and dump generated YAML document tree in block YAML form (and make whitespace visible) - $ fy-dump -V -mblock input.yaml - ... +We welcome contributions! Here's how to get involved: - Parse and dump generated YAML document from the input string - $ fy-dump -mjson ">foo: bar" - { - "foo": "bar" - } +1. **Report Issues**: [GitHub Issues](https://github.com/pantoniou/libfyaml/issues) +2. **Submit Pull Requests**: Fork, modify, and submit PRs +3. **Improve Documentation**: Help make libfyaml more accessible +4. **Add Examples**: Share your use cases with the community - Parse and dump generated YAML document from the input string (using streaming mode) - $ fy-dump --streaming ">foo: bar" - foo: bar +### Development Setup - Note that streaming mode can not perform document validity checks, like duplicate keys nor - support the sort keys option. +```bash +# Clone repository +git clone https://github.com/pantoniou/libfyaml.git +cd libfyaml + +# Build with CMake +mkdir build && cd build +cmake -DENABLE_ASAN=ON -DCMAKE_BUILD_TYPE=Debug .. +cmake --build . +ctest + +# Or build with Autotools +./bootstrap.sh +./configure --enable-asan +make check ``` -### fy-filter usage +### Running Tests + +```bash +# All tests +make check + +# Specific test suites +./test/libfyaml.test +./test/testemitter.test + +# With valgrind +./scripts/run-valgrind.sh +``` +For CMake ctest is supported +```bash +cd build +ctest --progress -j`nproc` ``` -Usage: fy-filter [options] [args] -Options: +--- - [common options] +## Missing Features - --sort, -s : Perform mapping key sort (valid for dump) (default false) - --comment, -c : Output comments (experimental) (default false) - --mode, -m : Output mode can be one of original, block, flow, flow-oneline, json, json-tp, json-oneline (default original) - --file, -f : Use given file instead of - Note that using a string with a leading '>' is equivalent to a file with the trailing content - --file ">foo: bar" is as --file file.yaml with file.yaml "foo: bar" +Current limitations: +- **Windows support**: Not yet implemented (contributions welcome) +- **Unicode**: UTF-8 only, no wide character input support - Parse and filter YAML document tree starting from the '/foo' path followed by the '/bar' path - $ fy-filter --file input.yaml /foo /bar - ... +--- - Parse and filter for two paths (note how a multi-document stream is produced) - $ fy-filter --file -mblock --filter --file ">{ foo: bar, baz: [ frooz, whee ] }" /foo /baz - bar - --- - - frooz - - whee +## License - Parse and filter YAML document in stdin (note how the key may be complex) - $ echo "{ foo: bar }: baz" | fy-filter "/{foo: bar}/" - baz -``` +libfyaml is released under the **MIT License**. -### fy-join usage +**As of version 0.9.1, all GPL-licensed code has been removed.** The entire library is now consistently MIT licensed, making it suitable for use in commercial and proprietary projects without any GPL licensing concerns. ``` -Usage: fy-join [options] [args] - -Options: - - [common options] - - --sort, -s : Perform mapping key sort (valid for dump) (default false) - --comment, -c : Output comments (experimental) (default false) - --mode, -m : Output mode can be one of original, block, flow, flow-oneline, json, json-tp, json-oneline (default original) - --file, -f : Use given file instead of - Note that using a string with a leading '>' is equivalent to a file with the trailing content - --file ">foo: bar" is as --file file.yaml with file.yaml "foo: bar" - --to, -T : Join to (default /) - --from, -F : Join from (default /) - --trim, -t : Output given path (default /) - - Parse and join two YAML files - $ fy-join file1.yaml file2.yaml - ... - - Parse and join two YAML maps - $ fy-join ">foo: bar" ">baz: frooz" - foo: bar - baz: frooz +Copyright (c) 2019-2025 Pantelis Antoniou + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. ``` -## Missing features and omissions +See [LICENSE](LICENSE) for complete terms. + +--- + +## Acknowledgments + +- **YAML Test Suite**: https://github.com/yaml/yaml-test-suite +- **Author**: Pantelis Antoniou [@pantoniou](https://github.com/pantoniou) +- **Contributors**: Thank you to all who have contributed code, bug reports, and feedback! -1. Windows - libfyaml is not supporting windows yet. -2. Unicode - libfyaml only supports UTF8 and has no support for wide character input. +--- -## Development and contributing -Feel free to send pull requests and raise issues. +**[Back to Top](#libfyaml)** diff --git a/bootstrap.sh b/bootstrap.sh index 5d39576..fc00c0d 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # potato autoconf touch build-aux/tap-driver.sh autoreconf -fvi diff --git a/cmake/CheckClangBlocks.cmake b/cmake/CheckClangBlocks.cmake new file mode 100644 index 0000000..a3b67b6 --- /dev/null +++ b/cmake/CheckClangBlocks.cmake @@ -0,0 +1,53 @@ +# --- CheckClangBlocks.cmake --- +# Detect support for Clang Blocks (Clang or AppleClang) +# Defines: HAVE_CLANG_BLOCKS (0 or 1) +# Defines: CLANG_BLOCKS_LIB (empty or "BlocksRuntime") + +# Default +set(HAVE_CLANG_BLOCKS 0) +set(CLANG_BLOCKS_LIB "") + +if (CMAKE_C_COMPILER_ID MATCHES "Clang") + + # AppleClang: Blocks are built-in + if (CMAKE_C_COMPILER_ID STREQUAL "AppleClang") + message(STATUS "AppleClang detected — Blocks support is built in") + + set(HAVE_CLANG_BLOCKS 1) + set(CLANG_BLOCKS_LIB "") + + else() + # LLVM Clang: must test for -fblocks and libBlocksRuntime + message(STATUS "LLVM Clang detected — checking for BlocksRuntime") + + include(CheckCSourceCompiles) + + set(CMAKE_REQUIRED_FLAGS "-fblocks") + set(CMAKE_REQUIRED_LIBRARIES "BlocksRuntime") + + check_c_source_compiles(" + #include + int main(void) { + void (^b)(void) = ^{ }; + b(); + return 0; + } + " HAVE_WORKING_CLANG_BLOCKS) + + unset(CMAKE_REQUIRED_FLAGS) + unset(CMAKE_REQUIRED_LIBRARIES) + + if (HAVE_WORKING_CLANG_BLOCKS) + message(STATUS "Clang Blocks support ENABLED") + set(HAVE_CLANG_BLOCKS 1) + set(CLANG_BLOCKS_LIB "BlocksRuntime") + else() + message(WARNING + "libBlocksRuntime not found — Clang Blocks disabled") + set(HAVE_CLANG_BLOCKS 0) + set(CLANG_BLOCKS_LIB "") + endif() + + endif() + +endif() diff --git a/cmake/CheckLibClang.cmake b/cmake/CheckLibClang.cmake new file mode 100644 index 0000000..65460b7 --- /dev/null +++ b/cmake/CheckLibClang.cmake @@ -0,0 +1,77 @@ +# CheckLibClang.cmake - Helper function to test if libclang works +# +# This module provides a function to test if libclang can be compiled and linked +# with the given compiler flags and libraries. +# +# Usage: +# check_libclang_works( +# CFLAGS +# INCLUDES +# LINK_DIRS +# LIBRARIES ) +# +# Arguments: +# result_var - Variable name to store the result (TRUE/FALSE) +# CFLAGS - Compiler flags to use (can be a list or string) +# INCLUDES - Include directories (list) +# LINK_DIRS - Link directories (list) +# LIBRARIES - Libraries to link (list or string) +# +# Example: +# check_libclang_works(LIBCLANG_WORKS +# CFLAGS "${LLVM_DEFINITIONS}" +# INCLUDES "${LLVM_INCLUDE_DIRS};${CLANG_INCLUDE_DIRS}" +# LINK_DIRS "${LLVM_LIBRARY_DIRS}" +# LIBRARIES "${LIBCLANG_LIBS}") + +include(CheckCSourceCompiles) + +function(check_libclang_works result_var) + # Parse arguments + set(options "") + set(oneValueArgs "") + set(multiValueArgs CFLAGS INCLUDES LINK_DIRS LIBRARIES) + cmake_parse_arguments(ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + # Save current CMAKE_REQUIRED_* variables + set(_SAVED_CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}") + set(_SAVED_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") + set(_SAVED_CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES}") + set(_SAVED_CMAKE_REQUIRED_LINK_DIRECTORIES "${CMAKE_REQUIRED_LINK_DIRECTORIES}") + + # Set up test environment + if(ARG_CFLAGS) + set(CMAKE_REQUIRED_FLAGS "${ARG_CFLAGS}") + endif() + + if(ARG_INCLUDES) + set(CMAKE_REQUIRED_INCLUDES "${ARG_INCLUDES}") + endif() + + if(ARG_LINK_DIRS) + set(CMAKE_REQUIRED_LINK_DIRECTORIES "${ARG_LINK_DIRS}") + endif() + + if(ARG_LIBRARIES) + set(CMAKE_REQUIRED_LIBRARIES "${ARG_LIBRARIES}") + endif() + + # Test if we can compile and link with libclang + check_c_source_compiles(" + #include + int main() { + CXIndex idx = clang_createIndex(0, 0); + clang_disposeIndex(idx); + return 0; + } + " _LIBCLANG_COMPILE_TEST) + + # Restore CMAKE_REQUIRED_* variables + set(CMAKE_REQUIRED_LIBRARIES "${_SAVED_CMAKE_REQUIRED_LIBRARIES}" PARENT_SCOPE) + set(CMAKE_REQUIRED_FLAGS "${_SAVED_CMAKE_REQUIRED_FLAGS}" PARENT_SCOPE) + set(CMAKE_REQUIRED_INCLUDES "${_SAVED_CMAKE_REQUIRED_INCLUDES}" PARENT_SCOPE) + set(CMAKE_REQUIRED_LINK_DIRECTORIES "${_SAVED_CMAKE_REQUIRED_LINK_DIRECTORIES}" PARENT_SCOPE) + + # Return result + set(${result_var} ${_LIBCLANG_COMPILE_TEST} PARENT_SCOPE) +endfunction() diff --git a/cmake/add-tap-subtests.cmake b/cmake/add-tap-subtests.cmake new file mode 100644 index 0000000..638ee2c --- /dev/null +++ b/cmake/add-tap-subtests.cmake @@ -0,0 +1,690 @@ +# Helper function to add individual TAP subtests from test scripts +# This mimics the behavior of autotools tap-driver.sh by registering each subtest individually + +# Function to run a single test from testerrors.test +function(add_testerrors_tests) + file(GLOB error_dirs "${CMAKE_CURRENT_SOURCE_DIR}/test/test-errors/[0-9][0-9][0-9][0-9]") + + foreach(dir ${error_dirs}) + get_filename_component(test_id "${dir}" NAME) + + # Read description from === file + set(desc_file "${dir}/===") + if(EXISTS "${desc_file}") + file(READ "${desc_file}" test_desc) + string(STRIP "${test_desc}" test_desc) + else() + set(test_desc "") + endif() + + # Build test name with description if available + if(test_desc) + set(test_name_full "testerrors/${test_id} - ${test_desc}") + else() + set(test_name_full "testerrors/${test_id}") + endif() + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + testerrors "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set properties + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test" + LABELS "testerrors" + ) + endforeach() +endfunction() + +# Function to add testemitter tests +function(add_testemitter_tests test_name extra_args) + file(GLOB yaml_files "${CMAKE_CURRENT_SOURCE_DIR}/test/emitter-examples/*.yaml") + + foreach(yaml_file ${yaml_files}) + get_filename_component(test_id "${yaml_file}" NAME) + + add_test( + NAME "${test_name}/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + "${test_name}" "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("${test_name}/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;EXTRA_DUMP_ARGS=${extra_args}" + LABELS "${test_name}" + ) + endforeach() +endfunction() + +# Function to add testsuite tests (yaml test suite) +# Can work either by scanning the test-suite-data directory (if it exists) +# or by using a pre-generated list of tests from testsuite-tests.cmake +function(add_testsuite_tests test_name test_script) + set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/test/test-suite-data") + set(use_pregenerated_list FALSE) + + # Define skip/xfail lists based on test suite variant + # These match the lists in test/*.test scripts + if(test_name STREQUAL "testsuite-json") + set(skip_list "UGM3") + set(xfail_list "C4HZ") + elseif(test_name STREQUAL "testsuite-resolution") + set(skip_list "2JQS" "X38W") + set(xfail_list "") + elseif(test_name STREQUAL "testsuite-evstream") + set(skip_list "2JQS") + set(xfail_list "") + else() + set(skip_list "") + set(xfail_list "") + endif() + + # Check if we should use the pre-generated list + if(NOT EXISTS "${test_dir}") + # Try to use pre-generated list if available + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/testsuite-tests.cmake") + message(STATUS "Using pre-generated test list from cmake/testsuite-tests.cmake for ${test_name}") + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/testsuite-tests.cmake") + set(use_pregenerated_list TRUE) + else() + # Neither data nor pre-generated list exists, register monolithic fallback + message(STATUS "Test suite data not found - registering monolithic ${test_name} test") + message(STATUS "Run 'ctest -R yaml-test-suite-setup' then re-run cmake to register individual subtests") + + add_test(NAME ${test_name} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test/${test_script} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + set_tests_properties(${test_name} PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + FIXTURES_REQUIRED yaml-test-suite-data + ) + return() + endif() + else() + message(STATUS "Registering individual ${test_name} subtests from filesystem") + endif() + + if(use_pregenerated_list) + # Use pre-generated TESTSUITE_ALL_TESTS list + foreach(test_id ${TESTSUITE_ALL_TESTS}) + # test_id is either "BASE" or "BASE/SUBTEST" + string(REPLACE "/" ";" test_parts "${test_id}") + list(LENGTH test_parts parts_count) + + add_test( + NAME "${test_name}/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + "${test_name}" "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("${test_name}/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "${test_name}" + FIXTURES_REQUIRED "yaml-test-suite-data" + ) + endforeach() + else() + # Scan the filesystem (original behavior) + file(GLOB base_tests "${test_dir}/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]") + + foreach(base_test ${base_tests}) + get_filename_component(test_id "${base_test}" NAME) + + # Check for main test + if(EXISTS "${base_test}/===") + # Read description from === file + file(READ "${base_test}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "${test_name}/${test_id} - ${test_desc}") + else() + set(test_name_full "${test_name}/${test_id}") + endif() + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + "${test_name}" "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "${test_name}") + set(test_disabled FALSE) + + # Check if test should be skipped + if(test_id IN_LIST skip_list) + set(test_disabled TRUE) + set(test_labels "${test_name};skipped") + endif() + + # Check if test is expected to fail (xfail) + # Note: TAP xfail (TODO) tests still pass (exit 0), they just mark the result as TODO + # So we don't use WILL_FAIL here, just add a label for filtering + if(test_id IN_LIST xfail_list) + set(test_labels "${test_name};xfail") + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "${test_labels}" + FIXTURES_REQUIRED "yaml-test-suite-data" + DISABLED ${test_disabled} + ) + endif() + + # Check for 2-digit subtests + file(GLOB subtests_2d "${base_test}/[0-9][0-9]") + foreach(subtest ${subtests_2d}) + if(EXISTS "${subtest}/===") + get_filename_component(subtest_id "${subtest}" NAME) + set(full_test_id "${test_id}/${subtest_id}") + + # Read description from === file + file(READ "${subtest}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "${test_name}/${full_test_id} - ${test_desc}") + else() + set(test_name_full "${test_name}/${full_test_id}") + endif() + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + "${test_name}" "${full_test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "${test_name}") + set(test_disabled FALSE) + + # Check if test should be skipped + if(full_test_id IN_LIST skip_list) + set(test_disabled TRUE) + set(test_labels "${test_name};skipped") + endif() + + # Check if test is expected to fail (xfail) + # Note: TAP xfail (TODO) tests still pass (exit 0), they just mark the result as TODO + # So we don't use WILL_FAIL here, just add a label for filtering + if(full_test_id IN_LIST xfail_list) + set(test_labels "${test_name};xfail") + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "${test_labels}" + FIXTURES_REQUIRED "yaml-test-suite-data" + DISABLED ${test_disabled} + ) + endif() + endforeach() + + # Check for 3-digit subtests + file(GLOB subtests_3d "${base_test}/[0-9][0-9][0-9]") + foreach(subtest ${subtests_3d}) + if(EXISTS "${subtest}/===") + get_filename_component(subtest_id "${subtest}" NAME) + set(full_test_id "${test_id}/${subtest_id}") + + # Read description from === file + file(READ "${subtest}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "${test_name}/${full_test_id} - ${test_desc}") + else() + set(test_name_full "${test_name}/${full_test_id}") + endif() + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + "${test_name}" "${full_test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "${test_name}") + set(test_disabled FALSE) + + # Check if test should be skipped + if(full_test_id IN_LIST skip_list) + set(test_disabled TRUE) + set(test_labels "${test_name};skipped") + endif() + + # Check if test is expected to fail (xfail) + # Note: TAP xfail (TODO) tests still pass (exit 0), they just mark the result as TODO + # So we don't use WILL_FAIL here, just add a label for filtering + if(full_test_id IN_LIST xfail_list) + set(test_labels "${test_name};xfail") + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "${test_labels}" + FIXTURES_REQUIRED "yaml-test-suite-data" + DISABLED ${test_disabled} + ) + endif() + endforeach() + endforeach() + endif() +endfunction() + +# Function to add jsontestsuite tests +# Can work either by scanning the json-test-suite-data directory (if it exists) +# or by using a pre-generated list of tests from jsontestsuite-tests.cmake +function(add_jsontestsuite_tests) + set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/test/json-test-suite-data/test_parsing") + set(use_pregenerated_list FALSE) + + # Check if we should use the pre-generated list + if(NOT EXISTS "${test_dir}") + # Try to use pre-generated list if available + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/jsontestsuite-tests.cmake") + message(STATUS "Using pre-generated test list from cmake/jsontestsuite-tests.cmake") + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/jsontestsuite-tests.cmake") + set(use_pregenerated_list TRUE) + else() + # Neither data nor pre-generated list exists, register monolithic fallback + message(STATUS "JSON test suite data not found - registering monolithic jsontestsuite test") + message(STATUS "Run 'ctest -R json-test-suite-setup' then re-run cmake to register individual subtests") + + add_test(NAME jsontestsuite + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test/jsontestsuite.test + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + set_tests_properties(jsontestsuite PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + FIXTURES_REQUIRED json-test-suite-data + ) + return() + endif() + else() + message(STATUS "Registering individual jsontestsuite subtests from filesystem") + endif() + + if(use_pregenerated_list) + # Use pre-generated lists + # Expected to pass (y_*.json) + foreach(test_id ${JSONTESTSUITE_PASS_TESTS}) + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-pass" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + + # Expected to fail (n_*.json) + foreach(test_id ${JSONTESTSUITE_FAIL_TESTS}) + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-fail" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + + # Implementation defined (i_*.json) + foreach(test_id ${JSONTESTSUITE_IMPL_TESTS}) + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-impl" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + else() + # Scan the filesystem (original behavior) + # Expected to pass (y_*.json) + file(GLOB pass_tests "${test_dir}/y_*.json") + foreach(test_file ${pass_tests}) + get_filename_component(test_id "${test_file}" NAME) + + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-pass" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + + # Expected to fail (n_*.json) + file(GLOB fail_tests "${test_dir}/n_*.json") + foreach(test_file ${fail_tests}) + get_filename_component(test_id "${test_file}" NAME) + + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-fail" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + + # Implementation defined (i_*.json) + file(GLOB impl_tests "${test_dir}/i_*.json") + foreach(test_file ${impl_tests}) + get_filename_component(test_id "${test_file}" NAME) + + add_test( + NAME "jsontestsuite/${test_id}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + jsontestsuite "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + set_tests_properties("jsontestsuite/${test_id}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test;JQ=${JQ_EXECUTABLE}" + LABELS "jsontestsuite;jsontestsuite-impl" + FIXTURES_REQUIRED "json-test-suite-data" + ) + endforeach() + endif() +endfunction() + +# Function to add reflection tests +function(add_testreflection_tests) + set(test_dir "${CMAKE_CURRENT_SOURCE_DIR}/test/reflection-data") + + # Skip/xfail lists from testreflection.test + # xfaillist="025PB08/00 025UXE6/00 025VL2J/00" + set(xfail_list "025PB08/00" "025UXE6/00" "025VL2J/00") + + # Probe system characteristics (matching testreflection.test logic) + include(CheckTypeSize) + check_type_size("unsigned int" SIZEOF_UINT) + check_type_size("unsigned short" SIZEOF_USHORT) + check_type_size("unsigned long" SIZEOF_ULONG) + check_type_size("unsigned char" SIZEOF_UCHAR) + check_type_size("long long" SIZEOF_LONGLONG) + + # Calculate bit sizes + math(EXPR intbits "${SIZEOF_UINT} * 8") + math(EXPR shortbits "${SIZEOF_USHORT} * 8") + math(EXPR longbits "${SIZEOF_ULONG} * 8") + math(EXPR charbits "${SIZEOF_UCHAR} * 8") + set(longlongbits 64) # Always 64 bits as per testreflection.test + + # Check if char is signed + include(CheckCSourceRuns) + check_c_source_runs(" + #include + int main() { return (CHAR_MIN < 0) ? 0 : 1; } + " CHAR_IS_SIGNED) + if(CHAR_IS_SIGNED) + set(charsigned "y") + else() + set(charsigned "n") + endif() + + message(STATUS "Reflection test system characteristics:") + message(STATUS " intbits=${intbits} shortbits=${shortbits} longbits=${longbits}") + message(STATUS " charbits=${charbits} longlongbits=${longlongbits} charsigned=${charsigned}") + + # Helper function to check if test should be skipped based on env file + macro(should_skip_test test_path result_var) + set(${result_var} FALSE) + set(env_file "${test_path}/env") + if(EXISTS "${env_file}") + file(READ "${env_file}" env_contents) + string(STRIP "${env_contents}" env_contents) + + # Parse env file for requirements + if(env_contents MATCHES "intbits=([0-9]+)") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${intbits}") + set(${result_var} TRUE) + endif() + endif() + if(env_contents MATCHES "shortbits=([0-9]+)") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${shortbits}") + set(${result_var} TRUE) + endif() + endif() + if(env_contents MATCHES "longbits=([0-9]+)") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${longbits}") + set(${result_var} TRUE) + endif() + endif() + if(env_contents MATCHES "charbits=([0-9]+)") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${charbits}") + set(${result_var} TRUE) + endif() + endif() + if(env_contents MATCHES "longlongbits=([0-9]+)") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${longlongbits}") + set(${result_var} TRUE) + endif() + endif() + if(env_contents MATCHES "charsigned=([yn])") + if(NOT "${CMAKE_MATCH_1}" STREQUAL "${charsigned}") + set(${result_var} TRUE) + endif() + endif() + endif() + endmacro() + + # Scan reflection-data directory for tests + # Note: CMake GLOB doesn't support [A-Z0-9] ranges like shell globs do + # We need to use a wildcard and filter in the loop + file(GLOB base_tests "${test_dir}/*") + + # Filter to only include directories matching the pattern [0-9][0-9][0-9][A-Z][A-Z0-9][A-Z0-9][A-Z0-9] + set(filtered_base_tests) + foreach(test_candidate ${base_tests}) + if(IS_DIRECTORY "${test_candidate}") + get_filename_component(test_name "${test_candidate}" NAME) + # Check if the name matches the pattern: 3 digits followed by 4 alphanumeric chars (starting with letter) + if(test_name MATCHES "^[0-9][0-9][0-9][A-Z][A-Z0-9][A-Z0-9][A-Z0-9]$") + list(APPEND filtered_base_tests "${test_candidate}") + endif() + endif() + endforeach() + set(base_tests ${filtered_base_tests}) + + foreach(base_test ${base_tests}) + get_filename_component(test_id "${base_test}" NAME) + + # Check for main test (base directory with === file) + if(EXISTS "${base_test}/===") + # Read description from === file + file(READ "${base_test}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "testreflection/${test_id} - ${test_desc}") + else() + set(test_name_full "testreflection/${test_id}") + endif() + + # Check if test should be skipped based on environment + should_skip_test("${base_test}" test_should_skip) + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + testreflection "${test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "testreflection") + set(test_disabled FALSE) + set(test_will_fail FALSE) + + # Check if test should be skipped + if(test_should_skip) + set(test_disabled TRUE) + set(test_labels "testreflection;skipped") + endif() + + # Check if test is expected to fail (xfail) + if(test_id IN_LIST xfail_list) + set(test_labels "testreflection;xfail") + set(test_will_fail TRUE) + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test" + LABELS "${test_labels}" + DISABLED ${test_disabled} + WILL_FAIL ${test_will_fail} + ) + endif() + + # Check for 2-digit subtests + file(GLOB subtests_2d "${base_test}/[0-9][0-9]") + foreach(subtest ${subtests_2d}) + if(EXISTS "${subtest}/===") + get_filename_component(subtest_id "${subtest}" NAME) + set(full_test_id "${test_id}/${subtest_id}") + + # Read description from === file + file(READ "${subtest}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "testreflection/${full_test_id} - ${test_desc}") + else() + set(test_name_full "testreflection/${full_test_id}") + endif() + + # Check if test should be skipped based on environment + should_skip_test("${subtest}" test_should_skip) + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + testreflection "${full_test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "testreflection") + set(test_disabled FALSE) + set(test_will_fail FALSE) + + # Check if test should be skipped + if(test_should_skip) + set(test_disabled TRUE) + set(test_labels "testreflection;skipped") + endif() + + # Check if test is expected to fail (xfail) + if(full_test_id IN_LIST xfail_list) + set(test_labels "testreflection;xfail") + set(test_will_fail TRUE) + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test" + LABELS "${test_labels}" + DISABLED ${test_disabled} + WILL_FAIL ${test_will_fail} + ) + endif() + endforeach() + + # Check for 3-digit subtests + file(GLOB subtests_3d "${base_test}/[0-9][0-9][0-9]") + foreach(subtest ${subtests_3d}) + if(EXISTS "${subtest}/===") + get_filename_component(subtest_id "${subtest}" NAME) + set(full_test_id "${test_id}/${subtest_id}") + + # Read description from === file + file(READ "${subtest}/===" test_desc) + string(STRIP "${test_desc}" test_desc) + + # Build test name with description if available + if(test_desc) + set(test_name_full "testreflection/${full_test_id} - ${test_desc}") + else() + set(test_name_full "testreflection/${full_test_id}") + endif() + + # Check if test should be skipped based on environment + should_skip_test("${subtest}" test_should_skip) + + add_test( + NAME "${test_name_full}" + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/cmake/run-single-tap-test.sh + testreflection "${full_test_id}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test + ) + + # Set basic properties + set(test_labels "testreflection") + set(test_disabled FALSE) + set(test_will_fail FALSE) + + # Check if test should be skipped + if(test_should_skip) + set(test_disabled TRUE) + set(test_labels "testreflection;skipped") + endif() + + # Check if test is expected to fail (xfail) + if(full_test_id IN_LIST xfail_list) + set(test_labels "testreflection;xfail") + set(test_will_fail TRUE) + endif() + + set_tests_properties("${test_name_full}" PROPERTIES + ENVIRONMENT "TOP_SRCDIR=${CMAKE_CURRENT_SOURCE_DIR};TOP_BUILDDIR=${CMAKE_CURRENT_BINARY_DIR};SRCDIR=${CMAKE_CURRENT_SOURCE_DIR}/test;BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/test" + LABELS "${test_labels}" + DISABLED ${test_disabled} + WILL_FAIL ${test_will_fail} + ) + endif() + endforeach() + endforeach() +endfunction() diff --git a/cmake/config.h.in b/cmake/config.h.in new file mode 100644 index 0000000..f287269 --- /dev/null +++ b/cmake/config.h.in @@ -0,0 +1,84 @@ +/* config.h - Generated by CMake */ + +#ifndef CONFIG_H +#define CONFIG_H + +/* Define to 1 if you have byteswap.h */ +#cmakedefine HAVE_BYTESWAP_H + +/* Define to 1 if you have __builtin_bswap16 */ +#cmakedefine HAVE___BUILTIN_BSWAP16 + +/* Define to 1 if you have __builtin_bswap32 */ +#cmakedefine HAVE___BUILTIN_BSWAP32 + +/* Define to 1 if you have __builtin_bswap64 */ +#cmakedefine HAVE___BUILTIN_BSWAP64 + +/* Define to 1 if you have qsort_r available */ +#cmakedefine01 HAVE_QSORT_R + +/* Define to 1 if you have mremap available */ +#cmakedefine01 HAVE_MREMAP + +/* Define to 1 if you have environ declared */ +#cmakedefine HAVE_DECL_ENVIRON + +/* Define to 1 if you have libyaml available */ +#cmakedefine HAVE_LIBYAML + +/* Define to 1 if you have check available */ +#cmakedefine HAVE_CHECK + +/* Define to 1 if you have a compatible version of check available */ +#cmakedefine HAVE_COMPATIBLE_CHECK + +/* Define to 1 if static linking is available */ +#cmakedefine HAVE_STATIC + +/* Define to 1 if tools are static */ +#cmakedefine HAVE_STATIC_TOOLS + +/* Define to 1 if ASAN is enabled */ +#cmakedefine HAVE_ASAN + +/* Define to 1 if developer mode is enabled */ +#cmakedefine HAVE_DEVMODE + +/* Define to 1 if PORTABLE_TARGET is enabled */ +#cmakedefine HAVE_PORTABLE_TARGET + +/* SSE2 target support */ +#cmakedefine01 TARGET_HAS_SSE2 + +/* SSE41 target support */ +#cmakedefine01 TARGET_HAS_SSE41 + +/* AVX2 target support */ +#cmakedefine01 TARGET_HAS_AVX2 + +/* AVX512 target support */ +#cmakedefine01 TARGET_HAS_AVX512 + +/* NEON target support */ +#cmakedefine01 TARGET_HAS_NEON + +/* Define to 1 if you have git available */ +#cmakedefine HAVE_GIT + +/* Define to 1 if you have jq available */ +#cmakedefine HAVE_JQ + +/* Define to 1 if you have libclang available */ +#cmakedefine01 HAVE_LIBCLANG + +/* Define to 1 if Clang (or AppleClang) Blocks are available */ +#cmakedefine HAVE_CLANG_BLOCKS @HAVE_CLANG_BLOCKS@ + +/* Define to 1 if GCC heap trampolines are enabled */ +#cmakedefine01 HAVE_HEAP_TRAMPOLINES + +/* Version string */ +#define VERSION "@PROJECT_VERSION@" + +#endif /* CONFIG_H */ diff --git a/cmake/generate-jsontestsuite-list.sh b/cmake/generate-jsontestsuite-list.sh new file mode 100755 index 0000000..d700f0b --- /dev/null +++ b/cmake/generate-jsontestsuite-list.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash + +# Script to generate a CMake fragment listing all tests in the JSON test suite +# This allows registering tests even before the test data is downloaded + +set -e + +# Default values +JSONTESTSUITEURL="${JSONTESTSUITEURL:-https://github.com/nst/JSONTestSuite}" +JSONTESTSUITECHECKOUT="${JSONTESTSUITECHECKOUT:-d64aefb55228d9584d3e5b2433f720ea8fd00c82}" +OUTPUT_FILE="${1:-jsontestsuite-tests.cmake}" + +usage() { + cat << EOF +Usage: $0 [OUTPUT_FILE] + +Generate a CMake fragment listing all tests in the JSON test suite. + +Arguments: + OUTPUT_FILE Output file path (default: jsontestsuite-tests.cmake) + +Environment variables: + JSONTESTSUITEURL URL of the test suite repository + JSONTESTSUITECHECKOUT Git commit/tag to use + +Example: + $0 cmake/jsontestsuite-tests.cmake +EOF + exit 1 +} + +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage +fi + +echo "Generating jsontestsuite list from ${JSONTESTSUITEURL} @ ${JSONTESTSUITECHECKOUT}" + +# Create a temporary directory +TMPDIR=$(mktemp -d) +trap "rm -rf $TMPDIR" EXIT + +# Check if we have a local json-test-suite-data directory to use +LOCAL_DATA="" +for dir in "build/test/json-test-suite-data/test_parsing" "json-test-suite-data/test_parsing" "../build/test/json-test-suite-data/test_parsing"; do + if [ -d "$dir" ]; then + LOCAL_DATA="$dir" + echo "Using local test data from: $LOCAL_DATA" + break + fi +done + +if [ -n "$LOCAL_DATA" ]; then + # Use local data + find "$LOCAL_DATA" -maxdepth 1 -type f -name 'y_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/pass_tests.txt" + find "$LOCAL_DATA" -maxdepth 1 -type f -name 'n_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/fail_tests.txt" + find "$LOCAL_DATA" -maxdepth 1 -type f -name 'i_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/impl_tests.txt" +else + # Clone from git + echo "No local data found, cloning from git repository..." + cd "$TMPDIR" + git clone --depth 1 --filter=blob:none --sparse "${JSONTESTSUITEURL}" repo + cd repo + git checkout "${JSONTESTSUITECHECKOUT}" + git sparse-checkout set test_parsing + git sparse-checkout disable + + # List test files + find test_parsing -maxdepth 1 -type f -name 'y_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/pass_tests.txt" + find test_parsing -maxdepth 1 -type f -name 'n_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/fail_tests.txt" + find test_parsing -maxdepth 1 -type f -name 'i_*.json' | \ + xargs -n1 basename | sort > "$TMPDIR/impl_tests.txt" +fi + +# Read test counts +PASS_COUNT=$(wc -l < "$TMPDIR/pass_tests.txt") +FAIL_COUNT=$(wc -l < "$TMPDIR/fail_tests.txt") +IMPL_COUNT=$(wc -l < "$TMPDIR/impl_tests.txt") +TOTAL_COUNT=$((PASS_COUNT + FAIL_COUNT + IMPL_COUNT)) + +echo "Found $PASS_COUNT tests expected to pass (y_*.json)" +echo "Found $FAIL_COUNT tests expected to fail (n_*.json)" +echo "Found $IMPL_COUNT implementation-defined tests (i_*.json)" +echo "Total: $TOTAL_COUNT tests" + +# Generate CMake fragment +cat > "$OUTPUT_FILE" << 'EOF' +# Auto-generated file - do not edit manually +# Generated by cmake/generate-jsontestsuite-list.sh +# +# This file contains lists of tests from the JSON test suite +# It allows registering individual tests without requiring the test data +# to be downloaded first. + +EOF + +echo "# Test suite checkout: ${JSONTESTSUITECHECKOUT}" >> "$OUTPUT_FILE" +echo "# Generated on: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> "$OUTPUT_FILE" +echo "# Pass tests: ${PASS_COUNT}" >> "$OUTPUT_FILE" +echo "# Fail tests: ${FAIL_COUNT}" >> "$OUTPUT_FILE" +echo "# Impl tests: ${IMPL_COUNT}" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +# Write pass tests +echo "# Tests expected to pass (y_*.json)" >> "$OUTPUT_FILE" +echo "set(JSONTESTSUITE_PASS_TESTS" >> "$OUTPUT_FILE" +while IFS= read -r test; do + echo " \"$test\"" >> "$OUTPUT_FILE" +done < "$TMPDIR/pass_tests.txt" +echo ")" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +# Write fail tests +echo "# Tests expected to fail (n_*.json)" >> "$OUTPUT_FILE" +echo "set(JSONTESTSUITE_FAIL_TESTS" >> "$OUTPUT_FILE" +while IFS= read -r test; do + echo " \"$test\"" >> "$OUTPUT_FILE" +done < "$TMPDIR/fail_tests.txt" +echo ")" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +# Write impl tests +echo "# Implementation-defined tests (i_*.json)" >> "$OUTPUT_FILE" +echo "set(JSONTESTSUITE_IMPL_TESTS" >> "$OUTPUT_FILE" +while IFS= read -r test; do + echo " \"$test\"" >> "$OUTPUT_FILE" +done < "$TMPDIR/impl_tests.txt" +echo ")" >> "$OUTPUT_FILE" + +echo "Generated $OUTPUT_FILE successfully!" +echo "Total tests: $TOTAL_COUNT" diff --git a/cmake/generate-testsuite-list.sh b/cmake/generate-testsuite-list.sh new file mode 100755 index 0000000..91eed21 --- /dev/null +++ b/cmake/generate-testsuite-list.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash + +# Script to generate a CMake fragment listing all tests in the YAML test suite +# This allows registering tests even before the test data is downloaded + +set -e + +# Default values +TESTSUITEURL="${TESTSUITEURL:-https://github.com/yaml/yaml-test-suite}" +TESTSUITECHECKOUT="${TESTSUITECHECKOUT:-6e6c296ae9c9d2d5c4134b4b64d01b29ac19ff6f}" +OUTPUT_FILE="${1:-testsuite-tests.cmake}" + +usage() { + cat << EOF +Usage: $0 [OUTPUT_FILE] + +Generate a CMake fragment listing all tests in the YAML test suite. + +Arguments: + OUTPUT_FILE Output file path (default: testsuite-tests.cmake) + +Environment variables: + TESTSUITEURL URL of the test suite repository + TESTSUITECHECKOUT Git commit/tag to use + +Example: + $0 cmake/testsuite-tests.cmake +EOF + exit 1 +} + +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage +fi + +echo "Generating testsuite list from ${TESTSUITEURL} @ ${TESTSUITECHECKOUT}" + +# Create a temporary directory +TMPDIR=$(mktemp -d) +trap "rm -rf $TMPDIR" EXIT + +# Check if we have a local test-suite-data directory to use +LOCAL_DATA="" +for dir in "build/test/test-suite-data" "test-suite-data" "../build/test/test-suite-data"; do + if [ -d "$dir" ]; then + LOCAL_DATA="$dir" + echo "Using local test data from: $LOCAL_DATA" + break + fi +done + +if [ -n "$LOCAL_DATA" ]; then + # Use local data + find "$LOCAL_DATA" -maxdepth 1 -type d -name '[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]' | \ + xargs -n1 basename | sort > "$TMPDIR/base_tests.txt" + + # For each base test, check for subtests + > "$TMPDIR/all_tests.txt" + while IFS= read -r base; do + # Check for base test + if [ -f "$LOCAL_DATA/$base/===" ]; then + echo "$base" >> "$TMPDIR/all_tests.txt" + fi + + # Check for 2-digit subtests + for subtest in "$LOCAL_DATA/$base"/[0-9][0-9]; do + if [ -d "$subtest" ] && [ -f "$subtest/===" ]; then + echo "$base/$(basename $subtest)" >> "$TMPDIR/all_tests.txt" + fi + done + + # Check for 3-digit subtests + for subtest in "$LOCAL_DATA/$base"/[0-9][0-9][0-9]; do + if [ -d "$subtest" ] && [ -f "$subtest/===" ]; then + echo "$base/$(basename $subtest)" >> "$TMPDIR/all_tests.txt" + fi + done + done < "$TMPDIR/base_tests.txt" +else + # Clone from git + echo "No local data found, cloning from git repository..." + cd "$TMPDIR" + git clone --depth 1 --filter=blob:none --sparse "${TESTSUITEURL}" repo + cd repo + git checkout "${TESTSUITECHECKOUT}" + git sparse-checkout disable + + # List all base test directories + find . -maxdepth 1 -type d -name '[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]' | \ + sed 's|^\./||' | sort > "$TMPDIR/base_tests.txt" + + # For each base test, check for subtests + > "$TMPDIR/all_tests.txt" + while IFS= read -r base; do + # Check for base test + if [ -f "$base/===" ]; then + echo "$base" >> "$TMPDIR/all_tests.txt" + fi + + # Check for 2-digit subtests + for subtest in "$base"/[0-9][0-9]; do + if [ -d "$subtest" ] && [ -f "$subtest/===" ]; then + echo "$base/$(basename $subtest)" >> "$TMPDIR/all_tests.txt" + fi + done + + # Check for 3-digit subtests + for subtest in "$base"/[0-9][0-9][0-9]; do + if [ -d "$subtest" ] && [ -f "$subtest/===" ]; then + echo "$base/$(basename $subtest)" >> "$TMPDIR/all_tests.txt" + fi + done + done < "$TMPDIR/base_tests.txt" +fi + +# Read test counts +BASE_COUNT=$(wc -l < "$TMPDIR/base_tests.txt") +if [ -f "$TMPDIR/all_tests.txt" ]; then + ALL_COUNT=$(wc -l < "$TMPDIR/all_tests.txt") +else + ALL_COUNT=0 +fi + +echo "Found $BASE_COUNT base tests" +echo "Found $ALL_COUNT total tests (including subtests)" + +# Generate CMake fragment +cat > "$OUTPUT_FILE" << 'EOF' +# Auto-generated file - do not edit manually +# Generated by cmake/generate-testsuite-list.sh +# +# This file contains lists of tests from the YAML test suite +# It allows registering individual tests without requiring the test data +# to be downloaded first. + +EOF + +echo "# Test suite checkout: ${TESTSUITECHECKOUT}" >> "$OUTPUT_FILE" +echo "# Generated on: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> "$OUTPUT_FILE" +echo "# Base tests: ${BASE_COUNT}" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +# Write base test list +echo "# List of base test IDs" >> "$OUTPUT_FILE" +echo "set(TESTSUITE_BASE_TESTS" >> "$OUTPUT_FILE" +while IFS= read -r test; do + echo " $test" >> "$OUTPUT_FILE" +done < "$TMPDIR/base_tests.txt" +echo ")" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +if [ -f "$TMPDIR/all_tests.txt" ]; then + echo "# List of all tests (including subtests)" >> "$OUTPUT_FILE" + echo "# Format: BASE or BASE/SUBTEST" >> "$OUTPUT_FILE" + echo "set(TESTSUITE_ALL_TESTS" >> "$OUTPUT_FILE" + while IFS= read -r test; do + echo " \"$test\"" >> "$OUTPUT_FILE" + done < "$TMPDIR/all_tests.txt" + echo ")" >> "$OUTPUT_FILE" +fi + +echo "Generated $OUTPUT_FILE successfully!" +echo "Total base tests: $BASE_COUNT" +[ $ALL_COUNT -gt 0 ] && echo "Total tests (with subtests): $ALL_COUNT" diff --git a/cmake/jsontestsuite-tests.cmake b/cmake/jsontestsuite-tests.cmake new file mode 100644 index 0000000..3737900 --- /dev/null +++ b/cmake/jsontestsuite-tests.cmake @@ -0,0 +1,342 @@ +# Auto-generated file - do not edit manually +# Generated by cmake/generate-jsontestsuite-list.sh +# +# This file contains lists of tests from the JSON test suite +# It allows registering individual tests without requiring the test data +# to be downloaded first. + +# Test suite checkout: d64aefb55228d9584d3e5b2433f720ea8fd00c82 +# Generated on: 2025-10-24 08:05:59 UTC +# Pass tests: 95 +# Fail tests: 188 +# Impl tests: 35 + +# Tests expected to pass (y_*.json) +set(JSONTESTSUITE_PASS_TESTS + "y_array_arraysWithSpaces.json" + "y_array_empty.json" + "y_array_empty-string.json" + "y_array_ending_with_newline.json" + "y_array_false.json" + "y_array_heterogeneous.json" + "y_array_null.json" + "y_array_with_1_and_newline.json" + "y_array_with_leading_space.json" + "y_array_with_several_null.json" + "y_array_with_trailing_space.json" + "y_number_0e+1.json" + "y_number_0e1.json" + "y_number_after_space.json" + "y_number_double_close_to_zero.json" + "y_number_int_with_exp.json" + "y_number.json" + "y_number_minus_zero.json" + "y_number_negative_int.json" + "y_number_negative_one.json" + "y_number_negative_zero.json" + "y_number_real_capital_e.json" + "y_number_real_capital_e_neg_exp.json" + "y_number_real_capital_e_pos_exp.json" + "y_number_real_exponent.json" + "y_number_real_fraction_exponent.json" + "y_number_real_neg_exp.json" + "y_number_real_pos_exponent.json" + "y_number_simple_int.json" + "y_number_simple_real.json" + "y_object_basic.json" + "y_object_duplicated_key_and_value.json" + "y_object_duplicated_key.json" + "y_object_empty.json" + "y_object_empty_key.json" + "y_object_escaped_null_in_key.json" + "y_object_extreme_numbers.json" + "y_object.json" + "y_object_long_strings.json" + "y_object_simple.json" + "y_object_string_unicode.json" + "y_object_with_newlines.json" + "y_string_1_2_3_bytes_UTF-8_sequences.json" + "y_string_accepted_surrogate_pair.json" + "y_string_accepted_surrogate_pairs.json" + "y_string_allowed_escapes.json" + "y_string_backslash_and_u_escaped_zero.json" + "y_string_backslash_doublequotes.json" + "y_string_comments.json" + "y_string_double_escape_a.json" + "y_string_double_escape_n.json" + "y_string_escaped_control_character.json" + "y_string_escaped_noncharacter.json" + "y_string_in_array.json" + "y_string_in_array_with_leading_space.json" + "y_string_last_surrogates_1_and_2.json" + "y_string_nbsp_uescaped.json" + "y_string_nonCharacterInUTF-8_U+10FFFF.json" + "y_string_nonCharacterInUTF-8_U+FFFF.json" + "y_string_null_escape.json" + "y_string_one-byte-utf-8.json" + "y_string_pi.json" + "y_string_reservedCharacterInUTF-8_U+1BFFF.json" + "y_string_simple_ascii.json" + "y_string_space.json" + "y_string_surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json" + "y_string_three-byte-utf-8.json" + "y_string_two-byte-utf-8.json" + "y_string_u+2028_line_sep.json" + "y_string_u+2029_par_sep.json" + "y_string_uescaped_newline.json" + "y_string_uEscape.json" + "y_string_unescaped_char_delete.json" + "y_string_unicode_2.json" + "y_string_unicodeEscapedBackslash.json" + "y_string_unicode_escaped_double_quote.json" + "y_string_unicode.json" + "y_string_unicode_U+10FFFE_nonchar.json" + "y_string_unicode_U+1FFFE_nonchar.json" + "y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json" + "y_string_unicode_U+2064_invisible_plus.json" + "y_string_unicode_U+FDD0_nonchar.json" + "y_string_unicode_U+FFFE_nonchar.json" + "y_string_utf8.json" + "y_string_with_del_character.json" + "y_structure_lonely_false.json" + "y_structure_lonely_int.json" + "y_structure_lonely_negative_real.json" + "y_structure_lonely_null.json" + "y_structure_lonely_string.json" + "y_structure_lonely_true.json" + "y_structure_string_empty.json" + "y_structure_trailing_newline.json" + "y_structure_true_in_array.json" + "y_structure_whitespace_array.json" +) + +# Tests expected to fail (n_*.json) +set(JSONTESTSUITE_FAIL_TESTS + "n_array_1_true_without_comma.json" + "n_array_a_invalid_utf8.json" + "n_array_colon_instead_of_comma.json" + "n_array_comma_after_close.json" + "n_array_comma_and_number.json" + "n_array_double_comma.json" + "n_array_double_extra_comma.json" + "n_array_extra_close.json" + "n_array_extra_comma.json" + "n_array_incomplete_invalid_value.json" + "n_array_incomplete.json" + "n_array_inner_array_no_comma.json" + "n_array_invalid_utf8.json" + "n_array_items_separated_by_semicolon.json" + "n_array_just_comma.json" + "n_array_just_minus.json" + "n_array_missing_value.json" + "n_array_newlines_unclosed.json" + "n_array_number_and_comma.json" + "n_array_number_and_several_commas.json" + "n_array_spaces_vertical_tab_formfeed.json" + "n_array_star_inside.json" + "n_array_unclosed.json" + "n_array_unclosed_trailing_comma.json" + "n_array_unclosed_with_new_lines.json" + "n_array_unclosed_with_object_inside.json" + "n_incomplete_false.json" + "n_incomplete_null.json" + "n_incomplete_true.json" + "n_multidigit_number_then_00.json" + "n_number_0.1.2.json" + "n_number_-01.json" + "n_number_0.3e+.json" + "n_number_0.3e.json" + "n_number_0_capital_E+.json" + "n_number_0_capital_E.json" + "n_number_0.e1.json" + "n_number_0e+.json" + "n_number_0e.json" + "n_number_1_000.json" + "n_number_1.0e+.json" + "n_number_1.0e-.json" + "n_number_1.0e.json" + "n_number_-1.0..json" + "n_number_1eE2.json" + "n_number_+1.json" + "n_number_.-1.json" + "n_number_2.e+3.json" + "n_number_2.e-3.json" + "n_number_2.e3.json" + "n_number_.2e-3.json" + "n_number_-2..json" + "n_number_9.e+.json" + "n_number_expression.json" + "n_number_hex_1_digit.json" + "n_number_hex_2_digits.json" + "n_number_infinity.json" + "n_number_+Inf.json" + "n_number_Inf.json" + "n_number_invalid+-.json" + "n_number_invalid-negative-real.json" + "n_number_invalid-utf-8-in-bigger-int.json" + "n_number_invalid-utf-8-in-exponent.json" + "n_number_invalid-utf-8-in-int.json" + "n_number_++.json" + "n_number_minus_infinity.json" + "n_number_minus_sign_with_trailing_garbage.json" + "n_number_minus_space_1.json" + "n_number_-NaN.json" + "n_number_NaN.json" + "n_number_neg_int_starting_with_zero.json" + "n_number_neg_real_without_int_part.json" + "n_number_neg_with_garbage_at_end.json" + "n_number_real_garbage_after_e.json" + "n_number_real_with_invalid_utf8_after_e.json" + "n_number_real_without_fractional_part.json" + "n_number_starting_with_dot.json" + "n_number_U+FF11_fullwidth_digit_one.json" + "n_number_with_alpha_char.json" + "n_number_with_alpha.json" + "n_number_with_leading_zero.json" + "n_object_bad_value.json" + "n_object_bracket_key.json" + "n_object_comma_instead_of_colon.json" + "n_object_double_colon.json" + "n_object_emoji.json" + "n_object_garbage_at_end.json" + "n_object_key_with_single_quotes.json" + "n_object_lone_continuation_byte_in_key_and_trailing_comma.json" + "n_object_missing_colon.json" + "n_object_missing_key.json" + "n_object_missing_semicolon.json" + "n_object_missing_value.json" + "n_object_no-colon.json" + "n_object_non_string_key_but_huge_number_instead.json" + "n_object_non_string_key.json" + "n_object_repeated_null_null.json" + "n_object_several_trailing_commas.json" + "n_object_single_quote.json" + "n_object_trailing_comma.json" + "n_object_trailing_comment.json" + "n_object_trailing_comment_open.json" + "n_object_trailing_comment_slash_open_incomplete.json" + "n_object_trailing_comment_slash_open.json" + "n_object_two_commas_in_a_row.json" + "n_object_unquoted_key.json" + "n_object_unterminated-value.json" + "n_object_with_single_string.json" + "n_object_with_trailing_garbage.json" + "n_single_space.json" + "n_string_1_surrogate_then_escape.json" + "n_string_1_surrogate_then_escape_u1.json" + "n_string_1_surrogate_then_escape_u1x.json" + "n_string_1_surrogate_then_escape_u.json" + "n_string_accentuated_char_no_quotes.json" + "n_string_backslash_00.json" + "n_string_escaped_backslash_bad.json" + "n_string_escaped_ctrl_char_tab.json" + "n_string_escaped_emoji.json" + "n_string_escape_x.json" + "n_string_incomplete_escaped_character.json" + "n_string_incomplete_escape.json" + "n_string_incomplete_surrogate_escape_invalid.json" + "n_string_incomplete_surrogate.json" + "n_string_invalid_backslash_esc.json" + "n_string_invalid_unicode_escape.json" + "n_string_invalid_utf8_after_escape.json" + "n_string_invalid-utf-8-in-escape.json" + "n_string_leading_uescaped_thinspace.json" + "n_string_no_quotes_with_bad_escape.json" + "n_string_single_doublequote.json" + "n_string_single_quote.json" + "n_string_single_string_no_double_quotes.json" + "n_string_start_escape_unclosed.json" + "n_string_unescaped_ctrl_char.json" + "n_string_unescaped_newline.json" + "n_string_unescaped_tab.json" + "n_string_unicode_CapitalU.json" + "n_string_with_trailing_garbage.json" + "n_structure_100000_opening_arrays.json" + "n_structure_angle_bracket_..json" + "n_structure_angle_bracket_null.json" + "n_structure_array_trailing_garbage.json" + "n_structure_array_with_extra_array_close.json" + "n_structure_array_with_unclosed_string.json" + "n_structure_ascii-unicode-identifier.json" + "n_structure_capitalized_True.json" + "n_structure_close_unopened_array.json" + "n_structure_comma_instead_of_closing_brace.json" + "n_structure_double_array.json" + "n_structure_end_array.json" + "n_structure_incomplete_UTF8_BOM.json" + "n_structure_lone-invalid-utf-8.json" + "n_structure_lone-open-bracket.json" + "n_structure_no_data.json" + "n_structure_null-byte-outside-string.json" + "n_structure_number_with_trailing_garbage.json" + "n_structure_object_followed_by_closing_object.json" + "n_structure_object_unclosed_no_value.json" + "n_structure_object_with_comment.json" + "n_structure_object_with_trailing_garbage.json" + "n_structure_open_array_apostrophe.json" + "n_structure_open_array_comma.json" + "n_structure_open_array_object.json" + "n_structure_open_array_open_object.json" + "n_structure_open_array_open_string.json" + "n_structure_open_array_string.json" + "n_structure_open_object_close_array.json" + "n_structure_open_object_comma.json" + "n_structure_open_object.json" + "n_structure_open_object_open_array.json" + "n_structure_open_object_open_string.json" + "n_structure_open_object_string_with_apostrophes.json" + "n_structure_open_open.json" + "n_structure_single_eacute.json" + "n_structure_single_star.json" + "n_structure_trailing_#.json" + "n_structure_U+2060_word_joined.json" + "n_structure_uescaped_LF_before_string.json" + "n_structure_unclosed_array.json" + "n_structure_unclosed_array_partial_null.json" + "n_structure_unclosed_array_unfinished_false.json" + "n_structure_unclosed_array_unfinished_true.json" + "n_structure_unclosed_object.json" + "n_structure_unicode-identifier.json" + "n_structure_UTF8_BOM_no_data.json" + "n_structure_whitespace_formfeed.json" + "n_structure_whitespace_U+2060_word_joiner.json" +) + +# Implementation-defined tests (i_*.json) +set(JSONTESTSUITE_IMPL_TESTS + "i_number_double_huge_neg_exp.json" + "i_number_huge_exp.json" + "i_number_neg_int_huge_exp.json" + "i_number_pos_double_huge_exp.json" + "i_number_real_neg_overflow.json" + "i_number_real_pos_overflow.json" + "i_number_real_underflow.json" + "i_number_too_big_neg_int.json" + "i_number_too_big_pos_int.json" + "i_number_very_big_negative_int.json" + "i_object_key_lone_2nd_surrogate.json" + "i_string_1st_surrogate_but_2nd_missing.json" + "i_string_1st_valid_surrogate_2nd_invalid.json" + "i_string_incomplete_surrogate_and_escape_valid.json" + "i_string_incomplete_surrogate_pair.json" + "i_string_incomplete_surrogates_escape_valid.json" + "i_string_invalid_lonely_surrogate.json" + "i_string_invalid_surrogate.json" + "i_string_invalid_utf-8.json" + "i_string_inverted_surrogates_U+1D11E.json" + "i_string_iso_latin_1.json" + "i_string_lone_second_surrogate.json" + "i_string_lone_utf8_continuation_byte.json" + "i_string_not_in_unicode_range.json" + "i_string_overlong_sequence_2_bytes.json" + "i_string_overlong_sequence_6_bytes.json" + "i_string_overlong_sequence_6_bytes_null.json" + "i_string_truncated-utf-8.json" + "i_string_utf16BE_no_BOM.json" + "i_string_utf16LE_no_BOM.json" + "i_string_UTF-16LE_with_BOM.json" + "i_string_UTF-8_invalid_sequence.json" + "i_string_UTF8_surrogate_U+D800.json" + "i_structure_500_nested_arrays.json" + "i_structure_UTF-8_BOM_empty_object.json" +) diff --git a/cmake/libfyaml-config.cmake.in b/cmake/libfyaml-config.cmake.in new file mode 100644 index 0000000..d67173a --- /dev/null +++ b/cmake/libfyaml-config.cmake.in @@ -0,0 +1,54 @@ +@PACKAGE_INIT@ + +# libfyaml CMake package configuration file +# This file provides libfyaml imported targets and dependencies + +include(CMakeFindDependencyMacro) + +# Provide version information +set(libfyaml_VERSION "@PROJECT_VERSION@") +set(libfyaml_VERSION_MAJOR "@VERSION_MAJOR@") +set(libfyaml_VERSION_MINOR "@VERSION_MINOR@") +set(libfyaml_VERSION_PATCH "@VERSION_PATCH@") + +# Provide feature flags (what was enabled when libfyaml was built) +set(libfyaml_HAS_LIBCLANG "@HAVE_LIBCLANG@") +set(libfyaml_HAS_LIBYAML "@HAVE_LIBYAML@") + +# Required dependency: Threads +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE) +find_dependency(Threads REQUIRED) + +# Optional dependency: libclang (only if libfyaml was built with reflection support) +if(libfyaml_HAS_LIBCLANG) + # Try CMake config first + find_dependency(LLVM QUIET CONFIG) + if(LLVM_FOUND) + # Help find Clang in the same LLVM installation + set(Clang_DIR "${LLVM_INSTALL_PREFIX}/lib/cmake/clang") + endif() + find_dependency(Clang QUIET CONFIG) + + # If CMake config not found, try pkg-config as fallback + if(NOT LLVM_FOUND OR NOT Clang_FOUND) + # User must provide libclang manually if needed + message(STATUS "libfyaml was built with libclang support, but libclang not found via CMake config") + message(STATUS " Set CMAKE_PREFIX_PATH or LLVM_DIR to locate LLVM/Clang") + endif() +endif() + +# Import libfyaml targets (defines libfyaml::fyaml) +include(${CMAKE_CURRENT_LIST_DIR}/libfyaml-targets.cmake) + +# Provide consistent naming: libfyaml::libfyaml as main target +if(NOT TARGET libfyaml::libfyaml AND TARGET libfyaml::fyaml) + add_library(libfyaml::libfyaml ALIAS libfyaml::fyaml) +endif() + +# Backward compatibility: provide non-namespaced target +if(NOT TARGET fyaml AND TARGET libfyaml::fyaml) + add_library(fyaml ALIAS libfyaml::fyaml) +endif() + +check_required_components(libfyaml) diff --git a/cmake/libfyaml.pc.in b/cmake/libfyaml.pc.in new file mode 100644 index 0000000..e57044c --- /dev/null +++ b/cmake/libfyaml.pc.in @@ -0,0 +1,10 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@ +includedir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@ + +Name: libfyaml +Description: Fancy YAML 1.3 parser library +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -lfyaml -lpthread +Cflags: -I${includedir} diff --git a/cmake/run-single-tap-test.sh b/cmake/run-single-tap-test.sh new file mode 100755 index 0000000..5382f56 --- /dev/null +++ b/cmake/run-single-tap-test.sh @@ -0,0 +1,314 @@ +#!/usr/bin/env bash + +# Wrapper script to run a single TAP subtest +# Usage: run-single-tap-test.sh + +test_suite="$1" +test_id="$2" + +case "$test_suite" in + testerrors) + dir="${SRCDIR}/test-errors/${test_id}" + desctxt=$(cat 2>/dev/null "$dir/===") + + t=$(mktemp) + res="not ok" + + pass_yaml=0 + ${TOP_BUILDDIR}/src/fy-tool --dump -r --no-streaming "$dir/in.yaml" >"$t" 2>&1 + if [ $? -eq 0 ]; then + pass_yaml=1 + fi + + errmsg=$(cat "$t" | head -n1 | sed -e 's/^[^:]*//') + echo "errmsg: $errmsg" >&2 + + # Replace with error message + echo "$errmsg" >"$t" + + # All error tests are expected to fail + if [ "$pass_yaml" == "0" ]; then + res="ok" + + # diff is pointless under valgrind + if [ "x$USE_VALGRIND" == "x" ]; then + diff_err=0 + diff -u "$dir/test.error" "$t" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + fi + else + res="not ok" + fi + + rm -f "$t" + + echo "$res 1 $test_id - $desctxt" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + testemitter) + f="${SRCDIR}/emitter-examples/${test_id}" + + t1=$(mktemp) + t2=$(mktemp) + + res="not ok" + + pass_parse=0 + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers "$f" >"$t1" + if [ $? -eq 0 ]; then + ${TOP_BUILDDIR}/src/fy-tool --dump "$f" | \ + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t2" + if [ $? -eq 0 ]; then + pass_parse=1 + fi + fi + + if [ "$pass_parse" == "1" ]; then + diff -u "$t1" "$t2" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + else + res="not ok" + fi + + rm -f "$t1" "$t2" + + echo "$res 1 - $test_id" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + testemitter-streaming) + f="${SRCDIR}/emitter-examples/${test_id}" + + t1=$(mktemp) + t2=$(mktemp) + + res="not ok" + + pass_parse=0 + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers "$f" >"$t1" + if [ $? -eq 0 ]; then + ${TOP_BUILDDIR}/src/fy-tool --dump --streaming "$f" | \ + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t2" + if [ $? -eq 0 ]; then + pass_parse=1 + fi + fi + + if [ "$pass_parse" == "1" ]; then + diff -u "$t1" "$t2" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + else + res="not ok" + fi + + rm -f "$t1" "$t2" + + echo "$res 1 - $test_id" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + testemitter-restreaming) + f="${SRCDIR}/emitter-examples/${test_id}" + + t1=$(mktemp) + t2=$(mktemp) + + res="not ok" + + pass_parse=0 + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers "$f" >"$t1" + if [ $? -eq 0 ]; then + ${TOP_BUILDDIR}/src/fy-tool --dump --streaming --recreating "$f" | \ + ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t2" + if [ $? -eq 0 ]; then + pass_parse=1 + fi + fi + + if [ "$pass_parse" == "1" ]; then + diff -u "$t1" "$t2" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + else + res="not ok" + fi + + rm -f "$t1" "$t2" + + echo "$res 1 - $test_id" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + testsuite|testsuite-json|testsuite-resolution) + tst="test-suite-data/${test_id}" + desctxt=$(cat 2>/dev/null "$tst/===") + + t=$(mktemp) + + res="not ok" + + pass_yaml=0 + ${TOP_BUILDDIR}/src/fy-tool --testsuite "$tst/in.yaml" >"$t" + if [ $? -eq 0 ]; then + pass_yaml=1 + fi + + if [ -e "$tst/error" ]; then + # Test is expected to fail + if [ $pass_yaml == "0" ]; then + res="ok" + else + res="not ok" + fi + else + # Test is expected to pass + if [ $pass_yaml == "1" ]; then + diff -u "$tst/test.event" "$t" + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + else + res="not ok" + fi + fi + + rm -f "$t" + + echo "$res 1 $test_id - $desctxt" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + testsuite-evstream) + tst="test-suite-data/${test_id}" + desctxt=$(cat 2>/dev/null "$tst/===") + + # Skip tests that are expected to fail + if [ -e "$tst/error" ]; then + # echo "ok 1 $test_id - $desctxt # SKIP expected to fail" + echo "skip 1 $test_id - $desctxt # SKIP expected to fail" + exit 0 + fi + + # xfails + case "${test_id}" in + 2JQS) + echo "skip 1 $test_id - $desctxt # SKIP expected to fail" + exit 0 + ;; + esac + + t=$(mktemp) + + res="not ok" + + # run the test using document-event-stream + ${TOP_BUILDDIR}/src/fy-tool --testsuite --document-event-stream "$tst/in.yaml" >"$t" 2>/dev/null + if [ $? -eq 0 ]; then + diff -u "$tst/test.event" "$t" + if [ $? -eq 0 ]; then + res="ok" + fi + fi + + rm -f "$t" + + echo "$res 1 $test_id - $desctxt" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + jsontestsuite) + tf="$test_id" + f="json-test-suite-data/test_parsing/${tf}" + + # Determine expected result based on prefix + case "$tf" in + y_*) + # Expected to pass + ${TOP_BUILDDIR}/src/fy-tool --testsuite --streaming "$f" >/dev/null 2>&1 + if [ $? -eq 0 ]; then + res="ok" + else + res="not ok" + fi + ;; + n_*) + # Expected to fail + ${TOP_BUILDDIR}/src/fy-tool --testsuite --streaming "$f" >/dev/null 2>&1 + if [ $? -eq 0 ]; then + res="not ok" + else + res="ok" + fi + ;; + i_*) + # Implementation defined - we'll accept either result as ok + ${TOP_BUILDDIR}/src/fy-tool --testsuite --streaming "$f" >/dev/null 2>&1 + res="ok" + ;; + *) + res="not ok" + ;; + esac + + echo "$res 1 - $tf" + + if [ "$res" == "ok" ]; then + exit 0 + else + exit 1 + fi + ;; + + *) + echo "Unknown test suite: $test_suite" >&2 + exit 1 + ;; +esac diff --git a/cmake/setup-yaml-test-suite.cmake b/cmake/setup-yaml-test-suite.cmake new file mode 100644 index 0000000..c8c7f2b --- /dev/null +++ b/cmake/setup-yaml-test-suite.cmake @@ -0,0 +1,42 @@ +# Script to set up test suite data in an idempotent way + +if(NOT DEFINED REPO_NAME) + set(REPO_NAME "test-suite-data") +endif() + +set(REPO_DIR "${TEST_DIR}/${REPO_NAME}") + +# Check if the repository already exists +if(EXISTS "${REPO_DIR}/.git") + message(STATUS "Repository ${REPO_NAME} already exists, updating...") + execute_process( + COMMAND ${GIT_EXECUTABLE} fetch -q origin + WORKING_DIRECTORY ${REPO_DIR} + RESULT_VARIABLE GIT_FETCH_RESULT + ) + if(NOT GIT_FETCH_RESULT EQUAL 0) + message(WARNING "Failed to fetch updates for ${REPO_NAME}") + endif() +else() + message(STATUS "Cloning ${REPO_NAME}...") + execute_process( + COMMAND ${GIT_EXECUTABLE} clone -q "${TESTSUITEURL}" ${REPO_NAME} + WORKING_DIRECTORY ${TEST_DIR} + RESULT_VARIABLE GIT_CLONE_RESULT + ) + if(NOT GIT_CLONE_RESULT EQUAL 0) + message(FATAL_ERROR "Failed to clone ${REPO_NAME}") + endif() +endif() + +# Checkout the specific commit +execute_process( + COMMAND ${GIT_EXECUTABLE} checkout -q --detach ${TESTSUITECHECKOUT} + WORKING_DIRECTORY ${REPO_DIR} + RESULT_VARIABLE GIT_CHECKOUT_RESULT +) +if(NOT GIT_CHECKOUT_RESULT EQUAL 0) + message(FATAL_ERROR "Failed to checkout ${TESTSUITECHECKOUT} in ${REPO_NAME}") +endif() + +message(STATUS "Test suite ${REPO_NAME} set up successfully") diff --git a/cmake/tags.cmake b/cmake/tags.cmake new file mode 100644 index 0000000..88ee25f --- /dev/null +++ b/cmake/tags.cmake @@ -0,0 +1,21 @@ +find_program(CTAGS ctags) + +if(CTAGS) + # Name of the build directory (for exclusion) + get_filename_component(_build_dir_name "${CMAKE_BINARY_DIR}" NAME) + + add_custom_target(ctags + COMMAND ${CTAGS} + -R + --exclude=${_build_dir_name} + --exclude=.git + --exclude=.cache + --extra=+q + --c-kinds=+lpx + --fields=afikmsSzt + -f "${CMAKE_BINARY_DIR}/tags" + "${CMAKE_SOURCE_DIR}" + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + VERBATIM + ) +endif() diff --git a/cmake/testsuite-tests.cmake b/cmake/testsuite-tests.cmake new file mode 100644 index 0000000..284aead --- /dev/null +++ b/cmake/testsuite-tests.cmake @@ -0,0 +1,771 @@ +# Auto-generated file - do not edit manually +# Generated by cmake/generate-testsuite-list.sh +# +# This file contains lists of tests from the YAML test suite +# It allows registering individual tests without requiring the test data +# to be downloaded first. + +# Test suite checkout: 6e6c296ae9c9d2d5c4134b4b64d01b29ac19ff6f +# Generated on: 2025-10-24 08:04:38 UTC +# Base tests: 350 + +# List of base test IDs +set(TESTSUITE_BASE_TESTS + 229Q + 236B + 26DV + 27NA + 2AUY + 2CMS + 2EBW + 2G84 + 2JQS + 2LFX + 2SXE + 2XXW + 33X3 + 35KP + 36F6 + 3ALJ + 3GZX + 3HFZ + 3MYT + 3R3P + 3RLN + 3UYS + 4ABK + 4CQQ + 4EJS + 4FJ6 + 4GC6 + 4H7K + 4HVU + 4JVG + 4MUZ + 4Q9F + 4QFQ + 4RWC + 4UYU + 4V8U + 4WA9 + 4ZYM + 52DL + 54T7 + 55WF + 565N + 57H4 + 58MP + 5BVJ + 5C5M + 5GBF + 5KJE + 5LLU + 5MUD + 5NYZ + 5T43 + 5TRB + 5TYM + 5U3A + 5WE3 + 62EZ + 652Z + 65WH + 6BCT + 6BFJ + 6CA3 + 6CK3 + 6FWR + 6H3V + 6HB6 + 6JQW + 6JTT + 6JWB + 6KGN + 6LVF + 6M2F + 6PBE + 6S55 + 6SLA + 6VJK + 6WLZ + 6WPF + 6XDY + 6ZKB + 735Y + 74H7 + 753E + 7A4E + 7BMT + 7BUB + 7FWL + 7LBH + 7MNF + 7T8X + 7TMG + 7W2P + 7Z25 + 7ZZ5 + 82AN + 87E4 + 8CWC + 8G76 + 8KB6 + 8MK2 + 8QBE + 8UDB + 8XDJ + 8XYN + 93JH + 93WF + 96L6 + 96NN + 98YD + 9BXH + 9C9N + 9CWY + 9DXL + 9FMG + 9HCY + 9J7A + 9JBA + 9KAX + 9KBC + 9MAG + 9MMA + 9MMW + 9MQT + 9SA2 + 9SHH + 9TFX + 9U5K + 9WXW + 9YRD + A2M4 + A6F9 + A984 + AB8U + AVM7 + AZ63 + AZW3 + B3HG + B63P + BD7L + BEC7 + BF9H + BS4K + BU8L + C2DT + C2SP + C4HZ + CC74 + CFD4 + CML9 + CN3R + CPZ3 + CQ3W + CT4Q + CTN5 + CUP7 + CVW2 + CXX2 + D49Q + D83L + D88J + D9TU + DBG4 + DC7X + DE56 + DFF7 + DHP8 + DK3J + DK4H + DK95 + DMG6 + DWX9 + E76Z + EB22 + EHF6 + EW3V + EX5H + EXG3 + F2C7 + F3CP + F6MC + F8F9 + FBC9 + FH7J + FP8R + FQ7F + FRK4 + FTA2 + FUP4 + G4RS + G5U8 + G7JE + G992 + G9HC + GDY7 + GH63 + GT5M + H2RW + H3Z8 + H7J7 + H7TQ + HM87 + HMK4 + HMQ5 + HRE5 + HS5T + HU3P + HWV9 + J3BT + J5UC + J7PZ + J7VC + J9HZ + JEF9 + JHB9 + JKF3 + JQ4R + JR7V + JS2J + JTV5 + JY7Z + K3WX + K4SU + K527 + K54U + K858 + KH5V + KK5P + KMK3 + KS4U + KSS4 + L24T + L383 + L94M + L9U5 + LE5A + LHL4 + LP6E + LQZ7 + LX3P + M29M + M2N8 + M5C3 + M5DY + M6YH + M7A3 + M7NX + M9B4 + MJS9 + MUS6 + MXS3 + MYW6 + MZX3 + N4JP + N782 + NAT4 + NB6Z + NHX8 + NJ66 + NKF9 + NP9H + P2AD + P2EQ + P76L + P94K + PBJ2 + PRH3 + PUW8 + PW8X + Q4CL + Q5MG + Q88A + Q8AD + Q9WF + QB6E + QF4Y + QLJ7 + QT73 + R4YG + R52L + RHX7 + RLU9 + RR7F + RTP8 + RXY3 + RZP5 + RZT7 + S3PD + S4GJ + S4JQ + S4T7 + S7BG + S98Z + S9E8 + SBG9 + SF5V + SKE5 + SM9W + SR86 + SSW6 + SU5Z + SU74 + SY6V + SYW4 + T26H + T4YY + T5N4 + T833 + TD5N + TE2A + TL85 + TS54 + U3C3 + U3XV + U44R + U99R + U9NS + UDM2 + UDR7 + UGM3 + UKK6 + UT92 + UV7Q + V55R + V9D5 + VJP3 + W42U + W4TN + W5VH + W9L4 + WZ62 + X38W + X4QW + X8DW + XLQ9 + XV9V + XW4D + Y2GN + Y79Y + YD5X + YJV2 + Z67P + Z9M4 + ZCZ6 + ZF4X + ZH7C + ZK9H + ZL4Z + ZVH3 + ZWK4 + ZXT5 +) + +# List of all tests (including subtests) +# Format: BASE or BASE/SUBTEST +set(TESTSUITE_ALL_TESTS + "229Q" + "236B" + "26DV" + "27NA" + "2AUY" + "2CMS" + "2EBW" + "2G84/00" + "2G84/01" + "2G84/02" + "2G84/03" + "2JQS" + "2LFX" + "2SXE" + "2XXW" + "33X3" + "35KP" + "36F6" + "3ALJ" + "3GZX" + "3HFZ" + "3MYT" + "3R3P" + "3RLN/00" + "3RLN/01" + "3RLN/02" + "3RLN/03" + "3RLN/04" + "3RLN/05" + "3UYS" + "4ABK" + "4CQQ" + "4EJS" + "4FJ6" + "4GC6" + "4H7K" + "4HVU" + "4JVG" + "4MUZ/00" + "4MUZ/01" + "4MUZ/02" + "4Q9F" + "4QFQ" + "4RWC" + "4UYU" + "4V8U" + "4WA9" + "4ZYM" + "52DL" + "54T7" + "55WF" + "565N" + "57H4" + "58MP" + "5BVJ" + "5C5M" + "5GBF" + "5KJE" + "5LLU" + "5MUD" + "5NYZ" + "5T43" + "5TRB" + "5TYM" + "5U3A" + "5WE3" + "62EZ" + "652Z" + "65WH" + "6BCT" + "6BFJ" + "6CA3" + "6CK3" + "6FWR" + "6H3V" + "6HB6" + "6JQW" + "6JTT" + "6JWB" + "6KGN" + "6LVF" + "6M2F" + "6PBE" + "6S55" + "6SLA" + "6VJK" + "6WLZ" + "6WPF" + "6XDY" + "6ZKB" + "735Y" + "74H7" + "753E" + "7A4E" + "7BMT" + "7BUB" + "7FWL" + "7LBH" + "7MNF" + "7T8X" + "7TMG" + "7W2P" + "7Z25" + "7ZZ5" + "82AN" + "87E4" + "8CWC" + "8G76" + "8KB6" + "8MK2" + "8QBE" + "8UDB" + "8XDJ" + "8XYN" + "93JH" + "93WF" + "96L6" + "96NN/00" + "96NN/01" + "98YD" + "9BXH" + "9C9N" + "9CWY" + "9DXL" + "9FMG" + "9HCY" + "9J7A" + "9JBA" + "9KAX" + "9KBC" + "9MAG" + "9MMA" + "9MMW" + "9MQT/00" + "9MQT/01" + "9SA2" + "9SHH" + "9TFX" + "9U5K" + "9WXW" + "9YRD" + "A2M4" + "A6F9" + "A984" + "AB8U" + "AVM7" + "AZ63" + "AZW3" + "B3HG" + "B63P" + "BD7L" + "BEC7" + "BF9H" + "BS4K" + "BU8L" + "C2DT" + "C2SP" + "C4HZ" + "CC74" + "CFD4" + "CML9" + "CN3R" + "CPZ3" + "CQ3W" + "CT4Q" + "CTN5" + "CUP7" + "CVW2" + "CXX2" + "D49Q" + "D83L" + "D88J" + "D9TU" + "DBG4" + "DC7X" + "DE56/00" + "DE56/01" + "DE56/02" + "DE56/03" + "DE56/04" + "DE56/05" + "DFF7" + "DHP8" + "DK3J" + "DK4H" + "DK95/00" + "DK95/01" + "DK95/02" + "DK95/03" + "DK95/04" + "DK95/05" + "DK95/06" + "DK95/07" + "DK95/08" + "DMG6" + "DWX9" + "E76Z" + "EB22" + "EHF6" + "EW3V" + "EX5H" + "EXG3" + "F2C7" + "F3CP" + "F6MC" + "F8F9" + "FBC9" + "FH7J" + "FP8R" + "FQ7F" + "FRK4" + "FTA2" + "FUP4" + "G4RS" + "G5U8" + "G7JE" + "G992" + "G9HC" + "GDY7" + "GH63" + "GT5M" + "H2RW" + "H3Z8" + "H7J7" + "H7TQ" + "HM87/00" + "HM87/01" + "HMK4" + "HMQ5" + "HRE5" + "HS5T" + "HU3P" + "HWV9" + "J3BT" + "J5UC" + "J7PZ" + "J7VC" + "J9HZ" + "JEF9/00" + "JEF9/01" + "JEF9/02" + "JHB9" + "JKF3" + "JQ4R" + "JR7V" + "JS2J" + "JTV5" + "JY7Z" + "K3WX" + "K4SU" + "K527" + "K54U" + "K858" + "KH5V/00" + "KH5V/01" + "KH5V/02" + "KK5P" + "KMK3" + "KS4U" + "KSS4" + "L24T/00" + "L24T/01" + "L383" + "L94M" + "L9U5" + "LE5A" + "LHL4" + "LP6E" + "LQZ7" + "LX3P" + "M29M" + "M2N8/00" + "M2N8/01" + "M5C3" + "M5DY" + "M6YH" + "M7A3" + "M7NX" + "M9B4" + "MJS9" + "MUS6/00" + "MUS6/01" + "MUS6/02" + "MUS6/03" + "MUS6/04" + "MUS6/05" + "MUS6/06" + "MXS3" + "MYW6" + "MZX3" + "N4JP" + "N782" + "NAT4" + "NB6Z" + "NHX8" + "NJ66" + "NKF9" + "NP9H" + "P2AD" + "P2EQ" + "P76L" + "P94K" + "PBJ2" + "PRH3" + "PUW8" + "PW8X" + "Q4CL" + "Q5MG" + "Q88A" + "Q8AD" + "Q9WF" + "QB6E" + "QF4Y" + "QLJ7" + "QT73" + "R4YG" + "R52L" + "RHX7" + "RLU9" + "RR7F" + "RTP8" + "RXY3" + "RZP5" + "RZT7" + "S3PD" + "S4GJ" + "S4JQ" + "S4T7" + "S7BG" + "S98Z" + "S9E8" + "SBG9" + "SF5V" + "SKE5" + "SM9W/00" + "SM9W/01" + "SR86" + "SSW6" + "SU5Z" + "SU74" + "SY6V" + "SYW4" + "T26H" + "T4YY" + "T5N4" + "T833" + "TD5N" + "TE2A" + "TL85" + "TS54" + "U3C3" + "U3XV" + "U44R" + "U99R" + "U9NS" + "UDM2" + "UDR7" + "UGM3" + "UKK6/00" + "UKK6/01" + "UKK6/02" + "UT92" + "UV7Q" + "V55R" + "V9D5" + "VJP3/00" + "VJP3/01" + "W42U" + "W4TN" + "W5VH" + "W9L4" + "WZ62" + "X38W" + "X4QW" + "X8DW" + "XLQ9" + "XV9V" + "XW4D" + "Y2GN" + "Y79Y/000" + "Y79Y/001" + "Y79Y/002" + "Y79Y/003" + "Y79Y/004" + "Y79Y/005" + "Y79Y/006" + "Y79Y/007" + "Y79Y/008" + "Y79Y/009" + "Y79Y/010" + "YD5X" + "YJV2" + "Z67P" + "Z9M4" + "ZCZ6" + "ZF4X" + "ZH7C" + "ZK9H" + "ZL4Z" + "ZVH3" + "ZWK4" + "ZXT5" +) diff --git a/configure.ac b/configure.ac index fd3d318..0c15913 100644 --- a/configure.ac +++ b/configure.ac @@ -14,6 +14,8 @@ fi AC_SUBST(ACLOCAL_AMFLAGS, "-I m4") AC_CANONICAL_TARGET +AC_CANONICAL_HOST +AC_CANONICAL_BUILD AC_CONFIG_SRCDIR([src/lib/fy-parse.c]) AC_CONFIG_HEADERS([config.h]) @@ -128,7 +130,10 @@ AC_CHECK_FUNCS([__builtin_bswap16 __builtin_bswap32 __builtin_bswap64]) dnl for old autoconf version AX_APPEND_COMPILE_FLAGS does not work m4_version_prereq(2.64, - [AX_APPEND_COMPILE_FLAGS([-Wall -Wsign-compare -Wno-stringop-overflow -fvisibility=hidden], + [AX_APPEND_COMPILE_FLAGS([-Wall -Wsign-compare -Wno-stringop-overflow \ + -fvisibility=hidden -Wno-unused-function \ + -Wno-tautological-constant-out-of-range-compare \ + -std=c2x], [CFLAGS], [-pedantic -Werror])], [CFLAGS="$CFLAGS -Wall -Wsign-compare"]) @@ -140,9 +145,9 @@ fi # need that as conditional because some internal tools compile as static # to have access to internal libfyaml APIs if test "x$enable_static" = "xyes" ; then -HAVE_STATIC=1 + HAVE_STATIC=1 else -HAVE_STATIC=0 + HAVE_STATIC=0 fi AC_SUBST(HAVE_STATIC) AC_DEFINE_UNQUOTED([HAVE_STATIC], [$HAVE_STATIC], [Define to 1 if static linking is available]) @@ -150,10 +155,15 @@ AM_CONDITIONAL([HAVE_STATIC], [ test x$HAVE_STATIC = x1 ]) dnl check whether warning flags are supported AX_CHECK_COMPILE_FLAG([-Wstringop-overread], , , [-Werror]) -AS_IF([test "x$ax_cv_check_cflags__Werror__wstringop_overread" = x"yes"], +AS_IF([test "x$ax_cv_check_cflags__Werror__Wstringop_overread" = x"yes"], [HAVE_WSTRINGOP_OVERREAD=1], [HAVE_WSTRINGOP_OVERREAD=0]) AC_DEFINE_UNQUOTED([HAVE_WSTRINGOP_OVERREAD], [$HAVE_WSTRINGOP_OVERREAD], [Define to 1 if -Wstringop-overread is supported]) +AX_CHECK_COMPILE_FLAG([-Wstrict-aliasing], , , [-Werror]) +AS_IF([test "x$ax_cv_check_cflags__Werror__Wstrict_aliasing" = x"yes"], + [HAVE_WSTRICT_ALIASING=1], [HAVE_WSTRICT_ALIASING=0]) +AC_DEFINE_UNQUOTED([HAVE_WSTRICT_ALIASING], [$HAVE_WSTRICT_ALIASING], [Define to 1 if -Wstrict-aliasing is supported]) + dnl Per target optimizations AC_ARG_ENABLE([portable-target], AS_HELP_STRING([--enable-portable-target], @@ -166,6 +176,16 @@ AC_SUBST(HAVE_PORTABLE_TARGET) AC_DEFINE_UNQUOTED([HAVE_PORTABLE_TARGET], [$HAVE_PORTABLE_TARGET], [Define to 1 if PORTABLE_TARGET is enabled]) AM_CONDITIONAL([HAVE_PORTABLE_TARGET], [ test x$HAVE_PORTABLE_TARGET == x1 ]) +dnl Detect cross-compilation +if test "x$build" != "x$host" ; then + CROSS_COMPILING=1 +else + CROSS_COMPILING=0 +fi +AC_SUBST(CROSS_COMPILING) +AC_DEFINE_UNQUOTED([CROSS_COMPILING], [$CROSS_COMPILING], [Define to 1 if cross compiling]) +AM_CONDITIONAL([CROSS_COMPILING], [ test x$CROSS_COMPILING = x1 ]) + AM_CONDITIONAL([TARGET_CPU_X86], [ test x$target_cpu = xx86 ]) AM_CONDITIONAL([TARGET_CPU_X86_64], [ test x$target_cpu = xx86_64 ]) AM_CONDITIONAL([TARGET_CPU_ANY_X86], [ test x$target_cpu = xx86 -o x$target_cpu = xx86_64 ]) @@ -239,6 +259,14 @@ AC_SUBST(HAVE_QSORT_R) AC_DEFINE_UNQUOTED([HAVE_QSORT_R], [$HAVE_QSORT_R], [Define to 1 if you have qsort_r available]) AM_CONDITIONAL([HAVE_QSORT_R], [ test x$HAVE_QSORT_R = x1 ]) +# check if there's a mremap available (macos or BSD's don't have it) +AC_CHECK_FUNC([mremap], + HAVE_MREMAP=1, + HAVE_MREMAP=0) +AC_SUBST(HAVE_MREMAP) +AC_DEFINE_UNQUOTED([HAVE_MREMAP], [$HAVE_MREMAP], [Define to 1 if you have mremap available]) +AM_CONDITIONAL([HAVE_MREMAP], [ test x$HAVE_MREMAP = x1 ]) + PKG_CHECK_MODULES(LIBYAML, [ yaml-0.1 ], HAVE_LIBYAML=1, HAVE_LIBYAML=0) # update with pkg-config's flags @@ -421,13 +449,13 @@ AC_DEFINE_UNQUOTED([HAVE_JQ], [$HAVE_JQ], [Define to 1 if you have jq available] AM_CONDITIONAL([HAVE_JQ], [ test x$HAVE_JQ = x1 ]) AC_SUBST(JQ, "$JQ") -# check for libclang -AC_ARG_WITH([libclang], - dnl enable internet available when checking + dnl enable internet available when checking AC_ARG_ENABLE([network], AS_HELP_STRING([--disable-network], [Disable tests requiring network access])) +# check for libclang +AC_ARG_WITH([libclang], [AS_HELP_STRING([--with-libclang=[]], [Enable support for libclang @<:@default=auto@:>@])], [with_libclang="$withval"], @@ -438,11 +466,18 @@ LIBCLANG_CFLAGS="" LIBCLANG_CPPFLAGS="" LIBCLANG_LDFLAGS="" LIBCLANG_LIBS="" -dnl TODO think about cross compilation +if test "x$with_libclang" != "xno"; then + if test "x$CROSS_COMPILING" = "x1" -a "x$with_libclang" = "xauto" ; then + AC_MSG_WARN([Cross-compiling detected: disabling libclang auto-detection.]) + AC_MSG_WARN([Use --with-libclang= to force libclang if needed (advanced use only).]) + with_libclang=no + fi +fi if test "x$with_libclang" != "xno"; then if test "x$with_libclang" == "xauto"; then - # try to auto detect (llvm 15 down to 8) - check="llvm-config llvm-config-15 llvm-config-14 llvm-config-13 llvm-config-12 llvm-config-11 llvm-config-10" + # try to auto detect (llvm 20 down to 10) + check="llvm-config llvm-config-20 llvm-config-19 llvm-config-18 llvm-config-17 llvm-config-16 \ + llvm-config-15 llvm-config-14 llvm-config-13 llvm-config-12 llvm-config-11 llvm-config-10" else if test "x$with_libclang" == "xlatest"; then check="llvm-config" @@ -456,18 +491,25 @@ if test "x$with_libclang" != "xno"; then orig_LIBS="$LIBS" for cfg in $check ; do # no caching + $as_unset ac_cv_path_LLVM_CONFIG AC_PATH_PROG([LLVM_CONFIG], [$cfg]) if test "x$LLVM_CONFIG" != "x" ; then LIBCLANG_CFLAGS=`$LLVM_CONFIG --cflags` LIBCLANG_CPPFLAGS=`$LLVM_CONFIG --cppflags` LIBCLANG_LDFLAGS=`$LLVM_CONFIG --ldflags` - LIBCLANG_LIBS="-lclang" + LIBCLANG_LIBS="-lclang `$LLVM_CONFIG --libs` `$LLVM_CONFIG --system-libs`" CPPFLAGS="$CPPFLAGS $LIBCLANG_CPPFLAGS" CFLAGS="$CFLAGS $LIBCLANG_CFLAGS" LDFLAGS="$LDFLAGS $LIBCLANG_LDFLAGS" LIBS="$LIBS $LIBCLANG_LIBS" - AC_CHECK_LIB([clang], [clang_createIndex], - [HAVE_LIBCLANG_LIB=1], [HAVE_LIBCLANG_LIB=0]) + + if test "x$CROSS_COMPILING" = "x1" ; then + AC_MSG_WARN([Cross-compiling: skipping libclang link test, checking headers only]) + HAVE_LIBCLANG_LIB=1 + else + AC_CHECK_LIB([clang], [clang_createIndex], + [HAVE_LIBCLANG_LIB=1], [HAVE_LIBCLANG_LIB=0]) + fi AC_CHECK_HEADER([clang-c/Index.h], [HAVE_LIBCLANG_HDR=1], [HAVE_LIBCLANG_HDR=0]) # if we found it, break out @@ -475,8 +517,13 @@ if test "x$with_libclang" != "xno"; then HAVE_LIBCLANG=1 break fi - $as_unset ac_cv_lib_clang_clang_createIndex $as_unset ac_cv_header_clang_c_Index_h \ - LIBCLANG_CFLAGS LIBCLANG_CPPFLAGS LIBCLANG_LIBS HAVE_LIBCLANG_LIB HAVE_LIBCLANG_HDR + $as_unset ac_cv_lib_clang_clang_createIndex + $as_unset ac_cv_header_clang_c_Index_h + $as_unset LIBCLANG_CFLAGS + $as_unset LIBCLANG_CPPFLAGS + $as_unset LIBCLANG_LIBS + $as_unset HAVE_LIBCLANG_LIB + $as_unset HAVE_LIBCLANG_HDR fi CFLAGS=$orig_CFLAGS LDFLAGS=$orig_LDFLAGS @@ -484,7 +531,7 @@ if test "x$with_libclang" != "xno"; then if test "x$HAVE_LIBCLANG" = "x1"; then break fi - $as_unset ac_cv_path_LLVM_CONFIG LLVM_CONFIG + $as_unset LLVM_CONFIG done CFLAGS=$orig_CFLAGS CPPFLAGS=$orig_CPPFLAGS @@ -492,9 +539,6 @@ if test "x$with_libclang" != "xno"; then LIBS=$orig_LIBS fi -dnl it is safe to update CFLAGS, CPPFLAGS and LDFLAGS from clang -dnl we will only link against libclang when we have to - AC_SUBST(HAVE_LIBCLANG) AC_SUBST(LIBCLANG_CFLAGS) AC_SUBST(LIBCLANG_CPPFLAGS) @@ -503,17 +547,105 @@ AC_SUBST(LIBCLANG_LIBS) AC_DEFINE_UNQUOTED([HAVE_LIBCLANG], [$HAVE_LIBCLANG], [Define to 1 if libclang is available]) AM_CONDITIONAL([HAVE_LIBCLANG], [ test x$HAVE_LIBCLANG = x1 ]) -dnl AS_IF([test "x$with_libclang" != xno], -dnl [AC_CHECK_LIB([libclang], [main], -dnl [AC_SUBST([LIBREADLINE], ["-lreadline -lncurses"]) -dnl AC_DEFINE([HAVE_LIBREADLINE], [1], -dnl [Define if you have libreadline]) -dnl ], -dnl [if test "x$with_readline" != xcheck; then -dnl AC_MSG_FAILURE( -dnl [--with-readline was given, but test for readline failed]) -dnl fi -dnl ], -lncurses)]) +dnl Check for Clang Blocks support (Clang or AppleClang) +dnl Blocks provide closure support for C, useful for callback-heavy APIs +HAVE_CLANG_BLOCKS=0 +CLANG_BLOCKS_CFLAGS="" +CLANG_BLOCKS_LIBS="" + +AC_MSG_CHECKING([for Clang compiler]) +case "$CC" in + *clang*) + IS_CLANG=yes + ;; + *) + IS_CLANG=no + ;; +esac +AC_MSG_RESULT([$IS_CLANG]) + +if test "x$IS_CLANG" = "xyes"; then + dnl Check if this is AppleClang (Blocks built-in) or LLVM Clang (needs BlocksRuntime) + AC_MSG_CHECKING([for AppleClang]) + if $CC --version 2>&1 | grep -q "Apple" ; then + IS_APPLECLANG=yes + else + IS_APPLECLANG=no + fi + AC_MSG_RESULT([$IS_APPLECLANG]) + + if test "x$IS_APPLECLANG" = "xyes"; then + dnl AppleClang has Blocks built-in + AC_MSG_CHECKING([for AppleClang Blocks support]) + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS -fblocks" + AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[]], [[ + void (^b)(void) = ^{ }; + b(); + ]])], + [HAVE_CLANG_BLOCKS=1 + CLANG_BLOCKS_CFLAGS="-fblocks" + AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no])]) + CFLAGS="$save_CFLAGS" + else + dnl LLVM Clang needs libBlocksRuntime + AC_MSG_CHECKING([for Clang Blocks with libBlocksRuntime]) + save_CFLAGS="$CFLAGS" + save_LIBS="$LIBS" + CFLAGS="$CFLAGS -fblocks" + LIBS="$LIBS -lBlocksRuntime" + AC_LINK_IFELSE( + [AC_LANG_PROGRAM([[#include ]], [[ + void (^b)(void) = ^{ }; + b(); + ]])], + [HAVE_CLANG_BLOCKS=1 + CLANG_BLOCKS_CFLAGS="-fblocks" + CLANG_BLOCKS_LIBS="-lBlocksRuntime" + AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no - libBlocksRuntime not found])]) + CFLAGS="$save_CFLAGS" + LIBS="$save_LIBS" + fi +fi + +AC_SUBST(HAVE_CLANG_BLOCKS) +AC_SUBST(CLANG_BLOCKS_CFLAGS) +AC_SUBST(CLANG_BLOCKS_LIBS) +AC_DEFINE_UNQUOTED([HAVE_CLANG_BLOCKS], [$HAVE_CLANG_BLOCKS], [Define to 1 if Clang Blocks are available]) +AM_CONDITIONAL([HAVE_CLANG_BLOCKS], [ test x$HAVE_CLANG_BLOCKS = x1 ]) + +dnl GCC heap trampolines support (for nested functions) +dnl Heap trampolines are more secure than executable stack trampolines +dnl Available in GCC 14+ with -ftrampoline-impl=heap +dnl Supported targets: x86_64/i386/aarch64 Linux, x86_64/i386 Darwin +HAVE_HEAP_TRAMPOLINES=0 + +AC_MSG_CHECKING([for GCC compiler]) +case "$CC" in + *gcc*) + IS_GCC=yes + ;; + *) + IS_GCC=no + ;; +esac +AC_MSG_RESULT([$IS_GCC]) + +if test "x$IS_GCC" = "xyes"; then + AX_CHECK_COMPILE_FLAG([-ftrampoline-impl=heap], + [HAVE_HEAP_TRAMPOLINES=1 + CFLAGS="$CFLAGS -ftrampoline-impl=heap" + AC_MSG_NOTICE([GCC heap trampolines enabled (-ftrampoline-impl=heap)])], + [AC_MSG_NOTICE([GCC heap trampolines not available (GCC 14+ required), using stack trampolines])], + [-Werror]) +fi + +AC_SUBST(HAVE_HEAP_TRAMPOLINES) +AC_DEFINE_UNQUOTED([HAVE_HEAP_TRAMPOLINES], [$HAVE_HEAP_TRAMPOLINES], [Define to 1 if GCC heap trampolines are enabled]) +AM_CONDITIONAL([HAVE_HEAP_TRAMPOLINES], [ test x$HAVE_HEAP_TRAMPOLINES = x1 ]) dnl trim cflags, cppflags, ldflags CFLAGS=`echo $CFLAGS | sed -e 's/^[ ]*//'` @@ -547,6 +679,10 @@ echo " EXTRA: ${EXTRA} LIBTOOL_VERSION: ${LIBTOOL_VERSION} prefix: ${prefix} + Build system: ${build} + Host system: ${host} + Target system: ${target} + Cross compiling: ${CROSS_COMPILING} C compiler: ${CC} Linker: ${LD} CFLAGS: ${CFLAGS} @@ -562,6 +698,8 @@ echo " DOCKER: ${DOCKER} HAVE_LIBCLANG: ${HAVE_LIBCLANG} LIBCLANG_LIBS: ${LIBCLANG_LIBS} + HAVE_CLANG_BLOCKS: ${HAVE_CLANG_BLOCKS} + HAVE_HEAP_TRAMPOLINES: ${HAVE_HEAP_TRAMPOLINES} TESTSUITEURL: $TESTSUITEURL TESTSUITECHECKOUT: $TESTSUITECHECKOUT JSONTESTSUITEURL: $JSONTESTSUITEURL diff --git a/doc/conf.py b/doc/conf.py index eb9f181..9e46afa 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -60,7 +60,7 @@ html_theme = 'sphinx_rtd_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] master_doc = 'index' diff --git a/doc/include b/doc/include new file mode 120000 index 0000000..3a1af68 --- /dev/null +++ b/doc/include @@ -0,0 +1 @@ +../include/ \ No newline at end of file diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 0000000..0390ef8 --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.12) +project(libfyaml-examples C) + +# Find libfyaml +find_package(libfyaml 0.9 QUIET) + +if(NOT libfyaml_FOUND) + # Fall back to pkg-config + find_package(PkgConfig REQUIRED) + pkg_check_modules(LIBFYAML REQUIRED libfyaml) + + # Create imported target for consistency + add_library(libfyaml::libfyaml INTERFACE IMPORTED) + set_target_properties(libfyaml::libfyaml PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${LIBFYAML_INCLUDE_DIRS}" + INTERFACE_LINK_LIBRARIES "${LIBFYAML_LIBRARIES}" + INTERFACE_LINK_DIRECTORIES "${LIBFYAML_LIBRARY_DIRS}" + INTERFACE_COMPILE_OPTIONS "${LIBFYAML_CFLAGS_OTHER}" + ) +endif() + +# List of all examples +set(EXAMPLES + quick-start + basic-parsing + path-queries + document-manipulation + event-streaming + build-from-scratch +) + +# Build each example +foreach(example ${EXAMPLES}) + add_executable(${example} ${example}.c) + target_link_libraries(${example} PRIVATE libfyaml::libfyaml) + + # Set C standard + set_property(TARGET ${example} PROPERTY C_STANDARD 99) + + # Enable warnings + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${example} PRIVATE -Wall -Wextra) + endif() +endforeach() + +# Installation (optional) +install(TARGETS ${EXAMPLES} + RUNTIME DESTINATION bin/libfyaml-examples +) + +# Install sample files +install(FILES + README.md + config.yaml + invoice.yaml + DESTINATION share/doc/libfyaml/examples +) + +# Show configuration info +message(STATUS "libfyaml examples configuration:") +message(STATUS " Examples to build: ${EXAMPLES}") +if(libfyaml_FOUND) + message(STATUS " Found libfyaml via CMake config") + if(libfyaml_HAS_LIBCLANG) + message(STATUS " libfyaml has reflection support") + endif() +else() + message(STATUS " Found libfyaml via pkg-config") +endif() diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..f67e682 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,281 @@ +# libfyaml Examples + +This directory contains practical examples demonstrating various features of libfyaml. + +## Building the Examples + +### Using CMake (Recommended) + +```bash +cd examples +mkdir build && cd build +cmake .. +cmake --build . +``` + +The compiled examples will be in the `build/` directory. + +### Using Make + +```bash +cd examples +make +``` + +The compiled examples will be in the current directory. + +### Manual Compilation + +```bash +gcc -o quick-start quick-start.c $(pkg-config --cflags --libs libfyaml) +gcc -o basic-parsing basic-parsing.c $(pkg-config --cflags --libs libfyaml) +# ... and so on for other examples +``` + +## Examples Overview + +### 1. quick-start.c + +Example showing: +- Parsing YAML from a file +- Extracting values using `fy_document_scanf()` (path-based queries) +- Modifying the document with `fy_document_insert_at()` +- Emitting the updated YAML + +**Usage**: +```bash +./quick-start [config.yaml] +``` + +**Required YAML structure** (for config.yaml): +```yaml +server: + host: localhost + port: 8080 +``` + +--- + +### 2. basic-parsing.c + +Basic YAML parsing and different output formats: +- Parse YAML from file +- Emit as compact flow format +- Emit as standard block YAML +- Emit as JSON + +**Usage**: +```bash +./basic-parsing [file.yaml] +``` + +Works with any valid YAML file. + +--- + +### 3. path-queries.c + +Query YAML documents using path expressions: +- Extract multiple values with `fy_document_scanf()` +- Query individual nodes with `fy_node_by_path()` +- Compare node values with strings +- Handle missing paths gracefully + +**Usage**: +```bash +./path-queries +``` + +This example uses inline YAML, so no external file is needed. + +--- + +### 4. document-manipulation.c + +Example of document manipulation: +- Read existing values from YAML +- Update values (replace invoice number) +- Add new fields (spouse, delivery address) +- Emit with sorted keys + +**Usage**: +```bash +./document-manipulation [invoice.yaml] +``` + +**Expected YAML structure** (for invoice.yaml): +```yaml +invoice: 34843 +date: 2001-01-23 +bill-to: + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 +``` + +If no file is provided, the example uses built-in default data. + +--- + +### 5. event-streaming.c + +Demonstrates the low-level event-based API: +- Create parser and set input +- Process events one by one +- Handle different event types (scalars, mappings, sequences) +- Memory-efficient parsing for large files + +**Usage**: +```bash +./event-streaming [file.yaml] +``` + +This is the most memory-efficient way to parse YAML, suitable for very large files. + +--- + +### 6. build-from-scratch.c + +Create YAML documents from scratch: +- Create empty document +- Build complex structures using `fy_node_buildf()` with printf-style formatting +- Add fields programmatically +- Emit as both JSON and YAML + +**Usage**: +```bash +./build-from-scratch +``` + +No input file needed - generates a complete configuration document. + +--- + +## Sample YAML Files + +Sample YAML files are provided for testing: + +- **config.yaml** - Simple server configuration +- **invoice.yaml** - Invoice document for manipulation example + +Create these files in the examples directory, or the programs will use sensible defaults. + +### Sample config.yaml + +```yaml +server: + host: localhost + port: 8080 + ssl: true + max_connections: 100 + +database: + host: db.example.com + port: 5432 + name: production_db + +logging: + level: info + file: /var/log/app.log +``` + +### Sample invoice.yaml + +```yaml +invoice: 34843 +date: 2001-01-23 +bill-to: + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + city: Royal Oak + state: MI + postal: 48046 +ship-to: + given: Chris + family: Dumars +product: + - sku: BL394D + quantity: 4 + description: Basketball + price: 450.00 + - sku: BL4438H + quantity: 1 + description: Super Hoop + price: 2392.00 +tax: 251.42 +total: 4443.52 +comments: > + Late afternoon is best. + Backup contact is Nancy + Billsmer @ 338-4338. +``` + +## API Reference + +For complete API documentation, visit: https://pantoniou.github.io/libfyaml/ + +## Common Patterns + +### Error Handling + +All examples demonstrate proper error handling: +```c +struct fy_document *fyd = fy_document_build_from_file(NULL, "file.yaml"); +if (!fyd) { + fprintf(stderr, "Failed to parse YAML\n"); + return EXIT_FAILURE; +} + +// ... use document ... + +fy_document_destroy(fyd); // Always clean up +``` + +### Path-Based Queries + +Extract multiple values efficiently: +```c +int count = fy_document_scanf(fyd, + "/path1 %s " + "/path2 %d", + string_var, &int_var); + +if (count != 2) { + // Handle error +} +``` + +### Document Modification + +Add or update fields: +```c +fy_document_insert_at(fyd, "/parent/path", FY_NT, + fy_node_buildf(fyd, "key: value")); +``` + +### Output Formatting + +Choose output mode based on needs: +```c +// Compact JSON +fy_emit_document_to_fp(fyd, FYECF_MODE_JSON, stdout); + +// Pretty YAML with sorted keys +fy_emit_document_to_fp(fyd, FYECF_MODE_BLOCK | FYECF_SORT_KEYS, stdout); + +// One-line flow format +fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stdout); +``` + +## License + +All examples are released under the MIT License, same as libfyaml. + +Copyright (c) 2019-2025 Pantelis Antoniou diff --git a/examples/basic-parsing.c b/examples/basic-parsing.c new file mode 100644 index 0000000..06c60e2 --- /dev/null +++ b/examples/basic-parsing.c @@ -0,0 +1,50 @@ +/* + * basic-parsing.c - Basic YAML parsing example + * + * Demonstrates: + * - Parsing YAML from a file + * - Basic error handling + * - Emitting in different output modes + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct fy_document *fyd = NULL; + const char *input_file = (argc > 1) ? argv[1] : "config.yaml"; + + // Parse YAML from file + fyd = fy_document_build_from_file(NULL, input_file); + if (!fyd) { + fprintf(stderr, "Failed to parse YAML from %s\n", input_file); + return EXIT_FAILURE; + } + + printf("Successfully parsed: %s\n\n", input_file); + + // Emit as flow-oneline (compact format) + printf("Compact format:\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_FLOW_ONELINE, stdout); + printf("\n\n"); + + // Emit as block (standard YAML) + printf("Block format:\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_BLOCK, stdout); + printf("\n"); + + // Emit as JSON + printf("JSON format:\n"); + fy_emit_document_to_fp(fyd, FYECF_MODE_JSON, stdout); + printf("\n"); + + fy_document_destroy(fyd); + + return EXIT_SUCCESS; +} diff --git a/examples/build-from-scratch.c b/examples/build-from-scratch.c new file mode 100644 index 0000000..e8f0a5b --- /dev/null +++ b/examples/build-from-scratch.c @@ -0,0 +1,111 @@ +/* + * build-from-scratch.c - Building YAML documents programmatically + * + * Demonstrates: + * - Creating empty documents + * - Building nodes using printf-style formatting + * - Setting document root + * - Adding fields programmatically + * - Emitting as both JSON and YAML + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include + +int main(void) +{ + struct fy_document *fyd = NULL; + struct fy_node *root = NULL; + time_t now = time(NULL); + struct tm *tm_info = localtime(&now); + char timestamp[32]; + int ret = EXIT_FAILURE; + + strftime(timestamp, sizeof(timestamp), "%Y-%m-%d %H:%M:%S", tm_info); + + // Create empty document + fyd = fy_document_create(NULL); + if (!fyd) { + fprintf(stderr, "Failed to create document\n"); + return EXIT_FAILURE; + } + + // Build root mapping using printf-style formatting + root = fy_node_buildf(fyd, + "application: MyApp\n" + "version: %d.%d.%d\n" + "build_date: %s\n" + "settings:\n" + " debug: %s\n" + " max_connections: %d\n" + " allowed_hosts:\n" + " - localhost\n" + " - 127.0.0.1\n", + 1, 2, 3, + timestamp, + "true", + 100); + + if (!root) { + fprintf(stderr, "Failed to build root node\n"); + goto cleanup; + } + + // Set as document root + fy_document_set_root(fyd, root); + + // Add more fields programmatically + if (fy_document_insert_at(fyd, "/settings", FY_NT, + fy_node_buildf(fyd, "log_level: info"))) { + fprintf(stderr, "Failed to add log_level\n"); + goto cleanup; + } + + if (fy_document_insert_at(fyd, "/settings", FY_NT, + fy_node_buildf(fyd, + "database:\n" + " host: localhost\n" + " port: 5432\n" + " name: myapp_db\n"))) { + fprintf(stderr, "Failed to add database config\n"); + goto cleanup; + } + + // Add feature flags + if (fy_document_insert_at(fyd, "/", FY_NT, + fy_node_buildf(fyd, + "features:\n" + " authentication: enabled\n" + " api_v2: disabled\n" + " metrics: enabled\n"))) { + fprintf(stderr, "Failed to add features\n"); + goto cleanup; + } + + // Output as JSON + printf("=== JSON Output ===\n"); + if (fy_emit_document_to_fp(fyd, FYECF_MODE_JSON, stdout)) { + fprintf(stderr, "Failed to emit as JSON\n"); + goto cleanup; + } + printf("\n\n"); + + // Output as YAML with sorted keys + printf("=== YAML Output (sorted) ===\n"); + if (fy_emit_document_to_fp(fyd, FYECF_MODE_BLOCK | FYECF_SORT_KEYS, stdout)) { + fprintf(stderr, "Failed to emit as YAML\n"); + goto cleanup; + } + + ret = EXIT_SUCCESS; + +cleanup: + fy_document_destroy(fyd); + return ret; +} diff --git a/examples/config.yaml b/examples/config.yaml new file mode 100644 index 0000000..73353c3 --- /dev/null +++ b/examples/config.yaml @@ -0,0 +1,40 @@ +# Sample configuration file for libfyaml examples +# This file demonstrates a typical application configuration + +server: + host: localhost + port: 8080 + ssl: true + max_connections: 100 + timeout: 30 + +database: + host: db.example.com + port: 5432 + name: production_db + username: dbuser + pool_size: 10 + +logging: + level: info + file: /var/log/app.log + rotate: daily + max_size: 100M + +cache: + enabled: true + type: redis + host: cache.example.com + port: 6379 + ttl: 3600 + +features: + authentication: enabled + api_v2: disabled + metrics: enabled + debug_mode: false + +allowed_origins: + - https://example.com + - https://www.example.com + - https://app.example.com diff --git a/examples/document-manipulation.c b/examples/document-manipulation.c new file mode 100644 index 0000000..8e5770d --- /dev/null +++ b/examples/document-manipulation.c @@ -0,0 +1,111 @@ +/* + * document-manipulation.c - Document manipulation example + * + * Demonstrates: + * - Reading values from a YAML document + * - Modifying existing values + * - Adding new fields to the document + * - Emitting with sorted keys + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct fy_document *fyd = NULL; + const char *input_file = (argc > 1) ? argv[1] : "invoice.yaml"; + unsigned int invoice_nr; + char given_name[256]; + int ret = EXIT_FAILURE; + + // Create a default invoice if file doesn't exist + const char *default_invoice = + "invoice: 34843\n" + "date: 2001-01-23\n" + "bill-to:\n" + " given: Chris\n" + " family: Dumars\n" + " address:\n" + " lines: |\n" + " 458 Walkman Dr.\n" + " Suite #292\n" + "product:\n" + " - sku: BL394D\n" + " quantity: 4\n" + " description: Basketball\n" + " price: 450.00\n"; + + // Try to load from file, fall back to default + fyd = fy_document_build_from_file(NULL, input_file); + if (!fyd) { + fprintf(stderr, "Note: Creating document from default data\n"); + fprintf(stderr, " (Create invoice.yaml or pass file as argument)\n\n"); + fyd = fy_document_build_from_string(NULL, default_invoice, FY_NT); + if (!fyd) { + fprintf(stderr, "Failed to create document\n"); + goto cleanup; + } + } + + // Read current invoice number and customer name + int count = fy_document_scanf(fyd, + "/invoice %u " + "/bill-to/given %255s", + &invoice_nr, given_name); + + if (count != 2) { + fprintf(stderr, "Failed to extract invoice data\n"); + goto cleanup; + } + + printf("Processing invoice #%u for %s\n\n", invoice_nr, given_name); + + // Update invoice number (replaces existing value) + if (fy_document_insert_at(fyd, "/invoice", FY_NT, + fy_node_buildf(fyd, "%u", invoice_nr + 1))) { + fprintf(stderr, "Failed to update invoice number\n"); + goto cleanup; + } + printf("Updated invoice number to %u\n", invoice_nr + 1); + + // Add spouse field to bill-to + if (fy_document_insert_at(fyd, "/bill-to", FY_NT, + fy_node_buildf(fyd, "spouse: Jane"))) { + fprintf(stderr, "Failed to add spouse\n"); + goto cleanup; + } + printf("Added spouse information\n"); + + // Add delivery address + if (fy_document_insert_at(fyd, "/bill-to", FY_NT, + fy_node_buildf(fyd, + "delivery-address:\n" + " street: 123 Main St\n" + " city: Springfield\n" + " state: IL\n" + " zip: 62701\n"))) { + fprintf(stderr, "Failed to add delivery address\n"); + goto cleanup; + } + printf("Added delivery address\n"); + + // Emit updated document with sorted keys + printf("\n--- Updated Invoice ---\n"); + if (fy_emit_document_to_fp(fyd, FYECF_DEFAULT | FYECF_SORT_KEYS, stdout)) { + fprintf(stderr, "Failed to emit document\n"); + goto cleanup; + } + + ret = EXIT_SUCCESS; + +cleanup: + fy_document_destroy(fyd); + return ret; +} diff --git a/examples/event-streaming.c b/examples/event-streaming.c new file mode 100644 index 0000000..5f2c441 --- /dev/null +++ b/examples/event-streaming.c @@ -0,0 +1,117 @@ +/* + * event-streaming.c - Event-based streaming parser example + * + * Demonstrates: + * - Event-based (streaming) YAML parsing + * - Processing different event types + * - Memory-efficient parsing of large files + * - Getting values from scalar events + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct fy_parser *fyp = NULL; + struct fy_event *fye = NULL; + const char *input_file = (argc > 1) ? argv[1] : "config.yaml"; + int indent = 0; + int ret = EXIT_FAILURE; + + // Create parser + fyp = fy_parser_create(NULL); + if (!fyp) { + fprintf(stderr, "Failed to create parser\n"); + return EXIT_FAILURE; + } + + // Set input file + if (fy_parser_set_input_file(fyp, input_file)) { + fprintf(stderr, "Failed to set input file: %s\n", input_file); + fprintf(stderr, "Note: Create config.yaml or pass file as argument\n"); + goto cleanup; + } + + printf("Parsing events from: %s\n\n", input_file); + + // Process events + while ((fye = fy_parser_parse(fyp)) != NULL) { + enum fy_event_type type = fye->type; + + // Print indentation + for (int i = 0; i < indent; i++) + printf(" "); + + switch (type) { + case FYET_STREAM_START: + printf("STREAM-START\n"); + break; + + case FYET_STREAM_END: + printf("STREAM-END\n"); + break; + + case FYET_DOCUMENT_START: + printf("DOCUMENT-START\n"); + indent++; + break; + + case FYET_DOCUMENT_END: + indent--; + printf("DOCUMENT-END\n"); + break; + + case FYET_MAPPING_START: + printf("MAPPING-START\n"); + indent++; + break; + + case FYET_MAPPING_END: + indent--; + printf("MAPPING-END\n"); + break; + + case FYET_SEQUENCE_START: + printf("SEQUENCE-START\n"); + indent++; + break; + + case FYET_SEQUENCE_END: + indent--; + printf("SEQUENCE-END\n"); + break; + + case FYET_SCALAR: { + // Get the scalar value + const char *value = fy_token_get_text0(fy_event_get_token(fye)); + printf("SCALAR: \"%s\"\n", value ? value : "(null)"); + break; + } + + case FYET_ALIAS: { + const char *anchor = fy_token_get_text0(fy_event_get_token(fye)); + printf("ALIAS: *%s\n", anchor ? anchor : "(null)"); + break; + } + + default: + printf("OTHER-EVENT (type=%d)\n", type); + break; + } + + fy_parser_event_free(fyp, fye); + } + + printf("\nParsing completed successfully\n"); + ret = EXIT_SUCCESS; + +cleanup: + fy_parser_destroy(fyp); + return ret; +} diff --git a/examples/invoice.yaml b/examples/invoice.yaml new file mode 100644 index 0000000..6de053b --- /dev/null +++ b/examples/invoice.yaml @@ -0,0 +1,42 @@ +# Sample invoice YAML for document manipulation example +# Based on the YAML specification examples + +invoice: 34843 +date: 2001-01-23 +bill-to: + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + city: Royal Oak + state: MI + postal: 48046 + country: USA +ship-to: + given: Chris + family: Dumars + address: + lines: | + 458 Walkman Dr. + Suite #292 + city: Royal Oak + state: MI + postal: 48046 + country: USA +product: + - sku: BL394D + quantity: 4 + description: Basketball + price: 450.00 + - sku: BL4438H + quantity: 1 + description: Super Hoop + price: 2392.00 +tax: 251.42 +total: 4443.52 +comments: > + Late afternoon is best. + Backup contact is Nancy + Billsmer @ 338-4338. diff --git a/examples/path-queries.c b/examples/path-queries.c new file mode 100644 index 0000000..19a97fb --- /dev/null +++ b/examples/path-queries.c @@ -0,0 +1,80 @@ +/* + * path-queries.c - Path-based YAML queries example + * + * Demonstrates: + * - Using YPATH expressions to query YAML + * - scanf-style value extraction + * - Single node lookup by path + * - String comparison with nodes + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +int main(void) +{ + const char *yaml = + "server:\n" + " host: localhost\n" + " port: 8080\n" + " ssl: true\n" + " max_connections: 100\n"; + + struct fy_document *fyd = fy_document_build_from_string(NULL, yaml, FY_NT); + if (!fyd) { + fprintf(stderr, "Failed to parse YAML\n"); + return EXIT_FAILURE; + } + + // Extract multiple values at once using scanf-style API + char host[256]; + unsigned int port; + int count = fy_document_scanf(fyd, + "/server/host %255s " + "/server/port %u", + host, &port); + + if (count == 2) { + printf("Server configuration:\n"); + printf(" Host: %s\n", host); + printf(" Port: %u\n", port); + } else { + fprintf(stderr, "Failed to extract server configuration (got %d/2)\n", count); + } + + // Query single node by path + struct fy_node *ssl_node = fy_node_by_path( + fy_document_root(fyd), "/server/ssl", FY_NT, FYNWF_DONT_FOLLOW); + + if (ssl_node) { + // Compare node value with string + if (fy_node_compare_string(ssl_node, "true", FY_NT) == 0) { + printf(" SSL: enabled\n"); + } else { + printf(" SSL: disabled\n"); + } + } + + // Extract another value + unsigned int max_conn; + count = fy_document_scanf(fyd, "/server/max_connections %u", &max_conn); + if (count == 1) { + printf(" Max connections: %u\n", max_conn); + } + + // Try to query a non-existent path + struct fy_node *missing = fy_node_by_path( + fy_document_root(fyd), "/server/timeout", FY_NT, FYNWF_DONT_FOLLOW); + + if (!missing) { + printf(" Timeout: not configured (using default)\n"); + } + + fy_document_destroy(fyd); + return EXIT_SUCCESS; +} diff --git a/examples/quick-start.c b/examples/quick-start.c new file mode 100644 index 0000000..b2cc515 --- /dev/null +++ b/examples/quick-start.c @@ -0,0 +1,70 @@ +/* + * quick-start.c - Quick start example for libfyaml + * + * Demonstrates: + * - Parsing YAML from a file + * - Extracting values using path-based scanf + * - Modifying the document + * - Emitting as YAML + * + * Copyright (c) 2019-2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct fy_document *fyd = NULL; + unsigned int port; + char hostname[256]; + int ret = EXIT_FAILURE; + const char *input_file = (argc > 1) ? argv[1] : "config.yaml"; + + // Parse YAML from string or file + fyd = fy_document_build_from_file(NULL, input_file); + if (!fyd) { + fprintf(stderr, "Failed to parse %s\n", input_file); + fprintf(stderr, "Note: You can create config.yaml or pass a file as argument\n"); + goto cleanup; + } + + // Extract values using path-based scanf + int count = fy_document_scanf(fyd, + "/server/port %u " + "/server/host %255s", + &port, hostname); + + if (count != 2) { + fprintf(stderr, "Failed to extract server configuration\n"); + fprintf(stderr, "Expected /server/port and /server/host in YAML\n"); + goto cleanup; + } + + printf("Current configuration: %s:%u\n", hostname, port); + + // Modify document using path-based insertion + if (fy_document_insert_at(fyd, "/server", FY_NT, + fy_node_buildf(fyd, "timeout: 30"))) { + fprintf(stderr, "Failed to insert timeout setting\n"); + goto cleanup; + } + + printf("Added timeout setting\n"); + + // Emit as YAML + printf("\nUpdated configuration:\n"); + if (fy_emit_document_to_fp(fyd, FYECF_SORT_KEYS, stdout)) { + fprintf(stderr, "Failed to emit document\n"); + goto cleanup; + } + + ret = EXIT_SUCCESS; + +cleanup: + fy_document_destroy(fyd); + return ret; +} diff --git a/include/libfyaml.h b/include/libfyaml.h index e1c03c4..af8c820 100644 --- a/include/libfyaml.h +++ b/include/libfyaml.h @@ -38,11 +38,14 @@ extern "C" { #include #include #include +#include #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) #include #endif +#include + /* opaque types for the user */ struct fy_token; struct fy_document_state; @@ -60,8 +63,9 @@ struct fy_path_expr; struct fy_path_exec; struct fy_path_component; struct fy_path; +struct fy_composer; struct fy_document_iterator; - +struct fy_document_builder; #ifndef FY_BIT #define FY_BIT(x) (1U << (x)) @@ -197,11 +201,24 @@ fy_version_supported_iterate(void **prevp) FY_EXPORT; /** - * struct fy_tag - The YAML tag structure. + * fy_shutdown() - Final cleanup before exit + * + * Some libraries *cough*libclang** need explicit cleanup calls + * at the end of program execution, even if you've never called + * any of their functions. * - * @handle: Handle of the tag (i.e. `"!!"` ) - * @prefix: The prefix of the tag (i.e. `"tag:yaml.org,2002:"` + * This method will make sure it calls their cleanup functions + * so that no memory leaks are reported in valgrind etc. + */ +void +fy_shutdown(void) + FY_EXPORT; + +/** + * struct fy_tag - The YAML tag structure. * + * @handle: Handle of the tag (i.e. `"!!"`) + * @prefix: The prefix of the tag (i.e. `"tag:yaml.org,2002:"`) * The parser fills it according to the \%TAG directives * encountered during parsing. */ @@ -255,6 +272,10 @@ enum fy_error_type { * @FYEM_BUILD: Build document module (after tree is constructed) * @FYEM_INTERNAL: Internal error/diagnostic module * @FYEM_SYSTEM: System error/diagnostic module + * @FYEM_EMIT: Emitter module + * @FYEM_TYPESET: Prepare types module (C reflection) + * @FYEM_DECODE: Decode, serialization -> internal form module + * @FYEM_ENCODE: Encode, internal form -> serialized form module * @FYEM_MAX: Non inclusive maximum fy_error_module value */ enum fy_error_module { @@ -266,6 +287,10 @@ enum fy_error_module { FYEM_BUILD, FYEM_INTERNAL, FYEM_SYSTEM, + FYEM_EMIT, + FYEM_TYPESET, + FYEM_DECODE, + FYEM_ENCODE, FYEM_MAX, }; @@ -474,7 +499,7 @@ struct fy_event_stream_start_data { struct fy_token *stream_start; }; -/* +/** * struct fy_event_stream_end_data - stream end event data * * @stream_end: The token that ended the stream @@ -483,7 +508,7 @@ struct fy_event_stream_end_data { struct fy_token *stream_end; }; -/* +/** * struct fy_event_document_start_data - doument start event data * * @document_start: The token that started the document, or NULL if @@ -498,7 +523,7 @@ struct fy_event_document_start_data { bool implicit; }; -/* +/** * struct fy_event_document_end_data - doument end event data * * @document_end: The token that ended the document, or NULL if the @@ -510,7 +535,7 @@ struct fy_event_document_end_data { bool implicit; }; -/* +/** * struct fy_event_alias_data - alias event data * * @anchor: The anchor token definining this alias. @@ -519,10 +544,10 @@ struct fy_event_alias_data { struct fy_token *anchor; }; -/* +/** * struct fy_event_scalar_data - scalar event data * - * @.anchor: anchor token or NULL + * @anchor: anchor token or NULL * @tag: tag token or NULL * @value: scalar value token (cannot be NULL) * @tag_implicit: true if the tag was implicit or explicit @@ -534,7 +559,7 @@ struct fy_event_scalar_data { bool tag_implicit; }; -/* +/** * struct fy_event_sequence_start_data - sequence start event data * * @anchor: anchor token or NULL @@ -548,7 +573,7 @@ struct fy_event_sequence_start_data { struct fy_token *sequence_start; }; -/* +/** * struct fy_event_sequence_end_data - sequence end event data * * @sequence_end: The token that ended the sequence, or NULL if @@ -558,7 +583,7 @@ struct fy_event_sequence_end_data { struct fy_token *sequence_end; }; -/* +/** * struct fy_event_mapping_start_data - mapping start event data * * @anchor: anchor token or NULL @@ -572,7 +597,7 @@ struct fy_event_mapping_start_data { struct fy_token *mapping_start; }; -/* +/** * struct fy_event_mapping_end_data - mapping end event data * * @mapping_end: The token that ended the mapping, or NULL if @@ -628,6 +653,35 @@ struct fy_event { }; }; +/** + * enum fy_event_part - Select part of the event + * + * @FYEP_VALUE: The value of the event (the main token) + * @FYEP_TAG: The tag of the event + * @FYEP_ANCHOR: The anchor of the event + */ +enum fy_event_part { + FYEP_VALUE, + FYEP_TAG, + FYEP_ANCHOR, +}; + +/** + * fy_event_get_type() - Get the event's type + * + * Return the type of the event + * + * @fye: The event + * + * Returns: + * The event type, or FYET_NONE when the event is invalid + */ +static inline enum fy_event_type +fy_event_get_type(const struct fy_event *fye) +{ + return fye ? fye->type : FYET_NONE; +} + /** * fy_event_data() - Get a pointer to the event data * @@ -939,7 +993,7 @@ int fy_parser_set_input_callback(struct fy_parser *fyp, void *user, FY_EXPORT; /** - * fy_parser_set_input_db() - Set the parser to process the given file descriptor + * fy_parser_set_input_fd() - Set the parser to process the given file descriptor * * Point the parser to use @fd for processing. * @@ -970,6 +1024,75 @@ struct fy_event * fy_parser_parse(struct fy_parser *fyp) FY_EXPORT; +/** + * fy_parser_parse_peek() - Parse and peek at the next event. + * + * It will return the next event that a call to fy_parser_parse + * would generate, but as read-only. + * + * You must not free this event. + * + * @fyp: The parser + * + * Returns: + * A peek at the next event in the stream or NULL. + */ +const struct fy_event * +fy_parser_parse_peek(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_skip() - Skip the current scalar/collection + * + * Skips the current scalar or collection. + * + * @fyp: The parser + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_parser_skip(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_count_sequence_items() - Count the sequence items + * + * Counts the number of sequence items. Parser must already + * be in a sequence state. + * Note that this uses backtracking so it might not be very + * efficient. + * + * @fyp: The parser + * + * Returns: + * The number of sequence items on success, -1 on error + * Note if the number of items exceeds INT_MAX, INT_MAX will + * be returned. + */ +int +fy_parser_count_sequence_items(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_count_mapping_items() - Count the mapping items + * + * Counts the number of mapping items. Parser must already + * be in a mapping state. + * Note that this uses backtracking so it might not be very + * efficient. + * + * @fyp: The parser + * + * Returns: + * The number of mapping items on success, -1 on error + * Note if the number of items exceeds INT_MAX, INT_MAX will + * be returned. + */ +int +fy_parser_count_mapping_items(struct fy_parser *fyp) + FY_EXPORT; + /** * fy_parser_event_free() - Free an event * @@ -994,6 +1117,48 @@ bool fy_parser_get_stream_error(struct fy_parser *fyp) FY_EXPORT; +enum fy_parser_mode { + fypm_invalid = -1, + fypm_none, /* scanning for mode, not committed yet */ + fypm_yaml_1_1, + fypm_yaml_1_2, + fypm_yaml_1_3, /* experimental */ + fypm_json, +}; + +static inline bool +fy_parser_mode_is_yaml(enum fy_parser_mode mode) +{ + return mode >= fypm_yaml_1_1 && mode <= fypm_yaml_1_3; +} + +/** + * fy_parser_get_mode() - Get the parser mode + * + * Retrieve the current mode of the parser, which indicates the YAML version + * or format being parsed (YAML 1.1, 1.2, 1.3, or JSON). + * + * @fyp: The parser + * + * Returns: + * The parser mode + */ +enum fy_parser_mode +fy_parser_get_mode(struct fy_parser *fyp) + FY_EXPORT; + +static inline bool +fy_parser_is_in_yaml_mode(struct fy_parser *fyp) +{ + return fy_parser_mode_is_yaml(fy_parser_get_mode(fyp)); +} + +static inline bool +fy_parser_is_in_json_mode(struct fy_parser *fyp) +{ + return fy_parser_get_mode(fyp) == fypm_json; +} + /** * fy_parser_vlog() - Log using the parsers diagnostics printf style (va_arg) * @@ -1068,7 +1233,7 @@ fy_parser_vreport(struct fy_parser *fyp, enum fy_error_type type, struct fy_toke * @type: The error type * @fyt: The token * @fmt: The printf format string - * @..: The extra arguments + * @...: The extra arguments */ void fy_parser_report(struct fy_parser *fyp, enum fy_error_type type, struct fy_token *fyt, @@ -1586,7 +1751,7 @@ struct fy_document * fy_node_document(struct fy_node *fyn) FY_EXPORT; -/* +/** * enum fy_emitter_write_type - Type of the emitted output * * Describes the kind of emitted output, which makes it @@ -1613,7 +1778,24 @@ fy_node_document(struct fy_node *fyn) * @fyewt_single_quoted_scalar_key: Output chunk is an single quoted scalar key * @fyewt_double_quoted_scalar_key: Output chunk is an double quoted scalar key * @fyewt_comment: Output chunk is a comment - * + * @fyewt_indicator_question_mark: ? indicator + * @fyewt_indicator_colon: : indicator + * @fyewt_indicator_dash: - indicator + * @fyewt_indicator_left_bracket: [ indicator + * @fyewt_indicator_right_bracket: ] indicator + * @fyewt_indicator_left_brace: { indicator + * @fyewt_indicator_right_brace: } indicator + * @fyewt_indicator_comma: , indicator + * @fyewt_indicator_bar: | indicator + * @fyewt_indicator_greater: > indicator + * @fyewt_indicator_single_quote_start: ' indicator + * @fyewt_indicator_single_quote_end: ' indicator + * @fyewt_indicator_double_quote_start: " indicator + * @fyewt_indicator_double_quote_end: " indicator + * @fyewt_indicator_ambersand: & indicator + * @fyewt_indicator_star: * indicator + * @fyewt_indicator_chomp: chomp indicator (- or +) + * @fyewt_indicator_explicit_indent: explicit indent indicator (0-9) */ enum fy_emitter_write_type { fyewt_document_indicator, @@ -1636,7 +1818,31 @@ enum fy_emitter_write_type { fyewt_single_quoted_scalar_key, fyewt_double_quoted_scalar_key, fyewt_comment, + /* extended indicators */ + fyewt_indicator_question_mark, + fyewt_indicator_colon, + fyewt_indicator_dash, + fyewt_indicator_left_bracket, + fyewt_indicator_right_bracket, + fyewt_indicator_left_brace, + fyewt_indicator_right_brace, + fyewt_indicator_comma, + fyewt_indicator_bar, + fyewt_indicator_greater, + fyewt_indicator_single_quote_start, + fyewt_indicator_single_quote_end, + fyewt_indicator_double_quote_start, + fyewt_indicator_double_quote_end, + fyewt_indicator_ambersand, + fyewt_indicator_star, + fyewt_indicator_chomp, + fyewt_indicator_explicit_indent, }; +#define FYEWT_EXTENDED_INDICATORS_FIRST fyewt_indicator_question_mark +#define FYEWT_EXTENDED_INDICATORS_LAST fyewt_indicator_explicit_indent + +#define FYEWT_EXTENDED_START fyewt_indicator_question_mark +#define FYEWT_COUNT (fyewt_indicator_explicit_indent + 1) #define FYECF_INDENT_SHIFT 8 #define FYECF_INDENT_MASK 0xf @@ -1678,6 +1884,7 @@ enum fy_emitter_write_type { * @FYECF_STRIP_DOC: Strip document tags and markers when emitting * @FYECF_NO_ENDING_NEWLINE: Do not output ending new line (useful for single line mode) * @FYECF_STRIP_EMPTY_KV: Remove all keys with empty values from the output (not available in streaming mode) + * @FYECF_EXTENDED_CFG: Extend configuration, this config structure is embedded in a fy_emitter_xcfg. * @FYECF_INDENT_DEFAULT: Default emit output indent * @FYECF_INDENT_1: Output indent is 1 * @FYECF_INDENT_2: Output indent is 2 @@ -1702,6 +1909,8 @@ enum fy_emitter_write_type { * @FYECF_MODE_DEJSON: Emit YAML trying to pretify JSON * @FYECF_MODE_PRETTY: Emit YAML that tries to look good * @FYECF_MODE_MANUAL: Emit YAML respecting all manual style hints (reformats if needed) + * @FYECF_MODE_FLOW_COMPACT: Emit using only the flow mode in as much as possible compact form + * @FYECF_MODE_JSON_COMPACT: Emit using JSON compact form * @FYECF_DOC_START_MARK_AUTO: Automatically generate document start markers if required * @FYECF_DOC_START_MARK_OFF: Do not generate document start markers * @FYECF_DOC_START_MARK_ON: Always generate document start markers @@ -1724,6 +1933,7 @@ enum fy_emitter_cfg_flags { FYECF_STRIP_DOC = FY_BIT(4), FYECF_NO_ENDING_NEWLINE = FY_BIT(5), FYECF_STRIP_EMPTY_KV = FY_BIT(6), + FYECF_EXTENDED_CFG = FY_BIT(7), FYECF_INDENT_DEFAULT = FYECF_INDENT(0), FYECF_INDENT_1 = FYECF_INDENT(1), FYECF_INDENT_2 = FYECF_INDENT(2), @@ -1748,6 +1958,8 @@ enum fy_emitter_cfg_flags { FYECF_MODE_DEJSON = FYECF_MODE(7), FYECF_MODE_PRETTY = FYECF_MODE(8), FYECF_MODE_MANUAL = FYECF_MODE(9), + FYECF_MODE_FLOW_COMPACT = FYECF_MODE(10), + FYECF_MODE_JSON_COMPACT = FYECF_MODE(11), FYECF_DOC_START_MARK_AUTO = FYECF_DOC_START_MARK(0), FYECF_DOC_START_MARK_OFF = FYECF_DOC_START_MARK(1), FYECF_DOC_START_MARK_ON = FYECF_DOC_START_MARK(2), @@ -1785,6 +1997,72 @@ struct fy_emitter_cfg { struct fy_diag *diag; }; +/* Shift amount of the color settings */ +#define FYEXCF_COLOR_SHIFT 0 +/* Mask of the color */ +#define FYEXCF_COLOR_MASK ((1U << 2) - 1) +/* Build a color version */ +#define FYEXCF_COLOR(x) (((unsigned int)(x) & FYEXCF_COLOR_MASK) << FYEXCF_COLOR_SHIFT) + +/* Shift amount of the output settings */ +#define FYEXCF_OUTPUT_SHIFT 2 +/* Mask of the output */ +#define FYEXCF_OUTPUT_MASK ((1U << 3) - 1) +/* Build an output */ +#define FYEXCF_OUTPUT(x) (((unsigned int)(x) & FYEXCF_OUTPUT_MASK) << FYEXCF_OUTPUT_SHIFT) + +/** + * enum fy_emitter_xcfg_flags - Emitter extended configuration flags + * + * These flags control the operation of the emitter. + * If no extended configuration mode is enabled all the flags are assumed 0. + * + * @FYEXCF_COLOR_AUTO: Automatically colorize if on a terminal + * @FYEXCF_COLOR_NONE: Do not colorize output + * @FYEXCF_COLOR_FORCE: Force color generation + * @FYEXCF_VISIBLE_WS: Make white space visible + * @FYEXCF_EXTENDED_INDICATORS: Extend the indicators generated + * @FYEXCF_OUTPUT_STDOUT: Output to stdout (default) + * @FYEXCF_OUTPUT_STDERR: Output to stderr, instead of stdout + * @FYEXCF_OUTPUT_FILE: Output to the fp FILE pointer + * @FYEXCF_OUTPUT_FD: Output to the fd file descriptor + * @FYEXCF_NULL_OUTPUT: No output + */ +enum fy_emitter_xcfg_flags { + FYEXCF_COLOR_AUTO = FYEXCF_COLOR(0), + FYEXCF_COLOR_NONE = FYEXCF_COLOR(1), + FYEXCF_COLOR_FORCE = FYEXCF_COLOR(2), + FYEXCF_OUTPUT_STDOUT = FYEXCF_OUTPUT(0), + FYEXCF_OUTPUT_STDERR = FYEXCF_OUTPUT(1), + FYEXCF_OUTPUT_FILE = FYEXCF_OUTPUT(2), + FYEXCF_OUTPUT_FD = FYEXCF_OUTPUT(2), + FYEXCF_NULL_OUTPUT = FYEXCF_OUTPUT(3), + FYEXCF_VISIBLE_WS = FY_BIT(5), + FYEXCF_EXTENDED_INDICATORS = FY_BIT(6), +}; + +/** + * struct fy_emitter_xcfg - emitter extended configuration structure. + * + * Argument to the fy_emitter create when FYECF_EXTENDED_CFG bit is + * set. + * + * @cfg: The standard emitter configuration + * @xflags: Extra configuration flags + * @colors: ANSI color overrides for the default output method + * @output_fp: The output FILE \*, FYEXCF\_FILE\_OUTPUT must be set + * @output_fd: The output file descriptor, FYEXCF\_FD\_OUTPUT must be set + */ +struct fy_emitter_xcfg { + struct fy_emitter_cfg cfg; + enum fy_emitter_xcfg_flags xflags; + const char *colors[FYEWT_COUNT]; + union { + FILE *output_fp; + int output_fd; + }; +}; + /** * fy_emitter_create() - Create an emitter * @@ -1979,7 +2257,7 @@ fy_emit_eventf(struct fy_emitter *emit, enum fy_event_type type, ...) * Queue and output using the emitter a scalar using a standard * write interface. * - * @emit: The emitter to use + * @fye: The emitter to use * @style: The scalar style to use * @anchor: The anchor or NULL * @tag: The tag or NULL @@ -2001,7 +2279,7 @@ fy_emit_scalar_write(struct fy_emitter *fye, enum fy_scalar_style style, * Queue and output using the emitter a scalar using a standard * vprintf interface. * - * @emit: The emitter to use + * @fye: The emitter to use * @style: The scalar style to use * @anchor: The anchor or NULL * @tag: The tag or NULL @@ -2023,7 +2301,7 @@ fy_emit_scalar_vprintf(struct fy_emitter *fye, enum fy_scalar_style style, * Queue and output using the emitter a scalar using a standard * printf interface. * - * @emit: The emitter to use + * @fye: The emitter to use * @style: The scalar style to use * @anchor: The anchor or NULL * @tag: The tag or NULL @@ -2139,6 +2417,23 @@ int fy_emit_root_node(struct fy_emitter *emit, struct fy_node *fyn) FY_EXPORT; +/** + * fy_emit_body_node() - Emit a single body node using the emitter + * + * Emits a single body node using the emitter. This is used in case + * you need finer control over the emitting output. + * No stream and document events will be generated at all. + * + * @emit: The emitter + * @fyn: The body node to emit + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_emit_body_node(struct fy_emitter *emit, struct fy_node *fyn) + FY_EXPORT; + /** * fy_emit_explicit_document_end() - Emit an explicit document end * @@ -2361,6 +2656,55 @@ char * fy_emit_to_string_collect(struct fy_emitter *emit, size_t *sizep) FY_EXPORT; +/** + * fy_emitter_vlog() - Log using the emitters diagnostics printf style (va_arg) + * + * Output a log on the emitter diagnostic output + * + * @emit: The emitter + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_emitter_vlog(struct fy_emitter *emit, enum fy_error_type type, const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_emitter_log() - Log using the emitters diagnostics printf style + * + * Output a report on the emitter's diagnostics + * + * @emit: The emitter + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments + */ +void +fy_emitter_log(struct fy_emitter *emit, enum fy_error_type type, const char *fmt, ...) + FY_FORMAT(printf, 3, 4) + FY_EXPORT; + +#ifndef NDEBUG + +#define fy_emitter_debug(_emit, _fmt, ...) \ + fy_emitter_log((_emit), FYET_DEBUG, (_fmt) , ## __VA_ARGS__) +#else + +#define fy_emitter_debug(_emit, _fmt, ...) \ + do { } while(0) + +#endif + +#define fy_emitter_info(_emit, _fmt, ...) \ + fy_emitter_log((_emit), FYET_INFO, (_fmt) , ## __VA_ARGS__) +#define fy_emitter_notice(_emit, _fmt, ...) \ + fy_emitter_log((_emit), FYET_NOTICE, (_fmt) , ## __VA_ARGS__) +#define fy_emitter_warning(_emit, _fmt, ...) \ + fy_emitter_log((_emit), FYET_WARNING, (_fmt) , ## __VA_ARGS__) +#define fy_emitter_error(_emit, _fmt, ...) \ + fy_emitter_log((_emit), FYET_ERROR, (_fmt) , ## __VA_ARGS__) + /** * fy_node_copy() - Copy a node, associating the new node with the given document * @@ -2446,6 +2790,29 @@ int fy_node_insert(struct fy_node *fyn_to, struct fy_node *fyn_from) FY_EXPORT; +/** + * fy_node_delete() - Delete a node from a document + * + * Delete the given node. + * If it's part of a sequence it will be removed from it. + * If it's the value of a node key value pair, it will + * be removed from the mapping. + * + * This is an alias for fy_document_insert_at(fyn, NULL) + * + * Note that it is expected this node is attached to a document. + * Do not call this to free a node, because if it's part of + * a collection it will not be properly removed. + * + * @fyn: The node to delete. + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_node_delete(struct fy_node *fyn) + FY_EXPORT; + /** * fy_document_insert_at() - Insert a node to the given path in the document * @@ -2935,14 +3302,12 @@ fy_flow_document_build_from_string(const struct fy_parse_cfg *cfg, * The document is a block document, and it terminates when indentation * appears to do so. * - * Example of block documents: + * Example of block documents:: * - * this-is-yaml - * foo: bar <- starts here - * baz: - * - 1 - * - 2 - * this-is-yaml-no-more + * this-is-yaml + * foo: bar <- starts here + * baz: [1, 2] + * this-is-yaml-no-more * * @cfg: The parse configuration to use or NULL for the default. * @str: The YAML source to use. @@ -3351,18 +3716,18 @@ fy_node_buildf(struct fy_document *fyd, const char *fmt, ...) * mappings, or indexes in brackets for traversing sequences. * Path specs may start with '/' which is silently ignored. * - * A few examples will make this clear + * A few examples will make this clear:: * - * fyn = { foo: bar } - fy_node_by_path(fyn, "/foo") -> bar - * fyn = [ foo, bar ] - fy_node_by_path(fyn, "1") -> bar - * fyn = { { foo: bar }: baz } - fy_node_by_path(fyn, "{foo: bar}") -> baz - * fyn = [ foo, { bar: baz } } - fy_node_by_path(fyn, "1/bar") -> baz + * fyn = { foo: bar } - fy_node_by_path(fyn, "/foo") -> bar + * fyn = [ foo, bar ] - fy_node_by_path(fyn, "1") -> bar + * fyn = { { foo: bar }: baz } - fy_node_by_path(fyn, "{foo: bar}") -> baz + * fyn = [ foo, { bar: baz } } - fy_node_by_path(fyn, "1/bar") -> baz * * Note that the special characters /{}[] are not escaped in plain style, * so you will not be able to use them as path traversal keys. - * In that case you can easily use either the single, or double quoted forms: + * In that case you can easily use either the single, or double quoted forms:: * - * fyn = { foo/bar: baz } - fy_node_by_path(fyn, "'foo/bar'") -> baz + * fyn = { foo/bar: baz } - fy_node_by_path(fyn, "'foo/bar'") -> baz * * @fyn: The node to use as start of the traversal operation * @path: The path spec to use in the traversal operation @@ -3449,8 +3814,8 @@ fy_node_get_parent_address(struct fy_node *fyn) FY_ALLOCA_COPY_FREE_NO_NULL(fy_node_get_parent_address((_fyn)), FY_NT) /** - * fy_node_get_path_relative_to() - Get a path address of a node - * relative to one of it's parents + * fy_node_get_path_relative_to() - Get a path address of a node relative + * to one of it's parents * * Retrieve the given node's path address relative to an arbitrary * parent in the tree. @@ -3471,22 +3836,23 @@ fy_node_get_path_relative_to(struct fy_node *fyn_parent, struct fy_node *fyn) FY_ALLOCA_COPY_FREE_NO_NULL(fy_node_get_path_relative_to((_fynp), (_fyn)), FY_NT) /** - * fy_node_get_short_path() - Get a path address of a node in the shortest - * path possible + * fy_node_get_short_path() - Get a path address of a node in the shortest path possible * - * Retrieve the given node's short path address relative to the - * closest anchor (either on this node, or it's parent). + * Retrieve the given nodes short path address relative to the + * closest anchor (either on this node, or its parent). * If no such parent is found then returns the absolute path * from the start of the document. * + * Example:: + * * --- &foo * foo: &bar * bar * baz * - * - The short path of /foo is \*foo - * - The short path of /foo/bar is \*bar - * - The short path of /baz is \*foo/baz + * - The short path of /foo is \*foo + * - The short path of /foo/bar is \*bar + * - The short path of /baz is \*foo/baz * * The address is dynamically allocated and should be freed when * you're done with it. @@ -3757,6 +4123,37 @@ const char * fy_node_get_tag(struct fy_node *fyn, size_t *lenp) FY_EXPORT; +/** + * fy_node_get_tag0() - Get the tag of the node + * + * This method will return a pointer to the text of a tag, + * which will be NULL terminated. + * + * @fyn: The node + * + * Returns: + * A pointer to the null terminated tag of the node. + * A NULL will be returned in case of an error. + */ +const char * +fy_node_get_tag0(struct fy_node *fyn) + FY_EXPORT; + +/** + * fy_node_get_tag_length() - Get the length of the tag of the node + * + * This method will return the size of the tag of the node. + * If the node is not tagged it will return 0. + * + * @fyn: The tagged node + * + * Returns: + * The size of the tag, or 0 if node is not tagged. + */ +size_t +fy_node_get_tag_length(struct fy_node *fyn) + FY_EXPORT; + /** * fy_node_get_scalar() - Get the scalar content of the node * @@ -5191,6 +5588,45 @@ fy_node_override_report(struct fy_node *fyn, enum fy_error_type type, FY_FORMAT(printf, 6, 7) FY_EXPORT; +/** + * fy_event_vreport() - Report about an event vprintf style + * + * Output a report about the given event via the specific + * error type, focusing at the given event part. + * + * @fyp: The parser of which the event was generated from + * @fye: The event + * @fyep: The event part which the report is about + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_event_vreport(struct fy_parser *fyp, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_event_report() - Report about an event printf style + * + * Output a report about the given event via the specific + * error type, focusing at the given event part. + * + * @fyp: The parser of which the event was generated from + * @fye: The event + * @fyep: The event part which the report is about + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_event_report(struct fy_parser *fyp, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, ...) + FY_FORMAT(printf, 5, 6) + FY_EXPORT; + typedef void (*fy_diag_output_fn)(struct fy_diag *diag, void *user, const char *buf, size_t len); @@ -5371,6 +5807,18 @@ bool fy_diag_got_error(struct fy_diag *diag) FY_EXPORT; +/** + * fy_diag_set_error() - Sets the error produced state + * + * Sets the error produced state + * + * @diag: The diagnostic object + * @on_error: The set error state + */ +void +fy_diag_set_error(struct fy_diag *diag, bool on_error) + FY_EXPORT; + /** * fy_diag_reset_error() - Reset the error flag of * the diagnostic object @@ -5600,7 +6048,7 @@ fy_diag_token_vreport(struct fy_diag *diag, struct fy_token *fyt, * document. * * @diag: The diag object - * @fyt: The token + * @fye: The token * @type: The error type * @fmt: The printf format string * @...: The extra arguments. @@ -5761,27 +6209,125 @@ fy_diag_node_override_report(struct fy_diag *diag, struct fy_node *fyn, FY_EXPORT; /** - * fy_diag_errors_iterate() - Iterate over the errors of a diagnostic object - * - * This method iterates over all the errors collected on the diagnostic object. - * The start of the iteration is signalled by a NULL in \*prevp. + * fy_diag_event_vreport() - Report about an event vprintf style using + * the given diagnostic object * - * @diag: The diagnostic object - * @prevp: The previous result iterator + * Output a report about the given event part via the specific + * error type. * - * Returns: - * The next errors or NULL when there are not any more. + * @diag: The diag object + * @fye: The event + * @fyep: The event part + * @type: The error type + * @fmt: The printf format string + * @ap: The argument list */ -struct fy_diag_error * -fy_diag_errors_iterate(struct fy_diag *diag, void **prevp) +void +fy_diag_event_vreport(struct fy_diag *diag, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, va_list ap) FY_EXPORT; /** - * enum fy_path_parse_cfg_flags - Path parse configuration flags + * fy_diag_event_report() - Report about a event printf style using + * the given diagnostic object * - * These flags control the operation of the path parse + * Output a report about the given event part via the specific + * error type. * - * @FYPPCF_QUIET: Quiet, do not output any information messages + * @diag: The diag object + * @fye: The event + * @fyep: The event part + * @type: The error type + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_diag_event_report(struct fy_diag *diag, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, ...) + FY_FORMAT(printf, 5, 6) + FY_EXPORT; + +/** + * fy_diag_event_override_vreport() - Report about a token vprintf style, + * overriding file, line and column info using + * the given diagnostic object + * + * Output a report about the given event part via the specific + * error type. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @diag: The diag object + * @fye: The event + * @fyep: The event part + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @ap: The argument list + */ +void +fy_diag_event_override_vreport(struct fy_diag *diag, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *file, int line, int column, + const char *fmt, va_list ap) + FY_EXPORT; + +/** + * fy_diag_event_override_report() - Report about a token printf style, + * overriding file, line and column info using + * the given diagnostic object + * + * Output a report about the given event part via the specific + * error type. This method will use the overrides provided in order + * to massage the reporting information. + * If @file is NULL, no file location will be reported. + * If either @line or @column is negative no location will be reported. + * + * @diag: The diag object + * @fyt: The event + * @fyep: The event part + * @type: The error type + * @file: The file override + * @line: The line override + * @column: The column override + * @fmt: The printf format string + * @...: The extra arguments. + */ +void +fy_diag_event_override_report(struct fy_diag *diag, struct fy_token *fyt, + enum fy_event_part fyep, enum fy_error_type type, + const char *file, int line, int column, + const char *fmt, ...) + FY_FORMAT(printf, 8, 9) + FY_EXPORT; + +/** + * fy_diag_errors_iterate() - Iterate over the errors of a diagnostic object + * + * This method iterates over all the errors collected on the diagnostic object. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * @diag: The diagnostic object + * @prevp: The previous result iterator + * + * Returns: + * The next errors or NULL when there are not any more. + */ +struct fy_diag_error * +fy_diag_errors_iterate(struct fy_diag *diag, void **prevp) + FY_EXPORT; + +/** + * enum fy_path_parse_cfg_flags - Path parse configuration flags + * + * These flags control the operation of the path parse + * + * @FYPPCF_QUIET: Quiet, do not output any information messages * @FYPPCF_DISABLE_RECYCLING: Disable recycling optimization * @FYPPCF_DISABLE_ACCELERATORS: Disable use of access accelerators (saves memory) */ @@ -5813,6 +6359,7 @@ struct fy_path_parse_cfg { * Creates a path parser with its configuration @cfg * The path parser may be destroyed by a corresponding call to * fy_path_parser_destroy(). + * If @cfg is NULL a default yaml parser is created. * * @cfg: The configuration for the path parser * @@ -6051,64 +6598,64 @@ fy_path_exec_results_iterate(struct fy_path_exec *fypx, void **prevp) /** * enum fy_token_type - Token types * - * The available token types that the tokenizer produces. - * * @FYTT_NONE: No token - * @FYTT_STREAM_START: Stream start - * @FYTT_STREAM_END: Stream end - * @FYTT_VERSION_DIRECTIVE: Version directive - * @FYTT_TAG_DIRECTIVE: Tag directive - * @FYTT_DOCUMENT_START: Document start - * @FYTT_DOCUMENT_END: Document end - * @FYTT_BLOCK_SEQUENCE_START: Start of a block sequence - * @FYTT_BLOCK_MAPPING_START: Start of a block mapping - * @FYTT_BLOCK_END: End of a block mapping or a sequence - * @FYTT_FLOW_SEQUENCE_START: Start of a flow sequence - * @FYTT_FLOW_SEQUENCE_END: End of a flow sequence - * @FYTT_FLOW_MAPPING_START: Start of a flow mapping - * @FYTT_FLOW_MAPPING_END: End of a flow mapping - * @FYTT_BLOCK_ENTRY: A block entry - * @FYTT_FLOW_ENTRY: A flow entry - * @FYTT_KEY: A key of a mapping - * @FYTT_VALUE: A value of a mapping - * @FYTT_ALIAS: An alias - * @FYTT_ANCHOR: An anchor - * @FYTT_TAG: A tag - * @FYTT_SCALAR: A scalar - * @FYTT_INPUT_MARKER: Internal input marker token - * @FYTT_PE_SLASH: A slash - * @FYTT_PE_ROOT: A root - * @FYTT_PE_THIS: A this - * @FYTT_PE_PARENT: A parent - * @FYTT_PE_MAP_KEY: A map key - * @FYTT_PE_SEQ_INDEX: A sequence index - * @FYTT_PE_SEQ_SLICE: A sequence slice - * @FYTT_PE_SCALAR_FILTER: A scalar filter - * @FYTT_PE_COLLECTION_FILTER: A collection filter - * @FYTT_PE_SEQ_FILTER: A sequence filter - * @FYTT_PE_MAP_FILTER: A mapping filter - * @FYTT_PE_UNIQUE_FILTER: Filters out duplicates - * @FYTT_PE_EVERY_CHILD: Every child - * @FYTT_PE_EVERY_CHILD_R: Every child recursive - * @FYTT_PE_ALIAS: An alias - * @FYTT_PE_SIBLING: A sibling marker - * @FYTT_PE_COMMA: A comma - * @FYTT_PE_BARBAR: A || - * @FYTT_PE_AMPAMP: A && - * @FYTT_PE_LPAREN: A left parenthesis - * @FYTT_PE_RPAREN: A right parenthesis - * @FYTT_PE_EQEQ: Equality operator - * @FYTT_PE_NOTEQ: Non-equality operator - * @FYTT_PE_LT: Less than operator - * @FYTT_PE_GT: Greater than operator - * @FYTT_PE_LTE: Less or equal than operator - * @FYTT_PE_GTE: Greater or equal than operator - * @FYTT_SE_PLUS: Plus operator - * @FYTT_SE_MINUS: Minus operator - * @FYTT_SE_MULT: Multiply operator - * @FYTT_SE_DIV: Divide operator - * @FYTT_PE_METHOD: Path expression method (chained) - * @FYTT_SE_METHOD: Scalar expression method (non chained) + * @FYTT_STREAM_START: Stream start token + * @FYTT_STREAM_END: Stream end token + * @FYTT_VERSION_DIRECTIVE: Version directive token + * @FYTT_TAG_DIRECTIVE: Tag directive token + * @FYTT_DOCUMENT_START: Document start token + * @FYTT_DOCUMENT_END: Document end token + * @FYTT_BLOCK_SEQUENCE_START: Block sequence start token + * @FYTT_BLOCK_MAPPING_START: Block mapping start token + * @FYTT_BLOCK_END: Block end token + * @FYTT_FLOW_SEQUENCE_START: Flow sequence start token + * @FYTT_FLOW_SEQUENCE_END: Flow sequence end token + * @FYTT_FLOW_MAPPING_START: Flow mapping start token + * @FYTT_FLOW_MAPPING_END: Flow mapping end token + * @FYTT_BLOCK_ENTRY: Block entry token + * @FYTT_FLOW_ENTRY: Flow entry token + * @FYTT_KEY: Key token + * @FYTT_VALUE: Value token + * @FYTT_ALIAS: Alias token + * @FYTT_ANCHOR: Anchor token + * @FYTT_TAG: Tag token + * @FYTT_SCALAR: Scalar token + * @FYTT_INPUT_MARKER: Input marker token + * @FYTT_PE_SLASH: Path expression slash token + * @FYTT_PE_ROOT: Path expression root token + * @FYTT_PE_THIS: Path expression this token + * @FYTT_PE_PARENT: Path expression parent token + * @FYTT_PE_MAP_KEY: Path expression map key token + * @FYTT_PE_SEQ_INDEX: Path expression sequence index token + * @FYTT_PE_SEQ_SLICE: Path expression sequence slice token + * @FYTT_PE_SCALAR_FILTER: Path expression scalar filter token + * @FYTT_PE_COLLECTION_FILTER: Path expression collection filter token + * @FYTT_PE_SEQ_FILTER: Path expression sequence filter token + * @FYTT_PE_MAP_FILTER: Path expression map filter token + * @FYTT_PE_UNIQUE_FILTER: Path expression unique filter token + * @FYTT_PE_EVERY_CHILD: Path expression every child token + * @FYTT_PE_EVERY_CHILD_R: Path expression every child recursive token + * @FYTT_PE_ALIAS: Path expression alias token + * @FYTT_PE_SIBLING: Path expression sibling token + * @FYTT_PE_COMMA: Path expression comma token + * @FYTT_PE_BARBAR: Path expression || token + * @FYTT_PE_AMPAMP: Path expression && token + * @FYTT_PE_LPAREN: Path expression ( token + * @FYTT_PE_RPAREN: Path expression ) token + * @FYTT_PE_EQEQ: Path expression == token + * @FYTT_PE_NOTEQ: Path expression != token + * @FYTT_PE_LT: Path expression < token + * @FYTT_PE_GT: Path expression > token + * @FYTT_PE_LTE: Path expression <= token + * @FYTT_PE_GTE: Path expression >= token + * @FYTT_SE_PLUS: Scalar expression + token + * @FYTT_SE_MINUS: Scalar expression - token + * @FYTT_SE_MULT: Scalar expression \* token + * @FYTT_SE_DIV: Scalar expression / token + * @FYTT_PE_METHOD: Path expression method token + * @FYTT_SE_METHOD: Scalar expression method token + * @FYTT_PE_BANG: Path expression ! token + * @FYTT_PE_AT: Path expression \@ token */ enum fy_token_type { /* non-content token types */ @@ -6163,12 +6710,12 @@ enum fy_token_type { FYTT_PE_RPAREN, /* comparison operators */ - FYTT_PE_EQEQ, /* == */ - FYTT_PE_NOTEQ, /* != */ - FYTT_PE_LT, /* < */ - FYTT_PE_GT, /* > */ - FYTT_PE_LTE, /* <= */ - FYTT_PE_GTE, /* >= */ + FYTT_PE_EQEQ, + FYTT_PE_NOTEQ, + FYTT_PE_LT, + FYTT_PE_GT, + FYTT_PE_LTE, + FYTT_PE_GTE, /* scalar expression tokens */ FYTT_SE_PLUS, @@ -6176,12 +6723,15 @@ enum fy_token_type { FYTT_SE_MULT, FYTT_SE_DIV, - FYTT_PE_METHOD, /* path expr method (chained) */ - FYTT_SE_METHOD, /* scalar expr method (non chained) */ + FYTT_PE_METHOD, + FYTT_SE_METHOD, + + FYTT_PE_BANG, + FYTT_PE_AT, }; /* The number of token types available */ -#define FYTT_COUNT (FYTT_SE_METHOD+1) +#define FYTT_COUNT (FYTT_PE_AT+1) /** * fy_token_type_is_valid() - Check token type validity @@ -7564,7 +8114,7 @@ fy_path_component_set_sequence_user_data(struct fy_path_component *fypc, void *d /** * fy_path_get_parent_user_data() - Return the userdata of the parent collection * - * @fypp: The path + * @path: The path * * Returns: * The user data associated with the parent collection of the path, or NULL if no path @@ -7578,7 +8128,7 @@ fy_path_get_parent_user_data(struct fy_path *path) * * Note, no error condition if not a path * - * @fypp: The path + * @path: The path * @data: The data to set as parent collection data */ void @@ -7588,7 +8138,7 @@ fy_path_set_parent_user_data(struct fy_path *path, void *data) /** * fy_path_get_last_user_data() - Return the userdata of the last collection * - * @fypp: The path + * @path: The path * * Returns: * The user data associated with the last collection of the path, or NULL if no path @@ -7602,7 +8152,7 @@ fy_path_get_last_user_data(struct fy_path *path) * * Note, no error condition if not a path * - * @fypp: The path + * @path: The path * @data: The data to set as last collection data */ void @@ -7642,6 +8192,204 @@ struct fy_path_component * fy_path_last_not_collection_root_component(struct fy_path *fypp) FY_EXPORT; +/** + * struct fy_composer_ops - Composer operation callbacks + * + * Callbacks used by the composer to process events and create document builders. + * + * @process_event: Callback for processing a single YAML event with path context + * @create_document_builder: Callback for creating a document builder instance + */ +struct fy_composer_ops { + /* single process event callback */ + enum fy_composer_return (*process_event)(struct fy_composer *fyc, struct fy_path *path, struct fy_event *fye); + struct fy_document_builder *(*create_document_builder)(struct fy_composer *fyc); +}; + +/** + * struct fy_composer_cfg - Composer configuration structure + * + * Configuration structure for creating a composer instance. + * + * @ops: Pointer to composer operation callbacks + * @userdata: Opaque user data pointer passed to callbacks + * @diag: Optional diagnostic interface to use, NULL for default + */ +struct fy_composer_cfg { + const struct fy_composer_ops *ops; + void *userdata; + struct fy_diag *diag; +}; + +/** + * fy_composer_create() - Create a composer + * + * Creates a composer with the given configuration. The composer processes + * YAML events using callback methods and maintains path information for + * intelligent document composition. The composer may be destroyed by a + * corresponding call to fy_composer_destroy(). + * + * @cfg: The configuration for the composer + * + * Returns: + * A pointer to the composer or NULL in case of an error. + */ +struct fy_composer * +fy_composer_create(struct fy_composer_cfg *cfg) + FY_EXPORT; + +/** + * fy_composer_destroy() - Destroy the given composer + * + * Destroy a composer created earlier via fy_composer_create(). + * + * @fyc: The composer to destroy + */ +void fy_composer_destroy(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_process_event() - Process a YAML event through the composer + * + * Process a YAML event by calling the configured process_event callback + * with path context. The composer maintains the current path and provides + * it to the callback for intelligent processing decisions. + * + * @fyc: The composer + * @fye: The event to process + * + * Returns: + * A fy_composer_return code indicating how to proceed (continue, stop, skip, or error) + */ +enum fy_composer_return +fy_composer_process_event(struct fy_composer *fyc, struct fy_event *fye) + FY_EXPORT; + +/** + * fy_composer_get_cfg() - Get the configuration of a composer + * + * @fyc: The composer + * + * Returns: + * The configuration of the composer + */ +struct fy_composer_cfg * +fy_composer_get_cfg(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_get_cfg_userdata() - Get the userdata from composer configuration + * + * Retrieve the opaque userdata pointer from the composer's configuration. + * + * @fyc: The composer + * + * Returns: + * The userdata pointer from the configuration + */ +void * +fy_composer_get_cfg_userdata(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_get_diag() - Get the diagnostic object of a composer + * + * Return a pointer to the diagnostic object of a composer object. + * Note that the returned diag object has a reference taken so + * you should fy_diag_unref() it when you're done with it. + * + * @fyc: The composer to get the diagnostic object + * + * Returns: + * A pointer to a ref'ed diagnostic object or NULL in case of an error. + */ +struct fy_diag * +fy_composer_get_diag(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_get_path() - Get the current path of the composer + * + * Retrieve the current path being processed by the composer. + * The path represents the location in the YAML document structure + * where the composer is currently positioned. + * + * @fyc: The composer + * + * Returns: + * The current path, or NULL if no path is active + */ +struct fy_path * +fy_composer_get_path(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_get_root_path() - Get the root path of the composer + * + * Retrieve the root path of the composer's path hierarchy. + * + * @fyc: The composer + * + * Returns: + * The root path, or NULL if no root exists + */ +struct fy_path * +fy_composer_get_root_path(struct fy_composer *fyc) + FY_EXPORT; + +/** + * fy_composer_get_next_path() - Get the next path in the composer's path list + * + * Iterate through the composer's path list. Pass NULL to get the first path, + * or pass the previous path to get the next one. + * + * @fyc: The composer + * @fypp: The previous path, or NULL to get the first path + * + * Returns: + * The next path in the list, or NULL if no more paths exist + */ +struct fy_path * +fy_composer_get_next_path(struct fy_composer *fyc, struct fy_path *fypp) + FY_EXPORT; + +/* Shift amount of the want mode */ +#define FYDICF_WANT_SHIFT 0 +/* Mask of the WANT mode */ +#define FYDICF_WANT_MASK ((1U << 2) - 1) +/* Build a WANT mode option */ +#define FYDICF_WANT(x) (((unsigned int)(x) & FYDICF_WANT_MASK) << FYDICF_WANT_SHIFT) + +/** + * enum fy_document_iterator_cfg_flags - Document iterator configuration flags + * + * These flags control the operation of the document iterator + * + * @FYDICF_WANT_BODY_EVENTS: Generate body events + * @FYDICF_WANT_DOCUMENT_BODY_EVENTS: Generate document and body events + * @FYDICF_WANT_STREAM_DOCUMENT_BODY_EVENTS: Generate stream, document and body events + */ +enum fy_document_iterator_cfg_flags { + FYDICF_WANT_BODY_EVENTS = FYDICF_WANT(0), + FYDICF_WANT_DOCUMENT_BODY_EVENTS = FYDICF_WANT(1), + FYDICF_WANT_STREAM_DOCUMENT_BODY_EVENTS = FYDICF_WANT(2), +}; + +/** + * struct fy_document_iterator_cfg - document iterator configuration structure. + * + * Argument to the fy_document_iterator_create_cfg() method. + * + * @flags: The document iterator flags + * @fyd: The document to iterate on (or NULL if iterate_root is set) + * @iterate_root: The root of iteration (NULL when fyd is not NULL) + */ +struct fy_document_iterator_cfg { + enum fy_document_iterator_cfg_flags flags; + struct fy_document *fyd; + struct fy_node *iterate_root; +}; + /** * fy_document_iterator_create() - Create a document iterator * @@ -7655,6 +8403,50 @@ struct fy_document_iterator * fy_document_iterator_create(void) FY_EXPORT; +/** + * fy_document_iterator_create_cfg() - Create a document iterator using config + * + * Creates a document iterator, that can trawl through a document + * without using recursion. The iterator will generate all the events + * that created the given document starting at iterator root. + * + * @cfg: The document iterator to destroy + * + * Returns: + * The newly created document iterator or NULL on error + */ +struct fy_document_iterator * +fy_document_iterator_create_cfg(const struct fy_document_iterator_cfg *cfg) + FY_EXPORT; + +/** + * fy_document_iterator_create_on_document() - Create a document iterator on document + * + * Creates a document iterator, starting at the root of the given document. + * + * @fyd: The document to iterate on + * + * Returns: + * The newly created document iterator or NULL on error + */ +struct fy_document_iterator * +fy_document_iterator_create_on_document(struct fy_document *fyd) + FY_EXPORT; + +/** + * fy_document_iterator_create_on_node() - Create a document iterator on node + * + * Creates a document iterator, starting at the given node + * + * @fyn: The node to iterate on + * + * Returns: + * The newly created document iterator or NULL on error + */ +struct fy_document_iterator * +fy_document_iterator_create_on_node(struct fy_node *fyn) + FY_EXPORT; + /** * fy_document_iterator_destroy() - Destroy the given document iterator * @@ -7794,6 +8586,24 @@ struct fy_node * fy_document_iterator_node_next(struct fy_document_iterator *fydi) FY_EXPORT; +/** + * fy_document_iterator_generate_next() - Create events from document iterator + * + * This is a method that will handle the complex state of generating + * stream, document and body events on the given iterator. + * + * When generation is complete a NULL event will be generated. + * + * @fydi: The document iterator to create the event + * + * Returns: + * The newly created event or NULL at an error, or an end of the + * event stream. Use fy_document_iterator_get_error() to check if an error occured. + */ +struct fy_event * +fy_document_iterator_generate_next(struct fy_document_iterator *fydi) + FY_EXPORT; + /** * fy_document_iterator_get_error() - Get the error state of the document iterator * @@ -7809,57 +8619,321 @@ bool fy_document_iterator_get_error(struct fy_document_iterator *fydi) FY_EXPORT; -/* - * The libfyaml's push-pull thread implementation +/** + * struct fy_document_builder_cfg - document builder configuration structure. * + * Argument to the fy_document_builder_create() method + * + * @parse_cfg: Parser configuration + * @userdata: Opaque user data pointer + * @diag: Optional diagnostic interface to use */ - -/* opaque types for the user */ -struct fy_thread_pool; -struct fy_thread; -struct fy_work_pool; +struct fy_document_builder_cfg { + struct fy_parse_cfg parse_cfg; + void *userdata; + struct fy_diag *diag; +}; /** - * typedef fy_work_exec_fn - Work exec function + * fy_document_builder_create() - Create a document builder * - * The callback executed on work submission + * Creates a document builder with its configuration @cfg + * The document builder may be destroyed by a corresponding call to + * fy_document_builder_destroy(). * - * @arg: The argument to the method + * @cfg: The configuration for the document builder * + * Returns: + * A pointer to the document builder or NULL in case of an error. */ -typedef void (*fy_work_exec_fn)(void *arg); +struct fy_document_builder * +fy_document_builder_create(const struct fy_document_builder_cfg *cfg) + FY_EXPORT; /** - * typedef fy_work_check_fn - Work check function + * fy_document_builder_create_on_parser() - Create a document builder + * pulling state from the parser * - * Work checker function to decide if it's worth to - * offload to a thread. + * Creates a document builder pulling state from the given parser * - * @arg: The argument to the method + * @fyp: The parser to associate with * * Returns: - * true if it should offload to thread, false otherwise - * + * A pointer to the document builder or NULL in case of an error. */ -typedef bool (*fy_work_check_fn)(const void *arg); +struct fy_document_builder * +fy_document_builder_create_on_parser(struct fy_parser *fyp) + FY_EXPORT; /** - * struct fy_thread_work - Work submitted to a thread for execution + * fy_document_builder_reset() - Reset a document builder * - * @fn: The execution function for this work - * @arg: The argument to the fn - * @wp: Used internally, must be set to NULL on entry + * Resets a document builder without destroying it * - * This is the structure describing the work submitted - * to a thread for execution. + * @fydb: The document builder */ -struct fy_thread_work { +void +fy_document_builder_reset(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_destroy() - Destroy a document builder + * + * Destroy a document builder + * + * @fydb: The document builder + */ +void +fy_document_builder_destroy(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_get_document() - Get the document of a builder + * + * Retrieve the document of a document builder. This document + * may be incomplete. If you need to take ownership use + * fy_document_builder_take_document(). + * + * @fydb: The document builder + * + * Returns: + * The document that the builder built, or NULL in case of an error + */ +struct fy_document * +fy_document_builder_get_document(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_is_in_stream() - Test document builder in stream + * + * Find out if the document builder is in 'stream' state, + * i.e. after stream start but before stream end events are generated. + * + * @fydb: The document builder + * + * Returns: + * true if in stream, false otherwise + */ +bool +fy_document_builder_is_in_stream(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_is_in_document() - Test document builder in document + * + * Find out if the document builder is in 'document' state, + * i.e. after document start but before document end events are generated. + * + * @fydb: The document builder + * + * Returns: + * true if in document, false otherwise + */ +bool +fy_document_builder_is_in_document(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_is_document_complete() - Test document builder complete + * + * Find out if the document of the builder is complete. + * If it is complete then a call to fy_document_builder_take_document() will + * transfer ownership of the document to the caller. + * + * @fydb: The document builder + * + * Returns: + * true if document complete, false otherwise + */ +bool +fy_document_builder_is_document_complete(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_take_document() - Take ownership the document of a builder + * + * Take ownership of the document of a document builder. + * The document builder's document must be complete. + * + * @fydb: The document builder + * + * Returns: + * The document that the builder built, or NULL in case of an error + */ +struct fy_document * +fy_document_builder_take_document(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_peek_document() - Peek at the document of a builder + * + * Peek at the document of a document builder. + * Ownership still remains with the builder. + * + * @fydb: The document builder + * + * Returns: + * A peek to the document that the builder built, or NULL in case of an error + */ +struct fy_document * +fy_document_builder_peek_document(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_set_in_stream() - Set the builders state in 'stream' + * + * Set the document builders state to in 'stream' + * + * @fydb: The document builder + */ +void +fy_document_builder_set_in_stream(struct fy_document_builder *fydb) + FY_EXPORT; + +/** + * fy_document_builder_set_in_document() - Set the builders state in 'document' + * + * Set the document builders state to in 'document' + * + * @fydb: The document builder + * @fyds: The document state + * @single: Single document mode + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_document_builder_set_in_document(struct fy_document_builder *fydb, struct fy_document_state *fyds, bool single) + FY_EXPORT; + +/** + * fy_document_builder_load_document() - Create a document from parser events + * + * Load a document by pumping the parser for events and then processing them + * with the builder. + * + * @fydb: The document builder + * @fyp: The parser + * + * Returns: + * The document that results from the parser, or NULL in case of an error (or EOF) + */ +struct fy_document * +fy_document_builder_load_document(struct fy_document_builder *fydb, struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parse_load_document_with_builder() - Parse a document via built-in builder + * + * Load a document by pumping the parser for events and then processing them + * with the in-parser builder. + * + * @fyp: The parser + * + * Returns: + * The document that results from the parser, or NULL in case of an error (or EOF) + */ +struct fy_document * +fy_parse_load_document_with_builder(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_document_builder_process_event() - Process an event with a builder + * + * Pump an event to a document builder for processing. + * + * @fydb: The document builder + * @fye: The event + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_document_builder_process_event(struct fy_document_builder *fydb, struct fy_event *fye) + FY_EXPORT; + +/** + * enum fy_parser_event_generator_flags - The parser event generator flags + * + * @FYPEGF_GENERATE_DOCUMENT_EVENTS: generate document events + * @FYPEGF_GENERATE_STREAM_EVENTS: generate stream events + * @FYPEGF_GENERATE_ALL_EVENTS: generate all events + */ +enum fy_parser_event_generator_flags { + FYPEGF_GENERATE_DOCUMENT_EVENTS = FY_BIT(0), + FYPEGF_GENERATE_STREAM_EVENTS = FY_BIT(1), + FYPEGF_GENERATE_ALL_EVENTS = FYPEGF_GENERATE_STREAM_EVENTS | FYPEGF_GENERATE_DOCUMENT_EVENTS, +}; + +/** + * fy_parser_set_document_iterator() - Associate a parser with a document iterator + * + * Associate a parser with a document iterator, that is instead of parsing the events + * will be generated by the document iterator. + * + * @fyp: The parser + * @flags: The event generation flags + * @fydi: The document iterator to associate + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_parser_set_document_iterator(struct fy_parser *fyp, enum fy_parser_event_generator_flags flags, + struct fy_document_iterator *fydi) + FY_EXPORT; + +/* + * The libfyaml's push-pull thread implementation + * + */ + +/* opaque types for the user */ +struct fy_thread_pool; +struct fy_thread; +struct fy_work_pool; + +/** + * typedef fy_work_exec_fn - Work exec function + * + * The callback executed on work submission + * + * @arg: The argument to the method + * + */ +typedef void (*fy_work_exec_fn)(void *arg); + +/** + * typedef fy_work_check_fn - Work check function + * + * Work checker function to decide if it's worth to + * offload to a thread. + * + * @arg: The argument to the method + * + * Returns: + * true if it should offload to thread, false otherwise + * + */ +typedef bool (*fy_work_check_fn)(const void *arg); + +/** + * struct fy_thread_work - Work submitted to a thread for execution + * + * @fn: The execution function for this work + * @arg: The argument to the fn + * @wp: Used internally, must be set to NULL on entry + * + * This is the structure describing the work submitted + * to a thread for execution. + */ +struct fy_thread_work { fy_work_exec_fn fn; void *arg; struct fy_work_pool *wp; }; -/* +/** * enum fy_thread_pool_cfg_flags - Thread pool configuration flags * * These flags control the operation of the thread pool. @@ -7942,7 +9016,7 @@ const struct fy_thread_pool_cfg * fy_thread_pool_get_cfg(struct fy_thread_pool *tp) FY_EXPORT; -/* +/** * fy_thread_reserve() - Reserve a thread from the pool. * * Reserve a thread from the pool and return it. @@ -7958,7 +9032,7 @@ struct fy_thread * fy_thread_reserve(struct fy_thread_pool *tp) FY_EXPORT; -/* +/** * fy_thread_unreserve() - Unreserve a previously reserved thread * * Unreserve a thread previously reserved via a call to fy_thread_reserve() @@ -7970,7 +9044,7 @@ void fy_thread_unreserve(struct fy_thread *t) FY_EXPORT; -/* +/** * fy_thread_submit_work() - Submit work for execution * * Submit work for execution. If successful the thread @@ -7991,7 +9065,7 @@ int fy_thread_submit_work(struct fy_thread *t, struct fy_thread_work *work) FY_EXPORT; -/* +/** * fy_thread_wait_work() - Wait for completion of submitted work * * Wait until submitted work to the thread has finished. @@ -8006,7 +9080,7 @@ int fy_thread_wait_work(struct fy_thread *t) FY_EXPORT; -/* +/** * fy_thread_work_join() - Submit works for execution and wait * * Submit works for possible parallel execution. If no offloading @@ -8025,7 +9099,7 @@ fy_thread_work_join(struct fy_thread_pool *tp, fy_work_check_fn check_fn) FY_EXPORT; -/* +/** * fy_thread_args_join() - Execute function in parallel using arguments as pointers * * Execute @fn possibly in parallel using the threads in the thread pool. @@ -8039,11 +9113,11 @@ fy_thread_work_join(struct fy_thread_pool *tp, */ void fy_thread_args_join(struct fy_thread_pool *tp, - fy_work_exec_fn, fy_work_check_fn check_fn, + fy_work_exec_fn fn, fy_work_check_fn check_fn, void **args, size_t count) FY_EXPORT; -/* +/** * fy_thread_arg_array_join() - Execute function in parallel using argument array * * Execute @fn possibly in parallel using the threads in the thread pool. @@ -8062,14 +9136,15 @@ fy_thread_arg_array_join(struct fy_thread_pool *tp, void *args, size_t argsize, size_t count) FY_EXPORT; -/* - * fy_thread_arg_array_join() - Execute function in parallel with the same argument +/** + * fy_thread_arg_join() - Execute function in parallel with the same argument * * Execute @fn possibly in parallel using the threads in the thread pool. * The argument of the functions is the same. * * @tp: The thread pool * @fn: The function to execute in parallel + * @check_fn: Pointer to a check function, or NULL for no checks * @arg: The common argument * @count: The count of executions */ @@ -8248,6 +9323,584 @@ const uint8_t * fy_blake3_hash_file(struct fy_blake3_hasher *fyh, const char *filename) FY_EXPORT; +/* forward decl of allocator interfaces */ +struct fy_allocator; + +/* A tag that represents the default tag */ +#define FY_ALLOC_TAG_DEFAULT 0 +/* A tag that denotes error */ +#define FY_ALLOC_TAG_ERROR -1 +/* A tag that represents 'none' */ +#define FY_ALLOC_TAG_NONE -2 + +/** + * fy_allocator_iterate() - Iterate over available allocator names + * + * This method iterates over all the available allocator names. + * The start of the iteration is signalled by a NULL in \*prevp. + * + * @prevp: The previous allocator iterator pointer + * + * Returns: + * The next allocator name in sequence or NULL at the end. + */ +const char * +fy_allocator_iterate(const char **prevp) + FY_EXPORT; + +/** + * fy_allocator_is_available() - Check if an allocator is available + * + * Check if the named allocator is available. + * + * @name: The name of the allocator to check + * + * Returns: + * true if the allocator is available, false otherwise + */ +bool +fy_allocator_is_available(const char *name) + FY_EXPORT; + +/** + * fy_allocator_create() - Create an allocator. + * + * Creates an allocator of the given type, using the configuration + * argument provided. + * The allocator may be destroyed by a corresponding call to + * fy_allocator_destroy(). + * + * You can retrieve the names of available allocators + * with the fy_allocator_get_names() method. + * + * @name: The name of the allocator + * @cfg: The type specific configuration for the allocator, or NULL + * for the default. + * + * Returns: + * A pointer to the allocator or NULL in case of an error. + */ +struct fy_allocator * +fy_allocator_create(const char *name, const void *cfg) + FY_EXPORT; + +/** + * fy_allocator_destroy() - Destroy the given allocator + * + * Destroy an allocator created earlier via fy_allocator_create(). + * Tracking allocators will release all memory allocated using them. + * + * @a: The allocator to destroy + */ +void +fy_allocator_destroy(struct fy_allocator *a) + FY_EXPORT; + +/** The minimum amount of memory for an inplace linear allocator */ +#define FY_LINEAR_ALLOCATOR_IN_PLACE_MIN_SIZE 256 + +/** + * fy_linear_allocator_create_in_place() - Create a linear allocator in place + * + * Creates a linear allocator in place, using the buffer provided. + * No memory allocations will be performed, so it's safe to embed. + * There is no need to call fy_allocator_destroy for this allocator. + * + * @buffer: The memory buffer to use for both storage and the allocator + * @size: The size of the memory buffer + * + * Returns: + * A pointer to the allocator, or NULL if there is no space + */ +struct fy_allocator * +fy_linear_allocator_create_in_place(void *buffer, size_t size) + FY_EXPORT; + +/** The minimum amount of memory for an inplace dedup allocator */ +#define FY_DEDUP_ALLOCATOR_IN_PLACE_MIN_SIZE 4096 + +/** + * fy_dedup_allocator_create_in_place() - Create a dedp allocator in place + * + * Creates a dedup allocator in place, using the buffer provided. + * No memory allocations will be performed, so it's safe to embed. + * There is no need to call fy_allocator_destroy for this allocator. + * The parent allocator of this will be a linear allocator. + * + * @buffer: The memory buffer to use for both storage and the allocator + * @size: The size of the memory buffer + * + * Returns: + * A pointer to the allocator, or NULL if there is no space + */ +struct fy_allocator * +fy_dedup_allocator_create_in_place(void *buffer, size_t size) + FY_EXPORT; + +/** + * fy_allocator_get_tag() - Get a tag from an allocator + * + * The allocator interface requires all allocation to belong + * to a tag. This call creates a tag and returns its value, + * or an error if not available. + * + * If an allocator only provides a single tag (like the linear + * allocator for instance), the same tag number, usually 0, is + * returned. + * + * @a: The allocator + * + * Returns: + * The created tag or -1 in case of an error. + */ +int +fy_allocator_get_tag(struct fy_allocator *a) + FY_EXPORT; + +/** + * fy_allocator_release_tag() - Release a tag from an allocator + * + * Releases a tag from an allocator and frees all memory it + * allocated (if such an operation is provided by the allocator). + * + * @a: The allocator + * @tag: The tag to release + */ +void +fy_allocator_release_tag(struct fy_allocator *a, int tag) + FY_EXPORT; + +/** + * fy_allocator_get_tag_count() - Get the maximum number of tags a + * allocator supports + * + * Get the maximum amount of tags an allocator supports. + * + * If an allocator only provides a single tag (like the linear + * allocator for instance), 1 will be returned. + * + * @a: The allocator + * + * Returns: + * The number of tags, or -1 on error + */ +int +fy_allocator_get_tag_count(struct fy_allocator *a) + FY_EXPORT; + +/** + * fy_allocator_set_tag_count() - Set the maximum number of tags a + * allocator supports + * + * Sets the maximum amount of tags an allocator supports. + * If the set allocator tag count is less than the current + * the additional tags will be released. + * + * @a: The allocator + * @count: The amount of tags the allocator should support + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_allocator_set_tag_count(struct fy_allocator *a, unsigned int count) + FY_EXPORT; + +/** + * fy_allocator_trim_tag() - Trim a tag + * + * Trim a tag, that is free any excess memory it allocator, fitting + * it to the size of the content it carries. + * Allocators that cannot perform this operation treat it as a NOP. + * + * @a: The allocator + * @tag: The tag to trim + */ +void +fy_allocator_trim_tag(struct fy_allocator *a, int tag) + FY_EXPORT; + +/** + * fy_allocator_reset_tag() - Reset a tag + * + * Reset a tag, that is free any content it carries, but do not + * release the tag. + * + * @a: The allocator + * @tag: The tag to reset + */ +void +fy_allocator_reset_tag(struct fy_allocator *a, int tag) + FY_EXPORT; + +/** + * fy_allocator_alloc() - Allocate memory from an allocator + * + * Allocate memory from the given allocator tag, satisfying the + * size and align restrictions. + * + * @a: The allocator + * @tag: The tag to allocate from + * @size: The size of the memory to allocate + * @align: The alignment of the object + * + * Returns: + * A pointer to the allocated memory or NULL + */ +void * +fy_allocator_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) + FY_EXPORT; + +/** + * fy_allocator_free() - Free memory allocated from an allocator + * + * Attempt to free the memory allocated previously by fy_allocator_alloc() + * Note that non per object tracking allocators treat this as a NOP + * + * @a: The allocator + * @tag: The tag used to allocate the memory + * @ptr: The pointer to the memory to free + */ +void +fy_allocator_free(struct fy_allocator *a, int tag, void *ptr) + FY_EXPORT; + +/** + * fy_allocator_store() - Store an object to an allocator + * + * Store an object to an allocator and return a pointer to the location + * it was stored. When using a deduplicating allocator no new allocation + * will take place and a pointer to the object already stored will be + * returned. + * + * The return pointer must not be modified, the objects stored are idempotent. + * + * @a: The allocator + * @tag: The tag used to allocate the memory + * @data: The pointer to object to store + * @size: The size of the object + * @align: The alignment restriction of the object + * + * Returns: + * A constant pointer to the object stored, or NULL in case of an error + */ +const void * +fy_allocator_store(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) + FY_EXPORT; + +/** + * fy_allocator_storev() - Store an object to an allocator (scatter gather) + * + * Store an object to an allocator and return a pointer to the location + * it was stored. When using a deduplicating allocator no new allocation + * will take place and a pointer to the object already stored will be + * returned. + * + * The object is created linearly from the scatter gather io vector provided. + * + * The return pointer must not be modified, the objects stored are immutable. + * + * @a: The allocator + * @tag: The tag used to allocate the memory from + * @iov: The I/O scatter gather vector + * @iovcnt: The number of vectors + * @align: The alignment restriction of the object + * + * Returns: + * A constant pointer to the object stored, or NULL in case of an error + */ +const void * +fy_allocator_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align) + FY_EXPORT; + +/** + * fy_allocator_lookup() - Lookup for object in an allocator. + * + * Lookup for the exact contents of an object stored in an allocator + * and return a pointer to the location it was stored. + * The allocator must have the FYACF_CAN_LOOKUP capability. + * + * @a: The allocator + * @tag: The tag used to locate the memory + * @data: The pointer to object to store + * @size: The size of the object + * @align: The alignment restriction of the object + * + * Returns: + * A constant pointer to the object stored, or NULL if the object does not exist + */ +const void * +fy_allocator_lookup(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) + FY_EXPORT; + +/** + * fy_allocator_lookupv() - Lookup for object in an allocator (scatter gather) + * + * Lookup for the exact contents of an object stored in an allocator + * and return a pointer to the location it was stored. + * The allocator must have the FYACF_CAN_LOOKUP capability. + * + * The scatter gather vector is used to recreate the object. + * + * @a: The allocator + * @tag: The tag used to search into + * @iov: The I/O scatter gather vector + * @iovcnt: The number of vectors + * @align: The alignment restriction of the object + * + * Returns: + * A constant pointer to the object stored, or NULL in case the object does not exist + */ +const void * +fy_allocator_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align) + FY_EXPORT; + +/** + * fy_allocator_dump() - Dump internal allocator state + * + * @a: The allocator + */ +void +fy_allocator_dump(struct fy_allocator *a) + FY_EXPORT; + +/** + * enum fy_allocator_cap_flags - Allocator capability flags + * + * @FYACF_CAN_FREE_INDIVIDUAL: Allocator supports freeing individual allocations + * @FYACF_CAN_FREE_TAG: Allocator supports releasing entire tags + * @FYACF_CAN_DEDUP: Allocator supports deduplication + * @FYACF_HAS_CONTAINS: Allocator can report if it contains a pointer (even if inefficiently) + * @FYACF_HAS_EFFICIENT_CONTAINS: Allocator can report if it contains a pointer (efficiently) + * @FYACF_HAS_TAGS: Allocator has individual tags or not + * @FYACF_CAN_LOOKUP: Allocator supports lookup for content + * + * These flags describe what operations an allocator supports. + */ +enum fy_allocator_cap_flags { + FYACF_CAN_FREE_INDIVIDUAL = FY_BIT(0), + FYACF_CAN_FREE_TAG = FY_BIT(1), + FYACF_CAN_DEDUP = FY_BIT(2), + FYACF_HAS_CONTAINS = FY_BIT(3), + FYACF_HAS_EFFICIENT_CONTAINS = FY_BIT(4), + FYACF_HAS_TAGS = FY_BIT(5), + FYACF_CAN_LOOKUP = FY_BIT(6), +}; + +/** + * fy_allocator_get_caps() - Get allocator capabilities + * + * Retrieve the capabilities of an allocator. + * + * @a: The allocator + * + * Returns: + * The capabilities of the allocator + */ +enum fy_allocator_cap_flags +fy_allocator_get_caps(struct fy_allocator *a) + FY_EXPORT; + +/** + * fy_allocator_contains() - Check if a allocator contains a pointer + * + * Report if an allocator contains the pointer + * + * @a: The allocator + * @tag: Tag to search in, -1 for all + * @ptr: The object pointer + * + * Returns: + * true if the pointer ptr is contained in the allocator, false otherwise + */ +bool +fy_allocator_contains(struct fy_allocator *a, int tag, const void *ptr) + FY_EXPORT; + +/** + * fy_allocator_get_tag_linear_size() - Get the linear size of an allocator tag + * + * Retrieve the linear size of the content of a tag. + * That is the size of a buffer if one was to copy the content of the tag in + * that buffer in a linear manner. + * + * @a: The allocator + * @tag: The tag + * + * Returns: + * The linear size of the content stored in the tag or -1 in case of an error. + */ +ssize_t +fy_allocator_get_tag_linear_size(struct fy_allocator *a, int tag) + FY_EXPORT; + +/** + * fy_allocator_get_tag_single_linear() - Get the linear extend of a tag + * + * If a tag stores it's content in a single linear buffer, retrieve it + * directly. This is possible only under careful arrangement of allocator + * configuration, but it is an important optimization case. + * + * @a: The allocator + * @tag: The tag + * @sizep: Pointer to a variable that will be filled with the size. + * + * Returns: + * A pointer to the linear content of the tag, or NULL if othersize. + */ +const void * +fy_allocator_get_tag_single_linear(struct fy_allocator *a, int tag, size_t *sizep) + FY_EXPORT; + +/** + * struct fy_linear_allocator_cfg - linear allocator configuration + * + * @buf: A pointer to a buffer that will be used, or NULL in order to allocate + * @size: Size of the buffer in bytes + */ +struct fy_linear_allocator_cfg { + void *buf; + size_t size; +}; + +/** + * enum fy_mremap_arena_type - The mremap allocator arena types + * + * @FYMRAT_DEFAULT: Use what's optimal for this platform + * @FYMRAT_MALLOC: Use malloc/realloc arena type (not recommended) + * @FYMRAT_MMAP: Use mmap/mremap arena type + * + */ +enum fy_mremap_arena_type { + FYMRAT_DEFAULT, + FYMRAT_MALLOC, + FYMRAT_MMAP, +}; + +/** + * struct fy_mremap_allocator_cfg - mremap allocator configuration + * + * If any of the fields is zero, then the system will provide (somewhat) + * reasonable defaults. + * + * @big_alloc_threshold: Threshold for immediately creating a new arena. + * @empty_threshold: The threshold under which an arena is moved to the full list. + * @minimum_arena_size: The minimum (and starting size) of an arena. + * @grow_ratio: The ratio which an arena will try to grow if full (>1.0) + * @balloon_ratio: The multiplier for the vm area first allocation + * @arena_type: The arena type + */ +struct fy_mremap_allocator_cfg { + size_t big_alloc_threshold; + size_t empty_threshold; + size_t minimum_arena_size; + float grow_ratio; + float balloon_ratio; + enum fy_mremap_arena_type arena_type; +}; + +/* malloc allocator has no configuration data, pass NULL */ + +/** + * struct fy_dedup_allocator_cfg - dedup allocator configuration + * + * @parent_allocator: The parent allocator (required) + * @bloom_filter_bits: Number of bits of the bloom filter (or 0 for default) + * @bucket_count_bits: Number of bits for the bucket count (or 0 for default) + * @dedup_threshold: Number of bytes over which dedup takes place (default 0=always) + * @chain_length_grow_trigger: Chain length of a bucket over which a grow takes place (or 0 for auto) + * @estimated_content_size: Estimated content size (or 0 for don't know) + * @minimum_bucket_occupancy: The minimum amount that a tag bucket must be full before + * growth is allowed (default 50%, or 0.0) + */ +struct fy_dedup_allocator_cfg { + struct fy_allocator *parent_allocator; + unsigned int bloom_filter_bits; + unsigned int bucket_count_bits; + size_t dedup_threshold; + unsigned int chain_length_grow_trigger; + size_t estimated_content_size; + float minimum_bucket_occupancy; +}; + +/** + * enum fy_auto_allocator_scenario_type - auto allocator scenario type + * + * @FYAST_PER_TAG_FREE: only per tag freeing, no individual obj free + * @FYAST_PER_TAG_FREE_DEDUP: per tag freeing, dedup obj store + * @FYAST_PER_OBJ_FREE: object freeing allowed, tag freeing still works + * @FYAST_PER_OBJ_FREE_DEDUP: per obj freeing, dedup obj store + * @FYAST_SINGLE_LINEAR_RANGE: just a single linear range, no frees at all + * @FYAST_SINGLE_LINEAR_RANGE_DEDUP: single linear range, with dedup + */ +enum fy_auto_allocator_scenario_type { + FYAST_PER_TAG_FREE, + FYAST_PER_TAG_FREE_DEDUP, + FYAST_PER_OBJ_FREE, + FYAST_PER_OBJ_FREE_DEDUP, + FYAST_SINGLE_LINEAR_RANGE, + FYAST_SINGLE_LINEAR_RANGE_DEDUP, +}; + +/** + * struct fy_auto_allocator_cfg - auto allocator configuration + * + * @scenario: Auto allocator scenario + * @estimated_max_size: Estimated max content size (or 0 for don't know) + */ +struct fy_auto_allocator_cfg { + enum fy_auto_allocator_scenario_type scenario; + size_t estimated_max_size; +}; + +struct fy_parser_checkpoint; + +/** + * fy_parser_checkpoint_create() - Create a parser checkpoint + * + * Create a checkpoint of the parser's current state. This allows you + * to save the parser state and potentially roll back to it later using + * fy_parser_rollback(). The checkpoint must be destroyed via + * fy_parser_checkpoint_destroy() when no longer needed. + * + * @fyp: The parser + * + * Returns: + * A pointer to the checkpoint, or NULL in case of an error + */ +struct fy_parser_checkpoint * +fy_parser_checkpoint_create(struct fy_parser *fyp) + FY_EXPORT; + +/** + * fy_parser_checkpoint_destroy() - Destroy a parser checkpoint + * + * Destroy a checkpoint created earlier via fy_parser_checkpoint_create(). + * + * @fypchk: The checkpoint to destroy + */ +void +fy_parser_checkpoint_destroy(struct fy_parser_checkpoint *fypchk) + FY_EXPORT; + +/** + * fy_parser_rollback() - Roll back the parser to a checkpoint + * + * Roll back the parser state to a previously created checkpoint. This allows + * you to revert the parser to an earlier state and re-parse from that point. + * The checkpoint remains valid after rollback and can be used again or + * destroyed via fy_parser_checkpoint_destroy(). + * + * @fyp: The parser + * @fypc: The checkpoint to roll back to + * + * Returns: + * 0 on success, -1 on error + */ +int +fy_parser_rollback(struct fy_parser *fyp, struct fy_parser_checkpoint *fypc) + FY_EXPORT; + #ifdef __cplusplus } #endif diff --git a/libfyaml.pc.in b/libfyaml.pc.in index 3ca5d42..ef73e76 100644 --- a/libfyaml.pc.in +++ b/libfyaml.pc.in @@ -6,5 +6,5 @@ includedir=@includedir@ Name: libfyaml Description: Fancy YAML 1.3 parser library Version: @PACKAGE_VERSION@ -Libs: -L${libdir} @ASAN_LIBS@ @PTHREAD_LIBS@ -lfyaml +Libs: -L${libdir} @ASAN_LIBS@ @PTHREAD_LIBS@ @LIBCLANG_LDFLAGS@ @LIBCLANG_LIBS@ -lfyaml Cflags: -I${includedir} @ASAN_CFLAGS@ @PTHREAD_CFLAGS@ diff --git a/scripts/get-gpg-key-id-by-email.sh b/scripts/get-gpg-key-id-by-email.sh index 300b479..6c551f4 100755 --- a/scripts/get-gpg-key-id-by-email.sh +++ b/scripts/get-gpg-key-id-by-email.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # either use the supplied email address, or the current git user's email if [ "x$1" != "x" ] ; then diff --git a/scripts/run-check-errors.sh b/scripts/run-check-errors.sh index 144b4b0..72a0e95 100755 --- a/scripts/run-check-errors.sh +++ b/scripts/run-check-errors.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ERROR=1 PARSER="./src/libfyaml-parser -mtestsuite" diff --git a/scripts/run-compare-dump.sh b/scripts/run-compare-dump.sh index 506cc12..0de6567 100755 --- a/scripts/run-compare-dump.sh +++ b/scripts/run-compare-dump.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # VALGRIND="" VALGRIND="valgrind" BN=`basename $1` diff --git a/scripts/run-compare-examples.sh b/scripts/run-compare-examples.sh index 655e0c5..dafbbc5 100755 --- a/scripts/run-compare-examples.sh +++ b/scripts/run-compare-examples.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash for i in $*; do echo "checking: $i" diff --git a/scripts/run-compare-parse.sh b/scripts/run-compare-parse.sh index fafa8cf..0410ed9 100755 --- a/scripts/run-compare-parse.sh +++ b/scripts/run-compare-parse.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # VALGRIND="" VALGRIND="valgrind" BN=`basename $1` diff --git a/scripts/run-compare-scan.sh b/scripts/run-compare-scan.sh index 68d0375..3509178 100755 --- a/scripts/run-compare-scan.sh +++ b/scripts/run-compare-scan.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # VALGRIND="" VALGRIND="valgrind" BN=`basename $1` diff --git a/scripts/run-compare-testsuite.sh b/scripts/run-compare-testsuite.sh index 5595bd5..bb9520d 100755 --- a/scripts/run-compare-testsuite.sh +++ b/scripts/run-compare-testsuite.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash declare -a rf diff --git a/scripts/run-emit-check.sh b/scripts/run-emit-check.sh index 5fd7186..6f6efc9 100755 --- a/scripts/run-emit-check.sh +++ b/scripts/run-emit-check.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash BN=`basename $1` ON="output/$BN" mkdir -p output diff --git a/scripts/run-list-testsuite.sh b/scripts/run-list-testsuite.sh index 4dc676e..08a84d6 100755 --- a/scripts/run-list-testsuite.sh +++ b/scripts/run-list-testsuite.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ERROR=1 PASS=1 diff --git a/scripts/run-test.sh b/scripts/run-test.sh index f4bbdb1..fcb0e59 100755 --- a/scripts/run-test.sh +++ b/scripts/run-test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash PREFIX="./src" PARSER="libfyaml-parser -d0 -mtestsuite" diff --git a/scripts/show-desc.sh b/scripts/show-desc.sh index cec4f88..5087dd7 100755 --- a/scripts/show-desc.sh +++ b/scripts/show-desc.sh @@ -1,2 +1,2 @@ -#!/bin/bash +#!/usr/bin/env bash for i in output/*.desc; do echo -n "$i: "; cat $i; done diff --git a/src/Makefile.am b/src/Makefile.am index 141209f..3d79cd5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -9,6 +9,14 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/include AM_CFLAGS = +INTERNAL_INCLUDES = \ + -I$(top_srcdir)/src/valgrind \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/xxhash \ + -I$(top_srcdir)/src/util \ + -I$(top_srcdir)/src/thread \ + -I$(top_srcdir)/src/allocator + lib_LTLIBRARIES += libfyaml.la libfyaml_la_SOURCES = \ lib/fy-parse.c lib/fy-parse.h \ @@ -27,7 +35,11 @@ libfyaml_la_SOURCES = \ lib/fy-walk.c lib/fy-walk.h \ lib/fy-path.c lib/fy-path.h \ lib/fy-composer.c lib/fy-composer.h \ - xxhash/xxhash.c xxhash/xxhash.h \ + lib/fy-parse-diag.c lib/fy-input-diag.c lib/fy-doc-diag.c \ + lib/fy-docbuilder-diag.c lib/fy-composer-diag.c + +# util +libfyaml_la_SOURCES += \ util/fy-list.h \ util/fy-typelist.h \ util/fy-ctype.c util/fy-ctype.h \ @@ -38,13 +50,27 @@ libfyaml_la_SOURCES = \ util/fy-id.h \ util/fy-align.h \ util/fy-bit64.h \ + util/fy-vlsize.h \ + util/fy-atomics.h + +# xxhash +libfyaml_la_SOURCES += \ + xxhash/xxhash.c xxhash/xxhash.h + +# thread +libfyaml_la_SOURCES += \ thread/fy-thread.h thread/fy-thread.c -libfyaml_la_CPPFLAGS = $(AM_CPPFLAGS) \ - -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/xxhash \ - -I$(top_srcdir)/src/util \ - -I$(top_srcdir)/src/thread +# allocator +libfyaml_la_SOURCES += \ + allocator/fy-allocator.c allocator/fy-allocator.h \ + allocator/fy-allocator-linear.c allocator/fy-allocator-linear.h \ + allocator/fy-allocator-malloc.c allocator/fy-allocator-malloc.h \ + allocator/fy-allocator-mremap.c allocator/fy-allocator-mremap.h \ + allocator/fy-allocator-dedup.c allocator/fy-allocator-dedup.h \ + allocator/fy-allocator-auto.c allocator/fy-allocator-auto.h + +libfyaml_la_CPPFLAGS = $(AM_CPPFLAGS) $(INTERNAL_INCLUDES) libfyaml_la_CFLAGS = $(AM_CFLAGS) libfyaml_la_LDFLAGS = -no-undefined $(AM_LDFLAGS) $(AM_LIBLDFLAGS) \ -version $(LIBTOOL_VERSION) @@ -114,22 +140,29 @@ endif libfyaml_la_LIBADD += libb3neon.la endif -# libfyaml-parser needs both LIBYAML and static -if HAVE_LIBYAML +if HAVE_LIBCLANG +libfyaml_la_CPPFLAGS += $(LIBCLANG_CPPFLAGS) +libfyaml_la_CFLAGS += $(LIBCLANG_CFLAGS) +libfyaml_la_LDFLAGS += $(LIBCLANG_LDFLAGS) $(LIBCLANG_LIBS) +endif + +if HAVE_CLANG_BLOCKS +libfyaml_la_CFLAGS += $(CLANG_BLOCKS_CFLAGS) +libfyaml_la_LDFLAGS += $(CLANG_BLOCKS_LIBS) +endif + if HAVE_STATIC +if HAVE_LIBYAML +if !CROSS_COMPILING + noinst_PROGRAMS += libfyaml-parser libfyaml_parser_SOURCES = \ internal/libfyaml-parser.c \ valgrind/fy-valgrind.h -libfyaml_parser_CPPFLAGS = $(AM_CPPFLAGS) \ - -I$(top_srcdir)/src/valgrind \ - -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/xxhash \ - -I$(top_srcdir)/src/util \ - -I$(top_srcdir)/src/thread +libfyaml_parser_CPPFLAGS = $(AM_CPPFLAGS) $(INTERNAL_INCLUDES) libfyaml_parser_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la libfyaml_parser_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS) @@ -137,8 +170,7 @@ libfyaml_parser_LDFLAGS = $(AM_LDFLAGS) -static endif endif -# fy-thread -if HAVE_STATIC +if !CROSS_COMPILING noinst_PROGRAMS += fy-thread @@ -146,20 +178,14 @@ fy_thread_SOURCES = \ internal/fy-thread.c \ valgrind/fy-valgrind.h -fy_thread_CPPFLAGS = $(AM_CPPFLAGS) \ - -I$(top_srcdir)/src/valgrind \ - -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/xxhash \ - -I$(top_srcdir)/src/util \ - -I$(top_srcdir)/src/thread +fy_thread_CPPFLAGS = $(AM_CPPFLAGS) $(INTERNAL_INCLUDES) fy_thread_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la fy_thread_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS) fy_thread_LDFLAGS = $(AM_LDFLAGS) -static endif -# fy-b3sum -if HAVE_STATIC +if !CROSS_COMPILING noinst_PROGRAMS += fy-b3sum @@ -167,23 +193,37 @@ fy_b3sum_SOURCES = \ internal/fy-b3sum.c \ valgrind/fy-valgrind.h -fy_b3sum_CPPFLAGS = $(AM_CPPFLAGS) \ - -I$(top_srcdir)/src/valgrind \ - -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/xxhash \ - -I$(top_srcdir)/src/util \ - -I$(top_srcdir)/src/thread \ - -I$(top_srcdir)/src/blake3 +fy_b3sum_CPPFLAGS = $(AM_CPPFLAGS) $(INTERNAL_INCLUDES) -I$(top_srcdir)/src/blake3 fy_b3sum_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la fy_b3sum_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS) fy_b3sum_LDFLAGS = $(AM_LDFLAGS) -static endif +if !CROSS_COMPILING + +noinst_PROGRAMS += fy-allocators + +fy_allocators_SOURCES = \ + internal/fy-allocators.c \ + valgrind/fy-valgrind.h + +fy_allocators_CPPFLAGS = $(AM_CPPFLAGS) $(INTERNAL_INCLUDES) +fy_allocators_LDADD = $(AM_LDADD) $(LIBYAML_LIBS) libfyaml.la +fy_allocators_CFLAGS = $(AM_CFLAGS) $(LIBYAML_CFLAGS) + +fy_allocators_LDFLAGS = $(AM_LDFLAGS) -static +endif + +endif # HAVE_STATIC + bin_PROGRAMS += fy-tool fy_tool_SOURCES = \ tool/fy-tool.c \ + tool/fy-tool-util.c \ + tool/fy-tool-dump.c \ + tool/fy-tool-util.h \ valgrind/fy-valgrind.h fy_tool_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/valgrind diff --git a/src/allocator/fy-allocator-auto.c b/src/allocator/fy-allocator-auto.c new file mode 100644 index 0000000..c5aa7c6 --- /dev/null +++ b/src/allocator/fy-allocator-auto.c @@ -0,0 +1,399 @@ +/* + * fy-allocator-auto.c - automatic allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-utils.h" +#include "fy-allocator-mremap.h" +#include "fy-allocator-dedup.h" +#include "fy-allocator-linear.h" +#include "fy-allocator-auto.h" + +/* fixed parameters */ +#ifndef AUTO_ALLOCATOR_BIG_ALLOC_THRESHOLD +#define AUTO_ALLOCATOR_BIG_ALLOC_THRESHOLD SIZE_MAX +#endif + +#ifndef AUTO_ALLOCATOR_EMPTY_THRESHOLD +#define AUTO_ALLOCATOR_EMPTY_THRESHOLD 64 +#endif + +#ifndef AUTO_ALLOCATOR_GROW_RATIO +#define AUTO_ALLOCATOR_GROW_RATIO 1.5 +#endif + +#ifndef AUTO_ALOCATOR_BALLOON_RATIO +#define AUTO_ALOCATOR_BALLOON_RATIO 8.0 +#endif + +#ifndef AUTO_ALLOCATOR_ARENA_TYPE +#define AUTO_ALLOCATOR_ARENA_TYPE FYMRAT_MMAP +#endif + +#ifndef AUTO_ALLOCATOR_MINIMUM_ARENA_SIZE +#define AUTO_ALLOCATOR_MINIMUM_ARENA_SIZE ((size_t)16 << 20) /* 16 MB */ +#endif + +#ifndef AUTO_ALLOCATOR_DEFAULT_ESTIMATED_MAX_SIZE +#define AUTO_ALLOCATOR_DEFAULT_ESTIMATED_MAX_SIZE ((size_t)1 << 20) /* 1MB */ +#endif + +#ifndef AUTO_ALLOCATOR_DEFAULT_BLOOM_FILTER_BITS +#define AUTO_ALLOCATOR_DEFAULT_BLOOM_FILTER_BITS 0 +#endif + +#ifndef AUTO_ALLOCATOR_DEFAULT_BUCKET_COUNT_BITS +#define AUTO_ALLOCATOR_DEFAULT_BUCKET_COUNT_BITS 0 +#endif + +#ifndef AUTO_ALLOCATOR_DEFAULT_DEDUP_THRESHOLD +#define AUTO_ALLOCATOR_DEFAULT_DEDUP_THRESHOLD 0 /* dedup everything */ +#endif + +#ifndef AUTO_ALLOCATOR_DEFAULT_CHAIN_LENGTH_GROW_TRIGGER +#define AUTO_ALLOCATOR_DEFAULT_CHAIN_LENGTH_GROW_TRIGGER 0 /* let dedup decide */ +#endif + +static const struct fy_auto_allocator_cfg default_cfg = { + .scenario = FYAST_PER_TAG_FREE_DEDUP, + .estimated_max_size = AUTO_ALLOCATOR_DEFAULT_ESTIMATED_MAX_SIZE, +}; + +static void fy_auto_cleanup(struct fy_allocator *a); + +static int fy_auto_setup(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *cfg_data) +{ + size_t pagesz, size; + struct fy_auto_allocator *aa; + const struct fy_auto_allocator_cfg *cfg; + struct fy_mremap_allocator_cfg mrcfg; + struct fy_dedup_allocator_cfg dcfg; + struct fy_linear_allocator_cfg lcfg; + struct fy_allocator *mra = NULL, *da = NULL, *ma = NULL, *la = NULL; + struct fy_allocator *topa = NULL, *suba = NULL; + + cfg = cfg_data ? cfg_data : &default_cfg; + + aa = container_of(a, struct fy_auto_allocator, a); + memset(aa, 0, sizeof(*aa)); + aa->a.name = "auto"; + aa->a.ops = &fy_auto_allocator_ops; + aa->a.parent = parent; + aa->a.parent_tag = parent_tag; + aa->cfg = *cfg; + + pagesz = sysconf(_SC_PAGESIZE); + size = cfg->estimated_max_size && cfg->estimated_max_size != SIZE_MAX ? + cfg->estimated_max_size : + AUTO_ALLOCATOR_MINIMUM_ARENA_SIZE; + + /* first allocator */ + switch (cfg->scenario) { + case FYAST_PER_TAG_FREE: + case FYAST_PER_TAG_FREE_DEDUP: + memset(&mrcfg, 0, sizeof(mrcfg)); + mrcfg.big_alloc_threshold = AUTO_ALLOCATOR_BIG_ALLOC_THRESHOLD; + mrcfg.empty_threshold = AUTO_ALLOCATOR_EMPTY_THRESHOLD; + mrcfg.grow_ratio = AUTO_ALLOCATOR_GROW_RATIO; + mrcfg.balloon_ratio = AUTO_ALOCATOR_BALLOON_RATIO; + mrcfg.arena_type = AUTO_ALLOCATOR_ARENA_TYPE; + + mrcfg.minimum_arena_size = fy_size_t_align(size, pagesz); + + mra = fy_allocator_create("mremap", &mrcfg); + if (!mra) + goto err_out; + topa = mra; + + break; + + case FYAST_PER_OBJ_FREE: + case FYAST_PER_OBJ_FREE_DEDUP: + ma = fy_allocator_create("malloc", NULL); + if (!ma) + goto err_out; + topa = ma; + break; + + case FYAST_SINGLE_LINEAR_RANGE: + case FYAST_SINGLE_LINEAR_RANGE_DEDUP: + memset(&lcfg, 0, sizeof(lcfg)); + lcfg.size = fy_size_t_align(size, pagesz); + + la = fy_allocator_create("linear", &lcfg); + if (!la) + goto err_out; + topa = la; + break; + + default: + goto err_out; + } + + if (!topa) + goto err_out; + + /* stack the dedup */ + switch (cfg->scenario) { + case FYAST_PER_TAG_FREE_DEDUP: + case FYAST_PER_OBJ_FREE_DEDUP: + case FYAST_SINGLE_LINEAR_RANGE_DEDUP: + memset(&dcfg, 0, sizeof(dcfg)); + dcfg.parent_allocator = topa; + dcfg.bloom_filter_bits = AUTO_ALLOCATOR_DEFAULT_BLOOM_FILTER_BITS; + dcfg.bucket_count_bits = AUTO_ALLOCATOR_DEFAULT_BUCKET_COUNT_BITS; + dcfg.dedup_threshold = AUTO_ALLOCATOR_DEFAULT_DEDUP_THRESHOLD; + dcfg.chain_length_grow_trigger = AUTO_ALLOCATOR_DEFAULT_CHAIN_LENGTH_GROW_TRIGGER; + dcfg.estimated_content_size = size; + dcfg.minimum_bucket_occupancy = 0.5; + + da = fy_allocator_create("dedup", &dcfg); + if (!da) + goto err_out; + /* the top is sub now */ + suba = topa; + topa = da; + + break; + + /* nothing to stack for these */ + case FYAST_PER_TAG_FREE: + case FYAST_PER_OBJ_FREE: + case FYAST_SINGLE_LINEAR_RANGE: + break; + + default: + goto err_out; + } + + /* OK, assign the allocators now */ + aa->parent_allocator = topa; + aa->sub_parent_allocator = suba; + + return 0; + +err_out: + fy_allocator_destroy(suba); + fy_allocator_destroy(topa); + fy_auto_cleanup(a); + return -1; +} + +static void fy_auto_cleanup(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + + /* order is important! */ + if (aa->parent_allocator) { + fy_allocator_destroy(aa->parent_allocator); + aa->parent_allocator = NULL; + } + + if (aa->sub_parent_allocator) { + fy_allocator_destroy(aa->sub_parent_allocator); + aa->sub_parent_allocator = NULL; + } +} + +struct fy_allocator *fy_auto_create(struct fy_allocator *parent, int parent_tag, const void *cfg) +{ + struct fy_auto_allocator *aa = NULL; + int rc; + + aa = fy_early_parent_allocator_alloc(parent, parent_tag, sizeof(*aa), _Alignof(struct fy_auto_allocator)); + + rc = fy_auto_setup(&aa->a, parent, parent_tag, cfg); + if (rc) + goto err_out; + + return &aa->a; + +err_out: + fy_early_parent_allocator_free(parent, parent_tag, aa); + + return NULL; +} + +void fy_auto_destroy(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + + fy_auto_cleanup(a); + + fy_parent_allocator_free(a, aa); +} + +void fy_auto_dump(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + + fy_allocator_dump_nocheck(aa->parent_allocator); +} + +static void *fy_auto_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_alloc_nocheck(aa->parent_allocator, tag, size, align); +} + +static void fy_auto_free(struct fy_allocator *a, int tag, void *data) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + fy_allocator_free_nocheck(aa->parent_allocator, tag, data); +} + +static int fy_auto_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_update_stats_nocheck(aa->parent_allocator, tag, stats); +} + +static const void *fy_auto_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_storev_hash_nocheck(aa->parent_allocator, tag, iov, iovcnt, align, hash); +} + +static const void *fy_auto_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_lookupv_nocheck(aa->parent_allocator, tag, iov, iovcnt, align, hash); +} + +static void fy_auto_release(struct fy_allocator *a, int tag, const void *data, size_t size) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + fy_allocator_release_nocheck(aa->parent_allocator, tag, data, size); +} + +static int fy_auto_get_tag(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + /* TODO, convert tag config? */ + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_get_tag_nocheck(aa->parent_allocator); +} + +static void fy_auto_release_tag(struct fy_allocator *a, int tag) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + fy_allocator_release_tag_nocheck(aa->parent_allocator, tag); +} + +static int fy_auto_get_tag_count(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_get_tag_count_nocheck(aa->parent_allocator); +} + +static int fy_auto_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_set_tag_count_nocheck(aa->parent_allocator, count); +} + +static void fy_auto_trim_tag(struct fy_allocator *a, int tag) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + fy_allocator_trim_tag_nocheck(aa->parent_allocator, tag); +} + +static void fy_auto_reset_tag(struct fy_allocator *a, int tag) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + fy_allocator_reset_tag_nocheck(aa->parent_allocator, tag); +} + +static struct fy_allocator_info * +fy_auto_get_info(struct fy_allocator *a, int tag) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_get_info_nocheck(aa->parent_allocator, tag); +} + +static enum fy_allocator_cap_flags +fy_auto_get_caps(struct fy_allocator *a) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_get_caps_nocheck(aa->parent_allocator); +} + +static bool fy_auto_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + struct fy_auto_allocator *aa; + + aa = container_of(a, struct fy_auto_allocator, a); + return fy_allocator_contains_nocheck(aa->parent_allocator, tag, ptr); +} + +const struct fy_allocator_ops fy_auto_allocator_ops = { + .setup = fy_auto_setup, + .cleanup = fy_auto_cleanup, + .create = fy_auto_create, + .destroy = fy_auto_destroy, + .dump = fy_auto_dump, + .alloc = fy_auto_alloc, + .free = fy_auto_free, + .update_stats = fy_auto_update_stats, + .storev = fy_auto_storev, + .lookupv = fy_auto_lookupv, + .release = fy_auto_release, + .get_tag = fy_auto_get_tag, + .release_tag = fy_auto_release_tag, + .get_tag_count = fy_auto_get_tag_count, + .set_tag_count = fy_auto_set_tag_count, + .trim_tag = fy_auto_trim_tag, + .reset_tag = fy_auto_reset_tag, + .get_info = fy_auto_get_info, + .get_caps = fy_auto_get_caps, + .contains = fy_auto_contains, +}; diff --git a/src/allocator/fy-allocator-auto.h b/src/allocator/fy-allocator-auto.h new file mode 100644 index 0000000..8027820 --- /dev/null +++ b/src/allocator/fy-allocator-auto.h @@ -0,0 +1,26 @@ +/* + * fy-allocator-auto.h - the auto allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_AUTO_H +#define FY_ALLOCATOR_AUTO_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "fy-allocator.h" + +struct fy_auto_allocator { + struct fy_allocator a; + struct fy_auto_allocator_cfg cfg; + struct fy_allocator *parent_allocator; + struct fy_allocator *sub_parent_allocator; +}; + +extern const struct fy_allocator_ops fy_auto_allocator_ops; + +#endif diff --git a/src/allocator/fy-allocator-dedup.c b/src/allocator/fy-allocator-dedup.c new file mode 100644 index 0000000..50193cb --- /dev/null +++ b/src/allocator/fy-allocator-dedup.c @@ -0,0 +1,1222 @@ +/* + * fy-allocator-dedup.c - dedup allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +/* for container_of */ +#include "fy-list.h" +#include "fy-utils.h" + +#include "fy-allocator-dedup.h" + +// #define DEBUG_GROWS + +#undef BEFORE +#define BEFORE() \ + do { \ + clock_gettime(CLOCK_MONOTONIC, &before); \ + } while(0) + +#undef AFTER +#define AFTER() \ + ({ \ + clock_gettime(CLOCK_MONOTONIC, &after); \ + (int64_t)(after.tv_sec - before.tv_sec) * (int64_t)1000000000UL + (int64_t)(after.tv_nsec - before.tv_nsec); \ + }) + +static const unsigned int bit_to_chain_length_map[] = { + [0] = 1, /* 1 */ + [1] = 1, /* 2 */ + [2] = 1, /* 4 */ + [3] = 1, /* 8 */ + [4] = 1, /* 16 */ + [5] = 1, /* 32 */ + [6] = 2, /* 64 */ + [7] = 2, /* 128 */ + [8] = 2, /* 256 */ + [9] = 2, /* 512 */ + [10] = 3, /* 1024 */ + [11] = 3, /* 2048 */ + [12] = 3, /* 2048 */ + [13] = 3, /* 2048 */ + [14] = 4, /* 4096 */ + [15] = 4, /* 8192 */ + [16] = 5, /* 16384 */ + [17] = 5, /* 32768 */ + [18] = 6, /* 65536 */ + [19] = 7, /* 65536 */ + [20] = 8, /* 131072 */ + [21] = 9, /* 262144*/ + [22] = 10, /* 524288 */ + [23] = INT_MAX /* infinite from now on */ +}; + +void fy_dedup_tag_data_destroy(struct fy_dedup_tag_data *dtd); + +static inline struct fy_dedup_tag * +fy_dedup_tag_from_tag(struct fy_dedup_allocator *da, int tag) +{ + if (!da) + return NULL; + + if ((unsigned int)tag >= da->tag_count) + return NULL; + + if (!fy_id_is_used(da->ids, da->tag_id_count, tag)) + return NULL; + + return &da->tags[tag]; +} + +static void fy_dedup_tag_data_cleanup(struct fy_dedup_tag_data *dtd) +{ + struct fy_dedup_allocator *da = dtd->cfg.da; + + fy_parent_allocator_free(&da->a, dtd->buckets); + fy_parent_allocator_free(&da->a, dtd->bloom_id); + fy_parent_allocator_free(&da->a, dtd->buckets_in_use); +} + +static int fy_dedup_tag_data_setup(struct fy_dedup_tag_data *dtd, + const struct fy_dedup_tag_data_cfg *cfg) +{ + struct fy_dedup_allocator *da; + unsigned int bloom_filter_bits, bucket_count_bits, chain_length_grow_trigger; + size_t dedup_threshold; + size_t tmpsz; + unsigned int i; + + assert(dtd); + assert(cfg); + + da = cfg->da; + + bloom_filter_bits = cfg->bloom_filter_bits; + bucket_count_bits = cfg->bucket_count_bits; + dedup_threshold = cfg->dedup_threshold; + chain_length_grow_trigger = cfg->chain_length_grow_trigger; + + memset(dtd, 0, sizeof(*dtd)); + dtd->cfg = *cfg; + + dtd->bloom_filter_bits = bloom_filter_bits; + dtd->bucket_count_bits = bucket_count_bits; + dtd->dedup_threshold = dedup_threshold; + dtd->chain_length_grow_trigger = chain_length_grow_trigger; + if (!dtd->chain_length_grow_trigger) { + if (bucket_count_bits >= ARRAY_SIZE(bit_to_chain_length_map)) + dtd->chain_length_grow_trigger = bit_to_chain_length_map[ARRAY_SIZE(bit_to_chain_length_map) - 1]; + else + dtd->chain_length_grow_trigger = bit_to_chain_length_map[bucket_count_bits]; + + if (dtd->chain_length_grow_trigger == 0) + dtd->chain_length_grow_trigger++; + } + + dtd->bloom_filter_mask = (1U << dtd->bloom_filter_bits) - 1; + dtd->bucket_count_mask = (1U << dtd->bucket_count_bits) - 1; + + dtd->bloom_id_count = ((1U << dtd->bloom_filter_bits) + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + assert(dtd->bloom_id_count > 0); + tmpsz = dtd->bloom_id_count * sizeof(*dtd->bloom_id); + dtd->bloom_id = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(fy_id_bits)); + if (!dtd->bloom_id) + goto err_out; + fy_id_reset(dtd->bloom_id, dtd->bloom_id_count); + + dtd->bucket_count = 1U << dtd->bucket_count_bits; + assert(dtd->bucket_count); + + tmpsz = sizeof(*dtd->buckets) * dtd->bucket_count; + dtd->buckets = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(struct fy_dedup_entry *)); + if (!dtd->buckets) + goto err_out; + for (i = 0; i < dtd->bucket_count; i++) + atomic_store(&dtd->buckets[i], NULL); + + dtd->bucket_id_count = ((1U << dtd->bucket_count_bits) + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + assert(dtd->bucket_id_count > 0); + tmpsz = dtd->bucket_id_count * sizeof(*dtd->buckets_in_use); + dtd->buckets_in_use = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(fy_id_bits)); + if (!dtd->buckets_in_use) + goto err_out; + fy_id_reset(dtd->buckets_in_use, dtd->bucket_id_count); + + return 0; + +err_out: + fy_dedup_tag_data_cleanup(dtd); + return -1; +} + +static void fy_dedup_tag_cleanup(struct fy_dedup_allocator *da, struct fy_dedup_tag *dt) +{ + struct fy_dedup_tag_data *dtd; + + if (!da || !dt) + return; + + while ((dtd = fy_atomic_load(&dt->tag_datas)) != NULL) { + + /* only do it if you are the winner */ + if (!fy_atomic_compare_exchange_strong(&dt->tag_datas, &dtd, dtd->next)) + continue; + +#ifdef DEBUG_GROWS + fprintf(stderr, "%s: dump of state at close\n", __func__); + fprintf(stderr, "%s: bloom count=%u used=%lu\n", + __func__, 1 << dtd->bloom_filter_bits, + fy_id_count_used(dtd->bloom_id, dtd->bloom_id_count)); + fprintf(stderr, "%s: bucket count=%u used=%lu\n", + __func__, 1 << dtd->bucket_count_bits, + fy_id_count_used(dtd->buckets_in_use, dtd->bucket_id_count)); +#endif + + fy_dedup_tag_data_destroy(dtd); + } + + /* we just release the tags, the underlying allocator should free everything */ + if (dt->content_tag != FY_ALLOC_TAG_NONE) + fy_allocator_release_tag(da->parent_allocator, dt->content_tag); + +} + +struct fy_dedup_tag_data * +fy_dedup_tag_data_create(const struct fy_dedup_tag_data_cfg *cfg) +{ + struct fy_dedup_allocator *da = cfg->da; + struct fy_dedup_tag_data *dtd; + int ret; + + dtd = fy_parent_allocator_alloc(&da->a, sizeof(*dtd), _Alignof(struct fy_dedup_tag_data)); + if (!dtd) + return NULL; + + ret = fy_dedup_tag_data_setup(dtd, cfg); + if (ret) + goto err_out; + + return dtd; + +err_out: + fy_parent_allocator_free(&da->a, dtd); + return NULL; +} + +void fy_dedup_tag_data_destroy(struct fy_dedup_tag_data *dtd) +{ + struct fy_dedup_allocator *da; + + if (!dtd) + return; + + da = dtd->cfg.da; + + fy_dedup_tag_data_cleanup(dtd); + fy_parent_allocator_free(&da->a, dtd); +} + +static int fy_dedup_tag_setup(struct fy_dedup_allocator *da, struct fy_dedup_tag *dt) +{ + struct fy_dedup_tag_data *dtd; + + assert(da); + assert(dt); + + memset(dt, 0, sizeof(*dt)); + + fy_atomic_flag_clear(&dt->growing); + if (da->parent_caps & FYACF_CAN_FREE_TAG) { + dt->content_tag = fy_allocator_get_tag(da->parent_allocator); + if (dt->content_tag == FY_ALLOC_TAG_ERROR) + goto err_out; + } else + dt->content_tag = FY_ALLOC_TAG_DEFAULT; + + fy_atomic_store(&dt->tag_datas, NULL); + + dtd = fy_dedup_tag_data_create( + &(struct fy_dedup_tag_data_cfg) { + .da = da, + .dt = dt, + .bloom_filter_bits = da->bloom_filter_bits, + .bucket_count_bits = da->bucket_count_bits, + .dedup_threshold = da->dedup_threshold, + .chain_length_grow_trigger = da->chain_length_grow_trigger + }); + if (!dtd) + goto err_out; + + /* start with this one */ + fy_atomic_store(&dt->tag_datas, dtd); + + return 0; + +err_out: + fy_dedup_tag_cleanup(da, dt); + return -1; +} + +static int +fy_dedup_tag_prepare_new(struct fy_dedup_tag_data *dtd, struct fy_dedup_tag_data *new_dtd, + int bloom_filter_adjust_bits, int bucket_adjust_bits) +{ + unsigned int bit_shift, new_bucket_count_bits, new_bloom_filter_bits; + int rc; + + bit_shift = (unsigned int)fy_id_ffs(FY_ID_BITS_BITS); + + new_bucket_count_bits = (unsigned int)((int)dtd->bucket_count_bits + bucket_adjust_bits); + if (new_bucket_count_bits > (sizeof(int) * 8 - 1)) + new_bucket_count_bits = (sizeof(int) * 8) - 1; + else if (new_bucket_count_bits < bit_shift) + new_bucket_count_bits = bit_shift; + + new_bloom_filter_bits = (unsigned int)((int)dtd->bloom_filter_bits + bloom_filter_adjust_bits); + if (new_bloom_filter_bits > (sizeof(int) * 8 - 1)) + new_bloom_filter_bits = (sizeof(int) * 8) - 1; + else if (new_bloom_filter_bits < new_bucket_count_bits) + new_bloom_filter_bits = new_bucket_count_bits; + + /* setup the new data */ + rc = fy_dedup_tag_data_setup(new_dtd, + &(struct fy_dedup_tag_data_cfg) { + .da = dtd->cfg.da, + .dt = dtd->cfg.dt, + .bloom_filter_bits = new_bloom_filter_bits, + .bucket_count_bits = new_bucket_count_bits, + .dedup_threshold = dtd->cfg.da->dedup_threshold, + .chain_length_grow_trigger = dtd->cfg.da->chain_length_grow_trigger + }); + return rc; +} + + +static int +fy_dedup_tag_adjust(struct fy_dedup_allocator *da, struct fy_dedup_tag *dt, + int bloom_filter_adjust_bits, int bucket_adjust_bits) +{ + size_t bucket_count, bucket_used; + struct fy_dedup_tag_data *dtd, *new_dtd = NULL; + float occupancy_ratio; + int rc; + + rc = -1; + + assert(da); + assert(dt); + + dtd = fy_atomic_load(&dt->tag_datas); + if (!dtd) + return -1; + + if (!fy_atomic_flag_test_and_set(&dt->growing)) { +#ifdef DEBUG_GROWS + fprintf(stderr, "grow: abort due another grow in progress\n"); +#endif + return -1; + } + + bucket_count = 1U << dtd->bucket_count_bits; + bucket_used = fy_id_count_used(dtd->buckets_in_use, dtd->bucket_id_count); + occupancy_ratio = (double)bucket_used/(double)bucket_count; + + /* do not grow until we're over 60% full */ + if (occupancy_ratio < da->cfg.minimum_bucket_occupancy) { +#ifdef DEBUG_GROWS + fprintf(stderr, "grow: abort due to less that %f%% full (is %f)\n", + 100 * da->cfg.minimum_bucket_occupancy, + 100 * occupancy_ratio); +#endif + goto out_ok; + } + + new_dtd = fy_parent_allocator_alloc(&da->a, sizeof(*new_dtd), _Alignof(struct fy_dedup_tag_data)); + if (!new_dtd) + goto err_out; + + /* prepare the new one */ + rc = fy_dedup_tag_prepare_new(dtd, new_dtd, bloom_filter_adjust_bits, bucket_adjust_bits); + if (rc) + goto err_out; + +#ifdef DEBUG_GROWS + { + size_t bloom_count, new_bloom_count, bloom_used; + size_t bucket_count, new_bucket_count, bucket_used; + + bloom_count = 1U << dtd->bloom_filter_bits; + new_bloom_count = 1U << new_dtd->bloom_filter_bits; + bloom_used = fy_id_count_used(dtd->bloom_id, dtd->bloom_id_count); + bucket_count = 1U << dtd->bucket_count_bits; + new_bucket_count = 1U << new_dtd->bucket_count_bits; + bucket_used = fy_id_count_used(dtd->buckets_in_use, dtd->bucket_id_count); + + fprintf(stderr, "grow: chain_length_grow_trigger=%u->%u bloom %zu->%zu used %zu (%2.2f%%) ", + dtd->chain_length_grow_trigger, new_dtd->chain_length_grow_trigger, + bloom_count, new_bloom_count, + bloom_used, 100.0*(double)bloom_used/(double)bloom_count); + fprintf(stderr, "bucket %zu->%zu used %zu (%2.2f%%)\n", bucket_count, new_bucket_count, + bucket_used, 100.0*(double)bucket_used/(double)bucket_count); + } +#endif + + /* try to add it, if the head changed, then drop our stuff and pretend nothing happened */ + new_dtd->next = dtd; + if (!fy_atomic_compare_exchange_strong(&dt->tag_datas, &dtd, new_dtd)) { +#ifdef DEBUG_GROWS + fprintf(stderr, "grow: Update cancelled, someone beat us to it\n"); +#endif + fy_dedup_tag_data_cleanup(new_dtd); + fy_parent_allocator_free(&da->a, new_dtd); + } + +out_ok: + rc = 0; +out: + fy_atomic_flag_clear(&dt->growing); + return rc; + +err_out: + fy_parent_allocator_free(&da->a, new_dtd); + rc = -1; + goto out; +} + +static void fy_dedup_tag_trim(struct fy_dedup_allocator *da, struct fy_dedup_tag *dt) +{ + if (!da || !dt) + return; + + /* just pass them trim down to the parent */ + if (da->parent_caps & FYACF_CAN_FREE_TAG) + fy_allocator_trim_tag(da->parent_allocator, dt->content_tag); +} + +static void fy_dedup_tag_reset(struct fy_dedup_allocator *da, struct fy_dedup_tag *dt) +{ + struct fy_dedup_tag_data *dtd, *dtd_head; + size_t i; + + if (!da || !dt) + return; + + /* just pass them reset down to the parent */ + if (da->parent_caps & FYACF_CAN_FREE_TAG) + fy_allocator_reset_tag(da->parent_allocator, dt->content_tag); + + /* pop the head which we will keep */ + + dtd_head = NULL; + while ((dtd = fy_atomic_load(&dt->tag_datas)) != NULL) { + if (!fy_atomic_compare_exchange_strong(&dt->tag_datas, &dtd, dtd->next)) + continue; + + dtd->next = NULL; + + /* keep the head as is */ + if (!dtd_head) + dtd_head = dtd; + else + fy_dedup_tag_data_destroy(dtd); + } + + if (dtd_head) { + dtd = dtd_head; + + dtd->next = NULL; + fy_id_reset(dtd->bloom_id, dtd->bloom_id_count); + for (i = 0; i < dtd->bucket_count; i++) + atomic_store(&dtd->buckets[i], NULL); + fy_id_reset(dtd->buckets_in_use, dtd->bucket_id_count); + fy_atomic_store(&dt->tag_datas, dtd); + } +} + +static void fy_dedup_cleanup(struct fy_allocator *a); + +#define BUCKET_ESTIMATE_DIV 1024 +#define BLOOM_ESTIMATE_DIV 128 + +static void fy_dedup_cleanup(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + unsigned int i; + + if (!a) + return; + + da = container_of(a, struct fy_dedup_allocator, a); + + for (i = 0; i < da->tag_count; i++) { + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + fy_dedup_tag_cleanup(da, dt); + } + + fy_parent_allocator_free(&da->a, da->ids); + fy_parent_allocator_free(&da->a, da->tags); +} + +static int fy_dedup_setup(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *cfg_data) +{ + struct fy_dedup_allocator *da = NULL; + const struct fy_dedup_allocator_cfg *cfg; + unsigned int bloom_filter_bits, bucket_count_bits; + unsigned int bit_shift, chain_length_grow_trigger; + size_t dedup_threshold, tmpsz; + bool has_estimate; + struct fy_dedup_tag *dt; + int rc; + + if (!a || !cfg_data) + return -1; + + cfg = cfg_data; + if (!cfg->parent_allocator) + return -1; + + has_estimate = cfg->estimated_content_size && cfg->estimated_content_size != SIZE_MAX; + + /* power of two so ffs = log2 */ + bit_shift = (unsigned int)fy_id_ffs(FY_ID_BITS_BITS); + + bucket_count_bits = cfg->bucket_count_bits; + if (!bucket_count_bits && has_estimate) { + bucket_count_bits = 1; + while ((1LU << bucket_count_bits) < (cfg->estimated_content_size / BUCKET_ESTIMATE_DIV)) + bucket_count_bits++; +#ifdef DEBUG_GROWS + fprintf(stderr, "bucket_count_bits %u\n", bucket_count_bits); +#endif + } + /* at least that amount */ + if (bucket_count_bits < bit_shift) + bucket_count_bits = bit_shift; + /* keep the bucket count bits in signed int range */ + if (bucket_count_bits > (sizeof(int) * 8 - 1)) + bucket_count_bits = (sizeof(int) * 8) - 1; + + bloom_filter_bits = cfg->bloom_filter_bits; + if (!bloom_filter_bits && has_estimate) { + bloom_filter_bits = 1; + while ((1LU << bloom_filter_bits) < (cfg->estimated_content_size / BLOOM_ESTIMATE_DIV)) + bloom_filter_bits++; +#ifdef DEBUG_GROWS + fprintf(stderr, "bloom_filter_bits %u\n", bloom_filter_bits); +#endif + } + /* must be more than bucket count bits */ + if (bloom_filter_bits < bucket_count_bits) + bloom_filter_bits = bucket_count_bits + 3; /* minimum fanout */ + /* keep the bloom filter bits in signed int range */ + if (bloom_filter_bits > (sizeof(int) * 8 - 1)) + bloom_filter_bits = (sizeof(int) * 8) - 1; + + dedup_threshold = cfg->dedup_threshold; + chain_length_grow_trigger = cfg->chain_length_grow_trigger; + + da = container_of(a, struct fy_dedup_allocator, a); + memset(da, 0, sizeof(*da)); + + da->a.name = "dedup"; + da->a.ops = &fy_dedup_allocator_ops; + da->a.parent = parent; + da->a.parent_tag = parent_tag; + da->cfg = *cfg; + + da->parent_allocator = cfg->parent_allocator; + da->parent_caps = fy_allocator_get_caps(da->parent_allocator); + + da->bloom_filter_bits = bloom_filter_bits; + da->bucket_count_bits = bucket_count_bits; + da->dedup_threshold = dedup_threshold; + da->chain_length_grow_trigger = chain_length_grow_trigger; + + /* we use as many tags as the parent allocator */ + rc = fy_allocator_get_tag_count(da->parent_allocator); + if (rc <= 0) + goto err_out; + da->tag_count = (unsigned int)rc; + da->tag_id_count = (da->tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + tmpsz = da->tag_id_count * sizeof(*da->ids); + da->ids = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(fy_id_bits)); + if (!da->ids) + goto err_out; + fy_id_reset(da->ids, da->tag_id_count); + + tmpsz = da->tag_count * sizeof(*da->tags); + da->tags = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(struct fy_dedup_tag)); + if (!da->tags) + goto err_out; + + /* start with tag 0 as general use */ + fy_id_set_used(da->ids, da->tag_id_count, 0); + dt = fy_dedup_tag_from_tag(da, 0); + rc = fy_dedup_tag_setup(da, dt); + if (rc) + goto err_out; + + return 0; +err_out: + fy_dedup_cleanup(a); + return -1; +} + +struct fy_allocator *fy_dedup_create(struct fy_allocator *parent, int parent_tag, const void *cfg) +{ + struct fy_dedup_allocator *da = NULL; + int rc; + + da = fy_early_parent_allocator_alloc(parent, parent_tag, sizeof(*da), _Alignof(struct fy_dedup_allocator)); + if (!da) + goto err_out; + + rc = fy_dedup_setup(&da->a, parent, parent_tag, cfg); + if (rc) + goto err_out; + + return &da->a; + +err_out: + fy_early_parent_allocator_free(parent, parent_tag, da); + return NULL; +} + +void fy_dedup_destroy(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + + da = container_of(a, struct fy_dedup_allocator, a); + fy_dedup_cleanup(a); + + fy_parent_allocator_free(a, da); +} + +void fy_dedup_dump(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + unsigned int i; + + da = container_of(a, struct fy_dedup_allocator, a); + + fprintf(stderr, "dedup: "); + for (i = 0; i < da->tag_count; i++) { + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + fprintf(stderr, "%c", fy_id_is_free(da->ids, da->tag_id_count, i) ? '.' : 'x'); + } + fprintf(stderr, "\n"); + + for (i = 0; i < da->tag_count; i++) { + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + fprintf(stderr, " %d: tags: content=%d\n", i, dt->content_tag); + } + + fprintf(stderr, "dedup: dumping parent allocator\n"); + fy_allocator_dump(da->parent_allocator); +} + +static void *fy_dedup_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + void *p; + + da = container_of(a, struct fy_dedup_allocator, a); + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + goto err_out; + + /* just pass to the parent allocator using the content tag */ + p = fy_allocator_alloc_nocheck(da->parent_allocator, dt->content_tag, size, align); + if (!p) + goto err_out; + + return p; + +err_out: + return NULL; +} + +static void fy_dedup_free(struct fy_allocator *a, int tag, void *data) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + + da = container_of(a, struct fy_dedup_allocator, a); + if (!(da->parent_caps & FYACF_CAN_FREE_INDIVIDUAL)) + return; + + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return; + + /* just pass to the parent allocator */ + fy_allocator_free(da->parent_allocator, dt->content_tag, data); +} + +static int fy_dedup_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + int r; + + da = container_of(a, struct fy_dedup_allocator, a); + + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return -1; + + r = fy_allocator_update_stats(da->parent_allocator, dt->content_tag, stats); + if (r) + return -1; + + stats->unique_stores += fy_atomic_get_and_clear_counter(&dt->unique_stores); + stats->dup_stores += fy_atomic_get_and_clear_counter(&dt->dup_stores); + stats->collisions += fy_atomic_get_and_clear_counter(&dt->collisions); + + return 0; +} + +/* we can lookup! */ +static const void *fy_dedup_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + struct fy_dedup_tag_data *dtd; + struct fy_dedup_entry *de; + int bloom_pos, bucket_pos; + bool bloom_hit; + size_t total_size; + + da = container_of(a, struct fy_dedup_allocator, a); + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return NULL; + + /* calculate data total size */ + total_size = fy_iovec_size(iov, iovcnt); + if (total_size == SIZE_MAX) + return NULL; + + /* if it's under the dedup threshold just allocate and copy */ + if (total_size < da->cfg.dedup_threshold) + return NULL; + + /* calculate hash if not given */ + if (!hash) + hash = fy_iovec_xxhash64(iov, iovcnt); + + for (dtd = fy_atomic_load(&dt->tag_datas); dtd; dtd = dtd->next) { + bloom_pos = (int)(hash & (uint64_t)dtd->bloom_filter_mask); + bloom_hit = fy_id_is_used(dtd->bloom_id, dtd->bloom_id_count, bloom_pos); + if (bloom_hit) { + bucket_pos = (int)(hash & (uint64_t)dtd->bucket_count_mask); + for (de = fy_atomic_load(&dtd->buckets[bucket_pos]); de; de = de->next) { + if (de->hash == hash && total_size == de->size && + !fy_iovec_cmp(iov, iovcnt, de->mem) && + ((uintptr_t)de->mem & (align - 1)) == 0) + return de->mem; + } + } + } + return NULL; +} + +static const void *fy_dedup_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + struct fy_dedup_tag_data *dtd, *dtd_best; + struct fy_dedup_entry *de, *de_head; + int bloom_pos, bucket_pos; + bool bloom_hit; + unsigned int chain_length; + void *mem = NULL; + size_t total_size, de_offset, max_align; + bool at_head; + + da = container_of(a, struct fy_dedup_allocator, a); + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return NULL; + + /* calculate data total size */ + total_size = fy_iovec_size(iov, iovcnt); + if (total_size == SIZE_MAX) + return NULL; + + /* if it's under the dedup threshold just allocate and copy */ + if (total_size < da->cfg.dedup_threshold) { + + /* just pass to the parent allocator using the content tag */ + void *p = fy_allocator_alloc_nocheck(da->parent_allocator, dt->content_tag, total_size, align); + if (!p) + return NULL; + + fy_iovec_copy_from(iov, iovcnt, p); + return p; + } + + /* calculate hash if not given */ + if (!hash) + hash = fy_iovec_xxhash64(iov, iovcnt); + +again: + de = NULL; + chain_length = 0; + + at_head = true; + dtd_best = NULL; + for (dtd = fy_atomic_load(&dt->tag_datas); dtd; dtd = dtd->next) { + + /* first check in the bloom filter */ + bloom_pos = (int)(hash & (uint64_t)dtd->bloom_filter_mask); + bucket_pos = (int)(hash & (uint64_t)dtd->bucket_count_mask); + + bloom_hit = fy_id_is_used(dtd->bloom_id, dtd->bloom_id_count, bloom_pos); + if (bloom_hit) { + for (de = fy_atomic_load(&dtd->buckets[bucket_pos]); de; de = de->next) { + if (de->hash == hash) { + if (total_size == de->size && !fy_iovec_cmp(iov, iovcnt, de->mem) && + ((uintptr_t)de->mem & (align - 1)) == 0) { + + /* only update stats if someone asked for it */ + if (a->flags & FYAF_KEEP_STATS) + fy_atomic_fetch_add(&dt->dup_stores, 1); + + if (a->flags & FYAF_TRACE) { + int i; + size_t j; + printf("%s: %p: dup-store %p 0x%016lx %zx:", __func__, a, de->mem, hash, total_size); + for (i = 0; i < iovcnt; i++) { + for (j = 0; j < iov[i].iov_len; j++) { + printf("%02x", (int)(((uint8_t *)iov[i].iov_base)[j]) & 0xff); + } + } + printf("\n"); + } + + return de->mem; + } + + /* mark that we had a collision here */ + if (a->flags & FYAF_KEEP_STATS) + fy_atomic_fetch_add(&dt->collisions, 1); + } + + if (at_head) + chain_length++; + } + } else { + if (!dtd_best) + dtd_best = dtd; /* keep whenever we had a empty bloom slot */ + } + at_head = false; + } + + /* use the one that had space, otherwise the top */ + if (dtd_best) + dtd = dtd_best; + else + dtd = fy_atomic_load(&dt->tag_datas); + + /* recalc positions for the delected dtd */ + bloom_pos = (int)(hash & (uint64_t)dtd->bloom_filter_mask); + bucket_pos = (int)(hash & (uint64_t)dtd->bucket_count_mask); + + /* we might be retrying; don't allocate and copy again */ + if (!mem) { + /* place the dedup entry at the aligned offset after the data */ + de_offset = fy_size_t_align(total_size, _Alignof(struct fy_dedup_entry)); + max_align = align > _Alignof(struct fy_dedup_entry) ? align : _Alignof(struct fy_dedup_entry); + mem = fy_allocator_alloc_nocheck(da->parent_allocator, dt->content_tag, de_offset + sizeof(*de), max_align); + if (!mem) + return NULL; + + /* verify it's aligned correctly */ + assert(((uintptr_t)mem & (align - 1)) == 0); + + de = mem + de_offset; + + de->hash = hash; + de->size = total_size; + de->mem = mem; + + /* and copy the data */ + fy_iovec_copy_from(iov, iovcnt, de->mem); + } + + /* set this bucket to used */ + fy_id_set_used(dtd->buckets_in_use, dtd->bucket_id_count, bucket_pos); + + /* turn the update bit for the bloom position */ + fy_id_set_used(dtd->bloom_id, dtd->bloom_id_count, bloom_pos); + + /* add to the bucket last atomically */ + de_head = fy_atomic_load(&dtd->buckets[bucket_pos]); + de->next = de_head; + if (!fy_atomic_compare_exchange_strong(&dtd->buckets[bucket_pos], &de_head, de)) + goto again; // we're leaking the entry, but that's fine + + /* adjust by one bit, if we've hit the trigger */ + if (chain_length > dtd->chain_length_grow_trigger) + fy_dedup_tag_adjust(da, dt, 1, 1); + + /* only update stats if someone asked for it */ + if (a->flags & FYAF_KEEP_STATS) + fy_atomic_fetch_add(&dt->unique_stores, 1); + + if (a->flags & FYAF_TRACE) { + int i; + size_t j; + printf("%s: %p: new-store %p 0x%016lx %zx:", __func__, a, de->mem, hash, total_size); + for (i = 0; i < iovcnt; i++) { + for (j = 0; j < iov[i].iov_len; j++) { + printf("%02x", (int)(((uint8_t *)iov[i].iov_base)[j]) & 0xff); + } + } + printf("\n"); + } + + return de->mem; +} + +static void fy_dedup_release(struct fy_allocator *a, int tag, const void *data, size_t size) +{ + /* we do nothing */ +} + +static int fy_dedup_get_tag(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt = NULL; + int tag; + int id, rc; + + da = container_of(a, struct fy_dedup_allocator, a); + + /* for a single tag, just return 0 */ + if (!(da->parent_caps & FYACF_CAN_FREE_TAG)) + return 0; + + /* and one from us */ + id = fy_id_alloc(da->ids, da->tag_id_count); + if (id < 0) + goto err_out; + + tag = (int)id; + + dt = fy_dedup_tag_from_tag(da, tag); + assert(dt); + + rc = fy_dedup_tag_setup(da, dt); + if (rc) + goto err_out; + + return tag; + +err_out: + fy_dedup_tag_cleanup(da, dt); + if (id >= 0) + fy_id_free(da->ids, da->tag_id_count, id); + return FY_ALLOC_TAG_ERROR; +} + +static void fy_dedup_release_tag(struct fy_allocator *a, int tag) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + + da = container_of(a, struct fy_dedup_allocator, a); + + /* for a single tag, just return */ + if (!(da->parent_caps & FYACF_CAN_FREE_TAG)) + return; + + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return; + + fy_dedup_tag_cleanup(da, dt); + + fy_id_free(da->ids, da->tag_count, tag); +} + +static int fy_dedup_get_tag_count(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + + da = container_of(a, struct fy_dedup_allocator, a); + return da->tag_count; +} + +static int fy_dedup_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag_data *dtd; + fy_id_bits *ids = NULL, *alloc_ids = NULL; + struct fy_dedup_tag *dt, *dt_new, *tags = NULL, *alloc_tags = NULL; + unsigned int i, tag_count, tag_id_count; + size_t tmpsz; + int rc; + + if (count < 1) + return -1; + + da = container_of(a, struct fy_dedup_allocator, a); + + if (count == da->tag_count) + return 0; + + tag_count = count; + tag_id_count = (tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + /* check if all content tag are within limits */ + for (i = 0; i < da->tag_count; i++) { + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + if (dt->content_tag >= (int)tag_count) + return -1; + } + + if (count > da->tag_count) { + /* we need to grow */ + tmpsz = tag_id_count * sizeof(*da->ids); + alloc_ids = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(fy_id_bits)); + if (!alloc_ids) + goto err_out; + fy_id_reset(alloc_ids, tag_id_count); + + tmpsz = tag_count * sizeof(*tags); + alloc_tags = fy_parent_allocator_alloc(&da->a, tmpsz, _Alignof(struct fy_dedup_tag)); + if (!alloc_tags) + goto err_out; + } + + /* shrink?, just clip */ + if (count < da->tag_count) { + tags = da->tags; + ids = da->ids; + for (i = count; i < da->tag_count; i++) { + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + fy_dedup_tag_cleanup(da, dt); + fy_id_free(da->ids, da->tag_id_count, i); + } + } else { + tags = alloc_tags; + ids = alloc_ids; + /* copy over the old entries */ + for (i = 0; i < da->tag_count; i++) { + + dt = fy_dedup_tag_from_tag(da, i); + if (!dt) + continue; + + dt_new = &tags[i]; + + fy_dedup_tag_setup(da, dt_new); + + /* move the old entries over */ + do { + dtd = fy_atomic_load(&dt->tag_datas); + } while (fy_atomic_compare_exchange_strong(&dt->tag_datas, &dtd, NULL)); + fy_atomic_store(&dt_new->tag_datas, dtd); + dt_new->content_tag = dt->content_tag; + fy_atomic_flag_clear(&dt_new->growing); + fy_atomic_store(&dt_new->unique_stores, fy_atomic_load(&dt->unique_stores)); + fy_atomic_store(&dt_new->dup_stores, fy_atomic_load(&dt->dup_stores)); + fy_atomic_store(&dt_new->collisions, fy_atomic_load(&dt->collisions)); + + /* clean the old entry */ + fy_dedup_tag_cleanup(da, dt); + + fy_id_set_used(ids, tag_id_count, i); + } + } + + /* ok, drop the parent bits first */ + rc = fy_allocator_set_tag_count(da->parent_allocator, tag_count); + if (rc) + goto err_out; + + /* switch */ + if (da->tags != tags) { + da->tags = tags; + fy_parent_allocator_free(&da->a, da->tags); + } + + if (da->ids != ids) { + da->ids = ids; + fy_parent_allocator_free(&da->a, da->ids); + } + + da->tag_count = tag_count; + da->tag_id_count = tag_id_count; + + return 0; + +err_out: + fy_parent_allocator_free(&da->a, alloc_tags); + fy_parent_allocator_free(&da->a, alloc_ids); + return -1; +} + +static void fy_dedup_trim_tag(struct fy_allocator *a, int tag) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + + da = container_of(a, struct fy_dedup_allocator, a); + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return; + + fy_dedup_tag_trim(da, dt); +} + +static void fy_dedup_reset_tag(struct fy_allocator *a, int tag) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + + da = container_of(a, struct fy_dedup_allocator, a); + + /* if it can't free an individual tag it can't reset it */ + if (!(da->parent_caps & FYACF_CAN_FREE_TAG)) + return; + + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return; + + fy_dedup_tag_reset(da, dt); +} + +static struct fy_allocator_info * +fy_dedup_get_info(struct fy_allocator *a, int tag) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + unsigned int i; + + /* full dump not supported yet */ + if (tag == FY_ALLOC_TAG_NONE) + return NULL; + + da = container_of(a, struct fy_dedup_allocator, a); + + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + return NULL; + + info = fy_allocator_get_info(da->parent_allocator, dt->content_tag); + if (!info) + return NULL; + + /* we will have to change the tag from content to this one */ + for (i = 0; i < info->num_tag_infos; i++) { + tag_info = &info->tag_infos[i]; + if (tag_info->tag == dt->content_tag) + tag_info->tag = tag; + } + + return info; +} + +static enum fy_allocator_cap_flags +fy_dedup_get_caps(struct fy_allocator *a) +{ + struct fy_dedup_allocator *da; + enum fy_allocator_cap_flags flags; + + da = container_of(a, struct fy_dedup_allocator, a); + + flags = da->parent_caps | FYACF_CAN_DEDUP | FYACF_CAN_LOOKUP; + flags &= ~FYACF_CAN_FREE_INDIVIDUAL; + + return flags; +} + +static bool fy_dedup_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + int tag_start, tag_end; + + da = container_of(a, struct fy_dedup_allocator, a); + if (!(da->parent_caps & FYACF_HAS_CONTAINS)) + return false; + + if (tag >= 0) { + if (tag >= (int)da->tag_count) + return false; + tag_start = tag; + tag_end = tag + 1; + } else { + tag_start = 0; + tag_end = (int)da->tag_count; + } + + for (tag = tag_start; tag < tag_end; tag++) { + dt = fy_dedup_tag_from_tag(da, tag); + if (!dt) + continue; + + if (fy_allocator_contains(da->parent_allocator, dt->content_tag, ptr)) + return true; + } + return false; +} + +const struct fy_allocator_ops fy_dedup_allocator_ops = { + .setup = fy_dedup_setup, + .cleanup = fy_dedup_cleanup, + .create = fy_dedup_create, + .destroy = fy_dedup_destroy, + .dump = fy_dedup_dump, + .alloc = fy_dedup_alloc, + .free = fy_dedup_free, + .update_stats = fy_dedup_update_stats, + .storev = fy_dedup_storev, + .lookupv = fy_dedup_lookupv, + .release = fy_dedup_release, + .get_tag = fy_dedup_get_tag, + .release_tag = fy_dedup_release_tag, + .get_tag_count = fy_dedup_get_tag_count, + .set_tag_count = fy_dedup_set_tag_count, + .trim_tag = fy_dedup_trim_tag, + .reset_tag = fy_dedup_reset_tag, + .get_info = fy_dedup_get_info, + .get_caps = fy_dedup_get_caps, + .contains = fy_dedup_contains, +}; diff --git a/src/allocator/fy-allocator-dedup.h b/src/allocator/fy-allocator-dedup.h new file mode 100644 index 0000000..967fe23 --- /dev/null +++ b/src/allocator/fy-allocator-dedup.h @@ -0,0 +1,89 @@ +/* + * fy-allocator-dedup.h - the dedup allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_DEDUP_H +#define FY_ALLOCATOR_DEDUP_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "fy-typelist.h" +#include "fy-id.h" +#include "xxhash.h" + +#include "fy-atomics.h" +#include "fy-allocator.h" + +#define FY_DEDUP_TAG_MAX 128 + +struct fy_dedup_entry { + struct fy_dedup_entry *next; + uint64_t hash; + size_t size; + void *mem; +}; + +struct fy_dedup_tag; + +struct fy_dedup_tag_data_cfg { + struct fy_dedup_allocator *da; + struct fy_dedup_tag *dt; + unsigned int bloom_filter_bits; + unsigned int bucket_count_bits; + size_t dedup_threshold; + unsigned int chain_length_grow_trigger; +}; + +struct fy_dedup_tag_data { + struct fy_dedup_tag_data *next; + struct fy_dedup_tag_data_cfg cfg; + unsigned int bloom_filter_bits; + unsigned int bloom_filter_mask; + size_t bloom_id_count; + fy_id_bits *bloom_id; + fy_id_bits *bloom_update_id; + unsigned int bucket_count_bits; + unsigned int bucket_count_mask; + size_t bucket_count; + FY_ATOMIC(struct fy_dedup_entry *) *buckets; + size_t bucket_id_count; + fy_id_bits *buckets_in_use; + size_t dedup_threshold; + unsigned int chain_length_grow_trigger; +}; + +struct fy_dedup_tag { + FY_ATOMIC(struct fy_dedup_tag_data *)tag_datas; + int content_tag; + fy_atomic_flag growing; + FY_ATOMIC(uint64_t) unique_stores; + FY_ATOMIC(uint64_t) dup_stores; + FY_ATOMIC(uint64_t) collisions; +}; + +struct fy_dedup_allocator { + struct fy_allocator a; + struct fy_dedup_allocator_cfg cfg; + struct fy_allocator *parent_allocator; + enum fy_allocator_cap_flags parent_caps; + int entries_tag; + unsigned int bloom_filter_bits; + unsigned int bucket_count_bits; + size_t dedup_threshold; + unsigned int chain_length_grow_trigger; + fy_id_bits *ids; + unsigned int tag_id_count; + struct fy_dedup_tag *tags; + unsigned int tag_count; +}; + +extern const struct fy_allocator_ops fy_dedup_allocator_ops; + +#define FY_DEDUP_XXHASH64_SEED FY_XXHASH64_SEED + +#endif diff --git a/src/allocator/fy-allocator-linear.c b/src/allocator/fy-allocator-linear.c new file mode 100644 index 0000000..96ae891 --- /dev/null +++ b/src/allocator/fy-allocator-linear.c @@ -0,0 +1,391 @@ +/* + * fy-allocator-linear.c - linear allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-utils.h" +#include "fy-allocator-linear.h" + +static int fy_linear_setup(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *cfg_data) +{ + struct fy_linear_allocator *la; + const struct fy_linear_allocator_cfg *cfg; + void *buf, *alloc = NULL; + bool need_zero; + + if (!a || !cfg_data) + return -1; + + cfg = cfg_data; + if (!cfg->size) + return -1; + + if (!cfg->buf) { + alloc = fy_early_parent_allocator_alloc(parent, parent_tag, cfg->size, 0); + if (!alloc) + goto err_out; + buf = alloc; + need_zero = true; + } else { + buf = cfg->buf; + need_zero = false; + } + + la = container_of(a, struct fy_linear_allocator, a); + memset(la, 0, sizeof(*la)); + la->a.name = "linear"; + la->a.ops = &fy_linear_allocator_ops; + la->a.parent = parent; + la->a.parent_tag = parent_tag; + la->cfg = *cfg; + la->alloc = alloc; + la->start = buf; + la->need_zero = need_zero; + + fy_atomic_store(&la->next, 0); + + return 0; +err_out: + fy_early_parent_allocator_free(parent, parent_tag, alloc); + return -1; +} + +static void fy_linear_cleanup(struct fy_allocator *a) +{ + struct fy_linear_allocator *la; + + if (!a) + return; + + la = container_of(a, struct fy_linear_allocator, a); + fy_parent_allocator_free(&la->a, la->alloc); + la->alloc = NULL; +} + +struct fy_allocator *fy_linear_create(struct fy_allocator *parent, int parent_tag, const void *cfg_data) +{ + struct fy_linear_allocator *la; + const struct fy_linear_allocator_cfg *cfg; + struct fy_linear_allocator_cfg newcfg; + void *s, *e, *buf, *alloc = NULL; + int rc; + + if (!cfg_data) + return NULL; + + cfg = cfg_data; + if (!cfg->size) + return NULL; + + if (!cfg->buf) { + alloc = fy_early_parent_allocator_alloc(parent, parent_tag, cfg->size, 0); + if (!alloc) + goto err_out; + buf = alloc; + } else + buf = cfg->buf; + + s = buf; + e = s + cfg->size; + + s = fy_ptr_align(s, _Alignof(struct fy_linear_allocator)); + if ((size_t)(e - s) < sizeof(*la)) + goto err_out; + + la = s; + s += sizeof(*la); + + memset(&newcfg, 0, sizeof(newcfg)); + newcfg.buf = s; + newcfg.size = (size_t)(e - s); + + rc = fy_linear_setup(&la->a, parent, parent_tag, &newcfg); + if (rc) + goto err_out; + + la->alloc = alloc; + + return &la->a; + +err_out: + fy_early_parent_allocator_free(parent, parent_tag, alloc); + return NULL; +} + +void fy_linear_destroy(struct fy_allocator *a) +{ + struct fy_linear_allocator *la; + struct fy_allocator *parent; + int parent_tag; + void *alloc; + + la = container_of(a, struct fy_linear_allocator, a); + + /* take out the allocation of create */ + alloc = la->alloc; + la->alloc = NULL; + parent = la->a.parent; + parent_tag = la->a.parent_tag; + + fy_linear_cleanup(a); + + fy_early_parent_allocator_free(parent, parent_tag, alloc); +} + +void fy_linear_dump(struct fy_allocator *a) +{ + struct fy_linear_allocator *la; + size_t next; + + la = container_of(a, struct fy_linear_allocator, a); + + next = fy_atomic_load(&la->next); + fprintf(stderr, "linear: total %zu used %zu free %zu\n", + la->cfg.size, next, la->cfg.size - la->next); +} + +static void *fy_linear_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + struct fy_linear_allocator *la; + size_t next, new_next, real_size; + void *s; + + assert(a); + + la = container_of(a, struct fy_linear_allocator, a); + + /* atomically update the pointer */ + do { + next = fy_atomic_load(&la->next); + s = fy_ptr_align(la->start + next, align); + new_next = (s + size) - la->start; + /* handle both overflow and underflow */ + if (new_next > la->cfg.size) + goto err_out; + } while (!fy_atomic_compare_exchange_strong(&la->next, &next, new_next)); + + real_size = new_next - next; + + /* only update stats if someone asked for it */ + if (a->flags & FYAF_KEEP_STATS) { + fy_atomic_fetch_add(&la->allocations, 1); + fy_atomic_fetch_add(&la->allocated, real_size); + } + + /* zero out buffer if not guaranteed */ + if (la->need_zero) + memset(s, 0, size); + + return s; + +err_out: + return NULL; +} + +static void fy_linear_free(struct fy_allocator *a, int tag, void *data) +{ + /* linear allocator does not free anything */ +} + +static int fy_linear_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + struct fy_linear_allocator *la; + + la = container_of(a, struct fy_linear_allocator, a); + + stats->allocations = fy_atomic_get_and_clear_counter(&la->allocations); + stats->allocated = fy_atomic_get_and_clear_counter(&la->allocated); + + return 0; +} + +static const void *fy_linear_storev(struct fy_allocator *a, int tag, + const struct iovec *iov, int iovcnt, size_t align, + uint64_t hash) +{ + struct fy_linear_allocator *la; + void *p; + size_t size; + + size = fy_iovec_size(iov, iovcnt); + if (size == SIZE_MAX) + return NULL; + + p = fy_linear_alloc(a, tag, size, align); + if (!p) + return NULL; + + fy_iovec_copy_from(iov, iovcnt, p); + + if (a->flags & FYAF_KEEP_STATS) { + la = container_of(a, struct fy_linear_allocator, a); + + fy_atomic_fetch_add(&la->stores, 1); + fy_atomic_fetch_add(&la->stores, size); + } + + return p; +} + +static const void *fy_linear_lookupv(struct fy_allocator *a, int tag, + const struct iovec *iov, int iovcnt, size_t align, + uint64_t hash) +{ + /* no way to lookup */ + return NULL; +} + +static void fy_linear_release(struct fy_allocator *a, int tag, const void *data, size_t size) +{ + /* nothing */ +} + +static int fy_linear_get_tag(struct fy_allocator *a) +{ + /* always return 0, we don't do tags for linear */ + return 0; +} + +static void fy_linear_release_tag(struct fy_allocator *a, int tag) +{ + /* nothing */ +} + +static int fy_linear_get_tag_count(struct fy_allocator *a) +{ + return 1; +} + +static int fy_linear_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + if (count != 1) + return -1; + return 0; +} + +static void fy_linear_trim_tag(struct fy_allocator *a, int tag) +{ + /* nothing */ +} + +static void fy_linear_reset_tag(struct fy_allocator *a, int tag) +{ + struct fy_linear_allocator *la; + + if (tag) + return; + + la = container_of(a, struct fy_linear_allocator, a); + + /* we just rewind */ + fy_atomic_store(&la->next, 0); +} + +static struct fy_allocator_info *fy_linear_get_info(struct fy_allocator *a, int tag) +{ + struct fy_linear_allocator *la; + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + struct fy_allocator_arena_info *arena_info; + size_t next, size; + + /* only single tag (or all tags with 0) */ + if (tag != 0 && tag != FY_ALLOC_TAG_NONE) + return NULL; + + la = container_of(a, struct fy_linear_allocator, a); + + /* one of each */ + size = sizeof(*info) + + sizeof(*tag_info) + + sizeof(*arena_info); + + info = malloc(size); + if (!info) + return NULL; + memset(info, 0, sizeof(*info)); + + tag_info = (void *)(info + 1); + assert(((uintptr_t)tag_info % _Alignof(struct fy_allocator_tag_info)) == 0); + arena_info = (void *)(tag_info + 1); + assert(((uintptr_t)arena_info % _Alignof(struct fy_allocator_arena_info)) == 0); + + /* fill-in the single arena */ + next = fy_atomic_load(&la->next); + + arena_info->free = la->cfg.size - next; + arena_info->used = next; + arena_info->total = la->cfg.size; + arena_info->data = la->start; + arena_info->size = arena_info->used; + + /* there's a single tag for linear */ + tag_info->tag = 0; + tag_info->free = arena_info->free; + tag_info->used = arena_info->used; + tag_info->total = arena_info->total; + tag_info->num_arena_infos = 1; + tag_info->arena_infos = arena_info; + + /* fill in the single tag */ + info->free = tag_info->free; + info->used = tag_info->used; + info->total = tag_info->total; + info->num_tag_infos = 1; + info->tag_infos = tag_info; + + return info; +} + +static enum fy_allocator_cap_flags +fy_linear_get_caps(struct fy_allocator *a) +{ + return FYACF_HAS_CONTAINS | FYACF_HAS_EFFICIENT_CONTAINS; +} + +static bool fy_linear_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + struct fy_linear_allocator *la; + + la = container_of(a, struct fy_linear_allocator, a); + return ptr >= la->start && ptr < (la->start + la->cfg.size); +} + +const struct fy_allocator_ops fy_linear_allocator_ops = { + .setup = fy_linear_setup, + .cleanup = fy_linear_cleanup, + .create = fy_linear_create, + .destroy = fy_linear_destroy, + .dump = fy_linear_dump, + .alloc = fy_linear_alloc, + .free = fy_linear_free, + .update_stats = fy_linear_update_stats, + .storev = fy_linear_storev, + .lookupv = fy_linear_lookupv, + .release = fy_linear_release, + .get_tag = fy_linear_get_tag, + .release_tag = fy_linear_release_tag, + .get_tag_count = fy_linear_get_tag_count, + .set_tag_count = fy_linear_set_tag_count, + .trim_tag = fy_linear_trim_tag, + .reset_tag = fy_linear_reset_tag, + .get_info = fy_linear_get_info, + .get_caps = fy_linear_get_caps, + .contains = fy_linear_contains, +}; diff --git a/src/allocator/fy-allocator-linear.h b/src/allocator/fy-allocator-linear.h new file mode 100644 index 0000000..a30a9e2 --- /dev/null +++ b/src/allocator/fy-allocator-linear.h @@ -0,0 +1,34 @@ +/* + * fy-allocator-linear.h - the linear allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_LINEAR_H +#define FY_ALLOCATOR_LINEAR_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "fy-atomics.h" +#include "fy-allocator.h" + +struct fy_linear_allocator { + FY_ATOMIC(size_t) next FY_CACHE_ALIGNED; // hot hot hot + struct fy_allocator a; + struct fy_linear_allocator_cfg cfg; + void *alloc; + void *start; + bool need_zero; + /* no need to keep anything else */ + FY_ATOMIC(uint64_t) allocations; + FY_ATOMIC(uint64_t) allocated; + FY_ATOMIC(uint64_t) stores; + FY_ATOMIC(uint64_t) stored; +}; + +extern const struct fy_allocator_ops fy_linear_allocator_ops; + +#endif diff --git a/src/allocator/fy-allocator-malloc.c b/src/allocator/fy-allocator-malloc.c new file mode 100644 index 0000000..7c06185 --- /dev/null +++ b/src/allocator/fy-allocator-malloc.c @@ -0,0 +1,717 @@ +/* + * fy-allocator-malloc.c - malloc allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include + +#include + +/* for container_of */ +#include "fy-list.h" +#include "fy-utils.h" + +#include "fy-allocator-malloc.h" + +static inline void +fy_malloc_tag_list_lock(struct fy_malloc_tag *mt) +{ + int loops FY_DEBUG_UNUSED; + + loops = 0; + while (!fy_atomic_flag_test_and_set(&mt->lock)) { + loops++; + assert(loops < 10000000); + fy_cpu_relax(); + } +} + +static inline void +fy_malloc_tag_list_unlock(struct fy_malloc_tag *mt) +{ + fy_atomic_flag_clear(&mt->lock); +} + +static inline struct fy_malloc_tag * +fy_malloc_tag_from_tag(struct fy_malloc_allocator *ma, int tag) +{ + if (!ma) + return NULL; + + if ((unsigned int)tag >= ma->tag_count) + return NULL; + + if (!fy_id_is_used(ma->ids, ma->tag_id_count, (int)tag)) + return NULL; + + return &ma->tags[tag]; +} + +static void fy_malloc_tag_setup(struct fy_malloc_allocator *ma, struct fy_malloc_tag *mt) +{ + fy_malloc_entry_list_init(&mt->entries); + atomic_flag_clear(&mt->lock); +} + +static void fy_malloc_tag_cleanup(struct fy_malloc_allocator *ma, struct fy_malloc_tag *mt) +{ + struct fy_malloc_entry *me; + + /* cleanup should happen from a single thread */ + while ((me = fy_malloc_entry_list_pop(&mt->entries)) != NULL) + free(me->mem); +} + +static void fy_malloc_cleanup(struct fy_allocator *a) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + unsigned int i; + + if (!a) + return; + + ma = container_of(a, struct fy_malloc_allocator, a); + + /* no need for locks, this is one thread only */ + for (i = 0; i < ma->tag_count; i++) { + mt = fy_malloc_tag_from_tag(ma, i); + if (!mt) + continue; + fy_malloc_tag_cleanup(ma, mt); + } + + fy_parent_allocator_free(&ma->a, ma->ids); + fy_parent_allocator_free(&ma->a, ma->tags); +} + +static int fy_malloc_setup(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *data) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + size_t tmpsz; + + if (!a) + return -1; + + ma = container_of(a, struct fy_malloc_allocator, a); + memset(ma, 0, sizeof(*ma)); + ma->a.name = "malloc"; + ma->a.ops = &fy_malloc_allocator_ops; + ma->a.parent = parent; + ma->a.parent_tag = parent_tag; + + ma->tag_count = FY_MALLOC_TAG_MAX; + ma->tag_id_count = (ma->tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + tmpsz = ma->tag_id_count * sizeof(*ma->ids); + ma->ids = fy_parent_allocator_alloc(&ma->a, tmpsz, _Alignof(fy_id_bits)); + if (!ma->ids) + goto err_out; + fy_id_reset(ma->ids, ma->tag_id_count); + + tmpsz = ma->tag_count * sizeof(*ma->tags); + ma->tags = fy_parent_allocator_alloc(&ma->a, tmpsz, _Alignof(struct fy_malloc_tag)); + if (!ma->tags) + goto err_out; + + /* start with tag 0 as general use */ + fy_id_set_used(ma->ids, ma->tag_id_count, 0); + mt = fy_malloc_tag_from_tag(ma, 0); + fy_malloc_tag_setup(ma, mt); + + return 0; + +err_out: + fy_malloc_cleanup(a); + return -1; +} + +struct fy_allocator *fy_malloc_create(struct fy_allocator *parent, int parent_tag, const void *cfg) +{ + struct fy_malloc_allocator *ma = NULL; + int rc; + + ma = fy_early_parent_allocator_alloc(parent, parent_tag, sizeof(*ma), _Alignof(struct fy_malloc_allocator)); + if (!ma) + goto err_out; + + rc = fy_malloc_setup(&ma->a, parent, parent_tag, cfg); + if (rc) + goto err_out; + + return &ma->a; + +err_out: + fy_early_parent_allocator_free(parent, parent_tag, ma); + return NULL; +} + +void fy_malloc_destroy(struct fy_allocator *a) +{ + struct fy_malloc_allocator *ma; + + ma = container_of(a, struct fy_malloc_allocator, a); + fy_malloc_cleanup(a); + + fy_parent_allocator_free(a, ma); +} + +void fy_malloc_dump(struct fy_allocator *a) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + struct fy_malloc_entry *me; + unsigned int i; + size_t count, total, system_total; + + ma = container_of(a, struct fy_malloc_allocator, a); + + fprintf(stderr, "malloc: "); + for (i = 0; i < ma->tag_count; i++) { + mt = fy_malloc_tag_from_tag(ma, i); + if (!mt) + continue; + fy_malloc_tag_list_lock(mt); + fprintf(stderr, "%c", fy_malloc_entry_list_empty(&mt->entries) ? '.' : 'x'); + fy_malloc_tag_list_unlock(mt); + } + fprintf(stderr, "\n"); + + for (i = 0; i < ma->tag_count; i++) { + mt = fy_malloc_tag_from_tag(ma, i); + if (!mt) + continue; + fy_malloc_tag_list_lock(mt); + count = total = system_total = 0; + if (!fy_malloc_entry_list_empty(&mt->entries)) { + for (me = fy_malloc_entry_list_head(&mt->entries); me; me = fy_malloc_entry_next(&mt->entries, me)) { + count++; + total += me->size; + system_total += sizeof(*me) + me->size; + } + } + fy_malloc_tag_list_unlock(mt); + if (!count) + continue; + fprintf(stderr, " %d: count %zu total %zu system %zu overhead %zu (%2.2f%%)\n", i, + count, total, system_total, system_total - total, + 100.0 * (double)(system_total - total) / (double)system_total); + } +} + +static void *fy_malloc_tag_alloc(struct fy_malloc_allocator *ma, struct fy_malloc_tag *mt, size_t size, size_t align) +{ + struct fy_malloc_entry *me; + size_t me_offset, max_align; + int r; + void *mem; + + me_offset = fy_size_t_align(size, _Alignof(struct fy_malloc_entry)); + max_align = align > _Alignof(struct fy_malloc_entry) ? align : _Alignof(struct fy_malloc_entry); + + r = posix_memalign(&mem, max_align, me_offset + sizeof(*me)); + if (r) + return NULL; + + me = mem + me_offset; + + me->size = size; + fy_malloc_tag_list_lock(mt); + fy_malloc_entry_list_add_tail(&mt->entries, me); + fy_malloc_tag_list_unlock(mt); + me->mem = mem; + + return mem; +} + +static void fy_malloc_tag_free(struct fy_malloc_allocator *ma, struct fy_malloc_tag *mt, void *data) +{ + struct fy_malloc_entry *me; + + me = container_of(data, struct fy_malloc_entry, mem); + + fy_malloc_tag_list_lock(mt); + fy_malloc_entry_list_del(&mt->entries, me); + fy_malloc_tag_list_unlock(mt); + + free(me->mem); +} + +static void *fy_malloc_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + void *p; + + ma = container_of(a, struct fy_malloc_allocator, a); + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + goto err_out; + + p = fy_malloc_tag_alloc(ma, mt, size, align); + if (!p) + goto err_out; + + mt->stats.allocations++; + mt->stats.allocated += size; + + return p; + +err_out: + return NULL; +} + +static void fy_malloc_free(struct fy_allocator *a, int tag, void *data) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + + ma = container_of(a, struct fy_malloc_allocator, a); + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + return; + + fy_malloc_tag_free(ma, mt, data); + + mt->stats.frees++; +} + +static int fy_malloc_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + unsigned int i; + + ma = container_of(a, struct fy_malloc_allocator, a); + + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + goto err_out; + + /* and update with this ones */ + for (i = 0; i < ARRAY_SIZE(mt->stats.counters); i++) { + stats->counters[i] += mt->stats.counters[i]; + mt->stats.counters[i] = 0; + } + + return 0; + +err_out: + return -1; +} + +static const void *fy_malloc_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + void *start; + size_t total_size; + + ma = container_of(a, struct fy_malloc_allocator, a); + + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + goto err_out; + + total_size = fy_iovec_size(iov, iovcnt); + if (total_size == SIZE_MAX) + goto err_out; + + start = fy_malloc_tag_alloc(ma, mt, total_size, align); + if (!start) + goto err_out; + + fy_iovec_copy_from(iov, iovcnt, start); + + mt->stats.stores++; + mt->stats.stored += total_size; + + return start; + +err_out: + return NULL; +} + +static const void *fy_malloc_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + /* no lookups */ + return NULL; +} + +static void fy_malloc_release(struct fy_allocator *a, int tag, const void *data, size_t size) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + + ma = container_of(a, struct fy_malloc_allocator, a); + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + return; + + /* the malloc's release is just a free */ + fy_malloc_tag_free(ma, mt, (void *)data); + + mt->stats.releases++; + mt->stats.released += size; +} + +static void fy_malloc_release_tag(struct fy_allocator *a, int tag) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + + ma = container_of(a, struct fy_malloc_allocator, a); + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + return; + + fy_malloc_tag_cleanup(ma, mt); + + fy_id_free(ma->ids, ma->tag_id_count, tag); +} + +static int fy_malloc_get_tag(struct fy_allocator *a) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + int id; + + ma = container_of(a, struct fy_malloc_allocator, a); + + id = fy_id_alloc(ma->ids, ma->tag_id_count); + if (id < 0) + goto err_out; + + mt = fy_malloc_tag_from_tag(ma, id); + if (!mt) + goto err_out; + + fy_malloc_tag_setup(ma, mt); + + return (int)id; + +err_out: + return FY_ALLOC_TAG_ERROR; +} + +static int fy_malloc_get_tag_count(struct fy_allocator *a) +{ + struct fy_malloc_allocator *ma; + + ma = container_of(a, struct fy_malloc_allocator, a); + return ma->tag_count; +} + +static int fy_malloc_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + struct fy_malloc_allocator *ma; + fy_id_bits *ids = NULL; + struct fy_malloc_tag *mt, *mt_new, *tags = NULL; + struct fy_malloc_entry *me; + unsigned int i, tag_count, tag_id_count; + size_t tmpsz; + + if (count < 1) + return -1; + + ma = container_of(a, struct fy_malloc_allocator, a); + if (count == ma->tag_count) + return 0; + + tag_count = count; + tag_id_count = (tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + /* shrink?, just clip */ + if (count < ma->tag_count) { + for (i = count; i < ma->tag_count; i++) { + mt = fy_malloc_tag_from_tag(ma, i); + if (!mt) + continue; + fy_malloc_tag_cleanup(ma, mt); + fy_id_free(ma->ids, ma->tag_id_count, i); + } + tags = ma->tags; + ids = ma->ids; + } else { + /* we need to grow */ + tmpsz = tag_id_count * sizeof(*ma->ids); + ids = fy_parent_allocator_alloc(&ma->a, tmpsz, _Alignof(fy_id_bits)); + if (!ids) + goto err_out; + fy_id_reset(ids, tag_id_count); + + tmpsz = tag_count * sizeof(*tags); + tags = fy_parent_allocator_alloc(&ma->a, tmpsz, _Alignof(struct fy_malloc_tag)); + if (!tags) + goto err_out; + + /* copy over the old entries */ + for (i = 0; i < ma->tag_count; i++) { + + mt = fy_malloc_tag_from_tag(ma, i); + if (!mt) + continue; + + mt_new = &tags[i]; + + fy_malloc_tag_setup(ma, mt_new); + /* move the old entries over */ + fy_malloc_tag_list_lock(mt); + while ((me = fy_malloc_entry_list_pop(&mt->entries)) != NULL) + fy_malloc_entry_list_add_tail(&mt_new->entries, me); + fy_malloc_tag_list_unlock(mt); + memcpy(&mt_new->stats, &mt->stats, sizeof(mt->stats)); + + /* clean the old entry */ + fy_malloc_tag_cleanup(ma, mt); + + fy_id_set_used(ids, tag_id_count, i); + } + } + + if (ma->tags != tags) { + ma->tags = tags; + fy_parent_allocator_free(&ma->a, ma->tags); + } + if (ma->ids != ids) { + ma->ids = ids; + fy_parent_allocator_free(&ma->a, ma->ids); + } + ma->tag_count = tag_count; + ma->tag_id_count = tag_id_count; + return 0; + +err_out: + fy_parent_allocator_free(&ma->a, tags); + fy_parent_allocator_free(&ma->a, ids); + return -1; +} + +static void fy_malloc_trim_tag(struct fy_allocator *a, int tag) +{ + /* nothing */ +} + +static void fy_malloc_reset_tag(struct fy_allocator *a, int tag) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + struct fy_malloc_entry *me; + + ma = container_of(a, struct fy_malloc_allocator, a); + + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + return; + + /* no lock, reset is single thread only */ + while ((me = fy_malloc_entry_list_pop(&mt->entries)) != NULL) + free(me->mem); +} + +static struct fy_allocator_info * +fy_malloc_get_info(struct fy_allocator *a, int tag) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + struct fy_malloc_entry *me; + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + struct fy_allocator_arena_info *arena_info; + size_t size, free, used, total; + size_t tag_free, tag_used, tag_total; + size_t arena_free, arena_used, arena_total; + unsigned int num_tags, num_arenas, i; + int id; + + ma = container_of(a, struct fy_malloc_allocator, a); + + /* allocate for the worst case always */ + num_tags = 0; + num_arenas = 0; + + free = 0; + used = 0; + total = 0; + + /* two passes */ + for (i = 0; i < 2; i++) { + + if (!i) { + info = NULL; + tag_info = NULL; + arena_info = NULL; + + } else { + size = sizeof(*info) + + sizeof(*tag_info) * num_tags + + sizeof(*arena_info) * num_arenas; + + info = malloc(size); + if (!info) + return NULL; + memset(info, 0, sizeof(*info)); + + tag_info = (void *)(info + 1); + assert(((uintptr_t)tag_info % _Alignof(struct fy_allocator_tag_info)) == 0); + arena_info = (void *)(tag_info + num_tags); + assert(((uintptr_t)arena_info % _Alignof(struct fy_allocator_arena_info)) == 0); + + info->free = free; + info->used = used; + info->total = total; + + info->num_tag_infos = 0; + info->tag_infos = tag_info; + } + + free = 0; + used = 0; + total = sizeof(*ma); + + for (id = 0; id < (int)ma->tag_count; id++) { + + if (!fy_id_is_used(ma->ids, ma->tag_id_count, id)) + continue; + + mt = &ma->tags[id]; + + tag_free = 0; + tag_used = 0; + tag_total = 0; + + if (i) { + tag_info->num_arena_infos = 0; + tag_info->arena_infos = arena_info; + } + + fy_malloc_tag_list_lock(mt); + + for (me = fy_malloc_entry_list_head(&mt->entries); me; me = fy_malloc_entry_next(&mt->entries, me)) { + arena_free = 0; + arena_used = me->size; + arena_total = sizeof(*me) + me->size; + + tag_free += arena_free; + tag_used += arena_used; + tag_total += arena_total; + + if (!i) { + num_arenas++; + } else { + arena_info->free = arena_free; + arena_info->used = arena_used; + arena_info->total = arena_total; + arena_info->data = me->mem; + arena_info->size = me->size; + arena_info++; + tag_info->num_arena_infos++; + } + } + fy_malloc_tag_list_unlock(mt); + + if (!i) { + num_tags++; + } else { + + /* only store the tag if there's a match */ + if (tag == FY_ALLOC_TAG_NONE || tag == id) { + tag_info->tag = id; + tag_info->free = tag_free; + tag_info->used = tag_used; + tag_info->total = tag_total; + tag_info++; + info->num_tag_infos++; + } + } + + free += tag_free; + used += tag_used; + total += tag_total; + } + } + + return info; +} + +static enum fy_allocator_cap_flags +fy_malloc_get_caps(struct fy_allocator *a) +{ + return FYACF_CAN_FREE_INDIVIDUAL | FYACF_CAN_FREE_TAG | + FYACF_HAS_CONTAINS | FYACF_HAS_TAGS; +} + +/* malloc allocator actually tracks allocations, it's just very inefficient + * to scan through. It might be good for debugging though + */ +static bool fy_malloc_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + struct fy_malloc_allocator *ma; + struct fy_malloc_tag *mt; + struct fy_malloc_entry *me; + int tag_start, tag_end; + + ma = container_of(a, struct fy_malloc_allocator, a); + if (tag >= 0) { + if (tag >= (int)ma->tag_count) + return false; + tag_start = tag; + tag_end = tag_start + 1; + } else { + tag_start = 0; + tag_end = ma->tag_count; + } + + for (tag = tag_start; tag < tag_end; tag++) { + mt = fy_malloc_tag_from_tag(ma, tag); + if (!mt) + continue; + + fy_malloc_tag_list_lock(mt); + for (me = fy_malloc_entry_list_head(&mt->entries); me; + me = fy_malloc_entry_next(&mt->entries, me)) { + + if (ptr >= (void *)me->mem && ptr < (void *)me->mem + me->size) { + fy_malloc_tag_list_unlock(mt); + return true; + } + } + fy_malloc_tag_list_unlock(mt); + } + + return false; +} + +const struct fy_allocator_ops fy_malloc_allocator_ops = { + .setup = fy_malloc_setup, + .cleanup = fy_malloc_cleanup, + .create = fy_malloc_create, + .destroy = fy_malloc_destroy, + .dump = fy_malloc_dump, + .alloc = fy_malloc_alloc, + .free = fy_malloc_free, + .update_stats = fy_malloc_update_stats, + .storev = fy_malloc_storev, + .lookupv = fy_malloc_lookupv, + .release = fy_malloc_release, + .get_tag = fy_malloc_get_tag, + .release_tag = fy_malloc_release_tag, + .get_tag_count = fy_malloc_get_tag_count, + .set_tag_count = fy_malloc_set_tag_count, + .trim_tag = fy_malloc_trim_tag, + .reset_tag = fy_malloc_reset_tag, + .get_info = fy_malloc_get_info, + .get_caps = fy_malloc_get_caps, + .contains = fy_malloc_contains, +}; diff --git a/src/allocator/fy-allocator-malloc.h b/src/allocator/fy-allocator-malloc.h new file mode 100644 index 0000000..745d85d --- /dev/null +++ b/src/allocator/fy-allocator-malloc.h @@ -0,0 +1,49 @@ +/* + * fy-allocator-malloc.h - the malloc allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_MALLOC_H +#define FY_ALLOCATOR_MALLOC_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "fy-typelist.h" +#include "fy-id.h" + +#include "fy-atomics.h" +#include "fy-allocator.h" + +struct fy_malloc_tag; + +FY_TYPE_FWD_DECL_LIST(malloc_entry); +struct fy_malloc_entry { + struct list_head node; + size_t size; + void *mem; +}; +FY_TYPE_DECL_LIST(malloc_entry); + +#define FY_MALLOC_TAG_MAX 32 + +struct fy_malloc_tag { + fy_atomic_flag lock; + struct fy_malloc_entry_list entries; + struct fy_allocator_stats stats; +}; + +struct fy_malloc_allocator { + struct fy_allocator a; + fy_id_bits *ids; + unsigned int tag_id_count; + struct fy_malloc_tag *tags; + unsigned int tag_count; +}; + +extern const struct fy_allocator_ops fy_malloc_allocator_ops; + +#endif diff --git a/src/allocator/fy-allocator-mremap.c b/src/allocator/fy-allocator-mremap.c new file mode 100644 index 0000000..cbf16fb --- /dev/null +++ b/src/allocator/fy-allocator-mremap.c @@ -0,0 +1,1180 @@ +/* + * fy-allocator-mremap.c - mremap allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* for container_of */ +#include "fy-list.h" +#include "fy-utils.h" + +#include "fy-allocator-mremap.h" + +// #define DEBUG_ARENA + +// #define DISABLE_MREMAP + +#if HAVE_MREMAP && !defined(DISABLE_MREMAP) +#define USE_MREMAP 1 +#else +#define USE_MREMAP 0 +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_BIG_ALLOC_THRESHOLD +#define MREMAP_ALLOCATOR_DEFAULT_BIG_ALLOC_THRESHOLD SIZE_MAX +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_EMPTY_THRESHOLD +#define MREMAP_ALLOCATOR_DEFAULT_EMPTY_THRESHOLD 64 +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_MINIMUM_ARENA_SIZE +#define MREMAP_ALLOCATOR_DEFAULT_MINIMUM_ARENA_SIZE (1U << 20) /* 1M */ +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_GROW_RATIO +#define MREMAP_ALLOCATOR_DEFAULT_GROW_RATIO 2.0 +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_BALLON_RATIO +#define MREMAP_ALLOCATOR_DEFAULT_BALLON_RATIO 32.0 +#endif + +#ifndef MREMAP_ALLOCATOR_DEFAULT_ARENA_TYPE +#define MREMAP_ALLOCATOR_DEFAULT_ARENA_TYPE FYMRAT_MMAP +#endif + +static inline size_t fy_mremap_useable_arena_size(struct fy_mremap_allocator *mra, size_t size) +{ + return fy_size_t_align(size + FY_MREMAP_ARENA_OVERHEAD, mra->pagesz) - + FY_MREMAP_ARENA_OVERHEAD; +} + +static inline bool fy_mremap_arena_available(struct fy_mremap_arena *mran) +{ + return mran->size - fy_atomic_load(&mran->next); +} + +static inline bool fy_mremap_arena_check_fit(struct fy_mremap_arena *mran, size_t size, size_t align) +{ + size_t old_next, new_next; + + old_next = fy_atomic_load(&mran->next); + new_next = fy_size_t_align(old_next, align) + size; + return new_next <= mran->size; +} + +static struct fy_mremap_arena * +fy_mremap_arena_create(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt, size_t size) +{ + struct fy_mremap_arena *mran = NULL; + uint64_t flags; + void *mem; + size_t size_page_align, balloon_size; +#if !USE_MREMAP + int rc FY_DEBUG_UNUSED; +#endif + + if (size < mra->minimum_arena_size) + size = mra->minimum_arena_size; + + size_page_align = fy_mremap_useable_arena_size(mra, size); + switch (mra->arena_type) { + case FYMRAT_MALLOC: + mran = malloc(size_page_align); + if (!mran) + return NULL; + memset(mran, 0, sizeof(*mran)); + break; + + case FYMRAT_MMAP: + /* allocate an initial ballooned size */ + balloon_size = fy_size_t_align((size_t)(size_page_align * mra->balloon_ratio), mra->pagesz); + if (balloon_size == size_page_align) + balloon_size = size_page_align + mra->pagesz; + mem = mmap(NULL, balloon_size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + /* first allocation failed, that's ok, try again */ + mran = mmap(NULL, size_page_align, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + } else { +#if USE_MREMAP + mran = mremap(mem, balloon_size, size_page_align, 0); + /* either it fails, or it moves we handle it */ +#else + /* we don't shrink, we just unmap over the limit */ + rc = munmap(mem + size_page_align, balloon_size - size_page_align); + if (rc) { +#ifdef DEBUG_ARENA + fprintf(stderr, "%s: failed to unmap for shink\n", __func__); +#endif + size_page_align = balloon_size; /* keep the balloon size? */ + } + mran = mem; +#endif + } + if (mran == MAP_FAILED) + return NULL; + break; + + default: + FY_IMPOSSIBLE_ABORT(); + } + + mran->next_arena = NULL; + flags = 0; + if (!fy_mremap_arena_type_is_growable(mra->arena_type)) + flags |= FYMRAF_CANT_GROW; + fy_atomic_store(&mran->flags, flags); + mran->size = size_page_align; + fy_atomic_store(&mran->next, FY_MREMAP_ARENA_OVERHEAD); + +#ifdef DEBUG_ARENA + fprintf(stderr, "%s: #%zu created arena %p size=%zu (%zuMB)\n", __func__, mrt - mra->tags, mran, mran->size, mran->size >> 20); +#endif + + return mran; +} + +static void fy_mremap_arena_destroy(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt, struct fy_mremap_arena *mran) +{ + if (!mran) + return; + +#ifdef DEBUG_ARENA + fprintf(stderr, "%s: #%zu destroy arena %p size=%zu (%zuMB)\n", __func__, mrt - mra->tags, mran, mran->size, mran->size >> 20); +#endif + switch (mra->arena_type) { + case FYMRAT_MALLOC: + free(mran); + break; + + case FYMRAT_MMAP: + (void)munmap(mran, mran->size); + break; + + default: + FY_IMPOSSIBLE_ABORT(); + } +} + +static inline bool +fy_mremap_arena_should_grow(struct fy_mremap_allocator *mra, struct fy_mremap_arena *mran, + size_t size, size_t align) +{ + size_t next; + + if (!mran || !size) + return false; + + if (!fy_mremap_arena_type_is_growable(mra->arena_type)) + return false; + + /* there's no point trying to grow something this big */ + if (mra->big_alloc_threshold && size >= mra->big_alloc_threshold) + return false; + + /* if the grow needs to be larger than double, don't bother */ + next = atomic_load(&mran->next); + if (fy_size_t_align(next, align) + size > 2 * mran->size) + return false; + + /* go for it */ + return true; +} + +static int fy_mremap_arena_grow(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt, + struct fy_mremap_arena *mran, size_t size, size_t align) +{ + void *mem; + + if (!fy_mremap_arena_should_grow(mra, mran, size, align)) + return -1; + + switch (mra->arena_type) { + case FYMRAT_MALLOC: + /* can't grow malloc without moving the pointer */ + break; + + case FYMRAT_MMAP: +#if USE_MREMAP + /* double the arena */ + mem = mremap(mran, mran->size, mran->size * 2, 0); + if (mem == MAP_FAILED) + break; + + assert(mem == mran); +#else + /* do a mmap right after the one we have, if it succeeds we have grown */ + mem = mmap((void *)mran + mran->size, mran->size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (mem != (void *)mran + mran->size) { + if (mem != MAP_FAILED) + munmap(mem, mran->size); + break; + } +#endif + mran->size *= 2; + +#ifdef DEBUG_ARENA + fprintf(stderr, "%s: #%zu grew arena %p size=%zu (%zuMB)\n", __func__, mrt - mra->tags, mran, mran->size, mran->size >> 20); +#endif + return 0; + + default: + FY_IMPOSSIBLE_ABORT(); + } + + return -1; +} + +static int fy_mremap_arena_trim(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt, struct fy_mremap_arena *mran) +{ + size_t next, new_size; +#if USE_MREMAP + void *mem FY_DEBUG_UNUSED; +#else + int rc FY_DEBUG_UNUSED; +#endif + + if (!mran) + return -1; + + switch (mra->arena_type) { + case FYMRAT_MALLOC: + /* trim not possible (or is it? should be possible to probe) */ + break; + + case FYMRAT_MMAP: + /* check the page size */ + next = fy_atomic_load(&mran->next); + new_size = fy_size_t_align(next, mra->pagesz); + if (new_size >= mran->size) + break; + +#ifdef DEBUG_ARENA + fprintf(stderr, "trim: %zu -> %zu\n", mran->size, new_size); +#endif + +#if USE_MREMAP + /* failure to shrink a mapping is unthinkable, but check anyway */ + mem = mremap(mran, mran->size, new_size, 0); + if (mem == MAP_FAILED || mem != mran) + return -1; +#else + /* we don't shrink, we just unmap over the limit */ + rc = munmap((void *)mran + new_size, mran->size - new_size); + if (rc) + return -1; +#endif + mran->size = new_size; + return 0; + + default: + FY_IMPOSSIBLE_ABORT(); + } + + return -1; +} + +static inline struct fy_mremap_tag * +fy_mremap_tag_from_tag(struct fy_mremap_allocator *mra, int tag) +{ + if (!mra) + return NULL; + + if ((unsigned int)tag >= mra->tag_count) + return NULL; + + if (!fy_id_is_used(mra->ids, mra->tag_id_count, (int)tag)) + return NULL; + + return &mra->tags[tag]; +} + +static void fy_mremap_tag_cleanup(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt) +{ + struct fy_mremap_arena *mran; +#ifdef DEBUG_ARENA + size_t total_sys_alloc, total_wasted, next; +#endif + + if (!mra || !mrt) + return; + +#ifdef DEBUG_ARENA + total_sys_alloc = 0; + total_wasted = 0; + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { + total_sys_alloc += mran->size; + next = fy_atomic_load(&mran->next); + total_wasted += (mran->size - next); + } + fprintf(stderr, "total_sys_alloc=%zu total_wasted=%zu\n", total_sys_alloc, total_wasted); +#endif + +#ifdef DEBUG_ARENA + fprintf(stderr, "%s: destroying active arenas\n", __func__); +#endif + /* pop and destroy */ + while ((mran = fy_atomic_load(&mrt->arenas)) != NULL) { + if (fy_atomic_compare_exchange_strong(&mrt->arenas, &mran, mran->next_arena)) + fy_mremap_arena_destroy(mra, mrt, mran); + } + + /* nuke it all */ + memset(mrt, 0, sizeof(*mrt)); +} + +static void fy_mremap_tag_trim(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt) +{ + struct fy_mremap_arena *mran; +#ifdef DEBUG_ARENA + size_t wasted_before, wasted_after, next; +#endif + + if (!mra || !mrt) + return; + + if (!fy_mremap_arena_type_is_trimmable(mra->arena_type)) + return; + +#ifdef DEBUG_ARENA + wasted_before = 0; + wasted_after = 0; +#endif + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { +#ifdef DEBUG_ARENA + next = fy_atomic_load(&mran->next); + wasted_before += (mran->size - next); +#endif + (void)fy_mremap_arena_trim(mra, mrt, mran); +#ifdef DEBUG_ARENA + next = fy_atomic_load(&mran->next); + wasted_after += (mran->size - next); +#endif + } + +#ifdef DEBUG_ARENA + fprintf(stderr, "wasted_before=%zu wasted_after=%zu\n", wasted_before, wasted_after); +#endif +} + +static void fy_mremap_tag_reset(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt) +{ + struct fy_mremap_arena *mran; + + if (!mra || !mrt) + return; + + /* pop and destroy */ + while ((mran = fy_atomic_load(&mrt->arenas)) != NULL) { + if (fy_atomic_compare_exchange_strong(&mrt->arenas, &mran, mran->next_arena)) + fy_mremap_arena_destroy(mra, mrt, mran); + } +} + +static void fy_mremap_tag_setup(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt) +{ + assert(mra); + assert(mrt); + + memset(mrt, 0, sizeof(*mrt)); + fy_atomic_store(&mrt->arenas, NULL); + fy_atomic_store(&mrt->next_arena_sz, mra->pagesz); + fy_atomic_store(&mrt->allocations, 0); + fy_atomic_store(&mrt->allocated, 0); + fy_atomic_store(&mrt->stores, 0); + fy_atomic_store(&mrt->stored, 0); +} + +static void *fy_mremap_tag_alloc(struct fy_mremap_allocator *mra, struct fy_mremap_tag *mrt, size_t size, size_t align) +{ + struct fy_mremap_arena *mran, *old_arenas; + size_t next_sz, next_arena_sz, old_next_arena_sz, old_next, new_next, data_pos; + uint64_t flags; + void *ptr; + int rc; + +again: + /* hot path, try to find an arena that fits first */ + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { + + flags = fy_atomic_load(&mran->flags); + if (flags & FYMRAF_FULL) + continue; + + if (fy_mremap_arena_check_fit(mran, size, align)) + goto do_alloc; + + /* if it cant grown and is under the threshold, mark it full */ + if ((flags & FYMRAF_CANT_GROW) && fy_mremap_arena_available(mran) < mra->empty_threshold) + fy_atomic_fetch_or(&mran->flags, FYMRAF_FULL); + } + + /* this arena type is not growable, skip growing step */ + if (!fy_mremap_arena_type_is_growable(mra->arena_type)) + goto create_arena; + + /* not found space in any arena, try to grow */ + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { + + flags = fy_atomic_load(&mran->flags); + if (flags & (FYMRAF_FULL | FYMRAF_CANT_GROW | FYMRAF_GROWING)) + continue; + + /* don't even try? */ + if (!fy_mremap_arena_should_grow(mra, mran, size, align)) + continue; + + /* try to grab the growing lock */ + if (!fy_atomic_compare_exchange_strong(&mran->flags, &flags, flags | FYMRAF_GROWING)) + continue; + + /* try to grow, note that the arena is still available */ + rc = fy_mremap_arena_grow(mra, mrt, mran, size, align); + if (rc) { +#ifdef DEBUG_ARENA + fprintf(stderr, "failed to grow %p\n", mran); +#endif + /* release the lock, and mark it as can't grow */ + fy_atomic_fetch_or(&mran->flags, FYMRAF_CANT_GROW); + fy_atomic_fetch_and(&mran->flags, ~FYMRAF_GROWING); + + continue; + } + +#ifdef DEBUG_ARENA + fprintf(stderr, "grow successful %p mran->size=%zu size=%zu\n", mran, mran->size, size); +#endif + + /* release the lock */ + fy_atomic_fetch_and(&mran->flags, ~FYMRAF_GROWING); + + if (fy_mremap_arena_check_fit(mran, size, align)) + goto do_alloc; + } + +create_arena: + /* everything failed, we have to allocate a new arena */ + + /* it's a relatively small allocation, try to resize until we fit */ + if (!mra->big_alloc_threshold || size < mra->big_alloc_threshold) { + + do { + /* increase by the ratio until we're over */ + old_next_arena_sz = fy_atomic_load(&mrt->next_arena_sz); + next_arena_sz = old_next_arena_sz; + while (fy_mremap_useable_arena_size(mra, next_arena_sz) < size) { + next_sz = (size_t)(next_arena_sz * mra->grow_ratio); + + /* very very unlikely */ + if (next_sz <= next_arena_sz) + goto err_out; + + next_arena_sz = next_sz; + } + + /* update to the next possible arena size */ + next_sz = (size_t)(next_arena_sz * mra->grow_ratio); + if (next_sz > next_arena_sz) + next_arena_sz = next_sz; + + } while (!fy_atomic_compare_exchange_strong(&mrt->next_arena_sz, + &old_next_arena_sz, next_arena_sz)); + } else + next_arena_sz = size; /* something big, just use it as is */ + + /* all failed, just new */ + mran = fy_mremap_arena_create(mra, mrt, next_arena_sz); + if (!mran) + goto err_out; + +#ifdef DEBUG_ARENA + fprintf(stderr, "allocated new %p mran->size=%zu size=%zu\n", mran, mran->size, size); +#endif + + /* atomically add it */ + do { + old_arenas = fy_atomic_load(&mrt->arenas); + mran->next_arena = old_arenas; + } while (!fy_atomic_compare_exchange_strong(&mrt->arenas, &old_arenas, mran)); + +do_alloc: + + do { + old_next = fy_atomic_load(&mran->next); + data_pos = fy_size_t_align(old_next, align); + new_next = data_pos + size; + if (new_next > mran->size) { +#ifdef DEBUG_ARENA + fprintf(stderr, "failed new %p mran->size=%zu size=%zu failed to fit!\n", mran, mran->size, size); +#endif + goto again; + } + ptr = (void *)mran + data_pos; + } while (!fy_atomic_compare_exchange_strong(&mran->next, &old_next, new_next)); + + /* malloc arenas need to zero out */ + if (mra->arena_type == FYMRAT_MALLOC) + memset(ptr, 0, size); + +#ifdef DEBUG_ARENA + fprintf(stderr, "allocated OK %p mran->size=%zu ptr=%p size=%zu align=%zu\n", mran, mran->size, ptr, size, align); +#endif + return ptr; + +err_out: + return NULL; +} + +static const struct fy_mremap_allocator_cfg default_cfg = { + .big_alloc_threshold = MREMAP_ALLOCATOR_DEFAULT_BIG_ALLOC_THRESHOLD, + .empty_threshold = MREMAP_ALLOCATOR_DEFAULT_EMPTY_THRESHOLD, + .minimum_arena_size = MREMAP_ALLOCATOR_DEFAULT_MINIMUM_ARENA_SIZE, + .grow_ratio = MREMAP_ALLOCATOR_DEFAULT_GROW_RATIO, + .balloon_ratio = MREMAP_ALLOCATOR_DEFAULT_BALLON_RATIO, + .arena_type = MREMAP_ALLOCATOR_DEFAULT_ARENA_TYPE, +}; + +static void fy_mremap_cleanup(struct fy_allocator *a) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + unsigned int i; + + if (!a) + return; + + mra = container_of(a, struct fy_mremap_allocator, a); + + for (i = 0; i < mra->tag_count; i++) { + mrt = fy_mremap_tag_from_tag(mra, i); + if (!mrt) + continue; + fy_mremap_tag_cleanup(mra, mrt); + } + + fy_parent_allocator_free(&mra->a, mra->ids); + fy_parent_allocator_free(&mra->a, mra->tags); +} + +static int fy_mremap_setup(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *cfg_data) +{ + const struct fy_mremap_allocator_cfg *cfg; + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + size_t tmpsz; + + if (!a) + return -1; + + cfg = cfg_data ? cfg_data : &default_cfg; + + mra = container_of(a, struct fy_mremap_allocator, a); + memset(mra, 0, sizeof(*mra)); + mra->a.name = "mremap"; + mra->a.ops = &fy_mremap_allocator_ops; + mra->a.parent = parent; + mra->a.parent_tag = parent_tag; + mra->cfg = *cfg; + + mra->pagesz = sysconf(_SC_PAGESIZE); + /* pagesz is size of 2 find the first set bit */ + mra->pageshift = fy_bit64_ffs(mra->pagesz); + + mra->big_alloc_threshold = cfg->big_alloc_threshold; + if (!mra->big_alloc_threshold) + mra->big_alloc_threshold = MREMAP_ALLOCATOR_DEFAULT_BIG_ALLOC_THRESHOLD; + + mra->empty_threshold = cfg->empty_threshold; + if (!mra->empty_threshold) + mra->empty_threshold = MREMAP_ALLOCATOR_DEFAULT_EMPTY_THRESHOLD; + + mra->minimum_arena_size = cfg->minimum_arena_size; + if (!mra->minimum_arena_size) + mra->minimum_arena_size = MREMAP_ALLOCATOR_DEFAULT_MINIMUM_ARENA_SIZE; + + mra->grow_ratio = cfg->grow_ratio; + if (mra->grow_ratio <= 1) + mra->grow_ratio = MREMAP_ALLOCATOR_DEFAULT_GROW_RATIO; + + mra->balloon_ratio = cfg->balloon_ratio; + if (mra->balloon_ratio <= 1) + mra->balloon_ratio = MREMAP_ALLOCATOR_DEFAULT_BALLON_RATIO; + + mra->arena_type = cfg->arena_type; + if (mra->arena_type == FYMRAT_DEFAULT) + mra->arena_type = MREMAP_ALLOCATOR_DEFAULT_ARENA_TYPE; + + mra->tag_count = FY_MREMAP_TAG_MAX; + mra->tag_id_count = (mra->tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + tmpsz = mra->tag_id_count * sizeof(*mra->ids); + mra->ids = fy_parent_allocator_alloc(&mra->a, tmpsz, _Alignof(fy_id_bits)); + if (!mra->ids) + goto err_out; + fy_id_reset(mra->ids, mra->tag_id_count); + + tmpsz = mra->tag_count * sizeof(*mra->tags); + mra->tags = fy_parent_allocator_alloc(&mra->a, tmpsz, _Alignof(struct fy_mremap_tag)); + if (!mra->tags) + goto err_out; + + /* start with tag 0 as general use */ + fy_id_set_used(mra->ids, mra->tag_id_count, 0); + mrt = fy_mremap_tag_from_tag(mra, 0); + fy_mremap_tag_setup(mra, mrt); + + return 0; + +err_out: + fy_mremap_cleanup(a); + return -1; +} + +struct fy_allocator *fy_mremap_create(struct fy_allocator *parent, int parent_tag, const void *cfg) +{ + struct fy_mremap_allocator *mra = NULL; + int rc; + + mra = fy_early_parent_allocator_alloc(parent, parent_tag, sizeof(*mra), _Alignof(struct fy_mremap_allocator)); + if (!mra) + goto err_out; + + rc = fy_mremap_setup(&mra->a, parent, parent_tag, cfg); + if (rc) + goto err_out; + + return &mra->a; + +err_out: + fy_early_parent_allocator_free(parent, parent_tag, mra); + return NULL; +} + +void fy_mremap_destroy(struct fy_allocator *a) +{ + struct fy_mremap_allocator *mra; + + mra = container_of(a, struct fy_mremap_allocator, a); + fy_mremap_cleanup(a); + fy_parent_allocator_free(a, mra); +} + +void fy_mremap_dump(struct fy_allocator *a) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + struct fy_mremap_arena *mran; + size_t count, active_count, full_count, total, system_total; + unsigned int i; + + mra = container_of(a, struct fy_mremap_allocator, a); + + fprintf(stderr, "mremap: "); + for (i = 0; i < mra->tag_count; i++) { + mrt = fy_mremap_tag_from_tag(mra, i); + if (!mrt) + continue; + fprintf(stderr, "%c", fy_id_is_free(mra->ids, mra->tag_id_count, i) ? '.' : 'x'); + } + fprintf(stderr, "\n"); + + for (i = 0; i < mra->tag_count; i++) { + mrt = fy_mremap_tag_from_tag(mra, i); + if (!mrt) + continue; + + count = full_count = active_count = total = system_total = 0; + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { + total += fy_atomic_load(&mran->next); + system_total += mran->size; + count++; + active_count++; + /* XXX full count */ + } + + fprintf(stderr, " %d: count %zu (a=%zu/f=%zu) total %zu system %zu overhead %zu (%2.2f%%)\n", i, + count, active_count, full_count, + total, system_total, system_total - total, + 100.0 * (double)(system_total - total) / (double)system_total); + } +} + +static void *fy_mremap_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + void *ptr; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + goto err_out; + + ptr = fy_mremap_tag_alloc(mra, mrt, size, align); + if (!ptr) + goto err_out; + + if (a->flags & FYAF_KEEP_STATS) { + fy_atomic_fetch_add(&mrt->allocations, 1); + fy_atomic_fetch_add(&mrt->allocated, size); + } + + return ptr; + +err_out: + return NULL; +} + +static void fy_mremap_free(struct fy_allocator *a, int tag, void *data) +{ + /* no frees */ +} + +static int fy_mremap_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + goto err_out; + + stats->allocations += fy_atomic_get_and_clear_counter(&mrt->allocations); + stats->allocated += fy_atomic_get_and_clear_counter(&mrt->allocated); + stats->stores += fy_atomic_get_and_clear_counter(&mrt->stores); + stats->stored += fy_atomic_get_and_clear_counter(&mrt->stored); + + return 0; + +err_out: + return -1; +} + +static const void *fy_mremap_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + void *start; + size_t total_size; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + goto err_out; + + total_size = fy_iovec_size(iov, iovcnt); + if (total_size == SIZE_MAX) + goto err_out; + + start = fy_mremap_tag_alloc(mra, mrt, total_size, align); + if (!start) + goto err_out; + + fy_iovec_copy_from(iov, iovcnt, start); + + if (a->flags & FYAF_KEEP_STATS) { + fy_atomic_fetch_add(&mrt->stores, 1); + fy_atomic_fetch_add(&mrt->stored, total_size); + } + + return start; + +err_out: + return NULL; +} + +static const void *fy_mremap_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + /* no lookups */ + return NULL; +} + +static void fy_mremap_release(struct fy_allocator *a, int tag, const void *data, size_t size) +{ + /* no releases */ +} + +static void fy_mremap_release_tag(struct fy_allocator *a, int tag) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + return; + + fy_mremap_tag_cleanup(mra, mrt); + + /* must be last */ + fy_id_free(mra->ids, mra->tag_id_count, tag); +} + +static int fy_mremap_get_tag(struct fy_allocator *a) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + int id = -1; + + mra = container_of(a, struct fy_mremap_allocator, a); + + /* this is atomic, so safe for multiple threads */ + id = fy_id_alloc(mra->ids, mra->tag_id_count); + if (id < 0) + goto err_out; + + mrt = fy_mremap_tag_from_tag(mra, id); + if (!mrt) + goto err_out; + + fy_mremap_tag_setup(mra, mrt); + + return (int)id; + +err_out: + if (id >= 0) + fy_id_free(mra->ids, mra->tag_id_count, id); + return FY_ALLOC_TAG_ERROR; +} + +static int fy_mremap_get_tag_count(struct fy_allocator *a) +{ + struct fy_mremap_allocator *mra; + + mra = container_of(a, struct fy_mremap_allocator, a); + return mra->tag_count; +} + +static int fy_mremap_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_arena *mran; + fy_id_bits *ids = NULL; + struct fy_mremap_tag *mrt, *mrt_new, *tags = NULL; + unsigned int i, tag_count, tag_id_count; + size_t tmpsz; + + if (count < 1) + return -1; + + mra = container_of(a, struct fy_mremap_allocator, a); + + if (count == mra->tag_count) + return 0; + + tag_count = count; + tag_id_count = (tag_count + FY_ID_BITS_BITS - 1) / FY_ID_BITS_BITS; + + /* shrink?, just clip */ + if (count < mra->tag_count) { + for (i = count; i < mra->tag_count; i++) { + mrt = fy_mremap_tag_from_tag(mra, i); + if (!mrt) + continue; + fy_mremap_tag_cleanup(mra, mrt); + fy_id_free(mra->ids, mra->tag_id_count, i); + } + tags = mra->tags; + ids = mra->ids; + } else { + /* we need to grow */ + tmpsz = tag_id_count * sizeof(*mra->ids); + ids = fy_parent_allocator_alloc(&mra->a, tmpsz, _Alignof(fy_id_bits)); + if (!ids) + goto err_out; + fy_id_reset(ids, tag_id_count); + + tmpsz = tag_count * sizeof(*tags); + tags = fy_parent_allocator_alloc(&mra->a, tmpsz, _Alignof(struct fy_mremap_tag)); + if (!tags) + goto err_out; + + /* copy over the old entries */ + for (i = 0; i < mra->tag_count; i++) { + + mrt = fy_mremap_tag_from_tag(mra, i); + if (!mrt) + continue; + + mrt_new = &tags[i]; + + fy_mremap_tag_setup(mra, mrt_new); + do { + mran = fy_atomic_load(&mrt->arenas); + } while (fy_atomic_compare_exchange_strong(&mrt->arenas, &mran, NULL)); + fy_atomic_store(&mrt_new->arenas, mran); + fy_atomic_store(&mrt_new->next_arena_sz, fy_atomic_load(&mrt->next_arena_sz)); + fy_atomic_store(&mrt_new->allocations, fy_atomic_load(&mrt->allocations)); + fy_atomic_store(&mrt_new->allocated, fy_atomic_load(&mrt->allocated)); + fy_atomic_store(&mrt_new->stores, fy_atomic_load(&mrt->stores)); + fy_atomic_store(&mrt_new->stored, fy_atomic_load(&mrt->stored)); + + /* clean the old entry */ + fy_mremap_tag_cleanup(mra, mrt); + + fy_id_set_used(ids, tag_id_count, i); + } + } + + if (mra->tags != tags) { + mra->tags = tags; + fy_parent_allocator_free(&mra->a, mra->tags); + } + if (mra->ids != ids) { + mra->ids = ids; + fy_parent_allocator_free(&mra->a, mra->ids); + } + mra->tag_count = tag_count; + mra->tag_id_count = tag_id_count; + return 0; + +err_out: + fy_parent_allocator_free(&mra->a, tags); + fy_parent_allocator_free(&mra->a, ids); + return -1; +} + +static void fy_mremap_trim_tag(struct fy_allocator *a, int tag) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + return; + + fy_mremap_tag_trim(mra, mrt); +} + +static void fy_mremap_reset_tag(struct fy_allocator *a, int tag) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + + mra = container_of(a, struct fy_mremap_allocator, a); + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + return; + + fy_mremap_tag_reset(mra, mrt); +} + +static struct fy_allocator_info * +fy_mremap_get_info(struct fy_allocator *a, int tag) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + struct fy_mremap_arena *mran; + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + struct fy_allocator_arena_info *arena_info; + size_t size, free, used, total; + size_t tag_free, tag_used, tag_total; + size_t arena_free, arena_used, arena_total; + unsigned int num_tags, num_arenas, i; + int id; + + mra = container_of(a, struct fy_mremap_allocator, a); + + /* allocate for the worst case always */ + num_tags = 0; + num_arenas = 0; + + free = 0; + used = 0; + total = 0; + + /* two passes */ + for (i = 0; i < 2; i++) { + + if (!i) { + tag_info = NULL; + arena_info = NULL; + + } else { + size = sizeof(*info) + + sizeof(*tag_info) * num_tags + + sizeof(*arena_info) * num_arenas; + + info = malloc(size); + if (!info) + return NULL; + memset(info, 0, sizeof(*info)); + + tag_info = (void *)(info + 1); + assert(((uintptr_t)tag_info % _Alignof(struct fy_allocator_tag_info)) == 0); + arena_info = (void *)(tag_info + num_tags); + assert(((uintptr_t)arena_info % _Alignof(struct fy_allocator_arena_info)) == 0); + + info->free = free; + info->used = used; + info->total = total; + + info->num_tag_infos = 0; + info->tag_infos = tag_info; + } + + free = 0; + used = 0; + total = sizeof(*mra); + + for (id = 0; id < (int)mra->tag_count; id++) { + + if (!fy_id_is_used(mra->ids, mra->tag_id_count, id)) + continue; + + mrt = &mra->tags[id]; + + tag_free = 0; + tag_used = 0; + tag_total = 0; + + if (i) { + tag_info->num_arena_infos = 0; + tag_info->arena_infos = arena_info; + } + + for (mran = fy_atomic_load(&mrt->arenas); mran; mran = mran->next_arena) { + arena_free = (size_t)(mran->size - mran->next); + arena_used = (size_t)(mran->next - FY_MREMAP_ARENA_OVERHEAD); + arena_total = mran->size; + + tag_free += arena_free; + tag_used += arena_used; + tag_total += arena_total; + + if (!i) { + num_arenas++; + } else { + arena_info->free = arena_free; + arena_info->used = arena_used; + arena_info->total = arena_total; + arena_info->data = (void *)mran + FY_MREMAP_ARENA_OVERHEAD; + arena_info->size = arena_info->used; + arena_info++; + tag_info->num_arena_infos++; + } + } + + if (!i) { + num_tags++; + } else { + + /* only store the tag if there's a match */ + if (tag == FY_ALLOC_TAG_NONE || tag == id) { + tag_info->tag = id; + tag_info->free = tag_free; + tag_info->used = tag_used; + tag_info->total = tag_total; + tag_info++; + info->num_tag_infos++; + } + } + + free += tag_free; + used += tag_used; + total += tag_total; + } + } + + return info; +} + +static enum fy_allocator_cap_flags +fy_mremap_get_caps(struct fy_allocator *a) +{ + return FYACF_CAN_FREE_TAG | FYACF_HAS_EFFICIENT_CONTAINS | \ + FYACF_HAS_CONTAINS | FYACF_HAS_TAGS; +} + +static bool mremap_tag_contains(struct fy_mremap_tag *mrt, const void *p) +{ + struct fy_mremap_arena *mra; + + for (mra = fy_atomic_load(&mrt->arenas); mra; mra = mra->next_arena) { + if (p >= (const void *)mra->mem && + p < (const void *)mra->mem + mra->size) + return true; + } + + return false; +} + +static bool fy_mremap_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + struct fy_mremap_allocator *mra; + struct fy_mremap_tag *mrt; + int tag_start, tag_end; + + mra = container_of(a, struct fy_mremap_allocator, a); + + if (tag >= 0) { + if (tag >= (int)mra->tag_count) + return false; + tag_start = tag; + tag_end = tag + 1; + } else { + tag_start = 0; + tag_end = (int)mra->tag_count; + } + + for (tag = tag_start; tag < tag_end; tag++) { + + mrt = fy_mremap_tag_from_tag(mra, tag); + if (!mrt) + continue; + + if (mremap_tag_contains(mrt, ptr)) + return true; + } + + return false; +} + +const struct fy_allocator_ops fy_mremap_allocator_ops = { + .setup = fy_mremap_setup, + .cleanup = fy_mremap_cleanup, + .create = fy_mremap_create, + .destroy = fy_mremap_destroy, + .dump = fy_mremap_dump, + .alloc = fy_mremap_alloc, + .free = fy_mremap_free, + .update_stats = fy_mremap_update_stats, + .storev = fy_mremap_storev, + .lookupv = fy_mremap_lookupv, + .release = fy_mremap_release, + .get_tag = fy_mremap_get_tag, + .release_tag = fy_mremap_release_tag, + .get_tag_count = fy_mremap_get_tag_count, + .set_tag_count = fy_mremap_set_tag_count, + .trim_tag = fy_mremap_trim_tag, + .reset_tag = fy_mremap_reset_tag, + .get_info = fy_mremap_get_info, + .get_caps = fy_mremap_get_caps, + .contains = fy_mremap_contains, +}; diff --git a/src/allocator/fy-allocator-mremap.h b/src/allocator/fy-allocator-mremap.h new file mode 100644 index 0000000..7503426 --- /dev/null +++ b/src/allocator/fy-allocator-mremap.h @@ -0,0 +1,77 @@ +/* + * fy-allocator-mremap.h - the mremap allocator + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_MREMAP_H +#define FY_ALLOCATOR_MREMAP_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "fy-typelist.h" +#include "fy-id.h" + +#include "fy-atomics.h" +#include "fy-allocator.h" + +struct fy_mremap_tag; + +#define FY_MREMAP_TAG_MAX 32 + +static inline bool fy_mremap_arena_type_is_growable(enum fy_mremap_arena_type type) +{ + return type == FYMRAT_MMAP; +} + +static inline bool fy_mremap_arena_type_is_trimmable(enum fy_mremap_arena_type type) +{ + return type == FYMRAT_MMAP; +} + +#define FYMRAF_FULL FY_BIT(0) +#define FYMRAF_GROWING FY_BIT(1) +#define FYMRAF_CANT_GROW FY_BIT(2) + +struct fy_mremap_arena { + struct fy_mremap_arena *next_arena; + size_t size; /* includes the arena header */ + FY_ATOMIC(uint64_t) flags; + FY_ATOMIC(size_t) next; + uint64_t mem[] __attribute__((aligned(16))); +}; + +#define FY_MREMAP_ARENA_OVERHEAD (offsetof(struct fy_mremap_arena, mem)) + +struct fy_mremap_tag { + FY_ATOMIC(struct fy_mremap_arena *) arenas; + FY_ATOMIC(size_t) next_arena_sz; + FY_ATOMIC(uint64_t) allocations; + FY_ATOMIC(uint64_t) allocated; + FY_ATOMIC(uint64_t) stores; + FY_ATOMIC(uint64_t) stored; +}; + +struct fy_mremap_allocator { + struct fy_allocator a; + struct fy_mremap_allocator_cfg cfg; + size_t pagesz; + size_t pageshift; + size_t big_alloc_threshold; /* bigger than that and a new allocation */ + size_t empty_threshold; /* less than that and get moved to full */ + size_t minimum_arena_size; /* the minimum arena size */ + float grow_ratio; + float balloon_ratio; + enum fy_mremap_arena_type arena_type; + fy_id_bits *ids; + unsigned int tag_id_count; + struct fy_mremap_tag *tags; + unsigned int tag_count; +}; + +extern const struct fy_allocator_ops fy_mremap_allocator_ops; + +#endif diff --git a/src/allocator/fy-allocator.c b/src/allocator/fy-allocator.c new file mode 100644 index 0000000..84fc059 --- /dev/null +++ b/src/allocator/fy-allocator.c @@ -0,0 +1,756 @@ +/* + * fy-allocator.c - allocators + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#include + +#include + +#ifndef FY_NON_LOCKING_REGISTRY +#include +#endif + +/* for container_of */ +#include "fy-list.h" +#include "fy-utils.h" + +#include "fy-allocator.h" +#include "fy-allocator-linear.h" +#include "fy-allocator-malloc.h" +#include "fy-allocator-mremap.h" +#include "fy-allocator-dedup.h" +#include "fy-allocator-auto.h" + +static struct fy_registered_allocator_entry_list allocator_registry_list; +static bool allocator_registry_initialized = false; +static bool allocator_registry_locked = false; + +#ifndef FY_NON_LOCKING_REGISTRY +static pthread_mutex_t allocator_registry_mutex = PTHREAD_MUTEX_INITIALIZER; +static inline void allocator_registry_lock(void) +{ + int rc FY_UNUSED; + + rc = pthread_mutex_lock(&allocator_registry_mutex); + assert(!rc); + + assert(!allocator_registry_locked); + allocator_registry_locked = true; +} + +static inline void allocator_registry_unlock(void) +{ + int rc FY_UNUSED; + + assert(allocator_registry_locked); + allocator_registry_locked = false; + + rc = pthread_mutex_unlock(&allocator_registry_mutex); + assert(!rc); +} +#else +static inline void allocator_registry_lock(void) +{ + /* nothing */ +} +static inline void allocator_registry_unlock(void) +{ + /* nothing */ +} +#endif + +static inline void allocator_registry_init(void) +{ + if (allocator_registry_initialized) + return; + + fy_registered_allocator_entry_list_init(&allocator_registry_list); + + allocator_registry_initialized = true; +} + +static struct fy_registered_allocator_entry * +fy_registered_allocator_entry_create(const char *name, const struct fy_allocator_ops *ops) +{ + struct fy_registered_allocator_entry *ae; + + ae = malloc(sizeof(*ae)); + if (!ae) + return NULL; + memset(ae, 0, sizeof(*ae)); + ae->name = name; + ae->ops = ops; + + return ae; +} + +static void fy_registered_allocator_entry_destroy(struct fy_registered_allocator_entry *ae) +{ + if (!ae) + return; + free(ae); +} + +static const struct { + const char *name; + const struct fy_allocator_ops *ops; +} builtin_allocators[] = { + { + .name = "linear", + .ops = &fy_linear_allocator_ops, + }, { + .name = "malloc", + .ops = &fy_malloc_allocator_ops, + }, { + .name = "mremap", + .ops = &fy_mremap_allocator_ops, + }, { + .name = "dedup", + .ops = &fy_dedup_allocator_ops, + }, { + .name = "auto", + .ops = &fy_auto_allocator_ops, + } +}; + +int fy_allocator_register(const char *name, const struct fy_allocator_ops *ops) +{ + struct fy_registered_allocator_entry *ae; + unsigned int i; + int ret; + + if (!name || !ops || + !ops->setup || + !ops->cleanup || + !ops->create || + !ops->destroy || + !ops->dump || + !ops->alloc || + !ops->free || + !ops->update_stats || + !ops->storev || + !ops->lookupv || + !ops->release | + !ops->get_tag || + !ops->release_tag || + !ops->get_tag_count || + !ops->set_tag_count || + !ops->trim_tag || + !ops->reset_tag || + !ops->get_info || + !ops->get_caps || + !ops->contains) + return -1; + + allocator_registry_lock(); + allocator_registry_init(); + + ret = -1; + + /* must not clash with the builtins */ + for (i = 0; i < ARRAY_SIZE(builtin_allocators); i++) { + if (!strcmp(builtin_allocators[i].name, name)) + goto out_unlock; + } + + /* must not clash with the other entries */ + for (ae = fy_registered_allocator_entry_list_head(&allocator_registry_list); ae; + ae = fy_registered_allocator_entry_next(&allocator_registry_list, ae)) { + if (!strcmp(ae->name, name)) + goto out_unlock; + } + + /* OK, create the entry */ + ae = fy_registered_allocator_entry_create(name, ops); + if (!ae) + goto out_unlock; + + /* and add it to the list */ + fy_registered_allocator_entry_list_add(&allocator_registry_list, ae); + + /* all clear */ + ret = 0; + +out_unlock: + allocator_registry_unlock(); + + return ret; +} + +int fy_allocator_unregister(const char *name) +{ + struct fy_registered_allocator_entry *ae; + unsigned int i; + int ret; + + ret = -1; + + allocator_registry_lock(); + allocator_registry_init(); + + /* must not try to unregister a builtin */ + for (i = 0; i < ARRAY_SIZE(builtin_allocators); i++) { + if (!strcmp(builtin_allocators[i].name, name)) + goto out_unlock; + } + + /* find the entry now */ + for (ae = fy_registered_allocator_entry_list_head(&allocator_registry_list); ae; + ae = fy_registered_allocator_entry_next(&allocator_registry_list, ae)) { + if (!strcmp(ae->name, name)) + break; + } + if (!ae) + goto out_unlock; + + fy_registered_allocator_entry_list_del(&allocator_registry_list, ae); + + /* and destroy it */ + fy_registered_allocator_entry_destroy(ae); + + ret = 0; + +out_unlock: + allocator_registry_unlock(); + + return ret; +} + +struct fy_allocator * +fy_allocator_create_internal(const char *name, struct fy_allocator *parent, int parent_tag, const void *cfg) +{ + struct fy_registered_allocator_entry *ae; + const struct fy_allocator_ops *ops = NULL; + unsigned int i; + + if (!name) + name = builtin_allocators[0].name; + + allocator_registry_lock(); + allocator_registry_init(); + + /* try the builtins first */ + for (i = 0; i < ARRAY_SIZE(builtin_allocators); i++) { + if (!strcmp(builtin_allocators[i].name, name)) { + ops = builtin_allocators[i].ops; + break; + } + } + + /* if not found there, try the registry */ + if (!ops) { + for (ae = fy_registered_allocator_entry_list_head(&allocator_registry_list); ae; + ae = fy_registered_allocator_entry_next(&allocator_registry_list, ae)) { + if (!strcmp(ae->name, name)) { + ops = ae->ops; + break; + } + } + } + allocator_registry_unlock(); + if (!ops) + return NULL; + + /* XXX just malloc for now */ + return ops->create(parent, parent_tag, cfg); +} + +struct fy_allocator *fy_allocator_create(const char *name, const void *cfg) +{ + /* for now just use internal */ + return fy_allocator_create_internal(name, NULL, FY_ALLOC_TAG_DEFAULT, cfg); +} + +/* special in place allocator */ +struct fy_allocator * +fy_linear_allocator_create_in_place(void *buffer, size_t size) +{ + const struct fy_allocator_ops *ops = &fy_linear_allocator_ops; + struct fy_linear_allocator_cfg cfg = { .buf = buffer, .size = size }; + + if (!buffer || size < FY_LINEAR_ALLOCATOR_IN_PLACE_MIN_SIZE) + return NULL; + + return ops->create(FY_PARENT_ALLOCATOR_INPLACE, FY_ALLOC_TAG_DEFAULT, &cfg); +} + +struct fy_allocator * +fy_dedup_allocator_create_in_place(void *buffer, size_t size) +{ + struct fy_allocator *pa; + struct fy_dedup_allocator_cfg dcfg; + size_t dedup_available; + + if (!buffer || size < FY_DEDUP_ALLOCATOR_IN_PLACE_MIN_SIZE) + return NULL; + + pa = fy_linear_allocator_create_in_place(buffer, size); + if (!pa) + return NULL; + + /* try to size the dedup structures to about 10% of the available space */ + dedup_available = size - FY_LINEAR_ALLOCATOR_IN_PLACE_MIN_SIZE; + + memset(&dcfg, 0, sizeof(dcfg)); + dcfg.parent_allocator = pa; + dcfg.bloom_filter_bits = 0; /* use default */ + dcfg.bucket_count_bits = 0; + dcfg.estimated_content_size = dedup_available; + dcfg.minimum_bucket_occupancy = 1.0; /* will never grow */ + + return fy_allocator_create_internal("dedup", pa, 0, &dcfg); +} + +void fy_allocator_registry_cleanup_internal(bool show_leftovers) +{ + struct fy_registered_allocator_entry *ae; + + if (!allocator_registry_initialized) + return; + + allocator_registry_lock(); + while ((ae = fy_registered_allocator_entry_list_pop(&allocator_registry_list)) != NULL) { + if (show_leftovers) + fprintf(stderr, "%s: destroying %s\n", __func__, ae->name); + fy_registered_allocator_entry_destroy(ae); + } + allocator_registry_unlock(); +} + +void fy_allocator_registry_cleanup(void) +{ + fy_allocator_registry_cleanup_internal(false); +} + +#ifdef FY_HAS_CONSTRUCTOR +static FY_CONSTRUCTOR void fy_allocator_registry_constructor(void) +{ + allocator_registry_init(); +} +#endif + +#ifdef FY_HAS_DESTRUCTOR +static FY_DESTRUCTOR void fy_allocator_registry_destructor(void) +{ + bool show_leftovers = false; + +#ifdef FY_DESTRUCTOR_SHOW_LEFTOVERS + show_leftovers = true; +#endif + + /* make sure the registry is not locked because we will hang */ + if (allocator_registry_locked) { + if (show_leftovers) + fprintf(stderr, "%s: refusing to work on locked registry\n", __func__); + return; + } + + fy_allocator_registry_cleanup_internal(show_leftovers); +} +#endif + +static const char **create_allocator_array_names(void) +{ + unsigned int i, j, count; + struct fy_registered_allocator_entry *ae; + const char **names = NULL; + + allocator_registry_lock(); + + /* two passes */ + for (j = 0; j < 2; j++) { + + count = 0; + + /* try the builtins first */ + for (i = 0; i < ARRAY_SIZE(builtin_allocators); i++) { + if (j) + names[count] = builtin_allocators[i].name; + count++; + + } + for (ae = fy_registered_allocator_entry_list_head(&allocator_registry_list); ae; + ae = fy_registered_allocator_entry_next(&allocator_registry_list, ae)) { + if (j) + names[count] = ae->name; + count++; + } + + if (!j) { + names = malloc(sizeof(*names) * (count + 1)); + if (!names) + return NULL; + memset(names, 0, sizeof(*names) * (count + 1)); + } else + names[count++] = NULL; + + } + + allocator_registry_unlock(); + + return names; +} + +const char *fy_allocator_iterate(const char **prevp) +{ + unsigned int i; + const char **names; + + if (!prevp) + return NULL; + + /* yeah, it's not fast, but should be negligible */ + names = create_allocator_array_names(); + if (!names) + return NULL; + + i = 0; + if (*prevp) { + while (names[i] && strcmp(*prevp, names[i])) + i++; + if (names[i]) + i++; + } + + *prevp = names[i]; + + free(names); + + return *prevp; +} + +bool fy_allocator_is_available(const char *allocator) +{ + const char *name, *prev; + + prev = NULL; + while ((name = fy_allocator_iterate(&prev)) != NULL) { + if (!strcmp(allocator, name)) + return true; + } + return false; +} + +char *fy_allocator_get_names(void) +{ + const char *name, *prev; + size_t size, len; + char *names, *s; + + size = 0; + prev = NULL; + while ((name = fy_allocator_iterate(&prev)) != NULL) + size += strlen(name) + 1; + + names = malloc(size + 1); + if (!names) + return NULL; + + s = names; + prev = NULL; + while ((name = fy_allocator_iterate(&prev)) != NULL) { + len = strlen(name); + assert((size_t)(s + len + 1 - names) <= size); + memcpy(s, name, len); + s += len; + *s++ = ' '; + } + if (s > names && s[-1] == ' ') + s[-1] = '\0'; + return names; +} + +ssize_t fy_allocator_get_tag_linear_size(struct fy_allocator *a, int tag) +{ + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + struct fy_allocator_arena_info *arena_info; + unsigned int i, j; + size_t size; + + if (!a || tag == FY_ALLOC_TAG_NONE) + return -1; + + info = fy_allocator_get_info(a, tag); + if (!info) + return -1; + + size = 0; + for (i = 0; i < info->num_tag_infos; i++) { + tag_info = &info->tag_infos[i]; + + for (j = 0; j < tag_info->num_arena_infos; j++) { + arena_info = &tag_info->arena_infos[j]; + + size = fy_size_t_align(size, 16); /* align at 16 always */ + size += arena_info->size; + } + } + + free(info); + + /* too large? really? */ + if ((ssize_t)size < 0) + return -1; + + return (ssize_t)size; +} + +const void *fy_allocator_get_tag_single_linear(struct fy_allocator *a, int tag, size_t *sizep) +{ + struct fy_allocator_info *info; + struct fy_allocator_arena_info *arena_info; + const void *data; + + if (!a || tag == FY_ALLOC_TAG_NONE || !sizep) + return NULL; + + *sizep = 0; + data = NULL; + + info = fy_allocator_get_info(a, tag); + if (!info) + return NULL; + + /* this is great, everything is linear */ + if (info->num_tag_infos == 1 && info->tag_infos[0].num_arena_infos == 1) { + arena_info = &info->tag_infos[0].arena_infos[0]; + data = arena_info->data; + *sizep = arena_info->size; + } + + free(info); + + return data; +} + +void fy_allocator_destroy(struct fy_allocator *a) +{ + if (!a) + return; + fy_allocator_destroy_nocheck(a); +} + +void fy_allocator_dump(struct fy_allocator *a) +{ + if (!a) + return; + fy_allocator_dump_nocheck(a); +} + +int fy_allocator_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + if (!a || !stats) + return -1; + return fy_allocator_update_stats_nocheck(a, tag, stats); +} + +void *fy_allocator_alloc(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + if (!a) + return NULL; + return fy_allocator_alloc_nocheck(a, tag, size, align); +} + +void fy_allocator_free(struct fy_allocator *a, int tag, void *ptr) +{ + if (!a || !ptr) + return; + return fy_allocator_free_nocheck(a, tag, ptr); +} + +const void *fy_allocator_storev_hash(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + if (!a || !iov) + return NULL; + return fy_allocator_storev_hash_nocheck(a, tag, iov, iovcnt, align, hash); +} + +const void *fy_allocator_store(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) +{ + struct iovec iov[1]; + + if (!a || !data) + return NULL; + + /* just call the storev */ + iov[0].iov_base = (void *)data; + iov[0].iov_len = size; + + return fy_allocator_storev_hash_nocheck(a, tag, iov, 1, align, 0); +} + +const void *fy_allocator_storev(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align) +{ + if (!a || !iov) + return NULL; + return fy_allocator_storev_nocheck(a, tag, iov, iovcnt, align); +} + +const void *fy_allocator_lookupv_hash(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + if (!a || !iov) + return NULL; + return fy_allocator_lookupv_nocheck(a, tag, iov, iovcnt, align, hash); +} + +const void *fy_allocator_lookupv(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align) +{ + if (!a || !iov) + return NULL; + return fy_allocator_lookupv_nocheck(a, tag, iov, iovcnt, align, 0); +} + +const void *fy_allocator_lookup(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) +{ + struct iovec iov[1]; + + if (!a || !data) + return NULL; + + /* just call the storev */ + iov[0].iov_base = (void *)data; + iov[0].iov_len = size; + + return fy_allocator_lookupv_nocheck(a, tag, iov, 1, align, 0); +} + +void fy_allocator_release(struct fy_allocator *a, int tag, const void *ptr, size_t size) +{ + if (!a || !ptr) + return; + fy_allocator_release(a, tag, ptr, size); +} + +int fy_allocator_get_tag(struct fy_allocator *a) +{ + if (!a) + return 0; + return fy_allocator_get_tag_nocheck(a); +} + +void fy_allocator_release_tag(struct fy_allocator *a, int tag) +{ + if (!a) + return; + fy_allocator_release_tag_nocheck(a, tag); +} + +int fy_allocator_get_tag_count(struct fy_allocator *a) +{ + if (!a) + return -1; + return fy_allocator_get_tag_count_nocheck(a); +} + +int fy_allocator_set_tag_count(struct fy_allocator *a, unsigned int count) +{ + if (!a) + return -1; + return fy_allocator_set_tag_count_nocheck(a, count); +} + +void fy_allocator_trim_tag(struct fy_allocator *a, int tag) +{ + if (!a) + return; + fy_allocator_trim_tag_nocheck(a, tag); +} + +void fy_allocator_reset_tag(struct fy_allocator *a, int tag) +{ + if (!a) + return; + fy_allocator_reset_tag_nocheck(a, tag); +} + +struct fy_allocator_info * +fy_allocator_get_info(struct fy_allocator *a, int tag) +{ + if (!a) + return NULL; + return fy_allocator_get_info_nocheck(a, tag); +} + +enum fy_allocator_cap_flags +fy_allocator_get_caps(struct fy_allocator *a) +{ + if (!a) + return 0; + + return fy_allocator_get_caps_nocheck(a); +} + +bool +fy_allocator_contains(struct fy_allocator *a, int tag, const void *ptr) +{ + if (!a || !ptr) + return false; + + return fy_allocator_contains_nocheck(a, tag, ptr); +} + +/* respects the parent allocator (or uses posix_memalign if NULL) */ +void *fy_early_parent_allocator_alloc(struct fy_allocator *parent, int parent_tag, size_t size, size_t align) +{ + void *ptr; + int r; + + /* yeah, not gonna work */ + if (parent == FY_PARENT_ALLOCATOR_INPLACE) + return NULL; + + if (!align) + align = _Alignof(max_align_t); + + if (parent) + return fy_allocator_alloc(parent, parent_tag, size, align); + + r = posix_memalign(&ptr, align, size); + if (r) + return NULL; + + return ptr; +} + +void fy_early_parent_allocator_free(struct fy_allocator *parent, int parent_tag, void *ptr) +{ + if (!ptr || parent == FY_PARENT_ALLOCATOR_INPLACE) + return; + + if (parent) + fy_allocator_free(parent, parent_tag, ptr); + else + free(ptr); +} + +/* respects the parent allocator (or uses posix_memalign if NULL) */ +void *fy_parent_allocator_alloc(struct fy_allocator *a, size_t size, size_t align) +{ + if (!a) + return NULL; + + return fy_early_parent_allocator_alloc(a->parent, a->parent_tag, size, align); +} + +void fy_parent_allocator_free(struct fy_allocator *a, void *ptr) +{ + if (!a || !ptr) + return; + + return fy_early_parent_allocator_free(a->parent, a->parent_tag, ptr); +} diff --git a/src/allocator/fy-allocator.h b/src/allocator/fy-allocator.h new file mode 100644 index 0000000..b53fea0 --- /dev/null +++ b/src/allocator/fy-allocator.h @@ -0,0 +1,284 @@ +/* + * fy-allocator.h - allocators + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_ALLOCATOR_H +#define FY_ALLOCATOR_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include + +#include "fy-typelist.h" +#include "fy-utils.h" + +#include + +struct fy_allocator; +struct fy_allocator_stats; + +struct fy_allocator_stats; +struct fy_allocator_info; + +struct fy_allocator_ops { + int (*setup)(struct fy_allocator *a, struct fy_allocator *parent, int parent_tag, const void *cfg); + void (*cleanup)(struct fy_allocator *a); + struct fy_allocator *(*create)(struct fy_allocator *parent, int parent_tag, const void *cfg); + void (*destroy)(struct fy_allocator *a); + void (*dump)(struct fy_allocator *a); + void *(*alloc)(struct fy_allocator *a, int tag, size_t size, size_t align); + void (*free)(struct fy_allocator *a, int tag, void *data); + int (*update_stats)(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats); + const void *(*storev)(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash); + const void *(*lookupv)(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash); + void (*release)(struct fy_allocator *a, int tag, const void *data, size_t size); + int (*get_tag)(struct fy_allocator *a); + void (*release_tag)(struct fy_allocator *a, int tag); + int (*get_tag_count)(struct fy_allocator *a); + int (*set_tag_count)(struct fy_allocator *a, unsigned int tag_count); + void (*trim_tag)(struct fy_allocator *a, int tag); + void (*reset_tag)(struct fy_allocator *a, int tag); + struct fy_allocator_info *(*get_info)(struct fy_allocator *a, int tag); + unsigned int (*get_caps)(struct fy_allocator *a); /* Get capability flags (FYACF_*) */ + bool (*contains)(struct fy_allocator *a, int tag, const void *ptr); +}; + +struct fy_allocator_stats { + union { + struct { + uint64_t allocations; + uint64_t allocated; + uint64_t frees; + uint64_t freed; + uint64_t stores; + uint64_t stored; + uint64_t releases; + uint64_t released; + uint64_t dup_stores; + uint64_t dup_saved; + uint64_t system_claimed; + uint64_t system_free; + uint64_t collisions; + uint64_t unique_stores; + }; + uint64_t counters[14]; + }; +}; + +struct fy_allocator_arena_info { + size_t free, used, total; + void *data; + size_t size; +}; + +struct fy_allocator_tag_info { + int tag; + size_t free, used, total; + unsigned int num_arena_infos; + struct fy_allocator_arena_info *arena_infos; +}; + +struct fy_allocator_info { + size_t free, used, total; + unsigned int num_tag_infos; + struct fy_allocator_tag_info *tag_infos; +}; + +enum fy_allocator_flags { + FYAF_KEEP_STATS = FY_BIT(0), + FYAF_TRACE = FY_BIT(1), +}; + +struct fy_allocator { + enum fy_allocator_flags flags; + const char *name; + const struct fy_allocator_ops *ops; + struct fy_allocator *parent; + int parent_tag; +}; + +/* these private still */ +int fy_allocator_update_stats(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats); +struct fy_allocator_info *fy_allocator_get_info(struct fy_allocator *a, int tag); + +static inline void fy_allocator_set_keep_stats(struct fy_allocator *a, bool keep_flags) +{ + if (!a) + return; + + if (keep_flags) + a->flags |= FYAF_KEEP_STATS; + else + a->flags &= ~FYAF_KEEP_STATS; +} + +static inline void fy_allocator_set_trace(struct fy_allocator *a, bool trace) +{ + if (!a) + return; + + if (trace) + a->flags |= FYAF_TRACE; + else + a->flags &= ~FYAF_TRACE; +} + +FY_TYPE_FWD_DECL_LIST(registered_allocator_entry); +struct fy_registered_allocator_entry { + struct list_head node; + const char *name; + const struct fy_allocator_ops *ops; +}; +FY_TYPE_DECL_LIST(registered_allocator_entry); + +/* these are private still */ +char *fy_allocator_get_names(void); +int fy_allocator_register(const char *name, const struct fy_allocator_ops *ops); +int fy_allocator_unregister(const char *name); +void fy_allocator_release(struct fy_allocator *a, int tag, const void *ptr, size_t size); + +/* respects the parent allocator (or uses posix_memalign if NULL) */ +void *fy_early_parent_allocator_alloc(struct fy_allocator *parent, int parent_tag, size_t size, size_t align); +void fy_early_parent_allocator_free(struct fy_allocator *parent, int parent_tag, void *ptr); +void *fy_parent_allocator_alloc(struct fy_allocator *a, size_t size, size_t align); +void fy_parent_allocator_free(struct fy_allocator *a, void *ptr); + +#define FY_PARENT_ALLOCATOR_MALLOC ((struct fy_allocator *)NULL) +#define FY_PARENT_ALLOCATOR_INPLACE ((struct fy_allocator *)(uintptr_t)-1) + +static inline struct fy_allocator * +fy_allocator_get_parent(struct fy_allocator *a) +{ + if (!a || !a->parent || a->parent == FY_PARENT_ALLOCATOR_INPLACE) + return NULL; + return a->parent; +} + +struct fy_allocator * +fy_allocator_create_internal(const char *name, struct fy_allocator *parent, int parent_tag, const void *cfg); + +static inline void fy_allocator_destroy_nocheck(struct fy_allocator *a) +{ + a->ops->destroy(a); +} + +static inline void fy_allocator_dump_nocheck(struct fy_allocator *a) +{ + a->ops->dump(a); +} + +static inline int fy_allocator_update_stats_nocheck(struct fy_allocator *a, int tag, struct fy_allocator_stats *stats) +{ + return a->ops->update_stats(a, tag, stats); +} + +static inline void *fy_allocator_alloc_nocheck(struct fy_allocator *a, int tag, size_t size, size_t align) +{ + return a->ops->alloc(a, tag, size, align); +} + +static inline void fy_allocator_free_nocheck(struct fy_allocator *a, int tag, void *ptr) +{ + a->ops->free(a, tag, ptr); +} + +static inline const void *fy_allocator_store_nocheck(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) +{ + struct iovec iov[1]; + + /* just call the storev */ + iov[0].iov_base = (void *)data; + iov[0].iov_len = size; + + return a->ops->storev(a, tag, iov, 1, align, 0); +} + +static inline const void *fy_allocator_lookup_nocheck(struct fy_allocator *a, int tag, const void *data, size_t size, size_t align) +{ + struct iovec iov[1]; + + /* just call the storev */ + iov[0].iov_base = (void *)data; + iov[0].iov_len = size; + + return a->ops->lookupv(a, tag, iov, 1, align, 0); +} +static inline const void *fy_allocator_lookupv_nocheck(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + return a->ops->lookupv(a, tag, iov, iovcnt, align, hash); +} + +static inline const void *fy_allocator_storev_hash_nocheck(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align, uint64_t hash) +{ + return a->ops->storev(a, tag, iov, iovcnt, align, hash); +} + +static inline const void *fy_allocator_storev_nocheck(struct fy_allocator *a, int tag, const struct iovec *iov, int iovcnt, size_t align) +{ + return a->ops->storev(a, tag, iov, iovcnt, align, 0); +} + +static inline void fy_allocator_release_nocheck(struct fy_allocator *a, int tag, const void *ptr, size_t size) +{ + a->ops->release(a, tag, ptr, size); +} + +static inline int fy_allocator_get_tag_nocheck(struct fy_allocator *a) +{ + return a->ops->get_tag(a); +} + +static inline void fy_allocator_release_tag_nocheck(struct fy_allocator *a, int tag) +{ + a->ops->release_tag(a, tag); +} + +static inline int fy_allocator_get_tag_count_nocheck(struct fy_allocator *a) +{ + return a->ops->get_tag_count(a); +} + +static inline int fy_allocator_set_tag_count_nocheck(struct fy_allocator *a, unsigned int count) +{ + return a->ops->set_tag_count(a, count); +} + +static inline void fy_allocator_trim_tag_nocheck(struct fy_allocator *a, int tag) +{ + a->ops->trim_tag(a, tag); +} + +static inline void fy_allocator_reset_tag_nocheck(struct fy_allocator *a, int tag) +{ + a->ops->reset_tag(a, tag); +} + +static inline struct fy_allocator_info * +fy_allocator_get_info_nocheck(struct fy_allocator *a, int tag) +{ + return a->ops->get_info(a, tag); +} + +static inline enum fy_allocator_cap_flags +fy_allocator_get_caps_nocheck(struct fy_allocator *a) +{ + return a->ops->get_caps(a); +} + +static inline bool +fy_allocator_contains_nocheck(struct fy_allocator *a, int tag, const void *ptr) +{ + return a->ops->contains(a, tag, ptr); +} + +#endif diff --git a/src/internal/fy-allocators.c b/src/internal/fy-allocators.c new file mode 100644 index 0000000..b1ed602 --- /dev/null +++ b/src/internal/fy-allocators.c @@ -0,0 +1,301 @@ +/* + * fy-allocators.c - allocators testing internal utility + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-utils.h" + +#include "fy-allocator.h" +#include "fy-allocator-linear.h" +#include "fy-allocator-malloc.h" +#include "fy-allocator-mremap.h" +#include "fy-allocator-dedup.h" +#include "fy-allocator-auto.h" + +static void dump_allocator_info(struct fy_allocator *a, int tag) +{ + struct fy_allocator_info *info; + struct fy_allocator_tag_info *tag_info; + struct fy_allocator_arena_info *arena_info; + unsigned int i, j; + + info = fy_allocator_get_info(a, tag); + if (!info) { + fprintf(stderr, "fy_allocator_get_info() failed\n"); + return; + } + + fprintf(stderr, "Allocator %p: free=%zu used=%zu total=%zu\n", a, + info->free, info->used, info->total); + for (i = 0; i < info->num_tag_infos; i++) { + tag_info = &info->tag_infos[i]; + + fprintf(stderr, "\ttag #%d: free=%zu used=%zu total=%zu\n", i, + tag_info->free, tag_info->used, tag_info->total); + for (j = 0; j < tag_info->num_arena_infos; j++) { + arena_info = &tag_info->arena_infos[j]; + + fprintf(stderr, "\t\tarena #%d: free=%zu used=%zu total=%zu data=%p-0x%zx\n", j, + arena_info->free, arena_info->used, arena_info->total, + arena_info->data, arena_info->size); + } + } + + free(info); +} + +static int allocator_test(const char *allocator, const char *parent_allocator, size_t size) +{ + struct fy_linear_allocator_cfg lcfg; + struct fy_dedup_allocator_cfg dcfg; + const void *gcfg = NULL, *pcfg = NULL; + struct fy_allocator *a = NULL, *pa = NULL; + int tag0; + char *names; + bool is_linear, is_dedup; + unsigned int *uintp[16]; + uint8_t *p; + const uint8_t *p1, *p2; + size_t psz = 4096; + unsigned int i; + const void *linear_data; + size_t linear_size; + + names = fy_allocator_get_names(); + assert(names); + fprintf(stderr, "Available allocators: %s\n", names); + free(names); + + is_linear = !strcmp(allocator, "linear"); + is_dedup = !strcmp(allocator, "dedup"); + + if (is_linear) { + memset(&lcfg, 0, sizeof(lcfg)); + lcfg.size = size ? size : 4096; + gcfg = &lcfg; + } else if (is_dedup) { + + fprintf(stderr, "Using parent-allocator: %s\n", parent_allocator); + + if (!strcmp(parent_allocator, "linear")) { + memset(&lcfg, 0, sizeof(lcfg)); + lcfg.size = size ? size : 4096; + + pcfg = &lcfg; + } else + pcfg = NULL; + + pa = fy_allocator_create(parent_allocator, pcfg); + assert(pa); + + memset(&dcfg, 0, sizeof(dcfg)); + dcfg.parent_allocator = pa; + gcfg = &dcfg; + } else + gcfg = NULL; + + fprintf(stderr, "Using allocator: %s\n", allocator); + + a = fy_allocator_create(allocator, gcfg); + assert(a); + + fprintf(stderr, "Allocator created: %p\n", a); + + tag0 = fy_allocator_get_tag(a); + assert(tag0 != FY_ALLOC_TAG_ERROR); + + fprintf(stderr, "tag0 created: %d\n", tag0); + + fprintf(stderr, "Allocating %u integers\n", (unsigned int)ARRAY_SIZE(uintp)); + for (i = 0; i < ARRAY_SIZE(uintp); i++) { + uintp[i] = fy_allocator_alloc(a, tag0, sizeof(unsigned int), _Alignof(unsigned int)); + assert(uintp[i] != NULL); + fprintf(stderr, "\t%u: %p\n", i, uintp[i]); + } + + /* fill in */ + for (i = 0; i < ARRAY_SIZE(uintp); i++) + *uintp[i] = i; + + fprintf(stderr, "Dumping allocator areas before trim\n"); + dump_allocator_info(a, tag0); + + fy_allocator_trim_tag(a, tag0); + + fprintf(stderr, "Dumping allocator areas after trim\n"); + dump_allocator_info(a, tag0); + + fprintf(stderr, "Allocating %zu bytes\n", psz); + p = fy_allocator_alloc(a, tag0, psz, 1); + if (!p) { + fprintf(stderr, "failed to allocate %zu bytes\n", psz); + } else { + for (i = 0; i < psz; i++) + p[i] = i % 251; + } + + /* verify */ + if (p) { + for (i = 0; i < psz; i++) + assert(p[i] == (i % 251)); + } + + for (i = 0; i < ARRAY_SIZE(uintp); i++) + assert(*uintp[i] == i); + + fprintf(stderr, "Dumping allocator areas after alloc\n"); + dump_allocator_info(a, tag0); + + fprintf(stderr, "Storing a copy of p (p1)\n"); + p1 = fy_allocator_store(a, tag0, p, psz, 1); + + fprintf(stderr, "Storing a copy of p (p2)\n"); + p2 = fy_allocator_store(a, tag0, p, psz, 1); + + fprintf(stderr, "Dumping allocator areas after double store p1=%p p2=%p\n", p1, p2); + dump_allocator_info(a, tag0); + + if (p1) { + for (i = 0; i < psz; i++) + assert(p1[i] == (i % 251)); + } + + if (p2) { + for (i = 0; i < psz; i++) + assert(p2[i] == (i % 251)); + } + + fprintf(stderr, "Allocator %p tag %u linear_size %zd\n", + a, tag0, fy_allocator_get_tag_linear_size(a, tag0)); + + linear_size = 0; + linear_data = fy_allocator_get_tag_single_linear(a, tag0, &linear_size); + fprintf(stderr, "Allocator %p tag %u linear_data %p linear_size 0x%zx\n", + a, tag0, linear_data, linear_size); + + fprintf(stderr, "Releasing tag0\n"); + fy_allocator_release_tag(a, tag0); + + fprintf(stderr, "Dumping allocator areas after release\n"); + dump_allocator_info(a, FY_ALLOC_TAG_NONE); + + if (a) + fy_allocator_destroy(a); + + if (pa) + fy_allocator_destroy(pa); + + + return 0; +} + +static struct option lopts[] = { + {"allocator", required_argument, 0, 'a' }, + {"parent", required_argument, 0, 'p' }, + {"size", required_argument, 0, 's' }, + {"help", no_argument, 0, 'h' }, + {0, 0, 0, 0 }, +}; + +static void display_usage(FILE *fp, const char *progname) +{ + char *names; + const char *s; + + names = fy_allocator_get_names(); + assert(names); + + s = strrchr(progname, '/'); + if (s != NULL) + progname = s + 1; + + fprintf(fp, "Usage:\n\t%s [options]\n", progname); + fprintf(fp, "\noptions:\n"); + fprintf(fp, "\t--allocator , -a : Use allocator, one of: %s\n", names); + fprintf(fp, "\t--parent , -p : Use parent allocator, one of: %s\n", names); + fprintf(fp, "\t--size , -s : Size for allocators that require one\n"); + fprintf(fp, "\t--help, -h : Display help message\n"); + fprintf(fp, "\n"); + + free(names); +} + +int main(int argc, char *argv[]) +{ + int opt, lidx, rc; + int exitcode = EXIT_FAILURE; + const char *allocator = "linear"; + const char *parent_allocator = "linear"; + size_t size = 0; + + while ((opt = getopt_long_only(argc, argv, "a:p:s:h", lopts, &lidx)) != -1) { + switch (opt) { + case 'a': + case 'p': + if (!fy_allocator_is_available(optarg)) { + fprintf(stderr, "Error: illegal allocator name \"%s\"\n", optarg); + goto err_out_usage; + } + if (opt == 'a') + allocator = optarg; + else + parent_allocator = optarg; + break; + case 's': + size = (size_t)atoi(optarg); + break; + case 'h' : + display_usage(stdout, argv[0]); + goto ok_out; + default: + goto err_out_usage; + } + } + + rc = allocator_test(allocator, parent_allocator, size); + if (rc) { + fprintf(stderr, "Error: allocator_test() failed\n"); + goto err_out; + } + +ok_out: + exitcode = EXIT_SUCCESS; + +out: + return exitcode; + +err_out_usage: + display_usage(stderr, argv[0]); +err_out: + exitcode = EXIT_FAILURE; + goto out; +} diff --git a/src/internal/fy-thread.c b/src/internal/fy-thread.c index 7bbac06..2880b00 100644 --- a/src/internal/fy-thread.c +++ b/src/internal/fy-thread.c @@ -33,18 +33,19 @@ #include +#include "fy-atomics.h" #include "fy-thread.h" static void test_worker_thread_fn(void *arg) { - _Atomic(int) *p = arg; + FY_ATOMIC(int) *p = arg; int v, exp_v; /* atomically increase the counter */ - v = atomic_load(p); + v = fy_atomic_load(p); for (;;) { exp_v = v; - if (atomic_compare_exchange_strong(p, &exp_v, v + 1)) + if (fy_atomic_compare_exchange_strong(p, &exp_v, v + 1)) return; v = exp_v; } @@ -290,16 +291,16 @@ void test_thread_latency(unsigned int num_threads) #define STEAL_LOOP_COUNT 10000 static void test_worker_thread_steal_fn(void *arg) { - _Atomic(int) *p = arg; + FY_ATOMIC(int) *p = arg; int v, exp_v; unsigned int i; /* atomically increase the counter STEAL_LOOP_COUNT times */ for (i = 0; i < STEAL_LOOP_COUNT; i++) { - v = atomic_load(p); + v = fy_atomic_load(p); for (;;) { exp_v = v; - if (atomic_compare_exchange_strong(p, &exp_v, v + 1)) + if (fy_atomic_compare_exchange_strong(p, &exp_v, v + 1)) break; v = exp_v; } diff --git a/src/internal/libfyaml-parser.c b/src/internal/libfyaml-parser.c index d24f03f..7253cce 100644 --- a/src/internal/libfyaml-parser.c +++ b/src/internal/libfyaml-parser.c @@ -357,7 +357,7 @@ void dump_event(struct fy_parser *fyp, struct fy_event *fye) printf("%-14s%s|\n", "MAPPING_END", mm); break; default: - assert(0); + FY_IMPOSSIBLE_ABORT(); } } @@ -458,7 +458,7 @@ void dump_testsuite_event(FILE *fp, struct fy_parser *fyp, struct fy_event *fye) fprintf(fp, " >"); break; default: - abort(); + FY_IMPOSSIBLE_ABORT(); } print_escaped(fp, value, value_len); fprintf(fp, "\n"); @@ -468,7 +468,7 @@ void dump_testsuite_event(FILE *fp, struct fy_parser *fyp, struct fy_event *fye) fprintf(fp, "=ALI *%.*s\n", (int)anchor_len, anchor); break; default: - assert(0); + FY_IMPOSSIBLE_ABORT(); } } @@ -1281,7 +1281,7 @@ void dump_libyaml_event(yaml_event_t *event) printf("%-14s%s|\n", "MAPPING_END", mm); break; default: - assert(0); + FY_IMPOSSIBLE_ABORT(); } } @@ -1374,7 +1374,7 @@ void dump_libyaml_testsuite_event(FILE *fp, yaml_event_t *event) fprintf(fp, " >"); break; case YAML_ANY_SCALAR_STYLE: - abort(); + FY_IMPOSSIBLE_ABORT(); } print_escaped(fp, (char *)event->data.scalar.value, event->data.scalar.length); fprintf(fp, "\n"); @@ -1383,7 +1383,7 @@ void dump_libyaml_testsuite_event(FILE *fp, yaml_event_t *event) fprintf(fp, "=ALI *%s\n", event->data.alias.anchor); break; default: - assert(0); + FY_IMPOSSIBLE_ABORT(); } } diff --git a/src/lib/fy-accel.c b/src/lib/fy-accel.c index 1040078..87a9b23 100644 --- a/src/lib/fy-accel.c +++ b/src/lib/fy-accel.c @@ -253,7 +253,8 @@ fy_accel_entry_lookup(struct fy_accel *xl, const void *key) struct fy_accel_entry *xle; xle = fy_accel_entry_iter_start(&xli, xl, key); - fy_accel_entry_iter_finish(&xli); + if (xle) + fy_accel_entry_iter_finish(&xli); return xle; } @@ -262,10 +263,13 @@ struct fy_accel_entry * fy_accel_entry_lookup_key_value(struct fy_accel *xl, const void *key, const void *value) { struct fy_accel_entry_iter xli; - struct fy_accel_entry *xle; + struct fy_accel_entry *xle, *xle_start; + + xle_start = fy_accel_entry_iter_start(&xli, xl, key); + if (!xle_start) + return NULL; - for (xle = fy_accel_entry_iter_start(&xli, xl, key); xle; - xle = fy_accel_entry_iter_next(&xli)) { + for (xle = xle_start; xle; xle = fy_accel_entry_iter_next(&xli)) { if (xle->value == value) break; diff --git a/src/lib/fy-atom.c b/src/lib/fy-atom.c index e0b08cf..c8d9e8e 100644 --- a/src/lib/fy-atom.c +++ b/src/lib/fy-atom.c @@ -23,6 +23,18 @@ #include "fy-parse.h" #include "fy-doc.h" +#include "fy-input.h" + +#include "fy-atom.h" + +const char *fy_atom_data(const struct fy_atom *atom) +{ + if (!atom) + return NULL; + + return fy_input_start(atom->fyi) + atom->start_mark.input_pos; +} + struct fy_atom *fy_reader_fill_atom(struct fy_reader *fyr, int advance, struct fy_atom *handle) { @@ -103,7 +115,7 @@ struct fy_atom *fy_reader_fill_atom_at(struct fy_reader *fyr, int advance, int c struct fy_mark start_mark, end_mark; int rc; - if (!fyr || !handle) + if (!fyr || !handle || !fyr->current_input) return NULL; /* start mark */ diff --git a/src/lib/fy-atom.h b/src/lib/fy-atom.h index 10ff7b7..3f32275 100644 --- a/src/lib/fy-atom.h +++ b/src/lib/fy-atom.h @@ -21,7 +21,7 @@ #include #include "fy-list.h" -#include "fy-input.h" +#include "fy-ctype.h" struct fy_reader; struct fy_input; @@ -66,11 +66,12 @@ struct fy_atom { uint64_t tozero; /* fast way to zero everything here */ struct { /* save a little bit of space with bitfields */ - enum fy_atom_style style : 8; /* note that it's a big perf win for bytes */ - enum fy_atom_chomp chomp : 8; + unsigned int style : 8; /* enum fy_atom_style, note that it's a big perf win for bytes */ + unsigned int chomp : 8; /* enum fy_atom_chomp */ unsigned int tabsize : 8; - enum fy_lb_mode lb_mode : 1; - enum fy_flow_ws_mode fws_mode : 1; + unsigned int lb_mode : 1; /* enum fy_lb_mode */ + unsigned int fws_mode : 1; /* enum fy_flow_ws_mode */ + unsigned int directive0_mode : 1; bool direct_output : 1; /* can directly output */ bool storage_hint_valid : 1; bool empty : 1; /* atom contains whitespace and linebreaks only if length > 0 */ @@ -86,6 +87,8 @@ struct fy_atom { bool json_mode : 1; /* atom was read in json mode */ bool ends_with_eof : 1; /* atom ends at EOF of input */ bool is_merge_key: 1; /* atom is just << */ + bool simple_key_allowed : 1; /* atom allows a simple key */ + bool high_ascii : 1; /* atom has utf code point >= 0x80 (only for plains) */ }; }; }; @@ -146,46 +149,6 @@ const char *fy_atom_format_text(struct fy_atom *atom, char *buf, size_t maxsz); int fy_atom_format_utf8_length(struct fy_atom *atom); -static inline void -fy_reader_fill_atom_start(struct fy_reader *fyr, struct fy_atom *handle) -{ - /* start mark */ - fy_reader_get_mark(fyr, &handle->start_mark); - handle->fyi = fy_reader_current_input(fyr); - handle->fyi_generation = fy_reader_current_input_generation(fyr); - - handle->increment = 0; - handle->tozero = 0; - - /* note that handle->data may be zero for empty input */ -} - -static inline void -fy_reader_fill_atom_end_at(struct fy_reader *fyr, struct fy_atom *handle, struct fy_mark *end_mark) -{ - if (end_mark) - handle->end_mark = *end_mark; - else - fy_reader_get_mark(fyr, &handle->end_mark); - - /* default is plain, modify at return */ - handle->style = FYAS_PLAIN; - handle->chomp = FYAC_CLIP; - /* by default we don't do storage hints, it's the job of the caller */ - handle->storage_hint = 0; - handle->storage_hint_valid = false; - handle->tabsize = fy_reader_tabsize(fyr); - handle->json_mode = fy_reader_json_mode(fyr); - handle->lb_mode = fy_reader_lb_mode(fyr); - handle->fws_mode = fy_reader_flow_ws_mode(fyr); -} - -static inline void -fy_reader_fill_atom_end(struct fy_reader *fyr, struct fy_atom *handle) -{ - fy_reader_fill_atom_end_at(fyr, handle, NULL); -} - static inline void fy_atom_reset_storage_hints(struct fy_atom *handle) { @@ -283,13 +246,7 @@ int fy_atom_strcmp(struct fy_atom *atom, const char *str); bool fy_atom_is_number(struct fy_atom *atom); int fy_atom_cmp(struct fy_atom *atom1, struct fy_atom *atom2); -static inline const char *fy_atom_data(const struct fy_atom *atom) -{ - if (!atom) - return NULL; - - return fy_input_start(atom->fyi) + atom->start_mark.input_pos; -} +const char *fy_atom_data(const struct fy_atom *atom); static inline size_t fy_atom_size(const struct fy_atom *atom) { diff --git a/src/lib/fy-composer-diag.c b/src/lib/fy-composer-diag.c new file mode 100644 index 0000000..bc75b24 --- /dev/null +++ b/src/lib/fy-composer-diag.c @@ -0,0 +1,91 @@ +/* + * fy-composer-diag.c - composer diagnostics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-diag.h" +#include "fy-composer.h" + +int fy_composer_vdiag(struct fy_composer *fyc, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap) +{ + struct fy_diag_ctx fydc; + int rc; + + if (!fyc || !fmt || !fyc->cfg.diag) + return -1; + + /* perform the enable tests early to avoid the overhead */ + if (((flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT) < fyc->cfg.diag->cfg.level) + return 0; + + /* fill in fy_diag_ctx */ + memset(&fydc, 0, sizeof(fydc)); + + fydc.level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; + fydc.module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; + fydc.source_file = file; + fydc.source_line = line; + fydc.source_func = func; + fydc.line = -1; + fydc.column = -1; + + rc = fy_vdiag(fyc->cfg.diag, &fydc, fmt, ap); + + return rc; +} + +int fy_composer_diag(struct fy_composer *fyc, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_composer_vdiag(fyc, flags, file, line, func, fmt, ap); + va_end(ap); + + return rc; +} + +void fy_composer_diag_vreport(struct fy_composer *fyc, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap) +{ + if (!fyc || !fyc->cfg.diag || !fydrc || !fmt) + return; + + fy_diag_vreport(fyc->cfg.diag, fydrc, fmt, ap); +} + +void fy_composer_diag_report(struct fy_composer *fyc, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_composer_diag_vreport(fyc, fydrc, fmt, ap); + va_end(ap); +} diff --git a/src/lib/fy-composer.c b/src/lib/fy-composer.c index 9bdfbfb..85b17bf 100644 --- a/src/lib/fy-composer.c +++ b/src/lib/fy-composer.c @@ -162,7 +162,7 @@ fy_composer_process_event_private(struct fy_composer *fyc, struct fy_event *fye, if (!stop_req) stop_req = ret == FYCR_OK_STOP; - rc = fy_document_builder_process_event(fypp->fydb, fyep); + rc = fy_document_builder_process_event(fypp->fydb, fyep ? &fyep->e : NULL); if (rc == 0) return FYCR_OK_CONTINUE; fyc_error_check(fyc, rc > 0, err_out, @@ -206,7 +206,7 @@ fy_composer_process_event_private(struct fy_composer *fyc, struct fy_event *fye, "ops->create_document_builder() failed\n"); /* and pass the current event; must return 0 since we know it's a collection start */ - rc = fy_document_builder_process_event(fypp->fydb, fyep); + rc = fy_document_builder_process_event(fypp->fydb, fyep ? &fyep->e : NULL); fyc_error_check(fyc, !rc, err_out, "fy_document_builder_process_event() failed\n"); @@ -313,8 +313,8 @@ err_out: return FYCR_ERROR; } -static void -fy_composer_halt(struct fy_composer *fyc, struct fy_path *fypp, enum fy_composer_return rc) +void +fy_composer_halt_one(struct fy_composer *fyc, struct fy_path *fypp, enum fy_composer_return rc) { const struct fy_composer_ops *ops; struct fy_eventp ev_none; @@ -344,6 +344,18 @@ fy_composer_halt(struct fy_composer *fyc, struct fy_path *fypp, enum fy_composer ops->process_event(fyc, fypp, &ev_none.e); } +void fy_composer_halt(struct fy_composer *fyc, enum fy_composer_return rc) +{ + struct fy_path *fypp; + + /* for normal stop, we don't teardown */ + if (rc == FYCR_OK_STOP) + return; + + for (fypp = fy_path_list_head(&fyc->paths); fypp; fypp = fy_path_next(&fyc->paths, fypp)) + fy_composer_halt_one(fyc, fypp, rc); +} + enum fy_composer_return fy_composer_process_event(struct fy_composer *fyc, struct fy_event *fye) { @@ -351,19 +363,19 @@ fy_composer_process_event(struct fy_composer *fyc, struct fy_event *fye) int rc; if (!fyc || !fye) - return -1; + return FYCR_ERROR; /* start at the head */ fypp = fy_path_list_head(&fyc->paths); /* no top? something's very out of order */ if (!fypp) - return -1; + return FYCR_ERROR; rc = fy_composer_process_event_private(fyc, fye, fypp); if (rc == FYCR_ERROR || rc == FYCR_OK_STOP) - fy_composer_halt(fyc, fypp, rc); + fy_composer_halt(fyc, rc); return rc; } @@ -388,3 +400,43 @@ struct fy_diag *fy_composer_get_diag(struct fy_composer *fyc) return NULL; return fyc->cfg.diag; } + +struct fy_path *fy_composer_get_root_path(struct fy_composer *fyc) +{ + if (!fyc) + return NULL; + return fy_path_list_head(&fyc->paths); +} + +struct fy_path *fy_composer_get_next_path(struct fy_composer *fyc, struct fy_path *fypp) +{ + if (!fyc || !fypp) + return NULL; + return fy_path_next(&fyc->paths, fypp); +} + +struct fy_path *fy_composer_get_path(struct fy_composer *fyc) +{ + struct fy_path *fypp, *fyppt; + struct fy_path_component *fypc_last; + + if (!fyc) + return NULL; + + fypp = fy_path_list_head(&fyc->paths); + if (!fypp) + return NULL; + + /* traverse until we find the last non complex key acculating */ + for (;;) { + fypc_last = fy_path_component_list_tail(&fypp->components); + if (!fy_path_component_is_mapping(fypc_last) || !fypc_last->map.accumulating_complex_key) + break; + fyppt = fy_path_next(&fyc->paths, fypp); + if (!fyppt) + break; + fypp = fyppt; + } + + return fypp; +} diff --git a/src/lib/fy-composer.h b/src/lib/fy-composer.h index d690f22..2c1634b 100644 --- a/src/lib/fy-composer.h +++ b/src/lib/fy-composer.h @@ -22,6 +22,8 @@ #include "fy-emit-accum.h" #include "fy-path.h" +#include "fy-diag.h" + struct fy_composer; struct fy_token; struct fy_diag; @@ -29,29 +31,107 @@ struct fy_event; struct fy_eventp; struct fy_document_builder; -struct fy_composer_ops { - /* single process event callback */ - enum fy_composer_return (*process_event)(struct fy_composer *fyc, struct fy_path *path, struct fy_event *fye); - struct fy_document_builder *(*create_document_builder)(struct fy_composer *fyc); -}; - -struct fy_composer_cfg { - const struct fy_composer_ops *ops; - void *userdata; - struct fy_diag *diag; -}; - struct fy_composer { struct fy_composer_cfg cfg; struct fy_path_list paths; }; -struct fy_composer *fy_composer_create(struct fy_composer_cfg *cfg); -void fy_composer_destroy(struct fy_composer *fyc); -int fy_composer_process_event(struct fy_composer *fyc, struct fy_event *fye); +void fy_composer_halt_one(struct fy_composer *fyc, struct fy_path *fypp, enum fy_composer_return rc); +void fy_composer_halt(struct fy_composer *fyc, enum fy_composer_return rc); + +/* diagnostics */ + +static inline bool +fyc_debug_log_level_is_enabled(struct fy_composer *fyc, enum fy_error_module module) +{ + return fyc && fy_diag_log_level_is_enabled(fyc->cfg.diag, FYET_DEBUG, module); +} + +int fy_composer_vdiag(struct fy_composer *fyc, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap); + +int fy_composer_diag(struct fy_composer *fyc, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) + __attribute__((format(printf, 6, 7))); + +void fy_composer_diag_vreport(struct fy_composer *fyc, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap); +void fy_composer_diag_report(struct fy_composer *fyc, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +#ifdef FY_DEVMODE + +#define fyc_debug(_fyc, _module, _fmt, ...) \ + do { \ + struct fy_composer *__fyc = (_fyc); \ + enum fy_error_module __module = (_module); \ + \ + if (fyc_debug_log_level_is_enabled(__fyc, __module)) \ + fy_composer_diag(__fyc, FYET_DEBUG | FYDF_MODULE(_module), \ + __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__); \ + } while(0) +#else + +#define fyc_debug(_fyc, _module, _fmt, ...) \ + do { } while(0) + +#endif + +#define fyc_info(_fyc, _fmt, ...) \ + fy_composer_diag((_fyc), FYET_INFO, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyc_notice(_fyc, _fmt, ...) \ + fy_composer_diag((_fyc), FYET_NOTICE, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyc_warning(_fyc, _fmt, ...) \ + fy_composer_diag((_fyc), FYET_WARNING, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyc_error(_fyc, _fmt, ...) \ + fy_composer_diag((_fyc), FYET_ERROR, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) + +#define fyc_error_check(_fyc, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + fyc_error((_fyc), _fmt, ## __VA_ARGS__); \ + goto _label ; \ + } \ + } while(0) + +#define _FYC_TOKEN_DIAG(_fyc, _fyt, _type, _module, _fmt, ...) \ + do { \ + struct fy_diag_report_ctx _drc; \ + memset(&_drc, 0, sizeof(_drc)); \ + _drc.type = (_type); \ + _drc.module = (_module); \ + _drc.fyt = (_fyt); \ + fy_composer_diag_report((_fyc), &_drc, (_fmt) , ## __VA_ARGS__); \ + } while(0) + +#define FYC_TOKEN_DIAG(_fyc, _fyt, _type, _module, _fmt, ...) \ + _FYC_TOKEN_DIAG(_fyc, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYC_NODE_DIAG(_fyc, _fyn, _type, _module, _fmt, ...) \ + _FYC_TOKEN_DIAG(_fyc, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYC_TOKEN_ERROR(_fyc, _fyt, _module, _fmt, ...) \ + FYC_TOKEN_DIAG(_fyc, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYC_TOKEN_ERROR_CHECK(_fyc, _fyt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYC_TOKEN_ERROR(_fyc, _fyt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) -struct fy_composer_cfg *fy_composer_get_cfg(struct fy_composer *fyc); -void *fy_composer_get_cfg_userdata(struct fy_composer *fyc); -struct fy_diag *fy_composer_get_diag(struct fy_composer *fyc); +#define FYC_TOKEN_WARNING(_fyc, _fyt, _module, _fmt, ...) \ + FYC_TOKEN_DIAG(_fyc, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) #endif diff --git a/src/lib/fy-diag.c b/src/lib/fy-diag.c index e30003f..27c920e 100644 --- a/src/lib/fy-diag.c +++ b/src/lib/fy-diag.c @@ -22,7 +22,11 @@ #include +#include "fy-diag.h" + #include "fy-parse.h" +#include "fy-input.h" + static const char *error_type_txt[] = { [FYET_DEBUG] = "debug", @@ -63,14 +67,18 @@ enum fy_error_type fy_string_to_error_type(const char *str) } static const char *error_module_txt[] = { - [FYEM_UNKNOWN] = "unknown", - [FYEM_ATOM] = "atom", - [FYEM_SCAN] = "scan", - [FYEM_PARSE] = "parse", - [FYEM_DOC] = "doc", - [FYEM_BUILD] = "build", - [FYEM_INTERNAL] = "internal", - [FYEM_SYSTEM] = "system", + [FYEM_UNKNOWN] = "unknown", + [FYEM_ATOM] = "atom", + [FYEM_SCAN] = "scan", + [FYEM_PARSE] = "parse", + [FYEM_DOC] = "doc", + [FYEM_BUILD] = "build", + [FYEM_INTERNAL] = "internal", + [FYEM_SYSTEM] = "system", + [FYEM_EMIT] = "emit", + [FYEM_TYPESET] = "typeset", + [FYEM_DECODE] = "decode", + [FYEM_ENCODE] = "encode", }; const char *fy_error_module_to_string(enum fy_error_module module) @@ -114,14 +122,18 @@ static const char *fy_error_level_str(enum fy_error_type level) static const char *fy_error_module_str(enum fy_error_module module) { static const char *txt[] = { - [FYEM_UNKNOWN] = "UNKWN", - [FYEM_ATOM] = "ATOM ", - [FYEM_SCAN] = "SCAN ", - [FYEM_PARSE] = "PARSE", - [FYEM_DOC] = "DOC ", - [FYEM_BUILD] = "BUILD", - [FYEM_INTERNAL] = "INTRL", - [FYEM_SYSTEM] = "SYSTM", + [FYEM_UNKNOWN] = "UNKWN", + [FYEM_ATOM] = "ATOM ", + [FYEM_SCAN] = "SCAN ", + [FYEM_PARSE] = "PARSE", + [FYEM_DOC] = "DOC ", + [FYEM_BUILD] = "BUILD", + [FYEM_INTERNAL] = "INTRL", + [FYEM_SYSTEM] = "SYSTM", + [FYEM_EMIT] = "EMIT ", + [FYEM_TYPESET] = "TYPES", + [FYEM_DECODE] = "DEC ", + [FYEM_ENCODE] = "ENC ", }; if ((unsigned int)module >= FYEM_MAX) @@ -250,6 +262,12 @@ bool fy_diag_got_error(struct fy_diag *diag) return diag && diag->on_error; } +void fy_diag_set_error(struct fy_diag *diag, bool on_error) +{ + if (diag) + diag->on_error = on_error; +} + void fy_diag_reset_error(struct fy_diag *diag) { struct fy_diag_errorp *errp; @@ -444,18 +462,13 @@ int fy_vdiag(struct fy_diag *diag, const struct fy_diag_ctx *fydc, if (level >= FYET_ERROR && diag->on_error) level = FYET_DEBUG; - if (level < diag->cfg.level) { + if (!fy_diag_log_level_is_enabled(diag, level, fydc->module)) { rc = 0; goto out; } - /* check module enable mask */ - if (!(diag->cfg.module_mask & FY_BIT(fydc->module))) { - rc = 0; - goto out; - } - - msg = alloca_vsprintf(fmt, ap); + msg = fy_vsprintfa(fmt, ap); + fy_strip_trailing_nl(msg); /* source part */ if (diag->cfg.show_source) { @@ -467,21 +480,21 @@ int fy_vdiag(struct fy_diag *diag, const struct fy_diag_ctx *fydc, file_stripped++; } else file_stripped = ""; - source = alloca_sprintf("%s:%d @%s()%s", + source = fy_sprintfa("%s:%d @%s()%s", file_stripped, fydc->source_line, fydc->source_func, " "); } /* position part */ if (diag->cfg.show_position && fydc->line >= 0 && fydc->column >= 0) - position = alloca_sprintf("<%3d:%2d>%s", fydc->line, fydc->column, ": "); + position = fy_sprintfa("<%3d:%2d>%s", fydc->line, fydc->column, ": "); /* type part */ if (diag->cfg.show_type) - typestr = alloca_sprintf("[%s]%s", fy_error_level_str(level), ": "); + typestr = fy_sprintfa("[%s]%s", fy_error_level_str(level), ": "); /* module part */ if (diag->cfg.show_module) - modulestr = alloca_sprintf("<%s>%s", fy_error_module_str(fydc->module), ": "); + modulestr = fy_sprintfa("<%s>%s", fy_error_module_str(fydc->module), ": "); if (diag->cfg.colorize) { switch (level) { @@ -600,6 +613,10 @@ void fy_diag_error_atom_display(struct fy_diag *diag, enum fy_error_type type, s fy_diag_get_error_colors(diag, type, &color_start, &color_end, &white); + /* pacify compilers that think these are uninitialized */ + cols = 0; + rowbufsz = 0; + /* two passes, first one collects extents */ start_col = -1; @@ -702,6 +719,11 @@ void fy_diag_error_atom_display(struct fy_diag *diag, enum fy_error_type type, s /* optimize by using the content end as a starting point */ s = l->content_start + l->content_len; e = l->line_start + l->line_count; + + /* guard against something stupid */ + if (s > e) + s = e; + col8 = l->content_end_col8; while ((c = fy_utf8_get(s, (e - s), &w)) >= 0) { s += w; @@ -834,15 +856,16 @@ void fy_diag_vreport(struct fy_diag *diag, } /* it will strip trailing newlines */ - msg_str = alloca_vsprintf(fmt, ap); + msg_str = fy_vsprintfa(fmt, ap); + fy_strip_trailing_nl(msg_str); /* get the colors */ fy_diag_get_error_colors(diag, fydrc->type, &color_start, &color_end, &white); if (name || (line > 0 && column > 0)) name_str = (line > 0 && column > 0) ? - alloca_sprintf("%s%s:%d:%d: ", white, name, line, column) : - alloca_sprintf("%s%s: ", white, name); + fy_sprintfa("%s%s:%d:%d: ", white, name, line, column) : + fy_sprintfa("%s%s: ", white, name); if (!diag->collect_errors) { fy_diag_printf(diag, "%s" "%s%s: %s" "%s\n", @@ -899,350 +922,6 @@ void fy_diag_report(struct fy_diag *diag, va_end(ap); } -/* parser */ - -int fy_parser_vdiag(struct fy_parser *fyp, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, va_list ap) -{ - struct fy_diag_ctx fydc; - int rc; - - if (!fyp || !fyp->diag || !fmt) - return -1; - - /* perform the enable tests early to avoid the overhead */ - if (((flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT) < fyp->diag->cfg.level) - return 0; - - /* fill in fy_diag_ctx */ - memset(&fydc, 0, sizeof(fydc)); - - fydc.level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; - fydc.module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; - fydc.source_file = file; - fydc.source_line = line; - fydc.source_func = func; - fydc.line = fyp_line(fyp); - fydc.column = fyp_column(fyp); - - rc = fy_vdiag(fyp->diag, &fydc, fmt, ap); - - if (fyp && !fyp->stream_error && fyp->diag->on_error) - fyp->stream_error = true; - - return rc; -} - -int fy_parser_diag(struct fy_parser *fyp, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, ...) -{ - va_list ap; - int rc; - - va_start(ap, fmt); - rc = fy_parser_vdiag(fyp, flags, file, line, func, fmt, ap); - va_end(ap); - - return rc; -} - -void fy_parser_diag_vreport(struct fy_parser *fyp, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, va_list ap) -{ - struct fy_diag *diag; - - if (!fyp || !fyp->diag || !fydrc || !fmt) - return; - - diag = fyp->diag; - - fy_diag_vreport(diag, fydrc, fmt, ap); - - if (fyp && !fyp->stream_error && diag->on_error) - fyp->stream_error = true; -} - -void fy_parser_diag_report(struct fy_parser *fyp, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, ...) -{ - va_list ap; - - va_start(ap, fmt); - fy_parser_diag_vreport(fyp, fydrc, fmt, ap); - va_end(ap); -} - -/* document */ - -int fy_document_vdiag(struct fy_document *fyd, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, va_list ap) -{ - struct fy_diag_ctx fydc; - int rc; - - if (!fyd || !fmt || !fyd->diag) - return -1; - - /* perform the enable tests early to avoid the overhead */ - if (((flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT) < fyd->diag->cfg.level) - return 0; - - /* fill in fy_diag_ctx */ - memset(&fydc, 0, sizeof(fydc)); - - fydc.level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; - fydc.module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; - fydc.source_file = file; - fydc.source_line = line; - fydc.source_func = func; - fydc.line = -1; - fydc.column = -1; - - rc = fy_vdiag(fyd->diag, &fydc, fmt, ap); - - return rc; -} - -int fy_document_diag(struct fy_document *fyd, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, ...) -{ - va_list ap; - int rc; - - va_start(ap, fmt); - rc = fy_document_vdiag(fyd, flags, file, line, func, fmt, ap); - va_end(ap); - - return rc; -} - -void fy_document_diag_vreport(struct fy_document *fyd, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, va_list ap) -{ - if (!fyd || !fyd->diag || !fydrc || !fmt) - return; - - fy_diag_vreport(fyd->diag, fydrc, fmt, ap); -} - -void fy_document_diag_report(struct fy_document *fyd, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, ...) -{ - va_list ap; - - va_start(ap, fmt); - fy_document_diag_vreport(fyd, fydrc, fmt, ap); - va_end(ap); -} - -/* composer */ -int fy_composer_vdiag(struct fy_composer *fyc, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, va_list ap) -{ - struct fy_diag_ctx fydc; - int rc; - - if (!fyc || !fmt || !fyc->cfg.diag) - return -1; - - /* perform the enable tests early to avoid the overhead */ - if (((flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT) < fyc->cfg.diag->cfg.level) - return 0; - - /* fill in fy_diag_ctx */ - memset(&fydc, 0, sizeof(fydc)); - - fydc.level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; - fydc.module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; - fydc.source_file = file; - fydc.source_line = line; - fydc.source_func = func; - fydc.line = -1; - fydc.column = -1; - - rc = fy_vdiag(fyc->cfg.diag, &fydc, fmt, ap); - - return rc; -} - -int fy_composer_diag(struct fy_composer *fyc, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, ...) -{ - va_list ap; - int rc; - - va_start(ap, fmt); - rc = fy_composer_vdiag(fyc, flags, file, line, func, fmt, ap); - va_end(ap); - - return rc; -} - -void fy_composer_diag_vreport(struct fy_composer *fyc, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, va_list ap) -{ - if (!fyc || !fyc->cfg.diag || !fydrc || !fmt) - return; - - fy_diag_vreport(fyc->cfg.diag, fydrc, fmt, ap); -} - -void fy_composer_diag_report(struct fy_composer *fyc, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, ...) -{ - va_list ap; - - va_start(ap, fmt); - fy_composer_diag_vreport(fyc, fydrc, fmt, ap); - va_end(ap); -} - -/* document_builder */ -int fy_document_builder_vdiag(struct fy_document_builder *fydb, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, va_list ap) -{ - struct fy_diag_ctx fydc; - int rc; - - if (!fydb || !fmt || !fydb->cfg.diag) - return -1; - - /* perform the enable tests early to avoid the overhead */ - if (((flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT) < fydb->cfg.diag->cfg.level) - return 0; - - /* fill in fy_diag_ctx */ - memset(&fydc, 0, sizeof(fydc)); - - fydc.level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; - fydc.module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; - fydc.source_file = file; - fydc.source_line = line; - fydc.source_func = func; - fydc.line = -1; - fydc.column = -1; - - rc = fy_vdiag(fydb->cfg.diag, &fydc, fmt, ap); - - return rc; -} - -int fy_document_builder_diag(struct fy_document_builder *fydb, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, ...) -{ - va_list ap; - int rc; - - va_start(ap, fmt); - rc = fy_document_builder_vdiag(fydb, flags, file, line, func, fmt, ap); - va_end(ap); - - return rc; -} - -void fy_document_builder_diag_vreport(struct fy_document_builder *fydb, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, va_list ap) -{ - if (!fydb || !fydb->cfg.diag || !fydrc || !fmt) - return; - - fy_diag_vreport(fydb->cfg.diag, fydrc, fmt, ap); -} - -void fy_document_builder_diag_report(struct fy_document_builder *fydb, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, ...) -{ - va_list ap; - - va_start(ap, fmt); - fy_document_builder_diag_vreport(fydb, fydrc, fmt, ap); - va_end(ap); -} - -/* reader */ - -int fy_reader_vdiag(struct fy_reader *fyr, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, va_list ap) -{ - struct fy_diag_ctx fydc; - int fydc_level, fyd_level; - - if (!fyr || !fyr->diag || !fmt) - return -1; - - /* perform the enable tests early to avoid the overhead */ - fydc_level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; - fyd_level = fyr->diag->cfg.level; - - if (fydc_level < fyd_level) - return 0; - - /* fill in fy_diag_ctx */ - memset(&fydc, 0, sizeof(fydc)); - - fydc.level = fydc_level; - fydc.module = FYEM_SCAN; /* reader is always scanner */ - fydc.source_file = file; - fydc.source_line = line; - fydc.source_func = func; - fydc.line = fyr->line; - fydc.column = fyr->column; - - return fy_vdiag(fyr->diag, &fydc, fmt, ap); -} - -int fy_reader_diag(struct fy_reader *fyr, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, ...) -{ - va_list ap; - int rc; - - va_start(ap, fmt); - rc = fy_reader_vdiag(fyr, flags, file, line, func, fmt, ap); - va_end(ap); - - return rc; -} - -void fy_reader_diag_vreport(struct fy_reader *fyr, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, va_list ap) -{ - if (!fyr || !fyr->diag || !fydrc || !fmt) - return; - - fy_diag_vreport(fyr->diag, fydrc, fmt, ap); -} - -void fy_reader_diag_report(struct fy_reader *fyr, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, ...) -{ - va_list ap; - - va_start(ap, fmt); - fy_reader_diag_vreport(fyr, fydrc, fmt, ap); - va_end(ap); -} - void fy_diag_token_vreport(struct fy_diag *diag, struct fy_token *fyt, enum fy_error_type type, const char *fmt, va_list ap) { @@ -1422,58 +1101,73 @@ void fy_node_override_report(struct fy_node *fyn, enum fy_error_type type, va_end(ap); } -void fy_parser_vlog(struct fy_parser *fyp, enum fy_error_type type, - const char *fmt, va_list ap) +void fy_diag_event_vreport(struct fy_diag *diag, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, va_list ap) { - struct fy_diag_ctx ctx = { - .level = type, - .module = FYEM_UNKNOWN, - .source_func = "", - .source_file = "", - .source_line = 0, - .file = NULL, - .line = 0, - .column = 0, - }; + struct fy_diag_report_ctx drc; + struct fy_token *fyt; + bool save_on_error; - if (!fyp || !fyp->diag) + if (!fye || !diag) return; - (void)fy_vdiag(fyp->diag, &ctx, fmt, ap); + save_on_error = diag->on_error; + diag->on_error = false; + + memset(&drc, 0, sizeof(drc)); + drc.type = type; + drc.module = FYEM_UNKNOWN; + drc.fyt = NULL; + switch (fyep) { + case FYEP_VALUE: + fyt = fy_event_get_token(fye); + break; + case FYEP_TAG: + fyt = fy_event_get_tag_token(fye); + break; + case FYEP_ANCHOR: + fyt = fy_event_get_anchor_token(fye); + break; + default: + fyt = NULL; + break; + } + drc.fyt = fy_token_ref(fyt); + fy_diag_vreport(diag, &drc, fmt, ap); + + diag->on_error = save_on_error; } -void fy_parser_log(struct fy_parser *fyp, enum fy_error_type type, - const char *fmt, ...) +void fy_diag_event_report(struct fy_diag *diag, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, ...) { va_list ap; va_start(ap, fmt); - fy_parser_vlog(fyp, type, fmt, ap); + fy_diag_event_vreport(diag, fye, fyep, type, fmt, ap); va_end(ap); } -void fy_parser_vreport(struct fy_parser *fyp, enum fy_error_type type, - struct fy_token *fyt, const char *fmt, va_list ap) +void fy_event_vreport(struct fy_parser *fyp, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, va_list ap) { - struct fy_diag_report_ctx fydrc_local, *fydrc = &fydrc_local; - - if (!fyp || !fyp->diag || !fyt) - return; - - memset(fydrc, 0, sizeof(*fydrc)); - fydrc->type = type; - fydrc->module = FYEM_UNKNOWN; - fydrc->fyt = fy_token_ref(fyt); - - fy_parser_diag_vreport(fyp, fydrc, fmt, ap); + if (fye) + fy_diag_event_vreport(fyp->diag, fye, fyep, type, fmt, ap); + else + fy_parser_vlog(fyp, type, fmt, ap); } -void fy_parser_report(struct fy_parser *fyp, enum fy_error_type type, - struct fy_token *fyt, const char *fmt, ...) +void +fy_event_report(struct fy_parser *fyp, struct fy_event *fye, + enum fy_event_part fyep, enum fy_error_type type, + const char *fmt, ...) { va_list ap; va_start(ap, fmt); - fy_parser_vreport(fyp, type, fyt, fmt, ap); + fy_event_vreport(fyp, fye, fyep, type, fmt, ap); va_end(ap); } diff --git a/src/lib/fy-diag.h b/src/lib/fy-diag.h index 62b6547..a37d356 100644 --- a/src/lib/fy-diag.h +++ b/src/lib/fy-diag.h @@ -19,15 +19,11 @@ #include +#include "fy-utils.h" + #include "fy-list.h" #include "fy-token.h" -#if !defined(NDEBUG) && defined(HAVE_DEVMODE) && HAVE_DEVMODE -#define FY_DEVMODE -#else -#undef FY_DEVMODE -#endif - /* error flags (above 0x100 is library specific) */ #define FYEF_SOURCE 0x0001 #define FYEF_POSITION 0x0002 @@ -100,634 +96,38 @@ void fy_diag_report(struct fy_diag *diag, const char *fmt, ...) __attribute__((format(printf, 3, 4))); -#ifdef FY_DEVMODE -#define __FY_DEBUG_UNUSED__ /* nothing */ -#else -#if defined(__GNUC__) && __GNUC__ >= 4 -#define __FY_DEBUG_UNUSED__ __attribute__((__unused__)) -#else -#define __FY_DEBUG_UNUSED__ /* nothing */ -#endif -#endif +static inline bool +fy_diag_log_level_is_enabled(struct fy_diag *diag, enum fy_error_type level, enum fy_error_module module) +{ + if (!diag) + return false; -/* parser diagnostics */ -struct fy_parser; + /* check level */ + if ((unsigned int)level < FYET_MAX) { -void fy_diag_cfg_from_parser_flags(struct fy_diag_cfg *cfg, enum fy_parse_cfg_flags pflags); + /* turn errors into debugs while not reset */ + if (level >= FYET_ERROR && diag->on_error) + level = FYET_DEBUG; -int fy_parser_vdiag(struct fy_parser *fyp, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, va_list ap); + if (level < diag->cfg.level) + return false; + } -int fy_parser_diag(struct fy_parser *fyp, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, ...) - __attribute__((format(printf, 6, 7))); + /* check module enable mask */ + if ((unsigned int)module < FYEM_MAX) { + if (!(diag->cfg.module_mask & FY_BIT(module))) + return false; + } + + /* ok, clear to generate */ + return true; +} void fy_diag_error_atom_display(struct fy_diag *diag, enum fy_error_type type, struct fy_atom *atom); void fy_diag_error_token_display(struct fy_diag *diag, enum fy_error_type type, struct fy_token *fyt); -void fy_parser_diag_vreport(struct fy_parser *fyp, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, va_list ap); -void fy_parser_diag_report(struct fy_parser *fyp, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, ...) - __attribute__((format(printf, 3, 4))); - -#ifdef FY_DEVMODE - -#define fyp_debug(_fyp, _module, _fmt, ...) \ - fy_parser_diag((_fyp), FYET_DEBUG | FYDF_MODULE(_module), \ - __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#else - -#define fyp_debug(_fyp, _module, _fmt, ...) \ - do { } while(0) - -#endif - -#define fyp_info(_fyp, _fmt, ...) \ - fy_parser_diag((_fyp), FYET_INFO, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyp_notice(_fyp, _fmt, ...) \ - fy_parser_diag((_fyp), FYET_NOTICE, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyp_warning(_fyp, _fmt, ...) \ - fy_parser_diag((_fyp), FYET_WARNING, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyp_error(_fyp, _fmt, ...) \ - fy_parser_diag((_fyp), FYET_ERROR, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) - -#define fyp_scan_debug(_fyp, _fmt, ...) \ - fyp_debug((_fyp), FYEM_SCAN, (_fmt) , ## __VA_ARGS__) -#define fyp_parse_debug(_fyp, _fmt, ...) \ - fyp_debug((_fyp), FYEM_PARSE, (_fmt) , ## __VA_ARGS__) -#define fyp_doc_debug(_fyp, _fmt, ...) \ - fyp_debug((_fyp), FYEM_DOC, (_fmt) , ## __VA_ARGS__) - -#define fyp_error_check(_fyp, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - fyp_error((_fyp), _fmt, ## __VA_ARGS__); \ - goto _label ; \ - } \ - } while(0) - -#define _FYP_TOKEN_DIAG(_fyp, _fyt, _type, _module, _fmt, ...) \ - do { \ - struct fy_diag_report_ctx _drc; \ - memset(&_drc, 0, sizeof(_drc)); \ - _drc.type = (_type); \ - _drc.module = (_module); \ - _drc.fyt = (_fyt); \ - fy_parser_diag_report((_fyp), &_drc, (_fmt) , ## __VA_ARGS__); \ - } while(0) - -#define FYP_TOKEN_DIAG(_fyp, _fyt, _type, _module, _fmt, ...) \ - _FYP_TOKEN_DIAG(_fyp, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) - -#define FYP_PARSE_DIAG(_fyp, _adv, _cnt, _type, _module, _fmt, ...) \ - _FYP_TOKEN_DIAG(_fyp, \ - fy_token_create(FYTT_INPUT_MARKER, \ - fy_fill_atom_at((_fyp), (_adv), (_cnt), \ - alloca(sizeof(struct fy_atom)))), \ - _type, _module, _fmt, ## __VA_ARGS__) - -#define FYP_MARK_DIAG(_fyp, _sm, _em, _type, _module, _fmt, ...) \ - _FYP_TOKEN_DIAG(_fyp, \ - fy_token_create(FYTT_INPUT_MARKER, \ - fy_fill_atom_mark(((_fyp)), (_sm), (_em), \ - alloca(sizeof(struct fy_atom)))), \ - _type, _module, _fmt, ## __VA_ARGS__) - -#define FYP_NODE_DIAG(_fyp, _fyn, _type, _module, _fmt, ...) \ - _FYP_TOKEN_DIAG(_fyp, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) - -#define FYP_TOKEN_ERROR(_fyp, _fyt, _module, _fmt, ...) \ - FYP_TOKEN_DIAG(_fyp, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYP_PARSE_ERROR(_fyp, _adv, _cnt, _module, _fmt, ...) \ - FYP_PARSE_DIAG(_fyp, _adv, _cnt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYP_MARK_ERROR(_fyp, _sm, _em, _module, _fmt, ...) \ - FYP_MARK_DIAG(_fyp, _sm, _em, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYP_NODE_ERROR(_fyp, _fyn, _module, _fmt, ...) \ - FYP_NODE_DIAG(_fyp, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYP_TOKEN_ERROR_CHECK(_fyp, _fyt, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYP_TOKEN_ERROR(_fyp, _fyt, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYP_PARSE_ERROR_CHECK(_fyp, _adv, _cnt, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYP_PARSE_ERROR(_fyp, _adv, _cnt, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYP_MARK_ERROR_CHECK(_fyp, _sm, _em, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYP_MARK_ERROR(_fyp, _sm, _em, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYP_NODE_ERROR_CHECK(_fyp, _fyn, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYP_NODE_ERROR(_fyp, _fyn, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYP_TOKEN_WARNING(_fyp, _fyt, _module, _fmt, ...) \ - FYP_TOKEN_DIAG(_fyp, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -#define FYP_PARSE_WARNING(_fyp, _adv, _cnt, _module, _fmt, ...) \ - FYP_PARSE_DIAG(_fyp, _adv, _cnt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -#define FYP_MARK_WARNING(_fyp, _sm, _em, _module, _fmt, ...) \ - FYP_MARK_DIAG(_fyp, _sm, _em, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -#define FYP_NODE_WARNING(_fyp, _fyn, _type, _module, _fmt, ...) \ - FYP_NODE_DIAG(_fyp, _fyn, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -/* reader diagnostics */ - -struct fy_reader; - -int fy_reader_vdiag(struct fy_reader *fyr, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, va_list ap); - -int fy_reader_diag(struct fy_reader *fyr, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, ...) - __attribute__((format(printf, 6, 7))); - -void fy_reader_diag_vreport(struct fy_reader *fyr, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, va_list ap); -void fy_reader_diag_report(struct fy_reader *fyr, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, ...) - __attribute__((format(printf, 3, 4))); - -#ifdef FY_DEVMODE - -#define fyr_debug(_fyr, _fmt, ...) \ - fy_reader_diag((_fyr), FYET_DEBUG, \ - __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#else - -#define fyr_debug(_fyr, _fmt, ...) \ - do { } while(0) - -#endif - -#define fyr_info(_fyr, _fmt, ...) \ - fy_reader_diag((_fyr), FYET_INFO, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyr_notice(_fyr, _fmt, ...) \ - fy_reader_diag((_fyr), FYET_NOTICE, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyr_warning(_fyr, _fmt, ...) \ - fy_reader_diag((_fyr), FYET_WARNING, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyr_error(_fyr, _fmt, ...) \ - fy_reader_diag((_fyr), FYET_ERROR, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) - -#define fyr_error_check(_fyr, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - fyr_error((_fyr), _fmt, ## __VA_ARGS__); \ - goto _label ; \ - } \ - } while(0) - -#define _FYR_TOKEN_DIAG(_fyr, _fyt, _type, _module, _fmt, ...) \ - do { \ - struct fy_diag_report_ctx _drc; \ - memset(&_drc, 0, sizeof(_drc)); \ - _drc.type = (_type); \ - _drc.module = (_module); \ - _drc.fyt = (_fyt); \ - fy_reader_diag_report((_fyr), &_drc, (_fmt) , ## __VA_ARGS__); \ - } while(0) - -#define FYR_TOKEN_DIAG(_fyr, _fyt, _type, _module, _fmt, ...) \ - _FYR_TOKEN_DIAG(_fyr, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) - -#define FYR_PARSE_DIAG(_fyr, _adv, _cnt, _type, _module, _fmt, ...) \ - _FYR_TOKEN_DIAG(_fyr, \ - fy_token_create(FYTT_INPUT_MARKER, \ - fy_reader_fill_atom_at((_fyr), (_adv), (_cnt), \ - alloca(sizeof(struct fy_atom)))), \ - _type, _module, _fmt, ## __VA_ARGS__) - -#define FYR_MARK_DIAG(_fyr, _sm, _em, _type, _module, _fmt, ...) \ - _FYR_TOKEN_DIAG(_fyr, \ - fy_token_create(FYTT_INPUT_MARKER, \ - fy_reader_fill_atom_mark(((_fyr)), (_sm), (_em), \ - alloca(sizeof(struct fy_atom)))), \ - _type, _module, _fmt, ## __VA_ARGS__) - -#define FYR_NODE_DIAG(_fyr, _fyn, _type, _module, _fmt, ...) \ - _FYR_TOKEN_DIAG(_fyr, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) - -#define FYR_TOKEN_ERROR(_fyr, _fyt, _module, _fmt, ...) \ - FYR_TOKEN_DIAG(_fyr, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYR_PARSE_ERROR(_fyr, _adv, _cnt, _module, _fmt, ...) \ - FYR_PARSE_DIAG(_fyr, _adv, _cnt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYR_MARK_ERROR(_fyr, _sm, _em, _module, _fmt, ...) \ - FYR_MARK_DIAG(_fyr, _sm, _em, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYR_NODE_ERROR(_fyr, _fyn, _module, _fmt, ...) \ - FYR_NODE_DIAG(_fyr, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYR_TOKEN_ERROR_CHECK(_fyr, _fyt, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYR_TOKEN_ERROR(_fyr, _fyt, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYR_PARSE_ERROR_CHECK(_fyr, _adv, _cnt, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYR_PARSE_ERROR(_fyr, _adv, _cnt, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYR_MARK_ERROR_CHECK(_fyr, _sm, _em, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYR_MARK_ERROR(_fyr, _sm, _em, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYR_NODE_ERROR_CHECK(_fyr, _fyn, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYR_NODE_ERROR(_fyr, _fyn, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYR_TOKEN_WARNING(_fyr, _fyt, _module, _fmt, ...) \ - FYR_TOKEN_DIAG(_fyr, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -#define FYR_PARSE_WARNING(_fyr, _adv, _cnt, _module, _fmt, ...) \ - FYR_PARSE_DIAG(_fyr, _adv, _cnt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -#define FYR_MARK_WARNING(_fyr, _sm, _em, _module, _fmt, ...) \ - FYR_MARK_DIAG(_fyr, _sm, _em, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -#define FYR_NODE_WARNING(_fyr, _fyn, _type, _module, _fmt, ...) \ - FYR_NODE_DIAG(_fyr, _fyn, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -/* doc */ -struct fy_document; - -int fy_document_vdiag(struct fy_document *fyd, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, va_list ap); - -int fy_document_diag(struct fy_document *fyd, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, ...) - __attribute__((format(printf, 6, 7))); - -void fy_document_diag_vreport(struct fy_document *fyd, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, va_list ap); -void fy_document_diag_report(struct fy_document *fyd, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, ...) - __attribute__((format(printf, 3, 4))); - -#ifdef FY_DEVMODE - -#define fyd_debug(_fyd, _module, _fmt, ...) \ - fy_document_diag((_fyd), FYET_DEBUG | FYDF_MODULE(_module), \ - __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) - -#else - -#define fyd_debug(_fyd, _module, _fmt, ...) \ - do { } while(0) - -#endif - -#define fyd_info(_fyd, _fmt, ...) \ - fy_document_diag((_fyd), FYET_INFO, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyd_notice(_fyd, _fmt, ...) \ - fy_document_diag((_fyd), FYET_NOTICE, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyd_warning(_fyd, _fmt, ...) \ - fy_document_diag((_fyd), FYET_WARNING, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyd_error(_fyd, _fmt, ...) \ - fy_document_diag((_fyd), FYET_ERROR, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) - -#define fyd_doc_debug(_fyd, _fmt, ...) \ - fyd_debug((_fyd), FYEM_DOC, (_fmt) , ## __VA_ARGS__) - -#define fyd_error_check(_fyd, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - fyd_error((_fyd), _fmt, ## __VA_ARGS__); \ - goto _label ; \ - } \ - } while(0) - -#define _FYD_TOKEN_DIAG(_fyd, _fyt, _type, _module, _fmt, ...) \ - do { \ - struct fy_diag_report_ctx _drc; \ - memset(&_drc, 0, sizeof(_drc)); \ - _drc.type = (_type); \ - _drc.module = (_module); \ - _drc.fyt = (_fyt); \ - fy_document_diag_report((_fyd), &_drc, (_fmt) , ## __VA_ARGS__); \ - } while(0) - -#define FYD_TOKEN_DIAG(_fyd, _fyt, _type, _module, _fmt, ...) \ - _FYD_TOKEN_DIAG(_fyd, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) - -#define FYD_NODE_DIAG(_fyd, _fyn, _type, _module, _fmt, ...) \ - _FYD_TOKEN_DIAG(_fyd, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) - -#define FYD_TOKEN_ERROR(_fyd, _fyt, _module, _fmt, ...) \ - FYD_TOKEN_DIAG(_fyd, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYD_NODE_ERROR(_fyd, _fyn, _module, _fmt, ...) \ - FYD_NODE_DIAG(_fyd, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYD_TOKEN_ERROR_CHECK(_fyd, _fyt, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYD_TOKEN_ERROR(_fyd, _fyt, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYD_NODE_ERROR_CHECK(_fyd, _fyn, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYD_NODE_ERROR(_fyd, _fyn, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYD_TOKEN_WARNING(_fyd, _fyt, _module, _fmt, ...) \ - FYD_TOKEN_DIAG(_fyd, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -#define FYD_NODE_WARNING(_fyd, _fyn, _type, _module, _fmt, ...) \ - FYD_NODE_DIAG(_fyd, _fyn, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -/* composer */ -struct fy_composer; - -int fy_composer_vdiag(struct fy_composer *fyc, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, va_list ap); - -int fy_composer_diag(struct fy_composer *fyc, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, ...) - __attribute__((format(printf, 6, 7))); - -void fy_composer_diag_vreport(struct fy_composer *fyc, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, va_list ap); -void fy_composer_diag_report(struct fy_composer *fyc, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, ...) - __attribute__((format(printf, 3, 4))); - -#ifdef FY_DEVMODE - -#define fyc_debug(_fyc, _module, _fmt, ...) \ - fy_composer_diag((_fyc), FYET_DEBUG | FYDF_MODULE(_module), \ - __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) - -#else - -#define fyc_debug(_fyc, _module, _fmt, ...) \ - do { } while(0) - -#endif - -#define fyc_info(_fyc, _fmt, ...) \ - fy_composer_diag((_fyc), FYET_INFO, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyc_notice(_fyc, _fmt, ...) \ - fy_composer_diag((_fyc), FYET_NOTICE, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyc_warning(_fyc, _fmt, ...) \ - fy_composer_diag((_fyc), FYET_WARNING, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fyc_error(_fyc, _fmt, ...) \ - fy_composer_diag((_fyc), FYET_ERROR, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) - -#define fyc_error_check(_fyc, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - fyc_error((_fyc), _fmt, ## __VA_ARGS__); \ - goto _label ; \ - } \ - } while(0) - -#define _FYC_TOKEN_DIAG(_fyc, _fyt, _type, _module, _fmt, ...) \ - do { \ - struct fy_diag_report_ctx _drc; \ - memset(&_drc, 0, sizeof(_drc)); \ - _drc.type = (_type); \ - _drc.module = (_module); \ - _drc.fyt = (_fyt); \ - fy_composer_diag_report((_fyc), &_drc, (_fmt) , ## __VA_ARGS__); \ - } while(0) - -#define FYC_TOKEN_DIAG(_fyc, _fyt, _type, _module, _fmt, ...) \ - _FYC_TOKEN_DIAG(_fyc, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) - -#define FYC_NODE_DIAG(_fyc, _fyn, _type, _module, _fmt, ...) \ - _FYC_TOKEN_DIAG(_fyc, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) - -#define FYC_TOKEN_ERROR(_fyc, _fyt, _module, _fmt, ...) \ - FYC_TOKEN_DIAG(_fyc, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYC_TOKEN_ERROR_CHECK(_fyc, _fyt, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYC_TOKEN_ERROR(_fyc, _fyt, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYC_TOKEN_WARNING(_fyc, _fyt, _module, _fmt, ...) \ - FYC_TOKEN_DIAG(_fyc, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -/* document builder */ -struct fy_document_builder; - -int fy_document_builder_vdiag(struct fy_document_builder *fydb, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, va_list ap); - -int fy_document_builder_diag(struct fy_document_builder *fydb, unsigned int flags, - const char *file, int line, const char *func, - const char *fmt, ...) - __attribute__((format(printf, 6, 7))); - -void fy_document_builder_diag_vreport(struct fy_document_builder *fydb, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, va_list ap); -void fy_document_builder_diag_report(struct fy_document_builder *fydb, - const struct fy_diag_report_ctx *fydrc, - const char *fmt, ...) - __attribute__((format(printf, 3, 4))); - -#ifdef FY_DEVMODE - -#define fydb_debug(_fydb, _module, _fmt, ...) \ - fy_document_builder_diag((_fydb), FYET_DEBUG | FYDF_MODULE(_module), \ - __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) - -#else - -#define fydb_debug(_fydb, _module, _fmt, ...) \ - do { } while(0) - -#endif - -#define fydb_info(_fydb, _fmt, ...) \ - fy_document_builder_diag((_fydb), FYET_INFO, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fydb_notice(_fydb, _fmt, ...) \ - fy_document_builder_diag((_fydb), FYET_NOTICE, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fydb_warning(_fydb, _fmt, ...) \ - fy_document_builder_diag((_fydb), FYET_WARNING, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) -#define fydb_error(_fydb, _fmt, ...) \ - fy_document_builder_diag((_fydb), FYET_ERROR, __FILE__, __LINE__, __func__, \ - (_fmt) , ## __VA_ARGS__) - -#define fydb_error_check(_fydb, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - fydb_error((_fydb), _fmt, ## __VA_ARGS__); \ - goto _label ; \ - } \ - } while(0) - -#define _FYDB_TOKEN_DIAG(_fydb, _fyt, _type, _module, _fmt, ...) \ - do { \ - struct fy_diag_report_ctx _drc; \ - memset(&_drc, 0, sizeof(_drc)); \ - _drc.type = (_type); \ - _drc.module = (_module); \ - _drc.fyt = (_fyt); \ - fy_document_builder_diag_report((_fydb), &_drc, (_fmt) , ## __VA_ARGS__); \ - } while(0) - -#define FYDB_TOKEN_DIAG(_fydb, _fyt, _type, _module, _fmt, ...) \ - _FYDB_TOKEN_DIAG(_fydb, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) - -#define FYDB_NODE_DIAG(_fydb, _fyn, _type, _module, _fmt, ...) \ - _FYDB_TOKEN_DIAG(_fydb, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) - -#define FYDB_TOKEN_ERROR(_fydb, _fyt, _module, _fmt, ...) \ - FYDB_TOKEN_DIAG(_fydb, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYDB_NODE_ERROR(_fydb, _fyn, _module, _fmt, ...) \ - FYDB_NODE_DIAG(_fydb, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) - -#define FYDB_TOKEN_ERROR_CHECK(_fydb, _fyt, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYDB_TOKEN_ERROR(_fydb, _fyt, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYDB_NODE_ERROR_CHECK(_fydb, _fyn, _module, _cond, _label, _fmt, ...) \ - do { \ - if (!(_cond)) { \ - FYDB_NODE_ERROR(_fydb, _fyn, _module, _fmt, ## __VA_ARGS__); \ - goto _label; \ - } \ - } while(0) - -#define FYDB_TOKEN_WARNING(_fydb, _fyt, _module, _fmt, ...) \ - FYDB_TOKEN_DIAG(_fydb, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) - -/* alloca formatted print methods */ -#define alloca_vsprintf(_fmt, _ap) \ - ({ \ - const char *__fmt = (_fmt); \ - va_list _ap_orig; \ - int _size; \ - int _sizew __FY_DEBUG_UNUSED__; \ - char *_buf = NULL, *_s; \ - \ - va_copy(_ap_orig, (_ap)); \ - _size = vsnprintf(NULL, 0, __fmt, _ap_orig); \ - va_end(_ap_orig); \ - if (_size != -1) { \ - _buf = alloca(_size + 1); \ - _sizew = vsnprintf(_buf, _size + 1, __fmt, _ap); \ - assert(_size == _sizew); \ - _s = _buf + strlen(_buf); \ - while (_s > _buf && _s[-1] == '\n') \ - *--_s = '\0'; \ - } \ - _buf; \ - }) - -#define alloca_sprintf(_fmt, ...) \ - ({ \ - const char *__fmt = (_fmt); \ - int _size; \ - int _sizew __FY_DEBUG_UNUSED__; \ - char *_buf = NULL, *_s; \ - \ - _size = snprintf(NULL, 0, __fmt, ## __VA_ARGS__); \ - if (_size != -1) { \ - _buf = alloca(_size + 1); \ - _sizew = snprintf(_buf, _size + 1, __fmt, __VA_ARGS__); \ - assert(_size == _sizew); \ - _s = _buf + strlen(_buf); \ - while (_s > _buf && _s[-1] == '\n') \ - *--_s = '\0'; \ - } \ - _buf; \ - }) +void fy_diag_cfg_from_parser_flags(struct fy_diag_cfg *cfg, enum fy_parse_cfg_flags pflags); #endif diff --git a/src/lib/fy-doc-diag.c b/src/lib/fy-doc-diag.c new file mode 100644 index 0000000..04247e3 --- /dev/null +++ b/src/lib/fy-doc-diag.c @@ -0,0 +1,91 @@ +/* + * fy-doc-diag.c - document diagnostics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-diag.h" +#include "fy-doc.h" + +int fy_document_vdiag(struct fy_document *fyd, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap) +{ + struct fy_diag_ctx fydc; + int rc; + + if (!fyd || !fmt || !fyd->diag) + return -1; + + /* perform the enable tests early to avoid the overhead */ + if (((flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT) < fyd->diag->cfg.level) + return 0; + + /* fill in fy_diag_ctx */ + memset(&fydc, 0, sizeof(fydc)); + + fydc.level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; + fydc.module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; + fydc.source_file = file; + fydc.source_line = line; + fydc.source_func = func; + fydc.line = -1; + fydc.column = -1; + + rc = fy_vdiag(fyd->diag, &fydc, fmt, ap); + + return rc; +} + +int fy_document_diag(struct fy_document *fyd, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_document_vdiag(fyd, flags, file, line, func, fmt, ap); + va_end(ap); + + return rc; +} + +void fy_document_diag_vreport(struct fy_document *fyd, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap) +{ + if (!fyd || !fyd->diag || !fydrc || !fmt) + return; + + fy_diag_vreport(fyd->diag, fydrc, fmt, ap); +} + +void fy_document_diag_report(struct fy_document *fyd, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_document_diag_vreport(fyd, fydrc, fmt, ap); + va_end(ap); +} diff --git a/src/lib/fy-doc.c b/src/lib/fy-doc.c index 5b3a942..826c562 100644 --- a/src/lib/fy-doc.c +++ b/src/lib/fy-doc.c @@ -188,14 +188,13 @@ static int fy_document_set_anchor_internal(struct fy_document *fyd, struct fy_no fyd_error_check(fyd, data_copy, err_out, "malloc() failed"); memcpy(data_copy, text, len); - fyi = fy_input_from_malloc_data(data_copy, len, &handle, true); } else if (malloced) data_copy = (char *)text; else data_copy = NULL; if (data_copy) - fyi = fy_input_from_malloc_data((void *)text, len, &handle, true); + fyi = fy_input_from_malloc_data(data_copy, len, &handle, true); else fyi = fy_input_from_data(text, len, &handle, true); fyd_error_check(fyd, fyi, err_out, @@ -278,7 +277,7 @@ int fy_node_set_vanchorf(struct fy_node *fyn, const char *fmt, va_list ap) if (!fyn || !fmt) return -1; - return fy_document_set_anchor_internal(fyn->fyd, fyn, alloca_vsprintf(fmt, ap), FY_NT, FYDSAF_COPY); + return fy_document_set_anchor_internal(fyn->fyd, fyn, fy_vsprintfa(fmt, ap), FY_NT, FYDSAF_COPY); } int fy_node_set_anchorf(struct fy_node *fyn, const char *fmt, ...) @@ -954,6 +953,7 @@ struct fy_token *fy_node_non_synthesized_token(struct fy_node *fyn) handle.json_mode = false; /* always false */ handle.lb_mode = fylb_cr_nl; /* always \r\n */ handle.fws_mode = fyfws_space_tab; /* always space + tab */ + handle.directive0_mode = false; handle.chomp = FYAC_STRIP; handle.increment = 0; @@ -1218,7 +1218,7 @@ bool fy_node_compare_user(struct fy_node *fyn1, struct fy_node *fyn2, fyni2 = fy_node_list_head(&fyn2->sequence); while (fyni1 && fyni2) { - ret = fy_node_compare(fyni1, fyni2); + ret = fy_node_compare_user(fyni1, fyni2, sort_fn, sort_fn_arg, cmp_fn, cmp_fn_arg); if (!ret) break; @@ -1252,11 +1252,11 @@ bool fy_node_compare_user(struct fy_node *fyn1, struct fy_node *fyn2, fynp1 = fynpp1[i]; fynp2 = fynpp2[i]; - ret = fy_node_compare(fynp1->key, fynp2->key); + ret = fy_node_compare_user(fynp1->key, fynp2->key, sort_fn, sort_fn_arg, cmp_fn, cmp_fn_arg); if (!ret) break; - ret = fy_node_compare(fynp1->value, fynp2->value); + ret = fy_node_compare_user(fynp1->value, fynp2->value, sort_fn, sort_fn_arg, cmp_fn, cmp_fn_arg); if (!ret) break; } @@ -1366,6 +1366,9 @@ int fy_node_mapping_get_pair_index(struct fy_node *fyn, const struct fy_node_pai struct fy_node_pair *fynpi; int i; + if (!fy_node_is_mapping(fyn)) + return -1; + for (i = 0, fynpi = fy_node_pair_list_head(&fyn->mapping); fynpi; fynpi = fy_node_pair_next(&fyn->mapping, fynpi), i++) { @@ -1452,7 +1455,7 @@ fy_parse_document_load_scalar(struct fy_parser *fyp, struct fy_document *fyd, fyn->scalar = fye->alias.anchor; fye->alias.anchor = NULL; } else - assert(0); + FY_IMPOSSIBLE_ABORT(); *fynp = fyn; fyn = NULL; @@ -1673,20 +1676,20 @@ fy_parse_document_load_mapping(struct fy_parser *fyp, struct fy_document *fyd, fynp_item->key = fyn_key; fynp_item->value = fyn_value; - fy_node_pair_list_add_tail(&fyn->mapping, fynp_item); + fyn_key = NULL; + fyn_value = NULL; if (fyn->xl) { rc = fy_accel_insert(fyn->xl, fynp_item->key, fynp_item); fyp_error_check(fyp, !rc, err_out_rc, "fy_accel_insert() failed"); } + fy_node_pair_list_add_tail(&fyn->mapping, fynp_item); if (fynp_item->key) fynp_item->key->attached = true; if (fynp_item->value) fynp_item->value->attached = true; fynp_item = NULL; - fyn_key = NULL; - fyn_value = NULL; } if (!fyep) @@ -2225,9 +2228,6 @@ int fy_node_insert(struct fy_node *fyn_to, struct fy_node *fyn_from) fyd_error_check(fyd, fyn_parent->type != FYNT_SCALAR, err_out, "Illegal scalar parent node type"); - fyd_error_check(fyd, fyn_from, err_out, - "Illegal NULL source node"); - if (fyn_parent->type == FYNT_MAPPING) { /* find mapping pair that contains the `to` node */ for (fynp = fy_node_pair_list_head(&fyn_parent->mapping); fynp; @@ -2450,6 +2450,11 @@ err_out_rc: return rc; } +int fy_node_delete(struct fy_node *fyn) +{ + return fy_node_insert(fyn, NULL); +} + int fy_document_insert_at(struct fy_document *fyd, const char *path, size_t pathlen, struct fy_node *fyn) @@ -2934,9 +2939,11 @@ void fy_document_purge_anchors(struct fy_document *fyd) if (fy_document_is_accelerated(fyd)) { fy_accel_cleanup(fyd->axl); free(fyd->axl); + fyd->axl = NULL; fy_accel_cleanup(fyd->naxl); free(fyd->naxl); + fyd->naxl = NULL; } } @@ -3537,6 +3544,9 @@ int fy_node_pair_set_value(struct fy_node_pair *fynp, struct fy_node *fyn) struct fy_node *fy_document_root(struct fy_document *fyd) { + if (!fyd) + return NULL; + return fyd->root; } @@ -3555,6 +3565,23 @@ const char *fy_node_get_tag(struct fy_node *fyn, size_t *lenp) return fy_token_get_text(fyn->tag, lenp); } +const char *fy_node_get_tag0(struct fy_node *fyn) +{ + if (!fyn || !fyn->tag) + return NULL; + + return fy_token_get_text0(fyn->tag); +} + +size_t fy_node_get_tag_length(struct fy_node *fyn) +{ + + if (!fyn || !fyn->tag) + return 0; + + return fy_token_get_text_length(fyn->tag); +} + const char *fy_node_get_scalar(struct fy_node *fyn, size_t *lenp) { size_t tmplen; @@ -4993,8 +5020,7 @@ char *fy_node_get_short_path(struct fy_node *fyn) struct fy_anchor *fya; const char *text; size_t len; - const char *str; - char *path; + char *str, *path; if (!fyn) return NULL; @@ -5011,10 +5037,11 @@ char *fy_node_get_short_path(struct fy_node *fyn) return NULL; if (fyn_anchor == fyn) - str = alloca_sprintf("*%.*s", (int)len, text); + str = fy_sprintfa("*%.*s", (int)len, text); else - str = alloca_sprintf("*%.*s/%s", (int)len, text, + str = fy_sprintfa("*%.*s/%s", (int)len, text, fy_node_get_path_relative_to_alloca(fyn_anchor, fyn)); + fy_strip_trailing_nl(str); path = strdup(str); return path; @@ -5320,7 +5347,7 @@ struct fy_node *fy_node_create_vscalarf(struct fy_document *fyd, const char *fmt if (!fyd || !fmt) return NULL; - return fy_node_create_scalar_internal(fyd, alloca_vsprintf(fmt, ap), FY_NT, FYNCSIF_COPY); + return fy_node_create_scalar_internal(fyd, fy_vsprintfa(fmt, ap), FY_NT, FYNCSIF_COPY); } struct fy_node *fy_node_create_scalarf(struct fy_document *fyd, const char *fmt, ...) @@ -6985,9 +7012,13 @@ fy_node_sequence_add_item(struct fy_node *fyn_parent, struct fy_node *fyn) return 0; } -void fy_document_iterator_setup(struct fy_document_iterator *fydi) +int fy_document_iterator_setup(struct fy_document_iterator *fydi, const struct fy_document_iterator_cfg *cfg) { memset(fydi, 0, sizeof(*fydi)); + + if (cfg) + fydi->cfg = *cfg; + fydi->state = FYDIS_WAITING_STREAM_START; fydi->fyd = NULL; fydi->iterate_root = NULL; @@ -7011,6 +7042,31 @@ void fy_document_iterator_setup(struct fy_document_iterator *fydi) fydi->stack_top = (unsigned int)-1; fydi->stack_alloc = sizeof(fydi->in_place) / sizeof(fydi->in_place[0]); fydi->stack = fydi->in_place; + + /* set generator state accordingly */ + fydi->generator_state = 0; + + /* without configuration, is the default */ + if (!cfg) + return 0; + + /* set the generator flags accordingly */ + switch (fydi->cfg.flags & FYDICF_WANT_MASK) { + case FYDICF_WANT_STREAM_DOCUMENT_BODY_EVENTS: + fydi->generator_state |= FYDIGF_WANTS_STREAM | FYDIGF_WANTS_DOC | FYDIGF_ENDS_AFTER_DOC; + break; + case FYDICF_WANT_DOCUMENT_BODY_EVENTS: + fydi->generator_state |= FYDIGF_WANTS_DOC | FYDIGF_ENDS_AFTER_DOC; + fydi->state = FYDIS_WAITING_DOCUMENT_START; + break; + case FYDICF_WANT_BODY_EVENTS: + fydi->generator_state |= FYDIGF_ENDS_AFTER_BODY; + fydi->state = FYDIS_WAITING_BODY_START_OR_DOCUMENT_END; + default: + break; + } + + return 0; } void fy_document_iterator_cleanup(struct fy_document_iterator *fydi) @@ -7036,15 +7092,62 @@ void fy_document_iterator_cleanup(struct fy_document_iterator *fydi) fydi->iterate_root = NULL; } -struct fy_document_iterator *fy_document_iterator_create(void) +struct fy_document_iterator * +fy_document_iterator_create_cfg(const struct fy_document_iterator_cfg *cfg) { - struct fy_document_iterator *fydi; + struct fy_document_iterator *fydi = NULL; + int rc; fydi = malloc(sizeof(*fydi)); if (!fydi) - return NULL; - fy_document_iterator_setup(fydi); + goto err_out; + + rc = fy_document_iterator_setup(fydi, cfg); + if (rc) + goto err_out; + return fydi; + +err_out: + fy_document_iterator_destroy(fydi); + return NULL; +} + +struct fy_document_iterator *fy_document_iterator_create(void) +{ + return fy_document_iterator_create_cfg(NULL); +} + +struct fy_document_iterator * +fy_document_iterator_create_on_document(struct fy_document *fyd) +{ + struct fy_document_iterator_cfg cfg; + + if (!fyd) + return NULL; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYDICF_WANT_STREAM_DOCUMENT_BODY_EVENTS; + cfg.fyd = fyd; + cfg.iterate_root = fyd->root; + + return fy_document_iterator_create_cfg(&cfg); +} + +struct fy_document_iterator * +fy_document_iterator_create_on_node(struct fy_node *fyn) +{ + struct fy_document_iterator_cfg cfg; + + if (!fyn) + return NULL; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYDICF_WANT_STREAM_DOCUMENT_BODY_EVENTS; + cfg.fyd = fy_node_document(fyn); + cfg.iterate_root = fyn; + + return fy_document_iterator_create_cfg(&cfg); } void fy_document_iterator_destroy(struct fy_document_iterator *fydi) @@ -7167,7 +7270,8 @@ err_out: } struct fy_event * -fy_document_iterator_document_start(struct fy_document_iterator *fydi, struct fy_document *fyd) +fy_document_iterator_document_start_internal(struct fy_document_iterator *fydi, struct fy_document *fyd, + struct fy_node *iterate_root) { struct fy_event *fye = NULL; struct fy_eventp *fyep; @@ -7190,8 +7294,8 @@ fy_document_iterator_document_start(struct fy_document_iterator *fydi, struct fy fydi->fyd = fyd; - /* the iteration root is the document root */ - fydi->iterate_root = fyd->root; + /* the iteration root is the document root if not given */ + fydi->iterate_root = iterate_root ? iterate_root : fyd->root; /* suppress recycling if we must */ fydi->suppress_recycling = (fyd->parse_cfg.flags & FYPCF_DISABLE_RECYCLING) || @@ -7221,6 +7325,12 @@ err_out: return NULL; } +struct fy_event * +fy_document_iterator_document_start(struct fy_document_iterator *fydi, struct fy_document *fyd) +{ + return fy_document_iterator_document_start_internal(fydi, fyd, NULL); +} + struct fy_event * fy_document_iterator_document_end(struct fy_document_iterator *fydi) { @@ -7312,8 +7422,7 @@ fydi_push_collection(struct fy_document_iterator *fydi, struct fy_node *fyn) break; default: - assert(0); - break; + FY_IMPOSSIBLE_ABORT(); } return true; @@ -7472,3 +7581,69 @@ bool fy_document_iterator_get_error(struct fy_document_iterator *fydi) return true; } + +struct fy_event * +fy_document_iterator_generate_next(struct fy_document_iterator *fydi) +{ + struct fy_event *fye = NULL; + + if (!fydi || fydi->state == FYDIS_ERROR) + return NULL; + + if (fydi->generator_state & FYDIGF_GENERATED_NULL) + return NULL; + + /* wants stream events and not generated yet */ + if ((fydi->generator_state & (FYDIGF_WANTS_STREAM | FYDIGF_GENERATED_SS)) == FYDIGF_WANTS_STREAM) { + fye = fy_document_iterator_stream_start(fydi); + if (!fye) + return NULL; + fydi->generator_state |= FYDIGF_GENERATED_SS; + return fye; + } + + /* wants document events and not generated yet */ + if ((fydi->generator_state & (FYDIGF_WANTS_DOC | FYDIGF_GENERATED_DS)) == FYDIGF_WANTS_DOC) { + if (!fydi->cfg.fyd) { + fydi->state = FYDIS_ERROR; + return NULL; + } + fye = fy_document_iterator_document_start_internal(fydi, fydi->cfg.fyd, fydi->cfg.iterate_root); + if (!fye) + return NULL; + fydi->generator_state |= FYDIGF_GENERATED_DS; + return fye; + } + + /* generate body events... */ + if (!(fydi->generator_state & FYDIGF_GENERATED_BODY)) { + if (!fydi->iterate_root) + fydi->iterate_root = fydi->cfg.iterate_root ? fydi->cfg.iterate_root : fy_document_root(fydi->fyd); + fye = fy_document_iterator_body_next(fydi); + if (fye) + return fye; + + fydi->generator_state |= FYDIGF_GENERATED_BODY; + } + + /* wants document events and not generated yet */ + if ((fydi->generator_state & (FYDIGF_WANTS_DOC | FYDIGF_GENERATED_DE)) == FYDIGF_WANTS_DOC) { + fye = fy_document_iterator_document_end(fydi); + if (!fye) + return NULL; + fydi->generator_state |= FYDIGF_GENERATED_DE; + return fye; + } + + /* wants stream events and not generated yet */ + if ((fydi->generator_state & (FYDIGF_WANTS_STREAM | FYDIGF_GENERATED_SE)) == FYDIGF_WANTS_STREAM) { + fye = fy_document_iterator_stream_end(fydi); + if (!fye) + return NULL; + fydi->generator_state |= FYDIGF_GENERATED_SE; + return fye; + } + + fydi->generator_state |= FYDIGF_GENERATED_NULL; + return NULL; +} diff --git a/src/lib/fy-doc.h b/src/lib/fy-doc.h index d368103..c9c46c9 100644 --- a/src/lib/fy-doc.h +++ b/src/lib/fy-doc.h @@ -61,7 +61,7 @@ struct fy_node { struct fy_node *parent; struct fy_document *fyd; unsigned int marks; - enum fy_node_type type : 2; /* 2 bits are enough for 3 types */ + unsigned int type : 2; /* enum fy_node_type: 2 bits are enough for 3 types */ bool has_meta : 1; bool attached : 1; /* when it's attached somewhere */ bool synthetic : 1; /* node has been modified programmaticaly */ @@ -161,9 +161,13 @@ bool fy_check_ref_loop(struct fy_document *fyd, struct fy_node *fyn, #define FYNWF_VISIT_MARKER (FYNWF_MAX_USER_MARKER + 1) #define FYNWF_REF_MARKER (FYNWF_MAX_USER_MARKER + 2) +#define FYNWF_INSET_MARKER (FYNWF_MAX_USER_MARKER + 3) #define FYNWF_SYSTEM_MARKS (FY_BIT(FYNWF_VISIT_MARKER) | \ - FY_BIT(FYNWF_REF_MARKER)) + FY_BIT(FYNWF_REF_MARKER) | \ + FY_BIT(FYNWF_INSET_MARKER) ) + +void fy_node_clear_system_marks(struct fy_node *fyn); bool fy_node_uses_single_input_only(struct fy_node *fyn, struct fy_input *fyi); struct fy_input *fy_node_get_first_input(struct fy_node *fyn); @@ -209,6 +213,7 @@ enum fy_document_iterator_state { FYDIS_BODY, FYDIS_WAITING_DOCUMENT_END, FYDIS_WAITING_STREAM_END_OR_DOCUMENT_START, + FYDIS_ENDED, FYDIS_ERROR, }; @@ -222,6 +227,7 @@ struct fy_document_iterator_body_state { }; struct fy_document_iterator { + struct fy_document_iterator_cfg cfg; enum fy_document_iterator_state state; struct fy_document *fyd; struct fy_node *iterate_root; @@ -238,14 +244,28 @@ struct fy_document_iterator { unsigned int stack_alloc; struct fy_document_iterator_body_state *stack; struct fy_document_iterator_body_state in_place[FYPCF_GUARANTEED_MINIMUM_DEPTH_LIMIT]; + +#define FYDIGF_GENERATED_SS FY_BIT(0) +#define FYDIGF_GENERATED_DS FY_BIT(1) +#define FYDIGF_GENERATED_DE FY_BIT(2) +#define FYDIGF_GENERATED_SE FY_BIT(3) +#define FYDIGF_GENERATED_BODY FY_BIT(4) +#define FYDIGF_GENERATED_NULL FY_BIT(5) +#define FYDIGF_ENDS_AFTER_BODY FY_BIT(6) +#define FYDIGF_ENDS_AFTER_DOC FY_BIT(7) +#define FYDIGF_WANTS_STREAM FY_BIT(8) +#define FYDIGF_WANTS_DOC FY_BIT(9) + unsigned int generator_state; }; -void fy_document_iterator_setup(struct fy_document_iterator *fydi); +int fy_document_iterator_setup(struct fy_document_iterator *fydi, const struct fy_document_iterator_cfg *cfg); void fy_document_iterator_cleanup(struct fy_document_iterator *fydi); struct fy_document_iterator *fy_document_iterator_create(void); void fy_document_iterator_destroy(struct fy_document_iterator *fydi); -void fy_document_iterator_start(struct fy_document_iterator *fydi, struct fy_document *fyd); -void fy_document_iterator_end(struct fy_document_iterator *fydi); + +struct fy_event * +fy_document_iterator_document_start_internal(struct fy_document_iterator *fydi, struct fy_document *fyd, + struct fy_node *iterate_root); struct fy_document_iterator_body_result { struct fy_node *fyn; @@ -256,4 +276,116 @@ bool fy_document_iterator_body_next_internal(struct fy_document_iterator *fydi, struct fy_document_iterator_body_result *res); +/* diagnostics */ + +static inline bool +fyd_debug_log_level_is_enabled(struct fy_document *fyd, enum fy_error_module module) +{ + return fyd && fy_diag_log_level_is_enabled(fyd->diag, FYET_DEBUG, module); +} + +int fy_document_vdiag(struct fy_document *fyd, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap); + +int fy_document_diag(struct fy_document *fyd, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) + __attribute__((format(printf, 6, 7))); + +void fy_document_diag_vreport(struct fy_document *fyd, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap); +void fy_document_diag_report(struct fy_document *fyd, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +#ifdef FY_DEVMODE + +#define fyd_debug(_fyd, _module, _fmt, ...) \ + do { \ + struct fy_document *__fyd = (_fyd); \ + enum fy_error_module __module = (_module); \ + \ + if (fyd_debug_log_level_is_enabled(__fyd, __module)) \ + fy_document_diag(__fyd, FYET_DEBUG | FYDF_MODULE(_module), \ + __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__); \ + } while(0) +#else + +#define fyd_debug(_fyd, _module, _fmt, ...) \ + do { } while(0) + +#endif + +#define fyd_info(_fyd, _fmt, ...) \ + fy_document_diag((_fyd), FYET_INFO, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyd_notice(_fyd, _fmt, ...) \ + fy_document_diag((_fyd), FYET_NOTICE, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyd_warning(_fyd, _fmt, ...) \ + fy_document_diag((_fyd), FYET_WARNING, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyd_error(_fyd, _fmt, ...) \ + fy_document_diag((_fyd), FYET_ERROR, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) + +#define fyd_doc_debug(_fyd, _fmt, ...) \ + fyd_debug((_fyd), FYEM_DOC, (_fmt) , ## __VA_ARGS__) + +#define fyd_error_check(_fyd, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + fyd_error((_fyd), _fmt, ## __VA_ARGS__); \ + goto _label ; \ + } \ + } while(0) + +#define _FYD_TOKEN_DIAG(_fyd, _fyt, _type, _module, _fmt, ...) \ + do { \ + struct fy_diag_report_ctx _drc; \ + memset(&_drc, 0, sizeof(_drc)); \ + _drc.type = (_type); \ + _drc.module = (_module); \ + _drc.fyt = (_fyt); \ + fy_document_diag_report((_fyd), &_drc, (_fmt) , ## __VA_ARGS__); \ + } while(0) + +#define FYD_TOKEN_DIAG(_fyd, _fyt, _type, _module, _fmt, ...) \ + _FYD_TOKEN_DIAG(_fyd, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYD_NODE_DIAG(_fyd, _fyn, _type, _module, _fmt, ...) \ + _FYD_TOKEN_DIAG(_fyd, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYD_TOKEN_ERROR(_fyd, _fyt, _module, _fmt, ...) \ + FYD_TOKEN_DIAG(_fyd, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYD_NODE_ERROR(_fyd, _fyn, _module, _fmt, ...) \ + FYD_NODE_DIAG(_fyd, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYD_TOKEN_ERROR_CHECK(_fyd, _fyt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYD_TOKEN_ERROR(_fyd, _fyt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYD_NODE_ERROR_CHECK(_fyd, _fyn, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYD_NODE_ERROR(_fyd, _fyn, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYD_TOKEN_WARNING(_fyd, _fyt, _module, _fmt, ...) \ + FYD_TOKEN_DIAG(_fyd, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYD_NODE_WARNING(_fyd, _fyn, _type, _module, _fmt, ...) \ + FYD_NODE_DIAG(_fyd, _fyn, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + #endif diff --git a/src/lib/fy-docbuilder-diag.c b/src/lib/fy-docbuilder-diag.c new file mode 100644 index 0000000..841469f --- /dev/null +++ b/src/lib/fy-docbuilder-diag.c @@ -0,0 +1,92 @@ +/* + * fy-docbuilder-diag.c - document builder diagnostics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-diag.h" +#include "fy-docbuilder.h" + +int fy_document_builder_vdiag(struct fy_document_builder *fydb, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap) +{ + struct fy_diag_ctx fydc; + int rc; + + if (!fydb || !fmt || !fydb->cfg.diag) + return -1; + + /* perform the enable tests early to avoid the overhead */ + if (((flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT) < fydb->cfg.diag->cfg.level) + return 0; + + /* fill in fy_diag_ctx */ + memset(&fydc, 0, sizeof(fydc)); + + fydc.level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; + fydc.module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; + fydc.source_file = file; + fydc.source_line = line; + fydc.source_func = func; + fydc.line = -1; + fydc.column = -1; + + rc = fy_vdiag(fydb->cfg.diag, &fydc, fmt, ap); + + return rc; +} + +int fy_document_builder_diag(struct fy_document_builder *fydb, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_document_builder_vdiag(fydb, flags, file, line, func, fmt, ap); + va_end(ap); + + return rc; +} + +void fy_document_builder_diag_vreport(struct fy_document_builder *fydb, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap) +{ + if (!fydb || !fydb->cfg.diag || !fydrc || !fmt) + return; + + fy_diag_vreport(fydb->cfg.diag, fydrc, fmt, ap); +} + +void fy_document_builder_diag_report(struct fy_document_builder *fydb, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_document_builder_diag_vreport(fydb, fydrc, fmt, ap); + va_end(ap); +} + diff --git a/src/lib/fy-docbuilder.c b/src/lib/fy-docbuilder.c index b1d78e8..f4a8d09 100644 --- a/src/lib/fy-docbuilder.c +++ b/src/lib/fy-docbuilder.c @@ -100,6 +100,39 @@ err_out: return NULL; } +struct fy_document_builder * +fy_document_builder_create_on_parser(struct fy_parser *fyp) +{ + struct fy_document_builder *fydb = NULL; + struct fy_document_state *fyds; + struct fy_document_builder_cfg cfg; + int rc; + + if (!fyp) + return NULL; + + memset(&cfg, 0, sizeof(cfg)); + cfg.parse_cfg = fyp->cfg; + cfg.diag = fy_diag_ref(fyp->diag); + + fydb = fy_document_builder_create(&cfg); + if (!fydb) { + fy_diag_unref(cfg.diag); + return NULL; + } + + fyds = fy_parser_get_document_state(fyp); + if (fyds) { + rc = fy_document_builder_set_in_document(fydb, fyds, true); + if (rc) { + fy_document_builder_destroy(fydb); + return NULL; + } + } + + return fydb; +} + void fy_document_builder_destroy(struct fy_document_builder *fydb) { @@ -223,9 +256,8 @@ fy_document_builder_set_in_document(struct fy_document_builder *fydb, struct fy_ } int -fy_document_builder_process_event(struct fy_document_builder *fydb, struct fy_eventp *fyep) +fy_document_builder_process_event(struct fy_document_builder *fydb, struct fy_event *fye) { - struct fy_event *fye; enum fy_event_type etype; struct fy_document *fyd; struct fy_document_builder_ctx *c, *cp; @@ -235,7 +267,6 @@ fy_document_builder_process_event(struct fy_document_builder *fydb, struct fy_ev struct fy_token *fyt; int rc; - fye = fyep ? &fyep->e : NULL; etype = fye ? fye->type : FYET_NONE; fyt = fye ? fy_event_get_token(fye) : NULL; @@ -269,7 +300,7 @@ fy_document_builder_process_event(struct fy_document_builder *fydb, struct fy_ev fydb_error_check(fydb, fydb->fyd, err_out, "fy_document_create() failed"); - rc = fy_document_set_document_state(fydb->fyd, fyep->e.document_start.document_state); + rc = fy_document_set_document_state(fydb->fyd, fye->document_start.document_state); fydb_error_check(fydb, !rc, err_out, "fy_document_set_document_state() failed"); @@ -342,7 +373,10 @@ fy_document_builder_process_event(struct fy_document_builder *fydb, struct fy_ev "fy_node_alloc() MAPPING failed"); c->fyn = fyn; - fyn->style = fye->mapping_start.mapping_start->type == FYTT_FLOW_MAPPING_START ? FYNS_FLOW : FYNS_BLOCK; + if (fye->mapping_start.mapping_start) + fyn->style = fye->mapping_start.mapping_start->type == FYTT_FLOW_MAPPING_START ? FYNS_FLOW : FYNS_BLOCK; + else + fyn->style = FYNS_ANY; fyn->tag = fy_token_ref(fye->mapping_start.tag); if (fye->mapping_start.anchor) { rc = fy_document_register_anchor(fyd, fyn, fy_token_ref(fye->mapping_start.anchor)); @@ -374,7 +408,10 @@ fy_document_builder_process_event(struct fy_document_builder *fydb, struct fy_ev "fy_node_alloc() SEQUENCE failed"); c->fyn = fyn; - fyn->style = fye->sequence_start.sequence_start->type == FYTT_FLOW_SEQUENCE_START ? FYNS_FLOW : FYNS_BLOCK; + if (fye->sequence_start.sequence_start) + fyn->style = fye->sequence_start.sequence_start->type == FYTT_FLOW_SEQUENCE_START ? FYNS_FLOW : FYNS_BLOCK; + else + fyn->style = FYNS_ANY; fyn->tag = fy_token_ref(fye->sequence_start.tag); if (fye->sequence_start.anchor) { rc = fy_document_register_anchor(fyd, fyn, fy_token_ref(fye->sequence_start.anchor)); @@ -527,7 +564,7 @@ fy_document_builder_load_document(struct fy_document_builder *fydb, while (!fy_document_builder_is_document_complete(fydb) && (fyep = fy_parse_private(fyp)) != NULL) { - rc = fy_document_builder_process_event(fydb, fyep); + rc = fy_document_builder_process_event(fydb, &fyep->e); fy_parse_eventp_recycle(fyp, fyep); if (rc < 0) { fyp->stream_error = true; @@ -553,7 +590,7 @@ fy_document_builder_event_document(struct fy_document_builder *fydb, struct fy_e if (fy_document_builder_is_document_complete(fydb)) break; - rc = fy_document_builder_process_event(fydb, fyep); + rc = fy_document_builder_process_event(fydb, &fyep->e); if (rc < 0) return NULL; diff --git a/src/lib/fy-docbuilder.h b/src/lib/fy-docbuilder.h index dd2d0a0..5c8780d 100644 --- a/src/lib/fy-docbuilder.h +++ b/src/lib/fy-docbuilder.h @@ -21,6 +21,8 @@ #include "fy-doc.h" +#include "fy-diag.h" + enum fy_document_builder_state { FYDBS_NODE, FYDBS_MAP_KEY, @@ -34,12 +36,6 @@ struct fy_document_builder_ctx { struct fy_node_pair *fynp; /* for mapping */ }; -struct fy_document_builder_cfg { - struct fy_parse_cfg parse_cfg; - void *userdata; - struct fy_diag *diag; -}; - struct fy_document_builder { struct fy_document_builder_cfg cfg; struct fy_document *fyd; @@ -52,50 +48,114 @@ struct fy_document_builder { struct fy_document_builder_ctx *stack; }; -struct fy_document_builder * -fy_document_builder_create(const struct fy_document_builder_cfg *cfg); - -void -fy_document_builder_reset(struct fy_document_builder *fydb); - -void -fy_document_builder_destroy(struct fy_document_builder *fydb); - -struct fy_document * -fy_document_builder_get_document(struct fy_document_builder *fydb); - -bool -fy_document_builder_is_in_stream(struct fy_document_builder *fydb); - -bool -fy_document_builder_is_in_document(struct fy_document_builder *fydb); - -bool -fy_document_builder_is_document_complete(struct fy_document_builder *fydb); - +/* internal only */ struct fy_document * -fy_document_builder_take_document(struct fy_document_builder *fydb); - -struct fy_document * -fy_document_builder_peek_document(struct fy_document_builder *fydb); - -void -fy_document_builder_set_in_stream(struct fy_document_builder *fydb); - -int -fy_document_builder_set_in_document(struct fy_document_builder *fydb, struct fy_document_state *fyds, bool single); +fy_document_builder_event_document(struct fy_document_builder *fydb, struct fy_eventp_list *evpl); -int -fy_document_builder_process_event(struct fy_document_builder *fydb, struct fy_eventp *fyep); +/* diagnostics */ + +static inline bool +fydb_debug_log_level_is_enabled(struct fy_document_builder *fydb, enum fy_error_module module) +{ + return fydb && fy_diag_log_level_is_enabled(fydb->cfg.diag, FYET_DEBUG, module); +} + +int fy_document_builder_vdiag(struct fy_document_builder *fydb, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap); + +int fy_document_builder_diag(struct fy_document_builder *fydb, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) + __attribute__((format(printf, 6, 7))); + +void fy_document_builder_diag_vreport(struct fy_document_builder *fydb, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap); +void fy_document_builder_diag_report(struct fy_document_builder *fydb, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +#ifdef FY_DEVMODE + +#define fydb_debug(_fydb, _module, _fmt, ...) \ + do { \ + struct fy_document_builder *__fydb = (_fydb); \ + enum fy_error_module __module = (_module); \ + \ + if (fydb_debug_log_level_is_enabled(__fydb, __module)) \ + fy_document_builder_diag(__fydb, FYET_DEBUG | FYDF_MODULE(_module), \ + __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__); \ + } while(0) +#else + +#define fydb_debug(_fydb, _module, _fmt, ...) \ + do { } while(0) -struct fy_document * -fy_document_builder_load_document(struct fy_document_builder *fydb, - struct fy_parser *fyp); - -struct fy_document * -fy_document_builder_event_document(struct fy_document_builder *fydb, struct fy_eventp_list *evpl); +#endif -struct fy_document * -fy_parse_load_document_with_builder(struct fy_parser *fyp); +#define fydb_info(_fydb, _fmt, ...) \ + fy_document_builder_diag((_fydb), FYET_INFO, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fydb_notice(_fydb, _fmt, ...) \ + fy_document_builder_diag((_fydb), FYET_NOTICE, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fydb_warning(_fydb, _fmt, ...) \ + fy_document_builder_diag((_fydb), FYET_WARNING, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fydb_error(_fydb, _fmt, ...) \ + fy_document_builder_diag((_fydb), FYET_ERROR, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) + +#define fydb_error_check(_fydb, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + fydb_error((_fydb), _fmt, ## __VA_ARGS__); \ + goto _label ; \ + } \ + } while(0) + +#define _FYDB_TOKEN_DIAG(_fydb, _fyt, _type, _module, _fmt, ...) \ + do { \ + struct fy_diag_report_ctx _drc; \ + memset(&_drc, 0, sizeof(_drc)); \ + _drc.type = (_type); \ + _drc.module = (_module); \ + _drc.fyt = (_fyt); \ + fy_document_builder_diag_report((_fydb), &_drc, (_fmt) , ## __VA_ARGS__); \ + } while(0) + +#define FYDB_TOKEN_DIAG(_fydb, _fyt, _type, _module, _fmt, ...) \ + _FYDB_TOKEN_DIAG(_fydb, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYDB_NODE_DIAG(_fydb, _fyn, _type, _module, _fmt, ...) \ + _FYDB_TOKEN_DIAG(_fydb, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYDB_TOKEN_ERROR(_fydb, _fyt, _module, _fmt, ...) \ + FYDB_TOKEN_DIAG(_fydb, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYDB_NODE_ERROR(_fydb, _fyn, _module, _fmt, ...) \ + FYDB_NODE_DIAG(_fydb, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYDB_TOKEN_ERROR_CHECK(_fydb, _fyt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYDB_TOKEN_ERROR(_fydb, _fyt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYDB_NODE_ERROR_CHECK(_fydb, _fyn, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYDB_NODE_ERROR(_fydb, _fyn, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYDB_TOKEN_WARNING(_fydb, _fyt, _module, _fmt, ...) \ + FYDB_TOKEN_DIAG(_fydb, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) #endif diff --git a/src/lib/fy-docstate.h b/src/lib/fy-docstate.h index 380c5f0..eb8df53 100644 --- a/src/lib/fy-docstate.h +++ b/src/lib/fy-docstate.h @@ -33,6 +33,7 @@ struct fy_document_state { bool tags_explicit : 1; bool start_implicit : 1; bool end_implicit : 1; + bool started_explicit : 1; bool json_mode : 1; struct fy_mark start_mark; struct fy_mark end_mark; diff --git a/src/lib/fy-dump.c b/src/lib/fy-dump.c index 24fb8c9..17a7543 100644 --- a/src/lib/fy-dump.c +++ b/src/lib/fy-dump.c @@ -89,6 +89,9 @@ const char *fy_token_type_txt[FYTT_COUNT] = { [FYTT_PE_METHOD] = "PE_METHOD", [FYTT_SE_METHOD] = "SE_METHOD", + + [FYTT_PE_BANG] = "PE_BANG", + [FYTT_PE_AT] = "PE_AT", }; char *fy_token_dump_format(struct fy_token *fyt, char *buf, size_t bufsz) diff --git a/src/lib/fy-emit.c b/src/lib/fy-emit.c index b382876..97ca397 100644 --- a/src/lib/fy-emit.c +++ b/src/lib/fy-emit.c @@ -23,6 +23,13 @@ #include "fy-parse.h" #include "fy-emit.h" +static void fy_emitter_fill_default_colors(struct fy_emitter *fye); +static FILE *fy_emitter_get_output_fp(struct fy_emitter *fye); +static int fy_emitter_get_output_fd(struct fy_emitter *fye); +static int fy_emitter_null_output(struct fy_emitter *fye, enum fy_emitter_write_type type, const char *str, int len, void *userdata); + +void fy_emit_save_ctx_cleanup(struct fy_emitter *emit, struct fy_emit_save_ctx *sc); + static inline struct fy_token_list *token_recycle_list(struct fy_emitter *emit, struct fy_parser *fyp) { if (fyp && fyp->recycled_token_list) @@ -49,48 +56,62 @@ static inline bool fy_emit_is_json_mode(const struct fy_emitter *emit) if (emit->force_json) return true; - flags = emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK); - return flags == FYECF_MODE_JSON || flags == FYECF_MODE_JSON_TP || flags == FYECF_MODE_JSON_ONELINE; + flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + return flags == FYECF_MODE_JSON || flags == FYECF_MODE_JSON_TP || + flags == FYECF_MODE_JSON_ONELINE || flags == FYECF_MODE_JSON_COMPACT; } static inline bool fy_emit_is_flow_mode(const struct fy_emitter *emit) { - enum fy_emitter_cfg_flags flags = emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); - return flags == FYECF_MODE_FLOW || flags == FYECF_MODE_FLOW_ONELINE || fy_emit_is_json_mode(emit); + return flags == FYECF_MODE_FLOW || flags == FYECF_MODE_FLOW_ONELINE || + flags == FYECF_MODE_FLOW_COMPACT || fy_emit_is_json_mode(emit); } static inline bool fy_emit_is_block_mode(const struct fy_emitter *emit) { - enum fy_emitter_cfg_flags flags = emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); return flags == FYECF_MODE_BLOCK || flags == FYECF_MODE_DEJSON || flags == FYECF_MODE_PRETTY; } static inline bool fy_emit_is_oneline(const struct fy_emitter *emit) { - enum fy_emitter_cfg_flags flags = emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); return flags == FYECF_MODE_FLOW_ONELINE || flags == FYECF_MODE_JSON_ONELINE; } +static inline bool fy_emit_is_compact(const struct fy_emitter *emit) +{ + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + + return flags == FYECF_MODE_FLOW_COMPACT || flags == FYECF_MODE_JSON_COMPACT; +} + +static inline bool fy_emit_is_oneline_or_compact(const struct fy_emitter *emit) +{ + return fy_emit_is_oneline(emit) || fy_emit_is_compact(emit); +} + static inline bool fy_emit_is_dejson_mode(const struct fy_emitter *emit) { - enum fy_emitter_cfg_flags flags = emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); return flags == FYECF_MODE_DEJSON; } static inline bool fy_emit_is_pretty_mode(const struct fy_emitter *emit) { - enum fy_emitter_cfg_flags flags = emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); return flags == FYECF_MODE_PRETTY; } static inline bool fy_emit_is_manual(const struct fy_emitter *emit) { - enum fy_emitter_cfg_flags flags = emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK); + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK); return flags == FYECF_MODE_MANUAL; } @@ -99,7 +120,7 @@ static inline int fy_emit_indent(struct fy_emitter *emit) { int indent; - indent = (emit->cfg.flags & FYECF_INDENT(FYECF_INDENT_MASK)) >> FYECF_INDENT_SHIFT; + indent = (emit->xcfg.cfg.flags & FYECF_INDENT(FYECF_INDENT_MASK)) >> FYECF_INDENT_SHIFT; return indent ? indent : 2; } @@ -107,7 +128,7 @@ static inline int fy_emit_width(struct fy_emitter *emit) { int width; - width = (emit->cfg.flags & FYECF_WIDTH(FYECF_WIDTH_MASK)) >> FYECF_WIDTH_SHIFT; + width = (emit->xcfg.cfg.flags & FYECF_WIDTH(FYECF_WIDTH_MASK)) >> FYECF_WIDTH_SHIFT; if (width == 0) return 80; if (width == FYECF_WIDTH_MASK) @@ -117,7 +138,7 @@ static inline int fy_emit_width(struct fy_emitter *emit) static inline bool fy_emit_output_comments(struct fy_emitter *emit) { - return !!(emit->cfg.flags & FYECF_OUTPUT_COMMENTS); + return !!(emit->xcfg.cfg.flags & FYECF_OUTPUT_COMMENTS); } static int fy_emit_node_check_json(struct fy_emitter *emit, struct fy_node *fyn) @@ -188,6 +209,42 @@ void fy_emit_scalar(struct fy_emitter *emit, struct fy_node *fyn, int flags, int void fy_emit_sequence(struct fy_emitter *emit, struct fy_node *fyn, int flags, int indent); void fy_emit_mapping(struct fy_emitter *emit, struct fy_node *fyn, int flags, int indent); +void fy_emit_write(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int len); +void fy_emit_puts(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str); +void fy_emit_putc(struct fy_emitter *emit, enum fy_emitter_write_type type, int c); + +/* simple write, just ascii, advance column */ +void fy_emit_write_simple(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int len) +{ + int outlen; + + if (!len) + return; + + outlen = emit->xcfg.cfg.output(emit, type, str, len, emit->xcfg.cfg.userdata); + if (outlen != len) + emit->output_error = true; + + emit->column += len; +} + +/* simple puts as above */ +void fy_emit_puts_simple(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str) +{ + fy_emit_write_simple(emit, type, str, strlen(str)); +} + +void fy_emit_putc_simple(struct fy_emitter *emit, enum fy_emitter_write_type type, int c) +{ + char cc = (char)c; + + fy_emit_write_simple(emit, type, &cc, 1); + if (cc == '\n') { + emit->column = 0; + emit->line++; + } +} + void fy_emit_write(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int len) { int c, w; @@ -197,7 +254,7 @@ void fy_emit_write(struct fy_emitter *emit, enum fy_emitter_write_type type, con if (!len) return; - outlen = emit->cfg.output(emit, type, str, len, emit->cfg.userdata); + outlen = emit->xcfg.cfg.output(emit, type, str, len, emit->xcfg.cfg.userdata); if (outlen != len) emit->output_error = true; @@ -276,7 +333,7 @@ void fy_emit_printf(struct fy_emitter *emit, enum fy_emitter_write_type type, co void fy_emit_write_ws(struct fy_emitter *emit) { - fy_emit_putc(emit, fyewt_whitespace, ' '); + fy_emit_putc_simple(emit, fyewt_whitespace, ' '); emit->flags |= FYEF_WHITESPACE; } @@ -289,14 +346,14 @@ void fy_emit_write_indent(struct fy_emitter *emit, int indent) if (!fy_emit_indentation(emit) || emit->column > indent || (emit->column == indent && !fy_emit_whitespace(emit))) - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); if (emit->column < indent) { len = indent - emit->column; ws = alloca(len + 1); memset(ws, ' ', len); ws[len] = '\0'; - fy_emit_write(emit, fyewt_indent, ws, len); + fy_emit_write_simple(emit, fyewt_indent, ws, len); } emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; @@ -326,30 +383,32 @@ void fy_emit_write_indicator(struct fy_emitter *emit, int flags, int indent, enum fy_emitter_write_type wtype) { + /* extended indicators mode? */ + if (wtype == fyewt_indicator && + (emit->xcfg.xflags & FYEXCF_EXTENDED_INDICATORS)) { + wtype = FYEWT_EXTENDED_INDICATORS_FIRST + (unsigned int)indicator; + assert(wtype >= FYEWT_EXTENDED_INDICATORS_FIRST && + wtype <= FYEWT_EXTENDED_INDICATORS_LAST); + } + switch (indicator) { case di_question_mark: if (!fy_emit_whitespace(emit)) fy_emit_write_ws(emit); - fy_emit_putc(emit, wtype, '?'); + fy_emit_putc_simple(emit, wtype, '?'); emit->flags &= ~(FYEF_WHITESPACE | FYEF_OPEN_ENDED); break; case di_colon: - if (!(flags & DDNF_SIMPLE)) { - if (!emit->flow_level && !fy_emit_is_oneline(emit)) - fy_emit_write_indent(emit, indent); - if (!fy_emit_whitespace(emit)) - fy_emit_write_ws(emit); - } - fy_emit_putc(emit, wtype, ':'); + fy_emit_putc_simple(emit, wtype, ':'); emit->flags &= ~(FYEF_WHITESPACE | FYEF_OPEN_ENDED); break; case di_dash: if (!fy_emit_whitespace(emit)) fy_emit_write_ws(emit); - fy_emit_putc(emit, wtype, '-'); + fy_emit_putc_simple(emit, wtype, '-'); emit->flags &= ~(FYEF_WHITESPACE | FYEF_OPEN_ENDED); break; @@ -358,7 +417,7 @@ void fy_emit_write_indicator(struct fy_emitter *emit, emit->flow_level++; if (!fy_emit_whitespace(emit)) fy_emit_write_ws(emit); - fy_emit_putc(emit, wtype, indicator == di_left_bracket ? '[' : '{'); + fy_emit_putc_simple(emit, wtype, indicator == di_left_bracket ? '[' : '{'); emit->flags |= FYEF_WHITESPACE; emit->flags &= ~(FYEF_INDENTATION | FYEF_OPEN_ENDED); break; @@ -366,12 +425,12 @@ void fy_emit_write_indicator(struct fy_emitter *emit, case di_right_bracket: case di_right_brace: emit->flow_level--; - fy_emit_putc(emit, wtype, indicator == di_right_bracket ? ']' : '}'); + fy_emit_putc_simple(emit, wtype, indicator == di_right_bracket ? ']' : '}'); emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION | FYEF_OPEN_ENDED); break; case di_comma: - fy_emit_putc(emit, wtype, ','); + fy_emit_putc_simple(emit, wtype, ','); emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION | FYEF_OPEN_ENDED); break; @@ -379,7 +438,7 @@ void fy_emit_write_indicator(struct fy_emitter *emit, case di_greater: if (!fy_emit_whitespace(emit)) fy_emit_write_ws(emit); - fy_emit_putc(emit, wtype, indicator == di_bar ? '|' : '>'); + fy_emit_putc_simple(emit, wtype, indicator == di_bar ? '|' : '>'); emit->flags &= ~(FYEF_INDENTATION | FYEF_WHITESPACE | FYEF_OPEN_ENDED); break; @@ -387,27 +446,27 @@ void fy_emit_write_indicator(struct fy_emitter *emit, case di_double_quote_start: if (!fy_emit_whitespace(emit)) fy_emit_write_ws(emit); - fy_emit_putc(emit, wtype, indicator == di_single_quote_start ? '\'' : '"'); + fy_emit_putc_simple(emit, wtype, indicator == di_single_quote_start ? '\'' : '"'); emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION | FYEF_OPEN_ENDED); break; case di_single_quote_end: case di_double_quote_end: - fy_emit_putc(emit, wtype, indicator == di_single_quote_end ? '\'' : '"'); + fy_emit_putc_simple(emit, wtype, indicator == di_single_quote_end ? '\'' : '"'); emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION | FYEF_OPEN_ENDED); break; case di_ambersand: if (!fy_emit_whitespace(emit)) fy_emit_write_ws(emit); - fy_emit_putc(emit, wtype, '&'); + fy_emit_putc_simple(emit, wtype, '&'); emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION); break; case di_star: if (!fy_emit_whitespace(emit)) fy_emit_write_ws(emit); - fy_emit_putc(emit, wtype, '*'); + fy_emit_putc_simple(emit, wtype, '*'); emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION); break; } @@ -495,14 +554,14 @@ void fy_emit_document_start_indicator(struct fy_emitter *emit) /* output linebreak anyway */ if (emit->column) - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); /* stripping doc indicators, do not emit */ - if (emit->cfg.flags & FYECF_STRIP_DOC) + if (emit->xcfg.cfg.flags & FYECF_STRIP_DOC) goto no_doc_emit; /* ok, emit document start indicator */ - fy_emit_puts(emit, fyewt_document_indicator, "---"); + fy_emit_puts_simple(emit, fyewt_document_indicator, "---"); emit->flags &= ~FYEF_WHITESPACE; emit->flags |= FYEF_HAD_DOCUMENT_START; return; @@ -597,6 +656,7 @@ void fy_emit_node_comment(struct fy_emitter *emit, struct fy_node *fyn, int flag } void fy_emit_common_node_preamble(struct fy_emitter *emit, + struct fy_token *fyt_value, struct fy_token *fyt_anchor, struct fy_token *fyt_tag, int flags, int indent) @@ -608,16 +668,21 @@ void fy_emit_common_node_preamble(struct fy_emitter *emit, size_t td_prefix_size, td_handle_size; size_t tag_len = 0, anchor_len = 0; bool json_mode = false; + struct fy_emit_save_ctx *sc = &emit->s_sc; json_mode = fy_emit_is_json_mode(emit); if (!json_mode) { - if (!(emit->cfg.flags & FYECF_STRIP_LABELS)) { + + if (fy_emit_token_has_comment(emit, fyt_value, fycp_top)) + fy_emit_token_comment(emit, fyt_value, sc->flags, sc->indent, fycp_top); + + if (!(emit->xcfg.cfg.flags & FYECF_STRIP_LABELS)) { if (fyt_anchor) anchor = fy_token_get_text(fyt_anchor, &anchor_len); } - if (!(emit->cfg.flags & FYECF_STRIP_TAGS)) { + if (!(emit->xcfg.cfg.flags & FYECF_STRIP_TAGS)) { if (fyt_tag) tag = fy_token_get_text(fyt_tag, &tag_len); } @@ -650,7 +715,7 @@ void fy_emit_common_node_preamble(struct fy_emitter *emit, /* content for root always starts on a new line */ if ((flags & DDNF_ROOT) && emit->column != 0 && !(emit->flags & FYEF_HAD_DOCUMENT_START)) { - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags = FYEF_WHITESPACE | FYEF_INDENTATION; } } @@ -660,19 +725,27 @@ void fy_emit_node_internal(struct fy_emitter *emit, struct fy_node *fyn, int fla enum fy_node_type type; struct fy_anchor *fya; struct fy_token *fyt_anchor = NULL; + struct fy_token *fyt_value = NULL; - if (!(emit->cfg.flags & FYECF_STRIP_LABELS)) { + if (!(emit->xcfg.cfg.flags & FYECF_STRIP_LABELS)) { fya = fy_document_lookup_anchor_by_node(emit->fyd, fyn); if (fya) fyt_anchor = fya->anchor; } - fy_emit_common_node_preamble(emit, fyt_anchor, fyn->tag, flags, indent); - type = fyn ? fyn->type : FYNT_SCALAR; + if (type == FYNT_SCALAR) + fyt_value = fyn ? fyn->scalar : NULL; + else if (type == FYNT_SEQUENCE) + fyt_value = fyn->sequence_start; + else if (type == FYNT_MAPPING) + fyt_value = fyn->mapping_start; + + fy_emit_common_node_preamble(emit, fyt_value, fyt_anchor, fyn->tag, flags, indent); + if (type != FYNT_SCALAR && (flags & DDNF_ROOT) && emit->column != 0) { - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags = FYEF_WHITESPACE | FYEF_INDENTATION; } @@ -721,8 +794,17 @@ void fy_emit_token_write_plain(struct fy_emitter *emit, struct fy_token *fyt, in atom = fy_token_atom(fyt); /* null and json mode */ - if (fy_emit_is_json_mode(emit) && (/* (!fyt || !atom || atom->size0) || */ fyt->scalar.is_null)) { - fy_emit_puts(emit, wtype, "null"); + if (fy_emit_is_json_mode(emit) && fyt->scalar.is_null) { + fy_emit_puts_simple(emit, wtype, "null"); + goto out; + } + + allow_breaks = !(flags & DDNF_SIMPLE) && !fy_emit_is_json_mode(emit) && !fy_emit_is_oneline(emit); + + /* very very simple case first (90% of cases) */ + str = fy_token_get_direct_simple_output(fyt, &len); + if (str && (!allow_breaks || (fy_emit_accum_column(&emit->ea) + (int)len <= fy_emit_width(emit)))) { + fy_emit_write_simple(emit, wtype, str, len); goto out; } @@ -736,8 +818,6 @@ void fy_emit_token_write_plain(struct fy_emitter *emit, struct fy_token *fyt, in if (!atom) goto out; - allow_breaks = !(flags & DDNF_SIMPLE) && !fy_emit_is_json_mode(emit) && !fy_emit_is_oneline(emit); - spaces = false; breaks = false; @@ -754,6 +834,7 @@ void fy_emit_token_write_plain(struct fy_emitter *emit, struct fy_token *fyt, in fy_emit_output_accum(emit, wtype, &emit->ea); emit->flags &= ~FYEF_INDENTATION; fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); } else fy_emit_accum_utf8_put(&emit->ea, c); @@ -769,17 +850,21 @@ void fy_emit_token_write_plain(struct fy_emitter *emit, struct fy_token *fyt, in if (!breaks) { fy_emit_output_accum(emit, wtype, &emit->ea); fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); } emit->flags &= ~FYEF_INDENTATION; fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); breaks = true; } else { - if (breaks) + if (breaks) { fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + } fy_emit_accum_utf8_put(&emit->ea, c); @@ -828,6 +913,7 @@ void fy_emit_token_write_alias(struct fy_emitter *emit, struct fy_token *fyt, in void fy_emit_token_write_quoted(struct fy_emitter *emit, struct fy_token *fyt, int flags, int indent, char qc) { + const struct fy_token_analysis *ta; bool allow_breaks, spaces, breaks; int c, i, w, digit; enum fy_emitter_write_type wtype; @@ -840,6 +926,7 @@ void fy_emit_token_write_quoted(struct fy_emitter *emit, struct fy_token *fyt, i uint32_t hi_surrogate, lo_surrogate; uint8_t non_utf8[4]; size_t non_utf8_len, k; + int emit_width; wtype = qc == '\'' ? ((flags & DDNF_SIMPLE_SCALAR_KEY) ? @@ -860,9 +947,15 @@ void fy_emit_token_write_quoted(struct fy_emitter *emit, struct fy_token *fyt, i */ target_style = qc == '"' ? FYAS_DOUBLE_QUOTED : FYAS_SINGLE_QUOTED; + allow_breaks = !(flags & DDNF_SIMPLE) && !fy_emit_is_json_mode(emit) && !fy_emit_is_oneline(emit); + emit_width = fy_emit_width(emit); + + ta = fy_token_text_analyze(fyt); + /* simple case of direct output (large amount of cases) */ str = fy_token_get_direct_output(fyt, &len); - if (str && fy_token_atom_style(fyt) == target_style) { + if (str && fy_token_atom_style(fyt) == target_style && + (ta->flags & FYTTAF_DIRECT_OUTPUT) && emit->column + ta->maxcol < emit_width) { fy_emit_write(emit, wtype, str, len); goto out; } @@ -872,8 +965,6 @@ void fy_emit_token_write_quoted(struct fy_emitter *emit, struct fy_token *fyt, i if (!atom) goto out; - allow_breaks = !(flags & DDNF_SIMPLE) && !fy_emit_is_json_mode(emit) && !fy_emit_is_oneline(emit); - spaces = false; breaks = false; @@ -902,7 +993,7 @@ void fy_emit_token_write_quoted(struct fy_emitter *emit, struct fy_token *fyt, i if (fy_is_space(c) || (qc == '\'' && fy_is_ws(c))) { should_indent = allow_breaks && !spaces && - fy_emit_accum_column(&emit->ea) > fy_emit_width(emit); + fy_emit_accum_column(&emit->ea) >= emit_width; if (should_indent && ((qc == '\'' && fy_is_ws(fy_atom_iter_utf8_peek(&iter))) || @@ -910,10 +1001,11 @@ void fy_emit_token_write_quoted(struct fy_emitter *emit, struct fy_token *fyt, i fy_emit_output_accum(emit, wtype, &emit->ea); if (qc == '"' && fy_is_ws(fy_atom_iter_utf8_peek(&iter))) - fy_emit_putc(emit, wtype, '\\'); + fy_emit_putc_simple(emit, wtype, '\\'); emit->flags &= ~FYEF_INDENTATION; fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); } else fy_emit_accum_utf8_put(&emit->ea, c); @@ -930,10 +1022,12 @@ void fy_emit_token_write_quoted(struct fy_emitter *emit, struct fy_token *fyt, i if (!breaks) { fy_emit_output_accum(emit, wtype, &emit->ea); fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); } emit->flags &= ~FYEF_INDENTATION; fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); breaks = true; } else { @@ -941,6 +1035,17 @@ void fy_emit_token_write_quoted(struct fy_emitter *emit, struct fy_token *fyt, i if (breaks) { fy_emit_output_accum(emit, wtype, &emit->ea); fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + + } else if (qc == '"' && allow_breaks && fy_emit_accum_column(&emit->ea) >= emit_width) { + + fy_emit_output_accum(emit, wtype, &emit->ea); + + fy_emit_putc_simple(emit, wtype, '\\'); + + emit->flags &= ~FYEF_INDENTATION; + fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); } /* escape */ @@ -1097,7 +1202,7 @@ bool fy_emit_token_write_block_hints(struct fy_emitter *emit, struct fy_token *f } if (atom->starts_with_ws || atom->starts_with_lb) { - fy_emit_putc(emit, fyewt_indicator, '0' + fy_emit_indent(emit)); + fy_emit_putc_simple(emit, fyewt_indicator, '0' + fy_emit_indent(emit)); explicit_chomp = true; } @@ -1116,7 +1221,7 @@ bool fy_emit_token_write_block_hints(struct fy_emitter *emit, struct fy_token *f out: if (chomp) - fy_emit_putc(emit, fyewt_indicator, chomp); + fy_emit_putc_simple(emit, fyewt_indicator, chomp); *chompp = chomp; return explicit_chomp; } @@ -1135,7 +1240,7 @@ void fy_emit_token_write_literal(struct fy_emitter *emit, struct fy_token *fyt, if (flags & DDNF_ROOT) indent += fy_emit_indent(emit); - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; atom = fy_token_atom(fyt); @@ -1150,6 +1255,7 @@ void fy_emit_token_write_literal(struct fy_emitter *emit, struct fy_token *fyt, if (breaks) { fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); breaks = false; } @@ -1182,7 +1288,7 @@ void fy_emit_token_write_folded(struct fy_emitter *emit, struct fy_token *fyt, i if (flags & DDNF_ROOT) indent += fy_emit_indent(emit); - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags |= FYEF_WHITESPACE | FYEF_INDENTATION; atom = fy_token_atom(fyt); @@ -1203,8 +1309,10 @@ void fy_emit_token_write_folded(struct fy_emitter *emit, struct fy_token *fyt, i fy_emit_output_accum(emit, fyewt_literal_scalar, &emit->ea); /* do not output a newline (indent) if at the end or * this is a leading spaces line */ - if (!fy_is_z(fy_atom_iter_utf8_peek(&iter)) && !leading_spaces) + if (!fy_is_z(fy_atom_iter_utf8_peek(&iter)) && !leading_spaces) { fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); + } } /* count the number of consecutive breaks */ @@ -1223,6 +1331,7 @@ void fy_emit_token_write_folded(struct fy_emitter *emit, struct fy_token *fyt, i while (nrbreaks-- > nrbreakslim) { emit->flags &= ~FYEF_INDENTATION; fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); } breaks = true; @@ -1232,6 +1341,7 @@ void fy_emit_token_write_folded(struct fy_emitter *emit, struct fy_token *fyt, i /* if we had a break, output an indent */ if (breaks) { fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); /* if this line starts with whitespace we need to know */ leading_spaces = fy_is_ws(c); @@ -1243,6 +1353,7 @@ void fy_emit_token_write_folded(struct fy_emitter *emit, struct fy_token *fyt, i fy_emit_output_accum(emit, fyewt_folded_scalar, &emit->ea); emit->flags &= ~FYEF_INDENTATION; fy_emit_write_indent(emit, indent); + fy_emit_output_col_sync(emit, &emit->ea); } else fy_emit_accum_utf8_put(&emit->ea, c); @@ -1259,9 +1370,9 @@ fy_emit_token_scalar_style(struct fy_emitter *emit, struct fy_token *fyt, int flags, int indent, enum fy_node_style style, struct fy_token *fyt_tag) { + const struct fy_token_analysis *ta = NULL; bool json, flow, is_null_scalar, is_json_plain; struct fy_atom *atom; - int aflags = -1; const char *tag; size_t tag_len; @@ -1308,23 +1419,18 @@ fy_emit_token_scalar_style(struct fy_emitter *emit, struct fy_token *fyt, return FYNS_DOUBLE_QUOTED; } - aflags = fy_token_text_analyze(fyt); + ta = fy_token_text_analyze(fyt); if (flow && (style == FYNS_ANY || style == FYNS_LITERAL || style == FYNS_FOLDED)) { /* if there's a linebreak, use double quoted style */ - if (aflags & FYTTAF_HAS_ANY_LB) { + if (ta->flags & FYTTAF_HAS_ANY_LB) { style = FYNS_DOUBLE_QUOTED; goto out; } - if (!(aflags & FYTTAF_HAS_NON_PRINT)) { - style = FYNS_SINGLE_QUOTED; - goto out; - } - /* anything not empty is double quoted here */ - style = !(aflags & FYTTAF_EMPTY) ? FYNS_PLAIN : FYNS_DOUBLE_QUOTED; + style = !(ta->flags & FYTTAF_EMPTY) ? FYNS_PLAIN : FYNS_DOUBLE_QUOTED; } /* try to pretify */ @@ -1332,21 +1438,20 @@ fy_emit_token_scalar_style(struct fy_emitter *emit, struct fy_token *fyt, (style == FYNS_ANY || style == FYNS_DOUBLE_QUOTED || style == FYNS_SINGLE_QUOTED)) { /* any original style can be a plain, but contains linebreaks, do a literal */ - if ((aflags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) == (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) { + if ((ta->flags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) == (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) { style = FYNS_LITERAL; goto out; } /* any style, can be just a plain, just make it so */ - if (style == FYNS_ANY && (aflags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) == FYTTAF_CAN_BE_PLAIN) { + if ((ta->flags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) == FYTTAF_CAN_BE_PLAIN) { style = FYNS_PLAIN; goto out; } - } if (!flow && emit->source_json && fy_emit_is_dejson_mode(emit)) { - if (is_json_plain || (aflags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) == FYTTAF_CAN_BE_PLAIN) { + if (is_json_plain || (ta->flags & (FYTTAF_CAN_BE_PLAIN | FYTTAF_HAS_LB)) == FYTTAF_CAN_BE_PLAIN) { style = FYNS_PLAIN; goto out; } @@ -1354,15 +1459,19 @@ fy_emit_token_scalar_style(struct fy_emitter *emit, struct fy_token *fyt, out: if (style == FYNS_ANY) - style = (aflags & FYTTAF_CAN_BE_PLAIN) ? FYNS_PLAIN : FYNS_DOUBLE_QUOTED; + style = (ta->flags & FYTTAF_CAN_BE_PLAIN) ? FYNS_PLAIN : FYNS_DOUBLE_QUOTED; if (style == FYNS_PLAIN) { /* plains in flow mode not being able to be plains * - plain in block mode that can't be plain in flow mode * - special handling for plains on start of line */ - if ((flow && !(aflags & FYTTAF_CAN_BE_PLAIN_FLOW) && !is_null_scalar) || - ((aflags & FYTTAF_QUOTE_AT_0) && indent == 0)) + if ((flow && !(ta->flags & FYTTAF_CAN_BE_PLAIN_FLOW) && !(ta->flags & FYTTAF_CAN_BE_SIMPLE_KEY) && !is_null_scalar) || + ((ta->flags & FYTTAF_QUOTE_AT_0) && indent == 0)) + style = FYNS_DOUBLE_QUOTED; + + /* make it double quoted, but only if not already over the width */ + if (style == FYNS_PLAIN && emit->column < fy_emit_width(emit) && (emit->column + ta->maxspan) > fy_emit_width(emit)) style = FYNS_DOUBLE_QUOTED; } @@ -1425,7 +1534,6 @@ void fy_emit_scalar(struct fy_emitter *emit, struct fy_node *fyn, int flags, int static void fy_emit_sequence_prolog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) { bool json = fy_emit_is_json_mode(emit); - bool oneline = fy_emit_is_oneline(emit); bool was_flow = sc->flow; sc->old_indent = sc->indent; @@ -1453,7 +1561,7 @@ static void fy_emit_sequence_prolog(struct fy_emitter *emit, struct fy_emit_save fy_emit_write_indicator(emit, di_left_bracket, sc->flags, sc->indent, fyewt_indicator); } - if (!oneline) { + if (!fy_emit_is_oneline(emit)) { if (was_flow || (sc->flags & (DDNF_ROOT | DDNF_SEQ))) sc->indent = fy_emit_increase_indent(emit, sc->flags, sc->indent); } @@ -1463,9 +1571,10 @@ static void fy_emit_sequence_prolog(struct fy_emitter *emit, struct fy_emit_save static void fy_emit_sequence_epilog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) { + if (sc->flow && (sc->flags & DDNF_HANGING_INDENT) && !fy_emit_is_oneline_or_compact(emit) && !sc->empty) + fy_emit_write_indent(emit, sc->old_indent); + if (sc->flow || fy_emit_is_json_mode(emit)) { - if (!fy_emit_is_oneline(emit) && !sc->empty) - fy_emit_write_indent(emit, sc->old_indent); fy_emit_write_indicator(emit, di_right_bracket, sc->flags, sc->old_indent, fyewt_indicator); } } @@ -1477,7 +1586,9 @@ static void fy_emit_sequence_item_prolog(struct fy_emitter *emit, struct fy_emit sc->flags |= DDNF_SEQ; - if (!fy_emit_is_oneline(emit)) + if (fy_emit_token_has_comment(emit, fyt_value, fycp_top) || + (!fy_emit_is_oneline_or_compact(emit) && !sc->flow) || + ((fy_emit_is_compact(emit) || sc->flow) && emit->column >= fy_emit_width(emit))) fy_emit_write_indent(emit, sc->indent); if (!sc->flow && !fy_emit_is_json_mode(emit)) @@ -1499,7 +1610,7 @@ static void fy_emit_sequence_item_epilog(struct fy_emitter *emit, struct fy_emit fy_emit_token_comment(emit, fyt_value, sc->flags, sc->indent, fycp_right); - if (last && (sc->flow || fy_emit_is_json_mode(emit)) && !fy_emit_is_oneline(emit) && !sc->empty) + if (last && sc->flow && (sc->flags & DDNF_HANGING_INDENT) && !fy_emit_is_oneline_or_compact(emit) && !sc->empty) fy_emit_write_indent(emit, sc->old_indent); sc->flags &= ~DDNF_SEQ; @@ -1541,7 +1652,6 @@ void fy_emit_sequence(struct fy_emitter *emit, struct fy_node *fyn, int flags, i static void fy_emit_mapping_prolog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) { bool json = fy_emit_is_json_mode(emit); - bool oneline = fy_emit_is_oneline(emit); bool was_flow = sc->flow; sc->old_indent = sc->indent; @@ -1569,7 +1679,7 @@ static void fy_emit_mapping_prolog(struct fy_emitter *emit, struct fy_emit_save_ fy_emit_write_indicator(emit, di_left_brace, sc->flags, sc->indent, fyewt_indicator); } - if (!oneline && !sc->empty) + if (!fy_emit_is_oneline(emit) && !sc->empty) sc->indent = fy_emit_increase_indent(emit, sc->flags, sc->indent); sc->flags &= ~DDNF_ROOT; @@ -1578,7 +1688,7 @@ static void fy_emit_mapping_prolog(struct fy_emitter *emit, struct fy_emit_save_ static void fy_emit_mapping_epilog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc) { if (sc->flow || fy_emit_is_json_mode(emit)) { - if (!fy_emit_is_oneline(emit) && !sc->empty) + if ((sc->flags & DDNF_HANGING_INDENT) && !fy_emit_is_oneline_or_compact(emit) && !sc->empty) fy_emit_write_indent(emit, sc->old_indent); fy_emit_write_indicator(emit, di_right_brace, sc->flags, sc->old_indent, fyewt_indicator); } @@ -1587,6 +1697,9 @@ static void fy_emit_mapping_epilog(struct fy_emitter *emit, struct fy_emit_save_ static void fy_emit_mapping_key_prolog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc, struct fy_token *fyt_key, bool simple_key) { + const struct fy_token_analysis *ta; + bool do_indent, key_over; + sc->flags = DDNF_MAP | (sc->flags & DDNF_FLOW); if (simple_key) { @@ -1599,7 +1712,24 @@ static void fy_emit_mapping_key_prolog(struct fy_emitter *emit, struct fy_emit_s sc->flags |= DDNF_SIMPLE; } - if (!fy_emit_is_oneline(emit)) + do_indent = false; + if (!fy_emit_is_oneline(emit)) { + if (fyt_key && fyt_key->type != FYTT_SCALAR) + /* always indent on non scalar keys */ + key_over = false; + else { + /* for scalar keys, check if key + 2 + quotes go over */ + ta = fy_token_text_analyze(fyt_key); + key_over = (emit->column + ta->maxspan + 2 + (!simple_key ? 2 : 0)) >= fy_emit_width(emit); + } + + do_indent = key_over; + if (!do_indent && !fy_emit_is_compact(emit)) + do_indent = !sc->flow || (sc->flags & DDNF_HANGING_INDENT); + } else + do_indent = false; + + if (do_indent) fy_emit_write_indent(emit, sc->indent); /* complex? */ @@ -1618,6 +1748,12 @@ static void fy_emit_mapping_key_epilog(struct fy_emitter *emit, struct fy_emit_s sc->flags &= ~DDNF_MAP; + if (!(sc->flags & DDNF_SIMPLE)) { + if (!emit->flow_level && !fy_emit_is_oneline_or_compact(emit)) + fy_emit_write_indent(emit, sc->indent); + if (!fy_emit_whitespace(emit)) + fy_emit_write_ws(emit); + } fy_emit_write_indicator(emit, di_colon, sc->flags, sc->indent, fyewt_indicator); tmp_indent = sc->indent; @@ -1636,7 +1772,20 @@ static void fy_emit_mapping_key_epilog(struct fy_emitter *emit, struct fy_emit_s static void fy_emit_mapping_value_prolog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc, struct fy_token *fyt_value) { - /* nothing */ + const struct fy_token_analysis *ta; + bool value_over; + + /* don't do anything for those cases */ + if (!sc->flow || fy_emit_is_oneline(emit) || !fyt_value || fyt_value->type != FYTT_SCALAR) + return; + + ta = fy_token_text_analyze(fyt_value); + value_over = (emit->column + ta->maxspan) >= fy_emit_width(emit); + + if (value_over) { + sc->indent = fy_emit_increase_indent(emit, sc->flags, sc->indent); + fy_emit_write_indent(emit, sc->indent); + } } static void fy_emit_mapping_value_epilog(struct fy_emitter *emit, struct fy_emit_save_ctx *sc, @@ -1647,7 +1796,7 @@ static void fy_emit_mapping_value_epilog(struct fy_emitter *emit, struct fy_emit fy_emit_token_comment(emit, fyt_value, sc->flags, sc->indent, fycp_right); - if (last && (sc->flow || fy_emit_is_json_mode(emit)) && !fy_emit_is_oneline(emit) && !sc->empty) + if (last && sc->flow && (sc->flags & DDNF_HANGING_INDENT) && !fy_emit_is_oneline_or_compact(emit) && !sc->empty) fy_emit_write_indent(emit, sc->old_indent); sc->flags &= ~DDNF_MAP; @@ -1658,7 +1807,8 @@ void fy_emit_mapping(struct fy_emitter *emit, struct fy_node *fyn, int flags, in struct fy_node_pair *fynp, *fynpn, **fynpp = NULL; struct fy_token *fyt_key, *fyt_value; bool last, simple_key, used_malloc = false; - int aflags, i, count; + const struct fy_token_analysis *tak = NULL, *tav; + int i, count; struct fy_emit_save_ctx sct, *sc = &sct; memset(sc, 0, sizeof(*sc)); @@ -1673,7 +1823,7 @@ void fy_emit_mapping(struct fy_emitter *emit, struct fy_node *fyn, int flags, in fy_emit_mapping_prolog(emit, sc); - if (!(emit->cfg.flags & (FYECF_SORT_KEYS | FYECF_STRIP_EMPTY_KV))) { + if (!(emit->xcfg.cfg.flags & (FYECF_SORT_KEYS | FYECF_STRIP_EMPTY_KV))) { fynp = fy_node_pair_list_head(&fyn->mapping); fynpp = NULL; } else { @@ -1694,7 +1844,7 @@ void fy_emit_mapping(struct fy_emitter *emit, struct fy_node *fyn, int flags, in fynp = fy_node_pair_next(&fyn->mapping, fynp)) { /* strip key/value pair from the output if it's empty */ - if ((emit->cfg.flags & FYECF_STRIP_EMPTY_KV) && fy_node_is_empty(fynp->value)) + if ((emit->xcfg.cfg.flags & FYECF_STRIP_EMPTY_KV) && fy_node_is_empty(fynp->value)) continue; fynpp[i++] = fynp; @@ -1703,7 +1853,7 @@ void fy_emit_mapping(struct fy_emitter *emit, struct fy_node *fyn, int flags, in fynpp[count] = NULL; /* sort the keys */ - if (emit->cfg.flags & FYECF_SORT_KEYS) + if (emit->xcfg.cfg.flags & FYECF_SORT_KEYS) fy_node_mapping_perform_sort(fyn, NULL, NULL, fynpp, count); i = 0; @@ -1730,9 +1880,9 @@ void fy_emit_mapping(struct fy_emitter *emit, struct fy_node *fyn, int flags, in if (fynp->key) { switch (fynp->key->type) { case FYNT_SCALAR: - aflags = fy_token_text_analyze(fynp->key->scalar); + tak = fy_token_text_analyze(fynp->key->scalar); simple_key = fy_emit_is_json_mode(emit) || - !!(aflags & FYTTAF_CAN_BE_SIMPLE_KEY); + !!(tak->flags & FYTTAF_CAN_BE_SIMPLE_KEY); break; case FYNT_SEQUENCE: simple_key = fy_node_list_empty(&fynp->key->sequence); @@ -1743,6 +1893,17 @@ void fy_emit_mapping(struct fy_emitter *emit, struct fy_node *fyn, int flags, in } } + /* special handling for compact mode simple key + scalar value*/ + if (fy_emit_is_compact(emit) && simple_key && + fynp->value && fynp->value->type == FYNT_SCALAR && tak) { + tav = fy_token_text_analyze(fynp->value->scalar); + + /* for plain key + value if we go over the width */ + if ((tav->flags & FYTTAF_CAN_BE_PLAIN) && + (emit->column + tak->maxspan + 2 + tav->maxspan + 1) >= fy_emit_width(emit)) + fy_emit_write_indent(emit, sc->indent); + } + fy_emit_mapping_key_prolog(emit, sc, fyt_key, simple_key); if (fynp->key) fy_emit_node_internal(emit, fynp->key, (sc->flags & ~DDNF_ROOT), sc->indent, true); @@ -1770,7 +1931,7 @@ int fy_emit_common_document_start(struct fy_emitter *emit, struct fy_token *fyt_chk; const char *td_handle, *td_prefix; size_t td_handle_size, td_prefix_size; - enum fy_emitter_cfg_flags flags = emit->cfg.flags; + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags; enum fy_emitter_cfg_flags vd_flags = flags & FYECF_VERSION_DIR(FYECF_VERSION_DIR_MASK); enum fy_emitter_cfg_flags td_flags = flags & FYECF_TAG_DIR(FYECF_TAG_DIR_MASK); enum fy_emitter_cfg_flags dsm_flags = flags & FYECF_DOC_START_MARK(FYECF_DOC_START_MARK_MASK); @@ -1780,23 +1941,23 @@ int fy_emit_common_document_start(struct fy_emitter *emit, if (!emit || !fyds || emit->fyds) return -1; - emit->fyds = fyds; + emit->fyds = fy_document_state_ref(fyds); vd = ((vd_flags == FYECF_VERSION_DIR_AUTO && fyds->version_explicit) || vd_flags == FYECF_VERSION_DIR_ON) && - !(emit->cfg.flags & FYECF_STRIP_DOC); + !(emit->xcfg.cfg.flags & FYECF_STRIP_DOC); td = ((td_flags == FYECF_TAG_DIR_AUTO && fyds->tags_explicit) || td_flags == FYECF_TAG_DIR_ON) && - !(emit->cfg.flags & FYECF_STRIP_DOC); + !(emit->xcfg.cfg.flags & FYECF_STRIP_DOC); /* if either a version or directive tags exist, and no previous * explicit document end existed, output one now */ if (!fy_emit_is_json_mode(emit) && (vd || td) && !(emit->flags & FYEF_HAD_DOCUMENT_END)) { if (emit->column) - fy_emit_putc(emit, fyewt_linebreak, '\n'); - if (!(emit->cfg.flags & FYECF_STRIP_DOC)) { - fy_emit_puts(emit, fyewt_document_indicator, "..."); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + if (!(emit->xcfg.cfg.flags & FYECF_STRIP_DOC)) { + fy_emit_puts_simple(emit, fyewt_document_indicator, "..."); emit->flags &= ~FYEF_WHITESPACE; emit->flags |= FYEF_HAD_DOCUMENT_END; } @@ -1804,10 +1965,10 @@ int fy_emit_common_document_start(struct fy_emitter *emit, if (!fy_emit_is_json_mode(emit) && vd) { if (emit->column) - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); fy_emit_printf(emit, fyewt_version_directive, "%%YAML %d.%d", fyds->version.major, fyds->version.minor); - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags = FYEF_WHITESPACE | FYEF_INDENTATION; } @@ -1825,11 +1986,11 @@ int fy_emit_common_document_start(struct fy_emitter *emit, had_non_default_tag = true; if (emit->column) - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); fy_emit_printf(emit, fyewt_tag_directive, "%%TAG %.*s %.*s", (int)td_handle_size, td_handle, (int)td_prefix_size, td_prefix); - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags = FYEF_WHITESPACE | FYEF_INDENTATION; } } @@ -1887,7 +2048,7 @@ int fy_emit_document_start(struct fy_emitter *emit, struct fy_document *fyd, int fy_emit_common_document_end(struct fy_emitter *emit, bool override_state, bool implicit_override) { const struct fy_document_state *fyds; - enum fy_emitter_cfg_flags flags = emit->cfg.flags; + enum fy_emitter_cfg_flags flags = emit->xcfg.cfg.flags; enum fy_emitter_cfg_flags dem_flags = flags & FYECF_DOC_END_MARK(FYECF_DOC_END_MARK_MASK); bool implicit, dem; @@ -1897,22 +2058,24 @@ int fy_emit_common_document_end(struct fy_emitter *emit, bool override_state, bo fyds = emit->fyds; implicit = fyds->end_implicit; - if (override_state) + if (fyds->started_explicit) + implicit = false; + else if (override_state) implicit = implicit_override; dem = ((dem_flags == FYECF_DOC_END_MARK_AUTO && !implicit) || dem_flags == FYECF_DOC_END_MARK_ON) && - !(emit->cfg.flags & FYECF_STRIP_DOC); + !(emit->xcfg.cfg.flags & FYECF_STRIP_DOC); - if (!(emit->cfg.flags & FYECF_NO_ENDING_NEWLINE)) { + if (!(emit->xcfg.cfg.flags & FYECF_NO_ENDING_NEWLINE)) { if (emit->column != 0) { - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags = FYEF_WHITESPACE | FYEF_INDENTATION; } if (!fy_emit_is_json_mode(emit) && dem) { - fy_emit_puts(emit, fyewt_document_indicator, "..."); - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_puts_simple(emit, fyewt_document_indicator, "..."); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags = FYEF_WHITESPACE | FYEF_INDENTATION; emit->flags |= FYEF_HAD_DOCUMENT_END; } else @@ -1920,10 +2083,10 @@ int fy_emit_common_document_end(struct fy_emitter *emit, bool override_state, bo } else { if (!fy_emit_is_json_mode(emit) && dem) { if (emit->column != 0) { - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags = FYEF_WHITESPACE | FYEF_INDENTATION; } - fy_emit_puts(emit, fyewt_document_indicator, "..."); + fy_emit_puts_simple(emit, fyewt_document_indicator, "..."); emit->flags &= ~(FYEF_WHITESPACE | FYEF_INDENTATION); emit->flags |= FYEF_HAD_DOCUMENT_END; } else @@ -1934,6 +2097,7 @@ int fy_emit_common_document_end(struct fy_emitter *emit, bool override_state, bo emit->flags |= FYEF_HAD_DOCUMENT_OUTPUT; /* stop our association with the document */ + fy_document_state_unref(emit->fyds); emit->fyds = NULL; return 0; @@ -1957,13 +2121,13 @@ int fy_emit_common_explicit_document_end(struct fy_emitter *emit) return -1; if (emit->column != 0) { - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags = FYEF_WHITESPACE | FYEF_INDENTATION; } if (!fy_emit_is_json_mode(emit)) { - fy_emit_puts(emit, fyewt_document_indicator, "..."); - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_puts_simple(emit, fyewt_document_indicator, "..."); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags = FYEF_WHITESPACE | FYEF_INDENTATION; emit->flags |= FYEF_HAD_DOCUMENT_END; } else @@ -1973,6 +2137,7 @@ int fy_emit_common_explicit_document_end(struct fy_emitter *emit) emit->flags |= FYEF_HAD_DOCUMENT_OUTPUT; /* stop our association with the document */ + fy_document_state_unref(emit->fyds); emit->fyds = NULL; return 0; @@ -2012,8 +2177,11 @@ void fy_emit_reset(struct fy_emitter *emit, bool reset_events) /* streaming mode flags */ emit->s_flags = DDNF_ROOT; + fy_emit_save_ctx_cleanup(emit, &emit->s_sc); + while (emit->sc_stack_top > 0) + fy_emit_save_ctx_cleanup(emit, &emit->sc_stack[--emit->sc_stack_top]); + emit->state_stack_top = 0; - emit->sc_stack_top = 0; /* and release any queued events */ if (reset_events) { @@ -2030,10 +2198,42 @@ int fy_emit_setup(struct fy_emitter *emit, const struct fy_emitter_cfg *cfg) return -1; memset(emit, 0, sizeof(*emit)); - - emit->cfg = *cfg; - if (!emit->cfg.output) - emit->cfg.output = fy_emitter_default_output; + emit->output_fd = -1; + + /* is it extended config or not? */ + if (!(cfg->flags & FYECF_EXTENDED_CFG)) + emit->xcfg.cfg = *cfg; + else + emit->xcfg = *container_of(cfg, struct fy_emitter_xcfg, cfg); + + fy_emitter_fill_default_colors(emit); + emit->output_fp = fy_emitter_get_output_fp(emit); + if (!emit->output_fp) + emit->output_fd = fy_emitter_get_output_fd(emit); + + if (!emit->xcfg.cfg.output) { + if (!emit->output_fp && emit->output_fd < 0) { + emit->xcfg.cfg.output = fy_emitter_null_output; + } else { + emit->xcfg.cfg.output = fy_emitter_default_output; + /* for the default, turn on extended indicators */ + emit->xcfg.xflags |= FYEXCF_EXTENDED_INDICATORS; + + switch (emit->xcfg.xflags & (FYEXCF_COLOR_MASK << FYEXCF_COLOR_SHIFT)) { + case FYEXCF_COLOR_AUTO: + emit->output_colorize = emit->output_fp ? + isatty(fileno(emit->output_fp)) : + isatty(emit->output_fd); + break; + case FYEXCF_COLOR_FORCE: + emit->output_colorize = true; + break; + default: + emit->output_colorize = false; + break; + } + } + } diag = cfg->diag; @@ -2079,7 +2279,9 @@ void fy_emit_save_ctx_cleanup(struct fy_emitter *emit, struct fy_emit_save_ctx * { sc = &emit->s_sc; fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_key); + sc->fyt_last_key = NULL; fy_token_unref_rl(emit->recycled_token_list, sc->fyt_last_value); + sc->fyt_last_value = NULL; memset(sc, 0, sizeof(*sc)); } @@ -2102,8 +2304,8 @@ void fy_emit_cleanup(struct fy_emitter *emit) while ((fyep = fy_eventp_list_pop(&emit->recycled_eventp)) != NULL) fy_eventp_free(fyep); - if (!emit->fyd && emit->fyds) - fy_document_state_unref(emit->fyds); + fy_document_state_unref(emit->fyds); + emit->fyds = NULL; fy_emit_accum_cleanup(&emit->ea); @@ -2177,7 +2379,7 @@ void fy_emit_prepare_document_state(struct fy_emitter *emit, struct fy_document_ /* if the original document was JSON and the mode is ORIGINAL turn on JSON mode */ emit->source_json = fyds && fyds->json_mode; - emit->force_json = (emit->cfg.flags & FYECF_MODE(FYECF_MODE_MASK)) == FYECF_MODE_ORIGINAL && + emit->force_json = (emit->xcfg.cfg.flags & FYECF_MODE(FYECF_MODE_MASK)) == FYECF_MODE_ORIGINAL && emit->source_json; } @@ -2254,7 +2456,7 @@ const struct fy_emitter_cfg *fy_emitter_get_cfg(struct fy_emitter *emit) if (!emit) return NULL; - return &emit->cfg; + return &emit->xcfg.cfg; } struct fy_diag *fy_emitter_get_diag(struct fy_emitter *emit) @@ -2305,7 +2507,7 @@ struct fy_emit_buffer_state { static int do_buffer_output(struct fy_emitter *emit, enum fy_emitter_write_type type, const char *str, int leni, void *userdata) { - struct fy_emit_buffer_state *state = emit->cfg.userdata; + struct fy_emit_buffer_state *state = emit->xcfg.cfg.userdata; size_t left, pagesize, size, len; char *bufnew; @@ -2349,7 +2551,7 @@ fy_emitter_str_finalizer(struct fy_emitter *emit) { struct fy_emit_buffer_state *state; - if (!emit || !(state = emit->cfg.userdata)) + if (!emit || !(state = emit->xcfg.cfg.userdata)) return; /* if the buffer is allowed to allocate_buffer... */ @@ -2357,7 +2559,7 @@ fy_emitter_str_finalizer(struct fy_emitter *emit) free(state->buf); free(state); - emit->cfg.userdata = NULL; + emit->xcfg.cfg.userdata = NULL; } static struct fy_emitter * @@ -2415,9 +2617,10 @@ fy_emitter_collect_str_internal(struct fy_emitter *emit, char **bufp, size_t *si { struct fy_emit_buffer_state *state; char *buf; + size_t size; int rc; - state = emit->cfg.userdata; + state = emit->xcfg.cfg.userdata; assert(state); /* if NULL, then use the values stored on the state */ @@ -2442,7 +2645,10 @@ fy_emitter_collect_str_internal(struct fy_emitter *emit, char **bufp, size_t *si } /* retreive the buffer and size */ - *sizep = state->size; + size = state->size; + if (size > 0 && state->buf[size-1] == '\0') + size--; + *sizep = size; *bufp = state->buf; /* reset the buffer, ownership now to the caller */ @@ -2906,7 +3112,7 @@ static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_parser *fyp if (fye->type != FYET_ALIAS && fye->type != FYET_SCALAR && (emit->s_flags & DDNF_ROOT) && emit->column != 0) { - fy_emit_putc(emit, fyewt_linebreak, '\n'); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); emit->flags = FYEF_WHITESPACE | FYEF_INDENTATION; } @@ -2921,9 +3127,11 @@ static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_parser *fyp case FYET_SCALAR: /* if we're pretty and at column 0 (meaning it's a single scalar document) output --- */ if ((emit->s_flags & DDNF_ROOT) && fy_emit_is_pretty_mode(emit) && !emit->column && - !fy_emit_is_flow_mode(emit) && !(emit->s_flags & DDNF_FLOW)) + !fy_emit_is_flow_mode(emit) && !(emit->s_flags & DDNF_FLOW)) { fy_emit_document_start_indicator(emit); - fy_emit_common_node_preamble(emit, fye->scalar.anchor, fye->scalar.tag, emit->s_flags, emit->s_indent); + emit->fyds->started_explicit = true; + } + fy_emit_common_node_preamble(emit, fye->scalar.value, fye->scalar.anchor, fye->scalar.tag, emit->s_flags, emit->s_indent); style = fye->scalar.value ? fy_node_style_from_scalar_style(fye->scalar.value->scalar.style) : FYNS_PLAIN; @@ -2947,7 +3155,9 @@ static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_parser *fyp else xstyle = FYNS_ANY; - fy_emit_common_node_preamble(emit, fye->sequence_start.anchor, fye->sequence_start.tag, emit->s_flags, emit->s_indent); + fy_emit_common_node_preamble(emit, fye->sequence_start.sequence_start, + fye->sequence_start.anchor, fye->sequence_start.tag, + emit->s_flags, emit->s_indent); /* create new context */ memset(sc, 0, sizeof(*sc)); @@ -2986,7 +3196,9 @@ static int fy_emit_streaming_node(struct fy_emitter *emit, struct fy_parser *fyp else xstyle = FYNS_ANY; - fy_emit_common_node_preamble(emit, fye->mapping_start.anchor, fye->mapping_start.tag, emit->s_flags, emit->s_indent); + fy_emit_common_node_preamble(emit, fye->mapping_start.mapping_start, + fye->mapping_start.anchor, fye->mapping_start.tag, + emit->s_flags, emit->s_indent); /* create new context */ memset(sc, 0, sizeof(*sc)); @@ -3027,6 +3239,9 @@ static int fy_emit_handle_stream_start(struct fy_emitter *emit, struct fy_parser return -1; } + fy_document_state_unref(emit->fyds); + emit->fyds = NULL; + fy_emit_reset(emit, false); fy_emit_goto_state(emit, FYES_FIRST_DOCUMENT_START); @@ -3052,7 +3267,7 @@ static int fy_emit_handle_document_start(struct fy_emitter *emit, struct fy_pars /* transfer ownership to the emitter */ fyds = fye->document_start.document_state; - fye->document_start.document_state = NULL; + // fye->document_start.document_state = NULL; /* prepare (i.e. adapt to the document state) */ fy_emit_prepare_document_state(emit, fyds); @@ -3066,7 +3281,6 @@ static int fy_emit_handle_document_start(struct fy_emitter *emit, struct fy_pars static int fy_emit_handle_document_end(struct fy_emitter *emit, struct fy_parser *fyp, struct fy_eventp *fyep) { - struct fy_document_state *fyds; struct fy_event *fye = &fyep->e; int ret; @@ -3075,14 +3289,10 @@ static int fy_emit_handle_document_end(struct fy_emitter *emit, struct fy_parser return -1; } - fyds = emit->fyds; - ret = fy_emit_common_document_end(emit, true, fye->document_end.implicit); if (ret) return ret; - fy_document_state_unref(fyds); - fy_emit_reset(emit, false); fy_emit_goto_state(emit, FYES_DOCUMENT_START); return 0; @@ -3192,12 +3402,13 @@ static int fy_emit_handle_mapping_key(struct fy_emitter *emit, struct fy_parser struct fy_event *fye = &fyep->e; struct fy_emit_save_ctx *sc = &emit->s_sc; struct fy_token *fyt_key = NULL; - int ret, aflags; + const struct fy_token_analysis *ta; + int ret; bool simple_key; fy_emit_token_unref(emit, fyp, sc->fyt_last_key); - fy_emit_token_unref(emit, fyp, sc->fyt_last_value); sc->fyt_last_key = NULL; + fy_emit_token_unref(emit, fyp, sc->fyt_last_value); sc->fyt_last_value = NULL; simple_key = false; @@ -3227,8 +3438,8 @@ static int fy_emit_handle_mapping_key(struct fy_emitter *emit, struct fy_parser break; case FYET_SCALAR: fyt_key = fye->scalar.value; - aflags = fy_token_text_analyze(fyt_key); - simple_key = !!(aflags & FYTTAF_CAN_BE_SIMPLE_KEY); + ta = fy_token_text_analyze(fyt_key); + simple_key = !!(ta->flags & FYTTAF_CAN_BE_SIMPLE_KEY); break; case FYET_SEQUENCE_START: fyt_key = fye->sequence_start.sequence_start; @@ -3349,6 +3560,15 @@ int fy_emit_event_from_parser(struct fy_emitter *emit, struct fy_parser *fyp, st if (emit->state == FYES_NONE) emit->state = FYES_STREAM_START; + /* handle reset (parser error recovery) */ + if (fye->type == FYET_STREAM_START && emit->state != FYES_STREAM_START) { + if (emit->column != 0) + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + fy_emit_puts_simple(emit, fyewt_document_indicator, "..."); + fy_emit_putc_simple(emit, fyewt_linebreak, '\n'); + emit->state = FYES_STREAM_START; + } + fyep = container_of(fye, struct fy_eventp, e); fy_eventp_list_add_tail(&emit->queued_events, fyep); @@ -3425,124 +3645,233 @@ fy_emitter_get_document_state(struct fy_emitter *emit) return emit ? emit->fyds : NULL; } +/* ANSI colors and escapes */ +#define A_RESET "\x1b[0m" +#define A_BLACK "\x1b[30m" +#define A_RED "\x1b[31m" +#define A_GREEN "\x1b[32m" +#define A_YELLOW "\x1b[33m" +#define A_BLUE "\x1b[34m" +#define A_MAGENTA "\x1b[35m" +#define A_CYAN "\x1b[36m" +#define A_LIGHT_GRAY "\x1b[37m" /* dark white is gray */ +#define A_GRAY "\x1b[1;30m" +#define A_BRIGHT_RED "\x1b[1;31m" +#define A_BRIGHT_GREEN "\x1b[1;32m" +#define A_BRIGHT_YELLOW "\x1b[1;33m" +#define A_BRIGHT_BLUE "\x1b[1;34m" +#define A_BRIGHT_MAGENTA "\x1b[1;35m" +#define A_BRIGHT_CYAN "\x1b[1;36m" +#define A_WHITE "\x1b[1;37m" + +static const char *default_colors[FYEWT_COUNT] = { + [fyewt_document_indicator] = A_CYAN, + [fyewt_tag_directive] = A_YELLOW, + [fyewt_version_directive] = A_YELLOW, + [fyewt_indent] = A_GREEN, // when visible only + [fyewt_indicator] = A_YELLOW, // in extended mode we produce more + [fyewt_whitespace] = A_GREEN, // when visible only + [fyewt_plain_scalar] = A_WHITE, + [fyewt_single_quoted_scalar] = A_YELLOW, + [fyewt_double_quoted_scalar] = A_YELLOW, + [fyewt_literal_scalar] = A_YELLOW, + [fyewt_folded_scalar] = A_YELLOW, + [fyewt_anchor] = A_BRIGHT_GREEN, + [fyewt_tag] = A_BRIGHT_GREEN, + [fyewt_linebreak] = A_GREEN, // when visible only + [fyewt_alias] = A_BRIGHT_GREEN, + [fyewt_terminating_zero] = A_BRIGHT_RED, // when visible only + [fyewt_plain_scalar_key] = A_BRIGHT_CYAN, + [fyewt_single_quoted_scalar_key] = A_BRIGHT_CYAN, + [fyewt_double_quoted_scalar_key] = A_BRIGHT_CYAN, + [fyewt_comment] = A_BRIGHT_BLUE, + // extended + [fyewt_indicator_question_mark] = A_MAGENTA, + [fyewt_indicator_colon] = A_MAGENTA, + [fyewt_indicator_dash] = A_MAGENTA, + [fyewt_indicator_left_bracket] = A_MAGENTA, + [fyewt_indicator_right_bracket] = A_MAGENTA, + [fyewt_indicator_left_brace] = A_MAGENTA, + [fyewt_indicator_right_brace] = A_MAGENTA, + [fyewt_indicator_comma] = A_MAGENTA, + [fyewt_indicator_bar] = A_MAGENTA, + [fyewt_indicator_greater] = A_MAGENTA, + [fyewt_indicator_single_quote_start] = A_YELLOW, + [fyewt_indicator_single_quote_end] = A_YELLOW, + [fyewt_indicator_double_quote_start] = A_YELLOW, + [fyewt_indicator_double_quote_end] = A_YELLOW, + [fyewt_indicator_ambersand] = A_BRIGHT_GREEN, + [fyewt_indicator_star] = A_MAGENTA, + [fyewt_indicator_chomp] = A_MAGENTA, + [fyewt_indicator_explicit_indent] = A_MAGENTA, +}; + +static void fy_emitter_fill_default_colors(struct fy_emitter *fye) +{ + unsigned int i; + + for (i = 0; i < FYEWT_COUNT; i++) { + if (!fye->xcfg.colors[i]) + fye->xcfg.colors[i] = default_colors[i]; + } +} + +static FILE *fy_emitter_get_output_fp(struct fy_emitter *fye) +{ + switch (fye->xcfg.xflags & (FYEXCF_OUTPUT_MASK << FYEXCF_OUTPUT_SHIFT)) { + case FYEXCF_OUTPUT_STDOUT: + return stdout; + case FYEXCF_OUTPUT_STDERR: + return stderr; + case FYEXCF_OUTPUT_FILE: + return fye->xcfg.output_fp; + default: + break; + } + return NULL; +} + +static int fy_emitter_get_output_fd(struct fy_emitter *fye) +{ + switch (fye->xcfg.xflags & (FYEXCF_OUTPUT_MASK << FYEXCF_OUTPUT_SHIFT)) { + case FYEXCF_OUTPUT_STDOUT: + return STDOUT_FILENO; + case FYEXCF_OUTPUT_STDERR: + return STDERR_FILENO; + case FYEXCF_OUTPUT_FD: + return fye->xcfg.output_fd; + default: + break; + } + return -1; +} + + +static int fy_emitter_null_output(struct fy_emitter *fye, enum fy_emitter_write_type type, const char *str, int len, void *userdata) +{ + return len; +} + +static inline ssize_t raw_default_output(FILE *fp, int fd, const char *data, size_t len) +{ + ssize_t wrn, twrn; + + if (fp) + return fwrite(data, 1, len, fp); + + wrn = 0; + if (fd >= 0) { + while (len > 0) { + do { + twrn = write(fd, data, len); + } while (twrn == -1 && errno == EAGAIN); + if (twrn <= 0) + return wrn; + data += twrn; + len -= (size_t)twrn; + wrn += twrn; + } + } + return wrn; +} + int fy_emitter_default_output(struct fy_emitter *fye, enum fy_emitter_write_type type, const char *str, int len, void *userdata) { struct fy_emitter_default_output_data d_local, *d; FILE *fp; + int fd; int ret, w; const char *color = NULL; const char *s, *e; + const char *visible; d = userdata; if (!d) { - /* kinda inneficient but should not matter */ d = &d_local; - d->fp = stdout; - d->colorize = isatty(STDOUT_FILENO); - d->visible = false; + fp = fye->output_fp; + fd = fye->output_fd; + d->colorize = fye->output_colorize; + d->visible = !!(fye->xcfg.xflags & FYEXCF_VISIBLE_WS); + } else { + fp = d->fp; + fd = -1; } - fp = d->fp; + + if (!fp && fd < 0) + return len; s = str; e = str + len; + if (d->colorize) { + + color = fye->xcfg.colors[type]; + + /* visible whitespace */ switch (type) { - case fyewt_document_indicator: - color = "\x1b[36m"; - break; - case fyewt_tag_directive: - case fyewt_version_directive: - color = "\x1b[33m"; - break; case fyewt_indent: - if (d->visible) { - fputs("\x1b[32m", fp); - while (s < e && (w = fy_utf8_width_by_first_octet(((uint8_t)*s))) > 0) { - /* open box - U+2423 */ - fputs("\xe2\x90\xa3", fp); - s += w; - } - fputs("\x1b[0m", fp); - return len; - } - break; - case fyewt_indicator: - if (len == 1 && (str[0] == '\'' || str[0] == '"')) - color = "\x1b[33m"; - else if (len == 1 && str[0] == '&') - color = "\x1b[32;1m"; - else - color = "\x1b[35m"; - break; case fyewt_whitespace: - if (d->visible) { - fputs("\x1b[32m", fp); - while (s < e && (w = fy_utf8_width_by_first_octet(((uint8_t)*s))) > 0) { - /* symbol for space - U+2420 */ - /* symbol for interpunct - U+00B7 */ - fputs("\xc2\xb7", fp); - s += w; - } - fputs("\x1b[0m", fp); - return len; - } - break; - case fyewt_plain_scalar: - color = "\x1b[37;1m"; - break; - case fyewt_single_quoted_scalar: - case fyewt_double_quoted_scalar: - color = "\x1b[33m"; - break; - case fyewt_literal_scalar: - case fyewt_folded_scalar: - color = "\x1b[33m"; - break; - case fyewt_anchor: - case fyewt_tag: - case fyewt_alias: - color = "\x1b[32;1m"; - break; case fyewt_linebreak: - if (d->visible) { - fputs("\x1b[32m", fp); - while (s < e && (w = fy_utf8_width_by_first_octet(((uint8_t)*s))) > 0) { - /* symbol for space - ^M */ - /* fprintf(fp, "^M\n"); */ - /* down arrow - U+2193 */ - fputs("\xe2\x86\x93\n", fp); - s += w; - } - fputs("\x1b[0m", fp); - return len; - } - color = NULL; - break; case fyewt_terminating_zero: - color = NULL; - break; - case fyewt_plain_scalar_key: - case fyewt_single_quoted_scalar_key: - case fyewt_double_quoted_scalar_key: - color = "\x1b[36;1m"; - break; - case fyewt_comment: - color = "\x1b[34;1m"; + + /* if not visible don't bother */ + if (!d->visible) { + color = NULL; + break; + } + + if (color) + raw_default_output(fp, fd, color, strlen(color)); + + switch (type) { + case fyewt_indent: + /* open box - U+2423 */ + visible = "\xe2\x90\xa3"; + break; + case fyewt_whitespace: + /* symbol for space - U+2420 */ + /* symbol for interpunct - U+00B7 */ + visible = "\xc2\xb7"; + break; + case fyewt_linebreak: + /* down arrow - U+2193 */ + visible = "\xe2\x86\x93" "\n"; + break; + case fyewt_terminating_zero: + visible = "\\0"; + break; + default: + visible = ""; /* should never happen */ + break; + + } + + while (s < e && (w = fy_utf8_width_by_first_octet(((uint8_t)*s))) > 0) { + raw_default_output(fp, fd, visible, strlen(visible)); + s += w; + } + + if (color) + raw_default_output(fp, fd, A_RESET, strlen(A_RESET)); + return len; + + default: break; } - } + } else + color = NULL; /* don't output the terminating zero */ if (type == fyewt_terminating_zero) return len; if (color) - fputs(color, fp); + raw_default_output(fp, fd, color, strlen(color)); - ret = fwrite(str, 1, len, fp); + ret = raw_default_output(fp, fd, str, len); if (color) - fputs("\x1b[0m", fp); + raw_default_output(fp, fd, A_RESET, strlen(A_RESET)); return ret; } @@ -3586,3 +3915,33 @@ err_emit: err_setup: return -1; } + +int fy_emit_body_node(struct fy_emitter *emit, struct fy_node *fyn) +{ + struct fy_document_iterator *fydi = NULL; + struct fy_event *fye; + struct fy_document_iterator_cfg cfg; + int rc; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYDICF_WANT_BODY_EVENTS; + cfg.fyd = fyn->fyd; + cfg.iterate_root = fyn; + + fydi = fy_document_iterator_create_cfg(&cfg); + if (!fydi) + goto err_out; + + while ((fye = fy_document_iterator_generate_next(fydi)) != NULL) { + rc = fy_emit_event(emit, fye); + if (rc) + goto err_out; + } + + fy_document_iterator_destroy(fydi); + return 0; + +err_out: + fy_document_iterator_destroy(fydi); + return -1; +} diff --git a/src/lib/fy-emit.h b/src/lib/fy-emit.h index 6afe399..5d23377 100644 --- a/src/lib/fy-emit.h +++ b/src/lib/fy-emit.h @@ -71,6 +71,7 @@ struct fy_emit_save_ctx { #define DDNF_FLOW 0x0010 #define DDNF_INDENTLESS 0x0020 #define DDNF_SIMPLE_SCALAR_KEY 0x0040 +#define DDNF_HANGING_INDENT 0x0080 struct fy_emitter { int line; @@ -84,7 +85,11 @@ struct fy_emitter { bool suppress_recycling : 1; /* current document */ - struct fy_emitter_cfg cfg; /* yeah, it isn't worth just to save a few bytes */ + struct fy_emitter_xcfg xcfg; + FILE *output_fp; + int output_fd; + bool output_colorize; + struct fy_document *fyd; struct fy_document_state *fyds; /* fyd->fyds when fyd != NULL */ struct fy_emit_accum ea; @@ -149,4 +154,10 @@ fy_emit_output_accum(struct fy_emitter *emit, enum fy_emitter_write_type type, s fy_emit_accum_reset(ea); } +static inline void +fy_emit_output_col_sync(struct fy_emitter *emit, struct fy_emit_accum *ea) +{ + ea->col += emit->column; +} + #endif diff --git a/src/lib/fy-event.c b/src/lib/fy-event.c index 42beab1..f042713 100644 --- a/src/lib/fy-event.c +++ b/src/lib/fy-event.c @@ -169,6 +169,9 @@ void fy_parser_event_free(struct fy_parser *fyp, struct fy_event *fye) if (!fyp || !fye) return; + if (fy_reader_generates_events(fyp->reader)) + return fy_reader_event_free(fyp->reader, fye); + fyep = container_of(fye, struct fy_eventp, e); fy_parse_eventp_recycle(fyp, fyep); diff --git a/src/lib/fy-input-diag.c b/src/lib/fy-input-diag.c new file mode 100644 index 0000000..46baf03 --- /dev/null +++ b/src/lib/fy-input-diag.c @@ -0,0 +1,92 @@ +/* + * fy-input-diag.c - input and reader diagnostics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-diag.h" +#include "fy-input.h" + +int fy_reader_vdiag(struct fy_reader *fyr, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap) +{ + struct fy_diag_ctx fydc; + int fydc_level, fyd_level; + + if (!fyr || !fyr->diag || !fmt) + return -1; + + /* perform the enable tests early to avoid the overhead */ + fydc_level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; + fyd_level = fyr->diag->cfg.level; + + if (fydc_level < fyd_level) + return 0; + + /* fill in fy_diag_ctx */ + memset(&fydc, 0, sizeof(fydc)); + + fydc.level = fydc_level; + fydc.module = FYEM_SCAN; /* reader is always scanner */ + fydc.source_file = file; + fydc.source_line = line; + fydc.source_func = func; + fydc.line = fyr->line; + fydc.column = fyr->column; + + return fy_vdiag(fyr->diag, &fydc, fmt, ap); +} + +int fy_reader_diag(struct fy_reader *fyr, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_reader_vdiag(fyr, flags, file, line, func, fmt, ap); + va_end(ap); + + return rc; +} + +void fy_reader_diag_vreport(struct fy_reader *fyr, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap) +{ + if (!fyr || !fyr->diag || !fydrc || !fmt) + return; + + fy_diag_vreport(fyr->diag, fydrc, fmt, ap); +} + +void fy_reader_diag_report(struct fy_reader *fyr, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_reader_diag_vreport(fyr, fydrc, fmt, ap); + va_end(ap); +} diff --git a/src/lib/fy-input.c b/src/lib/fy-input.c index cc29ae8..6095cfb 100644 --- a/src/lib/fy-input.c +++ b/src/lib/fy-input.c @@ -129,7 +129,7 @@ static void fy_input_from_data_setup(struct fy_input *fyi, handle->end_mark.line = 0; handle->end_mark.column = fy_utf8_count(data, size); /* if it's plain, all is good */ - if (simple || (aflags & FYACF_FLOW_PLAIN)) { + if ((simple || (aflags & FYACF_FLOW_PLAIN)) && !(aflags & FYACF_LB)) { handle->storage_hint = size; /* maximum */ handle->storage_hint_valid = false; handle->direct_output = !!(aflags & FYACF_JSON_ESCAPE); @@ -159,6 +159,7 @@ static void fy_input_from_data_setup(struct fy_input *fyi, handle->json_mode = false; /* XXX hardcoded */ handle->lb_mode = fylb_cr_nl; handle->fws_mode = fyfws_space_tab; + handle->directive0_mode = false; out: fyi->state = FYIS_PARSED; } @@ -255,6 +256,10 @@ void fy_input_close(struct fy_input *fyi) /* nothing */ break; + case fyit_dociter: + /* nothing */ + break; + default: break; } @@ -299,6 +304,7 @@ ssize_t fy_input_estimate_queued_size(const struct fy_input *fyi) break; case fyit_callback: + case fyit_dociter: default: return SSIZE_MAX; /* cannot determine */ } @@ -353,7 +359,6 @@ void fy_reader_reset(struct fy_reader *fyr) fyr->mode = fyrm_yaml; fyr->ops = ops; fyr->diag = diag; - fyr->current_c = -1; } void fy_reader_setup(struct fy_reader *fyr, const struct fy_reader_ops *ops) @@ -389,16 +394,19 @@ void fy_reader_apply_mode(struct fy_reader *fyr) fyr->json_mode = false; fyr->lb_mode = fylb_cr_nl; fyr->fws_mode = fyfws_space_tab; + fyr->directive0_mode = false; break; case fyrm_json: fyr->json_mode = true; fyr->lb_mode = fylb_cr_nl; fyr->fws_mode = fyfws_space; + fyr->directive0_mode = false; break; case fyrm_yaml_1_1: fyr->json_mode = false; fyr->lb_mode = fylb_cr_nl_N_L_P; fyr->fws_mode = fyfws_space_tab; + fyr->directive0_mode = true; break; } fyi = fyr->current_input; @@ -406,12 +414,14 @@ void fy_reader_apply_mode(struct fy_reader *fyr) fyi->json_mode = fyr->json_mode; fyi->lb_mode = fyr->lb_mode; fyi->fws_mode = fyr->fws_mode; + fyi->directive0_mode = fyr->directive0_mode; } } int fy_reader_input_open(struct fy_reader *fyr, struct fy_input *fyi, const struct fy_reader_input_cfg *icfg) { struct stat sb; + struct fy_document_iterator *fydi; int rc; if (!fyi) @@ -454,7 +464,7 @@ int fy_reader_input_open(struct fy_reader *fyr, struct fy_input *fyi, const stru "bad file.fd %d", fyi->cfg.fd.fd); break; default: - assert(0); // will never happen + FY_IMPOSSIBLE_ABORT(); } rc = fstat(fyi->fd, &sb); @@ -505,9 +515,14 @@ int fy_reader_input_open(struct fy_reader *fyr, struct fy_input *fyi, const stru case fyit_callback: break; - default: - assert(0); + case fyit_dociter: + fydi = fyi->cfg.dociter.fydi; + fyr_error_check(fyr, fydi, err_out, + "missing document iterator"); break; + + default: + FY_IMPOSSIBLE_ABORT(); } switch (fyi->cfg.type) { @@ -515,6 +530,7 @@ int fy_reader_input_open(struct fy_reader *fyr, struct fy_input *fyi, const stru /* those two need no memory */ case fyit_memory: case fyit_alloc: + case fyit_dociter: break; /* all the rest need it */ @@ -535,13 +551,11 @@ int fy_reader_input_open(struct fy_reader *fyr, struct fy_input *fyi, const stru } fyr->this_input_start = 0; - fyr->current_input_pos = 0; fyr->line = 0; fyr->column = 0; - fyr->current_c = -1; fyr->current_ptr = NULL; - fyr->current_w = 0; - fyr->current_left = 0; + fyr->current_ptr_start = NULL; + fyr->current_ptr_end = NULL; fyi->state = FYIS_PARSE_IN_PROGRESS; @@ -556,6 +570,7 @@ int fy_reader_input_done(struct fy_reader *fyr) { struct fy_input *fyi; void *buf; + size_t current_input_pos; if (!fyr) return -1; @@ -564,6 +579,8 @@ int fy_reader_input_done(struct fy_reader *fyr) if (!fyi) return 0; + current_input_pos = fy_reader_current_input_pos(fyr); + switch (fyi->cfg.type) { case fyit_file: case fyit_fd: @@ -575,14 +592,18 @@ int fy_reader_input_done(struct fy_reader *fyr) case fyit_stream: case fyit_callback: /* chop extra buffer */ - buf = realloc(fyi->buffer, fyr->current_input_pos); - fyr_error_check(fyr, buf || !fyr->current_input_pos, err_out, + buf = realloc(fyi->buffer, current_input_pos); + fyr_error_check(fyr, buf || !current_input_pos, err_out, "realloc() failed"); - fyi->buffer = buf; - fyi->allocated = fyr->current_input_pos; /* increate input generation; required for direct input to work */ - fyi->generation++; + if (fyi->buffer != buf) { + fyi->buffer = buf; + fyi->generation++; + } + + fyi->allocated = current_input_pos; + break; default: @@ -603,6 +624,7 @@ err_out: int fy_reader_input_scan_token_mark_slow_path(struct fy_reader *fyr) { struct fy_input *fyi, *fyi_new = NULL; + size_t current_input_pos; assert(fyr); @@ -634,21 +656,24 @@ int fy_reader_input_scan_token_mark_slow_path(struct fy_reader *fyr) fyi_new->lb_mode = fyi->lb_mode; fyi_new->fws_mode = fyi->fws_mode; + fyi_new->directive0_mode = fyi->directive0_mode; fyi_new->state = FYIS_PARSE_IN_PROGRESS; /* adjust and copy the left over reads */ - assert(fyi->read >= fyr->current_input_pos); - fyi_new->read = fyi->read - fyr->current_input_pos; + current_input_pos = fy_reader_current_input_pos(fyr); + assert(fyi->read >= current_input_pos); + fyi_new->read = fyi->read - current_input_pos; if (fyi_new->read > 0) - memcpy(fyi_new->buffer, fyi->buffer + fyr->current_input_pos, fyi_new->read); + memcpy(fyi_new->buffer, fyi->buffer + current_input_pos, fyi_new->read); - fyr->this_input_start += fyr->current_input_pos; + fyr->this_input_start += current_input_pos; /* update the reader to point to the new input */ fyr->current_input = fyi_new; - fyr->current_input_pos = 0; fyr->current_ptr = fyi_new->buffer; + fyr->current_ptr_start = fyi_new->buffer; + fyr->current_ptr_end = fyr->current_ptr_start + fyi_new->read; fyr_debug(fyr, "chop at this_input_start=%zu chop=%zu\n", fyr->this_input_start, fyi->chop); @@ -666,12 +691,12 @@ err_out: const void *fy_reader_ptr_slow_path(struct fy_reader *fyr, size_t *leftp) { struct fy_input *fyi; - const void *p; - size_t left; + const void *p, *start; + size_t size, left, current_input_pos; if (fyr->current_ptr) { if (leftp) - *leftp = fyr->current_left; + *leftp = fy_reader_current_left(fyr); return fyr->current_ptr; } @@ -680,49 +705,83 @@ const void *fy_reader_ptr_slow_path(struct fy_reader *fyr, size_t *leftp) return NULL; /* tokens cannot cross boundaries */ - switch (fyi->cfg.type) { - case fyit_file: - case fyit_fd: - if (fyi->addr) { - left = fyi->length - (fyr->this_input_start + fyr->current_input_pos); - p = fyi->addr + fyr->current_input_pos; - break; - } + start = fy_input_start_size(fyi, &size); - /* fall-through */ + current_input_pos = fy_reader_current_input_pos(fyr); + left = size - current_input_pos; + assert(left <= size); - case fyit_stream: - case fyit_callback: - left = fyi->read - (fyr->this_input_start + fyr->current_input_pos); - p = fyi->buffer + fyr->current_input_pos; - break; + p = start + current_input_pos; - case fyit_memory: - left = fyi->cfg.memory.size - fyr->current_input_pos; - p = fyi->cfg.memory.data + fyr->current_input_pos; - break; + if (leftp) + *leftp = left; - case fyit_alloc: - left = fyi->cfg.alloc.size - fyr->current_input_pos; - p = fyi->cfg.alloc.data + fyr->current_input_pos; - break; + if (!fyr->current_ptr_start) { + fyr->current_ptr_start = start; + fyr->current_ptr_end = start + size; + } + fyr->current_ptr = p; - default: - assert(0); /* no streams */ - p = NULL; - left = 0; - break; + assert(current_input_pos <= size); + + return p; +} + +int fy_reader_peek_at_offset_width_slow_path(struct fy_reader *fyr, size_t offset, int *wp) +{ + const uint8_t *p; + size_t left; + int w; + + assert(fyr); + + /* ensure that the first octet at least is pulled in */ + p = fy_reader_ensure_lookahead(fyr, offset + 1, &left); + if (!p) + return FYUG_EOF; + + /* get width by first octet */ + w = fy_utf8_width_by_first_octet(p[offset]); + if (!w) + return FYUG_INV; + + /* make sure that there's enough to cover the utf8 width */ + if (offset + w > left) { + p = fy_reader_ensure_lookahead(fyr, offset + w, &left); + if (!p) + return FYUG_PARTIAL; } - if (leftp) - *leftp = left; + return fy_utf8_get(p + offset, left - offset, wp); +} - fyr->current_ptr = p; - fyr->current_left = left; - fyr->current_c = fy_utf8_get(p, left, &fyr->current_w); +int64_t fy_reader_peek_at_offset_width_slow_path_64(struct fy_reader *fyr, size_t offset) +{ + const uint8_t *p; + size_t left; + int w; - return p; + assert(fyr); + + /* ensure that the first octet at least is pulled in */ + p = fy_reader_ensure_lookahead(fyr, offset + 1, &left); + if (!p) + return FYUG_EOF; + + /* get width by first octet */ + w = fy_utf8_width_by_first_octet(p[offset]); + if (!w) + return FYUG_INV; + + /* make sure that there's enough to cover the utf8 width */ + if (offset + w > left) { + p = fy_reader_ensure_lookahead(fyr, offset + w, &left); + if (!p) + return FYUG_PARTIAL; + } + + return fy_utf8_get_64(p + offset, left - offset); } const void *fy_reader_input_try_pull(struct fy_reader *fyr, struct fy_input *fyi, @@ -730,7 +789,7 @@ const void *fy_reader_input_try_pull(struct fy_reader *fyr, struct fy_input *fyi { const void *p; size_t left, pos, size, nread, nreadreq, missing; - ssize_t snread; + ssize_t snread, current_input_pos; size_t space __FY_DEBUG_UNUSED__; void *buf; @@ -742,7 +801,9 @@ const void *fy_reader_input_try_pull(struct fy_reader *fyr, struct fy_input *fyi p = NULL; left = 0; - pos = fyr->current_input_pos; + + current_input_pos = fy_reader_current_input_pos(fyr); + pos = current_input_pos; switch (fyi->cfg.type) { case fyit_file: @@ -807,10 +868,13 @@ const void *fy_reader_input_try_pull(struct fy_reader *fyr, struct fy_input *fyi fyr_debug(fyr, "input read allocated=%zu new-size=%zu", fyi->allocated, size); - fyi->buffer = buf; - fyi->allocated = size; - fyi->generation++; + /* increase input generation; required for direct input to work */ + if (fyi->buffer != buf) { + fyi->buffer = buf; + fyi->generation++; + } + fyi->allocated = size; space = fyi->allocated - pos; p = fyi->buffer + pos; } @@ -929,10 +993,11 @@ const void *fy_reader_input_try_pull(struct fy_reader *fyr, struct fy_input *fyi p = fyi->cfg.alloc.data + pos; break; + case fyit_dociter: + FY_IMPOSSIBLE_ABORT(); default: - assert(0); - break; + FY_IMPOSSIBLE_ABORT(); } if (leftp) @@ -949,11 +1014,11 @@ void fy_reader_advance_slow_path(struct fy_reader *fyr, int c) { bool is_line_break = false; - size_t w; - /* skip this character (optimize case of being the current) */ - w = c == fyr->current_c ? (size_t)fyr->current_w : fy_utf8_width(c); - fy_reader_advance_octets(fyr, w); + if (c < 0) + return; + + fy_reader_advance_octets(fyr, fy_utf8_width(c)); /* first check for CR/LF */ if (c == '\r' && fy_reader_peek(fyr) == '\n') { @@ -1023,9 +1088,15 @@ struct fy_input *fy_input_create(const struct fy_input_cfg *fyic) if (ret == -1) fyi->name = NULL; break; - default: - assert(0); + + case fyit_dociter: + ret = asprintf(&fyi->name, "", fyi->cfg.dociter.fydi); + if (ret == -1) + fyi->name = NULL; break; + + default: + FY_IMPOSSIBLE_ABORT(); } if (!fyi->name) goto err_out; @@ -1043,6 +1114,7 @@ struct fy_input *fy_input_create(const struct fy_input_cfg *fyic) /* default modes */ fyi->lb_mode = fylb_cr_nl; fyi->fws_mode = fyfws_space_tab; + fyi->directive0_mode = false; return fyi; @@ -1054,8 +1126,8 @@ err_out: /* ensure that there are at least size octets available */ const void *fy_reader_ensure_lookahead_slow_path(struct fy_reader *fyr, size_t size, size_t *leftp) { - const void *p; - size_t left; + const void *p, *new_start, *old_start; + size_t left, new_size, old_size; if (!leftp) leftp = &left; @@ -1063,19 +1135,87 @@ const void *fy_reader_ensure_lookahead_slow_path(struct fy_reader *fyr, size_t s p = fy_reader_ptr(fyr, leftp); if (!p || *leftp < size) { + if (!fyr->current_input) { + fyr_debug(fyr, "ensure lookahead size=%zd left=%zd no input left", + size, *leftp); + return NULL; + } fyr_debug(fyr, "ensure lookahead size=%zd left=%zd (%s - %zu)", size, *leftp, fy_input_get_filename(fyr->current_input), - fyr->current_input_pos); + fy_reader_current_input_pos(fyr)); + + /* update with what is new */ + old_start = fy_input_start_size(fyr->current_input, &old_size); p = fy_reader_input_try_pull(fyr, fyr->current_input, size, leftp); if (!p || *leftp < size) return NULL; + /* update with what is new */ + new_start = fy_input_start_size(fyr->current_input, &new_size); + + /* oops, input characteristics changed */ + if (old_start != new_start || old_size != new_size) { + + /* buffer just grew in place */ + if (old_start == new_start) { + fyr_debug(fyr, "buffer grew %zu -> %zu", + old_size, new_size); + } else { + fyr_debug(fyr, "buffer changed %p/%zu -> %p/%zu", + old_start, old_size, new_start, new_size); + + fyr->current_ptr_start = new_start; + } + + fyr->current_ptr_end = fyr->current_ptr_start + new_size; + } + fyr->current_ptr = p; - fyr->current_left = *leftp; - fyr->current_c = fy_utf8_get(fyr->current_ptr, fyr->current_left, &fyr->current_w); } return p; } +struct fy_event *fy_reader_generate_next_event(struct fy_reader *fyr) +{ + struct fy_input *fyi; + struct fy_document_iterator *fydi; + struct fy_event *fye; + + fyi = fy_reader_current_input(fyr); + if (!fyi || fyi->cfg.type != fyit_dociter) + return NULL; + + fydi = fyi->cfg.dociter.fydi; + + do { + fye = fy_document_iterator_generate_next(fydi); + if (!fye) + return NULL; + + /* remove if we don't want the event */ + if (((fye->type == FYET_STREAM_START || fye->type == FYET_STREAM_END) && + !(fyi->cfg.dociter.flags & FYPEGF_GENERATE_STREAM_EVENTS)) || + ((fye->type == FYET_DOCUMENT_START || fye->type == FYET_DOCUMENT_END) && + !(fyi->cfg.dociter.flags & FYPEGF_GENERATE_DOCUMENT_EVENTS))) { + fy_document_iterator_event_free(fydi, fye); + fye = NULL; + } + } while (!fye); + + return fye; +} + +void fy_reader_event_free(struct fy_reader *fyr, struct fy_event *fye) +{ + struct fy_input *fyi; + + fyi = fy_reader_current_input(fyr); + assert(fyi); + assert(fy_reader_generates_events(fyr)); + + assert (fyi->cfg.type == fyit_dociter); + + fy_document_iterator_event_free(fyi->cfg.dociter.fydi, fye); +} diff --git a/src/lib/fy-input.h b/src/lib/fy-input.h index 184590e..a401af2 100644 --- a/src/lib/fy-input.h +++ b/src/lib/fy-input.h @@ -24,6 +24,9 @@ #include "fy-typelist.h" #include "fy-ctype.h" +#include "fy-atom.h" +#include "fy-diag.h" + struct fy_atom; struct fy_parser; @@ -34,6 +37,7 @@ enum fy_input_type { fyit_alloc, fyit_callback, fyit_fd, + fyit_dociter, }; struct fy_input_cfg { @@ -66,6 +70,12 @@ struct fy_input_cfg { struct { int fd; } fd; + struct { + enum fy_parser_event_generator_flags flags; + struct fy_document_iterator *fydi; + struct fy_document *fyd; + bool owns_iterator; + } dociter; }; }; @@ -100,70 +110,64 @@ struct fy_input { bool json_mode; enum fy_lb_mode lb_mode; enum fy_flow_ws_mode fws_mode; + bool directive0_mode; }; FY_TYPE_DECL_LIST(input); -static inline const void *fy_input_start(const struct fy_input *fyi) -{ - const void *ptr = NULL; - - switch (fyi->cfg.type) { - case fyit_file: - if (fyi->addr) { - ptr = fyi->addr; - break; - } - /* fall-through */ - - case fyit_stream: - case fyit_callback: - ptr = fyi->buffer; - break; - - case fyit_memory: - ptr = fyi->cfg.memory.data; - break; - - case fyit_alloc: - ptr = fyi->cfg.alloc.data; - break; - - default: - break; - } - assert(ptr); - return ptr; -} - -static inline size_t fy_input_size(const struct fy_input *fyi) +static inline const void * +fy_input_start_size(const struct fy_input *fyi, size_t *sizep) { + const void *start; size_t size; + /* tokens cannot cross boundaries */ switch (fyi->cfg.type) { case fyit_file: + case fyit_fd: if (fyi->addr) { + start = fyi->addr; size = fyi->length; break; } + /* fall-through */ case fyit_stream: case fyit_callback: + start = fyi->buffer; size = fyi->read; break; case fyit_memory: + start = fyi->cfg.memory.data; size = fyi->cfg.memory.size; break; case fyit_alloc: + start = fyi->cfg.alloc.data; size = fyi->cfg.alloc.size; break; default: + start = NULL; size = 0; break; } + + *sizep = size; + return start; +} + +static inline const void *fy_input_start(const struct fy_input *fyi) +{ + size_t size; + return fy_input_start_size(fyi, &size); +} + +static inline size_t fy_input_size(const struct fy_input *fyi) +{ + size_t size; + (void)fy_input_start_size(fyi, &size); return size; } @@ -241,11 +245,9 @@ struct fy_reader { struct fy_input *current_input; size_t this_input_start; /* this input start */ - size_t current_input_pos; /* from start of input */ const void *current_ptr; /* current pointer into the buffer */ - int current_c; /* current utf8 character at current_ptr (-1 if not cached) */ - int current_w; /* current utf8 character width */ - size_t current_left; /* currently left characters into the buffer */ + const void *current_ptr_start; /* the start of this input */ + const void *current_ptr_end; /* the end of this input */ int line; /* always on input */ int column; @@ -258,6 +260,7 @@ struct fy_reader { bool json_mode; enum fy_lb_mode lb_mode; enum fy_flow_ws_mode fws_mode; + bool directive0_mode; }; void fy_reader_reset(struct fy_reader *fyr); @@ -268,6 +271,18 @@ int fy_reader_input_open(struct fy_reader *fyr, struct fy_input *fyi, const stru int fy_reader_input_done(struct fy_reader *fyr); int fy_reader_input_scan_token_mark_slow_path(struct fy_reader *fyr); +static FY_ALWAYS_INLINE inline size_t +fy_reader_current_input_pos(const struct fy_reader *fyr) +{ + return (size_t)(fyr->current_ptr - fyr->current_ptr_start); +} + +static FY_ALWAYS_INLINE inline size_t +fy_reader_current_left(const struct fy_reader *fyr) +{ + return (size_t)(fyr->current_ptr_end - fyr->current_ptr); +} + static inline bool fy_reader_input_chop_active(struct fy_reader *fyr) { @@ -302,7 +317,7 @@ fy_reader_input_scan_token_mark(struct fy_reader *fyr) { /* don't chop until ready */ if (!fy_reader_input_chop_active(fyr) || - fyr->current_input->chop > fyr->current_input_pos) + fyr->current_input->chop > fy_reader_current_input_pos(fyr)) return 0; return fy_reader_input_scan_token_mark_slow_path(fyr); @@ -328,6 +343,19 @@ fy_reader_set_mode(struct fy_reader *fyr, enum fy_reader_mode mode) fy_reader_apply_mode(fyr); } +static FY_ALWAYS_INLINE inline enum fy_reader_mode +fy_reader_calculate_mode(const struct fy_version *vers, bool json_mode) +{ + enum fy_reader_mode mode; + + if (!json_mode) + mode = fy_version_compare(vers, fy_version_make(1, 1)) <= 0 ? fyrm_yaml_1_1 : fyrm_yaml; + else + mode = fyrm_json; + + return mode; +} + static FY_ALWAYS_INLINE inline struct fy_input * fy_reader_current_input(const struct fy_reader *fyr) { @@ -364,6 +392,21 @@ fy_reader_line(const struct fy_reader *fyr) return fyr->line; } +static FY_ALWAYS_INLINE inline bool +fy_reader_generates_events(const struct fy_reader *fyr) +{ + struct fy_input *fyi; + + fyi = fy_reader_current_input(fyr); + if (!fyi) + return false; + + return fyi->cfg.type == fyit_dociter; +} + +struct fy_event *fy_reader_generate_next_event(struct fy_reader *fyr); +void fy_reader_event_free(struct fy_reader *fyr, struct fy_event *fye); + /* force new line at the end of stream */ static inline void fy_reader_stream_end(struct fy_reader *fyr) { @@ -380,7 +423,7 @@ static FY_ALWAYS_INLINE inline void fy_reader_get_mark(struct fy_reader *fyr, struct fy_mark *fym) { assert(fyr); - fym->input_pos = fyr->current_input_pos; + fym->input_pos = fy_reader_current_input_pos(fyr); fym->line = fyr->line; fym->column = fyr->column; } @@ -390,7 +433,7 @@ fy_reader_ptr(struct fy_reader *fyr, size_t *leftp) { if (fyr->current_ptr) { if (leftp) - *leftp = fyr->current_left; + *leftp = fy_reader_current_left(fyr); return fyr->current_ptr; } @@ -418,6 +461,13 @@ fy_reader_flow_ws_mode(const struct fy_reader *fyr) return fyr->fws_mode; } +static FY_ALWAYS_INLINE inline enum fy_flow_ws_mode +fy_reader_directive0_mode(const struct fy_reader *fyr) +{ + assert(fyr); + return fyr->directive0_mode; +} + static FY_ALWAYS_INLINE inline bool fy_reader_is_lb(const struct fy_reader *fyr, int c) { @@ -476,9 +526,10 @@ fy_reader_is_flow_blankz(const struct fy_reader *fyr, int c) static FY_ALWAYS_INLINE inline const void * fy_reader_ensure_lookahead(struct fy_reader *fyr, size_t size, size_t *leftp) { - if (fyr->current_ptr && fyr->current_left >= size) { - if (leftp) - *leftp = fyr->current_left; + size_t current_left = fy_reader_current_left(fyr); + + if (current_left >= size) { + *leftp = current_left; return fyr->current_ptr; } return fy_reader_ensure_lookahead_slow_path(fyr, size, leftp); @@ -489,69 +540,83 @@ static inline int fy_reader_strncmp(struct fy_reader *fyr, const char *str, size_t n) { const char *p; + size_t len; int ret; assert(fyr); - p = fy_reader_ensure_lookahead(fyr, n, NULL); + p = fy_reader_ensure_lookahead(fyr, n, &len); if (!p) return -1; + + /* not enough? */ + if (n > len) + return 1; + ret = strncmp(p, str, n); return ret ? 1 : 0; } +int fy_reader_peek_at_offset_width_slow_path(struct fy_reader *fyr, size_t offset, int *wp); + static FY_ALWAYS_INLINE inline int -fy_reader_peek_at_offset(struct fy_reader *fyr, size_t offset) +fy_reader_peek_at_offset_width(struct fy_reader *fyr, size_t offset, int *wp) { - const uint8_t *p; - size_t left; - int w; + assert(fyr); + + if ((ssize_t)(fy_reader_current_left(fyr) - offset) >= 4) + return fy_utf8_get(fyr->current_ptr + offset, 4, wp); + + return fy_reader_peek_at_offset_width_slow_path(fyr, offset, wp); +} + +int64_t fy_reader_peek_at_offset_width_slow_path_64(struct fy_reader *fyr, size_t offset); +static FY_ALWAYS_INLINE inline int64_t +fy_reader_peek_at_offset_width_64(struct fy_reader *fyr, size_t offset) +{ assert(fyr); - if (offset == 0 && fyr->current_c >= 0) - return fyr->current_c; - /* ensure that the first octet at least is pulled in */ - p = fy_reader_ensure_lookahead(fyr, offset + 1, &left); - if (!p) - return FYUG_EOF; - - /* get width by first octet */ - w = fy_utf8_width_by_first_octet(p[offset]); - if (!w) - return FYUG_INV; - - /* make sure that there's enough to cover the utf8 width */ - if (offset + w > left) { - p = fy_reader_ensure_lookahead(fyr, offset + w, &left); - if (!p) - return FYUG_PARTIAL; - } + if ((ssize_t)(fy_reader_current_left(fyr) - offset) >= 4) + return fy_utf8_get_64(fyr->current_ptr + offset, 4); - return fy_utf8_get(p + offset, left - offset, &w); + return fy_reader_peek_at_offset_width_slow_path_64(fyr, offset); } + static FY_ALWAYS_INLINE inline int -fy_reader_peek_at_internal(struct fy_reader *fyr, int pos, ssize_t *offsetp) +fy_reader_peek_at_offset(struct fy_reader *fyr, size_t offset) { - int i, c; + int w; + + return fy_reader_peek_at_offset_width(fyr, offset, &w); +} + +static FY_ALWAYS_INLINE inline int +fy_reader_peek_at_width_internal(struct fy_reader *fyr, int pos, ssize_t *offsetp, int *wp) +{ + int i, c, w; size_t offset; - assert(fyr); - if (!offsetp || *offsetp < 0) { - for (i = 0, offset = 0; i < pos; i++, offset += fy_utf8_width(c)) { - c = fy_reader_peek_at_offset(fyr, offset); + offset = (size_t)*offsetp; + if ((ssize_t)offset < 0) { + for (i = 0, offset = 0; i < pos; i++, offset += w) { + c = fy_reader_peek_at_offset_width(fyr, offset, &w); if (c < 0) return c; } - } else - offset = (size_t)*offsetp; + } + c = fy_reader_peek_at_offset_width(fyr, offset, wp); - c = fy_reader_peek_at_offset(fyr, offset); + *offsetp = offset + *wp; + return c; +} - if (offsetp) - *offsetp = offset + fy_utf8_width(c); +static FY_ALWAYS_INLINE inline int +fy_reader_peek_at_internal(struct fy_reader *fyr, int pos, ssize_t *offsetp) +{ + int w; - return c; + return fy_reader_peek_at_width_internal(fyr, pos, offsetp, &w); } static FY_ALWAYS_INLINE inline bool @@ -566,19 +631,49 @@ fy_reader_is_blankz_at_offset(struct fy_reader *fyr, size_t offset) return fy_reader_is_blankz(fyr, fy_reader_peek_at_offset(fyr, offset)); } +static FY_ALWAYS_INLINE inline int +fy_reader_peek_at_width(struct fy_reader *fyr, int pos, int *wp) +{ + int i, c, w; + size_t offset; + + assert(fyr); + for (i = 0, offset = 0; i < pos; i++, offset += w) { + c = fy_reader_peek_at_offset_width(fyr, offset, &w); + if (c < 0) + return c; + } + + return fy_reader_peek_at_offset_width(fyr, offset, wp); +} + static FY_ALWAYS_INLINE inline int fy_reader_peek_at(struct fy_reader *fyr, int pos) { - return fy_reader_peek_at_internal(fyr, pos, NULL); + int i, c, w; + size_t offset; + + assert(fyr); + for (i = 0, offset = 0; i < pos; i++, offset += w) { + c = fy_reader_peek_at_offset_width(fyr, offset, &w); + if (c < 0) + return c; + } + + return fy_reader_peek_at_offset(fyr, offset); } static FY_ALWAYS_INLINE inline int -fy_reader_peek(struct fy_reader *fyr) +fy_reader_peek_width(struct fy_reader *fyr, int *wp) { - if (fyr->current_c >= 0) - return fyr->current_c; + return fy_reader_peek_at_offset_width(fyr, 0, wp); +} - return fy_reader_peek_at_offset(fyr, 0); +static FY_ALWAYS_INLINE inline int +fy_reader_peek(struct fy_reader *fyr) +{ + int w; + return fy_reader_peek_width(fyr, &w); } static FY_ALWAYS_INLINE inline const void * @@ -600,14 +695,7 @@ fy_reader_peek_block(struct fy_reader *fyr, size_t *lenp) static FY_ALWAYS_INLINE inline void fy_reader_advance_octets(struct fy_reader *fyr, size_t advance) { - assert(fyr); - assert(fyr->current_left >= advance); - - fyr->current_input_pos += advance; fyr->current_ptr += advance; - fyr->current_left -= advance; - - fyr->current_c = fy_utf8_get(fyr->current_ptr, fyr->current_left, &fyr->current_w); } void fy_reader_advance_slow_path(struct fy_reader *fyr, int c); @@ -621,12 +709,29 @@ fy_reader_advance_printable_ascii(struct fy_reader *fyr, int c) } static FY_ALWAYS_INLINE inline void -fy_reader_advance(struct fy_reader *fyr, int c) +fy_reader_update_state_lb_mode(struct fy_reader *fyr, const int c, const enum fy_lb_mode lb_mode) { - if (fy_utf8_is_printable_ascii(c)) - fy_reader_advance_printable_ascii(fyr, c); + if (fy_is_lb_m(c, lb_mode)) { + fyr->column = 0; + fyr->line++; + } else if (fy_is_tab(c) && fyr->tabsize) + fyr->column += (fyr->tabsize - (fyr->column % fyr->tabsize)); else - fy_reader_advance_slow_path(fyr, c); + fyr->column++; +} + +static FY_ALWAYS_INLINE inline void +fy_reader_advance_lb_mode(struct fy_reader *fyr, const int c, const enum fy_lb_mode lb_mode) +{ + assert(fy_utf8_is_valid(c)); + fy_reader_advance_octets(fyr, fy_utf8_width(c)); + fy_reader_update_state_lb_mode(fyr, c, lb_mode); +} + +static FY_ALWAYS_INLINE inline void +fy_reader_advance(struct fy_reader *fyr, const int c) +{ + fy_reader_advance_lb_mode(fyr, c, fy_reader_lb_mode(fyr)); } static FY_ALWAYS_INLINE inline void @@ -635,7 +740,7 @@ fy_reader_advance_ws(struct fy_reader *fyr, int c) /* skip this character */ fy_reader_advance_octets(fyr, fy_utf8_width(c)); - if (fyr->tabsize && fy_is_tab(c)) + if (fy_is_tab(c) && fyr->tabsize) fyr->column += (fyr->tabsize - (fyr->column % fyr->tabsize)); else fyr->column++; @@ -682,4 +787,197 @@ fy_reader_strcmp(struct fy_reader *fyr, const char *str) return fy_reader_strncmp(fyr, str, strlen(str)); } +/* atom */ + +static inline void +fy_reader_fill_atom_start(struct fy_reader *fyr, struct fy_atom *handle) +{ + /* start mark */ + fy_reader_get_mark(fyr, &handle->start_mark); + handle->fyi = fy_reader_current_input(fyr); + handle->fyi_generation = fy_reader_current_input_generation(fyr); + + handle->increment = 0; + handle->tozero = 0; + + /* note that handle->data may be zero for empty input */ +} + +static inline void +fy_reader_fill_atom_end_at(struct fy_reader *fyr, struct fy_atom *handle, struct fy_mark *end_mark) +{ + if (end_mark) + handle->end_mark = *end_mark; + else + fy_reader_get_mark(fyr, &handle->end_mark); + + /* default is plain, modify at return */ + handle->style = FYAS_PLAIN; + handle->chomp = FYAC_CLIP; + /* by default we don't do storage hints, it's the job of the caller */ + handle->storage_hint = 0; + handle->storage_hint_valid = false; + handle->tabsize = fy_reader_tabsize(fyr); + handle->json_mode = fy_reader_json_mode(fyr); + handle->lb_mode = fy_reader_lb_mode(fyr); + handle->fws_mode = fy_reader_flow_ws_mode(fyr); + handle->directive0_mode = fy_reader_directive0_mode(fyr); +} + +static inline void +fy_reader_fill_atom_end(struct fy_reader *fyr, struct fy_atom *handle) +{ + fy_reader_fill_atom_end_at(fyr, handle, NULL); +} + +/* diag */ + +static inline bool +fyr_debug_log_level_is_enabled(struct fy_reader *fyr) +{ + return fyr && fy_diag_log_level_is_enabled(fyr->diag, FYET_DEBUG, FYEM_SCAN); +} + +int fy_reader_vdiag(struct fy_reader *fyr, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap); + +int fy_reader_diag(struct fy_reader *fyr, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) + __attribute__((format(printf, 6, 7))); + +void fy_reader_diag_vreport(struct fy_reader *fyr, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap); +void fy_reader_diag_report(struct fy_reader *fyr, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +#ifdef FY_DEVMODE + +#define fyr_debug(_fyr, _fmt, ...) \ + do { \ + struct fy_reader *__fyr = (_fyr); \ + \ + if (fyr_debug_log_level_is_enabled(__fyr)) \ + fy_reader_diag(__fyr, FYET_DEBUG, \ + __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__); \ + } while(0) +#else + +#define fyr_debug(_fyr, _fmt, ...) \ + do { } while(0) + +#endif + +#define fyr_info(_fyr, _fmt, ...) \ + fy_reader_diag((_fyr), FYET_INFO, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyr_notice(_fyr, _fmt, ...) \ + fy_reader_diag((_fyr), FYET_NOTICE, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyr_warning(_fyr, _fmt, ...) \ + fy_reader_diag((_fyr), FYET_WARNING, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyr_error(_fyr, _fmt, ...) \ + fy_reader_diag((_fyr), FYET_ERROR, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) + +#define fyr_error_check(_fyr, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + fyr_error((_fyr), _fmt, ## __VA_ARGS__); \ + goto _label ; \ + } \ + } while(0) + +#define _FYR_TOKEN_DIAG(_fyr, _fyt, _type, _module, _fmt, ...) \ + do { \ + struct fy_diag_report_ctx _drc; \ + memset(&_drc, 0, sizeof(_drc)); \ + _drc.type = (_type); \ + _drc.module = (_module); \ + _drc.fyt = (_fyt); \ + fy_reader_diag_report((_fyr), &_drc, (_fmt) , ## __VA_ARGS__); \ + } while(0) + +#define FYR_TOKEN_DIAG(_fyr, _fyt, _type, _module, _fmt, ...) \ + _FYR_TOKEN_DIAG(_fyr, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYR_PARSE_DIAG(_fyr, _adv, _cnt, _type, _module, _fmt, ...) \ + _FYR_TOKEN_DIAG(_fyr, \ + fy_token_create(FYTT_INPUT_MARKER, \ + fy_reader_fill_atom_at((_fyr), (_adv), (_cnt), \ + alloca(sizeof(struct fy_atom)))), \ + _type, _module, _fmt, ## __VA_ARGS__) + +#define FYR_MARK_DIAG(_fyr, _sm, _em, _type, _module, _fmt, ...) \ + _FYR_TOKEN_DIAG(_fyr, \ + fy_token_create(FYTT_INPUT_MARKER, \ + fy_reader_fill_atom_mark(((_fyr)), (_sm), (_em), \ + alloca(sizeof(struct fy_atom)))), \ + _type, _module, _fmt, ## __VA_ARGS__) + +#define FYR_NODE_DIAG(_fyr, _fyn, _type, _module, _fmt, ...) \ + _FYR_TOKEN_DIAG(_fyr, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYR_TOKEN_ERROR(_fyr, _fyt, _module, _fmt, ...) \ + FYR_TOKEN_DIAG(_fyr, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYR_PARSE_ERROR(_fyr, _adv, _cnt, _module, _fmt, ...) \ + FYR_PARSE_DIAG(_fyr, _adv, _cnt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYR_MARK_ERROR(_fyr, _sm, _em, _module, _fmt, ...) \ + FYR_MARK_DIAG(_fyr, _sm, _em, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYR_NODE_ERROR(_fyr, _fyn, _module, _fmt, ...) \ + FYR_NODE_DIAG(_fyr, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYR_TOKEN_ERROR_CHECK(_fyr, _fyt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYR_TOKEN_ERROR(_fyr, _fyt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYR_PARSE_ERROR_CHECK(_fyr, _adv, _cnt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYR_PARSE_ERROR(_fyr, _adv, _cnt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYR_MARK_ERROR_CHECK(_fyr, _sm, _em, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYR_MARK_ERROR(_fyr, _sm, _em, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYR_NODE_ERROR_CHECK(_fyr, _fyn, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYR_NODE_ERROR(_fyr, _fyn, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYR_TOKEN_WARNING(_fyr, _fyt, _module, _fmt, ...) \ + FYR_TOKEN_DIAG(_fyr, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYR_PARSE_WARNING(_fyr, _adv, _cnt, _module, _fmt, ...) \ + FYR_PARSE_DIAG(_fyr, _adv, _cnt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYR_MARK_WARNING(_fyr, _sm, _em, _module, _fmt, ...) \ + FYR_MARK_DIAG(_fyr, _sm, _em, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYR_NODE_WARNING(_fyr, _fyn, _type, _module, _fmt, ...) \ + FYR_NODE_DIAG(_fyr, _fyn, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + #endif diff --git a/src/lib/fy-parse-diag.c b/src/lib/fy-parse-diag.c new file mode 100644 index 0000000..a108312 --- /dev/null +++ b/src/lib/fy-parse-diag.c @@ -0,0 +1,162 @@ +/* + * fy-parse-diag.c - parser diagnostics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-diag.h" +#include "fy-parse.h" + +int fy_parser_vdiag(struct fy_parser *fyp, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap) +{ + enum fy_error_type level; + enum fy_error_module module; + struct fy_diag_ctx fydc; + int rc; + + if (!fyp || !fyp->diag || !fmt) + return -1; + + level = (flags & FYDF_LEVEL_MASK) >> FYDF_LEVEL_SHIFT; + module = (flags & FYDF_MODULE_MASK) >> FYDF_MODULE_SHIFT; + + if (!fy_diag_log_level_is_enabled(fyp->diag, level, module)) + return 0; + + /* fill in fy_diag_ctx */ + memset(&fydc, 0, sizeof(fydc)); + + fydc.level = level; + fydc.module = module; + fydc.source_file = file; + fydc.source_line = line; + fydc.source_func = func; + fydc.line = fyp_line(fyp); + fydc.column = fyp_column(fyp); + + rc = fy_vdiag(fyp->diag, &fydc, fmt, ap); + + if (fyp && !fyp->stream_error && fyp->diag->on_error) + fyp->stream_error = true; + + return rc; +} + +int fy_parser_diag(struct fy_parser *fyp, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = fy_parser_vdiag(fyp, flags, file, line, func, fmt, ap); + va_end(ap); + + return rc; +} + +void fy_parser_diag_vreport(struct fy_parser *fyp, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap) +{ + struct fy_diag *diag; + + if (!fyp || !fyp->diag || !fydrc || !fmt) + return; + + diag = fyp->diag; + + fy_diag_vreport(diag, fydrc, fmt, ap); + + if (fyp && !fyp->stream_error && diag->on_error) + fyp->stream_error = true; +} + +void fy_parser_diag_report(struct fy_parser *fyp, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_parser_diag_vreport(fyp, fydrc, fmt, ap); + va_end(ap); +} + +void fy_parser_vlog(struct fy_parser *fyp, enum fy_error_type type, + const char *fmt, va_list ap) +{ + struct fy_diag_ctx ctx = { + .level = type, + .module = FYEM_UNKNOWN, + .source_func = "", + .source_file = "", + .source_line = 0, + .file = NULL, + .line = 0, + .column = 0, + }; + + if (!fyp || !fyp->diag) + return; + + (void)fy_vdiag(fyp->diag, &ctx, fmt, ap); +} + +void fy_parser_log(struct fy_parser *fyp, enum fy_error_type type, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_parser_vlog(fyp, type, fmt, ap); + va_end(ap); +} + +void fy_parser_vreport(struct fy_parser *fyp, enum fy_error_type type, + struct fy_token *fyt, const char *fmt, va_list ap) +{ + struct fy_diag_report_ctx fydrc_local, *fydrc = &fydrc_local; + + if (!fyp || !fyp->diag || !fyt) + return; + + memset(fydrc, 0, sizeof(*fydrc)); + fydrc->type = type; + fydrc->module = FYEM_UNKNOWN; + fydrc->fyt = fy_token_ref(fyt); + + fy_parser_diag_vreport(fyp, fydrc, fmt, ap); +} + +void fy_parser_report(struct fy_parser *fyp, enum fy_error_type type, + struct fy_token *fyt, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + fy_parser_vreport(fyp, type, fyt, fmt, ap); + va_end(ap); +} + diff --git a/src/lib/fy-parse.c b/src/lib/fy-parse.c index 99fed7c..830e11b 100644 --- a/src/lib/fy-parse.c +++ b/src/lib/fy-parse.c @@ -37,7 +37,7 @@ const char *fy_library_version(void) { #ifndef VERSION -#warn No version defined +#warning No version defined return "UNKNOWN"; #else return VERSION; @@ -49,7 +49,7 @@ int fy_parse_input_append(struct fy_parser *fyp, const struct fy_input_cfg *fyic struct fy_input *fyi = NULL; fyi = fy_input_create(fyic); - fyp_error_check(fyp, fyp != NULL, err_out, + fyp_error_check(fyp, fyi != NULL, err_out, "fy_parse_input_create() failed!"); fyi->state = FYIS_QUEUED; @@ -105,11 +105,7 @@ int fy_parse_get_next_input(struct fy_parser *fyp) json_mode = true; /* set the initial reader mode according to json option and default version */ - if (!json_mode) - rdmode = fy_version_compare(&fyp->default_version, fy_version_make(1, 1)) <= 0 ? fyrm_yaml_1_1 : fyrm_yaml; - else - rdmode = fyrm_json; - + rdmode = fy_reader_calculate_mode(&fyp->stream_version, json_mode); fy_reader_set_mode(fyp->reader, rdmode); memset(&icfg, 0, sizeof(icfg)); @@ -459,11 +455,12 @@ bool fy_token_tag_directive_is_overridable(struct fy_token *fyt_td) int fy_reset_document_state(struct fy_parser *fyp) { struct fy_document_state *fyds_new = NULL; + enum fy_reader_mode rdmode; fyp_scan_debug(fyp, "resetting document state"); if (!fyp->default_document_state) { - fyds_new = fy_document_state_default(&fyp->default_version, NULL); + fyds_new = fy_document_state_default(&fyp->stream_version, NULL); fyp_error_check(fyp, fyds_new, err_out, "fy_document_state_default() failed"); } else { @@ -486,6 +483,9 @@ int fy_reset_document_state(struct fy_parser *fyp) /* and reset all aliases (TODO look for cross document aliases) */ fy_parse_streaming_aliases_reset(fyp); + rdmode = fy_reader_calculate_mode(&fyds_new->version, fyds_new->json_mode); + fy_reader_set_mode(fyp->reader, rdmode); + return 0; err_out: @@ -535,10 +535,10 @@ int fy_check_document_version(struct fy_parser *fyp) /* we only support YAML version 1.x */ if (major == 1) { /* 1.1 is supported without warnings */ - if (minor == 1) + if (minor == 1 || minor == 2) goto ok; - if (minor == 2 || minor == 3) + if (minor == 3) goto experimental; } @@ -787,6 +787,7 @@ int fy_parse_setup(struct fy_parser *fyp, const struct fy_parse_cfg *cfg) fyp->reader = &fyp->builtin_reader; fyp->default_version = *vers; + fyp->stream_version = fyp->default_version; fy_indent_list_init(&fyp->indent_stack); fy_indent_list_init(&fyp->recycled_indent); @@ -821,6 +822,8 @@ int fy_parse_setup(struct fy_parser *fyp, const struct fy_parse_cfg *cfg) fy_streaming_alias_list_init(&fyp->recycled_streaming_alias); fy_eventp_list_init(&fyp->mks.args); + fy_atom_reset(&fyp->last_comment); + fyp->suppress_recycling = !!(fyp->cfg.flags & FYPCF_DISABLE_RECYCLING) || (getenv("FY_VALGRIND") && !getenv("FY_VALGRIND_RECYCLING")); @@ -853,8 +856,13 @@ void fy_parse_cleanup(struct fy_parser *fyp) struct fy_eventp *fyep; struct fy_token *fyt; + fy_parse_eventp_recycle(fyp, fyp->fyep_peek); + fy_parse_streaming_aliases_reset(fyp); + fy_input_unref(fyp->last_comment.fyi); + fy_atom_reset(&fyp->last_comment); + fy_input_unref(fyp->last_event_handle.fyi); fy_atom_reset(&fyp->last_event_handle); @@ -1024,6 +1032,58 @@ int fy_scan_comment(struct fy_parser *fyp, struct fy_atom *handle, bool single_l return 0; } +bool +fy_comment_atoms_seperated_by_ws(struct fy_parser *fyp, struct fy_atom *a, struct fy_atom *b) +{ + struct fy_atom *tmpatom; + struct fy_atom inbetween; + struct fy_atom_iter iter; + int c; + bool ws_or_lb; + + if (!a || !b) + return NULL; + + /* must be on same input */ + if (a->fyi != b->fyi) + return NULL; + + /* must be comments */ + if (a->style != FYAS_COMMENT || b->style != FYAS_COMMENT) + return NULL; + + /* a must be before */ + if (a->end_mark.input_pos > b->start_mark.input_pos) { + tmpatom = a; + a = b; + b = tmpatom; + } + + memcpy(&inbetween, a, sizeof(inbetween)); + inbetween.start_mark = a->end_mark; + inbetween.end_mark = b->start_mark; + + fy_atom_iter_start(&inbetween, &iter); + ws_or_lb = true; + while (ws_or_lb && (c = fy_atom_iter_utf8_get(&iter)) >= 0) + ws_or_lb = fy_is_ws(c) || fy_is_lb_m(c, a->lb_mode); + fy_atom_iter_finish(&iter); + return ws_or_lb; +} + +static inline bool +fy_reset_last_comment(struct fy_parser *fyp) +{ + if (!(fyp->cfg.flags & FYPCF_PARSE_COMMENTS)) + return false; + + if (!fy_atom_is_set(&fyp->last_comment)) + return false; + + fy_atom_reset(&fyp->last_comment); + return true; +} + int fy_attach_comments_if_any(struct fy_parser *fyp, struct fy_token *fyt) { struct fy_atom *handle; @@ -1039,10 +1099,10 @@ int fy_attach_comments_if_any(struct fy_parser *fyp, struct fy_token *fyt) /* if a last comment exists and is valid */ if (fy_atom_is_set(&fyp->last_comment) && (handle = fy_token_comment_handle(fyt, fycp_top, true)) != NULL) { - assert (!fy_atom_is_set(handle)); + *handle = fyp->last_comment; - /* erase last comment */ - fy_atom_reset(&fyp->last_comment); + + fy_reset_last_comment(fyp); } /* right hand comment */ @@ -1075,7 +1135,7 @@ int fy_scan_to_next_token(struct fy_parser *fyp) int c, c_after_ws, i, rc = 0; bool tabs_allowed, sloppy_flow, no_indent; ssize_t offset; - struct fy_atom *handle; + struct fy_atom this_comment; struct fy_reader *fyr; fyr = fyp->reader; @@ -1085,7 +1145,7 @@ int fy_scan_to_next_token(struct fy_parser *fyp) "fy_reader_input_scan_token_mark() failed"); /* skip BOM at the start of the stream */ - if (fyr->current_input_pos == 0 && (c = fy_parse_peek(fyp)) == FY_UTF8_BOM) { + if (fy_reader_current_input_pos(fyr) == 0 && (c = fy_parse_peek(fyp)) == FY_UTF8_BOM) { fy_advance(fyp, c); /* reset column */ @@ -1167,13 +1227,36 @@ int fy_scan_to_next_token(struct fy_parser *fyp) /* comment? */ if (c == '#') { - handle = NULL; - if (fyp->cfg.flags & FYPCF_PARSE_COMMENTS) - handle = &fyp->last_comment; + if (!(fyp->cfg.flags & FYPCF_PARSE_COMMENTS)) { + rc = fy_scan_comment(fyp, NULL, false); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_scan_comment() failed"); - rc = fy_scan_comment(fyp, handle, false); - fyp_error_check(fyp, !rc, err_out_rc, - "fy_scan_comment() failed"); + + } else { + + memset(&this_comment, 0, sizeof(this_comment)); + fy_atom_reset(&this_comment); + rc = fy_scan_comment(fyp, &this_comment, false); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_scan_comment() failed"); + + if (fy_atom_is_set(&fyp->last_comment)) { + + /* merge consecutive comments */ + if (fy_comment_atoms_seperated_by_ws(fyp, &fyp->last_comment, &this_comment)) { + fyp->last_comment.end_mark = this_comment.end_mark; + fy_input_unref(this_comment.fyi); + fy_atom_reset(&this_comment); + } else { + fy_input_unref(fyp->last_comment.fyi); + fy_atom_reset(&fyp->last_comment); + fyp->last_comment = this_comment; + } + + } else + fyp->last_comment = this_comment; + } tabs_allowed = (fyp->flow_level || !fyp->simple_key_allowed) || fyp_tabsize(fyp); } @@ -2040,7 +2123,7 @@ int fy_scan_directive(struct fy_parser *fyp) /* skip until linebreak (or #) */ i = 0; lastc = -1; - while ((c = fy_parse_peek_at(fyp, i)) != -1 && !fyp_is_lb(fyp, c)) { + while ((c = fy_parse_peek_at(fyp, i)) >= 0 && !fyp_is_lb(fyp, c)) { if (fy_is_ws(lastc) && c == '#') break; lastc = c; @@ -2050,8 +2133,11 @@ int fy_scan_directive(struct fy_parser *fyp) FYP_PARSE_WARNING(fyp, 0, i, FYEM_SCAN, "Unsupported directive"); + FYP_PARSE_ERROR_CHECK(fyp, i, 1, FYEM_SCAN, c >= 0 || c == FYUG_EOF, err_out, + "Bad UTF8 input while scanning unknown %%TAG directive"); + if (fy_is_ws(lastc) && c == '#') { - while ((c = fy_parse_peek_at(fyp, i)) != -1 && !fyp_is_lb(fyp, c)) + while ((c = fy_parse_peek_at(fyp, i)) >= 0 && !fyp_is_lb(fyp, c)) i++; } @@ -2086,8 +2172,11 @@ int fy_scan_directive(struct fy_parser *fyp) fyp_error_check(fyp, version_length > 0, err_out, "fy_scan_yaml_version() failed"); + /* save the version per stream */ + fyp->stream_version = vers; + /* set the reader's mode according to the version just scanned */ - rdmode = fy_version_compare(&vers, fy_version_make(1, 1)) <= 0 ? fyrm_yaml_1_1 : fyrm_yaml; + rdmode = fy_reader_calculate_mode(&vers, false); fy_reader_set_mode(fyp->reader, rdmode); fy_advance_by(fyp, version_length); @@ -2136,7 +2225,7 @@ int fy_scan_directive(struct fy_parser *fyp) /* skip until linebreak (or #) */ i = 0; lastc = -1; - while ((c = fy_parse_peek_at(fyp, i)) != -1 && !fyp_is_lb(fyp, c)) { + while ((c = fy_parse_peek_at(fyp, i)) >= 0 && !fyp_is_lb(fyp, c)) { if (fy_is_ws(lastc) && c == '#') break; @@ -3213,7 +3302,7 @@ int fy_fetch_block_scalar(struct fy_parser *fyp, bool is_literal, int c) int breaks, breaks_length, presentation_breaks_length, first_break_length; bool doc_start_end_detected, empty, empty_line, prev_empty_line, indented, prev_indented, first; bool has_ws, has_lb, starts_with_ws, starts_with_lb, ends_with_ws, ends_with_lb, trailing_lb; - bool pending_nl, ends_with_eof, starts_with_eof; + bool pending_nl, ends_with_eof, starts_with_eof, content_is_eof; struct fy_token *fyt; size_t length, line_length, trailing_ws, trailing_breaks_length; size_t leading_ws; @@ -3221,7 +3310,7 @@ int fy_fetch_block_scalar(struct fy_parser *fyp, bool is_literal, int c) unsigned int chomp_amt; int actual_lb_length, pending_lb_length; struct fy_mark indicator_mark; - bool generated_indent; + bool generated_indent, final_lb; #ifdef ATOM_SIZE_CHECK size_t tlength; #endif @@ -3315,12 +3404,16 @@ int fy_fetch_block_scalar(struct fy_parser *fyp, bool is_literal, int c) } /* advance */ - fy_advance(fyp, c); + if (fy_utf8_is_valid(c)) + fy_advance(fyp, c); fy_fill_atom_start(fyp, &handle); starts_with_eof = c < 0; + c = fy_parse_peek(fyp); + content_is_eof = c < 0; /* any error or EOF */ + current_indent = fyp->indent >= 0 ? fyp->indent : 0; indent = increment ? current_indent + increment : 0; @@ -3553,42 +3646,24 @@ int fy_fetch_block_scalar(struct fy_parser *fyp, bool is_literal, int c) if (chomp_amt == (unsigned int)-1) chomp_amt = current_indent; + /* whether the final character is an lb (or implied to be one) */ + final_lb = pending_nl || (!starts_with_eof && ends_with_eof); + trailing_lb = trailing_breaks_length > 0; + ends_with_ws = fy_is_ws(lastc); + switch (chomp) { case FYAC_CLIP: - if (pending_nl || (!starts_with_eof && ends_with_eof)) { - if (actual_lb_length <= 2) - length += 1; - else - length += actual_lb_length; - - ends_with_lb = true; - ends_with_ws = false; - } else { - if (trailing_breaks_length > 0) - ends_with_lb = true; - else if (fy_is_ws(lastc)) - ends_with_ws = true; - } - break; case FYAC_KEEP: - if (pending_nl || (!starts_with_eof && ends_with_eof)) - length += actual_lb_length; - - length += breaks + presentation_breaks_length; - - trailing_lb = trailing_breaks_length > 0; - if (pending_nl || (!starts_with_eof && ends_with_eof) || trailing_breaks_length) { - ends_with_lb = true; - ends_with_ws = false; - } else if (fy_is_ws(lastc)) { - ends_with_ws = true; - ends_with_lb = false; - } + if (chomp == FYAC_KEEP) + length += breaks + presentation_breaks_length; + + if (final_lb && !content_is_eof) + length += (actual_lb_length == 0 && pending_nl) ? 1 : actual_lb_length; + + ends_with_lb = final_lb || trailing_lb; break; case FYAC_STRIP: ends_with_lb = false; - if (fy_is_ws(lastc)) - ends_with_ws = true; break; } @@ -3613,6 +3688,7 @@ int fy_fetch_block_scalar(struct fy_parser *fyp, bool is_literal, int c) handle.json_mode = fyp_json_mode(fyp); handle.lb_mode = fyp_lb_mode(fyp); handle.fws_mode = fyp_fws_mode(fyp); + handle.directive0_mode = fyp_directive0_mode(fyp); handle.tabsize = fyp_tabsize(fyp); handle.ends_with_eof = ends_with_eof; handle.is_merge_key = false; @@ -4050,9 +4126,11 @@ int fy_reader_fetch_flow_scalar_handle(struct fy_reader *fyr, int c, int indent, handle->json_mode = fy_reader_json_mode(fyr); handle->lb_mode = fy_reader_lb_mode(fyr); handle->fws_mode = fy_reader_flow_ws_mode(fyr); + handle->directive0_mode = fy_reader_directive0_mode(fyr); handle->tabsize = fy_reader_tabsize(fyr); handle->ends_with_eof = false; /* flow scalars never end with EOF and be valid */ handle->is_merge_key = false; + handle->simple_key_allowed = false; /* skip over block scalar end */ fy_reader_advance_by(fyr, 1); @@ -4080,66 +4158,58 @@ err_out: return -1; } -int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, int indent, int flow_level, struct fy_atom *handle, bool directive0) -{ +struct fy_fetch_plain_state { + struct fy_mark last_mark; size_t length; - int rc = -1, run, nextc, lastc, breaks_found, blanks_found; - int breaks_found_length, first_break_length, break_length, presentation_breaks_length; - bool has_leading_blanks; - bool last_ptr; - struct fy_mark mark, last_mark; - bool is_multiline, has_lb, has_ws, ends_with_eof, is_merge_key; + int c; + int lastc; + bool has_lb; + bool has_ws; bool has_json_esc; -#ifdef ATOM_SIZE_CHECK - size_t tlength; -#endif - - FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_SCAN, - !fy_reader_is_blankz(fyr, c), err_out, - "plain scalar cannot start with blank or zero"); - - /* may not start with any of ,[]{}#&*!|>'\"%@` */ - FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_SCAN, - !fy_utf8_strchr(",[]{}#&*!|>'\"%@`", c), err_out, - "plain scalar cannot start with '%c'", c); - - /* may not start with - not followed by blankz */ - FYR_PARSE_ERROR_CHECK(fyr, 0, 2, FYEM_SCAN, - c != '-' || !fy_reader_is_blank_at_offset(fyr, 1), err_out, - "plain scalar cannot start with '%c' followed by blank", c); - - /* may not start with -?: not followed by blankz (in block context) */ - FYR_PARSE_ERROR_CHECK(fyr, 0, 2, FYEM_SCAN, - flow_level > 0 || !((c == '?' || c == ':') && fy_reader_is_blank_at_offset(fyr, 1)), err_out, - "plain scalar cannot start with '%c' followed by blank (in block context)", c); - - /* may not start with - followed by ",[]{}" in flow context */ - FYR_PARSE_ERROR_CHECK(fyr, 0, 2, FYEM_SCAN, - flow_level == 0 || !(c == '-' && fy_utf8_strchr(",[]{}", fy_reader_peek_at(fyr, 1))), err_out, - "plain scalar cannot start with '%c' followed by ,[]{} (in flow context)", c); - - /* we can prime is_merge_key here */ - is_merge_key = c == '<' && fy_reader_peek_at(fyr, 1) == '<'; + bool has_high_ascii; + bool simple_key_allowed; + bool last_was_lb; +}; - fy_reader_get_mark(fyr, &mark); +static FY_ALWAYS_INLINE inline int +fy_reader_fetch_plain_scalar_handle_inline(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + const bool directive0, + const enum fy_lb_mode lb_mode, + const bool json_mode, + struct fy_fetch_plain_state *state) +{ + int nextc, width; + size_t length, length_advance; + int blanks_found_length, breaks_found_length, first_break_length; + union { + struct { + bool run_has_lb : 1; + bool run_has_ws : 1; + bool has_json_esc : 1; + bool has_high_ascii : 1; + bool has_lb : 1; + bool has_ws : 1; + bool had_run : 1; + bool flow_level_gt_0 : 1; + bool flow_level_le_0_indent_ge_0 : 1; + bool check_tab : 1; + bool last_was_lb : 1; + }; + unsigned short flags; + } u; + size_t run, len; + const char *s; - fy_reader_fill_atom_start(fyr, handle); + u.flags = 0; - has_leading_blanks = false; - has_lb = false; - has_ws = false; - has_json_esc = false; + u.flow_level_gt_0 = flow_level > 0; + u.flow_level_le_0_indent_ge_0 = flow_level <= 0 && indent >= 0; + u.check_tab = fy_reader_tabsize(fyr) || indent < 0; length = 0; - breaks_found = 0; - breaks_found_length = 0; - first_break_length = 0; - presentation_breaks_length = 0; - blanks_found = 0; - last_ptr = false; - memset(&last_mark, 0, sizeof(last_mark)); - c = FYUG_EOF; - lastc = FYUG_EOF; + length_advance = 0; for (;;) { /* break for document indicators */ @@ -4148,7 +4218,6 @@ int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, int indent fy_reader_is_blankz_at_offset(fyr, 3))) break; - c = fy_reader_peek(fyr); if (c == '#') break; @@ -4156,208 +4225,289 @@ int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, int indent if (directive0 && fy_reader_column(fyr) == 0 && c == '%') break; - /* quickly deal with runs */ - run = 0; - if (c >= 0 && c <= 0x7f && (fy_utf8_low_ascii_flags[c] & F_SIMPLE_SCALAR)) { - size_t len, consumed; - const char *p, *s, *e; - int8_t cc; - - while ((p = fy_reader_ensure_lookahead(fyr, 1, &len)) != NULL) { - - s = p; - e = s + len; - - while (s < e && (cc = (int8_t)*s) >= 0 && (fy_utf8_low_ascii_flags[cc] & F_SIMPLE_SCALAR)) - s++; - - consumed = s - p; - if (consumed) { - fy_reader_advance_octets(fyr, consumed); - fyr->column += consumed; - } - run += consumed; + u.had_run = false; + u.last_was_lb = false; + while (!fy_is_blankz_m(c, lb_mode)) { - /* we're done if stopped earlier */ - if (s < e) - break; - } + if (fy_utf8_is_simple_scalar(c)) { + s = fy_reader_ensure_lookahead(fyr, 1, &len); + run = 0; + while (run < len && fy_utf8_is_simple_scalar_no_check((int8_t)s[run])) + run++; - } - if (run > 0) { - length += run; - if (breaks_found) { - /* minimum 1 sep, or more for consecutive */ - length += breaks_found > 1 ? (breaks_found_length - first_break_length) : 1; - length += presentation_breaks_length; - breaks_found = 0; - blanks_found = 0; - presentation_breaks_length = 0; - } else if (blanks_found) { - /* just the blanks mam' */ - length += blanks_found; - blanks_found = 0; - } - } + fyr->column += run; + nextc = -1; + } else { - while (!fy_reader_is_blankz(fyr, c = fy_reader_peek(fyr))) { + width = fy_utf8_width(c); + if (c == ':') { - if (c == ':') { + nextc = fy_reader_peek_at_offset(fyr, width); - nextc = fy_reader_peek_at(fyr, 1); + /* ':' followed by space terminates */ + if (fy_is_blankz_m(nextc, lb_mode)) + break; - /* ':' followed by space terminates */ - if (fy_reader_is_blankz(fyr, nextc)) { - /* super rare case :: not followed by space */ - /* :: not followed by space */ - if (lastc != ':' || fy_reader_is_blankz(fyr, nextc)) + /* in flow context ':' followed by flow markers */ + if (u.flow_level_gt_0 && fy_is_flow_indicator(nextc)) break; - } + } else + nextc = -1; - /* in flow context ':' followed by flow markers */ - if (flow_level > 0 && fy_utf8_strchr(",[]{}", nextc)) + /* in flow context any or , [ ] { } */ + if (u.flow_level_gt_0 && fy_is_flow_indicator(c)) break; - } - /* in flow context any or , [ ] { } */ - if (flow_level > 0 && (c == ',' || c == '[' || c == ']' || c == '{' || c == '}')) - break; - - if (breaks_found) { - /* minimum 1 sep, or more for consecutive */ - length += breaks_found > 1 ? (breaks_found_length - first_break_length) : 1; - length += presentation_breaks_length; - breaks_found = 0; - blanks_found = 0; - presentation_breaks_length = 0; - } else if (blanks_found) { - /* just the blanks mam' */ - length += blanks_found; - blanks_found = 0; - } + /* check whether we have a JSON unescaped character */ + u.has_json_esc |= !fy_is_json_unescaped(c); + u.has_high_ascii |= c >= 0x80; - /* check whether we have a JSON unescaped character */ - if (!has_json_esc && !fy_is_json_unescaped(c)) - has_json_esc = true; + fyr->column++; - fy_reader_advance(fyr, c); - run++; + run = width; + } - length += fy_utf8_width(c); + u.had_run = true; + fy_reader_advance_octets(fyr, run); + length += run; - lastc = c; + c = nextc >= 0 ? nextc : fy_reader_peek(fyr); } /* save end mark if we processed more than one non-blank */ - if (run > 0) { - /* fyp_scan_debug(fyp, "saving mark"); */ - last_ptr = true; - fy_reader_get_mark(fyr, &last_mark); + if (u.had_run) { + length += length_advance; + + u.has_lb |= u.run_has_lb; + u.has_ws |= u.run_has_ws; + + fy_reader_get_mark(fyr, &state->last_mark); } /* end? */ - if (!(fy_is_blank(c) || fy_reader_is_lb(fyr, c))) + if (!(fy_is_blank_lb_m(c, lb_mode))) break; - has_json_esc = true; - /* consume blanks */ - breaks_found = 0; breaks_found_length = 0; first_break_length = 0; - blanks_found = 0; + blanks_found_length = 0; + u.run_has_ws = u.run_has_lb = false; + + width = fy_utf8_width(c); do { - fy_reader_advance(fyr, c); + fy_reader_advance_lb_mode(fyr, c, lb_mode); - if (!fy_reader_tabsize(fyr)) { - /* check for tab */ + if (u.check_tab) { FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_SCAN, - c != '\t' || !has_leading_blanks || indent < 0 || fy_reader_column(fyr) >= (indent + 1), err_out, + c != '\t' || fy_reader_column(fyr) >= (indent + 1), err_out, "invalid tab used as indentation"); } - nextc = fy_reader_peek(fyr); + u.last_was_lb = !fy_is_blank(c); /* if it's a break */ - if (fy_reader_is_lb(fyr, c)) { - - if (!fy_is_lb_LS_PS(c)) { - break_length = 1; - } else { - break_length = fy_utf8_width(c); - presentation_breaks_length += break_length; - } - - /* first break, turn on leading blanks */ - if (!has_leading_blanks) - has_leading_blanks = true; - if (!breaks_found) - first_break_length = break_length; - breaks_found++; - breaks_found_length += break_length; - blanks_found = 0; - has_lb = true; + if (!u.last_was_lb) { + blanks_found_length += width; + u.run_has_ws = true; } else { - blanks_found++; - has_ws = true; + /* first break, turn on leading blanks */ + if (!first_break_length) + first_break_length = width; + breaks_found_length += width; + u.run_has_lb = true; } - c = nextc; + c = fy_reader_peek_width(fyr, &width); - } while (fy_is_blank(c) || fy_reader_is_lb(fyr, c)); + } while (fy_is_blank_lb_m(c, lb_mode)); /* break out if indentation is less */ - if (flow_level <= 0 && indent >= 0 && fy_reader_column(fyr) < (indent + 1)) + if (u.flow_level_le_0_indent_ge_0 && fy_reader_column(fyr) <= indent) break; + + length_advance = u.run_has_lb ? + (breaks_found_length > first_break_length ? (breaks_found_length - first_break_length) : first_break_length) : + blanks_found_length; } + state->c = c; + state->length = length; + state->simple_key_allowed = u.run_has_lb; // simple key allowed if there was an lb + state->has_lb = u.has_lb; + state->has_ws = u.has_ws; + state->has_json_esc = u.has_json_esc; + state->has_high_ascii = u.has_high_ascii; + state->last_was_lb = u.last_was_lb; + + return 0; + +err_out: + return -1; +} + +int fy_reader_fetch_plain_scalar_handle_yaml(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + struct fy_fetch_plain_state *state) +{ + return fy_reader_fetch_plain_scalar_handle_inline(fyr, c, indent, flow_level, handle, + false, // directive0 + fylb_cr_nl, // lb_mode + false, // json + state); +} + +int fy_reader_fetch_plain_scalar_handle_json(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + struct fy_fetch_plain_state *state) +{ + return fy_reader_fetch_plain_scalar_handle_inline(fyr, c, indent, flow_level, handle, + false, // directive0 + fylb_cr_nl, // lb_mode + true, // json + state); +} + +int fy_reader_fetch_plain_scalar_handle_yaml_1_1(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + struct fy_fetch_plain_state *state) +{ + return fy_reader_fetch_plain_scalar_handle_inline(fyr, c, indent, flow_level, handle, + true, // directive0 + fylb_cr_nl_N_L_P, // lb_mode + false, // json + state); +} + +typedef int (*fy_reader_fetch_plain_scalar_handle_func)(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + struct fy_fetch_plain_state *state); + +static inline int +fy_reader_fetch_plain_scalar_handle_switch(struct fy_reader *fyr, int c, + const int indent, const int flow_level, + struct fy_atom *handle, + struct fy_fetch_plain_state *state) +{ + + static const fy_reader_fetch_plain_scalar_handle_func funcs[] = { + [fyrm_yaml] = fy_reader_fetch_plain_scalar_handle_yaml, + [fyrm_json] = fy_reader_fetch_plain_scalar_handle_json, + [fyrm_yaml_1_1] = fy_reader_fetch_plain_scalar_handle_yaml_1_1, + }; + fy_reader_fetch_plain_scalar_handle_func func; + + assert((unsigned int)fyr->mode <= ARRAY_SIZE(funcs)); + func = funcs[fyr->mode]; + assert(func != NULL); + + return func(fyr, c, indent, flow_level, handle, state); +} + +int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, + const int indent, int flow_level, + struct fy_atom *handle) +{ + struct fy_fetch_plain_state state_local, *state = &state_local; + const enum fy_lb_mode lb_mode = fy_reader_lb_mode(fyr); + bool is_merge_key, ends_with_eof, is_multiline; + int nextc, rc; + + nextc = fy_reader_peek_at(fyr, 1); + + FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_SCAN, + !fy_is_blankz_m(c, lb_mode), err_out, + "plain scalar cannot start with blank or zero"); + + /* may not start with any of ,[]{}#&*!|>'\"%@` */ + FYR_PARSE_ERROR_CHECK(fyr, 0, 1, FYEM_SCAN, + !fy_utf8_strchr(",[]{}#&*!|>'\"%@`", c), err_out, + "plain scalar cannot start with '%c'", c); + + /* may not start with - not followed by blankz */ + FYR_PARSE_ERROR_CHECK(fyr, 0, 2, FYEM_SCAN, + c != '-' || !fy_is_blank(nextc), err_out, + "plain scalar cannot start with '%c' followed by blank", c); + + /* may not start with -?: not followed by blankz (in block context) */ + FYR_PARSE_ERROR_CHECK(fyr, 0, 2, FYEM_SCAN, + flow_level > 0 || !((c == '?' || c == ':') && fy_is_blank(nextc)), err_out, + "plain scalar cannot start with '%c' followed by blank (in block context)", c); + + /* may not start with - followed by ",[]{}" in flow context */ + FYR_PARSE_ERROR_CHECK(fyr, 0, 2, FYEM_SCAN, + flow_level == 0 || !(c == '-' && fy_is_flow_indicator(nextc)), err_out, + "plain scalar cannot start with '%c' followed by ,[]{} (in flow context)", c); + + memset(state, 0, sizeof(*state)); + state->last_mark.input_pos = (size_t)-1; + + /* we can prime is_merge_key here */ + is_merge_key = c == '<' && nextc == '<'; + + fy_reader_fill_atom_start(fyr, handle); + + rc = fy_reader_fetch_plain_scalar_handle_switch(fyr, c, indent, flow_level, handle, state); + if (rc) + goto err_out; + /* end... */ - if (!last_ptr) + if (state->last_mark.input_pos == (size_t)-1) fy_reader_fill_atom_end(fyr, handle); else - fy_reader_fill_atom_end_at(fyr, handle, &last_mark); + fy_reader_fill_atom_end_at(fyr, handle, &state->last_mark); + c = state->c; if (c == FYUG_INV || c == FYUG_PARTIAL) { FYR_MARK_ERROR(fyr, &handle->start_mark, &handle->end_mark, FYEM_SCAN, "plain scalar is malformed UTF8"); goto err_out; } - ends_with_eof = c == FYUG_EOF && !fy_reader_is_lb(fyr, lastc); + + ends_with_eof = c == FYUG_EOF && !state->last_was_lb; is_multiline = handle->end_mark.line > handle->start_mark.line; handle->style = FYAS_PLAIN; handle->chomp = FYAC_STRIP; - handle->direct_output = !is_multiline && !has_json_esc && fy_atom_size(handle) == length; + handle->direct_output = !is_multiline && !state->has_json_esc && fy_atom_size(handle) == state->length; handle->empty = false; - handle->has_lb = has_lb; - handle->has_ws = has_ws; + handle->has_lb = state->has_lb; + handle->has_ws = state->has_ws; handle->starts_with_ws = false; handle->starts_with_lb = false; handle->ends_with_ws = false; handle->ends_with_lb = false; handle->trailing_lb = false; - handle->size0 = length == 0; + handle->size0 = state->length == 0; handle->valid_anchor = false; handle->json_mode = fy_reader_json_mode(fyr); handle->lb_mode = fy_reader_lb_mode(fyr); handle->fws_mode = fy_reader_flow_ws_mode(fyr); + handle->directive0_mode = fy_reader_directive0_mode(fyr); handle->tabsize = fy_reader_tabsize(fyr); handle->ends_with_eof = ends_with_eof; - handle->is_merge_key = is_merge_key && length == 2; + handle->is_merge_key = is_merge_key && state->length == 2; + handle->simple_key_allowed = state->simple_key_allowed; + handle->high_ascii = state->has_high_ascii; #ifdef ATOM_SIZE_CHECK + size_t tlength; tlength = fy_atom_format_text_length(handle); - if (tlength != length) { + if (tlength != state->length) { fyr_warning(fyr, "%s: storage hint calculation failed real %zu != hint %zu - \"%s\"", __func__, - tlength, length, + tlength, state->length, fy_utf8_format_text_a(fy_atom_data(handle), fy_atom_size(handle), fyue_doublequote)); - length = tlength; + state->length = tlength; } #endif - handle->storage_hint = length; + handle->storage_hint = state->length; handle->storage_hint_valid = true; /* extra check in json mode */ @@ -4374,6 +4524,7 @@ int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, int indent "Invalid JSON plain scalar"); } + return 0; err_out: @@ -4381,7 +4532,6 @@ err_out: return rc; } - int fy_fetch_flow_scalar(struct fy_parser *fyp, int c) { struct fy_atom handle; @@ -4766,8 +4916,7 @@ int fy_fetch_plain_scalar(struct fy_parser *fyp, int c) fy_get_simple_key_mark(fyp, &skm); - rc = fy_reader_fetch_plain_scalar_handle(fyp->reader, c, fyp->indent, fyp->flow_level, &handle, - fy_document_state_version_compare(fyp->current_document_state, fy_version_make(1, 1)) <= 0); + rc = fy_reader_fetch_plain_scalar_handle(fyp->reader, c, fyp->indent, fyp->flow_level, &handle); if (rc) { fyp->stream_error = true; goto err_out_rc; @@ -4816,7 +4965,7 @@ int fy_fetch_plain_scalar(struct fy_parser *fyp, int c) fyp_error_check(fyp, !rc, err_out_rc, "fy_save_simple_key_mark() failed"); - fyp->simple_key_allowed = handle.has_lb; + fyp->simple_key_allowed = handle.simple_key_allowed; fyp_scan_debug(fyp, "simple_key_allowed -> %s\n", fyp->simple_key_allowed ? "true" : "false"); if (fyp->cfg.flags & FYPCF_PARSE_COMMENTS) { @@ -5110,6 +5259,10 @@ struct fy_token *fy_scan_peek(struct fy_parser *fyp) int rc, last_token_activity_counter; bool have_simple_keys; + /* do not try to peek */ + if (fy_reader_generates_events(fyp->reader)) + return NULL; + /* nothing if stream end produced (and no stream end token in queue) */ if (fyp->stream_end_produced) { fyt = fy_token_list_head(&fyp->queued_tokens); @@ -5163,6 +5316,9 @@ struct fy_token *fy_scan_peek(struct fy_parser *fyp) fyp_error_check(fyp, !rc, err_out, "fy_fetch_tokens() failed"); + if (fy_reader_generates_events(fyp->reader)) + return NULL; + fyp_error_check(fyp, last_token_activity_counter != fyp->token_activity_counter, err_out, "out of tokens and failed to produce anymore"); } @@ -5626,14 +5782,36 @@ static struct fy_eventp *fy_parse_internal(struct fy_parser *fyp) fy_token_list_init(&tag_directives); /* are we done? */ - if (fyp->stream_error || fyp->state == FYPS_END) + if (fyp->stream_error || fyp->state == FYPS_END) { + fy_parser_debug(fyp, "%s: NULL: fyp->stream_error=%s fyp->state=%s", __func__, + fyp->stream_error ? "true" : "false", state_txt[fyp->state]); return NULL; + } fyt = fy_scan_peek(fyp); +#ifdef FY_DEVMODE + fy_parser_debug(fyp, "[%s]", state_txt[fyp->state]); + fyp_debug_dump_token(fyp, fyt, "scan_peek: "); +#endif + + if (!fyt && fy_reader_generates_events(fyp->reader)) { + fye = fy_reader_generate_next_event(fyp->reader); + if (!fye) { + rc = fy_reader_input_done(fyp->reader); + fyp_error_check(fyp, !rc, err_out, + "fy_parse_input_done() failed"); + + fy_parser_debug(fyp, "%s: NULL: !fyt && fy_reader_generates_events(fyp->reader)", __func__); + return NULL; + } + return container_of(fye, struct fy_eventp, e); + } /* special case without an error message for start */ - if (!fyt && fyp->state == FYPS_NONE) + if (!fyt && fyp->state == FYPS_NONE) { + fy_parser_debug(fyp, "%s: NULL: !fyt && fyp->state==FYPS_NONE", __func__); return NULL; + } /* keep a copy of stream end */ if (fyt && fyt->type == FYTT_STREAM_END && !fyp->stream_end_token) { @@ -6524,9 +6702,7 @@ static struct fy_eventp *fy_parse_internal(struct fy_parser *fyp) break; case FYPS_END: - /* should never happen */ - assert(0); - break; + FY_IMPOSSIBLE_ABORT(); } assert(fyep); @@ -6546,6 +6722,8 @@ err_out: fy_token_list_unref_all_rl(fyp->recycled_token_list, &tag_directives); fy_parse_eventp_recycle(fyp, fyep); fyp->stream_error = true; + + fy_parser_debug(fyp, "%s: NULL: stream_error=true", __func__); return NULL; } @@ -6570,12 +6748,248 @@ const char *fy_event_type_get_text(enum fy_event_type type) return fy_event_type_txt[type]; } +#ifndef NDEBUG + +static void fy_parse_dump_eventp(struct fy_parser *fyp, struct fy_eventp *fyep, const char *pfx) +{ + struct fy_event *fye = NULL; + FILE *fp; + char *mbuf = NULL; + const char *anchor = NULL; + const char *tag = NULL; + const char *text = NULL; + const char *alias = NULL; + size_t msize, anchor_len = 0, tag_len = 0, text_len = 0, alias_len = 0; + enum fy_scalar_style style; + const uint8_t *p; + int i, c, w; + + /* perform the enable tests early to avoid the overhead */ + if ((FYET_DEBUG >> FYDF_LEVEL_SHIFT) < fyp->diag->cfg.level) + return; + + fp = open_memstream(&mbuf, &msize); + if (!fp) + return; + + if (!fyep) { + fprintf(fp, ""); + goto out; + } + fye = &fyep->e; + + /* event type */ + switch (fye->type) { + case FYET_NONE: + fprintf(fp, "???"); + break; + case FYET_STREAM_START: + fprintf(fp, "+STR"); + break; + case FYET_STREAM_END: + fprintf(fp, "-STR"); + break; + case FYET_DOCUMENT_START: + fprintf(fp, "+DOC"); + break; + case FYET_DOCUMENT_END: + fprintf(fp, "-DOC"); + break; + case FYET_MAPPING_START: + fprintf(fp, "+MAP"); + if (fye->mapping_start.anchor) + anchor = fy_token_get_text(fye->mapping_start.anchor, &anchor_len); + if (fye->mapping_start.tag) + tag = fy_token_get_text(fye->mapping_start.tag, &tag_len); + if (fy_event_get_node_style(fye) == FYNS_FLOW) + fprintf(fp, " {}"); + break; + case FYET_MAPPING_END: + fprintf(fp, "-MAP"); + break; + case FYET_SEQUENCE_START: + fprintf(fp, "+SEQ"); + if (fye->sequence_start.anchor) + anchor = fy_token_get_text(fye->sequence_start.anchor, &anchor_len); + if (fye->sequence_start.tag) + tag = fy_token_get_text(fye->sequence_start.tag, &tag_len); + if (fy_event_get_node_style(fye) == FYNS_FLOW) + fprintf(fp, " []"); + break; + case FYET_SEQUENCE_END: + fprintf(fp, "-SEQ"); + break; + case FYET_SCALAR: + fprintf(fp, "=VAL"); + if (fye->scalar.anchor) + anchor = fy_token_get_text(fye->scalar.anchor, &anchor_len); + if (fye->scalar.tag) + tag = fy_token_get_text(fye->scalar.tag, &tag_len); + break; + case FYET_ALIAS: + fprintf(fp, "=ALI"); + break; + default: + break; + } + + /* (position) anchor and tag */ + if (anchor) + fprintf(fp, " &%.*s", (int)anchor_len, anchor); + if (tag) + fprintf(fp, " <%.*s>", (int)tag_len, tag); + + /* style hint */ + switch (fye->type) { + default: + break; + case FYET_DOCUMENT_START: + if (!fy_document_event_is_implicit(fye)) + fprintf(fp, " ---"); + break; + case FYET_DOCUMENT_END: + if (!fy_document_event_is_implicit(fye)) + fprintf(fp, " ..."); + break; + case FYET_MAPPING_START: + break; + case FYET_SEQUENCE_START: + break; + case FYET_SCALAR: + style = fy_token_scalar_style(fye->scalar.value); + switch (style) { + case FYSS_PLAIN: + fprintf(fp, " :"); + break; + case FYSS_SINGLE_QUOTED: + fprintf(fp, " '"); + break; + case FYSS_DOUBLE_QUOTED: + fprintf(fp, "\""); + break; + case FYSS_LITERAL: + fprintf(fp, " |"); + break; + case FYSS_FOLDED: + fprintf(fp, " >"); + break; + default: + break; + } + break; + case FYET_ALIAS: + break; + } + + /* content */ + switch (fye->type) { + default: + break; + case FYET_SCALAR: + text = fy_token_get_text(fye->scalar.value, &text_len); + for (p = (const uint8_t *)text; text_len > 0; p += w, text_len -= (size_t)w) { + + /* get width from the first octet */ + w = (p[0] & 0x80) == 0x00 ? 1 : + (p[0] & 0xe0) == 0xc0 ? 2 : + (p[0] & 0xf0) == 0xe0 ? 3 : + (p[0] & 0xf8) == 0xf0 ? 4 : 0; + + /* error, clip it */ + if ((size_t)w > text_len) + break; + + /* initial value */ + c = p[0] & (0xff >> w); + for (i = 1; i < w; i++) { + if ((p[i] & 0xc0) != 0x80) + break; + c = (c << 6) | (p[i] & 0x3f); + } + + /* check for validity */ + if ((w == 4 && c < 0x10000) || + (w == 3 && c < 0x800) || + (w == 2 && c < 0x80) || + (c >= 0xd800 && c <= 0xdfff) || c >= 0x110000) + break; + + switch (c) { + case '\\': + fprintf(fp, "\\\\"); + break; + case '\0': + fprintf(fp, "\\0"); + break; + case '\b': + fprintf(fp, "\\b"); + break; + case '\f': + fprintf(fp, "\\f"); + break; + case '\n': + fprintf(fp, "\\n"); + break; + case '\r': + fprintf(fp, "\\r"); + break; + case '\t': + fprintf(fp, "\\t"); + break; + case '\a': + fprintf(fp, "\\a"); + break; + case '\v': + fprintf(fp, "\\v"); + break; + case '\e': + fprintf(fp, "\\e"); + break; + case 0x85: + fprintf(fp, "\\N"); + break; + case 0xa0: + fprintf(fp, "\\_"); + break; + case 0x2028: + fprintf(fp, "\\L"); + break; + case 0x2029: + fprintf(fp, "\\P"); + break; + default: + if ((c >= 0x01 && c <= 0x1f) || c == 0x7f || /* C0 */ + (c >= 0x80 && c <= 0x9f)) /* C1 */ + fprintf(fp, "\\x%02x", c); + else + fprintf(fp, "%.*s", w, p); + break; + } + } + break; + case FYET_ALIAS: + alias = fy_token_get_text(fye->alias.anchor, &alias_len); + fprintf(fp, " %s%.*s", "*", (int)alias_len, alias); + break; + } + +out: + fclose(fp); + fyp_parse_debug(fyp, "%s%s", pfx, mbuf); + free(mbuf); +} + +#else +static inline void +fy_parse_dump_eventp(struct fy_parser *fyp, struct fy_eventp *fyep, const char *pfx) { } +#endif + struct fy_eventp *fy_parse_private(struct fy_parser *fyp) { struct fy_eventp *fyep = NULL; fyep = fy_parse_internal(fyp); - fyp_parse_debug(fyp, "> %s", fyep ? fy_event_type_txt[fyep->e.type] : "NULL"); + fy_parse_dump_eventp(fyp, fyep, "gen> "); return fyep; } @@ -6585,9 +6999,6 @@ struct fy_parser *fy_parser_create(const struct fy_parse_cfg *cfg) struct fy_parser *fyp; int rc; - if (!cfg) - return NULL; - fyp = malloc(sizeof(*fyp)); if (!fyp) return NULL; @@ -6654,6 +7065,7 @@ static void fy_parse_input_reset(struct fy_parser *fyp) fyin = fy_input_next(&fyp->queued_inputs, fyi); fy_input_unref(fyi); } + fy_input_list_init(&fyp->queued_inputs); fy_parse_parse_state_log_list_recycle_all(fyp, &fyp->state_stack); @@ -6876,10 +7288,42 @@ err_out_rc: return rc; } -int fy_parser_reset(struct fy_parser *fyp) +int fy_parser_set_document_iterator(struct fy_parser *fyp, enum fy_parser_event_generator_flags flags, + struct fy_document_iterator *fydi) { + struct fy_input_cfg fyic; int rc; + if (!fyp || !fydi) + return -1; + + memset(&fyic, 0, sizeof(fyic)); + + fyic.type = fyit_dociter; + fyic.dociter.flags = flags; + fyic.dociter.fydi = fydi; + fyic.dociter.owns_iterator = false; + + /* must not be in the middle of something */ + fyp_error_check(fyp, fyp->state == FYPS_NONE || fyp->state == FYPS_END, + err_out, "parser cannot be reset at state '%s'", + state_txt[fyp->state]); + + fy_parse_input_reset(fyp); + + rc = fy_parse_input_append(fyp, &fyic); + fyp_error_check(fyp, !rc, err_out_rc, + "fy_parse_input_append() failed"); + + return 0; +err_out: + rc = -1; +err_out_rc: + return rc; +} + +int fy_parser_reset(struct fy_parser *fyp) +{ if (!fyp) return -1; @@ -6900,13 +7344,29 @@ int fy_parser_reset(struct fy_parser *fyp) assert(fyp->diag); fyp->diag->on_error = false; - rc = fy_reset_document_state(fyp); - fyp_error_check(fyp, !rc, err_out_rc, - "fy_parse_input_reset() failed"); + fy_document_state_unref(fyp->current_document_state); + fyp->current_document_state = NULL; + + fy_document_state_unref(fyp->default_document_state); + fyp->default_document_state = NULL; + + fy_parse_streaming_aliases_reset(fyp); + + fy_input_unref(fyp->last_event_handle.fyi); + fy_atom_reset(&fyp->last_event_handle); + + fy_parse_indent_list_recycle_all(fyp, &fyp->indent_stack); + fy_parse_simple_key_list_recycle_all(fyp, &fyp->simple_keys); + fy_token_list_unref_all(&fyp->queued_tokens); + + fy_parse_parse_state_log_list_recycle_all(fyp, &fyp->state_stack); + fy_parse_flow_list_recycle_all(fyp, &fyp->flow_stack); + fy_parse_streaming_alias_list_recycle_all(fyp, &fyp->streaming_aliases); + + fy_token_unref_rl(fyp->recycled_token_list, fyp->stream_end_token); + fyp->stream_end_token = NULL; return 0; -err_out_rc: - return rc; } struct fy_event *fy_parser_parse(struct fy_parser *fyp) @@ -6917,6 +7377,13 @@ struct fy_event *fy_parser_parse(struct fy_parser *fyp) if (!fyp) return NULL; + if (fyp->fyep_peek) { + fyep = fyp->fyep_peek; + fyp->fyep_peek = NULL; + fy_parse_dump_eventp(fyp, fyep, "peek> "); + return &fyep->e; + } + if ((fyp->cfg.flags & FYPCF_RESOLVE_DOCUMENT) && fy_reader_get_mode(fyp->reader) != fyrm_json) { fyep = fy_parser_parse_resolve_prolog(fyp); if (fyep) @@ -6948,6 +7415,95 @@ struct fy_event *fy_parser_parse(struct fy_parser *fyp) return &fyep->e; } +const struct fy_event *fy_parser_parse_peek(struct fy_parser *fyp) +{ + struct fy_event *fye; + + if (!fyp->fyep_peek) { + fye = fy_parser_parse(fyp); + if (!fye) + return NULL; + + fyp->fyep_peek = container_of(fye, struct fy_eventp, e); + } + return &fyp->fyep_peek->e; +} + +int fy_parser_skip(struct fy_parser *fyp) +{ + struct fy_event *fye = 0; + enum fy_event_type start_type, end_type; + int nest, map_nest, seq_nest; + + /* check that we start on a scalar/seq/map */ + fye = fy_parser_parse(fyp); + if (!fye) + goto err_out; + + start_type = fye->type; + fy_parser_event_free(fyp, fye); + fye = NULL; + + if (start_type == FYET_SCALAR) + return 0; + + map_nest = 0; + seq_nest = 0; + if (start_type == FYET_SEQUENCE_START) { + end_type = FYET_SEQUENCE_END; + seq_nest = 1; + } else if (start_type == FYET_MAPPING_START) { + end_type = FYET_MAPPING_END; + map_nest = 1; + } else + goto err_out; + + nest = 1; + do { + fye = fy_parser_parse(fyp); + if (!fye) + goto err_out; + + switch (fye->type) { + case FYET_SEQUENCE_START: + seq_nest++; + break; + case FYET_MAPPING_START: + map_nest++; + break; + case FYET_SEQUENCE_END: + seq_nest--; + break; + case FYET_MAPPING_END: + map_nest--; + break; + default: + break; + } + + if (fye->type == start_type) + nest++; + else if (fye->type == end_type) + nest--; + + if (map_nest < 0 || seq_nest < 0) { + fy_event_report(fyp, fye, FYEP_VALUE, FYET_ERROR, + "unbalanced collection while skipping"); + goto err_out; + } + + fy_parser_event_free(fyp, fye); + fye = NULL; + + } while (nest > 0); + + return 0; + +err_out: + fy_parser_event_free(fyp, fye); + return -1; +} + bool fy_parser_get_stream_error(struct fy_parser *fyp) { if (!fyp) @@ -6979,7 +7535,7 @@ parse_process_event(struct fy_composer *fyc, struct fy_path *path, struct fy_eve return fyp->fyc_cb(fyp, fye, path, fyp->fyc_userdata); } -struct fy_document_builder * +static struct fy_document_builder * parse_create_document_builder(struct fy_composer *fyc) { struct fy_parser *fyp = fy_composer_get_cfg_userdata(fyc); @@ -7089,6 +7645,9 @@ static enum fy_composer_return fy_parse_compose_internal(struct fy_parser *fyp) if (ret != FYCR_OK_CONTINUE) break; } + if (fyp->stream_error) + fy_composer_halt(fyc, FYCR_ERROR); + return ret; } @@ -8016,3 +8575,554 @@ struct fy_eventp *fy_parser_parse_resolve_epilog(struct fy_parser *fyp, struct f return fyep; } + +struct fy_parser_checkpoint *fy_parser_checkpoint_create(struct fy_parser *fyp) +{ + struct fy_parser_checkpoint *fypchk = NULL; + struct fy_parser *fypc = NULL; + struct fy_reader *fyr, *fyrc; + struct fy_streaming_alias *fysa, *fysac; + struct fy_eventp *fyep, *fyepc; + struct fy_indent *fyit, *fyitc; + struct fy_simple_key *fysk, *fyskc; + struct fy_parse_state_log *fypsl, *fypslc; + struct fy_flow *fyf, *fyfc; + struct fy_token *fyt; + struct fy_input *fyi; + int i; + + /* must be a parser, in a document and not a checkpoint of one, + * with no composer or builder, neither in a merge key state */ + if (!fyp || /* fyp->state != FYPS_DOCUMENT_CONTENT || */ + fyp->is_checkpoint || fyp->fydb || fyp->fyc || + fyp->mks.active) { + fyp_parse_debug(fyp, "cannot take checkpoint at this state"); + return NULL; + } + + /* no queued inputs */ + if (!fy_input_list_empty(&fyp->queued_inputs)) + return NULL; + + fyp_parse_debug(fyp, "taking checkpoint"); + + fypchk = malloc(sizeof(*fypchk)); + fyp_error_check(fyp, fypchk != NULL, err_out, + "fy_parse_input_create() failed!"); + memset(fypchk, 0, sizeof(*fypchk)); + fypc = &fypchk->fyp_checkpoint; + + /* copy everything over and then fill out the pointer */ + *fypc = *fyp; + + fypc->is_checkpoint = true; /* mark it as a checkpoint */ + + /* all recycled lists are initialized */ + fy_indent_list_init(&fypc->recycled_indent); + fy_parse_state_log_list_init(&fypc->recycled_parse_state_log); + fy_eventp_list_init(&fypc->recycled_eventp); + fy_token_list_init(&fypc->recycled_token); + fy_flow_list_init(&fypc->flow_stack); + fy_flow_list_init(&fypc->recycled_flow); + fy_streaming_alias_list_init(&fypc->recycled_streaming_alias); + fy_simple_key_list_init(&fypc->recycled_simple_key); + fy_eventp_list_init(&fypc->mks.args); + fypc->recycled_eventp_list = NULL; + fypc->recycled_token_list = NULL; + + fy_input_list_init(&fypc->queued_inputs); + fypc->stream_end_token = fy_token_ref(fyp->stream_end_token); + + fypc->last_comment.fyi = fy_input_ref(fyp->last_comment.fyi); + fypc->current_document_state = fy_document_state_ref(fyp->current_document_state); + fypc->default_document_state = fy_document_state_ref(fyp->default_document_state); + fypc->next_single_document = fyp->next_single_document; + fypc->diag = NULL; + fypc->last_event_handle.fyi = fy_input_ref(fyp->last_event_handle.fyi); + + if (fyp->fyep_peek) { + fypc->fyep_peek = fy_parse_eventp_clone(fyp, fyp->fyep_peek, false); + fyp_parse_debug(fyp, "checkpoint has a peeked event"); + } + + /* copy the current state of the reader to the builtin reader space */ + fyr = fyp->reader; + assert(fyr); + fyrc = &fypc->builtin_reader; + fypc->reader = fyrc; + + *fyrc = *fyr; + fyrc->current_input = fy_input_ref(fyr->current_input); + assert(fyrc->current_input); + fyrc->diag = NULL; /* no diag */ + + /* copy the streaming alias list */ + if (fyp->sas.stack) { + fyp_parse_debug(fyp, "checkpoint has a streaming alias list"); + if (fyp->sas.alloc <= (int)ARRAY_SIZE(fyp->sas.local)) + fypc->sas.stack = fypc->sas.local; + else { + fypc->sas.stack = malloc(sizeof(*fyp->sas.stack) * fyp->sas.alloc); + assert(fypc->sas.stack); + memcpy(fypc->sas.stack, fyp->sas.stack, sizeof(*fyp->sas.stack) * fyp->sas.top); + } + } + + fy_streaming_alias_list_init(&fypc->streaming_aliases); + for (fysa = fy_streaming_alias_list_head(&fyp->streaming_aliases); fysa != NULL; + fysa = fy_streaming_alias_next(&fyp->streaming_aliases, fysa)) { + + fysac = fy_parse_streaming_alias_create(fyp, NULL); + assert(fysac); + fysac->anchor = fy_token_ref(fysa->anchor); + for (fyep = fy_eventp_list_head(&fysa->events); fyep; + fyep = fy_eventp_next(&fysa->events, fyep)) { + + fyepc = fy_parse_eventp_clone(fyp, fyep, false); + assert(fyepc); + fy_eventp_list_add_tail(&fysac->events, fyepc); + + /* scan stack and replace pointers */ + for (i = 0; i < fypc->sas.top; i++) { + if (fypc->sas.stack[i].fysa == fysa) + fypc->sas.stack[i].fysa = fysac; + if (fypc->sas.stack[i].next == fyep) + fypc->sas.stack[i].next = fyepc; + } + } + } + + if (fyp->cts.stack) { + fyp_parse_debug(fyp, "checkpoint has a cts stack"); + if (fyp->cts.alloc <= (int)ARRAY_SIZE(fyp->cts.local)) + fypc->cts.stack = fypc->cts.local; + else { + fypc->cts.stack = malloc(sizeof(*fyp->cts.stack) * fyp->cts.alloc); + assert(fypc->cts.stack); + memcpy(fypc->cts.stack, fyp->cts.stack, sizeof(*fyp->cts.stack) * fyp->cts.top); + } + } + + /* copy all remaining lists */ + fy_indent_list_init(&fypc->indent_stack); + for (fyit = fy_indent_list_head(&fyp->indent_stack); fyit != NULL; + fyit = fy_indent_next(&fyp->indent_stack, fyit)) { + + fyitc = fy_parse_indent_alloc(fyp); + assert(fyitc); + + *fyitc = *fyit; + + fy_indent_list_add_tail(&fypc->indent_stack, fyitc); + } + + fy_parse_state_log_list_init(&fypc->state_stack); + for (fypsl = fy_parse_state_log_list_head(&fyp->state_stack); fypsl != NULL; + fypsl = fy_parse_state_log_next(&fyp->state_stack, fypsl)) { + + fypslc = fy_parse_parse_state_log_alloc(fyp); + assert(fypslc); + + fypslc->state = fypsl->state; + + fyp_parse_debug(fyp, "checkpoint saving parse_state %s\n", state_txt[fypslc->state]); + + fy_parse_state_log_list_add_tail(&fypc->state_stack, fypslc); + } + + /* queued tokens and simple keys are 'special' */ + fy_token_list_init(&fypc->queued_tokens); + /* count, and take refs */ + for (fyt = fy_token_list_head(&fyp->queued_tokens), i = 0; fyt != NULL; + fyt = fy_token_next(&fyp->queued_tokens, fyt), i++) { } + fypchk->queued_token_count = i; + fypchk->queued_tokens = malloc(i * sizeof(*fypchk->queued_tokens)); + assert(fypchk->queued_tokens); + for (fyt = fy_token_list_head(&fyp->queued_tokens), i = 0; fyt != NULL; + fyt = fy_token_next(&fyp->queued_tokens, fyt), i++) { + fypchk->queued_tokens[i] = fy_token_ref(fyt); + } + fyp_parse_debug(fyp, "checkpoint has #%d queued tokens", fypchk->queued_token_count); + + fy_simple_key_list_init(&fypc->simple_keys); + + if (!fy_simple_key_list_empty(&fyp->simple_keys)) + fyp_parse_debug(fyp, "checkpoint has pending simple keys"); + + for (fysk = fy_simple_key_list_head(&fyp->simple_keys); fysk != NULL; + fysk = fy_simple_key_next(&fyp->simple_keys, fysk)) { + + fyskc = fy_parse_simple_key_alloc(fyp); + assert(fyskc); + + *fyskc = *fysk; + fyskc->token = fy_token_ref(fysk->token); + + fy_simple_key_list_add_tail(&fypc->simple_keys, fyskc); + } + + fy_input_list_init(&fypc->queued_inputs); + /* count, and take refs */ + for (fyi = fy_input_list_head(&fyp->queued_inputs), i = 0; fyi != NULL; + fyi = fy_input_next(&fyp->queued_inputs, fyi), i++) { } + fypchk->queued_input_count = i; + fypchk->queued_inputs = malloc(i * sizeof(*fypchk->queued_inputs)); + assert(fypchk->queued_inputs); + for (fyi = fy_input_list_head(&fyp->queued_inputs), i = 0; fyi != NULL; + fyi = fy_input_next(&fyp->queued_inputs, fyi), i++) { + fypchk->queued_inputs[i] = fy_input_ref(fyi); + } + + fyp_parse_debug(fyp, "checkpoint flow_level=%d", fypc->flow_level); + fypc->flow = fyp->flow; + if (!fy_flow_list_empty(&fyp->flow_stack)) + fyp_parse_debug(fyp, "checkpoint has flow state"); + + for (fyf = fy_flow_list_head(&fyp->flow_stack); fyf != NULL; + fyf = fy_flow_next(&fyp->flow_stack, fyf)) { + + fyfc = fy_parse_flow_alloc(fyp); + assert(fyfc); + + *fyfc = *fyf; + + fy_flow_list_add_tail(&fypc->flow_stack, fyfc); + } + + fypchk->diag_on_error = fyp->diag && fyp->diag->on_error; + + fyp_parse_debug(fyp, "took checkpoint"); + return fypchk; + +err_out: + if (fypchk) + free(fypchk); + return NULL; +} + +void fy_parser_checkpoint_destroy(struct fy_parser_checkpoint *fypchk) +{ + struct fy_parser *fypc; + int i; + + if (!fypchk) + return; + + if (fypchk->queued_inputs) { + for (i = 0; i < fypchk->queued_input_count; i++) + fy_input_unref(fypchk->queued_inputs[i]); + free(fypchk->queued_inputs); + } + + if (fypchk->queued_tokens) { + for (i = 0; i < fypchk->queued_token_count; i++) + fy_token_unref(fypchk->queued_tokens[i]); + free(fypchk->queued_tokens); + } + + fypc = &fypchk->fyp_checkpoint; + + fy_parse_cleanup(fypc); + + free(fypchk); +} + +int fy_parser_rollback(struct fy_parser *fyp, struct fy_parser_checkpoint *fypchk) +{ + struct fy_parser *fypc = NULL; + struct fy_reader *fyr, *fyrc; + struct fy_indent *fyit, *fyitc; + struct fy_flow *fyf, *fyfc; + struct fy_parse_state_log *fypsl, *fypslc; + struct fy_input *fyi, *fyic; + struct fy_token *fyt; + int i; + + if (!fypchk) + return -1; + + fyp_parse_debug(fyp, "rolling back checkpoint"); + + fypc = &fypchk->fyp_checkpoint; + + if (fyp->diag) + fyp->diag->on_error = fypchk->diag_on_error; + + /* reader restore */ + fyr = fyp->reader; + fyrc = fypc->reader; + + assert(fyr->ops == fyrc->ops); + fy_input_unref(fyr->current_input); + fyr->current_input = fy_input_ref(fyrc->current_input); + fyr->mode = fyrc->mode; + fyr->this_input_start = fyrc->this_input_start; + fyr->current_ptr = fyrc->current_ptr; + fyr->current_ptr_start =fyrc->current_ptr_start; + fyr->current_ptr_end =fyrc->current_ptr_end; + fyr->line = fyrc->line; + fyr->column = fyrc->column; + fyr->tabsize = fyrc->tabsize; + fyr->json_mode = fyrc->json_mode; + fyr->lb_mode = fyrc->lb_mode; + fyr->fws_mode = fyrc->fws_mode; + fyr->directive0_mode = fyrc->directive0_mode; + + fy_token_unref(fyp->stream_end_token); + fyp->stream_end_token = fy_token_ref(fypc->stream_end_token); + + /* inputs restore */ + while ((fyi = fy_input_list_pop(&fyp->queued_inputs)) != NULL) + fy_input_unref(fyi); + for (i = 0; i < fypchk->queued_input_count; i++) { + fyic = fypchk->queued_inputs[i]; + fy_input_list_add_tail(&fyp->queued_inputs, fy_input_ref(fyic)); + } + fyp->token_activity_counter = fypc->token_activity_counter; + + fy_input_unref(fyp->last_comment.fyi); + fyp->last_comment = fypc->last_comment; + fyp->last_comment.fyi = fy_input_ref(fypc->last_comment.fyi); + + /* indents */ + while ((fyit = fy_indent_list_pop(&fyp->indent_stack)) != NULL) + fy_parse_indent_recycle(fyp, fyit); + for (fyitc = fy_indent_list_head(&fypc->indent_stack); fyitc != NULL; + fyitc = fy_indent_next(&fypc->indent_stack, fyitc)) { + + fyit = fy_parse_indent_alloc(fyp); + assert(fyit); + + *fyit = *fyitc; + + fy_indent_list_add_tail(&fyp->indent_stack, fyit); + } + fyp->indent = fypc->indent; + fyp->parent_indent = fypc->parent_indent; + fyp->indent_line = fypc->indent_line; + fyp->starting_indent = fypc->starting_indent; + + /* parse state */ + fyp->state = fypc->state; + while ((fypsl = fy_parse_state_log_list_pop(&fyp->state_stack)) != NULL) + fy_parse_parse_state_log_recycle(fyp, fypsl); + for (fypslc = fy_parse_state_log_list_head(&fypc->state_stack); fypslc != NULL; + fypslc = fy_parse_state_log_next(&fypc->state_stack, fypslc)) { + + fypsl = fy_parse_parse_state_log_alloc(fyp); + assert(fypsl); + + fypsl->state = fypslc->state; + + fy_parse_state_log_list_add_tail(&fyp->state_stack, fypsl); + } + +#ifndef NDEBUG + for (fypsl = fy_parse_state_log_list_head(&fyp->state_stack); fypsl != NULL; + fypsl = fy_parse_state_log_next(&fyp->state_stack, fypsl)) { + fyp_parse_debug(fyp, "checkpoint restored parse_state %s\n", state_txt[fypsl->state]); + } +#endif + + while ((fyt = fy_token_list_pop(&fyp->queued_tokens)) != NULL) + fy_token_unref(fyt); + for (i = 0; i < fypchk->queued_token_count; i++) { + fyt = fypchk->queued_tokens[i]; + fy_token_list_add_tail(&fyp->queued_tokens, fy_token_ref(fyt)); + } + assert(fy_simple_key_list_empty(&fypc->simple_keys)); + assert(fy_streaming_alias_list_empty(&fypc->streaming_aliases)); + + fyp->default_version = fypc->default_version; + fyp->stream_version = fypc->stream_version; + fyp->parser_bitflags = fypc->parser_bitflags; + fyp->is_checkpoint = false; + + fyp_parse_debug(fyp, "checkpoint restoring flow_level=%d", fypc->flow_level); + + fyp->flow_level = fypc->flow_level; + fyp->flow = fypc->flow; + while ((fyf = fy_flow_list_pop(&fyp->flow_stack)) != NULL) + fy_parse_flow_recycle(fyp, fyf); + for (fyfc = fy_flow_list_head(&fypc->flow_stack); fyfc != NULL; + fyfc = fy_flow_next(&fypc->flow_stack, fyfc)) { + + fyf = fy_parse_flow_alloc(fyp); + assert(fyf); + + *fyf = *fyfc; + + fy_flow_list_add_tail(&fyp->flow_stack, fyf); + } + + fyp->pending_complex_key_column = fypc->pending_complex_key_column; + fyp->pending_complex_key_mark = fypc->pending_complex_key_mark; + fyp->last_block_mapping_key_line = fypc->last_block_mapping_key_line; + fyp->last_comma_mark = fypc->last_comma_mark; + fyp->last_tab_used_for_ws_mark = fypc->last_tab_used_for_ws_mark; + + fy_document_state_unref(fyp->current_document_state); + fyp->current_document_state = fy_document_state_ref(fypc->current_document_state); + fy_document_state_unref(fyp->default_document_state); + fyp->default_document_state = fy_document_state_ref(fypc->default_document_state); + + fyp->next_single_document = fypc->next_single_document; + fyp->last_event_handle = fypc->last_event_handle; + fy_input_unref(fyp->last_event_handle.fyi); + fyp->last_event_handle.fyi = fy_input_ref(fypc->last_event_handle.fyi); + + fy_parse_eventp_recycle(fyp, fyp->fyep_peek); + fyp->fyep_peek = NULL; + if (fypc->fyep_peek) { + fyp->fyep_peek = fy_parse_eventp_clone(fyp, fypc->fyep_peek, false); + assert(fypc->fyep_peek); + } + + assert(fyp->sas.top == 0); + + return 0; +} + +enum fy_parser_mode +fy_parser_get_mode(struct fy_parser *fyp) +{ + struct fy_version *versp; + int major, minor; + + if (!fyp) + return fypm_invalid; + + versp = fyp->current_document_state ? &fyp->current_document_state->version : &fyp->stream_version; + + switch (fy_reader_get_mode(fyp->reader)) { + case fyrm_yaml: + case fyrm_yaml_1_1: + major = versp->major; + minor = versp->minor; + /* 1.1, 1.2, 1.3 */ + if (major != 1 || minor < 1 || minor > 3) + break; + return fypm_yaml_1_1 + (minor - 1); + + case fyrm_json: + return fypm_json; + } + + return fypm_none; +} + +int fy_parser_count_sequence_items(struct fy_parser *fyp) +{ + const struct fy_event *fye_peek; + struct fy_parser_checkpoint *fypchk = NULL; + int rc, count; + + if (!fyp) + return -1; + + /* take a checkpoint and race ahead to find the size */ + fypchk = fy_parser_checkpoint_create(fyp); + fyp_error_check(fyp, fypchk != NULL, err_out, + "fy_parser_checkpoint_create() failed!"); + count = 0; + for (;;) { + fye_peek = fy_parser_parse_peek(fyp); + if (!fye_peek) + goto err_out; + if (fye_peek->type != FYET_SCALAR && + fye_peek->type != FYET_MAPPING_START && + fye_peek->type != FYET_SEQUENCE_START && + fye_peek->type != FYET_SEQUENCE_END) + goto err_out; + + if (fye_peek->type == FYET_SEQUENCE_END) + break; + + /* skip the item */ + rc = fy_parser_skip(fyp); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_skip() failed!"); + count++; + } + + rc = fy_parser_rollback(fyp, fypchk); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_rollback() failed!"); +out: + fy_parser_checkpoint_destroy(fypchk); + return count; + +err_out: + count = -1; + goto out; +} + +int fy_parser_count_mapping_items(struct fy_parser *fyp) +{ + const struct fy_event *fye_peek; + struct fy_parser_checkpoint *fypchk = NULL; + int rc, count; + + if (!fyp) + return -1; + + /* take a checkpoint and race ahead to find the size */ + fypchk = fy_parser_checkpoint_create(fyp); + fyp_error_check(fyp, fypchk != NULL, err_out, + "fy_parser_checkpoint_create() failed!"); + count = 0; + for (;;) { + fye_peek = fy_parser_parse_peek(fyp); + if (!fye_peek) + goto err_out; + if (fye_peek->type != FYET_SCALAR && + fye_peek->type != FYET_MAPPING_START && + fye_peek->type != FYET_SEQUENCE_START && + fye_peek->type != FYET_MAPPING_END) + goto err_out; + + if (fye_peek->type == FYET_MAPPING_END) + break; + + /* skip the key item */ + rc = fy_parser_skip(fyp); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_skip() failed!"); + + /* skip the value item */ + rc = fy_parser_skip(fyp); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_skip() failed!"); + count++; + } + + rc = fy_parser_rollback(fyp, fypchk); + fyp_error_check(fyp, !rc, err_out, + "fy_parser_rollback() failed!"); +out: + fy_parser_checkpoint_destroy(fypchk); + return count; + +err_out: + count = -1; + goto out; +} + +// just for LLVMShutdown +#if defined(HAVE_LIBCLANG) && HAVE_LIBCLANG +#include +#endif + +void fy_shutdown(void) +{ +#if defined(HAVE_LIBCLANG) && HAVE_LIBCLANG + LLVMShutdown(); +#endif +} + +#ifdef FY_HAS_DESTRUCTOR +FY_DESTRUCTOR void fy_do_cleanup(void) +{ + fy_shutdown(); +} +#endif diff --git a/src/lib/fy-parse.h b/src/lib/fy-parse.h index d7e502d..d269f58 100644 --- a/src/lib/fy-parse.h +++ b/src/lib/fy-parse.h @@ -175,23 +175,32 @@ struct fy_parser { struct fy_reader *reader; /* the external reader, or ptr to builtin_reader */ struct fy_version default_version; - bool suppress_recycling : 1; - bool stream_start_produced : 1; - bool stream_end_produced : 1; - bool stream_end_reached : 1; - bool simple_key_allowed : 1; - bool tab_used_for_ws : 1; - bool stream_error : 1; - bool generated_block_map : 1; - bool last_was_comma : 1; - bool document_has_content : 1; - bool document_first_content_token : 1; - bool bare_document_only : 1; /* no document start indicators allowed, no directives */ - bool stream_has_content : 1; - bool parse_flow_only : 1; /* document is in flow form, and stop parsing at the end */ - bool parse_block_only : 1; /* document is in embedded block form, and stop parsing at the end */ - bool colon_follows_colon : 1; /* "foo"::bar -> "foo": :bar */ - bool had_directives : 1; /* document had directives */ + struct fy_version stream_version; /* last version set in stream */ + + union { + struct { + bool suppress_recycling : 1; + bool stream_start_produced : 1; + bool stream_end_produced : 1; + bool stream_end_reached : 1; + bool simple_key_allowed : 1; + bool tab_used_for_ws : 1; + bool stream_error : 1; + bool generated_block_map : 1; + bool last_was_comma : 1; + bool document_has_content : 1; + bool document_first_content_token : 1; + bool bare_document_only : 1; /* no document start indicators allowed, no directives */ + bool stream_has_content : 1; + bool parse_flow_only : 1; /* document is in flow form, and stop parsing at the end */ + bool parse_block_only : 1; /* document is in embedded block form, and stop parsing at the end */ + bool colon_follows_colon : 1; /* "foo"::bar -> "foo": :bar */ + bool had_directives : 1; /* document had directives */ + bool is_checkpoint : 1; /* a parser checkpoint (not a real parser) */ + }; + unsigned int parser_bitflags; + }; + int flow_level; int pending_complex_key_column; struct fy_mark pending_complex_key_mark; @@ -281,6 +290,7 @@ struct fy_parser { struct fy_eventp_list args; struct fy_document *fyd; } mks; + struct fy_eventp *fyep_peek; }; static inline struct fy_input * @@ -331,6 +341,12 @@ static inline enum fy_flow_ws_mode fyp_fws_mode(const struct fy_parser *fyp) return fy_reader_flow_ws_mode(fyp->reader); } +static inline bool fyp_directive0_mode(const struct fy_parser *fyp) +{ + assert(fyp); + return fy_reader_directive0_mode(fyp->reader); +} + static inline bool fyp_block_mode(struct fy_parser *fyp) { return !fyp_json_mode(fyp) && !fyp->flow_level; @@ -390,13 +406,6 @@ static inline bool fyp_is_flow_blankz(const struct fy_parser *fyp, int c) return fy_reader_is_flow_blankz(fyp->reader, c); } -static inline const void * -fy_ptr_slow_path(struct fy_parser *fyp, size_t *leftp) -{ - assert(fyp); - return fy_reader_ptr_slow_path(fyp->reader, leftp); -} - static inline const void * fy_ensure_lookahead_slow_path(struct fy_parser *fyp, size_t size, size_t *leftp) { @@ -442,6 +451,13 @@ fy_parse_strncmp(struct fy_parser *fyp, const char *str, size_t n) return fy_reader_strncmp(fyp->reader, str, n); } +static FY_ALWAYS_INLINE inline int +fy_parse_peek_at_offset_width(struct fy_parser *fyp, size_t offset, int *wp) +{ + assert(fyp); + return fy_reader_peek_at_offset_width(fyp->reader, offset, wp); +} + static FY_ALWAYS_INLINE inline int fy_parse_peek_at_offset(struct fy_parser *fyp, size_t offset) { @@ -449,6 +465,13 @@ fy_parse_peek_at_offset(struct fy_parser *fyp, size_t offset) return fy_reader_peek_at_offset(fyp->reader, offset); } +static FY_ALWAYS_INLINE inline int +fy_parse_peek_at_width_internal(struct fy_parser *fyp, int pos, ssize_t *offsetp, int *wp) +{ + assert(fyp); + return fy_reader_peek_at_width_internal(fyp->reader, pos, offsetp, wp); +} + static FY_ALWAYS_INLINE inline int fy_parse_peek_at_internal(struct fy_parser *fyp, int pos, ssize_t *offsetp) { @@ -477,11 +500,25 @@ fy_is_generic_blankz_at_offset(struct fy_parser *fyp, size_t offset) return fy_reader_is_generic_blankz(fyp->reader, fy_reader_peek_at_offset(fyp->reader, offset)); } +static FY_ALWAYS_INLINE inline int +fy_parse_peek_at_width(struct fy_parser *fyp, int pos, int *wp) +{ + assert(fyp); + return fy_reader_peek_at_width(fyp->reader, pos, wp); +} + static FY_ALWAYS_INLINE inline int fy_parse_peek_at(struct fy_parser *fyp, int pos) { assert(fyp); - return fy_reader_peek_at_internal(fyp->reader, pos, NULL); + return fy_reader_peek_at(fyp->reader, pos); +} + +static FY_ALWAYS_INLINE inline int +fy_parse_peek_width(struct fy_parser *fyp, int *wp) +{ + assert(fyp); + return fy_reader_peek_width(fyp->reader, wp); } static FY_ALWAYS_INLINE inline int @@ -640,7 +677,7 @@ void fy_free_default(void *userdata, void *ptr); void *fy_realloc_default(void *userdata, void *ptr, size_t size); int fy_reader_fetch_flow_scalar_handle(struct fy_reader *fyr, int c, int indent, struct fy_atom *handle, bool sloppy_indent); -int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, int indent, int flow_level, struct fy_atom *handle, bool directive0); +int fy_reader_fetch_plain_scalar_handle(struct fy_reader *fyr, int c, int indent, int flow_level, struct fy_atom *handle); void fy_reader_skip_ws_cr_nl(struct fy_reader *fyr); @@ -662,4 +699,171 @@ void fy_parse_streaming_aliases_reset(struct fy_parser *fyp); struct fy_eventp *fy_parser_parse_resolve_prolog(struct fy_parser *fyp); struct fy_eventp *fy_parser_parse_resolve_epilog(struct fy_parser *fyp, struct fy_eventp *fyep); +struct fy_parser_checkpoint { + struct fy_parser fyp_checkpoint; + int queued_token_count; + struct fy_token **queued_tokens; + int queued_input_count; + struct fy_input **queued_inputs; + bool diag_on_error; +}; + +/* diagnostics */ + +static inline bool +fyp_debug_log_level_is_enabled(struct fy_parser *fyp, enum fy_error_module module) +{ + return fyp && fy_diag_log_level_is_enabled(fyp->diag, FYET_DEBUG, module); +} + +int fy_parser_vdiag(struct fy_parser *fyp, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, va_list ap); + +int fy_parser_diag(struct fy_parser *fyp, unsigned int flags, + const char *file, int line, const char *func, + const char *fmt, ...) + __attribute__((format(printf, 6, 7))); + +void fy_parser_diag_vreport(struct fy_parser *fyp, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, va_list ap); +void fy_parser_diag_report(struct fy_parser *fyp, + const struct fy_diag_report_ctx *fydrc, + const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +#ifdef FY_DEVMODE + +#define fyp_debug(_fyp, _module, _fmt, ...) \ + do { \ + struct fy_parser *__fyp = (_fyp); \ + enum fy_error_module __module = (_module); \ + \ + if (fyp_debug_log_level_is_enabled(__fyp, __module)) \ + fy_parser_diag(__fyp, FYET_DEBUG | FYDF_MODULE(__module), \ + __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__); \ + } while(0) +#else + +#define fyp_debug(_fyp, _module, _fmt, ...) \ + do { } while(0) + +#endif + +#define fyp_info(_fyp, _fmt, ...) \ + fy_parser_diag((_fyp), FYET_INFO, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyp_notice(_fyp, _fmt, ...) \ + fy_parser_diag((_fyp), FYET_NOTICE, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyp_warning(_fyp, _fmt, ...) \ + fy_parser_diag((_fyp), FYET_WARNING, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) +#define fyp_error(_fyp, _fmt, ...) \ + fy_parser_diag((_fyp), FYET_ERROR, __FILE__, __LINE__, __func__, \ + (_fmt) , ## __VA_ARGS__) + +#define fyp_scan_debug(_fyp, _fmt, ...) \ + fyp_debug((_fyp), FYEM_SCAN, (_fmt) , ## __VA_ARGS__) +#define fyp_parse_debug(_fyp, _fmt, ...) \ + fyp_debug((_fyp), FYEM_PARSE, (_fmt) , ## __VA_ARGS__) +#define fyp_doc_debug(_fyp, _fmt, ...) \ + fyp_debug((_fyp), FYEM_DOC, (_fmt) , ## __VA_ARGS__) + +#define fyp_error_check(_fyp, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + fyp_error((_fyp), _fmt, ## __VA_ARGS__); \ + goto _label ; \ + } \ + } while(0) + +#define _FYP_TOKEN_DIAG(_fyp, _fyt, _type, _module, _fmt, ...) \ + do { \ + struct fy_diag_report_ctx _drc; \ + memset(&_drc, 0, sizeof(_drc)); \ + _drc.type = (_type); \ + _drc.module = (_module); \ + _drc.fyt = (_fyt); \ + fy_parser_diag_report((_fyp), &_drc, (_fmt) , ## __VA_ARGS__); \ + } while(0) + +#define FYP_TOKEN_DIAG(_fyp, _fyt, _type, _module, _fmt, ...) \ + _FYP_TOKEN_DIAG(_fyp, fy_token_ref(_fyt), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYP_PARSE_DIAG(_fyp, _adv, _cnt, _type, _module, _fmt, ...) \ + _FYP_TOKEN_DIAG(_fyp, \ + fy_token_create(FYTT_INPUT_MARKER, \ + fy_fill_atom_at((_fyp), (_adv), (_cnt), \ + alloca(sizeof(struct fy_atom)))), \ + _type, _module, _fmt, ## __VA_ARGS__) + +#define FYP_MARK_DIAG(_fyp, _sm, _em, _type, _module, _fmt, ...) \ + _FYP_TOKEN_DIAG(_fyp, \ + fy_token_create(FYTT_INPUT_MARKER, \ + fy_fill_atom_mark(((_fyp)), (_sm), (_em), \ + alloca(sizeof(struct fy_atom)))), \ + _type, _module, _fmt, ## __VA_ARGS__) + +#define FYP_NODE_DIAG(_fyp, _fyn, _type, _module, _fmt, ...) \ + _FYP_TOKEN_DIAG(_fyp, fy_node_token(_fyn), _type, _module, _fmt, ## __VA_ARGS__) + +#define FYP_TOKEN_ERROR(_fyp, _fyt, _module, _fmt, ...) \ + FYP_TOKEN_DIAG(_fyp, _fyt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYP_PARSE_ERROR(_fyp, _adv, _cnt, _module, _fmt, ...) \ + FYP_PARSE_DIAG(_fyp, _adv, _cnt, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYP_MARK_ERROR(_fyp, _sm, _em, _module, _fmt, ...) \ + FYP_MARK_DIAG(_fyp, _sm, _em, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYP_NODE_ERROR(_fyp, _fyn, _module, _fmt, ...) \ + FYP_NODE_DIAG(_fyp, _fyn, FYET_ERROR, _module, _fmt, ## __VA_ARGS__) + +#define FYP_TOKEN_ERROR_CHECK(_fyp, _fyt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYP_TOKEN_ERROR(_fyp, _fyt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYP_PARSE_ERROR_CHECK(_fyp, _adv, _cnt, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYP_PARSE_ERROR(_fyp, _adv, _cnt, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYP_MARK_ERROR_CHECK(_fyp, _sm, _em, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYP_MARK_ERROR(_fyp, _sm, _em, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYP_NODE_ERROR_CHECK(_fyp, _fyn, _module, _cond, _label, _fmt, ...) \ + do { \ + if (!(_cond)) { \ + FYP_NODE_ERROR(_fyp, _fyn, _module, _fmt, ## __VA_ARGS__); \ + goto _label; \ + } \ + } while(0) + +#define FYP_TOKEN_WARNING(_fyp, _fyt, _module, _fmt, ...) \ + FYP_TOKEN_DIAG(_fyp, _fyt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYP_PARSE_WARNING(_fyp, _adv, _cnt, _module, _fmt, ...) \ + FYP_PARSE_DIAG(_fyp, _adv, _cnt, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYP_MARK_WARNING(_fyp, _sm, _em, _module, _fmt, ...) \ + FYP_MARK_DIAG(_fyp, _sm, _em, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + +#define FYP_NODE_WARNING(_fyp, _fyn, _type, _module, _fmt, ...) \ + FYP_NODE_DIAG(_fyp, _fyn, FYET_WARNING, _module, _fmt, ## __VA_ARGS__) + #endif diff --git a/src/lib/fy-path.c b/src/lib/fy-path.c index 12fb326..b770698 100644 --- a/src/lib/fy-path.c +++ b/src/lib/fy-path.c @@ -272,7 +272,7 @@ static int fy_path_component_get_text_internal(struct fy_emit_accum *ea, struct switch (fypc->type) { case FYPCT_NONE: - abort(); + FY_IMPOSSIBLE_ABORT(); case FYPCT_MAP: @@ -337,7 +337,7 @@ static int fy_path_get_text_internal(struct fy_emit_accum *ea, struct fy_path *f switch (fypc->type) { case FYPCT_NONE: - abort(); + FY_IMPOSSIBLE_ABORT(); case FYPCT_MAP: diff --git a/src/lib/fy-token.c b/src/lib/fy-token.c index bfcf27d..a16bc04 100644 --- a/src/lib/fy-token.c +++ b/src/lib/fy-token.c @@ -117,7 +117,7 @@ void fy_token_clean_rl(struct fy_token_list *fytl, struct fy_token *fyt) } fyt->type = FYTT_NONE; - fyt->analyze_flags = 0; + memset(&fyt->analysis, 0, sizeof(fyt->analysis)); fyt->text_len = 0; fyt->text = NULL; } @@ -641,20 +641,27 @@ const struct fy_mark *fy_token_end_mark(struct fy_token *fyt) return NULL; } -int fy_token_text_analyze(struct fy_token *fyt) +const struct fy_token_analysis * +fy_token_text_analyze(struct fy_token *fyt) { + static const struct fy_token_analysis null_analysis = { + .flags = FYTTAF_CAN_BE_SIMPLE_KEY | FYTTAF_DIRECT_OUTPUT | + FYTTAF_EMPTY | FYTTAF_CAN_BE_DOUBLE_QUOTED | + FYTTAF_ANALYZED, + .maxspan = 0, + .maxcol = 0, + }; struct fy_atom_iter iter; enum fy_atom_style style; int c, cn, cnn, cp, col; uint8_t col0si, col0ei; /* mask for --- ... at indent 0 */ - int flags; + int flags, span, maxspan, maxcol; if (!fyt) - return FYTTAF_CAN_BE_SIMPLE_KEY | FYTTAF_DIRECT_OUTPUT | - FYTTAF_EMPTY | FYTTAF_CAN_BE_DOUBLE_QUOTED; + return &null_analysis; - if (fyt->analyze_flags) - return fyt->analyze_flags; + if (fyt->analysis.flags & FYTTAF_ANALYZED) + return &fyt->analysis; /* only tokens that can generate text */ if (fyt->type != FYTT_SCALAR && @@ -662,14 +669,40 @@ int fy_token_text_analyze(struct fy_token *fyt) fyt->type != FYTT_ANCHOR && fyt->type != FYTT_ALIAS) { flags = FYTTAF_NO_TEXT_TOKEN; - fyt->analyze_flags = flags; - return flags; + + fyt->analysis.flags = flags | FYTTAF_ANALYZED; + fyt->analysis.maxspan = 0; + fyt->analysis.maxcol = 0; + + return &fyt->analysis; } flags = FYTTAF_TEXT_TOKEN; style = fy_token_atom_style(fyt); + /* hardwired and fast for regular plain scalars */ + if (style == FYAS_PLAIN && fyt->handle.storage_hint_valid && + fyt->handle.direct_output && !fyt->handle.high_ascii && + !fyt->handle.has_lb && !fyt->handle.has_ws && !fyt->handle.empty) { + + flags |= FYTTAF_DIRECT_OUTPUT; + + maxcol = fyt->handle.storage_hint; + maxspan = maxcol - 1; + flags |= + FYTTAF_DIRECT_OUTPUT | + FYTTAF_CAN_BE_SIMPLE_KEY | + FYTTAF_CAN_BE_PLAIN | + FYTTAF_CAN_BE_SINGLE_QUOTED | + FYTTAF_CAN_BE_DOUBLE_QUOTED | + FYTTAF_CAN_BE_LITERAL | + FYTTAF_CAN_BE_PLAIN_FLOW | + FYTTAF_CAN_BE_UNQUOTED_PATH_KEY; + + goto done; + } + /* can this token be a simple key initial condition */ if (!fy_atom_style_is_block(style) && style != FYAS_URI) flags |= FYTTAF_CAN_BE_SIMPLE_KEY; @@ -681,12 +714,15 @@ int fy_token_text_analyze(struct fy_token *fyt) fy_atom_iter_start(&fyt->handle, &iter); col = 0; + maxcol = 0; + maxspan = 0; + span = 0; /* get first character */ cn = fy_atom_iter_utf8_get(&iter); if (cn < 0) { /* empty? */ - flags |= FYTTAF_EMPTY | FYTTAF_CAN_BE_DOUBLE_QUOTED | FYTTAF_CAN_BE_UNQUOTED_PATH_KEY; + flags |= FYTTAF_EMPTY | FYTTAF_CAN_BE_DOUBLE_QUOTED | FYTTAF_CAN_BE_UNQUOTED_PATH_KEY | FYTTAF_CAN_BE_SIMPLE_KEY; goto out; } @@ -694,7 +730,6 @@ int fy_token_text_analyze(struct fy_token *fyt) FYTTAF_CAN_BE_SINGLE_QUOTED | FYTTAF_CAN_BE_DOUBLE_QUOTED | FYTTAF_CAN_BE_LITERAL | - FYTTAF_CAN_BE_LITERAL | FYTTAF_CAN_BE_FOLDED | FYTTAF_CAN_BE_PLAIN_FLOW | FYTTAF_CAN_BE_UNQUOTED_PATH_KEY; @@ -806,7 +841,16 @@ int fy_token_text_analyze(struct fy_token *fyt) (style == FYAS_DOUBLE_QUOTED && c == '\\'))) flags &= ~FYTTAF_DIRECT_OUTPUT; + if (cn < 0 || !c || fy_token_is_lb(fyt, c) || fy_is_ws(c)) { + if (span > maxspan) + maxspan = span; + span = 0; + } else + span++; + if (fy_token_is_lb(fyt, c)) { + if (col > maxcol) + maxcol = col; col = 0; col0si = col0ei = 0; } else @@ -823,10 +867,19 @@ int fy_token_text_analyze(struct fy_token *fyt) FYTTAF_CAN_BE_PLAIN_FLOW); } } + + if (col > maxcol) + maxcol = col; + if (span > maxspan) + maxspan = span; out: fy_atom_iter_finish(&iter); - fyt->analyze_flags = flags; - return flags; + +done: + fyt->analysis.flags = flags | FYTTAF_ANALYZED; + fyt->analysis.maxspan = maxspan; + fyt->analysis.maxcol = maxcol; + return &fyt->analysis; } const char *fy_tag_token_get_directive_handle(struct fy_token *fyt, size_t *td_handle_sizep) @@ -859,6 +912,23 @@ const char *fy_token_get_direct_output(struct fy_token *fyt, size_t *sizep) return fy_atom_data(fya); } +const char *fy_token_get_direct_simple_output(struct fy_token *fyt, size_t *sizep) +{ + const struct fy_atom *handle; + + handle = fy_token_atom(fyt); + + if (!(handle->style == FYAS_PLAIN && handle->storage_hint_valid && + handle->direct_output && !handle->high_ascii && + !handle->has_lb && !handle->has_ws && !handle->empty)) { + *sizep = 0; + return NULL; + } + + *sizep = fy_atom_size(handle); + return fy_atom_data(handle); +} + const char *fy_tag_token_handle(struct fy_token *fyt, size_t *lenp) { return fy_tag_token_get_directive_handle(fyt, lenp); @@ -1190,7 +1260,7 @@ const char *fy_token_get_scalar_path_key(struct fy_token *fyt, size_t *lenp) uint8_t non_utf8[4]; size_t non_utf8_len, k; int c, i, w, digit; - int aflags; + const struct fy_token_analysis *ta; if (!fyt || fyt->type != FYTT_SCALAR) { *lenp = 0; @@ -1204,10 +1274,10 @@ const char *fy_token_get_scalar_path_key(struct fy_token *fyt, size_t *lenp) } /* analyze the token */ - aflags = fy_token_text_analyze(fyt); + ta = fy_token_text_analyze(fyt); /* simple one? perfect */ - if ((aflags & FYTTAF_CAN_BE_UNQUOTED_PATH_KEY) == FYTTAF_CAN_BE_UNQUOTED_PATH_KEY) { + if ((ta->flags & FYTTAF_CAN_BE_UNQUOTED_PATH_KEY) == FYTTAF_CAN_BE_UNQUOTED_PATH_KEY) { fyt->scalar.path_key = fy_token_get_text(fyt, &fyt->scalar.path_key_len); *lenp = fyt->scalar.path_key_len; return fyt->scalar.path_key; @@ -1515,6 +1585,12 @@ unsigned int fy_analyze_scalar_content(const char *data, size_t size, if (break_run > 1) flags |= FYACF_TRAILING_LB; + if ((flags & FYACF_STARTS_WITH_WS) || + (flags & FYACF_STARTS_WITH_LB) || + (flags & FYACF_ENDS_WITH_WS) || + (flags & FYACF_ENDS_WITH_LB)) + flags &= ~FYACF_FLOW_PLAIN; + return flags; } diff --git a/src/lib/fy-token.h b/src/lib/fy-token.h index 8ab8462..93c1d94 100644 --- a/src/lib/fy-token.h +++ b/src/lib/fy-token.h @@ -19,6 +19,7 @@ #include +#include "fy-typelist.h" #include "fy-utils.h" #include "fy-atom.h" @@ -79,12 +80,43 @@ static inline bool fy_token_type_is_mapping_marker(enum fy_token_type type) #define FYACF_VALID_ANCHOR 0x040000 /* contains valid anchor (without & prefix) */ #define FYACF_JSON_ESCAPE 0x080000 /* contains a character that JSON escapes */ +/* analysis flags */ +#define FYTTAF_HAS_LB FY_BIT(0) +#define FYTTAF_HAS_WS FY_BIT(1) +#define FYTTAF_HAS_CONSECUTIVE_LB FY_BIT(2) +#define FYTTAF_HAS_CONSECUTIVE_WS FY_BIT(4) +#define FYTTAF_EMPTY FY_BIT(5) +#define FYTTAF_CAN_BE_SIMPLE_KEY FY_BIT(6) +#define FYTTAF_DIRECT_OUTPUT FY_BIT(7) +#define FYTTAF_NO_TEXT_TOKEN FY_BIT(8) +#define FYTTAF_TEXT_TOKEN FY_BIT(9) +#define FYTTAF_CAN_BE_PLAIN FY_BIT(10) +#define FYTTAF_CAN_BE_SINGLE_QUOTED FY_BIT(11) +#define FYTTAF_CAN_BE_DOUBLE_QUOTED FY_BIT(12) +#define FYTTAF_CAN_BE_LITERAL FY_BIT(13) +#define FYTTAF_CAN_BE_FOLDED FY_BIT(14) +#define FYTTAF_CAN_BE_PLAIN_FLOW FY_BIT(15) +#define FYTTAF_QUOTE_AT_0 FY_BIT(16) +#define FYTTAF_CAN_BE_UNQUOTED_PATH_KEY FY_BIT(17) +#define FYTTAF_HAS_ANY_LB FY_BIT(18) /* any LB including unicode, not per input */ +#define FYTTAF_HAS_START_IND FY_BIT(19) /* has --- at col 0 */ +#define FYTTAF_HAS_END_IND FY_BIT(20) /* has ... at col 0 */ +#define FYTTAF_HAS_NON_PRINT FY_BIT(21) /* has any non printable utf8 */ + +#define FYTTAF_ANALYZED FY_BIT(31) /* analyzed mark */ + +struct fy_token_analysis { + unsigned int flags; + int maxspan; + int maxcol; +}; + FY_TYPE_FWD_DECL_LIST(token); struct fy_token { struct list_head node; enum fy_token_type type; int refs; /* when on document, we switch to reference counting */ - int analyze_flags; /* cache of the analysis flags */ + struct fy_token_analysis analysis; size_t text_len; const char *text; char *text0; /* this is allocated */ @@ -168,8 +200,7 @@ fy_token_alloc_rl(struct fy_token_list *fytl) fyt->type = FYTT_NONE; fyt->refs = 1; - - fyt->analyze_flags = 0; + memset(&fyt->analysis, 0, sizeof(fyt->analysis)); fyt->text_len = 0; fyt->text = NULL; fyt->text0 = NULL; @@ -368,6 +399,7 @@ static inline bool fy_token_is_multiline(struct fy_token *fyt) } const char *fy_token_get_direct_output(struct fy_token *fyt, size_t *sizep); +const char *fy_token_get_direct_simple_output(struct fy_token *fyt, size_t *sizep); static inline struct fy_input *fy_token_get_input(struct fy_token *fyt) { @@ -425,29 +457,7 @@ static inline bool fy_token_is_flow_ws(struct fy_token *fyt, int c) return fy_atom_is_flow_ws(&fyt->handle, c); } -#define FYTTAF_HAS_LB FY_BIT(0) -#define FYTTAF_HAS_WS FY_BIT(1) -#define FYTTAF_HAS_CONSECUTIVE_LB FY_BIT(2) -#define FYTTAF_HAS_CONSECUTIVE_WS FY_BIT(4) -#define FYTTAF_EMPTY FY_BIT(5) -#define FYTTAF_CAN_BE_SIMPLE_KEY FY_BIT(6) -#define FYTTAF_DIRECT_OUTPUT FY_BIT(7) -#define FYTTAF_NO_TEXT_TOKEN FY_BIT(8) -#define FYTTAF_TEXT_TOKEN FY_BIT(9) -#define FYTTAF_CAN_BE_PLAIN FY_BIT(10) -#define FYTTAF_CAN_BE_SINGLE_QUOTED FY_BIT(11) -#define FYTTAF_CAN_BE_DOUBLE_QUOTED FY_BIT(12) -#define FYTTAF_CAN_BE_LITERAL FY_BIT(13) -#define FYTTAF_CAN_BE_FOLDED FY_BIT(14) -#define FYTTAF_CAN_BE_PLAIN_FLOW FY_BIT(15) -#define FYTTAF_QUOTE_AT_0 FY_BIT(16) -#define FYTTAF_CAN_BE_UNQUOTED_PATH_KEY FY_BIT(17) -#define FYTTAF_HAS_ANY_LB FY_BIT(18) /* any LB including unicode, not per input */ -#define FYTTAF_HAS_START_IND FY_BIT(19) /* has --- at col 0 */ -#define FYTTAF_HAS_END_IND FY_BIT(20) /* has ... at col 0 */ -#define FYTTAF_HAS_NON_PRINT FY_BIT(21) /* has any non printable utf8 */ - -int fy_token_text_analyze(struct fy_token *fyt); +const struct fy_token_analysis *fy_token_text_analyze(struct fy_token *fyt); unsigned int fy_analyze_scalar_content(const char *data, size_t size, bool json_mode, enum fy_lb_mode lb_mode, enum fy_flow_ws_mode fws_mode); diff --git a/src/lib/fy-types.h b/src/lib/fy-types.h index 6bddda3..24a2b26 100644 --- a/src/lib/fy-types.h +++ b/src/lib/fy-types.h @@ -35,7 +35,7 @@ struct fy_ ## _type *fy_ ## _type ## _alloc_simple_internal( \ return _n; \ _n = malloc(sizeof(*_n)); \ if (_n) \ - INIT_LIST_HEAD(&_n->node); \ + list_init(&_n->node); \ return _n; \ } \ \ diff --git a/src/lib/fy-walk.c b/src/lib/fy-walk.c index 736704d..b1daa56 100644 --- a/src/lib/fy-walk.c +++ b/src/lib/fy-walk.c @@ -144,12 +144,13 @@ struct fy_walk_result *fy_walk_result_alloc_rl(struct fy_walk_result_list *fwrl) if (!fwr) return NULL; memset(fwr, 0, sizeof(*fwr)); - } + } else + memset(&fwr->_clean, 0, sizeof(fwr->_clean)); fwr->type = fwrt_none; return fwr; } -struct fy_walk_result *fy_walk_result_clone_rl(struct fy_walk_result_list *fwrl, struct fy_walk_result *fwr) +struct fy_walk_result *fy_walk_result_clone_rl(struct fy_walk_result_list *fwrl, struct fy_walk_result *fwr, bool deep) { struct fy_walk_result *fwrn = NULL, *fwrn2 = NULL, *fwrn3; @@ -165,7 +166,15 @@ struct fy_walk_result *fy_walk_result_clone_rl(struct fy_walk_result_list *fwrl, case fwrt_none: break; case fwrt_node_ref: - fwrn->fyn = fwr->fyn; + if (!deep) { + fwrn->fyn = fwr->fyn; + fwrn->fyn_source = NULL; + } else { + fwrn->fyn = fy_node_copy(fy_node_document(fwr->fyn), fwr->fyn); + if (!fwrn->fyn) + goto err_out; + fwrn->fyn_source = fwr->fyn; + } break; case fwrt_number: fwrn->number = fwr->number; @@ -187,7 +196,7 @@ struct fy_walk_result *fy_walk_result_clone_rl(struct fy_walk_result_list *fwrl, for (fwrn2 = fy_walk_result_list_head(&fwr->refs); fwrn2; fwrn2 = fy_walk_result_next(&fwr->refs, fwrn2)) { - fwrn3 = fy_walk_result_clone_rl(fwrl, fwrn2); + fwrn3 = fy_walk_result_clone_rl(fwrl, fwrn2, deep); if (!fwrn3) goto err_out; @@ -210,7 +219,18 @@ struct fy_walk_result *fy_walk_result_clone(struct fy_walk_result *fwr) return NULL; fwrl = fy_path_exec_walk_result_rl(fwr->fypx); - return fy_walk_result_clone_rl(fwrl, fwr); + return fy_walk_result_clone_rl(fwrl, fwr, false); +} + +struct fy_walk_result *fy_walk_result_clone_deep(struct fy_walk_result *fwr) +{ + struct fy_walk_result_list *fwrl; + + if (!fwr) + return NULL; + + fwrl = fy_path_exec_walk_result_rl(fwr->fypx); + return fy_walk_result_clone_rl(fwrl, fwr, true); } void fy_walk_result_clean_rl(struct fy_walk_result_list *fwrl, struct fy_walk_result *fwr) @@ -224,6 +244,10 @@ void fy_walk_result_clean_rl(struct fy_walk_result_list *fwrl, struct fy_walk_re case fwrt_none: break; case fwrt_node_ref: + /* if deep copy, free */ + if (fwr->fyn_source) + fy_node_free(fwr->fyn); + break; case fwrt_number: break; case fwrt_string: @@ -241,6 +265,7 @@ void fy_walk_result_clean_rl(struct fy_walk_result_list *fwrl, struct fy_walk_re } fwr->type = fwrt_none; + memset(&fwr->_clean, 0, sizeof(fwr->_clean)); } void fy_walk_result_clean(struct fy_walk_result *fwr) @@ -474,6 +499,8 @@ const char *fy_path_expr_type_txt[FPET_COUNT] = { [fpet_chain] = "chain", [fpet_logical_or] = "logical-or", [fpet_logical_and] = "logical-and", + [fpet_select] = "select", + [fpet_unselect] = "unselect", [fpet_eq] = "equals", [fpet_neq] = "not-equals", @@ -1188,7 +1215,16 @@ int fy_path_fetch_tokens(struct fy_path_parser *fypp) simple_token_count = 2; break; } - /* may still be something else */ + if (cn == '/' || cn == '(' || fy_is_alnum(cn) || cn == '!') { + /* bang bang */ + type = FYTT_PE_BANG; + simple_token_count = 1; + } + break; + + case '@': + type = FYTT_PE_AT; + simple_token_count = 1; break; default: @@ -1202,8 +1238,7 @@ again: switch (fypp->expr_mode) { case fyem_none: - assert(0); /* should never happen */ - break; + FY_IMPOSSIBLE_ABORT(); case fyem_path: @@ -1352,8 +1387,7 @@ do_token: switch (fypp->expr_mode) { case fyem_none: - assert(0); /* should never happen */ - break; + FY_IMPOSSIBLE_ABORT(); case fyem_path: if (fy_is_first_alpha(c)) @@ -1722,10 +1756,14 @@ enum fy_path_expr_type fy_map_token_to_path_expr_type(enum fy_token_type type, e case FYTT_PE_METHOD: return fpet_method; + case FYTT_PE_BANG: + return fpet_unselect; + + case FYTT_PE_AT: + return fpet_select; + default: - /* note parentheses do not have an expression */ - assert(0); - break; + FY_IMPOSSIBLE_ABORT(); } return fpet_none; } @@ -1770,7 +1808,9 @@ bool fy_token_type_is_operator(enum fy_token_type type) type == FYTT_SE_PLUS || type == FYTT_SE_MINUS || type == FYTT_SE_MULT || - type == FYTT_SE_DIV; + type == FYTT_SE_DIV || + + type == FYTT_PE_BANG; } bool fy_token_type_is_operand_or_operator(enum fy_token_type type) @@ -1794,7 +1834,7 @@ int fy_path_expr_type_prec(enum fy_path_expr_type type) case fpet_logical_and: return 4; case fpet_multi: - return 11; + return 21; case fpet_eq: case fpet_neq: case fpet_lt: @@ -1809,7 +1849,10 @@ int fy_path_expr_type_prec(enum fy_path_expr_type type) case fpet_minus: return 8; case fpet_chain: - return 10; + return 20; + case fpet_select: + case fpet_unselect: + return 10; /* must be less than chain */ case fpet_lparen: case fpet_rparen: case fpet_method: @@ -1852,7 +1895,7 @@ pop_operand(struct fy_path_parser *fypp) #define PREFIX 0 #define INFIX 1 -#define SUFFIX 2 +#define POSTFIX 2 int fy_token_type_operator_placement(enum fy_token_type type) { @@ -1877,9 +1920,60 @@ int fy_token_type_operator_placement(enum fy_token_type type) case FYTT_PE_SEQ_FILTER: case FYTT_PE_MAP_FILTER: case FYTT_PE_UNIQUE_FILTER: - return SUFFIX; + return POSTFIX; case FYTT_PE_SIBLING: + case FYTT_PE_BANG: + return PREFIX; + default: + break; + } + return -1; +} + +int fy_path_expr_type_assoc(enum fy_path_expr_type type) +{ + switch (type) { + case fpet_root: + case fpet_select: + case fpet_unselect: return PREFIX; + + case fpet_this: + case fpet_parent: + case fpet_every_child: + case fpet_every_child_r: + case fpet_filter_collection: + case fpet_filter_scalar: + case fpet_filter_sequence: + case fpet_filter_mapping: + case fpet_filter_unique: + case fpet_seq_index: + case fpet_map_key: + case fpet_seq_slice: + case fpet_alias: + return POSTFIX; + + case fpet_logical_or: + case fpet_logical_and: + case fpet_eq: + case fpet_neq: + case fpet_lt: + case fpet_gt: + case fpet_lte: + case fpet_gte: + case fpet_plus: + case fpet_minus: + case fpet_mult: + case fpet_div: + case fpet_arg_separator: /* argument separator (comma in scalar mode) */ + return INFIX; + + case fpet_scalar: + case fpet_lparen: + case fpet_rparen: + case fpet_method: + case fpet_scalar_expr: + case fpet_path_expr: default: break; } @@ -2018,7 +2112,7 @@ int push_operand_lr(struct fy_path_parser *fypp, me = fy_token_end_mark(exprr->fyt); assert(me); } else { - me = fy_token_end_mark(exprr->fyt); + me = fy_token_end_mark(exprl->fyt); assert(me); } @@ -2686,6 +2780,7 @@ int evaluate_new(struct fy_path_parser *fypp) #ifdef DEBUG_EXPR FYR_TOKEN_DIAG(fyr, expr->fyt, FYDF_NOTICE, FYEM_PARSE, "poped operator expression"); + fy_path_expr_dump(expr, fypp->cfg.diag, FYET_NOTICE, 0, "poped expr: "); #endif exprl = NULL; @@ -2852,9 +2947,6 @@ int evaluate_new(struct fy_path_parser *fypp) break; case fpet_lparen: - abort(); - assert(0); - #ifdef DEBUG_EXPR FYR_TOKEN_DIAG(fyr, expr->fyt, FYDF_NOTICE, FYEM_PARSE, "("); @@ -2870,8 +2962,11 @@ int evaluate_new(struct fy_path_parser *fypp) /* evaluate until we hit a match to the rparen */ exprl = fy_expr_stack_peek(&fypp->operators); + if (!exprl) + goto err_out; if (!fy_path_expr_type_is_lparen(exprl->type)) { + exprl = NULL; ret = evaluate_new(fypp); if (ret) goto err_out; @@ -2895,6 +2990,7 @@ int evaluate_new(struct fy_path_parser *fypp) if (fy_path_expr_type_is_lparen(exprl->type)) break; + exprl = NULL; ret = evaluate_new(fypp); if (ret) goto err_out; @@ -2905,7 +3001,8 @@ int evaluate_new(struct fy_path_parser *fypp) "missing matching left parentheses"); exprl = fy_expr_stack_pop(&fypp->operators); - assert(exprl); + if (!exprl) + goto err_out; exprt = fy_expr_stack_peek(&fypp->operands); @@ -2949,6 +3046,8 @@ int evaluate_new(struct fy_path_parser *fypp) expr->fyt = expr_lr_to_token_mark(exprl, exprr, fypp->fyi); exprt = fy_expr_stack_pop(&fypp->operands); + if (!exprt) + goto err_out; FYR_TOKEN_ERROR_CHECK(fyr, exprr->fyt, FYEM_PARSE, exprt, err_out, @@ -2993,11 +3092,28 @@ int evaluate_new(struct fy_path_parser *fypp) case fpet_method: return evaluate_method(fypp, expr, NULL, NULL); + case fpet_select: + case fpet_unselect: + + /* we don't need the not operator now */ + fy_path_expr_free_recycle(fypp, expr); + expr = NULL; + + exprl = fy_expr_stack_pop(&fypp->operands); + fyr_error_check(fyr, exprl, err_out, + "fy_expr_stack_pop() failed for exprl\n"); + exprr = NULL; + + ret = push_operand_lr(fypp, type, exprl, exprr, true); + fyr_error_check(fyr, !ret, err_out, + "push_operand_lr() failed\n"); + + return 0; + /* shoud never */ case fpet_scalar_expr: case fpet_path_expr: - assert(0); - abort(); + FY_IMPOSSIBLE_ABORT(); default: fyr_error(fyr, "Unknown expression %s\n", fy_path_expr_type_txt[expr->type]); @@ -3206,8 +3322,10 @@ fy_path_parse_expression(struct fy_path_parser *fypp) #ifdef DEBUG_EXPR fy_notice(fypp->cfg.diag, "operator stack (before)\n"); fy_expr_stack_dump(fypp->cfg.diag, &fypp->operators); + fy_notice(fypp->cfg.diag, "operator stack (before) ends\n"); fy_notice(fypp->cfg.diag, "operand stack (before)\n"); fy_expr_stack_dump(fypp->cfg.diag, &fypp->operands); + fy_notice(fypp->cfg.diag, "operand stack (before) ends\n"); #endif old_scan_mode = fypp->expr_mode; @@ -3251,8 +3369,7 @@ fy_path_parse_expression(struct fy_path_parser *fypp) } else { switch (fypp->expr_mode) { case fyem_none: - assert(0); /* should never happen */ - break; + FY_IMPOSSIBLE_ABORT(); case fyem_path: if (fy_path_expr_type_is_conditional(expr->type)) { @@ -3281,6 +3398,7 @@ fy_path_parse_expression(struct fy_path_parser *fypp) break; } break; + } if (old_scan_mode != fypp->expr_mode) { @@ -3292,9 +3410,22 @@ fy_path_parse_expression(struct fy_path_parser *fypp) } ret = -1; - while ((expr_top = fy_expr_stack_peek(&fypp->operators)) != NULL && - fy_path_expr_type_prec(expr->type) <= fy_path_expr_type_prec(expr_top->type) && - !fy_path_expr_type_is_lparen(expr_top->type)) { + for (;;) { + expr_top = fy_expr_stack_peek(&fypp->operators); + if (!expr_top) + break; + + if (fy_path_expr_type_prec(expr->type) > fy_path_expr_type_prec(expr_top->type)) + break; + + if (fy_path_expr_type_is_lparen(expr_top->type)) + break; + + /* XXX same priority and prefix operator (have no arguments yet) */ + if (fy_path_expr_type_prec(expr->type) == fy_path_expr_type_prec(expr_top->type)) { + if (fy_path_expr_type_assoc(expr_top->type) == PREFIX) + break; + } ret = evaluate_new(fypp); /* evaluate will print diagnostic on error */ @@ -3311,8 +3442,10 @@ fy_path_parse_expression(struct fy_path_parser *fypp) #ifdef DEBUG_EXPR fy_notice(fypp->cfg.diag, "operator stack (after)\n"); fy_expr_stack_dump(fypp->cfg.diag, &fypp->operators); + fy_notice(fypp->cfg.diag, "operator stack (after) ends\n"); fy_notice(fypp->cfg.diag, "operand stack (after)\n"); fy_expr_stack_dump(fypp->cfg.diag, &fypp->operands); + fy_notice(fypp->cfg.diag, "operand stack (after) ends\n"); #endif prev_scan_mode = fypp->expr_mode; @@ -3358,9 +3491,13 @@ fy_path_parse_expression(struct fy_path_parser *fypp) expr != NULL, err_out, "No operands left on operand stack"); - FYR_TOKEN_ERROR_CHECK(fyr, expr->fyt, FYEM_PARSE, - fy_expr_stack_size(&fypp->operands) == 0, err_out, - "Operand stack contains more than 1 value at end"); + if (fy_expr_stack_size(&fypp->operands) != 0) { + FYR_TOKEN_ERROR(fyr, expr->fyt, FYEM_PARSE, + "Operand stack contains more than 1 value at end"); + fy_path_expr_free(expr); + expr = NULL; + goto err_out; + } /* check the expression for validity */ rc = fy_path_check_expression(fypp, expr); @@ -3810,8 +3947,7 @@ fy_walk_result_compare_simple(struct fy_path_exec *fypx, enum fy_path_expr_type switch (fwrl->type) { case fwrt_none: - abort(); /* should never happen */ - break; + FY_IMPOSSIBLE_ABORT(); case fwrt_node_ref: switch (type) { @@ -3831,8 +3967,7 @@ fy_walk_result_compare_simple(struct fy_path_exec *fypx, enum fy_path_expr_type break; case fwrt_refs: - assert(0); /* should not get here */ - break; + FY_IMPOSSIBLE_ABORT(); case fwrt_doc: switch (type) { @@ -4037,16 +4172,14 @@ fy_walk_result_arithmetic_simple(struct fy_path_exec *fypx, output->number = fwrr->number ? (fwrl->number / fwrr->number) : INFINITY; break; default: - assert(0); - break; + FY_IMPOSSIBLE_ABORT(); } fwrl = NULL; break; default: fy_error(diag, "fwrl->type=%s\n", fy_walk_result_type_txt[fwrl->type]); - assert(0); - break; + FY_IMPOSSIBLE_ABORT(); } } @@ -4162,9 +4295,8 @@ fy_walk_result_lhs_rhs(struct fy_path_exec *fypx, fwr = fy_walk_result_conditional_simple(fypx, expr, exprl, fwrlc, exprr, fwrrc); else if (fy_path_expr_type_is_arithmetic(expr->type)) fwr = fy_walk_result_arithmetic_simple(fypx, expr, exprl, fwrlc, exprr, fwrrc); - else { - assert(0); - } + else + FY_IMPOSSIBLE_ABORT(); fwrlc = NULL; fwrrc = NULL; @@ -4252,6 +4384,174 @@ fy_scalar_walk_result_to_expr(struct fy_path_exec *fypx, struct fy_walk_result * return exprt; } +enum fy_walk_result_set_op { + FYWRSO_SELECT, + FYWRSO_UNSELECT, +}; + +static void fy_node_delete_non_marked(struct fy_node *fyn) +{ + struct fy_node *fyni, *fynin; + struct fy_node_pair *fynp, *fynpn; + + if (!fyn) + return; + + if (!(fyn->marks & FY_BIT(FYNWF_INSET_MARKER))) { + fy_node_delete(fyn); + return; + } + + switch (fyn->type) { + case FYNT_SCALAR: + break; + + case FYNT_SEQUENCE: + for (fyni = fy_node_list_head(&fyn->sequence); fyni; fyni = fynin) { + fynin = fy_node_next(&fyn->sequence, fyni); + + fy_node_delete_non_marked(fyni); + } + break; + + case FYNT_MAPPING: + for (fynp = fy_node_pair_list_head(&fyn->mapping); fynp; fynp = fynpn) { + fynpn = fy_node_pair_next(&fyn->mapping, fynp); + + /* the mark is on the value */ + fy_node_delete_non_marked(fynp->value); + } + break; + } +} + +struct fy_walk_result * +fy_walk_result_perform_set_op(struct fy_path_exec *fypx, struct fy_walk_result *input, + struct fy_walk_result *set, enum fy_walk_result_set_op op) +{ + struct fy_walk_result *fwr, *fwrrm, *fwrin; + struct fy_walk_result *output = NULL; + char *relpath = NULL; + struct fy_node *fyn2, *fynt; + int rc; + + /* no input? return it */ + if (!input) + return NULL; + + if (!set) { + /* on unselect, return input, on select return NULL */ + return op == FYWRSO_UNSELECT ? input : NULL; + } + + assert(input->type == fwrt_node_ref || input->type == fwrt_refs); + +#ifdef DEBUG_EXPR + fy_walk_result_dump(input, fypx->cfg.diag, FYET_WARNING, 0, "set-op: input\n"); + fy_walk_result_dump(input, fypx->cfg.diag, FYET_WARNING, 0, "set-op: set\n"); +#endif + + for (fwrin = fy_walk_result_iter_start(input); fwrin; fwrin = fy_walk_result_iter_next(input, fwrin)) { + +#ifdef DEBUG_EXPR + fy_walk_result_dump(input, fypx->cfg.diag, FYET_WARNING, 0, "set-op: fwrin\n"); +#endif + + fwr = NULL; + + for (fwrrm = fy_walk_result_iter_start(set); fwrrm; fwrrm = fy_walk_result_iter_next(set, fwrrm)) { + +#ifdef DEBUG_EXPR + fy_walk_result_dump(fwrrm, fypx->cfg.diag, FYET_WARNING, 0, "set-op: fwrrm\n"); +#endif + + if (!output) { + output = fy_path_exec_walk_result_create(fypx, fwrt_refs); + if (!output) + goto err_out; + } + + /* if this fails, it means we don't have any output */ + if (fwrin->fyn == fwrrm->fyn || + (fwrrm->type == fwrt_node_ref && (relpath = fy_node_get_path_relative_to(fwrin->fyn, fwrrm->fyn)) == NULL)) + continue; +#ifdef DEBUG_EXPR + fy_diag_diag(fypx->cfg.diag, FYDF_WARNING, "relpath: %s", relpath); +#endif + + /* ok, do a deep copy now */ + if (!fwr) { + fwr = fy_walk_result_clone_deep(fwrin); + if (!fwr) + goto err_out; + } + + /* get the node at the relative path of the copy */ + fyn2 = fy_node_by_path(fwr->fyn, relpath, FY_NT, FYNWF_DONT_FOLLOW); + if (!fyn2) + goto err_out; + + free(relpath); + relpath = NULL; + + if (op == FYWRSO_UNSELECT) { + /* and remove it */ + rc = fy_node_delete(fyn2); + if (rc) + goto err_out; + } else { + + fynt = fyn2; + while (fynt && !(fynt->marks & FY_BIT(FYNWF_INSET_MARKER))) { + fynt->marks |= FY_BIT(FYNWF_INSET_MARKER); +#ifdef DEBUG_EXPR + fy_diag_diag(fypx->cfg.diag, FYDF_WARNING, "marking: %s", fy_node_get_path_alloca(fynt)); +#endif + fynt = fynt->parent; + } + } + } + + if (output && fwr) { + fy_walk_result_list_add_tail(&output->refs, fwr); + } + fwr = NULL; + } + + if (!output) { + /* nothing? nothing was removed or nothing was selected */ + return op == FYWRSO_UNSELECT ? input : NULL; + } + + /* simplify (might remove everything if empty) */ + output = fy_walk_result_simplify(output); + + /* for all marked nodes */ + if (output && op == FYWRSO_SELECT) { + + /* delete all not marked */ + for (fwr = fy_walk_result_iter_start(output); fwr; fwr = fy_walk_result_iter_next(output, fwr)) { +#ifdef DEBUG_EXPR + fy_diag_diag(fypx->cfg.diag, FYDF_WARNING, "deleting non marked: %s", fy_node_get_path_alloca(fwr->fyn)); +#endif + fy_node_delete_non_marked(fwr->fyn); + fy_node_clear_system_marks(fwr->fyn); + } + + } + + /* get rid of input */ + fy_walk_result_free(input); + return output; + +err_out: + fy_walk_result_free(input); + fy_walk_result_free(output); + if (relpath) + free(relpath); + return NULL; +} + struct fy_walk_result * fy_path_expr_execute(struct fy_path_exec *fypx, int level, struct fy_path_expr *expr, struct fy_walk_result *input, enum fy_path_expr_type ptype) @@ -4574,10 +4874,6 @@ fy_path_expr_execute(struct fy_path_exec *fypx, int level, struct fy_path_expr * if (!input) goto out; - /* flatten input */ - input = fy_walk_result_flatten(input); - assert(input); /* must work */ - /* for non refs, return input */ if (input->type != fwrt_refs) { output = input; @@ -4585,13 +4881,20 @@ fy_path_expr_execute(struct fy_path_exec *fypx, int level, struct fy_path_expr * break; } + /* nothing in there? */ + if (fy_walk_result_list_empty(&input->refs)) { + output = NULL; + break; + } + /* remove duplicates filter */ for (fwr = fy_walk_result_list_head(&input->refs); fwr; fwr = fy_walk_result_next(&input->refs, fwr)) { /* do not check recursively */ - if (fwr->type == fwrt_refs) + if (fwr->type == fwrt_refs) { continue; + } /* check the entries from this point forward */ for (fwrt = fy_walk_result_next(&input->refs, fwr); fwrt; fwrt = fwrtn) { @@ -4684,10 +4987,41 @@ fy_path_expr_execute(struct fy_path_exec *fypx, int level, struct fy_path_expr * break; + case fpet_select: + case fpet_unselect: + + /* pop the top in either case */ + assert(input); + if (!input) + goto out; + + /* only handle inputs of node and refs */ + if (input->type != fwrt_node_ref && input->type != fwrt_refs) + goto out; + + /* execute the expression */ + exprn = fy_path_expr_list_head(&expr->children); + assert(exprn); + if (!exprn) + goto out; + + fwrt = fy_walk_result_clone(input); + assert(fwrt); + if (!fwrt) + goto out; + + fwrn = fy_path_expr_execute(fypx, level + 1, exprn, fwrt, expr->type); + + output = fy_walk_result_perform_set_op(fypx, input, fwrn, expr->type == fpet_unselect ? FYWRSO_UNSELECT : FYWRSO_SELECT); + input = NULL; + + fy_walk_result_free(fwrn); + fwrn = NULL; + break; + default: fy_error(diag, "%s\n", fy_path_expr_type_txt[expr->type]); - assert(0); - break; + FY_IMPOSSIBLE_ABORT(); } out: @@ -4936,6 +5270,7 @@ int fy_node_setup_path_expr_data(struct fy_node *fyn) pxnd->fyi = fy_input_from_malloc_data(alloc, len + 1, NULL, false); if (!pxnd->fyi) goto err_no_input; + alloc = NULL; } fy_path_parser_reset(pxdd->fypp); diff --git a/src/lib/fy-walk.h b/src/lib/fy-walk.h index e42a76f..0578ac0 100644 --- a/src/lib/fy-walk.h +++ b/src/lib/fy-walk.h @@ -30,6 +30,7 @@ #include "fy-docstate.h" #include "fy-accel.h" #include "fy-token.h" +#include "fy-input.h" struct fy_document; @@ -53,7 +54,11 @@ struct fy_walk_result { struct fy_path_exec *fypx; enum fy_walk_result_type type; union { - struct fy_node *fyn; + uintptr_t _clean[2]; + struct { + struct fy_node *fyn; + struct fy_node *fyn_source; /* when deep */ + }; double number; char *string; struct fy_walk_result_list refs; @@ -123,6 +128,9 @@ enum fy_path_expr_type { fpet_logical_or, /* first non null result set */ fpet_logical_and, /* the last non null result set */ + fpet_select, /* set selection */ + fpet_unselect, /* negation of set selection */ + fpet_eq, /* equal expression */ fpet_neq, /* not equal */ fpet_lt, /* less than */ @@ -155,6 +163,7 @@ static inline bool fy_path_expr_type_is_valid(enum fy_path_expr_type type) return type >= fpet_root && type < FPET_COUNT; } +/* XXX do neg and reg check */ static inline bool fy_path_expr_type_is_single_result(enum fy_path_expr_type type) { return type == fpet_root || diff --git a/src/thread/fy-thread.c b/src/thread/fy-thread.c index d1f7eaa..bcd3214 100644 --- a/src/thread/fy-thread.c +++ b/src/thread/fy-thread.c @@ -50,17 +50,17 @@ static void fy_thread_work_join_steal_2(struct fy_thread_pool *tp, struct fy_thr #if defined(__linux__) && !defined(FY_THREAD_PORTABLE) /* linux pedal to the metal implementation */ -static inline int futex(_Atomic(uint32_t) *uaddr, int futex_op, uint32_t val, const struct timespec *timeout, uint32_t *uaddr2, uint32_t val3) +static inline int futex(FY_ATOMIC(uint32_t) *uaddr, int futex_op, uint32_t val, const struct timespec *timeout, uint32_t *uaddr2, uint32_t val3) { return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3); } -static inline int fwait(_Atomic(uint32_t) *futexp) +static inline int fwait(FY_ATOMIC(uint32_t) *futexp) { long s; uint32_t one = 1; - while (!atomic_compare_exchange_strong(futexp, &one, 0)) { + while (!fy_atomic_compare_exchange_strong(futexp, &one, 0)) { s = futex(futexp, FUTEX_WAIT_PRIVATE, 0, NULL, NULL, 0); if (s == -1 && errno != EAGAIN) return -1; @@ -68,12 +68,12 @@ static inline int fwait(_Atomic(uint32_t) *futexp) return 0; } -static inline int fpost(_Atomic(uint32_t) *futexp) +static inline int fpost(FY_ATOMIC(uint32_t) *futexp) { long s; uint32_t zero = 0; - if (atomic_compare_exchange_strong(futexp, &zero, 1)) { + if (fy_atomic_compare_exchange_strong(futexp, &zero, 1)) { s = futex(futexp, FUTEX_WAKE_PRIVATE, 1, NULL, NULL, 0); if (s == -1) return -1; @@ -84,8 +84,8 @@ static inline int fpost(_Atomic(uint32_t) *futexp) static inline void fy_thread_init_sync(struct fy_thread *t) { /* nothing more needed for futexes */ - atomic_store(&t->submit, 0); - atomic_store(&t->done, 0); + fy_atomic_store(&t->submit, 0); + fy_atomic_store(&t->done, 0); } static inline struct fy_thread_work *fy_worker_wait_for_work(struct fy_thread *t) @@ -93,7 +93,7 @@ static inline struct fy_thread_work *fy_worker_wait_for_work(struct fy_thread *t struct fy_thread_work *w; int rc __FY_DEBUG_UNUSED__; - while ((w = atomic_load(&t->work)) == NULL) { + while ((w = fy_atomic_load(&t->work)) == NULL) { rc = fwait(&t->submit); assert(!rc); } @@ -108,7 +108,7 @@ static inline void fy_worker_signal_work_done(struct fy_thread *t, struct fy_thr /* note that the work won't be replaced if it's a shutdown */ exp_work = work; - ok = atomic_compare_exchange_strong(&t->work, &exp_work, NULL); + ok = fy_atomic_compare_exchange_strong(&t->work, &exp_work, NULL); assert(ok); rc = fpost(&t->done); @@ -122,7 +122,7 @@ static inline int fy_thread_submit_work_internal(struct fy_thread *t, struct fy_ /* atomically update the work */ exp_work = NULL; - if (!atomic_compare_exchange_strong(&t->work, &exp_work, work)) + if (!fy_atomic_compare_exchange_strong(&t->work, &exp_work, work)) return -1; rc = fpost(&t->submit); @@ -135,10 +135,10 @@ static inline int fy_thread_wait_work_internal(struct fy_thread *t) { const struct fy_thread_work *work; - while ((work = atomic_load(&t->work)) != NULL) + while ((work = fy_atomic_load(&t->work)) != NULL) fwait(&t->done); - atomic_store(&t->done, 0); + fy_atomic_store(&t->done, 0); return 0; } @@ -147,7 +147,7 @@ void fy_worker_thread_shutdown(struct fy_thread *t) { int rc __FY_DEBUG_UNUSED__; - atomic_store(&t->work, WORK_SHUTDOWN); + fy_atomic_store(&t->work, WORK_SHUTDOWN); rc = fpost(&t->submit); assert(!rc); @@ -155,7 +155,7 @@ void fy_worker_thread_shutdown(struct fy_thread *t) assert(!rc); if (!(t->tp->cfg.flags & FYTPCF_STEAL_MODE)) - atomic_store(&t->work, NULL); + fy_atomic_store(&t->work, NULL); } #else @@ -176,7 +176,7 @@ static inline struct fy_thread_work *fy_worker_wait_for_work(struct fy_thread *t struct fy_thread_work *work; pthread_mutex_lock(&t->lock); - while ((work = atomic_load(&t->work)) == NULL) + while ((work = fy_atomic_load(&t->work)) == NULL) pthread_cond_wait(&t->cond, &t->lock); pthread_mutex_unlock(&t->lock); @@ -192,7 +192,7 @@ static inline void fy_worker_signal_work_done(struct fy_thread *t, struct fy_thr /* note that the work won't be replaced if it's a shutdown */ exp_work = work; - if (!atomic_compare_exchange_strong(&t->work, &exp_work, NULL)) + if (!fy_atomic_compare_exchange_strong(&t->work, &exp_work, NULL)) assert(exp_work == WORK_SHUTDOWN); pthread_cond_signal(&t->wait_cond); pthread_mutex_unlock(&t->wait_lock); @@ -210,7 +210,7 @@ static inline int fy_thread_submit_work_internal(struct fy_thread *t, struct fy_ pthread_mutex_lock(&t->lock); exp_work = NULL; - if (!atomic_compare_exchange_strong(&t->work, &exp_work, work)) { + if (!fy_atomic_compare_exchange_strong(&t->work, &exp_work, work)) { assert(exp_work == WORK_SHUTDOWN); ret = -1; } else { @@ -227,7 +227,7 @@ static inline int fy_thread_wait_work_internal(struct fy_thread *t) const struct fy_thread_work *work; pthread_mutex_lock(&t->wait_lock); - while ((work = atomic_load(&t->work)) != NULL) + while ((work = fy_atomic_load(&t->work)) != NULL) pthread_cond_wait(&t->wait_cond, &t->wait_lock); pthread_mutex_unlock(&t->wait_lock); @@ -237,7 +237,7 @@ static inline int fy_thread_wait_work_internal(struct fy_thread *t) void fy_worker_thread_shutdown(struct fy_thread *t) { pthread_mutex_lock(&t->lock); - atomic_store(&t->work, WORK_SHUTDOWN); + fy_atomic_store(&t->work, WORK_SHUTDOWN); pthread_cond_signal(&t->cond); pthread_mutex_unlock(&t->lock); pthread_join(t->tid, NULL); @@ -249,7 +249,7 @@ static inline struct fy_thread *fy_thread_reserve_internal(struct fy_thread_pool { struct fy_thread *t; unsigned int slot; - _Atomic(uint64_t) *free; + FY_ATOMIC(uint64_t) *free; uint64_t exp, v; unsigned int i, num_threads_words; @@ -257,13 +257,13 @@ static inline struct fy_thread *fy_thread_reserve_internal(struct fy_thread_pool num_threads_words = FY_BIT64_COUNT(tp->num_threads); for (i = 0, free = tp->freep; i < num_threads_words; i++, free++) { - v = atomic_load(free); + v = fy_atomic_load(free); while (v) { slot = FY_BIT64_LOWEST(v); assert(v & FY_BIT64(slot)); exp = v; /* expecting the previous value */ v &= ~FY_BIT64(slot); /* clear this bit */ - if (atomic_compare_exchange_strong(free, &exp, v)) { + if (fy_atomic_compare_exchange_strong(free, &exp, v)) { slot += i * 64; t = tp->threads + slot; assert(slot == t->id); @@ -279,52 +279,52 @@ static inline struct fy_thread *fy_thread_reserve_internal(struct fy_thread_pool static inline void fy_thread_unreserve_internal(struct fy_thread *t) { struct fy_thread_pool *tp; - _Atomic(uint64_t) *free; + FY_ATOMIC(uint64_t) *free; tp = t->tp; free = tp->freep + (unsigned int)(t->id / 64); - atomic_fetch_or(free, FY_BIT64(t->id & 63)); + fy_atomic_fetch_or(free, FY_BIT64(t->id & 63)); } static inline bool fy_thread_is_reserved_internal(struct fy_thread *t) { struct fy_thread_pool *tp = t->tp; - _Atomic(uint64_t) *free; + FY_ATOMIC(uint64_t) *free; free = tp->freep + (unsigned int)(t->id / 64); - return !(atomic_load(free) & FY_BIT64(t->id & 63)); + return !(fy_atomic_load(free) & FY_BIT64(t->id & 63)); } static inline bool fy_thread_pool_are_all_reserved_internal(struct fy_thread_pool *tp) { - _Atomic(uint64_t) *free; + FY_ATOMIC(uint64_t) *free; uint64_t v, m; unsigned int i, num_threads_words; num_threads_words = FY_BIT64_COUNT(tp->num_threads); for (i = 0, free = tp->freep; i < num_threads_words - 1; i++, free++) { - v = atomic_load(free); + v = fy_atomic_load(free); if (v != (uint64_t)-1) return false; } - v = atomic_load(free); + v = fy_atomic_load(free); m = FY_BIT64(tp->num_threads & 63) - 1; return (v & m) == m; } static inline bool fy_thread_pool_is_any_reserved_internal(struct fy_thread_pool *tp) { - _Atomic(uint64_t) *free; + FY_ATOMIC(uint64_t) *free; uint64_t v, m; unsigned int i, num_threads_words; num_threads_words = FY_BIT64_COUNT(tp->num_threads); for (i = 0, free = tp->freep; i < num_threads_words - 1; i++, free++) { - v = atomic_load(free); + v = fy_atomic_load(free); if (!v) return true; } - v = atomic_load(free); + v = fy_atomic_load(free); m = FY_BIT64(tp->num_threads & 63) - 1; return !(v & m); } @@ -722,9 +722,9 @@ static inline struct fy_work_pool *fy_work_pool_init(struct fy_work_pool *wp, si if (!wp) return NULL; - atomic_store(&wp->work_left, work_count); + fy_atomic_store(&wp->work_left, work_count); #if defined(__linux__) && !defined(FY_THREAD_PORTABLE) - atomic_store(&wp->done, !work_count); + fy_atomic_store(&wp->done, !work_count); #elif defined(__APPLE__) wp->sem = dispatch_semaphore_create(!work_count); assert(wp->sem != NULL); @@ -764,7 +764,7 @@ static inline bool fy_work_pool_signal(struct fy_work_pool *wp) if (!wp) return false; - prev_work_left = atomic_fetch_sub(&wp->work_left, 1); + prev_work_left = fy_atomic_fetch_sub(&wp->work_left, 1); if (prev_work_left == 1) { #if defined(__linux__) && !defined(FY_THREAD_PORTABLE) rc = fpost(&wp->done); @@ -789,12 +789,12 @@ static inline void fy_work_pool_wait(struct fy_work_pool *wp) return; /* if there's any work left, wait for it */ - while (atomic_load(&wp->work_left) > 0) { + while (fy_atomic_load(&wp->work_left) > 0) { #if defined(__linux__) && !defined(FY_THREAD_PORTABLE) rc = fwait(&wp->done); #elif defined(__APPLE__) dispatch_semaphore_wait(wp->sem, DISPATCH_TIME_FOREVER); - (void)rc; + rc = 0; #else rc = sem_wait(&wp->sem); #endif @@ -823,7 +823,7 @@ static inline struct fy_thread_work *fy_worker_thread_steal_work(struct fy_threa { struct fy_thread *t; unsigned int slot; - _Atomic(uint64_t) *loot; + FY_ATOMIC(uint64_t) *loot; uint64_t exp, v; unsigned int i, num_threads_words; struct fy_thread_work *w, *w_exp; @@ -833,19 +833,19 @@ static inline struct fy_thread_work *fy_worker_thread_steal_work(struct fy_threa num_threads_words = FY_BIT64_COUNT(tp->num_threads); for (i = 0, loot = tp->lootp; i < num_threads_words; i++, loot++) { - v = atomic_load(loot); + v = fy_atomic_load(loot); while (v) { slot = FY_BIT64_LOWEST(v); assert(v & FY_BIT64(slot)); exp = v; /* expecting the previous value */ v &= ~FY_BIT64(slot); /* clear this bit */ - if (atomic_compare_exchange_strong(loot, &exp, v)) { + if (fy_atomic_compare_exchange_strong(loot, &exp, v)) { slot += i * 64; t = tp->threads + slot; - if ((w = atomic_load(&t->next_work)) != NULL) { + if ((w = fy_atomic_load(&t->next_work)) != NULL) { w_exp = w; - if (atomic_compare_exchange_strong(&t->next_work, &w_exp, NULL)) + if (fy_atomic_compare_exchange_strong(&t->next_work, &w_exp, NULL)) return w; } } @@ -883,10 +883,10 @@ static void *fy_worker_thread_steal(void *arg) TDBG("%s: T#%u stole W:%p\n", __func__, t->id, w_stolen); w_exp = w_last; - if (!atomic_compare_exchange_strong(&t->work, &w_exp, w_stolen)) { + if (!fy_atomic_compare_exchange_strong(&t->work, &w_exp, w_stolen)) { assert(w_exp != WORK_SHUTDOWN); - TDBG("%s: T#%u t->work:%p w:%p w_stolen:%p\n", __func__, t->id, atomic_load(&t->work), w, w_stolen); - abort(); + TDBG("%s: T#%u t->work:%p w:%p w_stolen:%p\n", __func__, t->id, fy_atomic_load(&t->work), w, w_stolen); + FY_IMPOSSIBLE_ABORT(); } w = w_stolen; } @@ -895,7 +895,7 @@ static void *fy_worker_thread_steal(void *arg) fy_thread_unreserve_internal(t); w_exp = w_last; - if (!atomic_compare_exchange_strong(&t->work, &w_exp, NULL)) { + if (!fy_atomic_compare_exchange_strong(&t->work, &w_exp, NULL)) { assert(w_exp != WORK_SHUTDOWN); break; } @@ -971,7 +971,7 @@ static void fy_thread_work_join_steal(struct fy_thread_pool *tp, struct fy_threa resolved_t = true; } - if (t && atomic_load(&t->next_work) == NULL) { + if (t && fy_atomic_load(&t->next_work) == NULL) { TDBG("%s: T#%d could not post, available to steal W:%p\n", __func__, tid, works); assert(works->wp == NULL); @@ -981,13 +981,13 @@ static void fy_thread_work_join_steal(struct fy_thread_pool *tp, struct fy_threa works->wp = wp; expw = NULL; - if (!atomic_compare_exchange_strong(&t->next_work, &expw, works)) { + if (!fy_atomic_compare_exchange_strong(&t->next_work, &expw, works)) { TDBG("%s: T#%d could not update next_work: W:%p\n", __func__, tid, works); abort(); } /* set the has loot bit */ - atomic_fetch_or(tp->lootp + (unsigned int)(t->id / 64), FY_BIT64(t->id & 63)); + fy_atomic_fetch_or(tp->lootp + (unsigned int)(t->id / 64), FY_BIT64(t->id & 63)); has_loot = true; } @@ -1009,9 +1009,9 @@ static void fy_thread_work_join_steal(struct fy_thread_pool *tp, struct fy_threa expw = works; /* clear the has loot bit unconditionally */ - atomic_fetch_and(tp->lootp + (unsigned int)(t->id / 64), ~FY_BIT64(t->id & 63)); + fy_atomic_fetch_and(tp->lootp + (unsigned int)(t->id / 64), ~FY_BIT64(t->id & 63)); - if (!atomic_compare_exchange_strong(&t->next_work, &expw, NULL)) { + if (!fy_atomic_compare_exchange_strong(&t->next_work, &expw, NULL)) { TDBG("%s: T#%d had W:%p stolen, good\n", __func__, tid, works); work_count--; works++; @@ -1093,7 +1093,7 @@ static void fy_thread_work_join_steal_2(struct fy_thread_pool *tp, struct fy_thr resolved_t = true; } - if (t && atomic_load(&t->next_work) == NULL) { + if (t && fy_atomic_load(&t->next_work) == NULL) { TDBG("%s: T#%d could not post, available to steal W:%p\n", __func__, tid, &works[1]); assert(works[1].wp == NULL); @@ -1103,13 +1103,13 @@ static void fy_thread_work_join_steal_2(struct fy_thread_pool *tp, struct fy_thr works[1].wp = wp; expw = NULL; - if (!atomic_compare_exchange_strong(&t->next_work, &expw, &works[1])) { + if (!fy_atomic_compare_exchange_strong(&t->next_work, &expw, &works[1])) { TDBG("%s: T#%d could not update next_work: W:%p\n", __func__, tid, &works[1]); abort(); } /* set the has loot bit */ - atomic_fetch_or(tp->lootp + (unsigned int)(t->id / 64), FY_BIT64(t->id & 63)); + fy_atomic_fetch_or(tp->lootp + (unsigned int)(t->id / 64), FY_BIT64(t->id & 63)); has_loot = true; } @@ -1126,9 +1126,9 @@ static void fy_thread_work_join_steal_2(struct fy_thread_pool *tp, struct fy_thr expw = &works[1]; /* clear the has loot bit unconditionally */ - atomic_fetch_and(tp->lootp + (unsigned int)(t->id / 64), ~FY_BIT64(t->id & 63)); + fy_atomic_fetch_and(tp->lootp + (unsigned int)(t->id / 64), ~FY_BIT64(t->id & 63)); - if (!atomic_compare_exchange_strong(&t->next_work, &expw, NULL)) { + if (!fy_atomic_compare_exchange_strong(&t->next_work, &expw, NULL)) { TDBG("%s: T#%d had W:%p stolen\n", __func__, tid, &works[1]); } else { TDBG("%s: T#%d had W:%p not stolen\n", __func__, tid, &works[1]); diff --git a/src/thread/fy-thread.h b/src/thread/fy-thread.h index 4ce45ac..8b18025 100644 --- a/src/thread/fy-thread.h +++ b/src/thread/fy-thread.h @@ -26,6 +26,7 @@ #include "fy-bit64.h" #include "fy-align.h" +#include "fy-atomics.h" #include @@ -33,9 +34,9 @@ // #define FY_THREAD_PORTABLE /* define to use the portable implementation even on linux */ struct fy_work_pool { - _Atomic(size_t) work_left; + FY_ATOMIC(size_t) work_left; #if defined(__linux__) && !defined(FY_THREAD_PORTABLE) - _Atomic(uint32_t) done; + FY_ATOMIC(uint32_t) done; #elif defined(__APPLE__) dispatch_semaphore_t sem; #else @@ -47,11 +48,11 @@ struct fy_thread { struct fy_thread_pool *tp; unsigned int id; pthread_t tid; - _Atomic(struct fy_thread_work *)work; - _Atomic(struct fy_thread_work *)next_work; + FY_ATOMIC(struct fy_thread_work *)work; + FY_ATOMIC(struct fy_thread_work *)next_work; #if defined(__linux__) && !defined(FY_THREAD_PORTABLE) - _Atomic(uint32_t) submit; - _Atomic(uint32_t) done; + FY_ATOMIC(uint32_t) submit; + FY_ATOMIC(uint32_t) done; #else pthread_mutex_t lock; pthread_cond_t cond; @@ -64,8 +65,8 @@ struct fy_thread_pool { struct fy_thread_pool_cfg cfg; unsigned int num_threads; struct fy_thread *threads; - _Atomic(uint64_t) *freep; - _Atomic(uint64_t) *lootp; + FY_ATOMIC(uint64_t) *freep; + FY_ATOMIC(uint64_t) *lootp; pthread_key_t key; }; diff --git a/src/tool/fy-tool-dump.c b/src/tool/fy-tool-dump.c new file mode 100644 index 0000000..a66a4f4 --- /dev/null +++ b/src/tool/fy-tool-dump.c @@ -0,0 +1,767 @@ +/* + * fy-tool-dump.c - tool utils for dumping + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-tool-util.h" + +void print_escaped(const char *str, size_t length) +{ + const uint8_t *p; + int i, c, w; + + for (p = (const uint8_t *)str; length > 0; p += w, length -= (size_t)w) { + + /* get width from the first octet */ + w = (p[0] & 0x80) == 0x00 ? 1 : + (p[0] & 0xe0) == 0xc0 ? 2 : + (p[0] & 0xf0) == 0xe0 ? 3 : + (p[0] & 0xf8) == 0xf0 ? 4 : 0; + + /* error, clip it */ + if ((size_t)w > length) + goto err_out; + + /* initial value */ + c = p[0] & (0xff >> w); + for (i = 1; i < w; i++) { + if ((p[i] & 0xc0) != 0x80) + goto err_out; + c = (c << 6) | (p[i] & 0x3f); + } + + /* check for validity */ + if ((w == 4 && c < 0x10000) || + (w == 3 && c < 0x800) || + (w == 2 && c < 0x80) || + (c >= 0xd800 && c <= 0xdfff) || c >= 0x110000) + goto err_out; + + switch (c) { + case '\\': + printf("\\\\"); + break; + case '\0': + printf("\\0"); + break; + case '\b': + printf("\\b"); + break; + case '\f': + printf("\\f"); + break; + case '\n': + printf("\\n"); + break; + case '\r': + printf("\\r"); + break; + case '\t': + printf("\\t"); + break; + case '\a': + printf("\\a"); + break; + case '\v': + printf("\\v"); + break; + case '\e': + printf("\\e"); + break; + case 0x85: + printf("\\N"); + break; + case 0xa0: + printf("\\_"); + break; + case 0x2028: + printf("\\L"); + break; + case 0x2029: + printf("\\P"); + break; + default: + if ((c >= 0x01 && c <= 0x1f) || c == 0x7f || /* C0 */ + (c >= 0x80 && c <= 0x9f)) /* C1 */ + printf("\\x%02x", c); + else + printf("%.*s", w, p); + break; + } + } + + return; +err_out: + fprintf(stderr, "escape input error\n"); + abort(); +} + +void dump_token_comments(struct fy_token *fyt, bool colorize, const char *banner) +{ + static const char *placement_txt[] = { + [fycp_top] = "top", + [fycp_right] = "right", + [fycp_bottom] = "bottom", + }; + enum fy_comment_placement placement; + char buf[4096]; + const char *str; + + if (!fyt) + return; + + for (placement = fycp_top; placement < fycp_max; placement++) { + str = fy_token_get_comment(fyt, buf, sizeof(buf), placement); + if (!str) + continue; + fputs("\n", stdout); + if (colorize) + fputs(A_RED, stdout); + printf("\t%s %6s: ", banner, placement_txt[placement]); + print_escaped(str, strlen(str)); + if (colorize) + fputs(A_RESET, stdout); + } +} + +void dump_testsuite_event(struct fy_event *fye, enum dump_testsuite_event_flags dump_flags) +{ + bool colorize, disable_flow_markers, disable_doc_markers, disable_scalar_styles, tsv_format; + const char *anchor = NULL; + const char *tag = NULL; + const char *text = NULL; + const char *alias = NULL; + size_t anchor_len = 0, tag_len = 0, text_len = 0, alias_len = 0; + enum fy_scalar_style style; + const struct fy_mark *sm, *em = NULL; + char separator; + size_t spos, epos; + int sline, eline, scolumn, ecolumn; + + colorize = !!(dump_flags & DTEF_COLORIZE); + disable_flow_markers = !!(dump_flags & DTEF_DISABLE_FLOW_MARKERS); + disable_doc_markers = !!(dump_flags & DTEF_DISABLE_DOC_MARKERS); + disable_scalar_styles = !!(dump_flags & DTEF_DISABLE_SCALAR_STYLES); + tsv_format = !!(dump_flags & DTEF_TSV_FORMAT); + + if (!tsv_format) { + separator = ' '; + spos = epos = (size_t)-1; + sline = eline = -1; + scolumn = ecolumn = -1; + } else { + sm = fy_event_start_mark(fye); + if (sm) { + spos = sm->input_pos; + sline = sm->line + 1; + scolumn = sm->column + 1; + } else { + spos = (size_t)-1; + sline = -1; + scolumn = -1; + } + + + em = fy_event_end_mark(fye); + if (em) { + epos = em->input_pos; + eline = em->line + 1; + ecolumn = em->column + 1; + } else { + epos = (size_t)-1; + eline = -1; + ecolumn = -1; + } + separator = '\t'; + /* no colors for TSV */ + colorize = false; + /* no flow or doc markers for TSV */ + disable_flow_markers = true; + disable_doc_markers = true; + disable_scalar_styles = true; + } + + /* event type */ + switch (fye->type) { + case FYET_NONE: + if (colorize) + fputs(A_BRIGHT_RED, stdout); + printf("???"); + break; + case FYET_STREAM_START: + if (colorize) + fputs(A_CYAN, stdout); + printf("+%s", !tsv_format ? "STR" : "str"); + break; + case FYET_STREAM_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("-%s", !tsv_format ? "STR" : "str"); + break; + case FYET_DOCUMENT_START: + if (colorize) + fputs(A_CYAN, stdout); + printf("+%s", !tsv_format ? "DOC" : "doc"); + break; + case FYET_DOCUMENT_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("-%s", !tsv_format ? "DOC" : "doc"); + break; + case FYET_MAPPING_START: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("+%s", !tsv_format ? "MAP" : "map"); + if (fye->mapping_start.anchor) + anchor = fy_token_get_text(fye->mapping_start.anchor, &anchor_len); + if (fye->mapping_start.tag) + tag = fy_token_get_text(fye->mapping_start.tag, &tag_len); + if (!disable_flow_markers && fy_event_get_node_style(fye) == FYNS_FLOW) + printf("%c{}", separator); + break; + case FYET_MAPPING_END: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("-%s", !tsv_format ? "MAP" : "map"); + break; + case FYET_SEQUENCE_START: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("+%s", !tsv_format ? "SEQ" : "seq"); + if (fye->sequence_start.anchor) + anchor = fy_token_get_text(fye->sequence_start.anchor, &anchor_len); + if (fye->sequence_start.tag) + tag = fy_token_get_text(fye->sequence_start.tag, &tag_len); + if (!disable_flow_markers && fy_event_get_node_style(fye) == FYNS_FLOW) + printf("%c[]", separator); + break; + case FYET_SEQUENCE_END: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("-%s", !tsv_format ? "SEQ" : "seq"); + break; + case FYET_SCALAR: + if (colorize) + fputs(A_WHITE, stdout); + printf("=%s", !tsv_format ? "VAL" : "val"); + if (fye->scalar.anchor) + anchor = fy_token_get_text(fye->scalar.anchor, &anchor_len); + if (fye->scalar.tag) + tag = fy_token_get_text(fye->scalar.tag, &tag_len); + break; + case FYET_ALIAS: + if (colorize) + fputs(A_GREEN, stdout); + printf("=%s", !tsv_format ? "ALI" : "ali"); + break; + default: + abort(); + break; + } + + /* (position) anchor and tag */ + if (!tsv_format) { + if (anchor) { + if (colorize) + fputs(A_GREEN, stdout); + printf("%c&%.*s", separator, (int)anchor_len, anchor); + } + if (tag) { + if (colorize) + fputs(A_GREEN, stdout); + printf("%c<%.*s>", separator, (int)tag_len, tag); + } + } else { + if (!anchor) { + anchor = "-"; + anchor_len = 1; + } + if (!tag) { + tag = "-"; + tag_len = 1; + } + printf("%c%zd%c%d%c%d", separator, (ssize_t)spos, + separator, sline, separator, scolumn); + printf("%c%zd%c%d%c%d", separator, (ssize_t)epos, + separator, eline, separator, ecolumn); + printf("%c%.*s", separator, (int)anchor_len, anchor); + printf("%c%.*s", separator, (int)tag_len, tag); + } + + /* style hint */ + switch (fye->type) { + default: + break; + case FYET_DOCUMENT_START: + if (!fy_document_event_is_implicit(fye) && !disable_doc_markers) + printf("%c---", separator); + break; + case FYET_DOCUMENT_END: + if (!fy_document_event_is_implicit(fye) && !disable_doc_markers) + printf("%c...", separator); + break; + case FYET_MAPPING_START: + if (!tsv_format) + break; + printf("%c%s", separator, fy_event_get_node_style(fye) == FYNS_FLOW ? "{}" : ""); + break; + case FYET_SEQUENCE_START: + if (!tsv_format) + break; + printf("%c%s", separator, fy_event_get_node_style(fye) == FYNS_FLOW ? "[]" : ""); + break; + case FYET_SCALAR: + if (!disable_scalar_styles) + style = fy_token_scalar_style(fye->scalar.value); + else + style = FYSS_DOUBLE_QUOTED; /* double quoted can handle anything */ + switch (style) { + case FYSS_PLAIN: + if (colorize) + fputs(A_WHITE, stdout); + printf("%c:", separator); + break; + case FYSS_SINGLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf("%c'", separator); + break; + case FYSS_DOUBLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf("%c\"", separator); + break; + case FYSS_LITERAL: + if (colorize) + fputs(A_YELLOW, stdout); + printf("%c|", separator); + break; + case FYSS_FOLDED: + if (colorize) + fputs(A_YELLOW, stdout); + printf("%c>", separator); + break; + default: + abort(); + } + break; + case FYET_ALIAS: + if (tsv_format) + printf("%c*", separator); + break; + } + + /* content */ + switch (fye->type) { + default: + break; + case FYET_SCALAR: + if (tsv_format) + printf("%c", separator); + text = fy_token_get_text(fye->scalar.value, &text_len); + if (text && text_len > 0) + print_escaped(text, text_len); + break; + case FYET_ALIAS: + alias = fy_token_get_text(fye->alias.anchor, &alias_len); + printf("%c%s%.*s", separator, !tsv_format ? "*" : "", (int)alias_len, alias); + break; + } + + if (colorize) + fputs(A_RESET, stdout); + fputs("\n", stdout); +} + +void dump_parse_event(struct fy_parser *fyp, struct fy_event *fye, bool colorize) +{ + struct fy_token *fyt_tag = NULL, *fyt_anchor = NULL; + const char *anchor = NULL; + const char *tag = NULL; + const char *value = NULL; + size_t anchor_len = 0, tag_len = 0, len = 0; + enum fy_scalar_style style; + const struct fy_version *vers; + const struct fy_tag *tagp = NULL; + void *iterp; + struct fy_document_state *fyds; + + fyt_anchor = fy_event_get_anchor_token(fye); + if (fyt_anchor) { + anchor = fy_token_get_text(fyt_anchor, &anchor_len); + assert(anchor); + } + + fyt_tag = fy_event_get_tag_token(fye); + if (fyt_tag) { + tag = fy_token_get_text(fyt_tag, &tag_len); + assert(tag); + tagp = fy_tag_token_tag(fyt_tag); + assert(tagp); + } + + switch (fye->type) { + case FYET_NONE: + if (colorize) + fputs(A_BRIGHT_RED, stdout); + printf("???"); + break; + case FYET_STREAM_START: + if (colorize) + fputs(A_CYAN, stdout); + printf("STREAM_START"); + dump_token_comments(fye->stream_start.stream_start, colorize, ""); + break; + case FYET_STREAM_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("STREAM_END"); + dump_token_comments(fye->stream_end.stream_end, colorize, ""); + break; + case FYET_DOCUMENT_START: + if (colorize) + fputs(A_CYAN, stdout); + + printf("DOCUMENT_START implicit=%s", + fye->document_start.implicit ? "true" : "false"); + + fyds = fye->document_start.document_state; + assert(fyds); + vers = fy_document_state_version(fyds); + assert(vers); + printf("( V=%d.%d VE=%s TE=%s", vers->major, vers->minor, + fy_document_state_version_explicit(fyds) ? "true" : "false", + fy_document_state_tags_explicit(fyds) ? "true" : "false"); + iterp = NULL; + if ((tagp = fy_document_state_tag_directive_iterate(fyds, &iterp)) != NULL) { + printf(" TDs: ["); + do { + printf(" \"%s\",\"%s\"", tagp->handle, tagp->prefix); + } while ((tagp = fy_document_state_tag_directive_iterate(fyds, &iterp)) != NULL); + printf(" ]"); + } + printf(" )"); + dump_token_comments(fye->document_start.document_start, colorize, ""); + break; + + case FYET_DOCUMENT_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("DOCUMENT_END implicit=%s", + fye->document_end.implicit ? "true" : "false"); + dump_token_comments(fye->document_end.document_end, colorize, ""); + break; + case FYET_MAPPING_START: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("MAPPING_START"); + if (anchor) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" &%.*s", (int)anchor_len, anchor); + } + if (tag) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" <%.*s> (\"%s\",\"%s\")", + (int)tag_len, tag, + tagp->handle, tagp->prefix); + } + dump_token_comments(fye->mapping_start.mapping_start, colorize, ""); + break; + case FYET_MAPPING_END: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("MAPPING_END"); + dump_token_comments(fye->mapping_end.mapping_end, colorize, ""); + break; + case FYET_SEQUENCE_START: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("SEQUENCE_START"); + if (anchor) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" &%.*s", (int)anchor_len, anchor); + } + if (tag) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" <%.*s> (\"%s\",\"%s\")", + (int)tag_len, tag, + tagp->handle, tagp->prefix); + } + dump_token_comments(fye->sequence_start.sequence_start, colorize, ""); + break; + case FYET_SEQUENCE_END: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("SEQUENCE_END"); + dump_token_comments(fye->sequence_end.sequence_end, colorize, ""); + break; + case FYET_SCALAR: + if (colorize) + fputs(A_WHITE, stdout); + printf("SCALAR"); + if (anchor) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" &%.*s", (int)anchor_len, anchor); + } + if (tag) { + if (colorize) + fputs(A_GREEN, stdout); + printf(" <%.*s> (\"%s\",\"%s\")", + (int)tag_len, tag, + tagp->handle, tagp->prefix); + } + + style = fy_token_scalar_style(fye->scalar.value); + switch (style) { + case FYSS_PLAIN: + if (colorize) + fputs(A_WHITE, stdout); + printf(" "); + break; + case FYSS_SINGLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" '"); + break; + case FYSS_DOUBLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" \""); + break; + case FYSS_LITERAL: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" |"); + break; + case FYSS_FOLDED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" >"); + break; + default: + abort(); + } + value = fy_token_get_text(fye->scalar.value, &len); + if (value && len > 0) + print_escaped(value, len); + dump_token_comments(fye->scalar.value, colorize, ""); + break; + case FYET_ALIAS: + anchor = fy_token_get_text(fye->alias.anchor, &anchor_len); + if (colorize) + fputs(A_GREEN, stdout); + printf("ALIAS *%.*s", (int)anchor_len, anchor); + dump_token_comments(fye->alias.anchor, colorize, ""); + break; + default: + /* ignored */ + break; + } + if (colorize) + fputs(A_RESET, stdout); + fputs("\n", stdout); +} + +void dump_scan_token(struct fy_parser *fyp, struct fy_token *fyt, bool colorize) +{ + const char *anchor = NULL, *value = NULL; + size_t anchor_len = 0, len = 0; + enum fy_scalar_style style; + const struct fy_version *vers; + const struct fy_tag *tag; + + switch (fy_token_get_type(fyt)) { + case FYTT_NONE: + if (colorize) + fputs(A_BRIGHT_RED, stdout); + printf("NONE"); + break; + case FYTT_STREAM_START: + if (colorize) + fputs(A_CYAN, stdout); + printf("STREAM_START"); + break; + case FYTT_STREAM_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("STREAM_END"); + break; + case FYTT_VERSION_DIRECTIVE: + if (colorize) + fputs(A_CYAN, stdout); + vers = fy_version_directive_token_version(fyt); + assert(vers); + printf("VERSION_DIRECTIVE major=%d minor=%d", vers->major, vers->minor); + break; + case FYTT_TAG_DIRECTIVE: + if (colorize) + fputs(A_CYAN, stdout); + tag = fy_tag_directive_token_tag(fyt); + assert(tag); + printf("TAG_DIRECTIVE handle=\"%s\" prefix=\"%s\"", tag->handle, tag->prefix); + break; + case FYTT_DOCUMENT_START: + if (colorize) + fputs(A_CYAN, stdout); + printf("DOCUMENT_START"); + break; + case FYTT_DOCUMENT_END: + if (colorize) + fputs(A_CYAN, stdout); + printf("DOCUMENT_END"); + break; + case FYTT_BLOCK_SEQUENCE_START: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("BLOCK_SEQUENCE_START"); + break; + case FYTT_BLOCK_MAPPING_START: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("BLOCK_MAPPING_START"); + break; + case FYTT_BLOCK_END: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("BLOCK_END"); + break; + case FYTT_FLOW_SEQUENCE_START: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("FLOW_SEQUENCE_START"); + break; + case FYTT_FLOW_SEQUENCE_END: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("FLOW_SEQUENCE_END"); + break; + case FYTT_FLOW_MAPPING_START: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("FLOW_MAPPING_START"); + break; + case FYTT_FLOW_MAPPING_END: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("FLOW_MAPPING_END"); + break; + case FYTT_BLOCK_ENTRY: + if (colorize) + fputs(A_BRIGHT_CYAN, stdout); + printf("BLOCK_ENTRY"); + break; + case FYTT_FLOW_ENTRY: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("BLOCK_ENTRY"); + break; + case FYTT_KEY: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("KEY"); + break; + case FYTT_VALUE: + if (colorize) + fputs(A_BRIGHT_YELLOW, stdout); + printf("KEY"); + break; + case FYTT_ALIAS: + anchor = fy_token_get_text(fyt, &anchor_len); + assert(anchor); + if (colorize) + fputs(A_GREEN, stdout); + printf("ALIAS *%.*s", (int)anchor_len, anchor); + break; + case FYTT_ANCHOR: + anchor = fy_token_get_text(fyt, &anchor_len); + assert(anchor); + if (colorize) + fputs(A_GREEN, stdout); + printf("ANCHOR &%.*s", (int)anchor_len, anchor); + break; + case FYTT_TAG: + tag = fy_tag_token_tag(fyt); + assert(tag); + if (colorize) + fputs(A_GREEN, stdout); + /* prefix is a suffix for tag */ + printf("TAG handle=\"%s\" suffix=\"%s\"", tag->handle, tag->prefix); + break; + case FYTT_SCALAR: + if (colorize) + fputs(A_WHITE, stdout); + + printf("SCALAR "); + value = fy_token_get_text(fyt, &len); + assert(value); + style = fy_token_scalar_style(fyt); + switch (style) { + case FYSS_PLAIN: + if (colorize) + fputs(A_WHITE, stdout); + printf(" "); + break; + case FYSS_SINGLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" '"); + break; + case FYSS_DOUBLE_QUOTED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" \""); + break; + case FYSS_LITERAL: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" |"); + break; + case FYSS_FOLDED: + if (colorize) + fputs(A_YELLOW, stdout); + printf(" >"); + break; + default: + abort(); + } + printf("%.*s", (int)len, value); + break; + default: + /* not handled; should not be produced by scan */ + break; + } + if (colorize) + fputs(A_RESET, stdout); + fputs("\n", stdout); +} diff --git a/src/tool/fy-tool-util.c b/src/tool/fy-tool-util.c new file mode 100644 index 0000000..a3eac5d --- /dev/null +++ b/src/tool/fy-tool-util.c @@ -0,0 +1,251 @@ +/* + * fy-tool-util.c - tool utils + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "fy-tool-util.h" + +uintmax_t load_le(const void *ptr, size_t width, bool is_signed) +{ + uintmax_t v; + const uint8_t *p; + size_t off; + + assert(width <= sizeof(uintmax_t)); + + switch (width) { + case sizeof(uint8_t): + v = (uintmax_t)*(uint8_t *)ptr; + break; + case sizeof(uint16_t): + v = (uintmax_t)*(uint16_t *)ptr; + break; + case sizeof(uint32_t): + v = (uintmax_t)*(uint32_t *)ptr; + break; + case sizeof(uint64_t): + v = (uintmax_t)*(uint64_t *)ptr; + break; + default: + for (v = 0, p = ptr, off = 0; off < width; off++) + v |= (uintmax_t)p[off] << off; + break; + } + + /* sign extension? */ + if (is_signed && width < sizeof(uintmax_t) && (v & ((uintmax_t)1 << (width * 8 - 1)))) + v |= (uintmax_t)-1 << (width * 8); + + return v; +} + +void store_le(void *ptr, size_t width, uintmax_t v) +{ + uint8_t *p; + size_t off; + + switch (width) { + case sizeof(uint8_t): + *(uint8_t *)ptr = (uint8_t)v; + break; + case sizeof(uint16_t): + *(uint16_t *)ptr = (uint16_t)v; + break; + case sizeof(uint32_t): + *(uint32_t *)ptr = (uint32_t)v; + break; + case sizeof(uint64_t): + *(uint64_t *)ptr = (uint64_t)v; + break; + default: + for (p = ptr, off = 0; off < width; off++) + p[off] = (uint8_t)(v >> (8 * off)); + break; + } +} + +uintmax_t load_bitfield_le(const void *ptr, size_t bit_offset, size_t bit_width, bool is_signed) +{ + const uint8_t *p; + size_t off, width, space, use; + uint8_t bmask; + uintmax_t v; + + v = 0; + + width = bit_width; + p = ptr + bit_offset / 8; + off = bit_offset & 7; + if (off) { + space = 8 - off; + use = width > space ? space : width; + + bmask = (((uint8_t)1 << use) - 1) << off; + width -= use; + + v = (*p++ & bmask) >> off; + off = use; + + // fprintf(stderr, "%s: 0. [%02x] use=%zu v=%jx\n", __func__, p[-1] & 0xff, use, v); + } + while (width >= 8) { + v |= (uintmax_t)*p++ << off; + width -= 8; + off += 8; + // fprintf(stderr, "%s: 1. [%02x] v=%jx\n", __func__, p[-1] & 0xff, v); + } + if (width) { + v |= (uintmax_t)(*p & ((1 << width) - 1)) << off; + // fprintf(stderr, "%s: 2. [%02x] off=%zu v=%jx\n", __func__, p[0], off, v); + } + + /* sign extension? */ + if (bit_width < sizeof(uintmax_t) * 8) { + if (is_signed) { + if (v & ((uintmax_t)1 << (bit_width - 1))) + v |= (uintmax_t)-1 << bit_width; + } else { + v &= ~((uintmax_t)-1 << bit_width); + } + } + + return v; +} + +void store_bitfield_le(void *ptr, size_t bit_offset, size_t bit_width, uintmax_t v) +{ + uint8_t *p; + size_t off, width, space, use; + uint8_t bmask; + + width = bit_width; + p = ptr + bit_offset / 8; + off = bit_offset & 7; + if (off) { + space = 8 - off; + use = width > space ? space : width; + + bmask = (((uint8_t)1 << use) - 1) << off; + + *p = (*p & ~bmask) | ((uint8_t)(v << off) & bmask); + p++; + + // fprintf(stderr, "%s: 0. [%02x] bmask=%02x off=%zu %02x v=%jx\n", __func__, p[-1] & 0xff, bmask, off, (uint8_t)(v << off) & 0xff, v); + + v >>= use; + width -= use; + } + while (width >= 8) { + *p++ = (uint8_t)v; + + // fprintf(stderr, "%s: 1. [%02x] v=%jx\n", __func__, p[-1] & 0xff, v); + + v >>= 8; + width -= 8; + } + if (width) { + bmask = (1 << width) - 1; + *p = (*p & ~bmask) | ((uint8_t)v & bmask); + + // fprintf(stderr, "%s: 1. [%02x] v=%jx\n", __func__, p[0] & 0xff, v); + } +} + +#define STRTOXF_IS_UNSIGNED BIT(0) +#define STRTOXF_IS_SIGNED BIT(1) +#define STRTOXF_SKIP_UNDERSCORE BIT(2) +#define STRTOXF_ALLOW_BASE2 BIT(3) +#define STRTOXF_ALLOW_BASE8 BIT(4) +#define STRTOXF_ALLOW_BASE16 BIT(5) +#define STRTOXF_SINGLE_ZERO BIT(6) + +#define STRTOXF_YAML (STRTOXF_ALLOW_BASE8 | STRTOXF_ALLOW_BASE16) +#define STRTOXF_YAML_1_1 (STRTOXF_ALLOW_BASE2 | STRTOXF_ALLOW_BASE8 | \ + STRTOXF_ALLOW_BASE16 | STRTOXF_SKIP_UNDERSCORE) +#define STRTOXF_JSON (STRTOXF_SINGLE_ZERO) + +#define STRTOX_DECLARE(_type, _typename, _flags) \ +int str_to_ ## _typename (const char *str, _type *res) \ +{ \ + int base, dv; \ + bool negative; \ + char c; \ + _type v; \ + \ + negative = false; \ + if (*str == '+' || *str == '-') { \ + negative = *str == '-'; \ + str++; \ + if (!((_flags) & STRTOXF_IS_SIGNED) && negative) \ + return -EINVAL; \ + } \ + base = 10; \ + if (*str == '0') { \ + if (((_flags) & STRTOXF_ALLOW_BASE16) && str[1] == 'x') { \ + str += 2; \ + base = 16; \ + } else if (((_flags) & STRTOXF_ALLOW_BASE8) && str[1] == 'o') { \ + str += 2; \ + base = 8; \ + } else if (((_flags) & STRTOXF_ALLOW_BASE2) && str[1] == 'b') { \ + str += 2; \ + base = 2; \ + } else if (((_flags) & STRTOXF_SINGLE_ZERO) && str[1] == '\0') \ + return -EINVAL; \ + } else if (*str == '\0') \ + return -EINVAL; /* empty number without digits */ \ + v = 0; \ + while ((c = *str++) != '\0') { \ + if (((_flags) & STRTOXF_SKIP_UNDERSCORE) && c == '_') \ + continue; \ + dv = c >= '0' && c <= '9' ? (c - '0') : \ + c >= 'a' && c <= 'z' ? (10 + c - 'a') : \ + c >= 'A' && c <= 'Z' ? (10 + c - 'A') : -1; \ + if (dv < 0 || dv >= (int)base) \ + return -EINVAL; \ + if (MUL_OVERFLOW(v, (_type)base, &v) || \ + ADD_OVERFLOW(v, (_type)(!negative ? dv : -dv), &v)) \ + return -ERANGE; \ + } \ + *res = v; \ + return 0; \ +} \ +struct __useless_struct_to_allow_semicolon + +typedef int (*strtox_intmax_func)(const char *str, intmax_t *res); +typedef int (*strtox_uintmax_func)(const char *str, uintmax_t *res); + +STRTOX_DECLARE(intmax_t, intmax, STRTOXF_IS_SIGNED | STRTOXF_YAML); +STRTOX_DECLARE(uintmax_t, uintmax, STRTOXF_IS_UNSIGNED | STRTOXF_YAML); + +STRTOX_DECLARE(intmax_t, intmax_1_1, STRTOXF_IS_SIGNED | STRTOXF_YAML_1_1); +STRTOX_DECLARE(uintmax_t, uintmax_1_1, STRTOXF_IS_UNSIGNED | STRTOXF_YAML_1_1); + +STRTOX_DECLARE(intmax_t, intmax_json, STRTOXF_IS_SIGNED | STRTOXF_JSON); +STRTOX_DECLARE(uintmax_t, uintmax_json, STRTOXF_IS_UNSIGNED | STRTOXF_JSON); diff --git a/src/tool/fy-tool-util.h b/src/tool/fy-tool-util.h new file mode 100644 index 0000000..ca34287 --- /dev/null +++ b/src/tool/fy-tool-util.h @@ -0,0 +1,255 @@ +/* + * fy-tool-utils.h - internal utilities header file + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_TOOL_UTILS_H +#define FY_TOOL_UTILS_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#endif + +#ifndef BIT +#define BIT(x) (1U << (x)) +#endif + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) ((sizeof(x)/sizeof((x)[0]))) +#endif + +#if defined(NDEBUG) && (defined(__GNUC__) && __GNUC__ >= 4) +#define ALWAYS_INLINE __attribute__((always_inline)) +#else +#define ALWAYS_INLINE /* nothing */ +#endif + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define UNUSED __attribute__((unused)) +#else +#define UNUSED /* nothing */ +#endif + +#if defined(NDEBUG) && defined(__GNUC__) && __GNUC__ >= 4 +#define DEBUG_UNUSED __attribute__((unused)) +#else +#define DEBUG_UNUSED /* nothing */ +#endif + +/* if expression is zero, then the build will break */ +#define COMPILE_ERROR_ON_ZERO(_e) ((void)(sizeof(char[1 - 2*!!(_e)]))) + +/* true, if types are the same, false otherwise (depends on builtin_types_compatible_p) */ +#if defined(__has_builtin) && __has_builtin(__builtin_types_compatible_p) +#define SAME_TYPE(_a, _b) __builtin_types_compatible_p(__typeof__(_a), __typeof__(_b)) +#else +#define SAME_TYPE(_a, _b) true +#endif + +/* compile error if types are not the same */ +#define CHECK_SAME_TYPE(_a, _b) \ + COMPILE_ERROR_ON_ZERO(!SAME_TYPE(_a, _b)) + +/* type safe add overflow */ +#if defined(__has_builtin) && __has_builtin(__builtin_add_overflow) +#define ADD_OVERFLOW __builtin_add_overflow +#else +#define ADD_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + CHECK_SAME_TYPE(__a, __b); \ + \ + __res = __a + __b; \ + /* overflow when signs of a, b same, but results different */ \ + __overflow = ((__a ^ __result) & (__b & __result)) < 0; \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +/* type safe sub overflow */ +#if defined(__has_builtin) && __has_builtin(__builtin_sub_overflow) +#define SUB_OVERFLOW __builtin_sub_overflow +#else +#define SUB_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + CHECK_SAME_TYPE(__a, __b); \ + \ + __res = __a - __b; \ + /* overflow when signs of a, b differ, but results different from minuend */ \ + __overflow = ((__a ^ __b) & (__a & __result)) < 0; \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +/* type safe multiply overflow */ +#if defined(__has_builtin) && __has_builtin(__builtin_mul_overflow) +#define MUL_OVERFLOW __builtin_mul_overflow +#else +#define MUL_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + CHECK_SAME_TYPE(__a, __b); \ + \ + if (!__a || !__b) { \ + __overflow = false; \ + __res = 0; \ + } else { \ + __res = __a * __b; \ + /* overflow when division of the result differs */ \ + __overflow = (__res / __a) != __b; \ + } \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +/* ANSI colors and escapes */ +#define A_RESET "\x1b[0m" +#define A_BLACK "\x1b[30m" +#define A_RED "\x1b[31m" +#define A_GREEN "\x1b[32m" +#define A_YELLOW "\x1b[33m" +#define A_BLUE "\x1b[34m" +#define A_MAGENTA "\x1b[35m" +#define A_CYAN "\x1b[36m" +#define A_LIGHT_GRAY "\x1b[37m" /* dark white is gray */ +#define A_GRAY "\x1b[1;30m" +#define A_BRIGHT_RED "\x1b[1;31m" +#define A_BRIGHT_GREEN "\x1b[1;32m" +#define A_BRIGHT_YELLOW "\x1b[1;33m" +#define A_BRIGHT_BLUE "\x1b[1;34m" +#define A_BRIGHT_MAGENTA "\x1b[1;35m" +#define A_BRIGHT_CYAN "\x1b[1;36m" +#define A_WHITE "\x1b[1;37m" + +static inline bool +memiszero(const void *ptr, size_t size) +{ + const uint8_t *p; + size_t i; + + for (i = 0, p = ptr; i < size; i++) { + if (p[i]) + return false; + } + return true; +} + +/* in fy-tool-dump.c */ +void print_escaped(const char *str, size_t length); + +enum dump_testsuite_event_flags { + DTEF_COLORIZE = FY_BIT(0), + DTEF_DISABLE_FLOW_MARKERS = FY_BIT(1), + DTEF_DISABLE_DOC_MARKERS = FY_BIT(2), + DTEF_DISABLE_SCALAR_STYLES = FY_BIT(3), + DTEF_TSV_FORMAT = FY_BIT(4), +}; + +void dump_token_comments(struct fy_token *fyt, bool colorize, const char *banner); +void dump_testsuite_event(struct fy_event *fye, enum dump_testsuite_event_flags dump_flags); +void dump_parse_event(struct fy_parser *fyp, struct fy_event *fye, bool colorize); +void dump_scan_token(struct fy_parser *fyp, struct fy_token *fyt, bool colorize); + +/* */ + +static inline int parse_match_value(const char *text0, const char **check_vp) +{ + const char *check; + int i; + + for (i = 0; (check = *check_vp++) != NULL; i++) { + if (!strcmp(check, text0)) + return i; + } + return -1; +} + +union integer_scalar { + intmax_t sval; + uintmax_t uval; +}; + +union float_scalar { + float f; + double d; + long double ld; +}; + +uintmax_t load_le(const void *ptr, size_t width, bool is_signed); +void store_le(void *ptr, size_t width, uintmax_t v); +uintmax_t load_bitfield_le(const void *ptr, size_t bit_offset, size_t bit_width, bool is_signed); +void store_bitfield_le(void *ptr, size_t bit_offset, size_t bit_width, uintmax_t v); + +static inline intmax_t signed_integer_max_from_bit_width(int bit_width) +{ + assert((size_t)bit_width <= sizeof(intmax_t) * 8); + return INTMAX_MAX >> (sizeof(intmax_t) * 8 - bit_width); +} + +static inline intmax_t signed_integer_min_from_bit_width(int bit_width) +{ + assert((size_t)bit_width <= sizeof(intmax_t) * 8); + return INTMAX_MIN >> (sizeof(intmax_t) * 8 - bit_width); +} + +static inline uintmax_t unsigned_integer_max_from_bit_width(int bit_width) +{ + assert((size_t)bit_width <= sizeof(uintmax_t) * 8); + return UINTMAX_MAX >> (sizeof(uintmax_t) * 8 - bit_width); +} + +static inline intmax_t +signed_integer_min_from_size(size_t size) +{ + return signed_integer_min_from_bit_width(size * 8); +} + +static inline intmax_t +signed_integer_max_from_size(size_t size) +{ + return signed_integer_max_from_bit_width(size * 8); +} + +static inline uintmax_t +unsigned_integer_max_from_size(size_t size) +{ + return unsigned_integer_max_from_bit_width(size * 8); +} + +static inline bool str_null_eq(const char *s1, const char *s2) +{ + if (s1 == s2) + return true; + if (!s1 || !s2) + return false; + return !strcmp(s1, s2); +} + +#endif diff --git a/src/tool/fy-tool.c b/src/tool/fy-tool.c index 52d0d56..b8b15cd 100644 --- a/src/tool/fy-tool.c +++ b/src/tool/fy-tool.c @@ -18,10 +18,20 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include "fy-valgrind.h" +#include "fy-tool-util.h" #define QUIET_DEFAULT false #define INCLUDE_DEFAULT "" @@ -41,7 +51,8 @@ #define STRIP_LABELS_DEFAULT false #define STRIP_TAGS_DEFAULT false #define STRIP_DOC_DEFAULT false -#define STREAMING_DEFAULT false +#define STREAMING_DEFAULT true +#define RECREATING_DEFAULT false #define JSON_DEFAULT "auto" #define DISABLE_ACCEL_DEFAULT false #define DISABLE_BUFFERING_DEFAULT false @@ -50,6 +61,8 @@ #define PREFER_RECURSIVE_DEFAULT false #define YPATH_ALIASES_DEFAULT false #define DISABLE_FLOW_MARKERS_DEFAULT false +#define DISABLE_DOC_MARKERS_DEFAULT false +#define DISABLE_SCALAR_STYLES_DEFAULT false #define DUMP_PATH_DEFAULT false #define DOCUMENT_EVENT_STREAM_DEFAULT false #define COLLECT_ERRORS_DEFAULT false @@ -68,11 +81,13 @@ #define OPT_YAML_VERSION_DUMP 1008 #define OPT_COMPOSE 1009 #define OPT_B3SUM 1010 +#define OPT_REFLECT 1011 #define OPT_STRIP_LABELS 2000 #define OPT_STRIP_TAGS 2001 #define OPT_STRIP_DOC 2002 #define OPT_STREAMING 2003 +#define OPT_RECREATING 2004 #define OPT_DISABLE_ACCEL 2005 #define OPT_DISABLE_BUFFERING 2006 #define OPT_DISABLE_DEPTH_LIMIT 2007 @@ -90,6 +105,9 @@ #define OPT_STRIP_EMPTY_KV 2019 #define OPT_DISABLE_MMAP 2020 #define OPT_TSV_FORMAT 2021 +#define OPT_DISABLE_DOC_MARKERS 2022 +#define OPT_DISABLE_SCALAR_STYLES 2023 +#define OPT_NO_STREAMING 2024 #define OPT_DISABLE_DIAG 3000 #define OPT_ENABLE_DIAG 3001 @@ -144,6 +162,8 @@ static struct option lopts[] = { {"strip-tags", no_argument, 0, OPT_STRIP_TAGS }, {"strip-doc", no_argument, 0, OPT_STRIP_DOC }, {"streaming", no_argument, 0, OPT_STREAMING }, + {"no-streaming", no_argument, 0, OPT_NO_STREAMING }, + {"recreating", no_argument, 0, OPT_RECREATING }, {"disable-accel", no_argument, 0, OPT_DISABLE_ACCEL }, {"disable-buffering", no_argument, 0, OPT_DISABLE_BUFFERING }, {"disable-depth-limit", no_argument, 0, OPT_DISABLE_DEPTH_LIMIT }, @@ -159,6 +179,8 @@ static struct option lopts[] = { {"prefer-recursive", no_argument, 0, OPT_PREFER_RECURSIVE }, {"ypath-aliases", no_argument, 0, OPT_YPATH_ALIASES }, {"disable-flow-markers",no_argument, 0, OPT_DISABLE_FLOW_MARKERS }, + {"disable-doc-markers", no_argument, 0, OPT_DISABLE_DOC_MARKERS }, + {"disable-scalar-styles",no_argument, 0, OPT_DISABLE_SCALAR_STYLES }, {"dump-pathexpr", no_argument, 0, OPT_DUMP_PATHEXPR }, {"document-event-stream",no_argument, 0, OPT_DOCUMENT_EVENT_STREAM }, {"noexec", no_argument, 0, OPT_NOEXEC }, @@ -201,8 +223,8 @@ static void display_usage(FILE *fp, char *progname, int tool_mode) DEBUG_LEVEL_DEFAULT); fprintf(fp, "\t--disable-diag : Disable diag error module \n"); fprintf(fp, "\t--enable-diag : Enable diag error module \n"); - fprintf(fp, "\t--show-diag : Show diag option \n"); - fprintf(fp, "\t--hide-diag : Hide diag optione \n"); + fprintf(fp, "\t--show-diag : Show diag option (source, position, type, module)\n"); + fprintf(fp, "\t--hide-diag : Hide diag optione (source, position, type, module)\n"); fprintf(fp, "\t--indent, -i : Set dump indent to " " (default indent %d)\n", @@ -267,6 +289,7 @@ static void display_usage(FILE *fp, char *progname, int tool_mode) fprintf(fp, "\t--quiet, -q : Quiet operation, do not " "output messages (default %s)\n", QUIET_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--dry-run : Do not parse/emit\n"); fprintf(fp, "\t--version, -v : Display libfyaml version\n"); fprintf(fp, "\t--help, -h : Display help message\n"); @@ -277,22 +300,31 @@ static void display_usage(FILE *fp, char *progname, int tool_mode) fprintf(fp, "\t--comment, -c : Output comments (experimental)" " (default %s)\n", COMMENT_DEFAULT ? "true" : "false"); - fprintf(fp, "\t--mode, -m : Output mode can be one of original, block, flow, flow-oneline, json, json-tp, json-oneline, dejson, pretty|yamlfmt" + fprintf(fp, "\t--mode, -m : Output mode can be one of original, block, flow, flow-oneline, json, json-tp, json-oneline, dejson, pretty|yamlfmt, flow-compact, json-compact" " (default %s)\n", MODE_DEFAULT); fprintf(fp, "\t--disable-flow-markers : Disable testsuite's flow-markers" " (default %s)\n", DISABLE_FLOW_MARKERS_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--disable-doc-markers : Disable testsuite's document-markers" + " (default %s)\n", + DISABLE_DOC_MARKERS_DEFAULT ? "true" : "false"); + fprintf(fp, "\t--disable-scalar-styles : Disable testsuite's scalar styles (all are double quoted)" + " (default %s)\n", + DISABLE_SCALAR_STYLES_DEFAULT ? "true" : "false"); fprintf(fp, "\t--document-event-stream : Generate a document and then produce the event stream" " (default %s)\n", DOCUMENT_EVENT_STREAM_DEFAULT ? "true" : "false"); fprintf(fp, "\t--tsv-format : Display testsuite in TSV format" " (default %s)\n", TSV_FORMAT_DEFAULT ? "true" : "false"); - if (tool_mode == OPT_TOOL || tool_mode == OPT_DUMP) - fprintf(fp, "\t--streaming : Use streaming output mode" + if (tool_mode == OPT_TOOL || tool_mode == OPT_DUMP) { + fprintf(fp, "\t--streaming : Use streaming output mode (default)"); + fprintf(fp, "\t--no-streaming : Don't use streaming output mode"); + fprintf(fp, "\t--recreating : Recreate streaming events" " (default %s)\n", - STREAMING_DEFAULT ? "true" : "false"); + RECREATING_DEFAULT ? "true" : "false"); + } } if (tool_mode == OPT_TOOL || (tool_mode != OPT_DUMP && tool_mode != OPT_TESTSUITE)) { @@ -447,6 +479,8 @@ static int apply_mode_flags(const char *what, enum fy_emitter_cfg_flags *flagsp) { .name = "dejson", .value = FYECF_MODE_DEJSON }, { .name = "pretty", .value = FYECF_MODE_PRETTY }, { .name = "yamlfmt", .value = FYECF_MODE_PRETTY }, /* alias for pretty */ + { .name = "flow-compact", .value = FYECF_MODE_FLOW_COMPACT }, + { .name = "json-compact", .value = FYECF_MODE_JSON_COMPACT }, }; unsigned int i; @@ -472,7 +506,7 @@ int apply_flags_option(const char *arg, unsigned int *flagsp, { const char *s, *e, *sn; char *targ; - int len, ret; + int len, rc; if (!arg || !flagsp || !modify_flags) return -1; @@ -489,9 +523,9 @@ int apply_flags_option(const char *arg, unsigned int *flagsp, memcpy(targ, s, len); targ[len] = '\0'; - ret = modify_flags(targ, flagsp); - if (ret) - return ret; + rc = modify_flags(targ, flagsp); + if (rc) + return rc; s = sn < e ? (sn + 1) : sn; } @@ -499,877 +533,6 @@ int apply_flags_option(const char *arg, unsigned int *flagsp, return 0; } -struct dump_userdata { - FILE *fp; - bool colorize; - bool visible; -}; - -static inline int -utf8_width_by_first_octet(uint8_t c) -{ - return (c & 0x80) == 0x00 ? 1 : - (c & 0xe0) == 0xc0 ? 2 : - (c & 0xf0) == 0xe0 ? 3 : - (c & 0xf8) == 0xf0 ? 4 : 0; -} - -/* ANSI colors and escapes */ -#define A_RESET "\x1b[0m" -#define A_BLACK "\x1b[30m" -#define A_RED "\x1b[31m" -#define A_GREEN "\x1b[32m" -#define A_YELLOW "\x1b[33m" -#define A_BLUE "\x1b[34m" -#define A_MAGENTA "\x1b[35m" -#define A_CYAN "\x1b[36m" -#define A_LIGHT_GRAY "\x1b[37m" /* dark white is gray */ -#define A_GRAY "\x1b[1;30m" -#define A_BRIGHT_RED "\x1b[1;31m" -#define A_BRIGHT_GREEN "\x1b[1;32m" -#define A_BRIGHT_YELLOW "\x1b[1;33m" -#define A_BRIGHT_BLUE "\x1b[1;34m" -#define A_BRIGHT_MAGENTA "\x1b[1;35m" -#define A_BRIGHT_CYAN "\x1b[1;36m" -#define A_WHITE "\x1b[1;37m" - -static int do_output(struct fy_emitter *fye, enum fy_emitter_write_type type, const char *str, int len, void *userdata) -{ - struct dump_userdata *du = userdata; - FILE *fp = du->fp; - int ret, w; - const char *color = NULL; - const char *s, *e; - - s = str; - e = str + len; - if (du->colorize) { - switch (type) { - case fyewt_document_indicator: - color = A_CYAN; - break; - case fyewt_tag_directive: - case fyewt_version_directive: - color = A_YELLOW; - break; - case fyewt_indent: - if (du->visible) { - fputs(A_GREEN, fp); - while (s < e && (w = utf8_width_by_first_octet(((uint8_t)*s))) > 0) { - /* open box - U+2423 */ - fputs("\xe2\x90\xa3", fp); - s += w; - } - fputs(A_RESET, fp); - return len; - } - break; - case fyewt_indicator: - if (len == 1 && (str[0] == '\'' || str[0] == '"')) - color = A_YELLOW; - else if (len == 1 && str[0] == '&') - color = A_BRIGHT_GREEN; - else - color = A_MAGENTA; - break; - case fyewt_whitespace: - if (du->visible) { - fputs(A_GREEN, fp); - while (s < e && (w = utf8_width_by_first_octet(((uint8_t)*s))) > 0) { - /* symbol for space - U+2420 */ - /* symbol for interpunct - U+00B7 */ - fputs("\xc2\xb7", fp); - s += w; - } - fputs(A_RESET, fp); - return len; - } - break; - case fyewt_plain_scalar: - color = A_WHITE; - break; - case fyewt_single_quoted_scalar: - case fyewt_double_quoted_scalar: - color = A_YELLOW; - break; - case fyewt_literal_scalar: - case fyewt_folded_scalar: - color = A_YELLOW; - break; - case fyewt_anchor: - case fyewt_tag: - case fyewt_alias: - color = A_BRIGHT_GREEN; - break; - case fyewt_linebreak: - if (du->visible) { - fputs(A_GREEN, fp); - while (s < e && (w = utf8_width_by_first_octet(((uint8_t)*s))) > 0) { - /* symbol for space - ^M */ - /* fprintf(fp, "^M\n"); */ - /* down arrow - U+2193 */ - fputs("\xe2\x86\x93\n", fp); - s += w; - } - fputs(A_RESET, fp); - return len; - } - color = NULL; - break; - case fyewt_terminating_zero: - color = NULL; - break; - case fyewt_plain_scalar_key: - case fyewt_single_quoted_scalar_key: - case fyewt_double_quoted_scalar_key: - color = A_BRIGHT_CYAN; - break; - case fyewt_comment: - color = A_BRIGHT_BLUE; - break; - } - } - - /* don't output the terminating zero */ - if (type == fyewt_terminating_zero) - return len; - - if (color) - fputs(color, fp); - - ret = fwrite(str, 1, len, fp); - - if (color) - fputs(A_RESET, fp); - - return ret; -} - -void print_escaped(const char *str, size_t length) -{ - const uint8_t *p; - int i, c, w; - - for (p = (const uint8_t *)str; length > 0; p += w, length -= (size_t)w) { - - /* get width from the first octet */ - w = (p[0] & 0x80) == 0x00 ? 1 : - (p[0] & 0xe0) == 0xc0 ? 2 : - (p[0] & 0xf0) == 0xe0 ? 3 : - (p[0] & 0xf8) == 0xf0 ? 4 : 0; - - /* error, clip it */ - if ((size_t)w > length) - goto err_out; - - /* initial value */ - c = p[0] & (0xff >> w); - for (i = 1; i < w; i++) { - if ((p[i] & 0xc0) != 0x80) - goto err_out; - c = (c << 6) | (p[i] & 0x3f); - } - - /* check for validity */ - if ((w == 4 && c < 0x10000) || - (w == 3 && c < 0x800) || - (w == 2 && c < 0x80) || - (c >= 0xd800 && c <= 0xdfff) || c >= 0x110000) - goto err_out; - - switch (c) { - case '\\': - printf("\\\\"); - break; - case '\0': - printf("\\0"); - break; - case '\b': - printf("\\b"); - break; - case '\f': - printf("\\f"); - break; - case '\n': - printf("\\n"); - break; - case '\r': - printf("\\r"); - break; - case '\t': - printf("\\t"); - break; - case '\a': - printf("\\a"); - break; - case '\v': - printf("\\v"); - break; - case '\e': - printf("\\e"); - break; - case 0x85: - printf("\\N"); - break; - case 0xa0: - printf("\\_"); - break; - case 0x2028: - printf("\\L"); - break; - case 0x2029: - printf("\\P"); - break; - default: - if ((c >= 0x01 && c <= 0x1f) || c == 0x7f || /* C0 */ - (c >= 0x80 && c <= 0x9f)) /* C1 */ - printf("\\x%02x", c); - else - printf("%.*s", w, p); - break; - } - } - - return; -err_out: - fprintf(stderr, "escape input error\n"); - abort(); -} - -void dump_token_comments(struct fy_token *fyt, bool colorize, const char *banner) -{ - static const char *placement_txt[] = { - [fycp_top] = "top", - [fycp_right] = "right", - [fycp_bottom] = "bottom", - }; - enum fy_comment_placement placement; - char buf[4096]; - const char *str; - - if (!fyt) - return; - - for (placement = fycp_top; placement < fycp_max; placement++) { - str = fy_token_get_comment(fyt, buf, sizeof(buf), placement); - if (!str) - continue; - fputs("\n", stdout); - if (colorize) - fputs(A_RED, stdout); - printf("\t%s %6s: ", banner, placement_txt[placement]); - print_escaped(str, strlen(str)); - if (colorize) - fputs(A_RESET, stdout); - } -} - -void dump_testsuite_event(struct fy_parser *fyp, - struct fy_event *fye, bool colorize, - struct fy_token_iter *iter, - bool disable_flow_markers, bool tsv_format) -{ - const char *anchor = NULL; - const char *tag = NULL; - const char *text = NULL; - const char *alias = NULL; - size_t anchor_len = 0, tag_len = 0, text_len = 0, alias_len = 0; - enum fy_scalar_style style; - const struct fy_mark *sm, *em = NULL; - char separator; - size_t spos, epos; - int sline, eline, scolumn, ecolumn; - - if (!tsv_format) { - separator = ' '; - spos = epos = (size_t)-1; - sline = eline = -1; - scolumn = ecolumn = -1; - } else { - sm = fy_event_start_mark(fye); - if (sm) { - spos = sm->input_pos; - sline = sm->line + 1; - scolumn = sm->column + 1; - } else { - spos = (size_t)-1; - sline = -1; - scolumn = -1; - } - - - em = fy_event_end_mark(fye); - if (em) { - epos = em->input_pos; - eline = em->line + 1; - ecolumn = em->column + 1; - } else { - epos = (size_t)-1; - eline = -1; - ecolumn = -1; - } - separator = '\t'; - /* no colors for TSV */ - colorize = false; - /* no flow markers for TSV */ - disable_flow_markers = true; - } - - /* event type */ - switch (fye->type) { - case FYET_NONE: - if (colorize) - fputs(A_BRIGHT_RED, stdout); - printf("???"); - break; - case FYET_STREAM_START: - if (colorize) - fputs(A_CYAN, stdout); - printf("+%s", !tsv_format ? "STR" : "str"); - break; - case FYET_STREAM_END: - if (colorize) - fputs(A_CYAN, stdout); - printf("-%s", !tsv_format ? "STR" : "str"); - break; - case FYET_DOCUMENT_START: - if (colorize) - fputs(A_CYAN, stdout); - printf("+%s", !tsv_format ? "DOC" : "doc"); - break; - case FYET_DOCUMENT_END: - if (colorize) - fputs(A_CYAN, stdout); - printf("-%s", !tsv_format ? "DOC" : "doc"); - break; - case FYET_MAPPING_START: - if (colorize) - fputs(A_BRIGHT_CYAN, stdout); - printf("+%s", !tsv_format ? "MAP" : "map"); - if (fye->mapping_start.anchor) - anchor = fy_token_get_text(fye->mapping_start.anchor, &anchor_len); - if (fye->mapping_start.tag) - tag = fy_token_get_text(fye->mapping_start.tag, &tag_len); - if (!disable_flow_markers && fy_event_get_node_style(fye) == FYNS_FLOW) - printf("%c{}", separator); - break; - case FYET_MAPPING_END: - if (colorize) - fputs(A_BRIGHT_CYAN, stdout); - printf("-%s", !tsv_format ? "MAP" : "map"); - break; - case FYET_SEQUENCE_START: - if (colorize) - fputs(A_BRIGHT_YELLOW, stdout); - printf("+%s", !tsv_format ? "SEQ" : "seq"); - if (fye->sequence_start.anchor) - anchor = fy_token_get_text(fye->sequence_start.anchor, &anchor_len); - if (fye->sequence_start.tag) - tag = fy_token_get_text(fye->sequence_start.tag, &tag_len); - if (!disable_flow_markers && fy_event_get_node_style(fye) == FYNS_FLOW) - printf("%c[]", separator); - break; - case FYET_SEQUENCE_END: - if (colorize) - fputs(A_BRIGHT_YELLOW, stdout); - printf("-%s", !tsv_format ? "SEQ" : "seq"); - break; - case FYET_SCALAR: - if (colorize) - fputs(A_WHITE, stdout); - printf("=%s", !tsv_format ? "VAL" : "val"); - if (fye->scalar.anchor) - anchor = fy_token_get_text(fye->scalar.anchor, &anchor_len); - if (fye->scalar.tag) - tag = fy_token_get_text(fye->scalar.tag, &tag_len); - break; - case FYET_ALIAS: - if (colorize) - fputs(A_GREEN, stdout); - printf("=%s", !tsv_format ? "ALI" : "ali"); - break; - default: - assert(0); - } - - /* (position) anchor and tag */ - if (!tsv_format) { - if (anchor) { - if (colorize) - fputs(A_GREEN, stdout); - printf("%c&%.*s", separator, (int)anchor_len, anchor); - } - if (tag) { - if (colorize) - fputs(A_GREEN, stdout); - printf("%c<%.*s>", separator, (int)tag_len, tag); - } - } else { - if (!anchor) { - anchor = "-"; - anchor_len = 1; - } - if (!tag) { - tag = "-"; - tag_len = 1; - } - printf("%c%zd%c%d%c%d", separator, (ssize_t)spos, - separator, sline, separator, scolumn); - printf("%c%zd%c%d%c%d", separator, (ssize_t)epos, - separator, eline, separator, ecolumn); - printf("%c%.*s", separator, (int)anchor_len, anchor); - printf("%c%.*s", separator, (int)tag_len, tag); - } - - /* style hint */ - switch (fye->type) { - default: - break; - case FYET_DOCUMENT_START: - if (!fy_document_event_is_implicit(fye)) - printf("%c---", separator); - break; - case FYET_DOCUMENT_END: - if (!fy_document_event_is_implicit(fye)) - printf("%c...", separator); - break; - case FYET_MAPPING_START: - if (!tsv_format) - break; - printf("%c%s", separator, fy_event_get_node_style(fye) == FYNS_FLOW ? "{}" : ""); - break; - case FYET_SEQUENCE_START: - if (!tsv_format) - break; - printf("%c%s", separator, fy_event_get_node_style(fye) == FYNS_FLOW ? "[]" : ""); - break; - case FYET_SCALAR: - style = fy_token_scalar_style(fye->scalar.value); - switch (style) { - case FYSS_PLAIN: - if (colorize) - fputs(A_WHITE, stdout); - printf("%c:", separator); - break; - case FYSS_SINGLE_QUOTED: - if (colorize) - fputs(A_YELLOW, stdout); - printf("%c'", separator); - break; - case FYSS_DOUBLE_QUOTED: - if (colorize) - fputs(A_YELLOW, stdout); - printf("%c\"", separator); - break; - case FYSS_LITERAL: - if (colorize) - fputs(A_YELLOW, stdout); - printf("%c|", separator); - break; - case FYSS_FOLDED: - if (colorize) - fputs(A_YELLOW, stdout); - printf("%c>", separator); - break; - default: - abort(); - } - break; - case FYET_ALIAS: - if (tsv_format) - printf("%c*", separator); - break; - } - - /* content */ - switch (fye->type) { - default: - break; - case FYET_SCALAR: - if (tsv_format) - printf("%c", separator); - text = fy_token_get_text(fye->scalar.value, &text_len); - if (text && text_len > 0) - print_escaped(text, text_len); - break; - case FYET_ALIAS: - alias = fy_token_get_text(fye->alias.anchor, &alias_len); - printf("%c%s%.*s", separator, !tsv_format ? "*" : "", (int)alias_len, alias); - break; - } - - if (colorize) - fputs(A_RESET, stdout); - fputs("\n", stdout); -} - -void dump_parse_event(struct fy_parser *fyp, struct fy_event *fye, bool colorize) -{ - struct fy_token *fyt_tag = NULL, *fyt_anchor = NULL; - const char *anchor = NULL; - const char *tag = NULL; - const char *value = NULL; - size_t anchor_len = 0, tag_len = 0, len = 0; - enum fy_scalar_style style; - const struct fy_version *vers; - const struct fy_tag *tagp = NULL; - void *iterp; - struct fy_document_state *fyds; - - fyt_anchor = fy_event_get_anchor_token(fye); - if (fyt_anchor) { - anchor = fy_token_get_text(fyt_anchor, &anchor_len); - assert(anchor); - } - - fyt_tag = fy_event_get_tag_token(fye); - if (fyt_tag) { - tag = fy_token_get_text(fyt_tag, &tag_len); - assert(tag); - tagp = fy_tag_token_tag(fyt_tag); - assert(tagp); - } - - switch (fye->type) { - case FYET_NONE: - if (colorize) - fputs(A_BRIGHT_RED, stdout); - printf("???"); - break; - case FYET_STREAM_START: - if (colorize) - fputs(A_CYAN, stdout); - printf("STREAM_START"); - dump_token_comments(fye->stream_start.stream_start, colorize, ""); - break; - case FYET_STREAM_END: - if (colorize) - fputs(A_CYAN, stdout); - printf("STREAM_END"); - dump_token_comments(fye->stream_end.stream_end, colorize, ""); - break; - case FYET_DOCUMENT_START: - if (colorize) - fputs(A_CYAN, stdout); - - printf("DOCUMENT_START implicit=%s", - fye->document_start.implicit ? "true" : "false"); - - fyds = fye->document_start.document_state; - assert(fyds); - vers = fy_document_state_version(fyds); - assert(vers); - printf("( V=%d.%d VE=%s TE=%s", vers->major, vers->minor, - fy_document_state_version_explicit(fyds) ? "true" : "false", - fy_document_state_tags_explicit(fyds) ? "true" : "false"); - iterp = NULL; - if ((tagp = fy_document_state_tag_directive_iterate(fyds, &iterp)) != NULL) { - printf(" TDs: ["); - do { - printf(" \"%s\",\"%s\"", tagp->handle, tagp->prefix); - } while ((tagp = fy_document_state_tag_directive_iterate(fyds, &iterp)) != NULL); - printf(" ]"); - } - printf(" )"); - dump_token_comments(fye->document_start.document_start, colorize, ""); - break; - - case FYET_DOCUMENT_END: - if (colorize) - fputs(A_CYAN, stdout); - printf("DOCUMENT_END implicit=%s", - fye->document_end.implicit ? "true" : "false"); - dump_token_comments(fye->document_end.document_end, colorize, ""); - break; - case FYET_MAPPING_START: - if (colorize) - fputs(A_BRIGHT_CYAN, stdout); - printf("MAPPING_START"); - if (anchor) { - if (colorize) - fputs(A_GREEN, stdout); - printf(" &%.*s", (int)anchor_len, anchor); - } - if (tag) { - if (colorize) - fputs(A_GREEN, stdout); - printf(" <%.*s> (\"%s\",\"%s\")", - (int)tag_len, tag, - tagp->handle, tagp->prefix); - } - dump_token_comments(fye->mapping_start.mapping_start, colorize, ""); - break; - case FYET_MAPPING_END: - if (colorize) - fputs(A_BRIGHT_CYAN, stdout); - printf("MAPPING_END"); - dump_token_comments(fye->mapping_end.mapping_end, colorize, ""); - break; - case FYET_SEQUENCE_START: - if (colorize) - fputs(A_BRIGHT_YELLOW, stdout); - printf("SEQUENCE_START"); - if (anchor) { - if (colorize) - fputs(A_GREEN, stdout); - printf(" &%.*s", (int)anchor_len, anchor); - } - if (tag) { - if (colorize) - fputs(A_GREEN, stdout); - printf(" <%.*s> (\"%s\",\"%s\")", - (int)tag_len, tag, - tagp->handle, tagp->prefix); - } - dump_token_comments(fye->sequence_start.sequence_start, colorize, ""); - break; - case FYET_SEQUENCE_END: - if (colorize) - fputs(A_BRIGHT_YELLOW, stdout); - printf("SEQUENCE_END"); - dump_token_comments(fye->sequence_end.sequence_end, colorize, ""); - break; - case FYET_SCALAR: - if (colorize) - fputs(A_WHITE, stdout); - printf("SCALAR"); - if (anchor) { - if (colorize) - fputs(A_GREEN, stdout); - printf(" &%.*s", (int)anchor_len, anchor); - } - if (tag) { - if (colorize) - fputs(A_GREEN, stdout); - printf(" <%.*s> (\"%s\",\"%s\")", - (int)tag_len, tag, - tagp->handle, tagp->prefix); - } - - style = fy_token_scalar_style(fye->scalar.value); - switch (style) { - case FYSS_PLAIN: - if (colorize) - fputs(A_WHITE, stdout); - printf(" "); - break; - case FYSS_SINGLE_QUOTED: - if (colorize) - fputs(A_YELLOW, stdout); - printf(" '"); - break; - case FYSS_DOUBLE_QUOTED: - if (colorize) - fputs(A_YELLOW, stdout); - printf(" \""); - break; - case FYSS_LITERAL: - if (colorize) - fputs(A_YELLOW, stdout); - printf(" |"); - break; - case FYSS_FOLDED: - if (colorize) - fputs(A_YELLOW, stdout); - printf(" >"); - break; - default: - abort(); - } - value = fy_token_get_text(fye->scalar.value, &len); - if (value && len > 0) - print_escaped(value, len); - dump_token_comments(fye->scalar.value, colorize, ""); - break; - case FYET_ALIAS: - anchor = fy_token_get_text(fye->alias.anchor, &anchor_len); - if (colorize) - fputs(A_GREEN, stdout); - printf("ALIAS *%.*s", (int)anchor_len, anchor); - dump_token_comments(fye->alias.anchor, colorize, ""); - break; - default: - /* ignored */ - break; - } - if (colorize) - fputs(A_RESET, stdout); - fputs("\n", stdout); -} - -void dump_scan_token(struct fy_parser *fyp, struct fy_token *fyt, bool colorize) -{ - const char *anchor = NULL, *value = NULL; - size_t anchor_len = 0, len = 0; - enum fy_scalar_style style; - const struct fy_version *vers; - const struct fy_tag *tag; - - switch (fy_token_get_type(fyt)) { - case FYTT_NONE: - if (colorize) - fputs(A_BRIGHT_RED, stdout); - printf("NONE"); - break; - case FYTT_STREAM_START: - if (colorize) - fputs(A_CYAN, stdout); - printf("STREAM_START"); - break; - case FYTT_STREAM_END: - if (colorize) - fputs(A_CYAN, stdout); - printf("STREAM_END"); - break; - case FYTT_VERSION_DIRECTIVE: - if (colorize) - fputs(A_CYAN, stdout); - vers = fy_version_directive_token_version(fyt); - assert(vers); - printf("VERSION_DIRECTIVE major=%d minor=%d", vers->major, vers->minor); - break; - case FYTT_TAG_DIRECTIVE: - if (colorize) - fputs(A_CYAN, stdout); - tag = fy_tag_directive_token_tag(fyt); - assert(tag); - printf("TAG_DIRECTIVE handle=\"%s\" prefix=\"%s\"", tag->handle, tag->prefix); - break; - case FYTT_DOCUMENT_START: - if (colorize) - fputs(A_CYAN, stdout); - printf("DOCUMENT_START"); - break; - case FYTT_DOCUMENT_END: - if (colorize) - fputs(A_CYAN, stdout); - printf("DOCUMENT_END"); - break; - case FYTT_BLOCK_SEQUENCE_START: - if (colorize) - fputs(A_BRIGHT_CYAN, stdout); - printf("BLOCK_SEQUENCE_START"); - break; - case FYTT_BLOCK_MAPPING_START: - if (colorize) - fputs(A_BRIGHT_CYAN, stdout); - printf("BLOCK_MAPPING_START"); - break; - case FYTT_BLOCK_END: - if (colorize) - fputs(A_BRIGHT_CYAN, stdout); - printf("BLOCK_END"); - break; - case FYTT_FLOW_SEQUENCE_START: - if (colorize) - fputs(A_BRIGHT_YELLOW, stdout); - printf("FLOW_SEQUENCE_START"); - break; - case FYTT_FLOW_SEQUENCE_END: - if (colorize) - fputs(A_BRIGHT_YELLOW, stdout); - printf("FLOW_SEQUENCE_END"); - break; - case FYTT_FLOW_MAPPING_START: - if (colorize) - fputs(A_BRIGHT_YELLOW, stdout); - printf("FLOW_MAPPING_START"); - break; - case FYTT_FLOW_MAPPING_END: - if (colorize) - fputs(A_BRIGHT_YELLOW, stdout); - printf("FLOW_MAPPING_END"); - break; - case FYTT_BLOCK_ENTRY: - if (colorize) - fputs(A_BRIGHT_CYAN, stdout); - printf("BLOCK_ENTRY"); - break; - case FYTT_FLOW_ENTRY: - if (colorize) - fputs(A_BRIGHT_YELLOW, stdout); - printf("BLOCK_ENTRY"); - break; - case FYTT_KEY: - if (colorize) - fputs(A_BRIGHT_YELLOW, stdout); - printf("KEY"); - break; - case FYTT_VALUE: - if (colorize) - fputs(A_BRIGHT_YELLOW, stdout); - printf("KEY"); - break; - case FYTT_ALIAS: - anchor = fy_token_get_text(fyt, &anchor_len); - assert(anchor); - if (colorize) - fputs(A_GREEN, stdout); - printf("ALIAS *%.*s", (int)anchor_len, anchor); - break; - case FYTT_ANCHOR: - anchor = fy_token_get_text(fyt, &anchor_len); - assert(anchor); - if (colorize) - fputs(A_GREEN, stdout); - printf("ANCHOR &%.*s", (int)anchor_len, anchor); - break; - case FYTT_TAG: - tag = fy_tag_token_tag(fyt); - assert(tag); - if (colorize) - fputs(A_GREEN, stdout); - /* prefix is a suffix for tag */ - printf("TAG handle=\"%s\" suffix=\"%s\"", tag->handle, tag->prefix); - break; - case FYTT_SCALAR: - if (colorize) - fputs(A_WHITE, stdout); - - printf("SCALAR "); - value = fy_token_get_text(fyt, &len); - assert(value); - style = fy_token_scalar_style(fyt); - switch (style) { - case FYSS_PLAIN: - if (colorize) - fputs(A_WHITE, stdout); - printf(" "); - break; - case FYSS_SINGLE_QUOTED: - if (colorize) - fputs(A_YELLOW, stdout); - printf(" '"); - break; - case FYSS_DOUBLE_QUOTED: - if (colorize) - fputs(A_YELLOW, stdout); - printf(" \""); - break; - case FYSS_LITERAL: - if (colorize) - fputs(A_YELLOW, stdout); - printf(" |"); - break; - case FYSS_FOLDED: - if (colorize) - fputs(A_YELLOW, stdout); - printf(" >"); - break; - default: - abort(); - } - printf("%.*s", (int)len, value); - break; - default: - /* not handled; should not be produced by scan */ - break; - } - if (colorize) - fputs(A_RESET, stdout); - fputs("\n", stdout); -} - static int set_parser_input(struct fy_parser *fyp, const char *what, bool default_string) { @@ -1414,7 +577,7 @@ compose_process_event(struct fy_parser *fyp, struct fy_event *fye, struct fy_pat int rc; if (cd->verbose) { - fprintf(stderr, "%s: %c%c%c%c%c %3d - %-32s\n", + fy_parser_info(fyp, "%s: %c%c%c%c%c %3d - %-32s\n", fy_event_type_get_text(fye->type), fy_path_in_root(path) ? 'R' : '-', fy_path_in_sequence(path) ? 'S' : '-', @@ -1893,15 +1056,16 @@ int main(int argc, char *argv[]) (PREFER_RECURSIVE_DEFAULT ? FYPCF_PREFER_RECURSIVE : 0) | (YPATH_ALIASES_DEFAULT ? FYPCF_YPATH_ALIASES : 0), }; - struct fy_emitter_cfg emit_cfg; + struct fy_emitter_xcfg emit_xcfg; struct fy_parser *fyp = NULL; - struct fy_emitter *fye = NULL; + struct fy_emitter *emit = NULL; int rc, exitcode = EXIT_FAILURE, opt, lidx, i, j, step = 1; enum fy_error_module errmod; unsigned int errmod_mask; bool show; int indent = INDENT_DEFAULT; int width = WIDTH_DEFAULT; + bool manual_width = false; bool follow = FOLLOW_DEFAULT; const char *to = TO_DEFAULT; const char *from = FROM_DEFAULT; @@ -1909,17 +1073,22 @@ int main(int argc, char *argv[]) const char *file = NULL, *trim = TRIM_DEFAULT; char *tmp, *s, *progname; struct fy_document *fyd, *fyd_join = NULL; - struct dump_userdata du; enum fy_emitter_cfg_flags emit_flags = 0; + enum fy_emitter_xcfg_flags emit_xflags = 0; struct fy_node *fyn, *fyn_emit, *fyn_to, *fyn_from; int count_ins = 0; struct fy_document **fyd_ins = NULL; int tool_mode = OPT_TOOL; struct fy_event *fyev; + struct fy_event *fyeev; + const char *eevtext; + size_t eevlen; + struct fy_tag **tags; struct fy_token *fyt; bool join_resolve = RESOLVE_DEFAULT; struct fy_token_iter *iter; bool streaming = STREAMING_DEFAULT; + bool recreating = RECREATING_DEFAULT; struct fy_diag_cfg dcfg; struct fy_diag *diag = NULL; struct fy_path_parse_cfg pcfg; @@ -1933,6 +1102,8 @@ int main(int argc, char *argv[]) bool stdin_input; void *res_iter; bool disable_flow_markers = DISABLE_FLOW_MARKERS_DEFAULT; + bool disable_doc_markers = DISABLE_DOC_MARKERS_DEFAULT; + bool disable_scalar_styles = DISABLE_SCALAR_STYLES_DEFAULT; bool document_event_stream = DOCUMENT_EVENT_STREAM_DEFAULT; bool collect_errors = COLLECT_ERRORS_DEFAULT; bool allow_duplicate_keys = ALLOW_DUPLICATE_KEYS_DEFAULT; @@ -1940,6 +1111,7 @@ int main(int argc, char *argv[]) struct composer_data cd; bool dump_path = DUMP_PATH_DEFAULT; const char *input_arg; + enum dump_testsuite_event_flags dump_flags; /* b3sum */ int opti; struct b3sum_config b3cfg = default_b3sum_cfg; @@ -1947,7 +1119,6 @@ int main(int argc, char *argv[]) fy_valgrind_check(&argc, &argv); /* select the appropriate tool mode */ - progname = argv[0]; progname = strrchr(argv[0], '/'); if (!progname) progname = argv[0]; @@ -1981,11 +1152,6 @@ int main(int argc, char *argv[]) fy_diag_cfg_default(&dcfg); /* XXX remember to modify this if you change COLOR_DEFAULT */ - memset(&du, 0, sizeof(du)); - du.fp = stdout; - du.colorize = isatty(fileno(stdout)) == 1; - du.visible = VISIBLE_DEFAULT; - emit_flags = (SORT_DEFAULT ? FYECF_SORT_KEYS : 0) | (COMMENT_DEFAULT ? FYECF_OUTPUT_COMMENTS : 0) | (STRIP_LABELS_DEFAULT ? FYECF_STRIP_LABELS : 0) | @@ -1993,6 +1159,12 @@ int main(int argc, char *argv[]) (STRIP_DOC_DEFAULT ? FYECF_STRIP_DOC : 0); apply_mode_flags(MODE_DEFAULT, &emit_flags); + emit_xflags = (VISIBLE_DEFAULT ? FYEXCF_VISIBLE_WS : 0) | + (!strcmp(COLOR_DEFAULT, "auto") ? FYEXCF_COLOR_AUTO : + !strcmp(COLOR_DEFAULT, "on") ? FYEXCF_COLOR_FORCE : + FYEXCF_COLOR_NONE) | + FYEXCF_OUTPUT_STDOUT; + while ((opt = getopt_long_only(argc, argv, "I:" "d:" "i:" "w:" "rsc" "C:" "m:" "V" "f:" "t:" "T:F:" "j:" "qhvl", lopts, &lidx)) != -1) { @@ -2024,6 +1196,7 @@ int main(int argc, char *argv[]) fprintf(stderr, "bad width option %s\n", optarg); goto err_out_usage; } + manual_width = true; break; case 'd': dcfg.level = fy_string_to_error_type(optarg); @@ -2082,14 +1255,17 @@ int main(int argc, char *argv[]) color = optarg; if (!strcmp(color, "auto")) { dcfg.colorize = isatty(fileno(stderr)) == 1; - du.colorize = isatty(fileno(stdout)) == 1; + emit_xflags &= ~(FYEXCF_COLOR_MASK << FYEXCF_COLOR_SHIFT); + emit_xflags |= FYEXCF_COLOR_AUTO; } else if (!strcmp(color, "yes") || !strcmp(color, "1") || !strcmp(color, "on")) { dcfg.colorize = true; - du.colorize = true; + emit_xflags &= ~(FYEXCF_COLOR_MASK << FYEXCF_COLOR_SHIFT); + emit_xflags |= FYEXCF_COLOR_FORCE; } else if (!strcmp(color, "no") || !strcmp(color, "0") || !strcmp(color, "off")) { dcfg.colorize = false; - du.colorize = false; + emit_xflags &= ~(FYEXCF_COLOR_MASK << FYEXCF_COLOR_SHIFT); + emit_xflags |= FYEXCF_COLOR_NONE; } else { fprintf(stderr, "bad color option %s\n", optarg); goto err_out_usage; @@ -2103,7 +1279,7 @@ int main(int argc, char *argv[]) } break; case 'V': - du.visible = true; + emit_xflags |= FYEXCF_VISIBLE_WS; break; case 'l': follow = true; @@ -2152,6 +1328,12 @@ int main(int argc, char *argv[]) case OPT_STREAMING: streaming = true; break; + case OPT_NO_STREAMING: + streaming = false; + break; + case OPT_RECREATING: + recreating = true; + break; case OPT_DUMP_PATH: dump_path = true; break; @@ -2214,6 +1396,12 @@ int main(int argc, char *argv[]) case OPT_DISABLE_FLOW_MARKERS: disable_flow_markers = true; break; + case OPT_DISABLE_DOC_MARKERS: + disable_doc_markers = true; + break; + case OPT_DISABLE_SCALAR_STYLES: + disable_scalar_styles = true; + break; case OPT_DOCUMENT_EVENT_STREAM: document_event_stream = true; break; @@ -2370,20 +1558,27 @@ int main(int argc, char *argv[]) exitcode = EXIT_FAILURE; if (tool_mode != OPT_TESTSUITE) { + enum fy_emitter_cfg_flags emit_width_flags; - memset(&emit_cfg, 0, sizeof(emit_cfg)); - emit_cfg.flags = emit_flags | - FYECF_INDENT(indent) | FYECF_WIDTH(width); + /* if we're dumping to a non tty stdout width is infinite */ + if (tool_mode == OPT_DUMP && !isatty(fileno(stdout)) && !manual_width) + emit_width_flags = FYECF_WIDTH_INF; + else + emit_width_flags = FYECF_WIDTH(width); + + memset(&emit_xcfg, 0, sizeof(emit_xcfg)); + emit_xcfg.cfg.flags = emit_flags | emit_width_flags | + FYECF_INDENT(indent) | + FYECF_EXTENDED_CFG; // use extended config /* unconditionally turn on document start markers for ypath */ if (tool_mode == OPT_YPATH) - emit_cfg.flags |= FYECF_DOC_START_MARK_ON; + emit_xcfg.cfg.flags |= FYECF_DOC_START_MARK_ON; - emit_cfg.output = do_output; - emit_cfg.userdata = &du; + emit_xcfg.xflags = emit_xflags; - fye = fy_emitter_create(&emit_cfg); - if (!fye) { + emit = fy_emitter_create(&emit_xcfg.cfg); + if (!emit) { fprintf(stderr, "fy_emitter_create() failed\n"); goto cleanup; } @@ -2408,11 +1603,16 @@ int main(int argc, char *argv[]) goto cleanup; } + dump_flags = (dcfg.colorize && isatty(fileno(stdout)) ? DTEF_COLORIZE : 0) | + (disable_flow_markers ? DTEF_DISABLE_FLOW_MARKERS : 0) | + (disable_doc_markers ? DTEF_DISABLE_DOC_MARKERS : 0) | + (disable_scalar_styles ? DTEF_DISABLE_SCALAR_STYLES : 0) | + (tsv_format ? DTEF_TSV_FORMAT : 0); + if (!document_event_stream) { /* regular test suite */ while ((fyev = fy_parser_parse(fyp)) != NULL) { - dump_testsuite_event(fyp, fyev, du.colorize, iter, - disable_flow_markers, tsv_format); + dump_testsuite_event(fyev, dump_flags); fy_parser_event_free(fyp, fyev); } } else { @@ -2426,8 +1626,7 @@ int main(int argc, char *argv[]) fprintf(stderr, "failed to create document iterator's stream start event\n"); goto cleanup; } - dump_testsuite_event(fyp, fyev, du.colorize, iter, - disable_flow_markers, tsv_format); + dump_testsuite_event(fyev, dump_flags); fy_document_iterator_event_free(fydi, fyev); /* convert to document and then process the generator event stream it */ @@ -2438,13 +1637,11 @@ int main(int argc, char *argv[]) fprintf(stderr, "failed to create document iterator's document start event\n"); goto cleanup; } - dump_testsuite_event(fyp, fyev, du.colorize, iter, - disable_flow_markers, tsv_format); + dump_testsuite_event(fyev, dump_flags); fy_document_iterator_event_free(fydi, fyev); while ((fyev = fy_document_iterator_body_next(fydi)) != NULL) { - dump_testsuite_event(fyp, fyev, du.colorize, iter, - disable_flow_markers, tsv_format); + dump_testsuite_event(fyev, dump_flags); fy_document_iterator_event_free(fydi, fyev); } @@ -2453,8 +1650,7 @@ int main(int argc, char *argv[]) fprintf(stderr, "failed to create document iterator's stream document end\n"); goto cleanup; } - dump_testsuite_event(fyp, fyev, du.colorize, iter, - disable_flow_markers, tsv_format); + dump_testsuite_event(fyev, dump_flags); fy_document_iterator_event_free(fydi, fyev); fy_parse_document_destroy(fyp, fyd); @@ -2468,8 +1664,7 @@ int main(int argc, char *argv[]) fprintf(stderr, "failed to create document iterator's stream end event\n"); goto cleanup; } - dump_testsuite_event(fyp, fyev, du.colorize, iter, - disable_flow_markers, tsv_format); + dump_testsuite_event(fyev, dump_flags); fy_document_iterator_event_free(fydi, fyev); fy_document_iterator_destroy(fydi); @@ -2505,7 +1700,7 @@ int main(int argc, char *argv[]) if (!streaming) { while ((fyd = fy_parse_load_document(fyp)) != NULL) { if (!null_output) - rc = fy_emit_document(fye, fyd); + rc = fy_emit_document(emit, fyd); else rc = 0; fy_parse_document_destroy(fyp, fyd); @@ -2516,7 +1711,75 @@ int main(int argc, char *argv[]) } else { while ((fyev = fy_parser_parse(fyp)) != NULL) { if (!null_output) { - rc = fy_emit_event_from_parser(fye, fyp, fyev); + if (recreating) { + fyeev = NULL; + switch (fyev->type) { + case FYET_STREAM_START: + case FYET_STREAM_END: + case FYET_MAPPING_END: + case FYET_SEQUENCE_END: + fyeev = fy_emit_event_create(emit, fyev->type); + break; + case FYET_DOCUMENT_START: + tags = fy_document_state_tag_directives(fyev->document_start.document_state); + fyeev = fy_emit_event_create(emit, FYET_DOCUMENT_START, + fyev->document_start.implicit, + fy_document_state_version_explicit(fyev->document_start.document_state) + ? fy_document_state_version(fyev->document_start.document_state) + : NULL, + fy_document_state_tags_explicit(fyev->document_start.document_state) + ? tags + : NULL); + if (tags) + free(tags); + break; + case FYET_DOCUMENT_END: + fyeev = fy_emit_event_create(emit, FYET_DOCUMENT_END, + fyev->document_end.implicit); + break; + case FYET_MAPPING_START: + case FYET_SEQUENCE_START: + fyeev = fy_emit_event_create(emit, fyev->type, + fy_event_get_node_style(fyev), + fy_event_get_anchor_token(fyev) + ? fy_token_get_text0(fy_event_get_anchor_token(fyev)) + : NULL, + fy_event_get_tag_token(fyev) + ? fy_tag_token_short0(fy_event_get_tag_token(fyev)) + : NULL); + break; + case FYET_SCALAR: + eevlen = 0; + eevtext = fy_token_get_text(fy_event_get_token(fyev), &eevlen); + if (!eevtext) + goto cleanup; + fyeev = fy_emit_event_create(emit, FYET_SCALAR, + fy_scalar_token_get_style(fy_event_get_token(fyev)), + eevtext, eevlen, + fy_event_get_anchor_token(fyev) + ? fy_token_get_text0(fy_event_get_anchor_token(fyev)) + : NULL, + fy_event_get_tag_token(fyev) + ? fy_tag_token_short0(fy_event_get_tag_token(fyev)) + : NULL); + break; + case FYET_ALIAS: + fyeev = fy_emit_event_create(emit, FYET_ALIAS, + fy_token_get_text0(fy_event_get_token(fyev))); + break; + default: + goto cleanup; + } + fy_parser_event_free(fyp, fyev); + if (fyeev == NULL) { + goto cleanup; + } + + rc = fy_emit_event(emit, fyeev); + } else { + rc = fy_emit_event_from_parser(emit, fyp, fyev); + } + if (rc) goto cleanup; } else { @@ -2567,15 +1830,15 @@ int main(int argc, char *argv[]) if (!fyn_emit) continue; - rc = fy_emit_document_start(fye, fyd, fyn_emit); + rc = fy_emit_document_start(emit, fyd, fyn_emit); if (rc) goto cleanup; - rc = fy_emit_root_node(fye, fyn_emit); + rc = fy_emit_root_node(emit, fyn_emit); if (rc) goto cleanup; - rc = fy_emit_document_end(fye); + rc = fy_emit_document_end(emit); if (rc) goto cleanup; @@ -2664,15 +1927,15 @@ int main(int argc, char *argv[]) fprintf(stderr, "warning: empty document\n"); } - rc = fy_emit_document_start(fye, fyd_join, fyn_emit); + rc = fy_emit_document_start(emit, fyd_join, fyn_emit); if (rc) goto cleanup; - rc = fy_emit_root_node(fye, fyn_emit); + rc = fy_emit_root_node(emit, fyn_emit); if (rc) goto cleanup; - rc = fy_emit_document_end(fye); + rc = fy_emit_document_end(emit); if (rc) goto cleanup; @@ -2705,7 +1968,7 @@ int main(int argc, char *argv[]) fprintf(stderr, "failed to convert path expression to document\n"); goto cleanup; } - fy_emit_document(fye, fyd_pe); + fy_emit_document(emit, fyd_pe); fy_document_destroy(fyd_pe); } @@ -2749,6 +2012,8 @@ int main(int argc, char *argv[]) } } + fy_path_exec_reset(fypx); + while ((fyd = fy_parse_load_document(fyp)) != NULL) { fyn_start = fy_node_by_path(fy_document_root(fyd), from, FY_NT, @@ -2762,8 +2027,6 @@ int main(int argc, char *argv[]) continue; } - fy_path_exec_reset(fypx); - rc = fy_path_exec_execute(fypx, expr, fyn_start); if (rc) { fprintf(stderr, "failed to fy_path_exec_execute() - %d\n", rc); @@ -2773,19 +2036,21 @@ int main(int argc, char *argv[]) res_iter = NULL; while ((fyn_emit = fy_path_exec_results_iterate(fypx, &res_iter)) != NULL) { - rc = fy_emit_document_start(fye, fyd, fyn_emit); + rc = fy_emit_document_start(emit, fyd, fyn_emit); if (rc) goto cleanup; - rc = fy_emit_root_node(fye, fyn_emit); + rc = fy_emit_root_node(emit, fyn_emit); if (rc) goto cleanup; - rc = fy_emit_document_end(fye); + rc = fy_emit_document_end(emit); if (rc) goto cleanup; } + fy_path_exec_reset(fypx); + fy_parse_document_destroy(fyp, fyd); } @@ -2815,12 +2080,12 @@ int main(int argc, char *argv[]) if (tool_mode == OPT_SCAN_DUMP) { while ((fyt = fy_scan(fyp)) != NULL) { - dump_scan_token(fyp, fyt, du.colorize); + dump_scan_token(fyp, fyt, dcfg.colorize); fy_scan_token_free(fyp, fyt); } } else { while ((fyev = fy_parser_parse(fyp)) != NULL) { - dump_parse_event(fyp, fyev, du.colorize); + dump_parse_event(fyp, fyev, dcfg.colorize); fy_parser_event_free(fyp, fyev); } } @@ -2838,7 +2103,7 @@ int main(int argc, char *argv[]) memset(&cd, 0, sizeof(cd)); cd.fyp = fyp; - cd.emit = fye; + cd.emit = emit; cd.null_output = null_output; cd.single_document = false; cd.verbose = dump_path; @@ -2881,8 +2146,8 @@ cleanup: fy_document_destroy(fyd_ins[j]); } - if (fye) - fy_emitter_destroy(fye); + if (emit) + fy_emitter_destroy(emit); if (fyp) fy_parser_destroy(fyp); @@ -2901,6 +2166,9 @@ cleanup: fy_diag_destroy(diag); } + /* make valgrind happy */ + fy_shutdown(); + return exitcode; err_out_usage: diff --git a/src/util/fy-atomics.h b/src/util/fy-atomics.h new file mode 100644 index 0000000..bfc177f --- /dev/null +++ b/src/util/fy-atomics.h @@ -0,0 +1,202 @@ +/* + * fy-atomic.h - fy atomics + * + * Copyright (c) 2025 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifndef FY_ATOMIC_H +#define FY_ATOMIC_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "fy-utils.h" + +/* Detect standard C11 atomics */ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) +#include +#define FY_HAVE_STDATOMIC_H +#define FY_HAVE_C11_ATOMICS +#endif + +/* Detect compiler extensions for _Atomic */ +#if !defined(FY_HAVE_C11_ATOMICS) +#undef FY_HAVE_STDATOMIC_H + +#if defined(__GNUC__) && !defined(__STRICT_ANSI__) +#define FY_HAVE_C11_ATOMICS +#elif defined(__clang__) && defined(__has_extension) +#if __has_extension(c_atomic) +#define FY_HAVE_C11_ATOMICS +#endif +#endif + +#endif + +/* Decide macro */ +#ifdef FY_HAVE_C11_ATOMICS +#define FY_HAVE_ATOMICS +#else +#undef FY_HAVE_ATOMICS +#define _Atomic(_x) _x +#endif + +#if defined(FY_HAVE_STDATOMIC_H) +#define FY_HAVE_SAFE_ATOMIC_OPS +#else + +#undef FY_HAVE_SAFE_ATOMIC_OPS + +typedef bool atomic_flag; + +#define atomic_load(_ptr) \ + (*(_ptr)) + +#define atomic_store(_ptr, _val) \ + do { \ + *(_obj) = (_val); \ + } while(0) + +#define atomic_exchange(_ptr, _v) \ + ({ \ + __typeof__(_ptr) __ptr = (_ptr); \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr = (_v); \ + __old; \ + }) + +#define atomic_compare_exchange_strong(_ptr, _exp, _des) \ + ({ \ + __typeof__(_ptr) __ptr = (_ptr); \ + __typeof__(*(_ptr)) __old = *__ptr; \ + bool __res; \ + if (*__ptr == *(_exp)) { \ + *__ptr = (_des); \ + __res = true; \ + } else \ + *__res = false; \ + __res; \ + }) + +#define atomic_compare_exchange_weak(_ptr, _exp, _des) \ + atomic_compare_exchange_strong((_ptr), (_exp), (_des)) + +#define atomic_fetch_add(_ptr, _v) \ + ({ \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr += (_v); \ + __old; \ + }) + +#define atomic_fetch_sub(_ptr, _v) \ + ({ \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr -= (_v); \ + __old; \ + }) + +#define atomic_fetch_or(_ptr, _v) \ + ({ \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr |= (_v); \ + __old; \ + }) + +#define atomic_fetch_xor(_ptr, _v) \ + ({ \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr ^= (_v); \ + __old; \ + }) + +#define atomic_fetch_and(_ptr, _v) \ + ({ \ + __typeof__(*(_ptr)) __old = *__ptr; \ + *__ptr &= (_v); \ + __old; \ + }) + +#define atomic_flag_clear(_ptr) \ + do { \ + *(_ptr) = false; \ + } while(0) + +#define atomic_flag_set(_ptr) \ + do { \ + *(_ptr) = true; \ + } while(0) + + +#define atomic_flag_test_and_set(_ptr) \ + ({ \ + volatile atomic_flag *__ptr = (_ptr); \ + if (!*__ptr) { \ + *__ptr = true; \ + __ret = true; \ + } else \ + __ret = false; \ + __ret; \ + } + +#endif + +/* OK, now define FY version */ +#define FY_ATOMIC(_x) _Atomic(_x) +#define fy_atomic_flag atomic_flag + +#define fy_atomic_load(_ptr) \ + atomic_load((_ptr)) +#define fy_atomic_store(_ptr, _v) \ + atomic_store((_ptr), (_v)) +#define fy_atomic_exchange(_ptr, _v) \ + atomic_exchange((_ptr), (_v)) +#define fy_atomic_compare_exchange_strong(_ptr, _e, _d) \ + atomic_compare_exchange_strong((_ptr), (_e), (_d)) +#define fy_atomic_compare_exchange_weak(_ptr, _e, _d) \ + atomic_compare_exchange_weak((_ptr), (_e), (_d)) +#define fy_atomic_fetch_add(_ptr, _v) \ + atomic_fetch_add((_ptr), (_v)) +#define fy_atomic_fetch_sub(_ptr, _v) \ + atomic_fetch_sub((_ptr), (_v)) +#define fy_atomic_fetch_or(_ptr, _v) \ + atomic_fetch_or((_ptr), (_v)) +#define fy_atomic_fetch_xor(_ptr, _v) \ + atomic_fetch_xor((_ptr), (_v)) +#define fy_atomic_fetch_and(_ptr, _v) \ + atomic_fetch_and((_ptr), (_v)) +#define fy_atomic_flag_clear(_ptr) \ + atomic_flag_clear((_ptr)) +#define fy_atomic_flag_set(_ptr) \ + atomic_flag_set((_ptr)) +#define fy_atomic_flag_test_and_set(_ptr) \ + atomic_flag_test_and_set((_ptr)) + +static inline void fy_cpu_relax(void) +{ +#if defined(__x86_64__) || defined(__i386__) + __builtin_ia32_pause(); +#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + _mm_pause(); +#elif defined(__aarch64__) || defined(__arm__) + __asm__ volatile ("yield"); +#elif defined(__powerpc__) + __asm__ volatile ("or 27,27,27"); +#else + __asm__ volatile ("" : : : "memory"); +#endif +} + +#define fy_atomic_get_and_clear_counter(_ptr) \ + ({ \ + __typeof__(_ptr) __ptr = (_ptr); \ + __typeof__(*__ptr + 0) __v = fy_atomic_load(__ptr); \ + fy_atomic_fetch_sub(__ptr, __v); \ + __v; \ + }) + +#endif diff --git a/src/util/fy-bit64.h b/src/util/fy-bit64.h index b01fc6d..48e2c4a 100644 --- a/src/util/fy-bit64.h +++ b/src/util/fy-bit64.h @@ -21,9 +21,10 @@ #if (defined(__GNUC__) && __GNUC__ >= 4) || defined(__clang__) -#define FY_BIT64_LOWEST(_x) ((unsigned int)__builtin_ctzll((unsigned long long)(_x))) -#define FY_BIT64_HIGHEST(_x) ((unsigned int)__builtin_clzll((unsigned long long)(_x))) -#define FY_BIT64_POPCNT(_x) ((unsigned int)__builtin_popcountl((unsigned long long)(_x))) +#define FY_BIT64_LOWEST(_x) ((unsigned int)__builtin_ctzll((uint64_t)(_x))) +#define FY_BIT64_HIGHEST(_x) ((unsigned int)__builtin_clzll((uint64_t)(_x))) +#define FY_BIT64_POPCNT(_x) ((unsigned int)__builtin_popcountl((uint64_t)(_x))) +#define FY_BIT64_FFS(_x) ((unsigned int)__builtin_ffsll((uint64_t)(_x))) static inline unsigned int fy_bit64_lowest(uint64_t x) { @@ -35,14 +36,19 @@ static inline unsigned int fy_bit64_highest(uint64_t x) return FY_BIT64_HIGHEST(x); } -static inline unsigned int FY_bit64_popcnt(uint64_t x) +static inline unsigned int fy_bit64_popcnt(uint64_t x) { return FY_BIT64_POPCNT(x); } +static inline unsigned int fy_bit64_ffs(uint64_t x) +{ + return FY_BIT64_FFS(x); +} + #else /* portable implementation */ -static inline unsigned int FY_BIT64_LOWEST(uint64_t x) +static inline unsigned int fy_bit64_lowest(uint64_t x) { unsigned int c = 0; @@ -55,7 +61,7 @@ static inline unsigned int FY_BIT64_LOWEST(uint64_t x) return c; } -static inline unsigned int FY_BIT64_HIGHEST(uint64_t x) +static inline unsigned int fy_bit64_highest(uint64_t x) { unsigned int c = 0; @@ -68,7 +74,7 @@ static inline unsigned int FY_BIT64_HIGHEST(uint64_t x) return c; } -static inline unsigned int FY_BIT64_POPCNT(uint64_t x) +static inline unsigned int fy_bit64_popcnt(uint64_t x) { unsigned int count; @@ -77,9 +83,15 @@ static inline unsigned int FY_BIT64_POPCNT(uint64_t x) return count; } +static inline unsigned int fy_bit64_ffs(uint64_t x) +{ + return x ? (fy_bit64_lowest(x) + 1) : 0; +} + #define FY_BIT64_LOWEST(_x) fy_bit64_lowest(_x) #define FY_BIT64_HIGEST(_x) fy_bit64_highest(_x) #define FY_BIT64_POPCNT(_x) fy_bit64_popcnt(_x) +#define FY_BIT64_FFS(_x) fy_bit64_ffs(_x) #endif diff --git a/src/util/fy-blob.c b/src/util/fy-blob.c index 152afdd..9ed73a9 100644 --- a/src/util/fy-blob.c +++ b/src/util/fy-blob.c @@ -93,7 +93,7 @@ int fy_blob_write(const char *file, const void *blob, size_t size) if (!file || !blob) return -1; - fd = open(file, O_WRONLY|O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IRGRP | S_IROTH); + fd = open(file, O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IRGRP | S_IROTH); if (fd < 0) goto err_out; @@ -117,3 +117,27 @@ err_out: close(fd); return -1; } + +size_t br_wstr(struct blob_region *br, bool dedup, const char *str) +{ + const char *p, *e, *pt; + size_t tlen, len; + + len = strlen(str); + + /* search in the string table for duplicates */ + if (dedup && br->wstart) { + p = (const char *)br->wstart; + e = p + br->curr; + while (p < e) { + tlen = strlen(p); + if (tlen >= len) { + pt = p + (tlen - len); + if (!memcmp(pt, str, len)) + return (size_t)(pt - (const char *)br->wstart); + } + p += tlen + 1; + } + } + return br_write(br, str, len + 1); +} diff --git a/src/util/fy-blob.h b/src/util/fy-blob.h index 301455d..28631d6 100644 --- a/src/util/fy-blob.h +++ b/src/util/fy-blob.h @@ -14,7 +14,6 @@ #include #include -#include #ifdef HAVE_BYTESWAP_H #include #else @@ -44,6 +43,7 @@ #include #include +#include "fy-utils.h" #include "fy-endian.h" /* special unaligned types for pointer accesses */ @@ -72,13 +72,13 @@ enum blob_id_size { }; static inline enum blob_id_size -blob_count_to_id_size(unsigned long long count) +blob_count_to_id_size(uintmax_t count) { - if (count < (1LLU << 8)) + if (count <= (1LLU << 8)) return BID_U8; - if (count < (1LLU << 16)) + if (count <= (1LLU << 16)) return BID_U16; - if (count < (1LLU << 32)) + if (count <= (1LLU << 32)) return BID_U32; return BID_U64; } @@ -117,7 +117,6 @@ size_t _pfx ## _bits (struct blob_region *br, uint ## _bits ## _t v) \ { \ size_t pos; \ \ - assert(br->curr < br->size); \ if (_write) { \ if (_bswap) \ v = bswap_ ## _bits (v); \ @@ -134,8 +133,6 @@ uint ## _bits ## _t _pfx ## _bits (struct blob_region *br) \ { \ uint ## _bits ## _t v; \ \ - assert(br->curr + sizeof(v) <= br->size); \ - assert(br->rstart); \ v = ((const br_ ## _bits *)(br->rstart + br->curr))->v; \ if (_bswap) \ v = bswap_ ## _bits (v); \ @@ -186,7 +183,6 @@ static inline void br_wsetup(struct blob_region *br, void *data, size_t size, en static inline void br_rsetup(struct blob_region *br, const void *data, size_t size, enum blob_endian_type endian) { /* read makes no sense without data */ - assert(data); br_setup_common(br, size, endian); br->rstart = data; } @@ -205,7 +201,6 @@ static inline size_t br_write(struct blob_region *br, const void *data, size_t s { size_t pos; - assert(br->curr + size <= br->size); if (br->wstart) memcpy(br->wstart + br->curr, data, size); pos = br->curr; @@ -217,7 +212,6 @@ static inline size_t br_w0(struct blob_region *br, size_t size) { size_t pos; - assert(br->curr + size <= br->size); if (br->wstart) memset(br->wstart + br->curr, 0, size); pos = br->curr; @@ -232,7 +226,6 @@ static inline size_t br_wskip(struct blob_region *br, size_t size) static inline size_t br_wskip_to(struct blob_region *br, size_t offset) { - assert(br->curr <= offset); return br_w0(br, offset - br->curr); } @@ -245,25 +238,21 @@ typedef size_t (*br_wid_func)(struct blob_region *br, int id); static inline size_t br_wid8(struct blob_region *br, int id) { - assert((unsigned int)id < ((unsigned int)1 << 8)); return br_w8(br, (uint8_t)id); } static inline size_t br_wid16(struct blob_region *br, int id) { - assert((unsigned int)id < ((unsigned int)1 << 16)); return br_w16(br, (uint16_t)id); } static inline size_t br_wid32(struct blob_region *br, int id) { - assert((unsigned long)id < ((unsigned long)1 << 32)); return br_w32(br, (uint32_t)id); } static inline size_t br_wid64(struct blob_region *br, int id) { - assert(id >= 0); return br_w64(br, (uint64_t)id); } @@ -285,26 +274,19 @@ static inline br_wid_func br_wid_get_func(enum blob_id_size id_size) return NULL; } -static inline size_t br_wstr(struct blob_region *br, const char *str, size_t len) -{ - if (len == (size_t)-1) - len = strlen(str); - return br_write(br, str, len + 1); -} +/* write a string with optional dedup */ +size_t br_wstr(struct blob_region *br, bool dedup, const char *str); static inline size_t br_wX(struct blob_region *br, enum blob_id_size x_size, uint64_t x) { switch (x_size) { case BID_U8: - assert(x < ((uint64_t)1 << 8)); return br_w8(br, (uint8_t)x); case BID_U16: - assert(x < ((uint64_t)1 << 16)); return br_w16(br, (uint16_t)x); case BID_U32: - assert(x < ((uint64_t)1 << 32)); return br_w32(br, (uint32_t)x); case BID_U64: @@ -315,7 +297,6 @@ static inline size_t br_wX(struct blob_region *br, enum blob_id_size x_size, uin static inline size_t br_wid(struct blob_region *br, enum blob_id_size id_size, int id) { - assert(id >= 0); switch (id_size) { case BID_U8: return br_wid8(br, id); @@ -336,8 +317,6 @@ static inline size_t br_read(struct blob_region *br, void *data, size_t size) { size_t pos; - assert(br->curr + size <= br->size); - assert(br->rstart); memcpy(data, br->rstart + br->curr, size); pos = br->curr; br->curr += size; @@ -346,14 +325,11 @@ static inline size_t br_read(struct blob_region *br, void *data, size_t size) static inline void br_rskip(struct blob_region *br, size_t size) { - assert(br->curr + size <= br->size); br->curr += size; } static inline void br_rskip_to(struct blob_region *br, size_t offset) { - assert(br->curr <= offset); - assert(offset <= br->size); br->curr = offset; } @@ -417,7 +393,7 @@ static inline uint64_t br_rX(struct blob_region *br, enum blob_id_size x_size) case BID_U64: return br_r64(br); } - return (uint64_t)-1; + FY_IMPOSSIBLE_ABORT(); } static inline int br_rid(struct blob_region *br, enum blob_id_size id_size) @@ -443,9 +419,8 @@ static inline int br_rid(struct blob_region *br, enum blob_id_size id_size) default: id = -1; - break; + FY_IMPOSSIBLE_ABORT(); } - assert(id >= 0); return id; } diff --git a/src/util/fy-ctype.h b/src/util/fy-ctype.h index 06b5ecd..7c20791 100644 --- a/src/util/fy-ctype.h +++ b/src/util/fy-ctype.h @@ -29,110 +29,118 @@ enum fy_flow_ws_mode { fyfws_space, /* only space (json) */ }; -static inline bool fy_is_first_alpha(int c) +static inline bool fy_is_first_alpha(const int c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; } -static inline bool fy_is_alpha(int c) +static inline bool fy_is_alpha(const int c) { return fy_is_first_alpha(c) || c == '-'; } -static inline bool fy_is_num(int c) +static inline bool fy_is_num(const int c) { return c >= '0' && c <= '9'; } -static inline bool fy_is_digit(int c) +static inline bool fy_is_digit(const int c) { return c >= '0' && c <= '9'; } -static inline bool fy_is_xdigit(int c) +static inline bool fy_is_xdigit(const int c) { return fy_is_digit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } -static inline bool fy_is_first_alnum(int c) +static inline bool fy_is_first_alnum(const int c) { return fy_is_first_alpha(c); } -static inline bool fy_is_alnum(int c) +static inline bool fy_is_alnum(const int c) { return fy_is_alpha(c) || fy_is_num(c); } -static inline bool fy_is_space(int c) +static inline bool fy_is_space(const int c) { - return c == ' '; + return fy_utf8_is_space(c); } -static inline bool fy_is_tab(int c) +static inline bool fy_is_tab(const int c) { - return c == '\t'; + return fy_utf8_is_tab(c); } -static inline bool fy_is_ws(int c) +static inline bool fy_is_ws(const int c) { - return fy_is_space(c) || fy_is_tab(c); + return fy_utf8_is_ws(c); } -static inline bool fy_is_hex(int c) +static inline bool fy_is_hex(const int c) { - return (c >= '0' && c <= '9') || - (c >= 'a' && c <= 'f') || - (c >= 'A' && c <= 'F'); + return fy_utf8_is_hex(c); } -static inline bool fy_is_uri(int c) +static inline bool fy_is_flow_indicator(const int c) { - return fy_is_alnum(c) || fy_utf8_strchr(";/?:@&=+$,.!~*\'()[]%", c); + return fy_utf8_is_flow_indicator(c); +} + +static inline bool fy_is_lb_r_n(const int c) +{ + return fy_utf8_is_lb(c); } -static inline bool fy_is_lb_r_n(int c) +static inline bool fy_is_blank(const int c) { - return c == '\r' || c == '\n'; + return fy_utf8_is_ws(c); +} + +static inline bool fy_is_blank_lb_r_n(const int c) +{ + return fy_utf8_is_ws_lb(c); +} + +static inline bool fy_is_uri(const int c) +{ + return fy_is_alnum(c) || fy_utf8_strchr(";/?:@&=+$,.!~*\'()[]%", c); } -static inline bool fy_is_lb_NEL(int c) +static inline bool fy_is_lb_NEL(const int c) { return c == 0x85; } -static inline bool fy_is_lb_LS_PS(int c) +static inline bool fy_is_lb_LS_PS(const int c) { return c == 0x2028 || c == 0x2029; } -static inline bool fy_is_unicode_lb(int c) +static inline bool fy_is_unicode_lb(const int c) { /* note that YAML1.1 supports NEL #x85, LS #x2028 and PS #x2029 as linebreaks */ /* YAML1.2 and higher does not */ return fy_is_lb_NEL(c) || fy_is_lb_LS_PS(c); } -static inline bool fy_is_any_lb(int c) +static inline bool fy_is_any_lb(const int c) { return fy_is_lb_r_n(c) || fy_is_unicode_lb(c); } -static inline bool fy_is_z(int c) +static inline bool fy_is_z(const int c) { return c <= 0; } -static inline bool fy_is_blank(int c) -{ - return c == ' ' || c == '\t'; -} - #define FY_UTF8_BOM 0xfeff -static inline bool fy_is_print(int c) +static inline bool fy_is_print(const int c) { return c == '\n' || c == '\r' || (c >= 0x0020 && c <= 0x007e) || @@ -140,110 +148,112 @@ static inline bool fy_is_print(int c) (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM); } -static inline bool fy_is_printq(int c) +static inline bool fy_is_printq(const int c) { return c != '\t' && c != 0xa0 && !fy_is_any_lb(c) && fy_is_print(c); } -static inline bool fy_is_nb_char(int c) +static inline bool fy_is_nb_char(const int c) { return (c >= 0x0020 && c <= 0x007e) || (c >= 0x00a0 && c <= 0xd7ff) || (c >= 0xe000 && c <= 0xfffd && c != FY_UTF8_BOM); } -static inline bool fy_is_ns_char(int c) +static inline bool fy_is_ns_char(const int c) { return fy_is_nb_char(c) && !fy_is_ws(c); } -static inline bool fy_is_start_indicator(int c) +static inline bool fy_is_start_indicator(const int c) { return !!fy_utf8_strchr(",[]{}#&*!|>'\"%%@`", c); } -static inline bool fy_is_indicator_before_space(int c) +static inline bool fy_is_indicator_before_space(const int c) { return !!fy_utf8_strchr("-:?`", c); } -static inline bool fy_is_flow_indicator(int c) -{ - return !!fy_utf8_strchr(",[]{}", c); -} - -static inline bool fy_is_path_flow_scalar_start(int c) +static inline bool fy_is_path_flow_scalar_start(const int c) { return c == '\'' || c == '"'; } -static inline bool fy_is_path_flow_key_start(int c) +static inline bool fy_is_path_flow_key_start(const int c) { return c == '"' || c == '\'' || c == '{' || c == '['; } -static inline bool fy_is_path_flow_key_end(int c) +static inline bool fy_is_path_flow_key_end(const int c) { return c == '"' || c == '\'' || c == '}' || c == ']'; } -static inline bool fy_is_unicode_control(int c) +static inline bool fy_is_unicode_control(const int c) { return (c >= 0 && c <= 0x1f) || (c >= 0x80 && c <= 0x9f); } -static inline bool fy_is_unicode_space(int c) +static inline bool fy_is_unicode_space(const int c) { return c == 0x20 || c == 0xa0 || (c >= 0x2000 && c <= 0x200a) || c == 0x202f || c == 0x205f || c == 0x3000; } -static inline bool fy_is_json_unescaped(int c) +static inline bool fy_is_json_unescaped(const int c) { return c >= 0x20 && c <= 0x110000 && c != '"' && c != '\\'; } -static inline bool fy_is_json_unescaped_range_only(int c) +static inline bool fy_is_json_unescaped_range_only(const int c) { return c >= 0x20 && c <= 0x110000; } -static inline bool fy_is_lb_m(int c, enum fy_lb_mode lb_mode) +static inline bool fy_is_lb_m(const int c, const enum fy_lb_mode lb_mode) { if (fy_is_lb_r_n(c)) return true; return lb_mode == fylb_cr_nl_N_L_P && fy_is_unicode_lb(c); } -static inline bool fy_is_generic_lb_m(int c, enum fy_lb_mode lb_mode) +static inline bool fy_is_generic_lb_m(const int c, const enum fy_lb_mode lb_mode) { if (fy_is_lb_r_n(c)) return true; return lb_mode == fylb_cr_nl_N_L_P && fy_is_lb_NEL(c); } -static inline bool fy_is_lbz_m(int c, enum fy_lb_mode lb_mode) +static inline bool fy_is_blank_lb_m(const int c, const enum fy_lb_mode lb_mode) +{ + if (fy_is_blank_lb_r_n(c)) + return true; + return lb_mode == fylb_cr_nl_N_L_P && fy_is_unicode_lb(c); +} + +static inline bool fy_is_lbz_m(const int c, const enum fy_lb_mode lb_mode) { return fy_is_lb_m(c, lb_mode) || fy_is_z(c); } -static inline bool fy_is_generic_lbz_m(int c, enum fy_lb_mode lb_mode) +static inline bool fy_is_generic_lbz_m(const int c, const enum fy_lb_mode lb_mode) { return fy_is_generic_lb_m(c, lb_mode) || fy_is_z(c); } -static inline bool fy_is_blankz_m(int c, enum fy_lb_mode lb_mode) +static inline bool fy_is_blankz_m(const int c, const enum fy_lb_mode lb_mode) { return fy_is_ws(c) || fy_is_lbz_m(c, lb_mode); } -static inline bool fy_is_generic_blankz_m(int c, enum fy_lb_mode lb_mode) +static inline bool fy_is_generic_blankz_m(const int c, const enum fy_lb_mode lb_mode) { return fy_is_ws(c) || fy_is_generic_lbz_m(c, lb_mode); } -static inline bool fy_is_flow_ws_m(int c, enum fy_flow_ws_mode fws_mode) +static inline bool fy_is_flow_ws_m(const int c, const enum fy_flow_ws_mode fws_mode) { return fy_is_space(c) || (fws_mode == fyfws_space_tab && fy_is_tab(c)); } diff --git a/src/util/fy-id.h b/src/util/fy-id.h index 95c120e..aae5b9f 100644 --- a/src/util/fy-id.h +++ b/src/util/fy-id.h @@ -15,11 +15,15 @@ #include -typedef unsigned int fy_id_bits; +#include "fy-bit64.h" +#include "fy-atomics.h" + +typedef uint64_t fy_id_bits_non_atomic; +typedef FY_ATOMIC(fy_id_bits_non_atomic) fy_id_bits; #define FY_ID_BITS_SZ ((size_t)(sizeof(fy_id_bits))) #define FY_ID_BITS_BITS (FY_ID_BITS_SZ * 8) /* yes, every character is 8 bits in 2023 */ -#define FY_ID_BITS_MASK ((size_t)(FY_ID_BITS_BITS - 1)) +#define FY_ID_BITS_MASK ((uint64_t)(FY_ID_BITS_BITS - 1)) #define FY_ID_BITS_ARRAY_COUNT_BITS(_bits) \ (((_bits) + FY_ID_BITS_MASK) & ~FY_ID_BITS_MASK) @@ -28,58 +32,44 @@ typedef unsigned int fy_id_bits; (FY_ID_BITS_ARRAY_COUNT_BITS(_bits) / FY_ID_BITS_BITS) #define FY_ID_OFFSET(_id) ((_id) / FY_ID_BITS_BITS) -#define FY_ID_BIT_MASK(_id) ((fy_id_bits)1 << ((_id) & FY_ID_BITS_MASK)) +#define FY_ID_BIT_MASK(_id) ((uint64_t)1 << ((_id) & FY_ID_BITS_MASK)) -#if defined(__GNUC__) && __GNUC__ >= 4 -static inline int fy_id_ffs(const fy_id_bits id_bit) +static inline int fy_id_ffs(const fy_id_bits_non_atomic id_bit) { if (!id_bit) return -1; - return __builtin_ffs(id_bit) - 1; -} - -#else -static inline int fy_id_ffs(const fy_id_bits id_bit) -{ - return ffs((int)id_bits) - 1; -} -#endif - -#if defined(__GNUC__) && __GNUC__ >= 4 -static inline int fy_id_popcount(const fy_id_bits id_bit) -{ - return __builtin_popcount(id_bit); + return FY_BIT64_FFS(id_bit) - 1; } -#else -static inline int fy_id_popcount(fy_id_bits id_bit) +static inline int fy_id_popcount(const fy_id_bits_non_atomic id_bit) { - int count, pos; - - count = 0; - while ((pos = fy_id_ffs(id_bit)) >= 0) { - count++; - id_bit &= ~((fy_id_bits)1 << pos); - } - return count; + return FY_BIT64_POPCNT(id_bit); } -#endif static inline void fy_id_reset(fy_id_bits *bits, const size_t count) { - memset(bits, 0, sizeof(*bits) * count); + size_t i; + + for (i = 0; i < count; i++, bits++) + fy_atomic_store(bits, 0); } static inline int fy_id_alloc(fy_id_bits *bits, const size_t count) { size_t i; int pos; + fy_id_bits_non_atomic v, new_v; for (i = 0; i < count; i++, bits++) { - pos = fy_id_ffs(~*bits); - if (pos >= 0) { - *bits |= FY_ID_BIT_MASK(pos); - return (i * FY_ID_BITS_BITS) + pos; + + for (;;) { + v = fy_atomic_load(bits); + pos = fy_id_ffs(~v); + if (pos < 0) + break; + new_v = v | FY_ID_BIT_MASK(pos); + if (fy_atomic_compare_exchange_strong(bits, &v, new_v)) + return (i * FY_ID_BITS_BITS) + pos; } } return -1; @@ -88,50 +78,62 @@ static inline int fy_id_alloc(fy_id_bits *bits, const size_t count) static inline int fy_id_alloc_fixed(fy_id_bits *bits, const size_t count, const int id) { size_t offset; - fy_id_bits m; + fy_id_bits_non_atomic m, v, new_v; offset = FY_ID_OFFSET(id); if (offset >= count) return -1; bits += offset; + m = FY_ID_BIT_MASK(id); - if (*bits & m) + v = fy_atomic_load(bits); + if (v & m) + return -1; + new_v = v | m; + if (!fy_atomic_compare_exchange_strong(bits, &v, new_v)) return -1; - *bits |= m; + return id; } static inline bool fy_id_is_used(fy_id_bits *bits, const size_t count, const int id) { + fy_id_bits_non_atomic v; size_t offset; offset = FY_ID_OFFSET(id); if (offset >= count) return false; - return !!(bits[offset] & FY_ID_BIT_MASK(id)); + bits += offset; + v = fy_atomic_load(bits); + return !!(v & FY_ID_BIT_MASK(id)); } -static inline bool fy_id_is_used_no_check(fy_id_bits *bits, const size_t count, const int id) +static inline bool fy_id_is_free(fy_id_bits *bits, const size_t count, const int id) { + fy_id_bits_non_atomic v; size_t offset; offset = FY_ID_OFFSET(id); if (offset >= count) return false; - return !!(bits[offset] & FY_ID_BIT_MASK(id)); + bits += offset; + v = fy_atomic_load(bits); + return !(v & FY_ID_BIT_MASK(id)); } -static inline bool fy_id_is_free(fy_id_bits *bits, const size_t count, const int id) +static inline void fy_id_free(fy_id_bits *bits, const size_t count, int id) { size_t offset; offset = FY_ID_OFFSET(id); if (offset >= count) - return false; - return !(bits[offset] & FY_ID_BIT_MASK(id)); + return; + bits += offset; + fy_atomic_fetch_and(bits, ~FY_ID_BIT_MASK(id)); /* no need to check, if bit is free, it is still free */ } -static inline void fy_id_free(fy_id_bits *bits, const size_t count, int id) +static inline void fy_id_set_used(fy_id_bits *bits, const size_t count, const int id) { size_t offset; @@ -139,39 +141,36 @@ static inline void fy_id_free(fy_id_bits *bits, const size_t count, int id) if (offset >= count) return; bits += offset; - *bits &= ~FY_ID_BIT_MASK(id); /* no need to check, if bit is free, it is still free */ -} - -static inline void fy_id_set_used(fy_id_bits *bits, const size_t count, const int id) -{ - assert(FY_ID_OFFSET(id) < count); - bits[FY_ID_OFFSET(id)] |= FY_ID_BIT_MASK(id); + fy_atomic_fetch_or(bits, FY_ID_BIT_MASK(id)); } static inline void fy_id_set_free(fy_id_bits *bits, const size_t count, const int id) { - assert(FY_ID_OFFSET(id) < count); - bits[FY_ID_OFFSET(id)] &= ~FY_ID_BIT_MASK(id); + size_t offset; + + offset = FY_ID_OFFSET(id); + if (offset >= count) + return; + bits += offset; + fy_atomic_fetch_and(bits, ~FY_ID_BIT_MASK(id)); } static inline size_t fy_id_count_used(fy_id_bits *bits, const size_t count) { + fy_id_bits v; size_t i, popcount; popcount = 0; - for (i = 0; i < count; i++, bits++) - popcount += fy_id_popcount(*bits); + for (i = 0; i < count; i++, bits++) { + v = fy_atomic_load(bits); + popcount += fy_id_popcount(v); + } return popcount; } static inline size_t fy_id_count_free(fy_id_bits *bits, const size_t count) { - size_t i, popcount; - - popcount = 0; - for (i = 0; i < count; i++, bits++) - popcount += fy_id_popcount(~*bits); - return popcount; + return count - fy_id_count_used(bits, count); } struct fy_id_iter { @@ -182,13 +181,13 @@ struct fy_id_iter { static inline void fy_id_iter_begin(const fy_id_bits *bits, const size_t count, struct fy_id_iter *iterp) { iterp->s = bits; - iterp->m = ~(fy_id_bits)0; + iterp->m = ~(fy_id_bits_non_atomic)0; } static inline int fy_id_iter_next(const fy_id_bits *bits, const size_t count, struct fy_id_iter *iterp) { const fy_id_bits *s, *e; - fy_id_bits m; + fy_id_bits_non_atomic v, m; int pos; s = iterp->s; @@ -203,9 +202,13 @@ static inline int fy_id_iter_next(const fy_id_bits *bits, const size_t count, st /* scan until we find a set bit or hit the end */ pos = -1; - while (s < e && (pos = fy_id_ffs(*s & m)) < 0) { + while (s < e) { + v = fy_atomic_load(s); + pos = fy_id_ffs(v & m); + if (pos >= 0) + break; s++; - m = ~(fy_id_bits)0; + m = ~(fy_id_bits_non_atomic)0; } /* not found */ @@ -216,7 +219,7 @@ static inline int fy_id_iter_next(const fy_id_bits *bits, const size_t count, st } /* remove this bit from the mask */ - m &= ~((fy_id_bits)1 << pos); + m &= ~((fy_id_bits_non_atomic)1 << pos); /* advance by the number of fy_id_bits we scanned */ pos += (int)(s - bits) * FY_ID_BITS_BITS; @@ -224,14 +227,14 @@ static inline int fy_id_iter_next(const fy_id_bits *bits, const size_t count, st /* if we run out of mask, advance */ if (!m) { s++; - m = ~(fy_id_bits)0; + m = ~(fy_id_bits_non_atomic)0; } iterp->s = s; iterp->m = m; return pos; } -static inline void fy_id_iter_end(fy_id_bits *bits, const size_t count, const struct fy_id_iter *iterp) +static inline void fy_id_iter_end(const fy_id_bits *bits, const size_t count, const struct fy_id_iter *iterp) { /* nothing */ } diff --git a/src/util/fy-list.h b/src/util/fy-list.h index 1a0b5f9..892d6cd 100644 --- a/src/util/fy-list.h +++ b/src/util/fy-list.h @@ -1,277 +1,76 @@ /* - * fy-list.h - slightly modified Linux kernel list.h + * fy-list.h - Circular doubly-linked list implementation * - * All copyrights owned by their respective holders. + * Copyright (c) 2025 Pantelis Antoniou * - * SPDX-License-Identifier: GPL-2.0 + * SPDX-License-Identifier: MIT */ -#ifndef _FY_LIST_H -#define _FY_LIST_H +#ifndef FY_LIST_H +#define FY_LIST_H -#include - -/** - * container_of - cast a member of a structure out to the containing structure - * @ptr: the pointer to the member. - * @type: the type of the container struct this is embedded in. - * @member: the name of the member within the struct. - * - */ -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) - -#define LIST_POISON1 NULL -#define LIST_POISON2 NULL - -#ifndef ARCH_HAS_PREFETCH -#define ARCH_HAS_PREFETCH -static inline void prefetch(const void *x) {;} -#endif - -/* - * Simple doubly linked list implementation. - * - * Some of the internal functions ("__xxx") are useful when - * manipulating whole lists rather than single entries, as - * sometimes we already know the next/prev entries and we can - * generate better code by using them directly rather than - * using the generic single-entry routines. - */ +#include "fy-utils.h" struct list_head { - struct list_head *next, *prev; + struct list_head *next; + struct list_head *prev; }; -#define LIST_HEAD_INIT(name) { &(name), &(name) } - -#define LIST_HEAD(name) \ - struct list_head name = LIST_HEAD_INIT(name) - -static inline void INIT_LIST_HEAD(struct list_head *list) +static inline void list_init(struct list_head *list) { - list->next = list; - list->prev = list; + list->next = list->prev = list; } -/* - * Insert a new entry between two known consecutive entries. - * - * This is only for internal list manipulation where we know - * the prev/next entries already! - */ -static inline void __list_add(struct list_head *new, - struct list_head *prev, - struct list_head *next) +static inline void list_add(struct list_head *new_item, struct list_head *head) { - next->prev = new; - new->next = next; - new->prev = prev; - prev->next = new; -} + struct list_head *prev = head; + struct list_head *next = head->next; -/** - * list_add - add a new entry - * @new: new entry to be added - * @head: list head to add it after - * - * Insert a new entry after the specified head. - * This is good for implementing stacks. - */ -static inline void list_add(struct list_head *new, struct list_head *head) -{ - __list_add(new, head, head->next); + next->prev = new_item; + new_item->next = next; + new_item->prev = prev; + prev->next = new_item; } -/** - * list_add_tail - add a new entry - * @new: new entry to be added - * @head: list head to add it before - * - * Insert a new entry before the specified head. - * This is useful for implementing queues. - */ -static inline void list_add_tail(struct list_head *new, struct list_head *head) +static inline void list_add_tail(struct list_head *new_item, struct list_head *head) { - __list_add(new, head->prev, head); -} + struct list_head *prev = head->prev; + struct list_head *next = head; -/* - * Delete a list entry by making the prev/next entries - * point to each other. - * - * This is only for internal list manipulation where we know - * the prev/next entries already! - */ -static inline void __list_del(struct list_head *prev, struct list_head *next) -{ - next->prev = prev; - prev->next = next; + next->prev = new_item; + new_item->next = next; + new_item->prev = prev; + prev->next = new_item; } -/** - * list_del - deletes entry from list. - * @entry: the element to delete from the list. - * Note: list_empty() on entry does not return true after this, the entry is - * in an undefined state. - */ static inline void list_del(struct list_head *entry) { - __list_del(entry->prev, entry->next); - entry->next = LIST_POISON1; - entry->prev = LIST_POISON2; -} - -/** - * list_replace - replace old entry by new one - * @old : the element to be replaced - * @new : the new element to insert - * - * If @old was empty, it will be overwritten. - */ -static inline void list_replace(struct list_head *old, - struct list_head *new) -{ - new->next = old->next; - new->next->prev = new; - new->prev = old->prev; - new->prev->next = new; -} - -static inline void list_replace_init(struct list_head *old, - struct list_head *new) -{ - list_replace(old, new); - INIT_LIST_HEAD(old); -} - -/** - * list_del_init - deletes entry from list and reinitialize it. - * @entry: the element to delete from the list. - */ -static inline void list_del_init(struct list_head *entry) -{ - __list_del(entry->prev, entry->next); - INIT_LIST_HEAD(entry); -} - -/** - * list_move - delete from one list and add as another's head - * @list: the entry to move - * @head: the head that will precede our entry - */ -static inline void list_move(struct list_head *list, struct list_head *head) -{ - __list_del(list->prev, list->next); - list_add(list, head); -} - -/** - * list_move_tail - delete from one list and add as another's tail - * @list: the entry to move - * @head: the head that will follow our entry - */ -static inline void list_move_tail(struct list_head *list, - struct list_head *head) -{ - __list_del(list->prev, list->next); - list_add_tail(list, head); -} + struct list_head *prev = entry->prev; + struct list_head *next = entry->next; -/** - * list_is_last - tests whether @list is the last entry in list @head - * @list: the entry to test - * @head: the head of the list - */ -static inline int list_is_last(const struct list_head *list, - const struct list_head *head) -{ - return list->next == head; + next->prev = prev; + prev->next = next; } -/** - * list_empty - tests whether a list is empty - * @head: the list to test. - */ static inline int list_empty(const struct list_head *head) { return head->next == head; } -/** - * list_empty_careful - tests whether a list is empty and not being modified - * @head: the list to test - * - * Description: - * tests whether a list is empty _and_ checks that no other CPU might be - * in the process of modifying either member (next or prev) - * - * NOTE: using list_empty_careful() without synchronization - * can only be safe if the only activity that can happen - * to the list entry is list_del_init(). Eg. it cannot be used - * if another CPU could re-list_add() it. - */ -static inline int list_empty_careful(const struct list_head *head) -{ - struct list_head *next = head->next; - return (next == head) && (next == head->prev); -} - -/** - * list_is_singular - tests whether a list has just one entry. - * @head: the list to test. - */ static inline int list_is_singular(const struct list_head *head) { - return !list_empty(head) && (head->next == head->prev); -} - -static inline void __list_cut_position(struct list_head *list, - struct list_head *head, struct list_head *entry) -{ - struct list_head *new_first = entry->next; - list->next = head->next; - list->next->prev = list; - list->prev = entry; - entry->next = list; - head->next = new_first; - new_first->prev = head; + return !list_empty(head) && head->next == head->prev; } -/** - * list_cut_position - cut a list into two - * @list: a new list to add all removed entries - * @head: a list with entries - * @entry: an entry within head, could be the head itself - * and if so we won't cut the list - * - * This helper moves the initial part of @head, up to and - * including @entry, from @head to @list. You should - * pass on @entry an element you know is on @head. @list - * should be an empty list or a list you do not care about - * losing its data. - * - */ -static inline void list_cut_position(struct list_head *list, - struct list_head *head, struct list_head *entry) -{ - if (list_empty(head)) - return; - if (list_is_singular(head) && - (head->next != entry && head != entry)) - return; - if (entry == head) - INIT_LIST_HEAD(list); - else - __list_cut_position(list, head, entry); -} - -static inline void __list_splice(const struct list_head *list, - struct list_head *prev, - struct list_head *next) +static inline void list_splice(const struct list_head *list, + struct list_head *head) { struct list_head *first = list->next; struct list_head *last = list->prev; + struct list_head *prev = head; + struct list_head *next = head->next; + + if (list_empty(list)) + return; first->prev = prev; prev->next = first; @@ -280,426 +79,13 @@ static inline void __list_splice(const struct list_head *list, next->prev = last; } -/** - * list_splice - join two lists, this is designed for stacks - * @list: the new list to add. - * @head: the place to add it in the first list. - */ -static inline void list_splice(const struct list_head *list, - struct list_head *head) -{ - if (!list_empty(list)) - __list_splice(list, head, head->next); -} - -/** - * list_splice_tail - join two lists, each list being a queue - * @list: the new list to add. - * @head: the place to add it in the first list. - */ -static inline void list_splice_tail(struct list_head *list, - struct list_head *head) -{ - if (!list_empty(list)) - __list_splice(list, head->prev, head); -} - -/** - * list_splice_init - join two lists and reinitialise the emptied list. - * @list: the new list to add. - * @head: the place to add it in the first list. - * - * The list at @list is reinitialised - */ -static inline void list_splice_init(struct list_head *list, - struct list_head *head) -{ - if (!list_empty(list)) { - __list_splice(list, head, head->next); - INIT_LIST_HEAD(list); - } -} - -/** - * list_splice_tail_init - join two lists and reinitialise the emptied list - * @list: the new list to add. - * @head: the place to add it in the first list. - * - * Each of the lists is a queue. - * The list at @list is reinitialised - */ -static inline void list_splice_tail_init(struct list_head *list, - struct list_head *head) -{ - if (!list_empty(list)) { - __list_splice(list, head->prev, head); - INIT_LIST_HEAD(list); - } -} - -/** - * list_entry - get the struct for this entry - * @ptr: the &struct list_head pointer. - * @type: the type of the struct this is embedded in. - * @member: the name of the list_struct within the struct. - */ #define list_entry(ptr, type, member) \ container_of(ptr, type, member) -/** - * list_first_entry - get the first element from a list - * @ptr: the list head to take the element from. - * @type: the type of the struct this is embedded in. - * @member: the name of the list_struct within the struct. - * - * Note, that list is expected to be not empty. - */ #define list_first_entry(ptr, type, member) \ list_entry((ptr)->next, type, member) -/** - * list_last_entry - get the last element from a list - * @ptr: the list head to take the element from. - * @type: the type of the struct this is embedded in. - * @member: the name of the list_struct within the struct. - * - * Note, that list is expected to be not empty. - */ #define list_last_entry(ptr, type, member) \ list_entry((ptr)->prev, type, member) -/** - * list_for_each - iterate over a list - * @pos: the &struct list_head to use as a loop cursor. - * @head: the head for your list. - */ -#define list_for_each(pos, head) \ - for (pos = (head)->next; prefetch(pos->next), pos != (head); \ - pos = pos->next) - -/** - * __list_for_each - iterate over a list - * @pos: the &struct list_head to use as a loop cursor. - * @head: the head for your list. - * - * This variant differs from list_for_each() in that it's the - * simplest possible list iteration code, no prefetching is done. - * Use this for code that knows the list to be very short (empty - * or 1 entry) most of the time. - */ -#define __list_for_each(pos, head) \ - for (pos = (head)->next; pos != (head); pos = pos->next) - -/** - * list_for_each_prev - iterate over a list backwards - * @pos: the &struct list_head to use as a loop cursor. - * @head: the head for your list. - */ -#define list_for_each_prev(pos, head) \ - for (pos = (head)->prev; prefetch(pos->prev), pos != (head); \ - pos = pos->prev) - -/** - * list_for_each_safe - iterate over a list safe against removal of list entry - * @pos: the &struct list_head to use as a loop cursor. - * @n: another &struct list_head to use as temporary storage - * @head: the head for your list. - */ -#define list_for_each_safe(pos, n, head) \ - for (pos = (head)->next, n = pos->next; pos != (head); \ - pos = n, n = pos->next) - -/** - * list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry - * @pos: the &struct list_head to use as a loop cursor. - * @n: another &struct list_head to use as temporary storage - * @head: the head for your list. - */ -#define list_for_each_prev_safe(pos, n, head) \ - for (pos = (head)->prev, n = pos->prev; \ - prefetch(pos->prev), pos != (head); \ - pos = n, n = pos->prev) - -/** - * list_for_each_entry - iterate over list of given type - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - */ -#define list_for_each_entry(pos, head, member) \ - for (pos = list_entry((head)->next, typeof(*pos), member); \ - prefetch(pos->member.next), &pos->member != (head); \ - pos = list_entry(pos->member.next, typeof(*pos), member)) - -/** - * list_for_each_entry_reverse - iterate backwards over list of given type. - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - */ -#define list_for_each_entry_reverse(pos, head, member) \ - for (pos = list_entry((head)->prev, typeof(*pos), member); \ - prefetch(pos->member.prev), &pos->member != (head); \ - pos = list_entry(pos->member.prev, typeof(*pos), member)) - -/** - * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue() - * @pos: the type * to use as a start point - * @head: the head of the list - * @member: the name of the list_struct within the struct. - * - * Prepares a pos entry for use as a start point in list_for_each_entry_continue(). - */ -#define list_prepare_entry(pos, head, member) \ - ((pos) ? : list_entry(head, typeof(*pos), member)) - -/** - * list_for_each_entry_continue - continue iteration over list of given type - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Continue to iterate over list of given type, continuing after - * the current position. - */ -#define list_for_each_entry_continue(pos, head, member) \ - for (pos = list_entry(pos->member.next, typeof(*pos), member); \ - prefetch(pos->member.next), &pos->member != (head); \ - pos = list_entry(pos->member.next, typeof(*pos), member)) - -/** - * list_for_each_entry_continue_reverse - iterate backwards from the given point - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Start to iterate over list of given type backwards, continuing after - * the current position. - */ -#define list_for_each_entry_continue_reverse(pos, head, member) \ - for (pos = list_entry(pos->member.prev, typeof(*pos), member); \ - prefetch(pos->member.prev), &pos->member != (head); \ - pos = list_entry(pos->member.prev, typeof(*pos), member)) - -/** - * list_for_each_entry_from - iterate over list of given type from the current point - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Iterate over list of given type, continuing from current position. - */ -#define list_for_each_entry_from(pos, head, member) \ - for (; prefetch(pos->member.next), &pos->member != (head); \ - pos = list_entry(pos->member.next, typeof(*pos), member)) - -/** - * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry - * @pos: the type * to use as a loop cursor. - * @n: another type * to use as temporary storage - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - */ -#define list_for_each_entry_safe(pos, n, head, member) \ - for (pos = list_entry((head)->next, typeof(*pos), member), \ - n = list_entry(pos->member.next, typeof(*pos), member); \ - &pos->member != (head); \ - pos = n, n = list_entry(n->member.next, typeof(*n), member)) - -/** - * list_for_each_entry_safe_continue - * @pos: the type * to use as a loop cursor. - * @n: another type * to use as temporary storage - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Iterate over list of given type, continuing after current point, - * safe against removal of list entry. - */ -#define list_for_each_entry_safe_continue(pos, n, head, member) \ - for (pos = list_entry(pos->member.next, typeof(*pos), member), \ - n = list_entry(pos->member.next, typeof(*pos), member); \ - &pos->member != (head); \ - pos = n, n = list_entry(n->member.next, typeof(*n), member)) - -/** - * list_for_each_entry_safe_from - * @pos: the type * to use as a loop cursor. - * @n: another type * to use as temporary storage - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Iterate over list of given type from current point, safe against - * removal of list entry. - */ -#define list_for_each_entry_safe_from(pos, n, head, member) \ - for (n = list_entry(pos->member.next, typeof(*pos), member); \ - &pos->member != (head); \ - pos = n, n = list_entry(n->member.next, typeof(*n), member)) - -/** - * list_for_each_entry_safe_reverse - * @pos: the type * to use as a loop cursor. - * @n: another type * to use as temporary storage - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * Iterate backwards over list of given type, safe against removal - * of list entry. - */ -#define list_for_each_entry_safe_reverse(pos, n, head, member) \ - for (pos = list_entry((head)->prev, typeof(*pos), member), \ - n = list_entry(pos->member.prev, typeof(*pos), member); \ - &pos->member != (head); \ - pos = n, n = list_entry(n->member.prev, typeof(*n), member)) - -/* - * Double linked lists with a single pointer list head. - * Mostly useful for hash tables where the two pointer list head is - * too wasteful. - * You lose the ability to access the tail in O(1). - */ - -struct hlist_head { - struct hlist_node *first; -}; - -struct hlist_node { - struct hlist_node *next, **pprev; -}; - -#define HLIST_HEAD_INIT { .first = NULL } -#define HLIST_HEAD(name) struct hlist_head name = { .first = NULL } -#define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL) -static inline void INIT_HLIST_NODE(struct hlist_node *h) -{ - h->next = NULL; - h->pprev = NULL; -} - -static inline int hlist_unhashed(const struct hlist_node *h) -{ - return !h->pprev; -} - -static inline int hlist_empty(const struct hlist_head *h) -{ - return !h->first; -} - -static inline void __hlist_del(struct hlist_node *n) -{ - struct hlist_node *next = n->next; - struct hlist_node **pprev = n->pprev; - *pprev = next; - if (next) - next->pprev = pprev; -} - -static inline void hlist_del(struct hlist_node *n) -{ - __hlist_del(n); - n->next = LIST_POISON1; - n->pprev = LIST_POISON2; -} - -static inline void hlist_del_init(struct hlist_node *n) -{ - if (!hlist_unhashed(n)) { - __hlist_del(n); - INIT_HLIST_NODE(n); - } -} - -static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) -{ - struct hlist_node *first = h->first; - n->next = first; - if (first) - first->pprev = &n->next; - h->first = n; - n->pprev = &h->first; -} - -/* next must be != NULL */ -static inline void hlist_add_before(struct hlist_node *n, - struct hlist_node *next) -{ - n->pprev = next->pprev; - n->next = next; - next->pprev = &n->next; - *(n->pprev) = n; -} - -static inline void hlist_add_after(struct hlist_node *n, - struct hlist_node *next) -{ - next->next = n->next; - n->next = next; - next->pprev = &n->next; - - if(next->next) - next->next->pprev = &next->next; -} - -#define hlist_entry(ptr, type, member) container_of(ptr,type,member) - -#define hlist_for_each(pos, head) \ - for (pos = (head)->first; pos && ({ prefetch(pos->next); 1; }); \ - pos = pos->next) - -#define hlist_for_each_safe(pos, n, head) \ - for (pos = (head)->first; pos && ({ n = pos->next; 1; }); \ - pos = n) - -/** - * hlist_for_each_entry - iterate over list of given type - * @tpos: the type * to use as a loop cursor. - * @pos: the &struct hlist_node to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the hlist_node within the struct. - */ -#define hlist_for_each_entry(tpos, pos, head, member) \ - for (pos = (head)->first; \ - pos && ({ prefetch(pos->next); 1;}) && \ - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = pos->next) - -/** - * hlist_for_each_entry_continue - iterate over a hlist continuing after current point - * @tpos: the type * to use as a loop cursor. - * @pos: the &struct hlist_node to use as a loop cursor. - * @member: the name of the hlist_node within the struct. - */ -#define hlist_for_each_entry_continue(tpos, pos, member) \ - for (pos = (pos)->next; \ - pos && ({ prefetch(pos->next); 1;}) && \ - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = pos->next) - -/** - * hlist_for_each_entry_from - iterate over a hlist continuing from current point - * @tpos: the type * to use as a loop cursor. - * @pos: the &struct hlist_node to use as a loop cursor. - * @member: the name of the hlist_node within the struct. - */ -#define hlist_for_each_entry_from(tpos, pos, member) \ - for (; pos && ({ prefetch(pos->next); 1;}) && \ - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = pos->next) - -/** - * hlist_for_each_entry_safe - iterate over list of given type safe against removal of list entry - * @tpos: the type * to use as a loop cursor. - * @pos: the &struct hlist_node to use as a loop cursor. - * @n: another &struct hlist_node to use as temporary storage - * @head: the head for your list. - * @member: the name of the hlist_node within the struct. - */ -#define hlist_for_each_entry_safe(tpos, pos, n, head, member) \ - for (pos = (head)->first; \ - pos && ({ n = pos->next; 1; }) && \ - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = n) - #endif diff --git a/src/util/fy-typelist.h b/src/util/fy-typelist.h index 9739e73..a560e01 100644 --- a/src/util/fy-typelist.h +++ b/src/util/fy-typelist.h @@ -29,7 +29,7 @@ struct __useless_struct_to_allow_semicolon #define FY_TYPE_DECL_LIST(_type) \ static inline void fy_ ## _type ## _list_init(struct fy_ ## _type ## _list *_l) \ { \ - INIT_LIST_HEAD(&_l->_lh); \ + list_init(&_l->_lh); \ } \ static inline void fy_ ## _type ## _list_add(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ { \ @@ -61,10 +61,8 @@ static inline bool fy_ ## _type ## _list_is_singular(struct fy_ ## _type ## _lis } \ static inline void fy_ ## _type ## _list_del(struct fy_ ## _type ## _list *_l, struct fy_ ## _type *_n) \ { \ - if (_l && _n) { \ + if (_l && _n) \ list_del(&_n->node); \ - INIT_LIST_HEAD(&_n->node); \ - } \ } \ static inline void fy_ ## _type ## _list_insert_after(struct fy_ ## _type ## _list *_l, \ struct fy_ ## _type *_p, struct fy_ ## _type *_n) \ diff --git a/src/util/fy-utf8.c b/src/util/fy-utf8.c index 45c071c..228879a 100644 --- a/src/util/fy-utf8.c +++ b/src/util/fy-utf8.c @@ -50,11 +50,7 @@ int fy_utf8_get_generic(const void *ptr, size_t left, int *widthp) value = (value << 6) | (*p++ & 0x3f); } - /* check for validity */ - if ((width == 4 && value < 0x10000) || - (width == 3 && value < 0x800) || - (width == 2 && value < 0x80) || - (value >= 0xd800 && value <= 0xdfff) || value >= 0x110000) + if (!fy_utf8_is_valid(value)) return FYUG_INV; *widthp = width; @@ -463,135 +459,136 @@ out: return value; } -uint8_t fy_utf8_low_ascii_flags[0x80] = { - [0x00] = F_NON_PRINT, // NUL '\0' (null character) - [0x01] = F_NON_PRINT, // SOH (start of heading) - [0x02] = F_NON_PRINT, // STX (start of text) - [0x03] = F_NON_PRINT, // ETX (end of text) - [0x04] = F_NON_PRINT, // EOT (end of transmission) - [0x05] = F_NON_PRINT, // ENQ (enquiry) - [0x06] = F_NON_PRINT, // ACK (acknowledge) - [0x07] = F_NON_PRINT | F_QUOTE_ESC, // BEL '\a' (bell) - [0x08] = F_NON_PRINT | F_QUOTE_ESC, // BS '\b' (backspace) - [0x09] = F_NON_PRINT | F_QUOTE_ESC | F_WS, // HT '\t' (horizontal tab) - [0x0A] = F_NON_PRINT | F_QUOTE_ESC | F_LB, // LF '\n' (new line) - [0x0B] = F_NON_PRINT | F_QUOTE_ESC, // VT '\v' (vertical tab) - [0x0C] = F_NON_PRINT | F_QUOTE_ESC, // FF '\f' (form feed) - [0x0D] = F_NON_PRINT | F_QUOTE_ESC | F_LB, // CR '\r' (carriage ret) - [0x0E] = F_NON_PRINT, // SO (shift out) - [0x0F] = F_NON_PRINT, // SI (shift in) - [0x10] = F_NON_PRINT, // DLE (data link escape) - [0x11] = F_NON_PRINT, // DC1 (device control 1) - [0x12] = F_NON_PRINT, // DC2 (device control 2) - [0x13] = F_NON_PRINT, // DC3 (device control 3) - [0x14] = F_NON_PRINT, // DC4 (device control 4) - [0x15] = F_NON_PRINT, // NAK (negative ack.) - [0x16] = F_NON_PRINT, // SYN (synchronous idle) - [0x17] = F_NON_PRINT, // ETB (end of trans. blk) - [0x18] = F_NON_PRINT, // CAN (cancel) - [0x19] = F_NON_PRINT, // EM (end of medium) - [0x1A] = F_NON_PRINT, // SUB (substitute) - [0x1B] = F_NON_PRINT, // ESC (escape) - [0x1C] = F_NON_PRINT, // FS (file separator) - [0x1D] = F_NON_PRINT, // GS (group separator) - [0x1E] = F_NON_PRINT, // RS (record separator) - [0x1F] = F_NON_PRINT, // US (unit separator) - [' '] = F_WS, - ['!'] = F_PUNCT, - ['"'] = F_PUNCT, - ['#'] = F_PUNCT, - ['$'] = F_PUNCT, - ['%'] = F_PUNCT, - ['&'] = F_PUNCT, - ['\''] = F_PUNCT, - ['('] = F_PUNCT, - [')'] = F_PUNCT, - ['*'] = F_PUNCT, - ['+'] = F_PUNCT, - [','] = F_PUNCT, - ['-'] = F_PUNCT, - ['.'] = F_PUNCT, - ['/'] = F_PUNCT, - ['0'] = F_DIGIT | F_SIMPLE_SCALAR, - ['1'] = F_DIGIT | F_SIMPLE_SCALAR, - ['2'] = F_DIGIT | F_SIMPLE_SCALAR, - ['3'] = F_DIGIT | F_SIMPLE_SCALAR, - ['4'] = F_DIGIT | F_SIMPLE_SCALAR, - ['5'] = F_DIGIT | F_SIMPLE_SCALAR, - ['6'] = F_DIGIT | F_SIMPLE_SCALAR, - ['7'] = F_DIGIT | F_SIMPLE_SCALAR, - ['8'] = F_DIGIT | F_SIMPLE_SCALAR, - ['9'] = F_DIGIT | F_SIMPLE_SCALAR, - [':'] = F_PUNCT, - [';'] = F_PUNCT, - ['<'] = F_PUNCT, - ['='] = F_PUNCT, - ['>'] = F_PUNCT, - ['?'] = F_PUNCT, - ['@'] = F_PUNCT, - ['A'] = F_LETTER | F_SIMPLE_SCALAR, - ['B'] = F_LETTER | F_SIMPLE_SCALAR, - ['C'] = F_LETTER | F_SIMPLE_SCALAR, - ['D'] = F_LETTER | F_SIMPLE_SCALAR, - ['E'] = F_LETTER | F_SIMPLE_SCALAR, - ['F'] = F_LETTER | F_SIMPLE_SCALAR, - ['G'] = F_LETTER | F_SIMPLE_SCALAR, - ['H'] = F_LETTER | F_SIMPLE_SCALAR, - ['I'] = F_LETTER | F_SIMPLE_SCALAR, - ['J'] = F_LETTER | F_SIMPLE_SCALAR, - ['K'] = F_LETTER | F_SIMPLE_SCALAR, - ['L'] = F_LETTER | F_SIMPLE_SCALAR, - ['M'] = F_LETTER | F_SIMPLE_SCALAR, - ['N'] = F_LETTER | F_SIMPLE_SCALAR, - ['O'] = F_LETTER | F_SIMPLE_SCALAR, - ['P'] = F_LETTER | F_SIMPLE_SCALAR, - ['Q'] = F_LETTER | F_SIMPLE_SCALAR, - ['R'] = F_LETTER | F_SIMPLE_SCALAR, - ['S'] = F_LETTER | F_SIMPLE_SCALAR, - ['T'] = F_LETTER | F_SIMPLE_SCALAR, - ['U'] = F_LETTER | F_SIMPLE_SCALAR, - ['V'] = F_LETTER | F_SIMPLE_SCALAR, - ['W'] = F_LETTER | F_SIMPLE_SCALAR, - ['X'] = F_LETTER | F_SIMPLE_SCALAR, - ['Y'] = F_LETTER | F_SIMPLE_SCALAR, - ['Z'] = F_LETTER | F_SIMPLE_SCALAR, - ['['] = F_PUNCT, - ['\\'] = F_PUNCT, // '\\' - [']'] = F_PUNCT, - ['^'] = F_PUNCT, - ['_'] = F_PUNCT | F_SIMPLE_SCALAR, - ['`'] = F_PUNCT, - ['a'] = F_LETTER | F_SIMPLE_SCALAR, - ['b'] = F_LETTER | F_SIMPLE_SCALAR, - ['c'] = F_LETTER | F_SIMPLE_SCALAR, - ['d'] = F_LETTER | F_SIMPLE_SCALAR, - ['e'] = F_LETTER | F_SIMPLE_SCALAR, - ['f'] = F_LETTER | F_SIMPLE_SCALAR, - ['g'] = F_LETTER | F_SIMPLE_SCALAR, - ['h'] = F_LETTER | F_SIMPLE_SCALAR, - ['i'] = F_LETTER | F_SIMPLE_SCALAR, - ['j'] = F_LETTER | F_SIMPLE_SCALAR, - ['k'] = F_LETTER | F_SIMPLE_SCALAR, - ['l'] = F_LETTER | F_SIMPLE_SCALAR, - ['m'] = F_LETTER | F_SIMPLE_SCALAR, - ['n'] = F_LETTER | F_SIMPLE_SCALAR, - ['o'] = F_LETTER | F_SIMPLE_SCALAR, - ['p'] = F_LETTER | F_SIMPLE_SCALAR, - ['q'] = F_LETTER | F_SIMPLE_SCALAR, - ['r'] = F_LETTER | F_SIMPLE_SCALAR, - ['s'] = F_LETTER | F_SIMPLE_SCALAR, - ['t'] = F_LETTER | F_SIMPLE_SCALAR, - ['u'] = F_LETTER | F_SIMPLE_SCALAR, - ['v'] = F_LETTER | F_SIMPLE_SCALAR, - ['w'] = F_LETTER | F_SIMPLE_SCALAR, - ['x'] = F_LETTER | F_SIMPLE_SCALAR, - ['y'] = F_LETTER | F_SIMPLE_SCALAR, - ['z'] = F_LETTER | F_SIMPLE_SCALAR, - ['{'] = F_PUNCT, - ['|'] = F_PUNCT, - ['}'] = F_PUNCT, - ['~'] = F_PUNCT, - [0x7F] = F_NON_PRINT, // DEL +const uint8_t fy_utf8_low_ascii_flags[256] = { + [0x00] = 0, // NUL '\0' (null character) + [0x01] = 0, // SOH (start of heading) + [0x02] = 0, // STX (start of text) + [0x03] = 0, // ETX (end of text) + [0x04] = 0, // EOT (end of transmission) + [0x05] = 0, // ENQ (enquiry) + [0x06] = 0, // ACK (acknowledge) + [0x07] = 0, // BEL '\a' (bell) + [0x08] = 0, // BS '\b' (backspace) + [0x09] = F_WS, // HT '\t' (horizontal tab) + [0x0A] = F_LB, // LF '\n' (new line) + [0x0B] = 0, // VT '\v' (vertical tab) + [0x0C] = 0, // FF '\f' (form feed) + [0x0D] = F_LB, // CR '\r' (carriage ret) + [0x0E] = 0, // SO (shift out) + [0x0F] = 0, // SI (shift in) + [0x10] = 0, // DLE (data link escape) + [0x11] = 0, // DC1 (device control 1) + [0x12] = 0, // DC2 (device control 2) + [0x13] = 0, // DC3 (device control 3) + [0x14] = 0, // DC4 (device control 4) + [0x15] = 0, // NAK (negative ack.) + [0x16] = 0, // SYN (synchronous idle) + [0x17] = 0, // ETB (end of trans. blk) + [0x18] = 0, // CAN (cancel) + [0x19] = 0, // EM (end of medium) + [0x1A] = 0, // SUB (substitute) + [0x1B] = 0, // ESC (escape) + [0x1C] = 0, // FS (file separator) + [0x1D] = 0, // GS (group separator) + [0x1E] = 0, // RS (record separator) + [0x1F] = 0, // US (unit separator) + [' '] = F_DIRECT_PRINT | F_WS, + ['!'] = F_DIRECT_PRINT, + ['"'] = F_DIRECT_PRINT, + ['#'] = F_DIRECT_PRINT, + ['$'] = F_DIRECT_PRINT, + ['%'] = F_DIRECT_PRINT, + ['&'] = F_DIRECT_PRINT, + ['\''] = F_DIRECT_PRINT, + ['('] = F_DIRECT_PRINT, + [')'] = F_DIRECT_PRINT, + ['*'] = F_DIRECT_PRINT, + ['+'] = F_DIRECT_PRINT, + [','] = F_DIRECT_PRINT | F_FLOW_INDICATOR, + ['-'] = F_DIRECT_PRINT, + ['.'] = F_DIRECT_PRINT, + ['/'] = F_DIRECT_PRINT, + ['0'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['1'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['2'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['3'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['4'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['5'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['6'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['7'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['8'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + ['9'] = F_DIRECT_PRINT | F_DIGIT | F_XDIGIT | F_SIMPLE_SCALAR, + [':'] = F_DIRECT_PRINT, + [';'] = F_DIRECT_PRINT, + ['<'] = F_DIRECT_PRINT, + ['='] = F_DIRECT_PRINT, + ['>'] = F_DIRECT_PRINT, + ['?'] = F_DIRECT_PRINT, + ['@'] = F_DIRECT_PRINT, + ['A'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['B'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['C'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['D'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['E'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['F'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['G'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['H'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['I'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['J'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['K'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['L'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['M'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['N'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['O'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['P'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['Q'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['R'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['S'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['T'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['U'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['V'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['W'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['X'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['Y'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['Z'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['['] = F_DIRECT_PRINT | F_FLOW_INDICATOR, + ['\\'] = F_DIRECT_PRINT, // '\\' + [']'] = F_DIRECT_PRINT | F_FLOW_INDICATOR, + ['^'] = F_DIRECT_PRINT, + ['_'] = F_DIRECT_PRINT | F_SIMPLE_SCALAR, + ['`'] = F_DIRECT_PRINT, + ['a'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['b'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['c'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['d'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['e'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['f'] = F_DIRECT_PRINT | F_LETTER | F_XDIGIT | F_SIMPLE_SCALAR, + ['g'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['h'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['i'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['j'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['k'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['l'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['m'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['n'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['o'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['p'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['q'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['r'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['s'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['t'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['u'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['v'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['w'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['x'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['y'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['z'] = F_DIRECT_PRINT | F_LETTER | F_SIMPLE_SCALAR, + ['{'] = F_DIRECT_PRINT | F_FLOW_INDICATOR, + ['|'] = F_DIRECT_PRINT, + ['}'] = F_DIRECT_PRINT | F_FLOW_INDICATOR, + ['~'] = F_DIRECT_PRINT, + [0x7F] = 0, // DEL + /* the rest are zero */ }; void *fy_utf8_split_posix(const char *str, int *argcp, const char * const *argvp[]) @@ -944,7 +941,7 @@ void *fy_utf8_split_posix(const char *str, int *argcp, const char * const *argvp break; default: - abort(); + FY_IMPOSSIBLE_ABORT(); } break; @@ -963,8 +960,7 @@ void *fy_utf8_split_posix(const char *str, int *argcp, const char * const *argvp OUTC(outc); break; default: - abort(); - break; + FY_IMPOSSIBLE_ABORT(); } s += w; } diff --git a/src/util/fy-utf8.h b/src/util/fy-utf8.h index a877c95..923dc06 100644 --- a/src/util/fy-utf8.h +++ b/src/util/fy-utf8.h @@ -25,7 +25,7 @@ extern const int8_t fy_utf8_width_table[32]; static inline int -fy_utf8_width_by_first_octet_no_table(uint8_t c) +fy_utf8_width_by_first_octet_no_table(const uint8_t c) { return (c & 0x80) == 0x00 ? 1 : (c & 0xe0) == 0xc0 ? 2 : @@ -41,25 +41,17 @@ fy_utf8_width_by_first_octet(uint8_t c) /* assumes valid utf8 character */ static inline size_t -fy_utf8_width(int c) +fy_utf8_width(const int c) { - return c < 0x80 ? 1 : - c < 0x800 ? 2 : - c < 0x10000 ? 3 : 4; + return 1 + (c >= 0x80) + (c >= 0x800) + (c >= 0x10000); } static inline bool -fy_utf8_is_valid(int c) +fy_utf8_is_valid(const int c) { return c >= 0 && !((c >= 0xd800 && c <= 0xdfff) || c >= 0x110000); } -static inline bool -fy_utf8_is_printable_ascii(int c) -{ - return c >= 0x20 && c <= 0x7e; -} - /* generic utf8 decoder (not inlined) */ int fy_utf8_get_generic(const void *ptr, size_t left, int *widthp); @@ -68,21 +60,270 @@ int fy_utf8_get_generic(const void *ptr, size_t left, int *widthp); #define FYUG_INV -2 #define FYUG_PARTIAL -3 -static inline int fy_utf8_get(const void *ptr, size_t left, int *widthp) +#define FY_UTF8_64_C(_x) ((int)(int32_t)(_x)) +#define FY_UTF8_64_W(_x) (((int)((_x) >> 32))) + +#define FY_UTF8_64_MAKE(_w, _c) ((int64_t)(((int64_t)((_w)) << 32) | (int64_t)(_c))) + +static inline FY_ALWAYS_INLINE int64_t +fy_utf8_get_branch_64(const void *ptr, size_t left) { - const uint8_t *p = ptr; + const uint8_t *s = ptr; + unsigned int a, b, c, d; + uint32_t code; - /* single byte (hot path) */ - if (left <= 0) { - *widthp = 0; + if (!left) return FYUG_EOF; + + a = s[0]; + if (a < 0x80) // 1 byte: 0xxxxxxx + return FY_UTF8_64_MAKE(1, a); + + if (left < 2) + return FYUG_PARTIAL; + + b = s[1]; + if ((a & 0xe0) == 0xc0) { // 2 bytes: 110xxxxx 10xxxxxx + if ((b & 0xc0) != 0x80) + return FYUG_INV; + + code = (int)(((a & 0x1f) << 6) | (b & 0x3f)); + if (code < 0x80) + return FYUG_INV; // overlong + + return FY_UTF8_64_MAKE(2, code); } - if (!(p[0] & 0x80)) { + if (left < 3) + return FYUG_PARTIAL; + + c = s[2]; + if ((a & 0xf0) == 0xe0) { // 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx + + if (((b | c) & 0xc0) != 0x80) + return FYUG_INV; + + code = (int)(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f)); + if (code < 0x800 || (code >= 0xd800 && code <= 0xdfff)) + return FYUG_INV; // overlong or surrogate + + return FY_UTF8_64_MAKE(3, code); + } + + if (left < 4) + return FYUG_PARTIAL; + + d = s[3]; + if ((a & 0xf8) == 0xf0) { // 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + + if (((b | c | d) & 0xc0) != 0x80) + return FYUG_INV; + + code = ((a & 0x07) << 18) | ((b & 0x3f) << 12) | + ((c & 0x3f) << 6) | (d & 0x3f); + + if (code < 0x10000 || code > 0x10ffff) + return FYUG_INV; // overlong or out of range + + return FY_UTF8_64_MAKE(4, code); + } + + return FYUG_INV; +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get_branch(const void *ptr, size_t left, int *widthp) +{ + const uint8_t *s = ptr; + unsigned int a, b, c, d; + int code; + + if (!left) + goto err_eof; + + a = s[0]; + if (a < 0x80) { // 1 byte: 0xxxxxxx + code = (int)a; *widthp = 1; - return p[0] & 0x7f; + return code; + } + + if (left < 2) + goto err_partial; + + b = s[1]; + if ((a & 0xe0) == 0xc0) { // 2 bytes: 110xxxxx 10xxxxxx + if ((b & 0xc0) != 0x80) + goto err_inv; + + code = (int)(((a & 0x1f) << 6) | (b & 0x3f)); + if (code < 0x80) + goto err_inv; // overlong + + *widthp = 2; + return code; } - return fy_utf8_get_generic(ptr, left, widthp); + + if (left < 3) + goto err_partial; + + c = s[2]; + if ((a & 0xf0) == 0xe0) { // 3 bytes: 1110xxxx 10xxxxxx 10xxxxxx + + if (((b | c) & 0xc0) != 0x80) + goto err_inv; + + code = (int)(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f)); + if (code < 0x800 || (code >= 0xd800 && code <= 0xdfff)) + goto err_inv; // overlong or surrogate + + *widthp = 3; + return code; + } + + if (left < 4) + goto err_partial; + + d = s[3]; + if ((a & 0xf8) == 0xf0) { // 4 bytes: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + + if (((b | c | d) & 0xc0) != 0x80) + goto err_inv; + + code = ((a & 0x07) << 18) | ((b & 0x3f) << 12) | + ((c & 0x3f) << 6) | (d & 0x3f); + + if (code < 0x10000 || code > 0x10ffff) + goto err_inv; // overlong or out of range + + *widthp = 4; + return code; + } + +err_inv: + *widthp = 0; + return FYUG_INV; + +err_partial: + *widthp = 0; + return FYUG_PARTIAL; + +err_eof: + *widthp = 0; + return FYUG_EOF; +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get_table(const void *ptr, size_t left, int *widthp) +{ + const uint8_t *s = ptr; + unsigned int a, b, c, d; + int code, w; + + if (!left) + goto err_eof; + + w = fy_utf8_width_by_first_octet(s[0]); + if ((size_t)w > left) + goto err_partial; + + *widthp = w; + + switch (w) { + case 1: + a = s[0]; + code = (int)a; + return code; + case 2: + a = s[0]; + b = s[1]; + if ((b & 0xc0) != 0x80) + goto err_inv; + + code = (int)(((a & 0x1f) << 6) | (b & 0x3f)); + if (code < 0x80) + goto err_inv; // overlong + + return code; + case 3: + a = s[0]; + b = s[1]; + c = s[2]; + + if (((b | c) & 0xc0) != 0x80) + goto err_inv; + + code = (int)(((a & 0x0f) << 12) | ((b & 0x3f) << 6) | (c & 0x3f)); + if (code < 0x800 || (code >= 0xd800 && code <= 0xdfff)) + goto err_inv; // overlong or surrogate + + return code; + case 4: + a = s[0]; + b = s[1]; + c = s[2]; + d = s[3]; + + if (((b | c | d) & 0xc0) != 0x80) + goto err_inv; + + code = ((a & 0x07) << 18) | ((b & 0x3f) << 12) | + ((c & 0x3f) << 6) | (d & 0x3f); + + if (code < 0x10000 || code > 0x10ffff) + goto err_inv; // overlong or out of range + + return code; + + default: + break; + } + +err_inv: + *widthp = 0; + return FYUG_INV; + +err_partial: + *widthp = 0; + return FYUG_PARTIAL; + +err_eof: + *widthp = 0; + return FYUG_EOF; +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get(const void *ptr, size_t left, int *widthp) +{ + return fy_utf8_get_branch(ptr, left, widthp); +} + +static inline FY_ALWAYS_INLINE int64_t +fy_utf8_get_64(const void *ptr, size_t left) +{ + return fy_utf8_get_branch_64(ptr, left); +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get_no_width(const void *ptr, size_t left) +{ + int w; + + return fy_utf8_get(ptr, left, &w); +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get_end(const void *ptr, const void *ptr_end, int *widthp) +{ + return fy_utf8_get(ptr, (size_t)(ptr_end - ptr), widthp); +} + +static inline FY_ALWAYS_INLINE int +fy_utf8_get_end_no_width(const void *ptr, const void *ptr_end) +{ + int w; + + return fy_utf8_get(ptr, (size_t)(ptr_end - ptr), &w); } int fy_utf8_get_right_generic(const void *ptr, size_t left, int *widthp); @@ -143,12 +384,12 @@ enum fy_utf8_escape { fyue_doublequote_yaml_1_1, }; -static inline bool fy_utf8_escape_is_any_doublequote(enum fy_utf8_escape esc) +static inline bool fy_utf8_escape_is_any_doublequote(const enum fy_utf8_escape esc) { return esc >= fyue_doublequote && esc <= fyue_doublequote_yaml_1_1; } -char *fy_utf8_format(int c, char *buf, enum fy_utf8_escape esc); +char *fy_utf8_format(int c, char *buf, const enum fy_utf8_escape esc); #define fy_utf8_format_a(_c, _esc) \ ({ \ @@ -172,7 +413,7 @@ char *fy_utf8_format_text(const char *buf, size_t len, fy_utf8_format_text(__buf, __len, _out, _outsz, __esc); \ }) -char *fy_utf8_format_text_alloc(const char *buf, size_t len, enum fy_utf8_escape esc); +char *fy_utf8_format_text_alloc(const char *buf, size_t len, const enum fy_utf8_escape esc); const void *fy_utf8_memchr_generic(const void *s, int c, size_t n); @@ -214,19 +455,19 @@ static inline int fy_utf8_count(const void *ptr, size_t len) return count; } -int fy_utf8_parse_escape(const char **strp, size_t len, enum fy_utf8_escape esc); +int fy_utf8_parse_escape(const char **strp, size_t len, const enum fy_utf8_escape esc); #define F_NONE 0 -#define F_NON_PRINT FY_BIT(0) /* non printable */ -#define F_SIMPLE_SCALAR FY_BIT(1) /* part of simple scalar */ -#define F_QUOTE_ESC FY_BIT(2) /* escape form, i.e \n */ -#define F_LB FY_BIT(3) /* is a linebreak */ -#define F_WS FY_BIT(4) /* is a whitespace */ -#define F_PUNCT FY_BIT(5) /* is a punctuation mark */ -#define F_LETTER FY_BIT(6) /* is a letter a..z A..Z */ -#define F_DIGIT FY_BIT(7) /* is a digit 0..9 */ +#define F_SIMPLE_SCALAR FY_BIT(0) /* part of simple scalar 0-9a-zA-Z_ */ +#define F_DIRECT_PRINT FY_BIT(1) /* 0x20..0x7e */ +#define F_LB FY_BIT(2) /* is a linebreak */ +#define F_WS FY_BIT(3) /* is a whitespace */ +#define F_LETTER FY_BIT(4) /* is a letter a..z A..Z */ +#define F_DIGIT FY_BIT(5) /* is a digit 0..9 */ +#define F_XDIGIT FY_BIT(6) /* is a hex digit 0..9 a-f A-F */ +#define F_FLOW_INDICATOR FY_BIT(7) /* is ,[]{} */ -extern uint8_t fy_utf8_low_ascii_flags[0x80]; +extern const uint8_t fy_utf8_low_ascii_flags[256]; void *fy_utf8_split_posix(const char *str, int *argcp, const char * const *argvp[]); @@ -286,4 +527,117 @@ static inline struct fy_utf8_result fy_utf8_get_s_res(const void *ptr, const voi return (struct fy_utf8_result){ c, width }; } +static inline bool fy_utf8_is_space(const int c) +{ + return c == ' '; +} + +static inline bool fy_utf8_is_tab(const int c) +{ + return c == '\t'; +} + +static inline bool fy_utf8_is_simple_scalar_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_SIMPLE_SCALAR; +} + +static inline bool +fy_utf8_is_printable_ascii_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_DIRECT_PRINT; +} + +static inline bool fy_utf8_is_lb_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_LB; +} + +static inline bool fy_utf8_is_ws_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_WS; +} + +static inline bool fy_utf8_is_ws_lb_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & (F_WS | F_LB); +} + +static inline bool fy_utf8_is_letter_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_LETTER; +} + +static inline bool fy_utf8_is_digit_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_DIGIT; +} + +static inline bool fy_utf8_is_hex_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_XDIGIT; +} + +static inline bool fy_utf8_is_flow_indicator_no_check(const int c) +{ + return fy_utf8_low_ascii_flags[(uint8_t)c] & F_FLOW_INDICATOR; +} + +static inline bool fy_utf8_is_low_ascii(const int c) +{ + return (unsigned int)c < 128; +} + +static inline bool fy_utf8_is_simple_scalar(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_simple_scalar_no_check(c); +} + +static inline bool +fy_utf8_is_printable_ascii_x(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_printable_ascii_no_check(c); +} + +static inline bool +fy_utf8_is_printable_ascii(const int c) +{ + return c >= 0x20 && c <= 0x7e; +} + +static inline bool fy_utf8_is_lb(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_lb_no_check(c); +} + +static inline bool fy_utf8_is_ws(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_ws_no_check(c); +} + +static inline bool fy_utf8_is_ws_lb(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_ws_lb_no_check(c); +} + +static inline bool fy_utf8_is_letter(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_letter_no_check(c); +} + +static inline bool fy_utf8_is_digit(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_digit_no_check(c); +} + +static inline bool fy_utf8_is_hex(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_hex_no_check(c); +} + +static inline bool fy_utf8_is_flow_indicator(const int c) +{ + return fy_utf8_is_low_ascii(c) && fy_utf8_is_flow_indicator_no_check(c); +} + #endif diff --git a/src/util/fy-utils.c b/src/util/fy-utils.c index 6b45f15..fa4cff8 100644 --- a/src/util/fy-utils.c +++ b/src/util/fy-utils.c @@ -26,6 +26,8 @@ #include "fy-ctype.h" #include "fy-utils.h" +#include "xxhash.h" + #if defined(__APPLE__) && (_POSIX_C_SOURCE < 200809L) /* @@ -295,6 +297,9 @@ int fy_tag_handle_length(const char *data, size_t len) s += w; c = fy_utf8_get(s, e - s, &w); + if (c == -1) + return len; + if (fy_is_ws(c)) return s - data; /* if first character is !, empty handle */ @@ -364,7 +369,7 @@ int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info) /* either !suffix or !handle!suffix */ /* we scan back to back, and split handle/suffix */ handle_length = fy_tag_handle_length(s, e - s); - if (handle_length <= 0) + if (handle_length < 0) return -1; s += handle_length; } @@ -875,3 +880,21 @@ void fy_keyword_iter_end(struct fy_keyword_iter *iter) { /* nothing */ } + +#define FY_XXHASH64_SEED ((uint64_t)0x1973198120142019U) + +uint64_t fy_iovec_xxhash64(const struct iovec *iov, int iovcnt) +{ + XXH64_state_t xxstate; + uint64_t hash; + int i; + + XXH64_reset(&xxstate, FY_XXHASH64_SEED); + for (i = 0; i < iovcnt; i++) + XXH64_update(&xxstate, iov[i].iov_base, iov[i].iov_len); + hash = XXH64_digest(&xxstate); + /* XXX we never return a hash value of zero */ + if (!hash) + hash++; + return hash; +} diff --git a/src/util/fy-utils.h b/src/util/fy-utils.h index 20758b2..8e5fa4e 100644 --- a/src/util/fy-utils.h +++ b/src/util/fy-utils.h @@ -15,8 +15,29 @@ #include #include +#include #include #include +#include +#include +#include +#include +#include + +/* + * Define FY_CPP_SHORT_NAMES to use shortened macro names in the implementation. + * This significantly reduces expansion buffer usage during recursive macro expansion, + * which can help avoid internal compiler errors and speed up compilation. + * + * The public API (FY_CPP_*) remains the same regardless of this setting. + */ + +#define FY_CPP_SHORT_NAMES + +/* to avoid dragging in libfyaml.h */ +#ifndef FY_BIT +#define FY_BIT(x) (1U << (x)) +#endif #if defined(__linux__) #include @@ -40,11 +61,19 @@ struct fy_tag_scan_info { int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info); +#ifndef container_of +#define container_of(ptr, type, member) \ + ({ \ + const __typeof__(((type *)0)->member) *__mptr = (ptr); \ + (type *)((void *)__mptr - offsetof(type,member)); \ + }) +#endif + #ifndef ARRAY_SIZE #define ARRAY_SIZE(x) ((sizeof(x)/sizeof((x)[0]))) #endif -#if !defined(NDEBUG) && (defined(__GNUC__) && __GNUC__ >= 4) +#if defined(NDEBUG) && (defined(__GNUC__) && __GNUC__ >= 4) #define FY_ALWAYS_INLINE __attribute__((always_inline)) #else #define FY_ALWAYS_INLINE /* nothing */ @@ -56,6 +85,43 @@ int fy_tag_scan(const char *data, size_t len, struct fy_tag_scan_info *info); #define FY_UNUSED /* nothing */ #endif +#if defined(NDEBUG) && defined(__GNUC__) && __GNUC__ >= 4 +#define FY_DEBUG_UNUSED __attribute__((unused)) +#else +#define FY_DEBUG_UNUSED /* nothing */ +#endif + +#if defined(__GNUC__) && __GNUC__ >= 4 +#define FY_CONSTRUCTOR __attribute__((constructor)) +#define FY_DESTRUCTOR __attribute__((destructor)) +#define FY_HAS_CONSTRUCTOR +#define FY_HAS_DESTRUCTOR +#else +#define FY_CONSTRUCTOR /* nothing */ +#define FY_DESTRUCTOR /* nothing */ +#endif + +#if defined(FY_DESTRUCTOR) && defined(NDEBUG) +#define FY_DESTRUCTOR_SHOW_LEFTOVERS +#endif + +/* something impossible is happening, assert/abort */ +#define FY_IMPOSSIBLE_ABORT() \ + do { \ + assert(0); \ + abort(); \ + } while(0) + +static inline void *fy_ptr_align(void *p, size_t align) +{ + return (void *)(((uintptr_t)p + (align - 1)) & ~(align - 1)); +} + +static inline size_t fy_size_t_align(size_t size, size_t align) +{ + return (size + (align - 1)) & ~(align - 1); +} + int fy_term_set_raw(int fd, struct termios *oldt); int fy_term_restore(int fd, const struct termios *oldt); ssize_t fy_term_write(int fd, const void *data, size_t count); @@ -106,4 +172,448 @@ void fy_keyword_iter_end(struct fy_keyword_iter *iter); #define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) #endif +/* if expression is zero, then the build will break */ +#define FY_COMPILE_ERROR_ON_ZERO(_e) ((void)(sizeof(char[1 - 2*!!(_e)]))) + +/* make things fail easy on non GCC/clang */ +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +/* true, if types are the same, false otherwise (depends on builtin_types_compatible_p) */ +#if __has_builtin(__builtin_types_compatible_p) +#define FY_SAME_TYPE(_a, _b) __builtin_types_compatible_p(__typeof__(_a), __typeof__(_b)) +#else +#define FY_SAME_TYPE(_a, _b) true +#endif + +/* compile error if types are not the same */ +#define FY_CHECK_SAME_TYPE(_a, _b) \ + FY_COMPILE_ERROR_ON_ZERO(!FY_SAME_TYPE(_a, _b)) + +/* type safe add overflow */ +#if __has_builtin(__builtin_add_overflow) +#define FY_ADD_OVERFLOW __builtin_add_overflow +#else +#define FY_ADD_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + FY_CHECK_SAME_TYPE(__a, __b); \ + \ + __res = __a + __b; \ + /* overflow when signs of a, b same, but results different */ \ + __overflow = ((__a ^ __result) & (__b & __result)) < 0; \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +/* type safe sub overflow */ +#if __has_builtin(__builtin_sub_overflow) +#define FY_SUB_OVERFLOW __builtin_sub_overflow +#else +#define FY_SUB_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + FY_CHECK_SAME_TYPE(__a, __b); \ + \ + __res = __a - __b; \ + /* overflow when signs of a, b differ, but results different from minuend */ \ + __overflow = ((__a ^ __b) & (__a & __result)) < 0; \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +/* type safe multiply overflow */ +#if __has_builtin(__builtin_mul_overflow) +#define FY_MUL_OVERFLOW __builtin_mul_overflow +#else +#define FY_MUL_OVERFLOW(_a, _b, _resp) \ +({ \ + __typeof__(_a) __a = (_a), __res; \ + __typeof__(_b) __b = (_b); \ + bool __overflow; \ + \ + FY_CHECK_SAME_TYPE(__a, __b); \ + \ + if (!__a || !__b) { \ + __overflow = false; \ + __res = 0; \ + } else { \ + __res = __a * __b; \ + /* overflow when division of the result differs */ \ + __overflow = (__res / __a) != __b; \ + } \ + *(_resp) = __res; \ + __overflow; \ +}) +#endif + +static inline void fy_strip_trailing_nl(char *str) +{ + char *s; + + if (str) { + s = str + strlen(str); + while (s > str && s[-1] == '\n') + *--s = '\0'; + } +} + +/* alloca formatted print methods */ +#define fy_vsprintfa(_fmt, _ap) \ + ({ \ + const char *__fmt = (_fmt); \ + va_list _ap_orig; \ + int _size; \ + char *_buf; \ + \ + va_copy(_ap_orig, (_ap)); \ + _size = vsnprintf(NULL, 0, __fmt, _ap_orig); \ + va_end(_ap_orig); \ + _buf = alloca(_size + 1); \ + (void)vsnprintf(_buf, _size + 1, __fmt, (_ap)); \ + _buf; \ + }) + +#define fy_sprintfa(_fmt, ...) \ + ({ \ + const char *__fmt = (_fmt); \ + int _size; \ + char *_buf; \ + \ + _size = snprintf(NULL, 0, __fmt, ## __VA_ARGS__); \ + _buf = alloca(_size + 1); \ + (void)snprintf(_buf, _size + 1, __fmt, __VA_ARGS__); \ + _buf; \ + }) + +#if !defined(NDEBUG) && defined(HAVE_DEVMODE) && HAVE_DEVMODE +#define FY_DEVMODE +#else +#undef FY_DEVMODE +#endif + +#ifdef FY_DEVMODE +#define __FY_DEBUG_UNUSED__ /* nothing */ +#else +#if defined(__GNUC__) && __GNUC__ >= 4 +#define __FY_DEBUG_UNUSED__ __attribute__((__unused__)) +#else +#define __FY_DEBUG_UNUSED__ /* nothing */ +#endif +#endif + +// C preprocessor magic follows (pilfered and adapted from h4x0r.org) + +/* applies an expansion over a varargs list */ + +#ifdef FY_CPP_SHORT_NAMES + +/* Short-name implementation - reduces expansion buffer usage by ~60% */ + +#define _E1(...) __VA_ARGS__ +#define _E2(...) _E1(_E1(__VA_ARGS__)) +#define _E4(...) _E2(_E2(__VA_ARGS__)) +#define _E8(...) _E4(_E4(__VA_ARGS__)) +#define _E16(...) _E8(_E8(__VA_ARGS__)) +#define _E32(...) _E16(_E16(__VA_ARGS__)) +#define _E(...) _E32(_E32(__VA_ARGS__)) + +#define _FMT() +#define _FP1(m) m _FMT() +#define _FP2(a, m) m _FMT() + +#define _F1(_1, ...) _1 +#define _F1F(...) _F1(__VA_OPT__(__VA_ARGS__ ,) 0) + +#define _F2(_1, _2, ...) _2 +#define _F2F(...) _F2(__VA_OPT__(__VA_ARGS__ ,) 0, 0) + +#define _F3(_1, _2, _3, ...) _3 +#define _F3F(...) _F3(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0) + +#define _F4(_1, _2, _3, _4, ...) _4 +#define _F4F(...) _F4(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0) + +#define _F5(_1, _2, _3, _4, _5, ...) _5 +#define _F5F(...) \ + _F5(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0, 0) + +#define _F6(_1, _2, _3, _4, _5, _6, ...) _6 +#define _F6F(...) \ + _F6(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0, 0, 0) + +#define _FR(x, ...) __VA_ARGS__ +#define _FRF(...) __VA_OPT__(_FR(__VA_ARGS__)) + +#define _FM(m, ...) \ + __VA_OPT__(_E(_FM1(m, __VA_ARGS__))) +#define _FM1(m, x, ...) m(x) \ + __VA_OPT__(_FP1(_FMI)()(m, __VA_ARGS__)) +#define _FMI() _FM1 + +#define _FCB(x) +1 +#define _FVC(...) (__VA_OPT__((_FM(_FCB, __VA_ARGS__) + 0)) + 0) + +#define _FI1(a) a +#define _FIL(a) , _FI1(a) +#define _FILS(...) _FM(_FIL, __VA_ARGS__) + +#define _FVIS(...) \ + _FI1(_F1F(__VA_ARGS__)) \ + _FILS(_FRF(__VA_ARGS__)) + +#define _FVISF(_t, ...) \ + ((_t [_FVC(__VA_ARGS__)]) { _FVIS(__VA_ARGS__) }) + +#define _FM2(a, m, ...) \ + __VA_OPT__(_E(_FM12(a, m, __VA_ARGS__))) + +#define _FM12(a, m, x, ...) m(a, x) \ + __VA_OPT__(_FP2(a, _FMI2)()(a, m, __VA_ARGS__)) +#define _FMI2() _FM12 + +/* Public API - maps to short names */ +#define FY_CPP_EVAL1 _E1 +#define FY_CPP_EVAL2 _E2 +#define FY_CPP_EVAL4 _E4 +#define FY_CPP_EVAL8 _E8 +#if !defined(__clang__) +#define FY_CPP_EVAL16 _E16 +#endif +#define FY_CPP_EVAL _E +#define FY_CPP_EMPTY _FMT +#define FY_CPP_POSTPONE1 _FP1 +#define FY_CPP_POSTPONE2 _FP2 + +#define FY_CPP_FIRST _F1F +#define FY_CPP_SECOND _F2F +#define FY_CPP_THIRD _F3F +#define FY_CPP_FOURTH _F4F +#define FY_CPP_FIFTH _F5F +#define FY_CPP_SIXTH _F6F +#define FY_CPP_REST _FRF + +#define FY_CPP_MAP _FM +#define _FY_CPP_MAP_ONE _FM1 +#define _FY_CPP_MAP_INDIRECT _FMI +#define _FY_CPP_COUNT_BODY _FCB +#define FY_CPP_VA_COUNT _FVC +#define _FY_CPP_ITEM_ONE _FI1 +#define _FY_CPP_ITEM_LATER_ARG _FIL +#define _FY_CPP_ITEM_LIST _FILS +#define _FY_CPP_VA_ITEMS _FVIS +#define FY_CPP_VA_ITEMS _FVISF +#define FY_CPP_MAP2 _FM2 +#define _FY_CPP_MAP_ONE2 _FM12 +#define _FY_CPP_MAP_INDIRECT2 _FMI2 + +#else /* !FY_CPP_SHORT_NAMES */ + +/* Original long-name implementation */ + +#define FY_CPP_EVAL1(...) __VA_ARGS__ +#define FY_CPP_EVAL2(...) FY_CPP_EVAL1(FY_CPP_EVAL1(__VA_ARGS__)) +#define FY_CPP_EVAL4(...) FY_CPP_EVAL2(FY_CPP_EVAL2(__VA_ARGS__)) +#define FY_CPP_EVAL8(...) FY_CPP_EVAL4(FY_CPP_EVAL4(__VA_ARGS__)) +#if !defined(__clang__) +// gcc is better, goes to 16 +#define FY_CPP_EVAL16(...) FY_CPP_EVAL8(FY_CPP_EVAL8(__VA_ARGS__)) +#define FY_CPP_EVAL(...) FY_CPP_EVAL16(FY_CPP_EVAL16(__VA_ARGS__)) +#else +// clang craps out at 8 +#define FY_CPP_EVAL(...) FY_CPP_EVAL8(FY_CPP_EVAL8(__VA_ARGS__)) +#endif + +#define FY_CPP_EMPTY() +#define FY_CPP_POSTPONE1(macro) macro FY_CPP_EMPTY() + +#define FY_CPP_MAP(macro, ...) \ + __VA_OPT__(FY_CPP_EVAL(_FY_CPP_MAP_ONE(macro, __VA_ARGS__))) +#define _FY_CPP_MAP_ONE(macro, x, ...) macro(x) \ + __VA_OPT__(FY_CPP_POSTPONE1(_FY_CPP_MAP_INDIRECT)()(macro, __VA_ARGS__)) +#define _FY_CPP_MAP_INDIRECT() _FY_CPP_MAP_ONE + +#define FY_CPP_POSTPONE2(a, macro) macro FY_CPP_EMPTY() + +#define _FY_CPP_FIRST(_1, ...) _1 +#define FY_CPP_FIRST(...) _FY_CPP_FIRST(__VA_OPT__(__VA_ARGS__ ,) 0) + +#define _FY_CPP_SECOND(_1, _2, ...) _2 +#define FY_CPP_SECOND(...) _FY_CPP_SECOND(__VA_OPT__(__VA_ARGS__ ,) 0, 0) + +#define _FY_CPP_THIRD(_1, _2, _3, ...) _3 +#define FY_CPP_THIRD(...) _FY_CPP_THIRD(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0) + +#define _FY_CPP_FOURTH(_1, _2, _3, _4, ...) _4 +#define FY_CPP_FOURTH(...) _FY_CPP_FOURTH(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0) + +#define _FY_CPP_FIFTH(_1, _2, _3, _4, _5, ...) _5 +#define FY_CPP_FIFTH(...) \ + _FY_CPP_FIFTH(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0, 0) + +#define _FY_CPP_SIXTH(_1, _2, _3, _4, _5, _6, ...) _6 +#define FY_CPP_SIXTH(...) \ + _FY_CPP_SIXTH(__VA_OPT__(__VA_ARGS__ ,) 0, 0, 0, 0, 0, 0) + +#define _FY_CPP_REST(x, ...) __VA_ARGS__ +#define FY_CPP_REST(...) __VA_OPT__(_FY_CPP_REST(__VA_ARGS__)) + +#define _FY_CPP_COUNT_BODY(x) +1 +#define FY_CPP_VA_COUNT(...) (__VA_OPT__((FY_CPP_MAP(_FY_CPP_COUNT_BODY, __VA_ARGS__) + 0)) + 0) + +#define _FY_CPP_ITEM_ONE(arg) arg +#define _FY_CPP_ITEM_LATER_ARG(arg) , _FY_CPP_ITEM_ONE(arg) +#define _FY_CPP_ITEM_LIST(...) FY_CPP_MAP(_FY_CPP_ITEM_LATER_ARG, __VA_ARGS__) + +#define _FY_CPP_VA_ITEMS(...) \ + _FY_CPP_ITEM_ONE(FY_CPP_FIRST(__VA_ARGS__)) \ + _FY_CPP_ITEM_LIST(FY_CPP_REST(__VA_ARGS__)) + +#define FY_CPP_VA_ITEMS(_type, ...) \ + ((_type [FY_CPP_VA_COUNT(__VA_ARGS__)]) { _FY_CPP_VA_ITEMS(__VA_ARGS__) }) + +#define FY_CPP_MAP2(a, macro, ...) \ + __VA_OPT__(FY_CPP_EVAL(_FY_CPP_MAP_ONE2(a, macro, __VA_ARGS__))) + +#define _FY_CPP_MAP_ONE2(a, macro, x, ...) macro(a, x) \ + __VA_OPT__(FY_CPP_POSTPONE2(a, _FY_CPP_MAP_INDIRECT2)()(a, macro, __VA_ARGS__)) +#define _FY_CPP_MAP_INDIRECT2() _FY_CPP_MAP_ONE2 + +#endif /* FY_CPP_SHORT_NAMES */ + +/* + * example usage: + * + * int do_sum(int count, int *items) + * { + * int i; + * int sum; + * + * sum = 0; + * for (i = 0; i < count; i++) + * sum += items[i]; + * + * return sum; + * } + * + * #define do_sum_macro(...) \ + * do_sum( \ + * FY_CPP_VA_COUNT(__VA_ARGS__), \ + * FY_CPP_VA_ITEMS(int, __VA_ARGS__)) + * + * sum = do_sum_macro(1, 2, 5, 100); + * + * will expand to: + * + * sum = do_sum(4, ((int [4]){ 1, 2, 5, 100 })); + * + * printf("sum=%d\n", sum); + */ + +#define fy_alloca_align(_sz, _align) \ + ({ \ + const size_t __sz = (_sz); \ + const size_t __align = (_align); \ + void *__p; \ + __p = __align <= sizeof(max_align_t) ? alloca(__sz) : fy_ptr_align(alloca(__sz + __align - 1), __align); \ + __p; \ + }) + +static inline size_t +fy_iovec_size(const struct iovec *iov, int iovcnt) +{ + size_t size; + int i; + + size = 0; + for (i = 0; i < iovcnt; i++) { + if (FY_ADD_OVERFLOW(size, iov[i].iov_len, &size)) + return SIZE_MAX; + } + return size; +} + +static inline void * +fy_iovec_copy_from(const struct iovec *iov, int iovcnt, void *dst) +{ + size_t size; + int i; + + for (i = 0; i < iovcnt; i++, dst += size) { + size = iov[i].iov_len; + memcpy(dst, iov[i].iov_base, size); + } + return dst; +} + +static inline const void * +fy_iovec_copy_to(const struct iovec *iov, int iovcnt, const void *src) +{ + size_t size; + int i; + + for (i = 0; i < iovcnt; i++, src += size) { + size = iov[i].iov_len; + memcpy(iov[i].iov_base, src, size); + } + return src; +} + +static inline int +fy_iovec_cmp(const struct iovec *iov, int iovcnt, const void *data) +{ + const void *s = data; + size_t size; + int i, ret; + + for (i = 0; i < iovcnt; i++) { + size = iov[i].iov_len; + ret = memcmp(iov[i].iov_base, s, size); + if (ret) + return ret; + s += size; + } + return 0; +} + +/* safe bet for 2025 */ +#define FY_CACHE_LINE_SZ 64 +#define FY_CACHE_ALIGNED __attribute__((aligned(FY_CACHE_LINE_SZ))) + +uint64_t fy_iovec_xxhash64(const struct iovec *iov, int iovcnt); + +#define FY_CONCAT(_a, _b) FY_CONCAT_INNER(_a, _b) +#define FY_CONCAT_INNER(_a, _b) _a ## _b +#define FY_UNIQUE(_base) FY_CONCAT(_base, __COUNTER__) +#define FY_LUNIQUE(_base) FY_CONCAT(_base, __LINE__) + +#if __has_builtin(__builtin_stack_save) && __has_builtin(__builtin_stack_restore) +#define FY_STACK_SAVE() __builtin_stack_save() +#define FY_STACK_RESTORE(_x) __builtin_stack_restore((_x)) +#else +/* no builtin? just ignore */ +#define FY_STACK_SAVE() ((void *)NULL) +#define FY_STACK_RESTORE(_x) do { (void)(_x); } while(0) +#endif + +/* possible have lambdas, either gcc or clang with block support */ +#ifdef __clang__ +#ifdef __BLOCKS__ +#define FY_HAVE_LAMBDAS +#define FY_HAVE_BLOCK_LAMBDAS +#endif +#elif defined(__GNUC__) +#define FY_HAVE_LAMBDAS +#define FY_HAVE_NESTED_FUNC_LAMBDAS +#endif + #endif diff --git a/src/util/fy-vlsize.h b/src/util/fy-vlsize.h new file mode 100644 index 0000000..568407d --- /dev/null +++ b/src/util/fy-vlsize.h @@ -0,0 +1,564 @@ +/* + * fy-vlsize.h - variable length size encoding + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ +#ifndef FY_VLSIZE_H +#define FY_VLSIZE_H + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +// size encoding for 64 bit +// +// high bit is set, more follow with 0 ending the run +// The final 9th byte always terminates the run +// +// 0 1 2 3 4 5 6 7 8 +// -- -- -- -- -- -- -- -- -- +// 0k 7 bit length k +// 1k 0l 14 bit length kl +// 1k 1l 0m 21 bit length klm +// 1k 1l 1m 0n 28 bit length klmn +// 1k 1l 1m 1n 0o 35 bit length klmno +// 1k 1l 1m 1n 1o 0p 42 bit length klmnop +// 1k 1l 1m 1n 1o 1p 0q 49 bit length klmnopq +// 1k 1l 1m 1n 1o 1p 1q 0r 56 bit length klmnopqr +// 1k 1l 1m 1n 1o 1p 1q 1r t 64 bit length klmnopqrt +// +// size encoding for 32 bit +// +// high bit is set, more follow with 0 ending the run +// The final 9th byte always terminates the run +// +// 0 1 2 3 4 5 6 7 8 +// -- -- -- -- -- -- -- -- -- +// 0k 7 bit length k +// 1k 0l 14 bit length kl +// 1k 1l 0m 21 bit length klm +// 1k 1l 1m 0n 28 bit length klmn +// 1xk 1l 1m 1n o 32 bit length klmno 4 high bits ignored (at byte 0) +// + +#define FYVL_SIZE_ENCODING_MAX_64 9 // 7 * 8 + 8 = 64 bits +#define FYVL_SIZE_ENCODING_MAX_32 5 // 7 * 4 + 4 = 32 bits + +/* 32 bit specific */ +static inline unsigned int +fy_encode_size32_bytes(uint32_t size) +{ + if (size < ((uint32_t)1 << 7)) + return 1; + if (size < ((uint32_t)1 << 14)) + return 2; + if (size < ((uint32_t)1 << 21)) + return 3; + if (size < ((uint32_t)1 << 28)) + return 4; + return 5; +} + +static inline uint8_t * +fy_encode_size32(uint8_t *p, uint32_t bufsz, uint32_t size) +{ + uint8_t *end = p + bufsz; + + if (size < ((uint32_t)1 << 7)) { + if (p + 1 > end) + return NULL; + p[0] = (uint8_t)size; + return p + 1; + } + if (size < ((uint32_t)1 << 14)) { + if (p + 2 > end) + return NULL; + p[0] = (uint8_t)(size >> 7) | 0x80; + p[1] = (uint8_t)size & 0x7f; + return p + 2; + } + if (size < ((uint32_t)1 << 21)) { + if (p + 3 > end) + return NULL; + p[0] = (uint8_t)(size >> 14) | 0x80; + p[1] = (uint8_t)(size >> 7) | 0x80; + p[2] = (uint8_t)size & 0x7f; + return p + 3; + } + if (size < ((uint32_t)1 << 28)) { + if (p + 4 > end) + return NULL; + p[0] = (uint8_t)(size >> 21) | 0x80; + p[1] = (uint8_t)(size >> 14) | 0x80; + p[2] = (uint8_t)(size >> 7) | 0x80; + p[3] = (uint8_t)size & 0x7f; + return p + 4; + } + if (p + 5 > end) + return NULL; + p[0] = (uint8_t)(size >> 29) | 0x80; + p[1] = (uint8_t)(size >> 22) | 0x80; + p[2] = (uint8_t)(size >> 15) | 0x80; + p[3] = (uint8_t)(size >> 8) | 0x80; + p[4] = (uint8_t)size; + return p + 5; +} + +static inline const uint8_t * +fy_decode_size32(const uint8_t *start, size_t bufsz, uint32_t *sizep) +{ + const uint8_t *p, *end, *end_scan, *end_max_scan; + uint32_t size; + + end = start + bufsz; + + end_max_scan = start + FYVL_SIZE_ENCODING_MAX_32; + + if (end_max_scan < end) + end_scan = end_max_scan; + else + end_scan = end; + + end_max_scan--; + + p = start; + size = 0; + while (p < end_scan) { + if (p < end_max_scan) { + size <<= 7; + size |= (*p & 0x7f); + if (!(*p & 0x80)) + goto done; + } else { + /* last one is always the full 8 bit */ + size <<= 8; + size |= *p; + goto done; + } + p++; + } + + *sizep = (uint32_t)-1; + return NULL; +done: + if (++p >= end) + p = end; + *sizep = size; + return p; +} + +static inline const uint8_t * +fy_decode_size32_nocheck(const uint8_t *start, uint64_t *sizep) +{ + const uint8_t *p; + size_t size; + unsigned int i; + + p = start; + size = 0; + for (i = 0; i < FYVL_SIZE_ENCODING_MAX_32-1; i++) { + size <<= 7; + size |= (p[i] & 0x7f); + if ((int8_t)p[i] >= 0) + goto out; + } + size <<= 8; + size |= p[i]; +out: + *sizep = size; + return p + i + 1; +} + +static inline const uint8_t * +fy_skip_size32(const uint8_t *start, size_t bufsz) +{ + const uint8_t *p, *end, *end_scan, *end_max_scan; + + end = start + bufsz; + + end_max_scan = start + FYVL_SIZE_ENCODING_MAX_32; + + if (end_max_scan < end) + end_scan = end_max_scan; + else + end_scan = end; + + end_max_scan--; + + p = start; + while (p < end_scan) { + if (p < end_max_scan) { + if (!(*p & 0x80)) + goto done; + } else + goto done; + p++; + } + + return NULL; +done: + if (++p >= end) + p = end; + return p; +} + +static inline const uint8_t * +fy_skip_size32_nocheck(const uint8_t *p) +{ + unsigned int i; + + for (i = 0; i < FYVL_SIZE_ENCODING_MAX_32; ) { + if ((int8_t)p[i++] >= 0) + break; + } + return p + i; +} + +static inline unsigned int +fy_encode_size64_bytes(uint64_t size) +{ + if (size < ((uint64_t)1 << 7)) + return 1; + if (size < ((uint64_t)1 << 14)) + return 2; + if (size < ((uint64_t)1 << 21)) + return 3; + if (size < ((uint64_t)1 << 28)) + return 4; + if (size < ((uint64_t)1 << 35)) + return 5; + if (size < ((uint64_t)1 << 42)) + return 6; + if (size < ((uint64_t)1 << 49)) + return 7; + if (size < ((uint64_t)1 << 56)) + return 8; + return 9; +} + +static inline uint8_t * +fy_encode_size64(uint8_t *p, size_t bufsz, uint64_t size) +{ + uint8_t *end = p + bufsz; + + if (size < ((uint64_t)1 << 7)) { + if (p + 1 > end) + return NULL; + p[0] = (uint8_t)size; + return p + 1; + } + if (size < ((uint64_t)1 << 14)) { + if (p + 2 > end) + return NULL; + p[0] = (uint8_t)(size >> 7) | 0x80; + p[1] = (uint8_t)size & 0x7f; + return p + 2; + } + if (size < ((uint64_t)1 << 21)) { + if (p + 3 > end) + return NULL; + p[0] = (uint8_t)(size >> 14) | 0x80; + p[1] = (uint8_t)(size >> 7) | 0x80; + p[2] = (uint8_t)size & 0x7f; + return p + 3; + } + if (size < ((uint64_t)1 << 28)) { + if (p + 4 > end) + return NULL; + p[0] = (uint8_t)(size >> 21) | 0x80; + p[1] = (uint8_t)(size >> 14) | 0x80; + p[2] = (uint8_t)(size >> 7) | 0x80; + p[3] = (uint8_t)size & 0x7f; + return p + 4; + } + if (size < ((uint64_t)1 << 35)) { + if (p + 5 > end) + return NULL; + p[0] = (uint8_t)(size >> 28) | 0x80; + p[1] = (uint8_t)(size >> 21) | 0x80; + p[2] = (uint8_t)(size >> 14) | 0x80; + p[3] = (uint8_t)(size >> 7) | 0x80; + p[4] = (uint8_t)size & 0x7f; + return p + 5; + } + if (size < ((uint64_t)1 << 42)) { + if (p + 6 > end) + return NULL; + p[0] = (uint8_t)(size >> 35) | 0x80; + p[1] = (uint8_t)(size >> 28) | 0x80; + p[2] = (uint8_t)(size >> 21) | 0x80; + p[3] = (uint8_t)(size >> 14) | 0x80; + p[4] = (uint8_t)(size >> 7) | 0x80; + p[5] = (uint8_t)size & 0x7f; + return p + 6; + } + if (size < ((uint64_t)1 << 49)) { + if (p + 7 > end) + return NULL; + p[0] = (uint8_t)(size >> 42) | 0x80; + p[1] = (uint8_t)(size >> 35) | 0x80; + p[2] = (uint8_t)(size >> 28) | 0x80; + p[3] = (uint8_t)(size >> 21) | 0x80; + p[4] = (uint8_t)(size >> 14) | 0x80; + p[5] = (uint8_t)(size >> 7) | 0x80; + p[6] = (uint8_t)size & 0x7f; + return p + 7; + } + if (size < ((uint64_t)1 << 56)) { + if (p + 8 > end) + return NULL; + p[0] = (uint8_t)(size >> 49) | 0x80; + p[1] = (uint8_t)(size >> 42) | 0x80; + p[2] = (uint8_t)(size >> 35) | 0x80; + p[3] = (uint8_t)(size >> 28) | 0x80; + p[4] = (uint8_t)(size >> 21) | 0x80; + p[5] = (uint8_t)(size >> 14) | 0x80; + p[6] = (uint8_t)(size >> 7) | 0x80; + p[7] = (uint8_t)size & 0x7f; + return p + 8; + } + if (p + 9 > end) + return NULL; + p[0] = (uint8_t)(size >> 57) | 0x80; + p[1] = (uint8_t)(size >> 50) | 0x80; + p[2] = (uint8_t)(size >> 43) | 0x80; + p[3] = (uint8_t)(size >> 36) | 0x80; + p[4] = (uint8_t)(size >> 29) | 0x80; + p[5] = (uint8_t)(size >> 22) | 0x80; + p[6] = (uint8_t)(size >> 15) | 0x80; + p[7] = (uint8_t)(size >> 8) | 0x80; + p[8] = (uint8_t)size; + return p + 9; +} + +static inline const uint8_t * +fy_decode_size64(const uint8_t *start, size_t bufsz, uint64_t *sizep) +{ + const uint8_t *p, *end, *end_scan, *end_max_scan; + size_t size; + + end = start + bufsz; + + end_max_scan = start + FYVL_SIZE_ENCODING_MAX_64; + + if (end_max_scan < end) + end_scan = end_max_scan; + else + end_scan = end; + + end_max_scan--; + + p = start; + size = 0; + while (p < end_scan) { + if (p < end_max_scan) { + size <<= 7; + size |= (*p & 0x7f); + if (!(*p & 0x80)) + goto done; + } else { + /* last one is always 8 bit */ + size <<= 8; + size |= *p; + goto done; + } + p++; + } + + *sizep = (size_t)-1; + return NULL; +done: + if (++p >= end) + p = end; + *sizep = size; + return p; +} + +static inline const uint8_t * +fy_decode_size64_nocheck(const uint8_t *start, uint64_t *sizep) +{ + const uint8_t *p; + size_t size; + unsigned int i; + + p = start; + size = 0; + for (i = 0; i < FYVL_SIZE_ENCODING_MAX_64-1; i++) { + size <<= 7; + size |= (p[i] & 0x7f); + if ((int8_t)p[i] >= 0) + goto out; + } + size <<= 8; + size |= p[i]; +out: + *sizep = size; + return p + i + 1; +} + +static inline const uint8_t * +fy_skip_size64(const uint8_t *start, size_t bufsz) +{ + const uint8_t *p, *end, *end_scan, *end_max_scan; + + end = start + bufsz; + + end_max_scan = start + FYVL_SIZE_ENCODING_MAX_64; + + if (end_max_scan < end) + end_scan = end_max_scan; + else + end_scan = end; + + end_max_scan--; + + p = start; + while (p < end_scan) { + if (p < end_max_scan) { + if (!(*p & 0x80)) + goto done; + } else + goto done; + p++; + } + + return NULL; +done: + if (++p >= end) + p = end; + return p; +} + +static inline const uint8_t * +fy_skip_size64_nocheck(const uint8_t *p) +{ + unsigned int i; + + for (i = 0; i < FYVL_SIZE_ENCODING_MAX_64; ) { + if ((int8_t)p[i++] >= 0) + break; + } + return p + i; +} + +/* is pointless to pretend size_t is anything other than 64 or 32 bits */ +#if SIZE_MAX == UINT64_MAX + +static inline unsigned int +fy_encode_size_bytes(size_t size) +{ + return fy_encode_size64_bytes(size); +} + +static inline uint8_t * +fy_encode_size(uint8_t *p, size_t bufsz, size_t size) +{ + return fy_encode_size64(p, bufsz, size); +} + +static inline const uint8_t * +fy_decode_size(const uint8_t *start, size_t bufsz, size_t *sizep) +{ + uint64_t sz; + const uint8_t *ret; + + ret = fy_decode_size64(start, bufsz, &sz); + if (!ret) { + *sizep = 0; + return NULL; + } + *sizep = sz; + return ret; +} + +static inline const uint8_t * +fy_decode_size_nocheck(const uint8_t *start, size_t *sizep) +{ + uint64_t size64; + const uint8_t *s; + + switch (sizeof(size_t)) { + case sizeof(uint64_t): + s = fy_decode_size64_nocheck(start, &size64); + break; + case sizeof(uint32_t): + s = fy_decode_size32_nocheck(start, &size64); + break; + default: + size64 = 0; + s = NULL; + break; + } + *sizep = (size_t)size64; + return s; +} + +static inline const uint8_t * +fy_skip_size(const uint8_t *start, size_t bufsz) +{ + return fy_skip_size64(start, bufsz); +} + +static inline const uint8_t * +fy_skip_size_nocheck(const uint8_t *start) +{ + return fy_skip_size64_nocheck(start); +} + +#define FYVL_SIZE_ENCODING_MAX FYVL_SIZE_ENCODING_MAX_64 + +#else + +static inline unsigned int +fy_encode_size_bytes(size_t size) +{ + return fy_encode_size32_bytes(size); +} + +static inline uint8_t * +fy_encode_size(uint8_t *p, size_t bufsz, size_t size) +{ + return fy_encode_size32(p, bufsz, size); +} + +static inline const uint8_t * +fy_decode_size(const uint8_t *start, size_t bufsz, size_t *sizep) +{ + uint32_t sz; + const uint8_t *ret; + + ret = fy_decode_size32(start, bufsz, &sz); + if (!ret) + return NULL; + *sizep = sz; + return ret; +} + +static inline const uint8_t * +fy_decode_size_nocheck(const uint8_t *start, size_t *sizep) +{ + return fy_decode_size32_nocheck(start, sizep); +} + +static inline const uint8_t * +fy_skip_size(const uint8_t *start, size_t bufsz) +{ + return fy_skip_size32(start, bufsz, &sz); +} + +static inline const uint8_t * +fy_skip_size_nocheck(const uint8_t *start) +{ + return fy_skip_size32_nocheck(start); +} + +#define FYVL_SIZE_ENCODING_MAX FYVL_SIZE_ENCODING_MAX_32 + +#endif + +#endif diff --git a/src/valgrind/fy-valgrind.h b/src/valgrind/fy-valgrind.h index 0c7074b..80176f3 100644 --- a/src/valgrind/fy-valgrind.h +++ b/src/valgrind/fy-valgrind.h @@ -20,6 +20,10 @@ #include #include #include +#include +#ifdef __linux__ +#include +#endif enum fy_valgrind_tool { fyvt_none, @@ -105,14 +109,15 @@ do_valgrind_no_opt: switch (tool) { case fyvt_valgrind: - va_argc = 1 + 4 + argc - 1; + va_argc = 1 + 5 + argc - 1; va_argv = alloca(sizeof(*va_argv) * (va_argc + 1)); va_argv[0] = valgrind; va_argv[1] = "--leak-check=full"; - va_argv[2] = "--track-origins=yes"; - va_argv[3] = "--error-exitcode=5"; - va_argv[4] = argv0; - memcpy(va_argv + 1 + 4, argv + 1, argc * sizeof(*va_argv)); + va_argv[2] = "--show-leak-kinds=all"; + va_argv[3] = "--track-origins=yes"; + va_argv[4] = "--error-exitcode=5"; + va_argv[5] = argv0; + memcpy(va_argv + 1 + 5, argv + 1, argc * sizeof(*va_argv)); break; case fyvt_callgrind: diff --git a/test/Makefile.am b/test/Makefile.am index f70ddc1..6da175b 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -25,12 +25,19 @@ TEST_LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) $(top_srcdir)/build-aux/tap-d TESTS = +# When cross-compiling, we can build the tests but cannot execute them +# since they are compiled for the target architecture +if CROSS_COMPILING +check_PROGRAMS = +else + if HAVE_COMPATIBLE_CHECK check_PROGRAMS = libfyaml-test libfyaml_test_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/src/valgrind/ \ -I$(top_srcdir)/src/lib/ \ - -I$(top_srcdir)/src/util + -I$(top_srcdir)/src/util \ + -I$(top_srcdir)/src/allocator libfyaml_test_LDADD = $(AM_LDADD) $(CHECK_LIBS) $(top_builddir)/src/libfyaml.la libfyaml_test_CFLAGS = $(AM_CFLAGS) $(CHECK_CFLAGS) libfyaml_test_LDFLAGS = $(AM_LDFLAGS) $(CHECK_LDFLAGS) @@ -39,14 +46,17 @@ libfyaml_test_SOURCES = \ libfyaml-test.c \ libfyaml-test-core.c \ libfyaml-test-meta.c \ - libfyaml-test-emit.c + libfyaml-test-emit.c \ + libfyaml-test-allocator.c # the private tests require static compilation # if we're compiling shared only, disable the private parts if HAVE_STATIC libfyaml_test_SOURCES += \ libfyaml-test-private.c \ - libfyaml-test-private-id.c + libfyaml-test-private-id.c \ + libfyaml-test-parser.c \ + libfyaml-test-thread.c libfyaml_test_LDFLAGS += -static endif @@ -104,6 +114,9 @@ TESTS += testerrors.test TESTS += testemitter.test TESTS += testemitter-streaming.test +TESTS += testemitter-restreaming.test + +endif # !CROSS_COMPILING EXTRA_DIST = test-env $(TESTS) diff --git a/test/jsontestsuite.test b/test/jsontestsuite.test index 9383f60..4b0cbc1 100755 --- a/test/jsontestsuite.test +++ b/test/jsontestsuite.test @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash count=0 diff --git a/test/libfyaml-test-allocator.c b/test/libfyaml-test-allocator.c new file mode 100644 index 0000000..b5435cb --- /dev/null +++ b/test/libfyaml-test-allocator.c @@ -0,0 +1,356 @@ +/* + * libfyaml-test-allocator.c - libfyaml test public allocator interface + * + * Copyright (c) 2024 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) ((sizeof(x)/sizeof((x)[0]))) +#endif + +static const char *builtin_allocators[] = { + "linear", + "malloc", + "mremap", + "dedup", + "auto", + NULL, +}; + +START_TEST(allocator_builtins) +{ + const char **pp, *p; + + /* verify that all the builtins are available */ + pp = builtin_allocators; + while ((p = *pp++) != NULL) { + fprintf(stderr, "checking builtin allocator: %s\n", p); + ck_assert(fy_allocator_is_available(p)); + } +} +END_TEST + +static void test_allocator_alignment(struct fy_allocator *a, int tag) +{ + size_t sz, align; + void *p; + + /* 1, 2, 4, 8, 16 bytes allocations */ + for (sz = 1; sz <= 16; sz <<= 1) { + + align = sz; + + /* allocate and check alignment */ + p = fy_allocator_alloc(a, tag, sz, align); + ck_assert_ptr_ne(p, NULL); + ck_assert(((uintptr_t)p & (align - 1)) == 0); + } +} + +START_TEST(allocator_linear_buf) +{ + /* align to 64 bits */ + union { + char buf[1024]; + uint64_t dummy; + } u; + struct fy_linear_allocator_cfg lcfg; + struct fy_allocator *a = NULL; + int tag = -1; + void *p; + + memset(&lcfg, 0, sizeof(lcfg)); + lcfg.buf = u.buf; + lcfg.size = sizeof(u.buf); + + /* create */ + a = fy_allocator_create("linear", &lcfg); + ck_assert_ptr_ne(a, NULL); + + /* get the tag */ + tag = fy_allocator_get_tag(a); + ck_assert_int_ne(tag, -1); + + test_allocator_alignment(a, tag); + + /* allocate something too large to fit */ + p = fy_allocator_alloc(a, tag, sizeof(u.buf) + 1, 16); + ck_assert_ptr_eq(p, NULL); + + /* destroy */ + fy_allocator_destroy(a); +} +END_TEST + +START_TEST(allocator_linear_alloc) +{ + struct fy_linear_allocator_cfg lcfg; + struct fy_allocator *a = NULL; + int tag = -1; + void *p; + + memset(&lcfg, 0, sizeof(lcfg)); + lcfg.size = 1024; + + /* create */ + a = fy_allocator_create("linear", &lcfg); + ck_assert_ptr_ne(a, NULL); + + /* get the tag */ + tag = fy_allocator_get_tag(a); + ck_assert_int_ne(tag, -1); + + test_allocator_alignment(a, tag); + + /* allocate something too large to fit */ + p = fy_allocator_alloc(a, tag, 1024 + 1, 16); + ck_assert_ptr_eq(p, NULL); + + /* destroy */ + fy_allocator_destroy(a); +} +END_TEST + +START_TEST(allocator_malloc) +{ + struct fy_allocator *a = NULL; + int tag0 = -1, tag1 = -1; + + /* create */ + a = fy_allocator_create("malloc", NULL); + ck_assert_ptr_ne(a, NULL); + + /* get the first tag */ + tag0 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag0, -1); + + /* get the second tag */ + tag1 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag1, -1); + + /* tags must be different */ + ck_assert_int_ne(tag0, tag1); + + test_allocator_alignment(a, tag0); + + test_allocator_alignment(a, tag1); + + /* destroy */ + fy_allocator_destroy(a); +} +END_TEST + +START_TEST(allocator_mremap) +{ + struct fy_allocator *a = NULL; + int tag0 = -1, tag1 = -1; + + /* create (default everything) */ + a = fy_allocator_create("mremap", NULL); + ck_assert_ptr_ne(a, NULL); + + /* get the first tag */ + tag0 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag0, -1); + + /* get the second tag */ + tag1 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag1, -1); + + /* tags must be different */ + ck_assert_int_ne(tag0, tag1); + + test_allocator_alignment(a, tag0); + + test_allocator_alignment(a, tag1); + + /* destroy */ + fy_allocator_destroy(a); +} +END_TEST + +static inline bool +scenario_is_single_tagged(int scenario) +{ + return scenario >= FYAST_SINGLE_LINEAR_RANGE && + scenario <= FYAST_SINGLE_LINEAR_RANGE_DEDUP; +} + +static inline bool +scenario_is_dedup(int scenario) +{ + return scenario == FYAST_PER_TAG_FREE_DEDUP || + scenario == FYAST_PER_OBJ_FREE_DEDUP || + scenario == FYAST_SINGLE_LINEAR_RANGE_DEDUP; +} + +static inline bool +scenario_is_fixed_size(int scenario) +{ + return scenario >= FYAST_SINGLE_LINEAR_RANGE && + scenario <= FYAST_SINGLE_LINEAR_RANGE_DEDUP; +} + +START_TEST(allocator_auto) +{ + static const struct { + int scenario; + const char *name; + } auto_scenarios[] = { + { + .scenario = FYAST_PER_TAG_FREE, + .name = "per-tag-free", + }, { + .scenario = FYAST_PER_TAG_FREE_DEDUP, + .name = "per-tag-free-dedup", + }, { + .scenario = FYAST_PER_OBJ_FREE, + .name = "per-obj-free", + }, { + .scenario = FYAST_PER_OBJ_FREE_DEDUP, + .name = "per-obj-free-dedup", + }, { + .scenario = FYAST_SINGLE_LINEAR_RANGE, + .name = "single-linear-range", + }, { + .scenario = FYAST_SINGLE_LINEAR_RANGE_DEDUP, + .name = "single-linear-range-dedup", + } + }; + struct fy_auto_allocator_cfg acfg; + struct fy_allocator *a = NULL; + enum fy_auto_allocator_scenario_type sc; + int tag0 = -1, tag1 = -1; + unsigned int i; + + /* create (default everything) */ + for (i = 0; i < ARRAY_SIZE(auto_scenarios); i++) { + + printf("scenario #%d %s\n", i, auto_scenarios[i].name); + sc = auto_scenarios[i].scenario; + + memset(&acfg, 0, sizeof(acfg)); + acfg.scenario = sc; + + /* for fixed sizes, make it 1MB */ + if (scenario_is_fixed_size(sc)) + acfg.estimated_max_size = 1 << 20; + + a = fy_allocator_create("auto", &acfg); + ck_assert_ptr_ne(a, NULL); + + /* get the first tag */ + tag0 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag0, -1); + + if (!scenario_is_single_tagged(sc)) { + /* get the second tag */ + tag1 = fy_allocator_get_tag(a); + ck_assert_int_ne(tag1, -1); + + /* tags must be different */ + ck_assert_int_ne(tag0, tag1); + } + + test_allocator_alignment(a, tag0); + + if (!scenario_is_single_tagged(sc)) + test_allocator_alignment(a, tag1); + + /* destroy */ + fy_allocator_destroy(a); + a = NULL; + } +} +END_TEST + +START_TEST(allocator_linear_inplace) +{ + struct fy_allocator *a; + char buf[FY_LINEAR_ALLOCATOR_IN_PLACE_MIN_SIZE]; + int tag; + const void *p; + + a = fy_linear_allocator_create_in_place(buf, sizeof(buf)); + ck_assert_ptr_ne(a, NULL); + + tag = fy_allocator_get_tag(a); + ck_assert_int_ne(tag, -1); + + p = fy_allocator_store(a, tag, "Hello", 6, 1); + ck_assert_ptr_ne(p, NULL); + + ck_assert(!strcmp(p, "Hello")); + + /* no release */ +} +END_TEST + +START_TEST(allocator_dedup_inplace) +{ + struct fy_allocator *a; + char buf[FY_DEDUP_ALLOCATOR_IN_PLACE_MIN_SIZE]; + int tag; + const void *p1, *p2; + + a = fy_dedup_allocator_create_in_place(buf, sizeof(buf)); + ck_assert_ptr_ne(a, NULL); + + tag = fy_allocator_get_tag(a); + ck_assert_int_ne(tag, -1); + + p1 = fy_allocator_store(a, tag, "Hello", 6, 1); + ck_assert_ptr_ne(p1, NULL); + + ck_assert(!strcmp(p1, "Hello")); + + p2 = fy_allocator_store(a, tag, "Hello", 6, 1); + ck_assert_ptr_ne(p2, NULL); + + ck_assert(!strcmp(p2, "Hello")); + + /* dedup must return the same pointer */ + ck_assert(p1 == p2); + + /* no release */ +} +END_TEST + +TCase *libfyaml_case_allocator(void) +{ + TCase *tc; + + tc = tcase_create("allocator"); + + tcase_add_test(tc, allocator_builtins); + + tcase_add_test(tc, allocator_linear_buf); + tcase_add_test(tc, allocator_linear_alloc); + tcase_add_test(tc, allocator_malloc); + tcase_add_test(tc, allocator_mremap); + tcase_add_test(tc, allocator_auto); + + tcase_add_test(tc, allocator_linear_inplace); + tcase_add_test(tc, allocator_dedup_inplace); + + return tc; +} diff --git a/test/libfyaml-test-core.c b/test/libfyaml-test-core.c index 8d3e4e6..47d8fac 100644 --- a/test/libfyaml-test-core.c +++ b/test/libfyaml-test-core.c @@ -1125,6 +1125,50 @@ START_TEST(doc_sort) } END_TEST +START_TEST(doc_insert_at) +{ + struct fy_document *fyd; + struct fy_node *fyn; + int rc; + + /* build document */ + fyd = fy_document_build_from_string(NULL, + "apiVersion: apps/v1\n" + "kind: Deployment\n" + "metadata:\n" + " annotations:\n" + " x: 2024-06-12t14:50:43z\n", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* verify that x exists */ + fyn = fy_node_by_path(fy_document_root(fyd), "/metadata/annotations/x", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + + /* delete the x node */ + rc = fy_document_insert_at(fyd, "/metadata/annotations/x", FY_NT, NULL); + ck_assert_int_eq(rc, 0); + + /* verify that x now gone */ + fyn = fy_node_by_path(fy_document_root(fyd), "/metadata/annotations/x", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_eq(fyn, NULL); + + /* verify that y does not exist */ + fyn = fy_node_by_path(fy_document_root(fyd), "/metadata/annotations/y", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_eq(fyn, NULL); + + /* insert a y node */ + rc = fy_document_insert_at(fyd, "/metadata/annotations", FY_NT, + fy_node_buildf(fyd, "y: 2024-06-12t14:50:43z")); + ck_assert_int_eq(rc, 0); + + /* verify that y exists now */ + fyn = fy_node_by_path(fy_document_root(fyd), "/metadata/annotations/y", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_destroy(fyd); +} +END_TEST + static char *join_docs(const char *tgt_text, const char *tgt_path, const char *src_text, const char *src_path, const char *emit_path) @@ -1922,7 +1966,8 @@ START_TEST(scanf_check) } END_TEST -START_TEST(token_test) { +START_TEST(token_test) +{ struct fy_document *fyd; struct fy_node *fyn_sequence, *fyn_mapping, *fyn_scalar; struct fy_token *fyn_sequence_start, *fyn_sequence_end, @@ -2031,6 +2076,7 @@ TCase *libfyaml_case_core(void) tcase_add_test(tc, doc_insert_remove_map); tcase_add_test(tc, doc_sort); + tcase_add_test(tc, doc_insert_at); tcase_add_test(tc, doc_join_scalar_to_scalar); tcase_add_test(tc, doc_join_scalar_to_map); diff --git a/test/libfyaml-test-parser.c b/test/libfyaml-test-parser.c new file mode 100644 index 0000000..13927bd --- /dev/null +++ b/test/libfyaml-test-parser.c @@ -0,0 +1,2165 @@ +/* + * libfyaml-test-parser.c - libfyaml parser testing (extracted from libfyaml-parser.c) + * + * Copyright (c) 2019 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include + +#include +#include "fy-parse.h" +#include "fy-utf8.h" + +/* Test: Mapping iterator (forward and reverse) */ +START_TEST(parser_mapping_iterator) +{ + struct fy_document *fyd; + struct fy_node_pair *fynp; + int count; + void *iter; + + /* Build a mapping with multiple entries including complex keys */ + fyd = fy_document_build_from_string(NULL, + "{ foo: 10, bar: 20, baz: [100, 101], [frob, 1]: boo }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Verify count */ + count = fy_node_mapping_item_count(fy_document_root(fyd)); + ck_assert_int_eq(count, 4); + + /* Test forward iterator */ + iter = NULL; + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "foo"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "10"); + + fynp = fy_node_mapping_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "bar"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "20"); + + /* Test reverse iterator */ + iter = NULL; + fynp = fy_node_mapping_reverse_iterate(fy_document_root(fyd), &iter); + ck_assert_ptr_ne(fynp, NULL); + /* Last item should be the complex key */ + + /* Test index-based access */ + fynp = fy_node_mapping_get_by_index(fy_document_root(fyd), 0); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "foo"); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_value(fynp)), "10"); + + fynp = fy_node_mapping_get_by_index(fy_document_root(fyd), 1); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "bar"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Mapping key lookup */ +START_TEST(parser_mapping_key_lookup) +{ + struct fy_document *fyd; + struct fy_node *fyn; + + fyd = fy_document_build_from_string(NULL, + "{ foo: 10, bar: 20, baz: [100, 101], [frob, 1]: boo }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Lookup simple keys */ + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fyd), "foo", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fyd), "bar", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "20"); + + /* Lookup key with sequence value */ + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fyd), "baz", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_sequence(fyn)); + + /* Lookup complex key */ + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fyd), "[ frob, 1 ]", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "boo"); + + /* Lookup non-existent key */ + fyn = fy_node_mapping_lookup_by_string(fy_document_root(fyd), "nonexistent", FY_NT); + ck_assert_ptr_eq(fyn, NULL); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Path-based node queries */ +START_TEST(parser_path_queries) +{ + struct fy_document *fyd; + struct fy_node *fyn; + + fyd = fy_document_build_from_string(NULL, + "{ foo: 10, bar: 20, baz:{ frob: boo }, " + "frooz: [ seq1, { key: value} ], \"zero\\0zero\": 0, " + "{ key2: value2 }: { key3: value3 } }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Query root */ + fyn = fy_node_by_path(fy_document_root(fyd), "/", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + /* Query simple keys */ + fyn = fy_node_by_path(fy_document_root(fyd), "foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + fyn = fy_node_by_path(fy_document_root(fyd), "bar", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "20"); + + /* Query nested path */ + fyn = fy_node_by_path(fy_document_root(fyd), "baz/frob", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "boo"); + + /* Query sequence elements by index */ + fyn = fy_node_by_path(fy_document_root(fyd), "/frooz/[0]", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "seq1"); + + /* Query nested in sequence */ + fyn = fy_node_by_path(fy_document_root(fyd), "/frooz/[1]/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "value"); + + /* Query with quoted key */ + fyn = fy_node_by_path(fy_document_root(fyd), "\"foo\"", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + /* Query with null byte in key */ + fyn = fy_node_by_path(fy_document_root(fyd), "\"zero\\0zero\"", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "0"); + + /* Query complex key mapping */ + fyn = fy_node_by_path(fy_document_root(fyd), "/{ key2: value2 }", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + /* Query nested in complex key */ + fyn = fy_node_by_path(fy_document_root(fyd), "/{ key2: value2 }/key3", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "value3"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node path generation */ +START_TEST(parser_node_path_generation) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *path; + + fyd = fy_document_build_from_string(NULL, + "{ foo: 10, frooz: [ seq1, { key: value} ], " + "{ key2: value2 }: { key3: value3 } }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Get path of root */ + fyn = fy_node_by_path(fy_document_root(fyd), "/", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + path = fy_node_get_path(fyn); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "/"); + free(path); + + /* Get path of simple key */ + fyn = fy_node_by_path(fy_document_root(fyd), "/frooz", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + path = fy_node_get_path(fyn); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "/frooz"); + free(path); + + /* Get path of sequence element */ + fyn = fy_node_by_path(fy_document_root(fyd), "/frooz/[0]", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + path = fy_node_get_path(fyn); + ck_assert_ptr_ne(path, NULL); + ck_assert_str_eq(path, "/frooz/0"); + free(path); + + /* Get path of nested element in complex key */ + fyn = fy_node_by_path(fy_document_root(fyd), "/{ key2: value2 }/key3", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + path = fy_node_get_path(fyn); + ck_assert_ptr_ne(path, NULL); + /* Path should be valid */ + ck_assert_ptr_ne(path, NULL); + free(path); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation from scratch */ +START_TEST(parser_node_creation_scalar) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + /* Create document and scalar node */ + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_scalar(fyd, "foo", 3); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* Emit and verify */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + ck_assert_str_eq(buf, "foo\n"); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation - multiline scalar */ +START_TEST(parser_node_creation_multiline_scalar) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_scalar(fyd, "foo\nfoo", 7); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* Emit and verify - multiline scalars should be emitted with literal or folded style */ + buf = fy_emit_document_to_string(fyd, 0); + ck_assert_ptr_ne(buf, NULL); + /* Just verify it emits successfully and contains the content */ + ck_assert(strstr(buf, "foo") != NULL); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation - empty sequence */ +START_TEST(parser_node_creation_empty_sequence) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* Emit and verify */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + ck_assert_str_eq(buf, "[]\n"); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation - empty mapping */ +START_TEST(parser_node_creation_empty_mapping) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn = fy_node_create_mapping(fyd); + ck_assert_ptr_ne(fyn, NULL); + + fy_document_set_root(fyd, fyn); + + /* Emit and verify */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + ck_assert_str_eq(buf, "{}\n"); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation - populated sequence */ +START_TEST(parser_node_creation_populated_sequence) +{ + struct fy_document *fyd; + struct fy_node *fyn, *fyn_seq; + int ret; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn_seq = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn_seq, NULL); + + /* Append elements */ + fyn = fy_node_create_scalar(fyd, "foo", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ret = fy_node_sequence_append(fyn_seq, fyn); + ck_assert_int_eq(ret, 0); + + fyn = fy_node_create_scalar(fyd, "bar", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ret = fy_node_sequence_append(fyn_seq, fyn); + ck_assert_int_eq(ret, 0); + + fyn = fy_node_build_from_string(fyd, "{ baz: frooz }", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ret = fy_node_sequence_append(fyn_seq, fyn); + ck_assert_int_eq(ret, 0); + + fy_document_set_root(fyd, fyn_seq); + + /* Verify count */ + ck_assert_int_eq(fy_node_sequence_item_count(fyn_seq), 3); + + /* Verify content */ + fyn = fy_node_sequence_get_by_index(fyn_seq, 0); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "foo"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 1); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "bar"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 2); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node creation - populated mapping */ +START_TEST(parser_node_creation_populated_mapping) +{ + struct fy_document *fyd; + struct fy_node *fyn, *fyn_key, *fyn_val, *fyn_map; + int ret; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn_map = fy_node_create_mapping(fyd); + ck_assert_ptr_ne(fyn_map, NULL); + + /* Append key-value pairs */ + fyn_key = fy_node_create_scalar(fyd, "foo", FY_NT); + ck_assert_ptr_ne(fyn_key, NULL); + fyn_val = fy_node_create_scalar(fyd, "10", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + ret = fy_node_mapping_append(fyn_map, fyn_key, fyn_val); + ck_assert_int_eq(ret, 0); + + fyn_key = fy_node_create_scalar(fyd, "bar", FY_NT); + ck_assert_ptr_ne(fyn_key, NULL); + fyn_val = fy_node_build_from_string(fyd, "[ 1, 2, 3 ]", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + ret = fy_node_mapping_append(fyn_map, fyn_key, fyn_val); + ck_assert_int_eq(ret, 0); + + fy_document_set_root(fyd, fyn_map); + + /* Verify count */ + ck_assert_int_eq(fy_node_mapping_item_count(fyn_map), 2); + + /* Verify lookup */ + fyn = fy_node_mapping_lookup_by_string(fyn_map, "foo", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "10"); + + fyn = fy_node_mapping_lookup_by_string(fyn_map, "bar", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_sequence(fyn)); + ck_assert_int_eq(fy_node_sequence_item_count(fyn), 3); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Build node from string within document */ +START_TEST(parser_build_node_from_string) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* Build a complex node from string */ + fyn = fy_node_build_from_string(fyd, "{ }", FY_NT); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + fy_document_set_root(fyd, fyn); + + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + ck_assert_str_eq(buf, "{}\n"); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Sequence operations - reverse index access */ +START_TEST(parser_sequence_negative_index) +{ + struct fy_document *fyd; + struct fy_node *fyn; + + fyd = fy_document_build_from_string(NULL, "[ first, second, third ]", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Access from end using negative indices */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -1); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "third"); + + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -2); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "second"); + + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -3); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "first"); + + /* Out of bounds negative index */ + fyn = fy_node_sequence_get_by_index(fy_document_root(fyd), -4); + ck_assert_ptr_eq(fyn, NULL); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Complex nested structure */ +START_TEST(parser_complex_nested_structure) +{ + struct fy_document *fyd; + struct fy_node *fyn; + int count; + + /* Build a complex nested structure */ + fyd = fy_document_build_from_string(NULL, + "root:\n" + " level1:\n" + " level2:\n" + " - item1\n" + " - item2\n" + " - key: value\n" + " nested: data\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Navigate to nested sequence */ + fyn = fy_node_by_path(fy_document_root(fyd), "root/level1/level2", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_sequence(fyn)); + + count = fy_node_sequence_item_count(fyn); + ck_assert_int_eq(count, 3); + + /* Check first scalar item */ + fyn = fy_node_by_path(fy_document_root(fyd), "root/level1/level2/[0]", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "item1"); + + /* Check nested mapping in sequence */ + fyn = fy_node_by_path(fy_document_root(fyd), "root/level1/level2/[2]/nested", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "data"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Anchor and alias resolution */ +START_TEST(parser_anchor_alias_resolution) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + int rc; + + /* Build document with anchor and alias */ + fyd = fy_document_build_from_string(NULL, + "base: &base\n" + " name: this-is-a-name\n" + " value: 42\n" + "copy: *base\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Before resolution, alias should exist */ + fyn = fy_node_by_path(fy_document_root(fyd), "copy", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_alias(fyn)); + + /* Resolve the document */ + rc = fy_document_resolve(fyd); + ck_assert_int_eq(rc, 0); + + /* After resolution, we should be able to access through the alias */ + fyn = fy_node_by_path(fy_document_root(fyd), "copy/name", FY_NT, FYNWF_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "this-is-a-name"); + + /* Emit resolved document */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document insertion at path */ +START_TEST(parser_document_insert_at) +{ + struct fy_document *fyd; + struct fy_node *fyn, *fyn_inserted; + int rc; + + /* Create base document */ + fyd = fy_document_build_from_string(NULL, + "base:\n" + " name: original\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Build a mapping node to insert (key: value) */ + fyn = fy_node_buildf(fyd, "new-key: inserted-value"); + ck_assert_ptr_ne(fyn, NULL); + + /* Insert the mapping at /base (merges into existing mapping) */ + rc = fy_document_insert_at(fyd, "/base", FY_NT, fyn); + ck_assert_int_eq(rc, 0); + + /* Verify insertion */ + fyn_inserted = fy_node_by_path(fy_document_root(fyd), "/base/new-key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_inserted, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn_inserted), "inserted-value"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document emit with different flags */ +START_TEST(parser_document_emit_flags) +{ + struct fy_document *fyd; + char *buf; + + /* Build test document */ + fyd = fy_document_build_from_string(NULL, + "{ z: 1, a: 2, m: 3 }", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Emit with default flags */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE); + ck_assert_ptr_ne(buf, NULL); + ck_assert(strstr(buf, "z") != NULL); + ck_assert(strstr(buf, "a") != NULL); + ck_assert(strstr(buf, "m") != NULL); + free(buf); + + /* Emit with sorted keys */ + buf = fy_emit_document_to_string(fyd, FYECF_MODE_FLOW_ONELINE | FYECF_SORT_KEYS); + ck_assert_ptr_ne(buf, NULL); + /* In sorted output, 'a' should come before 'z' */ + ck_assert(strstr(buf, "a") < strstr(buf, "z")); + ck_assert(strstr(buf, "m") < strstr(buf, "z")); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Multi-document stream parsing */ +START_TEST(parser_multi_document_stream) +{ + struct fy_parser fyp_data, *fyp = &fyp_data; + struct fy_parse_cfg cfg = { + .flags = FYPCF_DEFAULT_DOC, + }; + struct fy_document *fyd; + int count; + int rc; + + /* Setup parser */ + rc = fy_parse_setup(fyp, &cfg); + ck_assert_int_eq(rc, 0); + + /* Create multi-document input */ + const char *yaml_multi = + "---\n" + "doc: 1\n" + "---\n" + "doc: 2\n" + "---\n" + "doc: 3\n"; + + /* Add input */ + struct fy_input_cfg fyic = { + .type = fyit_memory, + .memory.data = yaml_multi, + .memory.size = strlen(yaml_multi), + }; + rc = fy_parse_input_append(fyp, &fyic); + ck_assert_int_eq(rc, 0); + + /* Parse all documents */ + count = 0; + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + count++; + + /* Verify document content */ + struct fy_node *fyn = fy_node_by_path(fy_document_root(fyd), "doc", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + + int doc_num = atoi(fy_node_get_scalar0(fyn)); + ck_assert_int_eq(doc_num, count); + + fy_parse_document_destroy(fyp, fyd); + } + + ck_assert_int_eq(count, 3); + + fy_parse_cleanup(fyp); +} +END_TEST + +/* Test: Empty document handling */ +START_TEST(parser_empty_document) +{ + struct fy_document *fyd; + char *buf; + + /* Null document (YAML null/empty) */ + fyd = fy_document_build_from_string(NULL, "null", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Should have scalar null root */ + ck_assert_ptr_ne(fy_document_root(fyd), NULL); + + /* Should emit */ + buf = fy_emit_document_to_string(fyd, 0); + ck_assert_ptr_ne(buf, NULL); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document with comments (requires FYPCF_PARSE_COMMENTS) */ +START_TEST(parser_document_with_comments) +{ + struct fy_parse_cfg cfg = { + .flags = FYPCF_PARSE_COMMENTS, + }; + struct fy_document *fyd; + + /* Build document with comments */ + fyd = fy_document_build_from_string(&cfg, + "# Top comment\n" + "key: value # Right comment\n" + "# Bottom comment\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Verify content (comments should be preserved in structure) */ + struct fy_node *fyn = fy_node_by_path(fy_document_root(fyd), "key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "value"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Sequence append and prepend */ +START_TEST(parser_sequence_append_prepend) +{ + struct fy_document *fyd; + struct fy_node *fyn_seq, *fyn; + int rc; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* Create sequence */ + fyn_seq = fy_node_create_sequence(fyd); + ck_assert_ptr_ne(fyn_seq, NULL); + fy_document_set_root(fyd, fyn_seq); + + /* Append items */ + fyn = fy_node_create_scalar(fyd, "second", FY_NT); + rc = fy_node_sequence_append(fyn_seq, fyn); + ck_assert_int_eq(rc, 0); + + /* Prepend item */ + fyn = fy_node_create_scalar(fyd, "first", FY_NT); + rc = fy_node_sequence_prepend(fyn_seq, fyn); + ck_assert_int_eq(rc, 0); + + /* Append another */ + fyn = fy_node_create_scalar(fyd, "third", FY_NT); + rc = fy_node_sequence_append(fyn_seq, fyn); + ck_assert_int_eq(rc, 0); + + /* Verify order */ + fyn = fy_node_sequence_get_by_index(fyn_seq, 0); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "first"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 1); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "second"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 2); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "third"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Mapping prepend */ +START_TEST(parser_mapping_prepend) +{ + struct fy_document *fyd; + struct fy_node *fyn_map, *fyn_key, *fyn_val; + struct fy_node_pair *fynp; + int rc; + + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + fyn_map = fy_node_create_mapping(fyd); + ck_assert_ptr_ne(fyn_map, NULL); + fy_document_set_root(fyd, fyn_map); + + /* Append a pair */ + fyn_key = fy_node_create_scalar(fyd, "second", FY_NT); + fyn_val = fy_node_create_scalar(fyd, "2", FY_NT); + rc = fy_node_mapping_append(fyn_map, fyn_key, fyn_val); + ck_assert_int_eq(rc, 0); + + /* Prepend a pair */ + fyn_key = fy_node_create_scalar(fyd, "first", FY_NT); + fyn_val = fy_node_create_scalar(fyd, "1", FY_NT); + rc = fy_node_mapping_prepend(fyn_map, fyn_key, fyn_val); + ck_assert_int_eq(rc, 0); + + /* Verify order */ + fynp = fy_node_mapping_get_by_index(fyn_map, 0); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "first"); + + fynp = fy_node_mapping_get_by_index(fyn_map, 1); + ck_assert_ptr_ne(fynp, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fy_node_pair_key(fynp)), "second"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node removal from sequence */ +START_TEST(parser_sequence_remove) +{ + struct fy_document *fyd; + struct fy_node *fyn_seq, *fyn; + int count; + + /* Build a sequence */ + fyd = fy_document_build_from_string(NULL, "[ a, b, c, d ]", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_seq = fy_document_root(fyd); + ck_assert_int_eq(fy_node_sequence_item_count(fyn_seq), 4); + + /* Remove middle element */ + fyn = fy_node_sequence_get_by_index(fyn_seq, 1); + ck_assert_ptr_ne(fyn, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "b"); + + fyn = fy_node_sequence_remove(fyn_seq, fyn); + ck_assert_ptr_ne(fyn, NULL); + fy_node_free(fyn); + + /* Verify count and order */ + count = fy_node_sequence_item_count(fyn_seq); + ck_assert_int_eq(count, 3); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 0); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "a"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 1); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "c"); + + fyn = fy_node_sequence_get_by_index(fyn_seq, 2); + ck_assert_str_eq(fy_node_get_scalar0(fyn), "d"); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Node removal from mapping */ +START_TEST(parser_mapping_remove) +{ + struct fy_document *fyd; + struct fy_node *fyn_map, *fyn_key, *fyn_val; + int count; + + /* Build a mapping */ + fyd = fy_document_build_from_string(NULL, "{ a: 1, b: 2, c: 3 }", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fyn_map = fy_document_root(fyd); + ck_assert_int_eq(fy_node_mapping_item_count(fyn_map), 3); + + /* Remove by key */ + fyn_key = fy_node_build_from_string(fyd, "b", FY_NT); + ck_assert_ptr_ne(fyn_key, NULL); + + fyn_val = fy_node_mapping_remove_by_key(fyn_map, fyn_key); + ck_assert_ptr_ne(fyn_val, NULL); + fy_node_free(fyn_val); + + /* Verify count */ + count = fy_node_mapping_item_count(fyn_map); + ck_assert_int_eq(count, 2); + + /* Verify 'b' is gone */ + fyn_val = fy_node_mapping_lookup_by_string(fyn_map, "b", FY_NT); + ck_assert_ptr_eq(fyn_val, NULL); + + /* Verify others remain */ + fyn_val = fy_node_mapping_lookup_by_string(fyn_map, "a", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + + fyn_val = fy_node_mapping_lookup_by_string(fyn_map, "c", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document iterator functionality */ +START_TEST(parser_document_iterator) +{ + struct fy_document *fyd; + struct fy_document_iterator *fydi; + struct fy_node *fyn; + int count; + + /* Build test document with nested structure */ + fyd = fy_document_build_from_string(NULL, + "root:\n" + " scalar: value\n" + " seq:\n" + " - item1\n" + " - item2\n" + " map:\n" + " key: val\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Create iterator */ + fydi = fy_document_iterator_create(); + ck_assert_ptr_ne(fydi, NULL); + + /* Start iteration from root */ + fy_document_iterator_node_start(fydi, fy_document_root(fyd)); + + /* Count all nodes */ + count = 0; + while ((fyn = fy_document_iterator_node_next(fydi)) != NULL) { + count++; + + /* Verify node type detection works */ + if (fy_node_is_scalar(fyn)) { + const char *text; + size_t len; + text = fy_node_get_scalar(fyn, &len); + ck_assert_ptr_ne(text, NULL); + } else if (fy_node_is_sequence(fyn)) { + /* Verify it's a sequence */ + ck_assert(fy_node_is_sequence(fyn)); + } else if (fy_node_is_mapping(fyn)) { + /* Verify it's a mapping */ + ck_assert(fy_node_is_mapping(fyn)); + } + } + + /* We should have iterated through multiple nodes */ + ck_assert_int_gt(count, 0); + + /* Cleanup */ + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document iterator with key detection */ +START_TEST(parser_document_iterator_key_detection) +{ + struct fy_document *fyd; + struct fy_document_iterator *fydi; + struct fy_node *fyn; + + /* Build mapping document */ + fyd = fy_document_build_from_string(NULL, + "key1: value1\n" + "key2: value2\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fydi = fy_document_iterator_create(); + ck_assert_ptr_ne(fydi, NULL); + + fy_document_iterator_node_start(fydi, fy_document_root(fyd)); + + /* Iterate and check for key nodes */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_ne(fyn, NULL); + /* must be a mapping */ + ck_assert(fy_node_is_mapping(fyn)); + /* must be the root */ + ck_assert_ptr_eq(fyn, fy_document_root(fyd)); + + /* get the first key */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_ne(fyn, NULL); + /* must be a scalar */ + ck_assert(fy_node_is_scalar(fyn)); + ck_assert(!strcmp(fy_node_get_scalar0(fyn), "key1")); + + /* get the first value */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_ne(fyn, NULL); + /* must be a scalar */ + ck_assert(fy_node_is_scalar(fyn)); + ck_assert(!strcmp(fy_node_get_scalar0(fyn), "value1")); + + /* get the second key */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_ne(fyn, NULL); + /* must be a scalar */ + ck_assert(fy_node_is_scalar(fyn)); + ck_assert(!strcmp(fy_node_get_scalar0(fyn), "key2")); + + /* get the second value */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_ne(fyn, NULL); + /* must be a scalar */ + ck_assert(fy_node_is_scalar(fyn)); + ck_assert(!strcmp(fy_node_get_scalar0(fyn), "value2")); + + /* final, must be out of nodes */ + fyn = fy_document_iterator_node_next(fydi); + ck_assert_ptr_eq(fyn, NULL); + + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Comment retrieval from tokens */ +START_TEST(parser_comment_retrieval) +{ + struct fy_parse_cfg cfg = { + .flags = FYPCF_PARSE_COMMENTS, + }; + struct fy_document *fyd; + struct fy_document_iterator *fydi; + struct fy_node *fyn; + struct fy_token *fyt; + char buf[256]; + bool found_comment = false; + + /* Build document with comments */ + fyd = fy_document_build_from_string(&cfg, + "# Top comment\n" + "scalar: value # Right comment\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fydi = fy_document_iterator_create(); + ck_assert_ptr_ne(fydi, NULL); + + fy_document_iterator_node_start(fydi, fy_document_root(fyd)); + + /* Iterate and check for comments */ + while ((fyn = fy_document_iterator_node_next(fydi)) != NULL) { + if (!fy_node_is_scalar(fyn)) + continue; + + fyt = fy_node_get_scalar_token(fyn); + if (!fyt || !fy_token_has_any_comment(fyt)) + continue; + + /* Try to get comments at different placements */ + for (int placement = fycp_top; placement < fycp_max; placement++) { + if (fy_token_get_comment(fyt, buf, sizeof(buf), placement)) { + ck_assert_ptr_ne(buf, NULL); + ck_assert_int_gt(strlen(buf), 0); + found_comment = true; + } + } + } + + ck_assert(found_comment); + + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Alias node detection in iterator */ +START_TEST(parser_iterator_alias_detection) +{ + struct fy_document *fyd; + struct fy_document_iterator *fydi; + struct fy_node *fyn; + bool found_alias = false; + + /* Build document with anchor and alias */ + fyd = fy_document_build_from_string(NULL, + "anchor: &ref value\n" + "alias: *ref\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + fydi = fy_document_iterator_create(); + ck_assert_ptr_ne(fydi, NULL); + + fy_document_iterator_node_start(fydi, fy_document_root(fyd)); + + /* Iterate and check for alias nodes */ + while ((fyn = fy_document_iterator_node_next(fydi)) != NULL) { + if (fy_node_is_scalar(fyn) && fy_node_is_alias(fyn)) { + found_alias = true; + } + } + + ck_assert(found_alias); + + fy_document_iterator_destroy(fydi); + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Event-based parsing */ +START_TEST(parser_event_generation) +{ + struct fy_parser fyp_data, *fyp = &fyp_data; + struct fy_parse_cfg cfg = { + .flags = FYPCF_DEFAULT_DOC, + }; + struct fy_event *event; + int rc; + bool got_stream_start = false; + bool got_doc_start = false; + bool got_scalar = false; + bool got_doc_end = false; + bool got_stream_end = false; + + /* Setup parser */ + rc = fy_parse_setup(fyp, &cfg); + ck_assert_int_eq(rc, 0); + + /* Add simple YAML input */ + const char *yaml = "key: value\n"; + struct fy_input_cfg fyic = { + .type = fyit_memory, + .memory.data = yaml, + .memory.size = strlen(yaml), + }; + rc = fy_parse_input_append(fyp, &fyic); + ck_assert_int_eq(rc, 0); + + /* Parse events */ + while ((event = fy_parser_parse(fyp)) != NULL) { + switch (event->type) { + case FYET_STREAM_START: + got_stream_start = true; + break; + case FYET_DOCUMENT_START: + got_doc_start = true; + break; + case FYET_SCALAR: + got_scalar = true; + break; + case FYET_DOCUMENT_END: + got_doc_end = true; + break; + case FYET_STREAM_END: + got_stream_end = true; + break; + default: + break; + } + fy_parser_event_free(fyp, event); + } + + /* Verify we got expected events */ + ck_assert(got_stream_start); + ck_assert(got_doc_start); + ck_assert(got_scalar); + ck_assert(got_doc_end); + ck_assert(got_stream_end); + + fy_parse_cleanup(fyp); +} +END_TEST + +/* Test: Scalar style detection */ +START_TEST(parser_scalar_styles) +{ + struct fy_document *fyd; + struct fy_node *fyn; + struct fy_token *fyt; + enum fy_scalar_style style; + + /* Build document with different scalar styles */ + fyd = fy_document_build_from_string(NULL, + "plain: plain value\n" + "single: 'single quoted'\n" + "double: \"double quoted\"\n" + "literal: |\n" + " literal block\n" + "folded: >\n" + " folded block\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Check plain style */ + fyn = fy_node_by_path(fy_document_root(fyd), "plain", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + fyt = fy_node_get_scalar_token(fyn); + ck_assert_ptr_ne(fyt, NULL); + style = fy_token_scalar_style(fyt); + ck_assert_int_eq(style, FYSS_PLAIN); + + /* Check single quoted style */ + fyn = fy_node_by_path(fy_document_root(fyd), "single", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + fyt = fy_node_get_scalar_token(fyn); + ck_assert_ptr_ne(fyt, NULL); + style = fy_token_scalar_style(fyt); + ck_assert_int_eq(style, FYSS_SINGLE_QUOTED); + + /* Check double quoted style */ + fyn = fy_node_by_path(fy_document_root(fyd), "double", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + fyt = fy_node_get_scalar_token(fyn); + ck_assert_ptr_ne(fyt, NULL); + style = fy_token_scalar_style(fyt); + ck_assert_int_eq(style, FYSS_DOUBLE_QUOTED); + + /* Check literal style */ + fyn = fy_node_by_path(fy_document_root(fyd), "literal", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + fyt = fy_node_get_scalar_token(fyn); + ck_assert_ptr_ne(fyt, NULL); + style = fy_token_scalar_style(fyt); + ck_assert_int_eq(style, FYSS_LITERAL); + + /* Check folded style */ + fyn = fy_node_by_path(fy_document_root(fyd), "folded", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + fyt = fy_node_get_scalar_token(fyn); + ck_assert_ptr_ne(fyt, NULL); + style = fy_token_scalar_style(fyt); + ck_assert_int_eq(style, FYSS_FOLDED); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Tag handling */ +START_TEST(parser_tag_handling) +{ + struct fy_document *fyd; + struct fy_node *fyn; + const char *tag; + + /* Build document with tags */ + fyd = fy_document_build_from_string(NULL, + "string: !!str tagged string\n" + "integer: !!int 42\n" + "custom: !custom custom tag\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Check string tag */ + fyn = fy_node_by_path(fy_document_root(fyd), "string", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + tag = fy_node_get_tag0(fyn); + ck_assert_ptr_ne(tag, NULL); + ck_assert(strstr(tag, "str") != NULL); + + /* Check integer tag */ + fyn = fy_node_by_path(fy_document_root(fyd), "integer", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + tag = fy_node_get_tag0(fyn); + ck_assert_ptr_ne(tag, NULL); + ck_assert(strstr(tag, "int") != NULL); + + /* Check custom tag */ + fyn = fy_node_by_path(fy_document_root(fyd), "custom", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + tag = fy_node_get_tag0(fyn); + ck_assert_ptr_ne(tag, NULL); + ck_assert(strstr(tag, "custom") != NULL); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: YAML version directives */ +START_TEST(parser_yaml_version) +{ + struct fy_parse_cfg cfg_11 = { + .flags = FYPCF_DEFAULT_VERSION_1_1, + }; + struct fy_parse_cfg cfg_12 = { + .flags = FYPCF_DEFAULT_VERSION_1_2, + }; + struct fy_document *fyd; + + /* Parse with YAML 1.1 */ + fyd = fy_document_build_from_string(&cfg_11, "key: value", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + fy_document_destroy(fyd); + + /* Parse with YAML 1.2 */ + fyd = fy_document_build_from_string(&cfg_12, "key: value", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + fy_document_destroy(fyd); + + /* Parse with explicit version directive */ + fyd = fy_document_build_from_string(NULL, "%YAML 1.2\n---\nkey: value", FY_NT); + ck_assert_ptr_ne(fyd, NULL); + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Flow and block styles */ +START_TEST(parser_flow_block_styles) +{ + struct fy_document *fyd; + struct fy_node *fyn; + char *buf; + + /* Build document with mixed flow and block styles */ + fyd = fy_document_build_from_string(NULL, + "block_map:\n" + " key: value\n" + "flow_map: {key: value}\n" + "block_seq:\n" + " - item\n" + "flow_seq: [item]\n", + FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Verify block mapping */ + fyn = fy_node_by_path(fy_document_root(fyd), "block_map", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + /* Verify flow mapping */ + fyn = fy_node_by_path(fy_document_root(fyd), "flow_map", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_mapping(fyn)); + + /* Verify block sequence */ + fyn = fy_node_by_path(fy_document_root(fyd), "block_seq", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_sequence(fyn)); + + /* Verify flow sequence */ + fyn = fy_node_by_path(fy_document_root(fyd), "flow_seq", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_sequence(fyn)); + + /* Emit and verify output */ + buf = fy_emit_document_to_string(fyd, 0); + ck_assert_ptr_ne(buf, NULL); + free(buf); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Document builder API */ +START_TEST(parser_document_builder) +{ + struct fy_document *fyd; + struct fy_node *fyn_root, *fyn_key, *fyn_val; + int rc; + + /* Create document using builder pattern */ + fyd = fy_document_create(NULL); + ck_assert_ptr_ne(fyd, NULL); + + /* Build root mapping */ + fyn_root = fy_node_create_mapping(fyd); + ck_assert_ptr_ne(fyn_root, NULL); + fy_document_set_root(fyd, fyn_root); + + /* Add key-value pairs using builder */ + fyn_key = fy_node_build_from_string(fyd, "key1", FY_NT); + ck_assert_ptr_ne(fyn_key, NULL); + fyn_val = fy_node_build_from_string(fyd, "value1", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + rc = fy_node_mapping_append(fyn_root, fyn_key, fyn_val); + ck_assert_int_eq(rc, 0); + + /* Add another pair with complex value */ + fyn_key = fy_node_build_from_string(fyd, "key2", FY_NT); + ck_assert_ptr_ne(fyn_key, NULL); + fyn_val = fy_node_build_from_string(fyd, "[1, 2, 3]", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + rc = fy_node_mapping_append(fyn_root, fyn_key, fyn_val); + ck_assert_int_eq(rc, 0); + + /* Verify the built document */ + ck_assert_int_eq(fy_node_mapping_item_count(fyn_root), 2); + + fyn_val = fy_node_mapping_lookup_by_string(fyn_root, "key1", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + ck_assert_str_eq(fy_node_get_scalar0(fyn_val), "value1"); + + fyn_val = fy_node_mapping_lookup_by_string(fyn_root, "key2", FY_NT); + ck_assert_ptr_ne(fyn_val, NULL); + ck_assert(fy_node_is_sequence(fyn_val)); + ck_assert_int_eq(fy_node_sequence_item_count(fyn_val), 3); + + fy_document_destroy(fyd); +} +END_TEST + +/* Test: Shell-like string splitting (from do_shell_split) */ +START_TEST(parser_shell_split) +{ + const char * const *argv; + int argc; + void *mem; + + /* Test simple splitting */ + mem = fy_utf8_split_posix("arg1 arg2 arg3", &argc, &argv); + ck_assert_ptr_ne(mem, NULL); + ck_assert_int_eq(argc, 3); + ck_assert_str_eq(argv[0], "arg1"); + ck_assert_str_eq(argv[1], "arg2"); + ck_assert_str_eq(argv[2], "arg3"); + ck_assert_ptr_eq(argv[argc], NULL); + free(mem); + + /* Test quoted strings */ + mem = fy_utf8_split_posix("'single quoted' \"double quoted\"", &argc, &argv); + ck_assert_ptr_ne(mem, NULL); + ck_assert_int_eq(argc, 2); + ck_assert_str_eq(argv[0], "single quoted"); + ck_assert_str_eq(argv[1], "double quoted"); + ck_assert_ptr_eq(argv[argc], NULL); + free(mem); + + /* Test escape sequences */ + mem = fy_utf8_split_posix("arg1\\ with\\ spaces arg2", &argc, &argv); + ck_assert_ptr_ne(mem, NULL); + ck_assert_int_eq(argc, 2); + ck_assert_str_eq(argv[0], "arg1 with spaces"); + ck_assert_str_eq(argv[1], "arg2"); + ck_assert_ptr_eq(argv[argc], NULL); + free(mem); + + /* Test empty string */ + mem = fy_utf8_split_posix("", &argc, &argv); + ck_assert_ptr_ne(mem, NULL); + ck_assert_int_eq(argc, 0); + ck_assert_ptr_eq(argv[argc], NULL); + free(mem); + + /* Test multiple spaces */ + mem = fy_utf8_split_posix(" arg1 arg2 ", &argc, &argv); + ck_assert_ptr_ne(mem, NULL); + ck_assert_int_eq(argc, 2); + ck_assert_str_eq(argv[0], "arg1"); + ck_assert_str_eq(argv[1], "arg2"); + ck_assert_ptr_eq(argv[argc], NULL); + free(mem); +} +END_TEST + +/* Test: UTF-8 validation (from do_bad_utf8) */ +START_TEST(parser_utf8_validation) +{ + const char *valid_ascii = "hello world"; + const char *s, *e; + int c, w, pos; + + /* Test valid ASCII forward */ + s = valid_ascii; + e = s + strlen(valid_ascii); + pos = 0; + while (s < e) { + c = fy_utf8_get(s, e - s, &w); + ck_assert_int_ge(c, 0); + ck_assert_int_ge(w, 1); + ck_assert_int_le(w, 4); + s += w; + pos++; + } + ck_assert_int_eq(pos, 11); + + /* Test valid UTF-8 forward (Greek characters) */ + const char *greek = "\xCE\xA4\xCE\xB9\xCE\xBC\xCE\xAE"; /* Τιμή */ + s = greek; + e = s + strlen(greek); + pos = 0; + while (s < e) { + c = fy_utf8_get(s, e - s, &w); + ck_assert_int_ge(c, 0); + ck_assert_int_eq(w, 2); /* Greek chars are 2 bytes */ + s += w; + pos++; + } + ck_assert_int_eq(pos, 4); + + /* Test valid ASCII backward */ + s = valid_ascii; + e = s + strlen(valid_ascii); + pos = 0; + while (s < e) { + c = fy_utf8_get_right(s, e - s, &w); + ck_assert_int_ge(c, 0); + ck_assert_int_ge(w, 1); + ck_assert_int_le(w, 4); + e -= w; + pos++; + } + ck_assert_int_eq(pos, 11); + + /* Test invalid UTF-8 sequence */ + const char invalid[] = { 0x67, 0xe7, 0x67, 0x00 }; /* 'g' followed by incomplete UTF-8 */ + s = invalid; + e = s + 3; /* Don't include null terminator */ + c = fy_utf8_get(s, 1, &w); /* First char should be valid */ + ck_assert_int_eq(c, 0x67); + ck_assert_int_eq(w, 1); + s += w; + c = fy_utf8_get(s, e - s, &w); /* Second char should be invalid or partial */ + ck_assert_int_lt(c, 0); /* Should return FYUG_INV or FYUG_PARTIAL */ + + /* Test EOF condition */ + c = fy_utf8_get(valid_ascii, 0, &w); + ck_assert_int_eq(c, FYUG_EOF); + ck_assert_int_eq(w, 0); +} +END_TEST + +/* Test: Token scanning - basic scalars */ +START_TEST(parser_token_scan_scalars) +{ + struct fy_parser *fyp; + struct fy_parse_cfg cfg; + struct fy_token *fyt; + const char *yaml; + int token_count; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + /* Test simple scalar */ + yaml = "hello"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + ck_assert_ptr_ne(fyt, NULL); + token_count++; + fy_token_unref(fyt); + } + ck_assert_int_gt(token_count, 0); + fy_parser_destroy(fyp); + + /* Test multiple scalars */ + yaml = "foo bar baz"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + token_count++; + fy_token_unref(fyt); + } + ck_assert_int_gt(token_count, 0); + fy_parser_destroy(fyp); +} +END_TEST + +/* Test: Token scanning - mappings */ +START_TEST(parser_token_scan_mapping) +{ + struct fy_parser *fyp; + struct fy_parse_cfg cfg; + struct fy_token *fyt; + const char *yaml; + int token_count; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + yaml = "key: value"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + ck_assert_ptr_ne(fyt, NULL); + token_count++; + fy_token_unref(fyt); + } + /* Should have multiple tokens: STREAM-START, BLOCK-MAPPING-START, KEY, SCALAR, VALUE, SCALAR, etc. */ + ck_assert_int_gt(token_count, 5); + fy_parser_destroy(fyp); + + /* Test nested mapping */ + yaml = "outer:\n inner: value"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + token_count++; + fy_token_unref(fyt); + } + ck_assert_int_gt(token_count, 5); + fy_parser_destroy(fyp); +} +END_TEST + +/* Test: Token scanning - sequences */ +START_TEST(parser_token_scan_sequence) +{ + struct fy_parser *fyp; + struct fy_parse_cfg cfg; + struct fy_token *fyt; + const char *yaml; + int token_count; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + /* Test flow sequence */ + yaml = "[1, 2, 3]"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + token_count++; + fy_token_unref(fyt); + } + ck_assert_int_gt(token_count, 5); + fy_parser_destroy(fyp); + + /* Test block sequence */ + yaml = "- item1\n- item2\n- item3"; + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + token_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + token_count++; + fy_token_unref(fyt); + } + ck_assert_int_gt(token_count, 5); + fy_parser_destroy(fyp); +} +END_TEST + +/* Test: Token scanning - scalar styles */ +START_TEST(parser_token_scan_scalar_styles) +{ + struct fy_parser *fyp; + struct fy_parse_cfg cfg; + struct fy_token *fyt; + const char *yaml; + int scalar_count; + enum fy_scalar_style style; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + /* Test different scalar styles */ + yaml = "plain: value\n" + "single: 'quoted value'\n" + "double: \"quoted value\"\n"; + + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + fy_parser_set_string(fyp, yaml, FY_NT); + + scalar_count = 0; + while ((fyt = fy_scan(fyp)) != NULL) { + if (fyt->type == FYTT_SCALAR) { + style = fy_token_scalar_style(fyt); + /* Verify style is valid */ + ck_assert(style == FYSS_PLAIN || + style == FYSS_SINGLE_QUOTED || + style == FYSS_DOUBLE_QUOTED || + style == FYSS_LITERAL || + style == FYSS_FOLDED); + scalar_count++; + } + fy_token_unref(fyt); + } + ck_assert_int_ge(scalar_count, 6); /* 6 scalars: 3 keys + 3 values */ + fy_parser_destroy(fyp); +} +END_TEST + +/* Test: Manual emitter mode - emit events directly */ +START_TEST(parser_manual_emitter) +{ + struct fy_emitter_cfg ecfg; + struct fy_emitter *emit; + + memset(&ecfg, 0, sizeof(ecfg)); + ecfg.flags = FYECF_MODE_MANUAL; + + /* Test block style manual emission */ + emit = fy_emitter_create(&ecfg); + ck_assert_ptr_ne(emit, NULL); + + /* Emit: key: [{ a: 1 }] in block style */ + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_START)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_START, false, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "key", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_BLOCK, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "a", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "1", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_END, true, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_END)); + + /* If we got here without crashes, the manual emitter block style works */ + fy_emitter_destroy(emit); + + /* Test flow style manual emission */ + emit = fy_emitter_create(&ecfg); + ck_assert_ptr_ne(emit, NULL); + + /* Emit: {key: [{a: 1}]} in flow style */ + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_START)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_START, false, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_FLOW, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "key", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_START, FYNS_FLOW, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_START, FYNS_FLOW, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "a", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SCALAR, FYSS_PLAIN, "1", FY_NT, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_SEQUENCE_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_MAPPING_END)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_DOCUMENT_END, true, NULL, NULL)); + fy_emit_event(emit, fy_emit_event_create(emit, FYET_STREAM_END)); + + /* If we got here without crashes, the manual emitter flow style works */ + fy_emitter_destroy(emit); +} +END_TEST + +START_TEST(parser_parse_load_document) +{ + struct fy_parse_cfg cfg; + struct fy_parser *fyp; + struct fy_document *fyd; + const char *yaml = "---\nfoo: bar\n---\nbaz: qux\n"; + int count; + + /* Test loading multiple documents from a parser stream */ + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + + fy_parser_set_string(fyp, yaml, FY_NT); + + /* Load first document */ + count = 0; + while ((fyd = fy_parse_load_document(fyp)) != NULL) { + const char *value; + struct fy_node *fyn; + + count++; + + /* Verify document content */ + if (count == 1) { + fyn = fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + value = fy_node_get_scalar0(fyn); + ck_assert_str_eq(value, "bar"); + } else if (count == 2) { + fyn = fy_node_by_path(fy_document_root(fyd), "/baz", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + value = fy_node_get_scalar0(fyn); + ck_assert_str_eq(value, "qux"); + } + + fy_parse_document_destroy(fyp, fyd); + } + + ck_assert_int_eq(count, 2); + + fy_parser_destroy(fyp); +} +END_TEST + +START_TEST(parser_document_resolve) +{ + struct fy_document *fyd; + struct fy_node *fyn; + const char *yaml = "anchor: &data\n key: value\nref: *data\n"; + const char *val; + int rc; + + /* Test document resolution (resolving aliases) */ + fyd = fy_document_build_from_string(NULL, yaml, FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Before resolution, alias nodes exist */ + fyn = fy_node_by_path(fy_document_root(fyd), "/ref", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(fy_node_is_alias(fyn)); + + /* Resolve all aliases in the document */ + rc = fy_document_resolve(fyd); + ck_assert_int_eq(rc, 0); + + /* After resolution, verify both paths are accessible and have same values */ + fyn = fy_node_by_path(fy_document_root(fyd), "/anchor/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + val = fy_node_get_scalar0(fyn); + ck_assert_str_eq(val, "value"); + + fyn = fy_node_by_path(fy_document_root(fyd), "/ref/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + val = fy_node_get_scalar0(fyn); + ck_assert_str_eq(val, "value"); + + /* After resolution, the ref node should no longer be an alias */ + fyn = fy_node_by_path(fy_document_root(fyd), "/ref", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + ck_assert(!fy_node_is_alias(fyn)); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(parser_document_clone) +{ + struct fy_document *fyd, *fyd_clone; + struct fy_node *fyn, *fyn_clone; + const char *yaml = "foo: bar\nbaz: [1, 2, 3]\n"; + const char *val1, *val2; + + /* Test document cloning */ + fyd = fy_document_build_from_string(NULL, yaml, FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Clone the document */ + fyd_clone = fy_document_clone(fyd); + ck_assert_ptr_ne(fyd_clone, NULL); + + /* Verify original document content */ + fyn = fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + val1 = fy_node_get_scalar0(fyn); + ck_assert_str_eq(val1, "bar"); + + /* Verify cloned document has same content */ + fyn_clone = fy_node_by_path(fy_document_root(fyd_clone), "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_clone, NULL); + val2 = fy_node_get_scalar0(fyn_clone); + ck_assert_str_eq(val2, "bar"); + + /* But they are different node objects */ + ck_assert_ptr_ne(fyn, fyn_clone); + + fy_document_destroy(fyd); + fy_document_destroy(fyd_clone); +} +END_TEST + +START_TEST(parser_node_copy) +{ + struct fy_document *fyd; + struct fy_node *fyn_src, *fyn_copy, *fyn_root; + const char *yaml = "original: {key: value}\n"; + const char *val; + + /* Test node copying */ + fyd = fy_document_build_from_string(NULL, yaml, FY_NT); + ck_assert_ptr_ne(fyd, NULL); + + /* Get the source node */ + fyn_src = fy_node_by_path(fy_document_root(fyd), "/original", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_src, NULL); + + /* Copy the node */ + fyn_copy = fy_node_copy(fyd, fyn_src); + ck_assert_ptr_ne(fyn_copy, NULL); + + /* Verify the copy has same content */ + fyn_root = fy_node_by_path(fyn_copy, "/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_root, NULL); + val = fy_node_get_scalar0(fyn_root); + ck_assert_str_eq(val, "value"); + + /* Add the copy to the document with a new key */ + fy_node_mapping_append(fy_document_root(fyd), + fy_node_build_from_string(fyd, "copied", FY_NT), + fyn_copy); + + /* Verify it exists at new location */ + fyn_root = fy_node_by_path(fy_document_root(fyd), "/copied/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn_root, NULL); + val = fy_node_get_scalar0(fyn_root); + ck_assert_str_eq(val, "value"); + + fy_document_destroy(fyd); +} +END_TEST + +START_TEST(parser_document_builder_load) +{ + struct fy_parse_cfg parse_cfg; + struct fy_document_builder_cfg builder_cfg; + struct fy_parser *fyp; + struct fy_document_builder *fydb; + struct fy_document *fyd; + const char *yaml = "---\nkey: value\n---\nfoo: bar\n"; + int count; + + /* Test document builder API */ + memset(&parse_cfg, 0, sizeof(parse_cfg)); + parse_cfg.flags = FYPCF_DEFAULT_PARSE; + + memset(&builder_cfg, 0, sizeof(builder_cfg)); + builder_cfg.parse_cfg = parse_cfg; + + fyp = fy_parser_create(&parse_cfg); + ck_assert_ptr_ne(fyp, NULL); + + fy_parser_set_string(fyp, yaml, FY_NT); + + /* Create document builder with configuration */ + fydb = fy_document_builder_create(&builder_cfg); + ck_assert_ptr_ne(fydb, NULL); + + /* Load documents using builder */ + count = 0; + while ((fyd = fy_document_builder_load_document(fydb, fyp)) != NULL) { + const char *value; + struct fy_node *fyn; + + count++; + + /* Verify document content */ + if (count == 1) { + fyn = fy_node_by_path(fy_document_root(fyd), "/key", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + value = fy_node_get_scalar0(fyn); + ck_assert_str_eq(value, "value"); + } else if (count == 2) { + fyn = fy_node_by_path(fy_document_root(fyd), "/foo", FY_NT, FYNWF_DONT_FOLLOW); + ck_assert_ptr_ne(fyn, NULL); + value = fy_node_get_scalar0(fyn); + ck_assert_str_eq(value, "bar"); + } + + fy_document_destroy(fyd); + } + + ck_assert_int_eq(count, 2); + + fy_document_builder_destroy(fydb); + fy_parser_destroy(fyp); +} +END_TEST + +/* Composer callback record entry */ +struct compose_record { + enum fy_event_type type; + char path[128]; + int depth; + char scalar_value[64]; +}; + +/* Composer callback data for parser_compose_callback test */ +struct compose_test_data { + struct compose_record records[100]; + int record_count; +}; + +/* Composer callback function for testing */ +static enum fy_composer_return +compose_test_callback(struct fy_parser *fyp, struct fy_event *fye, + struct fy_path *path, void *userdata) +{ + struct compose_test_data *data = userdata; + struct compose_record *rec; + char *path_text; + const char *scalar; + size_t len; + + if (data->record_count >= 100) + return FYCR_OK_CONTINUE; + + rec = &data->records[data->record_count++]; + rec->type = fye->type; + rec->depth = fy_path_depth(path); + + /* Get path as text */ + path_text = fy_path_get_text(path); + if (path_text) { + snprintf(rec->path, sizeof(rec->path), "%s", path_text); + free(path_text); + } else { + rec->path[0] = '\0'; + } + + /* For scalar events, capture the value */ + rec->scalar_value[0] = '\0'; + if (fye->type == FYET_SCALAR && fye->scalar.value) { + scalar = fy_token_get_text(fye->scalar.value, &len); + if (scalar && len < sizeof(rec->scalar_value) - 1) { + memcpy(rec->scalar_value, scalar, len); + rec->scalar_value[len] = '\0'; + } + } + + return FYCR_OK_CONTINUE; +} + +START_TEST(parser_compose_callback) +{ + struct fy_parse_cfg cfg; + struct fy_parser *fyp; + struct compose_test_data data; + const char *yaml = + "invoice: 34843\n" + "date : !!str 2001-01-23\n" + "bill-to: &id001\n" + " given : Chris\n" + " family : Dumars\n" + " address:\n" + " lines: |\n" + " 458 Walkman Dr.\n" + " Suite #292\n"; + int rc; + int i; + bool found_invoice_key = false; + bool found_invoice_value = false; + bool found_bill_to_path = false; + + /* Test compose callback mechanism */ + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYPCF_DEFAULT_PARSE; + + fyp = fy_parser_create(&cfg); + ck_assert_ptr_ne(fyp, NULL); + + fy_parser_set_string(fyp, yaml, FY_NT); + + /* Initialize test data */ + memset(&data, 0, sizeof(data)); + + /* Compose with callback */ + rc = fy_parse_compose(fyp, compose_test_callback, &data); + ck_assert_int_eq(rc, 0); + + /* Verify we got events */ + ck_assert(data.record_count > 0); + + /* Verify we have the expected events and paths */ + for (i = 0; i < data.record_count; i++) { + struct compose_record *rec = &data.records[i]; + + /* Look for specific events we expect */ + if (rec->type == FYET_SCALAR && strcmp(rec->scalar_value, "invoice") == 0) { + found_invoice_key = true; + /* Verify path for invoice key */ + ck_assert(strstr(rec->path, "invoice") != NULL || strcmp(rec->path, "/") == 0); + } + + if (rec->type == FYET_SCALAR && strcmp(rec->scalar_value, "34843") == 0) { + found_invoice_value = true; + } + + /* Check for bill-to path */ + if (rec->type == FYET_MAPPING_START && strstr(rec->path, "bill-to") != NULL) { + found_bill_to_path = true; + /* Verify depth is reasonable (should be at least 1) */ + ck_assert(rec->depth > 0); + } + } + + /* Verify we found the expected elements */ + ck_assert(found_invoice_key); + ck_assert(found_invoice_value); + ck_assert(found_bill_to_path); + + /* Verify first event is stream start */ + ck_assert_int_eq(data.records[0].type, FYET_STREAM_START); + + /* Verify last event is stream end */ + ck_assert_int_eq(data.records[data.record_count - 1].type, FYET_STREAM_END); + + fy_parser_destroy(fyp); +} +END_TEST + +TCase *libfyaml_case_parser(void) +{ + TCase *tc; + + tc = tcase_create("parser"); + + /* Mapping tests */ + tcase_add_test(tc, parser_mapping_iterator); + tcase_add_test(tc, parser_mapping_key_lookup); + tcase_add_test(tc, parser_mapping_prepend); + tcase_add_test(tc, parser_mapping_remove); + + /* Path query tests */ + tcase_add_test(tc, parser_path_queries); + tcase_add_test(tc, parser_node_path_generation); + + /* Node creation tests */ + tcase_add_test(tc, parser_node_creation_scalar); + tcase_add_test(tc, parser_node_creation_multiline_scalar); + tcase_add_test(tc, parser_node_creation_empty_sequence); + tcase_add_test(tc, parser_node_creation_empty_mapping); + tcase_add_test(tc, parser_node_creation_populated_sequence); + tcase_add_test(tc, parser_node_creation_populated_mapping); + tcase_add_test(tc, parser_build_node_from_string); + + /* Sequence tests */ + tcase_add_test(tc, parser_sequence_negative_index); + tcase_add_test(tc, parser_sequence_append_prepend); + tcase_add_test(tc, parser_sequence_remove); + + /* Complex structure tests */ + tcase_add_test(tc, parser_complex_nested_structure); + + /* Anchor/alias tests */ + tcase_add_test(tc, parser_anchor_alias_resolution); + + /* Document operations */ + tcase_add_test(tc, parser_document_insert_at); + tcase_add_test(tc, parser_document_emit_flags); + tcase_add_test(tc, parser_multi_document_stream); + tcase_add_test(tc, parser_empty_document); + tcase_add_test(tc, parser_document_with_comments); + + /* Iterator tests */ + tcase_add_test(tc, parser_document_iterator); + tcase_add_test(tc, parser_document_iterator_key_detection); + tcase_add_test(tc, parser_iterator_alias_detection); + + /* Comment tests */ + tcase_add_test(tc, parser_comment_retrieval); + + /* Event and parsing tests */ + tcase_add_test(tc, parser_event_generation); + tcase_add_test(tc, parser_scalar_styles); + tcase_add_test(tc, parser_tag_handling); + tcase_add_test(tc, parser_yaml_version); + tcase_add_test(tc, parser_flow_block_styles); + tcase_add_test(tc, parser_document_builder); + + /* Utility function tests */ + tcase_add_test(tc, parser_shell_split); + tcase_add_test(tc, parser_utf8_validation); + + /* Token scanning tests */ + tcase_add_test(tc, parser_token_scan_scalars); + tcase_add_test(tc, parser_token_scan_mapping); + tcase_add_test(tc, parser_token_scan_sequence); + tcase_add_test(tc, parser_token_scan_scalar_styles); + + /* Manual emitter tests */ + tcase_add_test(tc, parser_manual_emitter); + + /* Parser and document builder tests */ + tcase_add_test(tc, parser_parse_load_document); + tcase_add_test(tc, parser_document_resolve); + tcase_add_test(tc, parser_document_clone); + tcase_add_test(tc, parser_node_copy); + tcase_add_test(tc, parser_document_builder_load); + + /* Compose callback tests */ + tcase_add_test(tc, parser_compose_callback); + + return tc; +} diff --git a/test/libfyaml-test-private-id.c b/test/libfyaml-test-private-id.c index a016211..7936f87 100644 --- a/test/libfyaml-test-private-id.c +++ b/test/libfyaml-test-private-id.c @@ -34,17 +34,19 @@ START_TEST(id_ffs) int r; } ffs_check[] = { { .v = 0, .r = -1 }, - { .v = ~(fy_id_bits)0, .r = 0 }, - { .v = ((fy_id_bits)1 << 0), .r = 0 }, - { .v = ((fy_id_bits)1 << (FY_ID_BITS_BITS - 1)), .r = (FY_ID_BITS_BITS - 1) }, - { .v = ((fy_id_bits)1 << 0) | ((fy_id_bits)1 << (FY_ID_BITS_BITS - 1)), .r = 0 }, - { .v = ((fy_id_bits)1 << (FY_ID_BITS_BITS / 2)), .r = (FY_ID_BITS_BITS / 2), }, + { .v = ~(fy_id_bits_non_atomic)0, .r = 0 }, + { .v = ((fy_id_bits_non_atomic)1 << 0), .r = 0 }, + { .v = ((fy_id_bits_non_atomic)1 << (FY_ID_BITS_BITS - 1)), .r = (FY_ID_BITS_BITS - 1) }, + { .v = ((fy_id_bits_non_atomic)1 << 0) | ((fy_id_bits_non_atomic)1 << (FY_ID_BITS_BITS - 1)), .r = 0 }, + { .v = ((fy_id_bits_non_atomic)1 << (FY_ID_BITS_BITS / 2)), .r = (FY_ID_BITS_BITS / 2), }, }; unsigned int i; + fy_id_bits v; int r; for (i = 0; i < ARRAY_SIZE(ffs_check); i++) { - r = fy_id_ffs(ffs_check[i].v); + fy_atomic_store(&v, ffs_check[i].v); + r = fy_id_ffs(v); ck_assert_int_eq(r, ffs_check[i].r); } } @@ -83,6 +85,7 @@ START_TEST(id_alloc_full) expected_id = 0; for (i = 0; i < ba_bits_actual; i++) { id = fy_id_alloc(ba, ba_count); + ck_assert_int_ne(id, -1); ck_assert_int_eq(id, expected_id); expected_id++; } diff --git a/test/libfyaml-test-thread.c b/test/libfyaml-test-thread.c new file mode 100644 index 0000000..581b4df --- /dev/null +++ b/test/libfyaml-test-thread.c @@ -0,0 +1,267 @@ +/* + * libfyaml-test-thread.c - libfyaml threading tests + * + * Copyright (c) 2023 Pantelis Antoniou + * + * SPDX-License-Identifier: MIT + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include + +/* Test: Basic thread pool creation and destruction */ +START_TEST(thread_pool_create_destroy) +{ + struct fy_thread_pool_cfg cfg; + struct fy_thread_pool *tp; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = 0; + cfg.num_threads = 2; + cfg.userdata = NULL; + + tp = fy_thread_pool_create(&cfg); + ck_assert_ptr_ne(tp, NULL); + + /* Verify we can get the configuration */ + const struct fy_thread_pool_cfg *got_cfg = fy_thread_pool_get_cfg(tp); + ck_assert_ptr_ne(got_cfg, NULL); + ck_assert_int_eq(got_cfg->num_threads, 2); + + /* Verify we can get the number of threads */ + int num_threads = fy_thread_pool_get_num_threads(tp); + ck_assert_int_eq(num_threads, 2); + + fy_thread_pool_destroy(tp); +} +END_TEST + +/* Worker function that atomically increments a counter */ +static void atomic_increment_worker(void *arg) +{ + _Atomic(int) *p = arg; + int v, exp_v; + + /* Atomically increase the counter */ + v = atomic_load(p); + for (;;) { + exp_v = v; + if (atomic_compare_exchange_strong(p, &exp_v, v + 1)) + return; + v = exp_v; + } +} + +/* Test: Thread reserve, submit work, wait, unreserve */ +START_TEST(thread_reserve_submit_wait) +{ + struct fy_thread_pool_cfg cfg; + struct fy_thread_pool *tp; + struct fy_thread *threads[4]; + struct fy_thread_work works[4]; + _Atomic(int) counter = 0; + unsigned int i; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = 0; + cfg.num_threads = 4; + cfg.userdata = NULL; + + tp = fy_thread_pool_create(&cfg); + ck_assert_ptr_ne(tp, NULL); + + /* Reserve all threads */ + for (i = 0; i < 4; i++) { + threads[i] = fy_thread_reserve(tp); + ck_assert_ptr_ne(threads[i], NULL); + } + + /* Submit work to all threads */ + for (i = 0; i < 4; i++) { + works[i].fn = atomic_increment_worker; + works[i].arg = &counter; + fy_thread_submit_work(threads[i], &works[i]); + } + + /* Wait for all threads to complete */ + for (i = 0; i < 4; i++) { + fy_thread_wait_work(threads[i]); + } + + /* Verify counter was incremented 4 times */ + ck_assert_int_eq(atomic_load(&counter), 4); + + /* Unreserve all threads */ + for (i = 0; i < 4; i++) { + fy_thread_unreserve(threads[i]); + } + + fy_thread_pool_destroy(tp); +} +END_TEST + +/* Test: Thread arg join */ +START_TEST(thread_arg_join) +{ + struct fy_thread_pool_cfg cfg; + struct fy_thread_pool *tp; + _Atomic(int) counter = 0; + unsigned int num_tasks = 8; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = 0; + cfg.num_threads = 4; + cfg.userdata = NULL; + + tp = fy_thread_pool_create(&cfg); + ck_assert_ptr_ne(tp, NULL); + + /* Use arg_join to execute the same function multiple times */ + fy_thread_arg_join(tp, atomic_increment_worker, NULL, &counter, num_tasks); + + /* Verify counter was incremented num_tasks times */ + ck_assert_int_eq(atomic_load(&counter), (int)num_tasks); + + fy_thread_pool_destroy(tp); +} +END_TEST + +/* Worker function for sum test */ +struct sum_arg { + const int *values; + int count; + int result; +}; + +static void sum_worker(void *arg) +{ + struct sum_arg *s = arg; + int i; + int sum = 0; + + for (i = 0; i < s->count; i++) { + sum += s->values[i]; + } + s->result = sum; +} + +/* Test: Thread array join with different arguments */ +START_TEST(thread_arg_array_join) +{ + struct fy_thread_pool_cfg cfg; + struct fy_thread_pool *tp; + int values[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + struct sum_arg args[2]; + int total_sum; + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = 0; + cfg.num_threads = 2; + cfg.userdata = NULL; + + tp = fy_thread_pool_create(&cfg); + ck_assert_ptr_ne(tp, NULL); + + /* Split the work into two tasks */ + args[0].values = values; + args[0].count = 5; + args[0].result = 0; + + args[1].values = values + 5; + args[1].count = 5; + args[1].result = 0; + + /* Execute both tasks in parallel */ + fy_thread_arg_array_join(tp, sum_worker, NULL, args, sizeof(struct sum_arg), 2); + + /* Verify results */ + total_sum = args[0].result + args[1].result; + ck_assert_int_eq(total_sum, 55); /* 1+2+3+...+10 = 55 */ + ck_assert_int_eq(args[0].result, 15); /* 1+2+3+4+5 = 15 */ + ck_assert_int_eq(args[1].result, 40); /* 6+7+8+9+10 = 40 */ + + fy_thread_pool_destroy(tp); +} +END_TEST + +/* Worker function for steal mode test */ +static void steal_mode_worker(void *arg) +{ + _Atomic(int) *p = arg; + int v, exp_v; + int i; + + /* Increment counter 100 times */ + for (i = 0; i < 100; i++) { + v = atomic_load(p); + for (;;) { + exp_v = v; + if (atomic_compare_exchange_strong(p, &exp_v, v + 1)) + break; + v = exp_v; + } + } +} + +/* Test: Work stealing mode */ +START_TEST(thread_steal_mode) +{ + struct fy_thread_pool_cfg cfg; + struct fy_thread_pool *tp; + _Atomic(int) counter = 0; + unsigned int num_tasks = 16; /* More tasks than threads */ + + memset(&cfg, 0, sizeof(cfg)); + cfg.flags = FYTPCF_STEAL_MODE; + cfg.num_threads = 4; + cfg.userdata = NULL; + + tp = fy_thread_pool_create(&cfg); + ck_assert_ptr_ne(tp, NULL); + + /* Execute many tasks with work stealing enabled */ + fy_thread_arg_join(tp, steal_mode_worker, NULL, &counter, num_tasks); + + /* Verify all tasks completed: 16 tasks * 100 increments each */ + ck_assert_int_eq(atomic_load(&counter), (int)(num_tasks * 100)); + + fy_thread_pool_destroy(tp); +} +END_TEST + +TCase *libfyaml_case_thread(void) +{ + TCase *tc; + + tc = tcase_create("thread"); + + /* Basic thread pool tests */ + tcase_add_test(tc, thread_pool_create_destroy); + + /* Thread work submission tests */ + tcase_add_test(tc, thread_reserve_submit_wait); + + /* Join API tests */ + tcase_add_test(tc, thread_arg_join); + tcase_add_test(tc, thread_arg_array_join); + + /* Work stealing tests */ + tcase_add_test(tc, thread_steal_mode); + + return tc; +} diff --git a/test/libfyaml-test.c b/test/libfyaml-test.c index 0f45c42..1ac79fb 100644 --- a/test/libfyaml-test.c +++ b/test/libfyaml-test.c @@ -44,6 +44,9 @@ extern TCase *libfyaml_case_private_id(void); extern TCase *libfyaml_case_core(void); extern TCase *libfyaml_case_meta(void); extern TCase *libfyaml_case_emit(void); +extern TCase *libfyaml_case_allocator(void); +extern TCase *libfyaml_case_parser(void); +extern TCase *libfyaml_case_thread(void); Suite *libfyaml_suite(void) { @@ -58,6 +61,9 @@ Suite *libfyaml_suite(void) suite_add_tcase(s, libfyaml_case_core()); suite_add_tcase(s, libfyaml_case_meta()); suite_add_tcase(s, libfyaml_case_emit()); + suite_add_tcase(s, libfyaml_case_allocator()); + suite_add_tcase(s, libfyaml_case_parser()); + suite_add_tcase(s, libfyaml_case_thread()); return s; } diff --git a/test/libfyaml.test b/test/libfyaml.test index cb691aa..885f310 100755 --- a/test/libfyaml.test +++ b/test/libfyaml.test @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash if [ "x$builddir" == "x" ]; then T=`realpath ./libfyaml-test` diff --git a/test/testemitter-restreaming.test b/test/testemitter-restreaming.test new file mode 100755 index 0000000..4a48178 --- /dev/null +++ b/test/testemitter-restreaming.test @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +${SRCDIR}/testemitter.test --restreaming diff --git a/test/testemitter-streaming.test b/test/testemitter-streaming.test index f5830ff..84aa774 100755 --- a/test/testemitter-streaming.test +++ b/test/testemitter-streaming.test @@ -1,2 +1,2 @@ -#!/bin/bash +#!/usr/bin/env bash ${SRCDIR}/testemitter.test --streaming diff --git a/test/testemitter.test b/test/testemitter.test index 8dfb377..be9d4f2 100755 --- a/test/testemitter.test +++ b/test/testemitter.test @@ -1,9 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash EXTRA_DUMP_ARGS="" if [ "x$1" == "x--streaming" ]; then EXTRA_DUMP_ARGS="$EXTRA_DUMP_ARGS --streaming" +elif [ "x$1" == "x--restreaming" ]; then + EXTRA_DUMP_ARGS="$EXTRA_DUMP_ARGS --streaming --recreating" fi count=0 diff --git a/test/testerrors.test b/test/testerrors.test index e904ceb..55f8627 100755 --- a/test/testerrors.test +++ b/test/testerrors.test @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash count=0 for dir in "${SRCDIR}"/test-errors/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do @@ -20,7 +20,7 @@ for dir in "${SRCDIR}"/test-errors/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do res="not ok" pass_yaml=0 - ${TOP_BUILDDIR}/src/fy-tool --dump -r "$dir/in.yaml" >"$t" 2>&1 + ${TOP_BUILDDIR}/src/fy-tool --dump --no-streaming -r "$dir/in.yaml" >"$t" 2>&1 if [ $? -eq 0 ]; then pass_yaml=1 fi diff --git a/test/testsuite-evstream.test b/test/testsuite-evstream.test index a6f931f..c60b0be 100755 --- a/test/testsuite-evstream.test +++ b/test/testsuite-evstream.test @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash count=0 for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do @@ -18,8 +18,10 @@ done # output plan echo 1..$count -skiplist="2JQS" -xfaillist="" +declare -A skips=( + [2JQS]="duplicate keys in testcase; cannot load as document" +) +declare -A xfails=() i=0 for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do @@ -47,19 +49,18 @@ for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do desctxt=`cat 2>/dev/null "$tst/==="` - t=`mktemp` - directive="" - for skip in $skiplist; do - if [ "$test_subtest_id" == "$skip" ]; then - directive=" # skip: duplicate keys in testcase; cannot load as document" - break - fi - done + skip_reason="${skips[$test_subtest_id]}" + if [ "x$skip_reason" != "x" ]; then + directive=" # SKIP: ${skip_reason}" + fi res="ok" - if [ "x$directive" == "x" ]; then + if [ "x$skip_reason" == "x" ]; then res="not ok" + + t=`mktemp` + # run the test using document-event-stream ${TOP_BUILDDIR}/src/fy-tool --testsuite --document-event-stream "$tst/in.yaml" >"$t" if [ $? -eq 0 ]; then @@ -75,12 +76,10 @@ for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do rm -f "$t" - for xfail in $xfaillist; do - if [ "$test_subtest_id" == "$xfail" ]; then - directive=" # TODO: known failure." - break - fi - done + xfail_reason="${xfails[$test_subtest_id]}" + if [ "x$xfail_reason" != "x" ]; then + directive=" # TODO: ${xfail_reason}" + fi fi echo "$res $i $test_subtest_id - $desctxt$directive" diff --git a/test/testsuite-json.test b/test/testsuite-json.test index 503f993..f5435a5 100755 --- a/test/testsuite-json.test +++ b/test/testsuite-json.test @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash count=0 for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do @@ -22,8 +22,12 @@ done # output plan echo 1..$count -skiplist="C4HZ" -xfaillist="" +declare -A skips=( + [UGM3]="Later jq versions rewrite numbers like 12.00 -> 12 which breaks diff" +) +declare -A xfails=( + [C4HZ]="requires schema support which libfyaml does not support yet." +) i=0 for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do @@ -51,16 +55,13 @@ for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do desctxt=`cat 2>/dev/null "$tst/==="` directive="" - for skip in $skiplist; do - if [ "$test_subtest_id" == "$skip" ]; then - directive="# SKIP: does not apply to libfyaml" - break - fi - done + skip_reason="${skips[$test_subtest_id]}" + if [ "x$skip_reason" != "x" ]; then + directive=" # SKIP: ${skip_reason}" + fi res="ok" - - if [ "x$directive" == "x" ]; then + if [ "x$skip_reason" == "x" ]; then t1=`mktemp` t2=`mktemp` @@ -79,12 +80,10 @@ for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do rm -f "$t1" "$t2" - for xfail in $xfaillist; do - if [ "$test_subtest_id" == "$xfail" ]; then - directive=" # TODO: known failure." - break - fi - done + xfail_reason="${xfails[$test_subtest_id]}" + if [ "x$xfail_reason" != "x" ]; then + directive=" # TODO: ${xfail_reason}" + fi fi echo "$res $i $test_subtest_id - $desctxt (JSON)$directive" diff --git a/test/testsuite-resolution.test b/test/testsuite-resolution.test index 2aaf07d..d33f4a1 100755 --- a/test/testsuite-resolution.test +++ b/test/testsuite-resolution.test @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash count=0 for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do @@ -23,9 +23,12 @@ done # output plan echo 1..$count -# X38W has duplicate keys after resolution -skiplist="2JQS X38W" -xfaillist="" +declare -A skips=( + [2JQS]="duplicate keys in testcase; cannot load as document" + [X38W]="duplicate keys after resolution" +) +declare -A xfails=( +) i=0 for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do @@ -63,15 +66,13 @@ for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do t2=`mktemp` directive="" - for skip in $skiplist; do - if [ "$test_subtest_id" == "$skip" ]; then - directive=" # skip: duplicate keys in testcase; cannot load as document" - break - fi - done + skip_reason="${skips[$test_subtest_id]}" + if [ "x$skip_reason" != "x" ]; then + directive=" # SKIP: ${skip_reason}" + fi res="ok" - if [ "x$directive" == "x" ]; then + if [ "x$skip_reason" == "x" ]; then res="not ok" ${TOP_BUILDDIR}/src/fy-tool --dump --resolve "$tst/in.yaml" | ${TOP_BUILDDIR}/src/fy-tool --testsuite --disable-flow-markers - >"$t1" @@ -86,12 +87,10 @@ for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do rm -f "$t1" "$t2" - for xfail in $xfaillist; do - if [ "$test_subtest_id" == "$xfail" ]; then - directive=" # TODO: known failure." - break - fi - done + xfail_reason="${xfails[$test_subtest_id]}" + if [ "x$xfail_reason" != "x" ]; then + directive=" # TODO: ${xfail_reason}" + fi fi echo "$res $i $test_subtest_id - $desctxt$directive" diff --git a/test/testsuite.test b/test/testsuite.test index 4f82f4d..adee528 100755 --- a/test/testsuite.test +++ b/test/testsuite.test @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash count=0 for basetst in test-suite-data/[A-Z0-9][A-Z0-9][A-Z0-9][A-Z0-9]/; do -- 2.39.5